diff --git a/CODEOWNERS b/CODEOWNERS index 12c0a54ac1..5f89315d23 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -560,6 +560,7 @@ esphome/components/tc74/* @sethgirvan esphome/components/tca9548a/* @andreashergert1984 esphome/components/tca9555/* @mobrembski esphome/components/tcl112/* @glmnet +esphome/components/tcp_uart/* @Bascht74 esphome/components/tee501/* @Stock-M esphome/components/teleinfo/* @0hax esphome/components/tem3200/* @bakerkj diff --git a/esphome/__main__.py b/esphome/__main__.py index cbeb9ff01c..40457dbcc2 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1300,10 +1300,9 @@ def _choose_ota_platform(config: ConfigType, requested: str | None) -> str: The native API uses challenge-response auth with MD5/SHA256 hashing of a server-issued nonce, so the password is never sent over the wire; the ``web_server`` path uses HTTP Basic auth which transmits credentials in - cleartext over the LAN. (The native path also supports gzip compression - on ESP8266, where flash space is tight; on ESP32/RP2040/LibreTiny the - backend reports ``supports_compression() == false`` and the firmware is - sent uncompressed regardless of which platform is used.) Falls back to + cleartext over the LAN. (The native path also compresses the upload: + gzip on ESP8266 and RP2040, which inflate it at reboot, and a deflate + stream on ESP32/LibreTiny, which inflate it as it arrives.) Falls back to ``web_server`` only when that is the only available platform. """ # Use a dict (insertion-ordered) instead of a list so error messages and diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 3838d07602..42a1de5a4b 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -66,6 +66,43 @@ else() "app edits will regenerate sections.ld.") endif()""" +# lwip sources that compile to empty objects with the option off (their own +# #if guard). (option, regex valid for both Python and CMake); a source is +# only dropped when its option is defined and off, so a renamed option +# keeps it. +LWIP_EMPTY_SOURCES: tuple[tuple[str, str], ...] = ( + ("CONFIG_LWIP_PPP_SUPPORT", "/netif/ppp/"), + ("CONFIG_LWIP_IPV6", "/core/ipv6/"), + ("CONFIG_LWIP_AUTOIP", "/core/ipv4/autoip[.]c$"), + ("CONFIG_LWIP_STATS", "/core/stats[.]c$"), +) +# Drift guard only: keep every lwip source. +LWIP_FULL_SOURCES_ENV = "ESPHOME_LWIP_FULL_SOURCES" + +# Drops the empty objects after project(), once the lwip target exists. +_LWIP_EMPTY_SOURCES_FILTER = f"""\ +idf_build_get_property(esphome_build_components BUILD_COMPONENTS) +if(lwip IN_LIST esphome_build_components AND NOT DEFINED ENV{{{LWIP_FULL_SOURCES_ENV}}}) + idf_component_get_property(esphome_lwip_lib lwip COMPONENT_LIB) + get_target_property(esphome_lwip_srcs ${{esphome_lwip_lib}} SOURCES) +@FILTERS@ + set_property(TARGET ${{esphome_lwip_lib}} PROPERTY SOURCES ${{esphome_lwip_srcs}}) +endif()""" + + +def lwip_empty_source_gate(option: str, regex: str) -> str: + return ( + f" if(DEFINED {option} AND NOT {option})\n" + f' list(FILTER esphome_lwip_srcs EXCLUDE REGEX "{regex}")\n' + " endif()" + ) + + +def _lwip_empty_sources_filter() -> str: + gates = "\n".join(lwip_empty_source_gate(*entry) for entry in LWIP_EMPTY_SOURCES) + return _LWIP_EMPTY_SOURCES_FILTER.replace("@FILTERS@", gates) + + # Runs after project() so the walk has happened; catches the remaining # silent path where the top-level out-var was renamed. _LDGEN_OVERRIDE_CHECK = """\ @@ -348,14 +385,20 @@ project({CORE.name}) {ldgen_override_check} +{_lwip_empty_sources_filter()} + # Emit per-memory-type JSON size data for ESPHome to read post-build. # json2 stays small; raw dumps every symbol (~2s on a large map) and # this command runs inside the link edge, blocking everything downstream. +# The map is a BYPRODUCT so ninja knows the link writes it; IDF's size +# target depends on the map and can then be built in the same run as all. +# IDF's cmakev2 declares the map itself, so drop this line on that switch. add_custom_command( TARGET ${{CMAKE_PROJECT_NAME}}.elf POST_BUILD COMMAND ${{PYTHON}} -m esp_idf_size {size_ng_flag} --format=json2 -o ${{CMAKE_BINARY_DIR}}/esp_idf_size.json ${{CMAKE_PROJECT_NAME}}.map + BYPRODUCTS ${{CMAKE_BINARY_DIR}}/${{CMAKE_PROJECT_NAME}}.map WORKING_DIRECTORY ${{CMAKE_BINARY_DIR}} VERBATIM ) diff --git a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp index 1d55b06123..b7017bc105 100644 --- a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp +++ b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp @@ -17,6 +17,13 @@ #include "esphome/core/log.h" #include +#include + +// cancel_open exists from ESP-IDF 5.5.5 and 6.0.1. +#if (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 5, 5) && ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(6, 0, 0)) || \ + ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 1) +#define BLUEDROID_HAS_CANCEL_OPEN +#endif namespace esphome::bluetooth_connection { @@ -63,6 +70,12 @@ void BluedroidGattClient::loop() { // The one teardown safety net: a lost CLOSE_EVT, or a scheduled // teardown whose OPEN_EVT never arrives. if (millis() - this->disconnecting_started_ > ble_device_base::GATT_DISCONNECT_TIMEOUT_MS) { + if (this->conn_id_ == UNSET_CONN_ID) { + this->cancel_pending_open_(); + } else if (st == ClientState::CONNECTING) { + // CONNECT_EVT came but OPEN_EVT was lost, so our open still holds the ACL link. + this->check_and_log_error_("esp_ble_gap_disconnect", esp_ble_gap_disconnect(this->remote_bda_)); + } ESP_LOGE(TAG, "[%d] Timeout waiting for teardown, forcing IDLE", this->connection_index_); // Release before idling: a lost completion must not leak the cache. this->release_services(); @@ -140,6 +153,8 @@ void BluedroidGattClient::tracker_connect_() { this->services_released_ = false; this->seen_mtu_ = false; this->mtu_failed_ = false; + this->cancel_open_sent_ = false; + this->conn_id_ = UNSET_CONN_ID; this->enable_loop(); this->set_state(ClientState::CONNECTING); if (this->connection_type_ == ConnectionType::V3_WITHOUT_CACHE) { @@ -183,12 +198,28 @@ int BluedroidGattClient::gatt_disconnect() { // Arm the safety window: a lost OPEN_EVT must not leak the teardown. this->disconnecting_started_ = millis(); this->enable_loop(); + if (this->conn_id_ == UNSET_CONN_ID) { + // CANCEL_OPEN_EVT or a racing OPEN_EVT settles the slot. + this->cancel_pending_open_(); + } return 0; } this->unconditional_disconnect_(); return 0; } +void BluedroidGattClient::cancel_pending_open_() { +#ifdef BLUEDROID_HAS_CANCEL_OPEN + esp_ble_gattc_cancel_open_params_t params{}; + params.gattc_if = this->gattc_if_; + memcpy(params.remote_bda, this->remote_bda_, sizeof(esp_bd_addr_t)); + // A refused cancel schedules no CANCEL_OPEN_EVT, so the teardown stays cancellable. + if (this->check_and_log_error_("esp_ble_gattc_cancel_open", esp_ble_gattc_cancel_open(¶ms)) == ESP_OK) { + this->cancel_open_sent_ = true; + } +#endif +} + void BluedroidGattClient::unconditional_disconnect_() { ESP_LOGI(TAG, "[%d] Disconnecting (conn_id: %d)", this->connection_index_, this->conn_id_); if (this->conn_id_ == UNSET_CONN_ID) { @@ -214,6 +245,10 @@ bool BluedroidGattClient::cancel_gatt_disconnect() { if (this->state() != ClientState::CONNECTING || !this->disconnect_pending()) { return false; } + if (this->cancel_open_sent_) { + // The cancel already went out, so CANCEL_OPEN_EVT ends it. + return false; + } this->want_disconnect_ = false; return true; } @@ -574,6 +609,8 @@ void BluedroidGattClient::handle_open_evt_(esp_ble_gattc_cb_param_t *param) { this->listener_->on_connection_state(false, 0, param->open.status); return; } + // ALREADY_OPEN on an existing link sends no CONNECT_EVT, so take the id here. + this->conn_id_ = param->open.conn_id; if (this->disconnect_pending()) { // Open resolved with a teardown scheduled: close now (conn_id_ stays set // so CLOSE_EVT still matches). @@ -617,6 +654,11 @@ void BluedroidGattClient::handle_disconnect_evt_(esp_ble_gattc_cb_param_t *param // Active close delivers CLOSE_EVT first; never walk back to DISCONNECTING. return; } + if (this->conn_id_ == UNSET_CONN_ID) { + // No CONNECT_EVT for this attempt yet, so the event belongs to an earlier + // link, such as one the teardown timeout forced down. + return; + } // Passive disconnect: wait for CLOSE_EVT before going IDLE (reconnecting // earlier makes the controller reject with 133 or assert) and before // reporting - the wrapper frees the slot on the report, and a freed slot @@ -645,6 +687,10 @@ bool BluedroidGattClient::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga case ESP_GATTC_CONNECT_EVT: { if (!this->check_addr_(param->connect.remote_bda)) return false; + // Every client interface gets CONNECT_EVT for every new link; only an + // attempt in flight owns it. An idle slot must not adopt another slot's link. + if (this->state() != ClientState::CONNECTING) + break; this->conn_id_ = param->connect.conn_id; // MTU request here rather than OPEN_EVT, matching the IDF examples. auto ret = esp_ble_gattc_send_mtu_req(this->gattc_if_, param->connect.conn_id); @@ -656,8 +702,15 @@ bool BluedroidGattClient::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga break; } case ESP_GATTC_OPEN_EVT: { - if (!this->check_addr_(param->open.remote_bda)) + if (!this->check_addr_(param->open.remote_bda)) { + // Our interface (filtered above), stale address, so nobody tracks this link. + if (param->open.status == ESP_GATT_OK || param->open.status == ESP_GATT_ALREADY_OPEN) { + ESP_LOGW(TAG, "[%d] Closing link left by an abandoned open", this->connection_index_); + this->check_and_log_error_("esp_ble_gattc_close", esp_ble_gattc_close(this->gattc_if_, param->open.conn_id)); + return true; + } return false; + } this->handle_open_evt_(param); break; } @@ -748,6 +801,24 @@ bool BluedroidGattClient::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga this->listener_->on_notify_data(param->notify.handle, param->notify.value, param->notify.value_len); break; } +#ifdef BLUEDROID_HAS_CANCEL_OPEN + case ESP_GATTC_CANCEL_OPEN_EVT: { + if (!this->check_addr_(param->cancel_open.remote_bda)) + return false; + if (param->cancel_open.status != ESP_GATT_OK) { + // Too late to cancel: OPEN_EVT follows and settles the slot. The latch + // stays set because the event cannot be tied to this attempt's cancel. + break; + } + if (this->state() == ClientState::CONNECTING && this->disconnect_pending()) { + ESP_LOGD(TAG, "[%d] Pending open cancelled", this->connection_index_); + this->release_services(); + this->set_idle_(); + this->listener_->on_connection_state(false, 0, ESP_GATT_CONN_CONN_CANCEL); + } + break; + } +#endif default: break; } diff --git a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h index a4e9edec23..ef0ca223cb 100644 --- a/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h +++ b/esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h @@ -100,6 +100,7 @@ class BluedroidGattClient final : public esp32_ble_tracker::ESPBTClient, public int handle_search_cmpl_(esp_gatt_status_t status); void deliver_pending_search_(); void unconditional_disconnect_(); + void cancel_pending_open_(); void set_idle_(); void set_disconnecting_(); esp_err_t update_conn_params_(uint16_t min_interval, uint16_t max_interval, uint16_t latency, uint16_t timeout, @@ -133,6 +134,9 @@ class BluedroidGattClient final : public esp32_ble_tracker::ESPBTClient, public bool seen_mtu_ : 1 {false}; // The MTU request was refused at CONNECT_EVT; OPEN_EVT reports instead. bool mtu_failed_ : 1 {false}; + // esp_ble_gattc_cancel_open() was accepted; CANCEL_OPEN_EVT or OPEN_EVT ends + // the attempt, so the scheduled teardown can no longer be cancelled. + bool cancel_open_sent_ : 1 {false}; // Search issued at OPEN_EVT overlaps the MTU exchange; discover_services() // completes from it. Reset by set_idle_(). static_assert(static_cast(SearchState::REPORT_PENDING) < (1 << 4), "search_state_ bitfield too narrow"); diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index e14f5e7e86..6ecae24e29 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -315,10 +315,19 @@ FINAL_VALIDATE_SCHEMA = ota_esphome_final_validate FILTER_SOURCE_FILES = filter_source_files_from_defines( - {"ota_esphome_noise.cpp": "USE_OTA_ENCRYPTION"} + { + "ota_esphome_noise.cpp": "USE_OTA_ENCRYPTION", + "ota_esphome_inflate_session.cpp": "USE_OTA_DEFLATE", + "ota_esphome_inflate.c": "USE_OTA_DEFLATE", + } ) +def enable_deflate() -> None: + """Compile the on-the-fly inflater for compressed uploads.""" + cg.add_define("USE_OTA_DEFLATE") + + @coroutine_with_priority(CoroPriority.OTA_UPDATES) async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) @@ -340,6 +349,10 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_ALLOW_PARTITION_ACCESS): cg.add_define("USE_OTA_PARTITIONS") + # ESP8266 and RP2040 inflate gzip at reboot; the rest inflate on the fly + if not (CORE.is_esp8266 or CORE.is_rp2): + enable_deflate() + # One key per device: an api encryption block supplies it (static or # runtime) and offers; the ota block only adds the requirement api_conf = CORE.config.get(CONF_API) or {} diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 22580343e8..c399c0901c 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -22,8 +22,10 @@ #include "esphome/core/lwip_fast_select.h" #endif +#include #include #include +#include #include namespace esphome { @@ -47,6 +49,8 @@ static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds // practice for a lost chunk ack (1.5 + 3 + 6 + 12 + 24 + 48 s); the CLI waits // longer (espota2.DATA_PHASE_TIMEOUT) so the device is free before it retries static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 105000; +static constexpr uint32_t OTA_PROGRESS_INTERVAL_MS = 1000; +static constexpr size_t OTA_SIZE_FIELD_BYTES = 4; // sizes on the wire are 4 bytes MSB first // Single-instance pointer — multi-port configs are rejected in final_validate. // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) @@ -194,12 +198,23 @@ static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_NOISE = 0x08; +static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_DEFLATE = 0x10; // Noise needs the extended protocol: the prologue binds the 2-byte feature ack static constexpr uint8_t CLIENT_NOISE_FEATURES = CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_NOISE = 0x04; +// Raw deflate, window <= OTA_INFLATE_WINDOW_SIZE. Binding once offered: the +// client must then send the image size frame and a deflate stream. +static constexpr uint8_t SERVER_FEATURE_SUPPORTS_DEFLATE = 0x08; + +#ifdef USE_OTA_ENCRYPTION +inline bool ESPHomeOTAComponent::noise_offered_() const { + return (this->handshake_buf_[1] & SERVER_FEATURE_SUPPORTS_NOISE) != 0 && + (this->ota_features_ & CLIENT_NOISE_FEATURES) == CLIENT_NOISE_FEATURES; +} +#endif inline bool ESPHomeOTAComponent::extended_proto_() const { #ifdef USE_OTA_ENCRYPTION_REQUIRED @@ -305,7 +320,7 @@ void ESPHomeOTAComponent::handle_handshake_() { this->transition_ota_state_(OTAState::FEATURE_ACK); const bool supports_compression = - (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_COMPRESSION) != 0 && this->backend_->supports_compression(); + (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_COMPRESSION) != 0 && ota::OTABackend::supports_compression(); // Compose the feature-ack response. When the client negotiates the extended protocol we emit // a 2-byte response (marker + server feature flags); otherwise we emit the single-byte @@ -325,6 +340,28 @@ void ESPHomeOTAComponent::handle_handshake_() { #elif defined(USE_OTA_ENCRYPTION) // A yaml key always exists: validation rejects the all-zeros key this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; +#endif +#ifdef USE_OTA_ENCRYPTION + // Reserve the noise session before the optional inflate buffer, so the + // required allocation is not starved by the compression window + if (this->noise_offered_()) { + this->noise_reserve_session_(); + } +#endif +#ifdef USE_OTA_DEFLATE + // Offered only once the session memory is in hand; else uncompressed + if ((this->ota_features_ & CLIENT_FEATURE_SUPPORTS_DEFLATE) != 0) { + // Value initialized: a corrupt stream that back references the + // window before it is filled then copies zeros, never stale memory. + // Default placement, PSRAM first where present: the session lives for one + // upload and keeps 4.9 KB of internal heap free while it runs + this->inflate_ = RAMAllocator().make_unique(); + if (this->inflate_ != nullptr) { + this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_DEFLATE; + } else { + ESP_LOGW(TAG, "No memory to inflate"); + } + } #endif } else { this->handshake_buf_[0] = @@ -343,8 +380,7 @@ void ESPHomeOTAComponent::handle_handshake_() { #ifdef USE_OTA_ENCRYPTION // Latch the offer actually sent: a key activating between the two // states must not start a session the client never expects - if ((this->handshake_buf_[1] & SERVER_FEATURE_SUPPORTS_NOISE) != 0 && - (this->ota_features_ & CLIENT_NOISE_FEATURES) == CLIENT_NOISE_FEATURES) { + if (this->noise_offered_()) { // handshake_buf_ still holds the feature ack composed above; a // would-block re-entry lands here without rebuilding it if (!this->noise_start_session_(this->handshake_buf_[1])) { @@ -442,16 +478,11 @@ void ESPHomeOTAComponent::handle_data_() { // Backend calls overwrite this with OK; reset to UNKNOWN before any // goto error that follows a successful begin()/write() ota::OTAResponseTypes error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; - size_t total = 0; - uint32_t last_progress = 0; - uint32_t last_data_ms = 0; + DataTransfer xfer; uint8_t buf[OTA_BUFFER_SIZE]; char *sbuf = reinterpret_cast(buf); - size_t ota_size; + size_t image_size; ota::OTAType ota_type = ota::OTA_TYPE_UPDATE_APP; -#if USE_OTA_VERSION == 2 - size_t size_acknowledged = 0; -#endif // Set socket timeouts and blocking mode (see strategy table above) struct timeval tv; @@ -480,16 +511,20 @@ void ESPHomeOTAComponent::handle_data_() { ESP_LOGV(TAG, "OTA type is 0x%02x", ota_type); // Read size, 4 bytes MSB first - if (!this->data_readall_(buf, 4)) { + if (!this->read_size_(buf, xfer.ota_size, LOG_STR("size"))) { // The first request byte is the type on the extended protocol; a close after it was a cut-off request if (!this->extended_proto_() && this->client_left_before_start_()) return; this->log_read_error_(LOG_STR("size")); goto error; // NOLINT(cppcoreguidelines-avoid-goto) } - ota_size = (static_cast(buf[0]) << 24) | (static_cast(buf[1]) << 16) | - (static_cast(buf[2]) << 8) | buf[3]; - ESP_LOGV(TAG, "Size is %zu bytes", ota_size); + image_size = xfer.ota_size; +#ifdef USE_OTA_DEFLATE + if (this->inflate_ != nullptr && !this->read_size_(buf, image_size, LOG_STR("image size"))) { + this->log_read_error_(LOG_STR("image size")); + goto error; // NOLINT(cppcoreguidelines-avoid-goto) + } +#endif #ifndef USE_OTA_PARTITIONS if (ota_type != ota::OTA_TYPE_UPDATE_APP) { @@ -509,7 +544,7 @@ void ESPHomeOTAComponent::handle_data_() { #endif // begin() returns quickly; flash sectors are erased incrementally during write(). - error_code = this->backend_->begin(ota_size, ota_type); + error_code = this->backend_->begin(image_size, ota_type); if (error_code != ota::OTA_RESPONSE_OK) goto error; // NOLINT(cppcoreguidelines-avoid-goto) @@ -529,77 +564,25 @@ void ESPHomeOTAComponent::handle_data_() { // Acknowledge MD5 OK - 1 byte this->data_write_byte_(ota::OTA_RESPONSE_BIN_MD5_OK); - // Track when we last received data so a silently-vanished peer (no FIN/RST - // delivered, e.g. uploader killed mid-transfer or NAT/router dropped state) - // can't wedge the device indefinitely. Without this, the loop only exits - // on actual data, EOF, or a non-EWOULDBLOCK error from read(), and lwIP - // TCP keepalive isn't enabled here. - last_data_ms = millis(); - while (total < ota_size) { - if (millis() - last_data_ms > OTA_SOCKET_TIMEOUT_DATA) { - ESP_LOGW(TAG, "No data received for %u ms", (unsigned) OTA_SOCKET_TIMEOUT_DATA); - error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; + xfer.last_data_ms = millis(); +#ifdef USE_OTA_DEFLATE + if (this->inflate_ != nullptr) { + error_code = this->inflate_data_(buf, image_size, xfer); + if (error_code != ota::OTA_RESPONSE_OK) goto error; // NOLINT(cppcoreguidelines-avoid-goto) - } - size_t remaining = ota_size - total; - size_t requested = remaining < OTA_BUFFER_SIZE ? remaining : OTA_BUFFER_SIZE; - ssize_t read; -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ != nullptr) { - // One frame per call; noise_read_data_ waits internally (readall_), so - // there is no would-block retry here and failures are already logged. - read = this->noise_read_data_(buf, requested); - if (read <= 0) { - if (this->remote_closed_) - this->log_remote_closed_(LOG_STR("data")); + } else +#endif + { + while (xfer.total < xfer.ota_size) { + ssize_t read = this->receive_data_(buf, xfer); + if (read < 0) { error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; goto error; // NOLINT(cppcoreguidelines-avoid-goto) } - } else -#endif - { - read = this->client_->read(buf, requested); - if (read == -1) { - const int err = errno; - if (this->would_block_(err)) { - // read() already waited up to SO_RCVTIMEO for data, just feed WDT - App.feed_wdt(); - continue; - } - ESP_LOGW(TAG, "Read err %d", err); - error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; + error_code = this->write_flash_(buf, read); + if (error_code != ota::OTA_RESPONSE_OK) goto error; // NOLINT(cppcoreguidelines-avoid-goto) - } else if (read == 0) { - ESP_LOGW(TAG, "Remote closed"); - error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; - goto error; // NOLINT(cppcoreguidelines-avoid-goto) - } - } - - last_data_ms = millis(); - error_code = this->backend_->write(buf, read); - if (error_code != ota::OTA_RESPONSE_OK) { - ESP_LOGW(TAG, "Flash write err %d", error_code); - goto error; // NOLINT(cppcoreguidelines-avoid-goto) - } - total += read; -#if USE_OTA_VERSION == 2 - while (size_acknowledged + OTA_BLOCK_SIZE <= total || (total == ota_size && size_acknowledged < ota_size)) { - this->data_write_byte_(ota::OTA_RESPONSE_CHUNK_OK); - size_acknowledged += OTA_BLOCK_SIZE; - } -#endif - - uint32_t now = millis(); - if (now - last_progress > 1000) { - last_progress = now; - float percentage = (total * 100.0f) / ota_size; - ESP_LOGD(TAG, "Progress: %0.1f%%", percentage); -#ifdef USE_OTA_STATE_LISTENER - this->notify_state_(ota::OTA_IN_PROGRESS, percentage, 0); -#endif - // feed watchdog and give other tasks a chance to run - this->yield_and_feed_watchdog_(); + this->ack_written_(xfer); } } @@ -815,6 +798,90 @@ bool ESPHomeOTAComponent::try_write_(size_t to_write, const LogString *desc) { return this->handshake_buf_pos_ >= to_write; } +bool ESPHomeOTAComponent::read_size_(uint8_t *buf, size_t &size, const LogString *desc) { + if (!this->data_readall_(buf, OTA_SIZE_FIELD_BYTES)) + return false; + size = encode_uint32(buf[0], buf[1], buf[2], buf[3]); + ESP_LOGV(TAG, "%s is %zu bytes", LOG_STR_ARG(desc), size); + return true; +} + +ota::OTAResponseTypes ESPHomeOTAComponent::write_flash_(uint8_t *data, size_t len) { + ota::OTAResponseTypes result = this->backend_->write(data, len); + if (result != ota::OTA_RESPONSE_OK) { + ESP_LOGW(TAG, "Flash write err %d", result); + } + return result; +} + +ssize_t ESPHomeOTAComponent::receive_data_(uint8_t *buf, DataTransfer &xfer) { + const size_t remaining = xfer.ota_size - xfer.total; + const size_t requested = std::min(remaining, OTA_BUFFER_SIZE); + ssize_t read; + for (;;) { + // A silently-vanished peer (no FIN/RST delivered, e.g. uploader killed + // mid-transfer or NAT/router dropped state) must not wedge the device: + // read() only fails on EOF or a real error, and lwIP TCP keepalive isn't + // enabled here. + if (millis() - xfer.last_data_ms > OTA_SOCKET_TIMEOUT_DATA) { + ESP_LOGW(TAG, "No data received for %u ms", (unsigned) OTA_SOCKET_TIMEOUT_DATA); + return -1; + } +#ifdef USE_OTA_ENCRYPTION + if (this->noise_ != nullptr) { + // One frame per call; noise_read_data_ waits internally (readall_), so + // there is no would-block retry here and failures are already logged. + read = this->noise_read_data_(buf, requested); + if (read <= 0) { + if (this->remote_closed_) + this->log_remote_closed_(LOG_STR("data")); + return -1; + } + break; + } +#endif + read = this->client_->read(buf, requested); + if (read > 0) + break; + if (read == 0) { + this->log_remote_closed_(LOG_STR("data")); + return -1; + } + if (!this->would_block_(errno)) { + this->log_socket_error_(LOG_STR("data")); + return -1; + } + // read() already waited up to SO_RCVTIMEO for data, just feed WDT + App.feed_wdt(); + } + + const uint32_t now = millis(); + xfer.last_data_ms = now; + xfer.total += read; + this->ack_received_(xfer); + if (now - xfer.last_progress > OTA_PROGRESS_INTERVAL_MS) { + xfer.last_progress = now; + float percentage = (xfer.total * 100.0f) / xfer.ota_size; + ESP_LOGD(TAG, "Progress: %0.1f%%", percentage); +#ifdef USE_OTA_STATE_LISTENER + this->notify_state_(ota::OTA_IN_PROGRESS, percentage, 0); +#endif + // feed watchdog and give other tasks a chance to run + this->yield_and_feed_watchdog_(); + } + return read; +} + +void ESPHomeOTAComponent::send_chunk_acks_(DataTransfer &xfer) { +#if USE_OTA_VERSION == 2 + while (xfer.acknowledged + OTA_BLOCK_SIZE <= xfer.total || + (xfer.total == xfer.ota_size && xfer.acknowledged < xfer.ota_size)) { + this->data_write_byte_(ota::OTA_RESPONSE_CHUNK_OK); + xfer.acknowledged += OTA_BLOCK_SIZE; + } +#endif +} + void ESPHomeOTAComponent::cleanup_connection_() { this->client_->close(); this->client_ = nullptr; @@ -829,6 +896,9 @@ void ESPHomeOTAComponent::cleanup_connection_() { #endif #ifdef USE_OTA_ENCRYPTION this->noise_ = nullptr; +#endif +#ifdef USE_OTA_DEFLATE + this->inflate_ = nullptr; #endif // Intentionally no disable_loop() — letting loop() run one more iteration catches // any connection that queued on the listener mid-session (otherwise the wake flag, diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index 6f04b78da5..910a5b3d99 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -7,6 +7,9 @@ #ifdef USE_OTA_ENCRYPTION #include "esphome/components/noise/noise_handshake.h" #endif +#ifdef USE_OTA_DEFLATE +#include "ota_esphome_inflate.h" +#endif #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/preferences.h" @@ -89,6 +92,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { // The api server's live context when it exists, otherwise our own (a build // time key, or the saved key loaded in safe mode) const noise::NoiseContext &noise_context_() const; + // True once the feature ack offers noise and the client asked for it + bool noise_offered_() const; + void noise_reserve_session_(); bool noise_start_session_(uint8_t server_feature_flags); bool handle_noise_handshake_(); bool noise_try_read_frame_(); @@ -120,6 +126,38 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { return this->readall_(buf, len); } + // Upload accounting shared by the data loop and the inflate read callback + struct DataTransfer { + size_t ota_size{0}; // bytes the client sends + size_t total{0}; // bytes received so far +#if USE_OTA_VERSION == 2 + size_t acknowledged{0}; +#endif + uint32_t last_data_ms{0}; + uint32_t last_progress{0}; + }; + // Up to OTA_BUFFER_SIZE bytes into buf; returns bytes read, -1 on failure (logged) + ssize_t receive_data_(uint8_t *buf, DataTransfer &xfer); + // Raw lwIP cannot service the radio during a sector write, so the ack waits + // for the write there; a socket task lets the next block arrive meanwhile +#ifdef USE_SOCKET_IMPL_LWIP_TCP + static constexpr bool ACK_AFTER_WRITE = true; +#else + static constexpr bool ACK_AFTER_WRITE = false; +#endif + void send_chunk_acks_(DataTransfer &xfer); + inline void ack_received_(DataTransfer &xfer) { + if (!ACK_AFTER_WRITE) + this->send_chunk_acks_(xfer); + } + inline void ack_written_(DataTransfer &xfer) { + if (ACK_AFTER_WRITE) + this->send_chunk_acks_(xfer); + } + inline bool read_size_(uint8_t *buf, size_t &size, const LogString *desc); + // Writes to the backend and logs a failure + ota::OTAResponseTypes write_flash_(uint8_t *data, size_t len); + bool try_read_(size_t to_read, const LogString *desc); bool try_write_(size_t to_write, const LogString *desc); @@ -175,6 +213,34 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { static_assert(OTA_BUFFER_SIZE >= NOISE_CLIENT_MAX_PLAINTEXT + noise::MAC_SIZE, "OTA_BUFFER_SIZE must fit a full encrypted data frame"); #endif +#ifdef USE_OTA_DEFLATE + // At least 1 << espota2.DEFLATE_WINDOW_BITS; also the inflate output buffer + static constexpr size_t OTA_INFLATE_WINDOW_SIZE = 4096; + // Heap-allocated only while a deflate upload is negotiated; the decoder + // state is the base so the read callback can recover the session + struct InflateSession : OtaInflateState { + // The session outlives the upload it serves, but these three are borrowed + // from inflate_data_'s caller and dangle once that call returns; only that + // call, and the flush and read callback it drives, may read them + ESPHomeOTAComponent *self; + DataTransfer *xfer; + uint8_t *in; // caller's buffer for the compressed input + size_t image_size; + size_t written; // inflated bytes in flash + size_t flushed; // bytes of the current window already in flash + ota::OTAResponseTypes error; // first failure inside the read callback + uint8_t window[OTA_INFLATE_WINDOW_SIZE]; + }; +#ifndef CLANG_TIDY // static analysis sets every define at once + static_assert(!ota::OTABackend::supports_compression(), + "USE_OTA_DEFLATE is for backends that cannot store a gzip image"); +#endif + // Writes the decoded bytes not yet in flash without moving dest + ota::OTAResponseTypes inflate_flush_(InflateSession &session); + ota::OTAResponseTypes inflate_data_(uint8_t *in, size_t image_size, DataTransfer &xfer); + RAMUniquePtr inflate_; +#endif + static constexpr uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45}; // Derived from the feature byte; storing it would pad the trailing bytes bool extended_proto_() const; diff --git a/esphome/components/esphome/ota/ota_esphome_inflate.c b/esphome/components/esphome/ota/ota_esphome_inflate.c new file mode 100644 index 0000000000..1208f82ae8 --- /dev/null +++ b/esphome/components/esphome/ota/ota_esphome_inflate.c @@ -0,0 +1,498 @@ +/* + * uzlib - tiny deflate/inflate library (deflate, gzip, zlib) + * + * Copyright (c) 2003 by Joergen Ibsen / Jibz + * All Rights Reserved + * http://www.ibsensoftware.com/ + * + * Copyright (c) 2014-2018 by Paul Sokolovsky + * + * This software is provided 'as-is', without any express + * or implied warranty. In no event will the authors be + * held liable for any damages arising from the use of + * this software. + * + * Permission is granted to anyone to use this software + * for any purpose, including commercial applications, + * and to alter it and redistribute it freely, subject to + * the following restrictions: + * + * 1. The origin of this software must not be + * misrepresented; you must not claim that you + * wrote the original software. If you use this + * software in a product, an acknowledgment in + * the product documentation would be appreciated + * but is not required. + * + * 2. Altered source versions must be plainly marked + * as such, and must not be misrepresented as + * being the original software. + * + * 3. This notice may not be removed or altered from + * any source distribution. + */ + +/* + * Altered for ESPHome: this is the raw deflate decoder from uzlib's + * tinflate.c (v2.9.5) with the gzip/zlib header parsers, checksums, + * runtime table builder and in-memory (non ring window) output path + * removed, and the public names prefixed with ota_inflate. + */ + +#include "ota_esphome_inflate.h" + +#include + +#define TINF_OK OTA_INFLATE_OK +#define TINF_DONE OTA_INFLATE_DONE +#define TINF_DATA_ERROR OTA_INFLATE_DATA_ERROR +#define TINF_DICT_ERROR OTA_INFLATE_DICT_ERROR +#define TINF_DATA struct OtaInflateState +#define TINF_TREE struct OtaInflateTree +#define TINF_ARRAY_SIZE(arr) (sizeof(arr) / sizeof(*(arr))) + +/* every output byte also goes into the ring window */ +#define TINF_PUT(d, c) \ + { \ + *d->dest++ = c; \ + d->dict_ring[d->dict_idx++] = c; \ + if (d->dict_idx == d->dict_size) \ + d->dict_idx = 0; \ + } + +/* --------------------------------------------------- * + * -- constant tables (upstream builds them at runtime) -- * + * --------------------------------------------------- */ + +static const unsigned char LENGTH_BITS[30] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, + 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5}; +static const unsigned short LENGTH_BASE[30] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, + 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258}; + +static const unsigned char DIST_BITS[30] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, + 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13}; +static const unsigned short DIST_BASE[30] = {1, 2, 3, 4, 5, 7, 9, 13, 17, 25, + 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, + 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577}; + +/* special ordering of code length codes */ +static const unsigned char CLCIDX[] = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + +/* ----------------------- * + * -- utility functions -- * + * ----------------------- */ + +/* given an array of code lengths, build a tree */ +static void tinf_build_tree(TINF_TREE *t, const unsigned char *lengths, unsigned int num) { + unsigned short offs[16]; + unsigned int i, sum; + + /* clear code length count table */ + for (i = 0; i < 16; ++i) + t->table[i] = 0; + + /* scan symbol lengths, and sum code length counts */ + for (i = 0; i < num; ++i) + t->table[lengths[i]]++; + + /* In the lengths array, 0 means unused code. So, t->table[0] now contains + number of unused codes. But table's purpose is to contain # of codes of + particular length, and there're 0 codes of length 0. */ + t->table[0] = 0; + + /* compute offset table for distribution sort */ + for (sum = 0, i = 0; i < 16; ++i) { + offs[i] = sum; + sum += t->table[i]; + } + + /* create code->symbol translation table (symbols sorted by code) */ + for (i = 0; i < num; ++i) { + if (lengths[i]) + t->trans[offs[lengths[i]]++] = i; + } +} + +/* ---------------------- * + * -- decode functions -- * + * ---------------------- */ + +static unsigned char uzlib_get_byte(TINF_DATA *d) { + /* If end of source buffer is not reached, return next byte from source + buffer. */ + if (d->source < d->source_limit) { + return *d->source++; + } + + /* Otherwise if there's callback and we haven't seen EOF yet, try to + read next byte using it. (Note: the callback can also update ->source + and ->source_limit). */ + if (!d->eof) { + int val = d->source_read_cb(d); + if (val >= 0) { + return (unsigned char) val; + } + } + + /* Otherwise, we hit EOF (either from ->source_read_cb() or from exhaustion + of the buffer), and it will be "sticky", i.e. further calls to this + function will end up here too. */ + d->eof = true; + + return 0; +} + +/* get one bit from source stream */ +static int tinf_getbit(TINF_DATA *d) { + unsigned int bit; + + /* check if tag is empty */ + if (!d->bitcount--) { + /* load next tag */ + d->tag = uzlib_get_byte(d); + d->bitcount = 7; + } + + /* shift bit out of tag */ + bit = d->tag & 0x01; + d->tag >>= 1; + + return bit; +} + +/* read a num bit value from a stream and add base */ +static unsigned int tinf_read_bits(TINF_DATA *d, int num, int base) { + unsigned int val = 0; + + /* read num bits */ + if (num) { + unsigned int limit = 1 << (num); + unsigned int mask; + + for (mask = 1; mask < limit; mask *= 2) + if (tinf_getbit(d)) + val += mask; + } + + return val + base; +} + +/* given a data stream and a tree, decode a symbol */ +static int tinf_decode_symbol(TINF_DATA *d, TINF_TREE *t) { + int sum = 0, cur = 0, len = 0; + + /* get more bits while code value is above sum */ + do { + cur = 2 * cur + tinf_getbit(d); + + if (++len == TINF_ARRAY_SIZE(t->table)) { + return TINF_DATA_ERROR; + } + + sum += t->table[len]; + cur -= t->table[len]; + + } while (cur >= 0); + + sum += cur; + if (sum < 0 || sum >= t->size) { + return TINF_DATA_ERROR; + } + + return t->trans[sum]; +} + +/* given a data stream, decode dynamic trees from it */ +static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) { + /* code lengths for 288 literal/len symbols and 32 dist symbols */ + unsigned char lengths[288 + 32]; + unsigned int hlit, hdist, hclen, hlimit; + unsigned int i, num, length; + + /* get 5 bits HLIT (257-286) */ + hlit = tinf_read_bits(d, 5, 257); + + /* get 5 bits HDIST (1-32) */ + hdist = tinf_read_bits(d, 5, 1); + + /* get 4 bits HCLEN (4-19) */ + hclen = tinf_read_bits(d, 4, 4); + + for (i = 0; i < 19; ++i) + lengths[i] = 0; + + /* read code lengths for code length alphabet */ + for (i = 0; i < hclen; ++i) { + /* get 3 bits code length (0-7) */ + unsigned int clen = tinf_read_bits(d, 3, 0); + + lengths[CLCIDX[i]] = clen; + } + + /* build code length tree, temporarily use length tree */ + tinf_build_tree(lt, lengths, 19); + + /* decode code lengths for the dynamic trees */ + hlimit = hlit + hdist; + for (num = 0; num < hlimit;) { + int sym = tinf_decode_symbol(d, lt); + unsigned char fill_value = 0; + int lbits, lbase = 3; + + /* error decoding */ + if (sym < 0) + return sym; + + switch (sym) { + case 16: + /* copy previous code length 3-6 times (read 2 bits) */ + if (num == 0) + return TINF_DATA_ERROR; + fill_value = lengths[num - 1]; + lbits = 2; + break; + case 17: + /* repeat code length 0 for 3-10 times (read 3 bits) */ + lbits = 3; + break; + case 18: + /* repeat code length 0 for 11-138 times (read 7 bits) */ + lbits = 7; + lbase = 11; + break; + default: + /* values 0-15 represent the actual code lengths */ + lengths[num++] = sym; + /* continue the for loop */ + continue; + } + + /* special code length 16-18 are handled here */ + length = tinf_read_bits(d, lbits, lbase); + if (num + length > hlimit) + return TINF_DATA_ERROR; + for (; length; --length) { + lengths[num++] = fill_value; + } + } + + /* Check that there's "end of block" symbol */ + if (lengths[256] == 0) { + return TINF_DATA_ERROR; + } + + /* build dynamic trees */ + tinf_build_tree(lt, lengths, hlit); + tinf_build_tree(dt, lengths + hlit, hdist); + + return TINF_OK; +} + +/* build the fixed huffman trees (RFC 1951 3.2.6) through the generic tree + builder; altered from upstream, which unrolls them by hand */ +static void tinf_build_fixed_trees(TINF_TREE *lt, TINF_TREE *dt) { + unsigned char lengths[288]; + unsigned int i; + + for (i = 0; i < 144; ++i) + lengths[i] = 8; + for (; i < 256; ++i) + lengths[i] = 9; + for (; i < 280; ++i) + lengths[i] = 7; + for (; i < 288; ++i) + lengths[i] = 8; + tinf_build_tree(lt, lengths, 288); + + for (i = 0; i < 32; ++i) + lengths[i] = 5; + tinf_build_tree(dt, lengths, 32); +} + +/* ----------------------------- * + * -- block inflate functions -- * + * ----------------------------- */ + +/* given a stream and two trees, inflate next chunk of output (a byte or more) */ +static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) { + if (d->curlen == 0) { + unsigned int offs; + int dist; + int sym = tinf_decode_symbol(d, lt); + + if (d->eof) { + return TINF_DATA_ERROR; + } + + if (sym < 0) { + return sym; + } + + /* literal byte */ + if (sym < 256) { + TINF_PUT(d, sym); + return TINF_OK; + } + + /* end of block */ + if (sym == 256) { + return TINF_DONE; + } + + /* substring from sliding dictionary */ + sym -= 257; + if (sym >= 29) { + return TINF_DATA_ERROR; + } + + /* possibly get more bits from length code */ + d->curlen = tinf_read_bits(d, LENGTH_BITS[sym], LENGTH_BASE[sym]); + + dist = tinf_decode_symbol(d, dt); + if (dist < 0 || dist >= 30) { + return TINF_DATA_ERROR; + } + + /* possibly get more bits from distance code */ + offs = tinf_read_bits(d, DIST_BITS[dist], DIST_BASE[dist]); + + /* calculate and validate actual LZ offset to use */ + if (offs > d->dict_size) { + return TINF_DICT_ERROR; + } + /* Note: we don't try to catch offset which points to not yet filled + part of the dictionary here. Doing so would require keeping another + variable to track "filled in" size of the dictionary. Appearance of + such an offset cannot lead to accessing memory outside of the + dictionary buffer, and clients which don't want to leak unrelated + information, should explicitly initialize dictionary buffer passed + to uzlib. */ + + d->lz_off = d->dict_idx - offs; + if (d->lz_off < 0) { + d->lz_off += d->dict_size; + } + } + + /* copy next byte from dict substring */ + TINF_PUT(d, d->dict_ring[d->lz_off]); + if ((unsigned) ++d->lz_off == d->dict_size) { + d->lz_off = 0; + } + d->curlen--; + return TINF_OK; +} + +/* inflate next byte from uncompressed block of data */ +static int tinf_inflate_uncompressed_block(TINF_DATA *d) { + if (d->curlen == 0) { + unsigned int length, invlength; + + /* get length */ + length = uzlib_get_byte(d); + length += 256 * uzlib_get_byte(d); + /* get one's complement of length */ + invlength = uzlib_get_byte(d); + invlength += 256 * uzlib_get_byte(d); + /* check length */ + if (length != (~invlength & 0x0000ffff)) + return TINF_DATA_ERROR; + + /* increment length to properly return TINF_DONE below, without + producing data at the same time */ + d->curlen = length + 1; + + /* make sure we start next block on a byte boundary */ + d->bitcount = 0; + } + + if (--d->curlen == 0) { + return TINF_DONE; + } + + unsigned char c = uzlib_get_byte(d); + TINF_PUT(d, c); + return TINF_OK; +} + +/* ---------------------- * + * -- public functions -- * + * ---------------------- */ + +/* initialize decompression structure */ +void ota_inflate_init(TINF_DATA *d, unsigned char *dict, unsigned int dict_len) { + d->source = NULL; + d->source_limit = NULL; + d->tag = 0; + d->eof = 0; + d->bitcount = 0; + d->lz_off = 0; + d->bfinal = 0; + d->btype = -1; + d->dict_size = dict_len; + d->dict_ring = dict; + d->dict_idx = 0; + d->curlen = 0; + d->ltree.trans = d->ltrans; + d->ltree.size = TINF_ARRAY_SIZE(d->ltrans); + d->dtree.trans = d->dtrans; + d->dtree.size = TINF_ARRAY_SIZE(d->dtrans); +} + +/* inflate next output bytes from compressed stream */ +int ota_inflate(TINF_DATA *d) { + do { + int res; + + /* start a new block */ + if (d->btype == -1) { + int old_btype; + next_blk: + old_btype = d->btype; + /* read final block flag */ + d->bfinal = tinf_getbit(d); + /* read block type (2 bits) */ + d->btype = tinf_read_bits(d, 2, 0); + + if (d->btype == 1 && old_btype != 1) { + /* build fixed huffman trees */ + tinf_build_fixed_trees(&d->ltree, &d->dtree); + } else if (d->btype == 2) { + /* decode trees from stream */ + res = tinf_decode_trees(d, &d->ltree, &d->dtree); + if (res != TINF_OK) { + return res; + } + } + } + + /* process current block */ + switch (d->btype) { + case 0: + /* decompress uncompressed block */ + res = tinf_inflate_uncompressed_block(d); + break; + case 1: + case 2: + /* decompress block with fixed/dynamic huffman trees */ + /* trees were decoded previously, so it's the same routine for both */ + res = tinf_inflate_block_data(d, &d->ltree, &d->dtree); + break; + default: + return TINF_DATA_ERROR; + } + + if (res == TINF_DONE && !d->bfinal) { + /* the block has ended (without producing more data), but we + can't return without data, so start procesing next block */ + goto next_blk; + } + + if (res != TINF_OK) { + return res; + } + + } while (d->dest < d->dest_limit); + + return TINF_OK; +} diff --git a/esphome/components/esphome/ota/ota_esphome_inflate.h b/esphome/components/esphome/ota/ota_esphome_inflate.h new file mode 100644 index 0000000000..9726508a30 --- /dev/null +++ b/esphome/components/esphome/ota/ota_esphome_inflate.h @@ -0,0 +1,65 @@ +#pragma once +// Raw deflate decoder cut down from uzlib (https://github.com/pfalcon/uzlib, +// zlib licence, see the .c file); output goes through a ring window. + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +enum OtaInflateResult { + OTA_INFLATE_OK = 0, /* more data produced, call again */ + OTA_INFLATE_DONE = 1, /* end of compressed stream reached */ + OTA_INFLATE_DATA_ERROR = -3, + OTA_INFLATE_DICT_ERROR = -5, +}; + +struct OtaInflateTree { + uint16_t table[16]; /* table of code length counts */ + uint16_t *trans; /* code -> symbol translation table, size entries */ + uint16_t size; +}; + +struct OtaInflateState { + /* Next byte in the input buffer and one past its end */ + const unsigned char *source; + const unsigned char *source_limit; + /* Called when source is exhausted; returns the next byte or -1 at EOF. + It may refill source/source_limit for buffered operation. */ + int (*source_read_cb)(struct OtaInflateState *d); + /* Output cursor and one past the end of the output buffer */ + unsigned char *dest; + unsigned char *dest_limit; + /* Ring window holding the last dict_size output bytes for back references */ + unsigned char *dict_ring; + + unsigned int tag; + unsigned int curlen; + int lz_off; + unsigned int dict_size; + unsigned int dict_idx; + + /* One word: btype is -1 between blocks, bitcount never exceeds 7 */ + int8_t btype; + uint8_t bfinal; + uint8_t bitcount; + bool eof; + + struct OtaInflateTree ltree; /* dynamic length/symbol tree */ + struct OtaInflateTree dtree; /* dynamic distance tree */ + uint16_t ltrans[288]; + uint16_t dtrans[32]; /* the distance alphabet has 30 symbols, so the tree is kept small */ +}; + +/* dict must cover the encoder's window (its largest back reference) */ +void ota_inflate_init(struct OtaInflateState *d, unsigned char *dict, unsigned int dict_len); +/* Fills dest up to dest_limit (OK) or to the end of the stream (DONE). dest may + alias dict only if dest_limit - dest == dict_len and dest is reset to dict + exactly when a call returns OK, so the ring index and dest stay in lockstep */ +int ota_inflate(struct OtaInflateState *d); + +#ifdef __cplusplus +} +#endif diff --git a/esphome/components/esphome/ota/ota_esphome_inflate_session.cpp b/esphome/components/esphome/ota/ota_esphome_inflate_session.cpp new file mode 100644 index 0000000000..f3005cac48 --- /dev/null +++ b/esphome/components/esphome/ota/ota_esphome_inflate_session.cpp @@ -0,0 +1,94 @@ +#include "ota_esphome.h" +#ifdef USE_OTA +#ifdef USE_OTA_DEFLATE +#include "esphome/components/ota/ota_backend.h" +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +namespace esphome { + +static const char *const TAG = "esphome.ota"; + +// The window doubles as the output buffer; flushed bytes stay as back +// reference history for the next windowful. +ota::OTAResponseTypes ESPHomeOTAComponent::inflate_flush_(InflateSession &session) { + const size_t produced = session.dest - session.window; + const size_t pending = produced - session.flushed; + if (pending != 0) { + if (pending > session.image_size - session.written) { + ESP_LOGW(TAG, "Inflate overrun"); + return ota::OTA_RESPONSE_ERROR_UNKNOWN; + } + ota::OTAResponseTypes result = this->write_flash_(session.window + session.flushed, pending); + if (result != ota::OTA_RESPONSE_OK) + return result; + session.flushed = produced; + session.written += pending; + // A compressible region yields many windows per socket read + App.feed_wdt(); + } + // Even with nothing new written: a block boundary can fall inside a header + this->ack_written_(*session.xfer); + return ota::OTA_RESPONSE_OK; +} + +ota::OTAResponseTypes ESPHomeOTAComponent::inflate_data_(uint8_t *in, size_t image_size, DataTransfer &xfer) { + InflateSession &session = *this->inflate_; + session.self = this; + session.xfer = &xfer; + session.in = in; + session.image_size = image_size; + session.written = 0; + session.error = ota::OTA_RESPONSE_OK; + ota_inflate_init(&session, session.window, OTA_INFLATE_WINDOW_SIZE); + // Where the ack must follow the write, flush and ack before waiting for + // input, or the client waits for an ack while the decoder waits for data + session.source_read_cb = [](OtaInflateState *d) -> int { + auto *s = static_cast(d); + if (ACK_AFTER_WRITE) { + s->error = s->self->inflate_flush_(*s); + if (s->error != ota::OTA_RESPONSE_OK) + return -1; + } + // More input than announced; reported by the size check below + if (s->xfer->total >= s->xfer->ota_size) + return -1; + ssize_t read = s->self->receive_data_(s->in, *s->xfer); + if (read <= 0) { + // Already logged by receive_data_ + s->error = ota::OTA_RESPONSE_ERROR_UNKNOWN; + return -1; + } + d->source = s->in + 1; + d->source_limit = s->in + read; + return s->in[0]; + }; + + int res; + do { + // The ring index wrapped to 0 exactly when the window filled + session.dest = session.window; + session.dest_limit = session.window + OTA_INFLATE_WINDOW_SIZE; + session.flushed = 0; + res = ota_inflate(&session); + // A stored block keeps emitting zeros after a failed read, hence eof + if (res < 0 || session.eof) + break; + session.error = this->inflate_flush_(session); + } while (res != OTA_INFLATE_DONE && session.error == ota::OTA_RESPONSE_OK); + + // Transport and flash failures are logged where they happen + if (session.error != ota::OTA_RESPONSE_OK) + return session.error; + if (res != OTA_INFLATE_DONE || session.written != image_size || xfer.total != xfer.ota_size) { + ESP_LOGW(TAG, "Inflate err %d, %zu of %zu B from %zu of %zu", res, session.written, image_size, xfer.total, + xfer.ota_size); + return ota::OTA_RESPONSE_ERROR_UNKNOWN; + } + ESP_LOGD(TAG, "Inflated %zu bytes from %zu", session.written, xfer.total); + return ota::OTA_RESPONSE_OK; +} + +} // namespace esphome +#endif // USE_OTA_DEFLATE +#endif // USE_OTA diff --git a/esphome/components/esphome/ota/ota_esphome_noise.cpp b/esphome/components/esphome/ota/ota_esphome_noise.cpp index 65476572a1..b3d8501f10 100644 --- a/esphome/components/esphome/ota/ota_esphome_noise.cpp +++ b/esphome/components/esphome/ota/ota_esphome_noise.cpp @@ -32,7 +32,12 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() { } } -/** Allocate the session and start the responder handshake. +void ESPHomeOTAComponent::noise_reserve_session_() { + // Default placement, PSRAM first where present: the session lives for one upload + this->noise_ = RAMAllocator().make_unique(); +} + +/** Start the responder handshake, on the session reserved at offer time. * * The prologue binds the whole plaintext preamble, so any tampering with the * negotiation (a stripped feature flag, a changed version) breaks the first @@ -41,9 +46,7 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() { */ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { // A provisioned key cleared between the offer and here is not guarded: the - // session runs on the zero key load_psk fills in and fails the client's MAC. - // Default placement, PSRAM first where present: the session only lives for one upload - this->noise_ = RAMAllocator().make_unique(); + // session runs on the zero key load_psk fills in and fails the client's MAC static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1; static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags diff --git a/esphome/components/ota/ota_backend.h b/esphome/components/ota/ota_backend.h index 7348a0ce90..a505d58efe 100644 --- a/esphome/components/ota/ota_backend.h +++ b/esphome/components/ota/ota_backend.h @@ -7,6 +7,7 @@ #include #include #include +#include #ifdef USE_OTA_STATE_LISTENER #include @@ -102,6 +103,8 @@ enum OTAType : uint8_t { // - set_update_md5: expected digest of the incoming image, hex string. // - write: consume the next chunk; end: finalize and mark bootable. // - abort: safe to call in any state, including after end(). +// - supports_compression: constexpr, whether a gzip image is stored as is and +// inflated at reboot. template concept OTABackendContract = requires(T backend, size_t image_size, uint8_t *data, size_t len, const char *md5) { { backend.begin(image_size, OTA_TYPE_UPDATE_APP) } -> std::same_as; @@ -110,7 +113,9 @@ concept OTABackendContract = requires(T backend, size_t image_size, uint8_t *dat { backend.write(data, len) } -> std::same_as; { backend.end() } -> std::same_as; backend.abort(); - { backend.supports_compression() } -> std::same_as; + { T::supports_compression() } -> std::same_as; + // The value must be a constant expression + typename std::bool_constant; }; /** Listener interface for OTA state changes. diff --git a/esphome/components/ota/ota_backend_arduino_libretiny.h b/esphome/components/ota/ota_backend_arduino_libretiny.h index c2716a44d1..c322ed21f2 100644 --- a/esphome/components/ota/ota_backend_arduino_libretiny.h +++ b/esphome/components/ota/ota_backend_arduino_libretiny.h @@ -13,7 +13,7 @@ class ArduinoLibreTinyOTABackend final { OTAResponseTypes write(uint8_t *data, size_t len); OTAResponseTypes end(); void abort(); - bool supports_compression() { return false; } + static constexpr bool supports_compression() { return false; } private: bool md5_set_{false}; diff --git a/esphome/components/ota/ota_backend_arduino_rp2.h b/esphome/components/ota/ota_backend_arduino_rp2.h index f7c0037bd2..15142869ac 100644 --- a/esphome/components/ota/ota_backend_arduino_rp2.h +++ b/esphome/components/ota/ota_backend_arduino_rp2.h @@ -15,7 +15,10 @@ class ArduinoRP2OTABackend final { OTAResponseTypes write(uint8_t *data, size_t len); OTAResponseTypes end(); void abort(); - bool supports_compression() { return false; } + // The core's OTA stub inflates a staged gzip image at reboot, on every chip + // from 4.0.3 (ESPHome pins 6.0.0). begin() only sees the gzip size; the + // inflated size is known when the stub reads the trailer. + static constexpr bool supports_compression() { return USE_ARDUINO_VERSION_CODE >= VERSION_CODE(4, 0, 3); } private: bool md5_set_{false}; diff --git a/esphome/components/ota/ota_backend_esp8266.h b/esphome/components/ota/ota_backend_esp8266.h index 21b5c12c2d..1f1ec37eee 100644 --- a/esphome/components/ota/ota_backend_esp8266.h +++ b/esphome/components/ota/ota_backend_esp8266.h @@ -20,7 +20,7 @@ class ESP8266OTABackend final { OTAResponseTypes end(); void abort(); // Compression supported in all ESP8266 Arduino versions ESPHome supports (>= 2.7.0) - bool supports_compression() { return true; } + static constexpr bool supports_compression() { return true; } protected: /// Erase flash sector if current address is at sector boundary diff --git a/esphome/components/ota/ota_backend_esp_idf.h b/esphome/components/ota/ota_backend_esp_idf.h index c991f896e8..4f4093a594 100644 --- a/esphome/components/ota/ota_backend_esp_idf.h +++ b/esphome/components/ota/ota_backend_esp_idf.h @@ -33,7 +33,7 @@ class IDFOTABackend final { OTAResponseTypes write(uint8_t *data, size_t len); OTAResponseTypes end(); void abort(); - bool supports_compression() { return false; } + static constexpr bool supports_compression() { return false; } protected: #ifdef USE_OTA_PARTITIONS diff --git a/esphome/components/ota/ota_backend_factory.h b/esphome/components/ota/ota_backend_factory.h index 82d001ed9e..06c58582b5 100644 --- a/esphome/components/ota/ota_backend_factory.h +++ b/esphome/components/ota/ota_backend_factory.h @@ -25,7 +25,7 @@ struct StubOTABackend { OTAResponseTypes write(uint8_t *data, size_t len) { return OTA_RESPONSE_ERROR_UNKNOWN; } OTAResponseTypes end() { return OTA_RESPONSE_ERROR_UNKNOWN; } void abort() {} - bool supports_compression() { return false; } + static constexpr bool supports_compression() { return false; } }; std::unique_ptr make_ota_backend(); } // namespace esphome::ota @@ -33,6 +33,7 @@ std::unique_ptr make_ota_backend(); namespace esphome::ota { using OTABackendPtr = decltype(make_ota_backend()); -static_assert(OTABackendContract, +using OTABackend = OTABackendPtr::element_type; +static_assert(OTABackendContract, "The platform's OTA backend is missing part of the backend surface (ota_backend.h)"); } // namespace esphome::ota diff --git a/esphome/components/ota/ota_backend_host.h b/esphome/components/ota/ota_backend_host.h index 51ffdaeda3..e53868f102 100644 --- a/esphome/components/ota/ota_backend_host.h +++ b/esphome/components/ota/ota_backend_host.h @@ -19,7 +19,7 @@ class HostOTABackend final { OTAResponseTypes write(uint8_t *data, size_t len); OTAResponseTypes end(); void abort(); - bool supports_compression() { return false; } + static constexpr bool supports_compression() { return false; } protected: md5::MD5Digest md5_{}; diff --git a/esphome/components/socket/__init__.py b/esphome/components/socket/__init__.py index edebed5d40..a814f46e3b 100644 --- a/esphome/components/socket/__init__.py +++ b/esphome/components/socket/__init__.py @@ -140,6 +140,17 @@ def require_wake_loop_threadsafe() -> None: cg.add_define("USE_SOCKET_SELECT_SUPPORT") +def require_ipv4_resolve() -> None: + """Compile the shared IPv4 lookup; call from a consumer's to_code.""" + cg.add_define("USE_SOCKET_IPV4_RESOLVE") + + +def require_tcp_client_link() -> None: + """Compile the reconnecting TCP client link; call from a consumer's to_code.""" + require_ipv4_resolve() + cg.add_define("USE_SOCKET_TCP_CLIENT_LINK") + + CONFIG_SCHEMA = cv.Schema( { cv.SplitDefault( @@ -184,11 +195,14 @@ async def to_code(config: ConfigType) -> None: # Each implementation file is fully #ifdef'd on the define set in to_code -# for the selected implementation. +# for the selected implementation. The helper files compile only for +# consumers that called the matching require_ function. FILTER_SOURCE_FILES = filter_source_files_from_defines( { "lwip_raw_tcp_impl.cpp": "USE_SOCKET_IMPL_LWIP_TCP", "bsd_sockets_impl.cpp": "USE_SOCKET_IMPL_BSD_SOCKETS", "lwip_sockets_impl.cpp": "USE_SOCKET_IMPL_LWIP_SOCKETS", + "ipv4_resolve.cpp": "USE_SOCKET_IPV4_RESOLVE", + "tcp_client_link.cpp": "USE_SOCKET_TCP_CLIENT_LINK", } ) diff --git a/esphome/components/socket/tcp_client_link.cpp b/esphome/components/socket/tcp_client_link.cpp new file mode 100644 index 0000000000..f2c1b8e4c1 --- /dev/null +++ b/esphome/components/socket/tcp_client_link.cpp @@ -0,0 +1,150 @@ +#include "tcp_client_link.h" + +#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) + +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +#include +#include + +namespace esphome::socket { + +// After this long in SYN, the stack's own retries are cut short. +static constexpr uint32_t CONNECT_TIMEOUT_MS = 10000; + +// Non-blocking options and TCP keepalive for a bridged stream socket. +// Keepalive is best-effort: the raw lwIP implementation (ESP8266, RP2040) +// rejects it, so a half-open link there is only detected by a failed write. +static void set_stream_options(Socket *sock) { + int yes = 1; + sock->setblocking(false); + sock->setsockopt(IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)); + sock->setsockopt(SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes)); +#ifdef TCP_KEEPIDLE + int idle = 30; + int interval = 10; + int count = 3; + sock->setsockopt(IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle)); + sock->setsockopt(IPPROTO_TCP, TCP_KEEPINTVL, &interval, sizeof(interval)); + sock->setsockopt(IPPROTO_TCP, TCP_KEEPCNT, &count, sizeof(count)); +#endif +} + +void TcpClientLink::begin(const char *tag) { + this->tag_ = tag; + // The first attempt must not wait out a full interval. + this->last_attempt_ms_ = App.get_loop_component_start_time() - this->reconnect_interval_ms_; +} + +void TcpClientLink::poll_slow_() { + if (this->sock_ == nullptr) { + this->try_connect_(); + return; + } + int err = 0; + switch (poll_connect(*this->sock_, err)) { + case ConnectPollResult::CONNECT_POLL_RESULT_PENDING: + // Give up before the stack's SYN retries do, so the interval stays honest + // and the next attempt resolves the host again. + if (App.get_loop_component_start_time() - this->last_attempt_ms_ >= + std::max(this->reconnect_interval_ms_, CONNECT_TIMEOUT_MS)) { + this->drop_(LOG_STR("Connect failed"), ETIMEDOUT); + } + return; + case ConnectPollResult::CONNECT_POLL_RESULT_ERROR: + this->drop_(LOG_STR("Connect failed"), err); + return; + default: + break; + } + this->connected_ = true; + ESP_LOGI(this->tag_, "Connected to %s:%u", this->host_.c_str(), this->port_); +} + +void TcpClientLink::try_connect_() { + if (this->resolved_.consume_failure()) { + this->note_attempt(); + return; + } + this->resolved_.start(this->host_.c_str(), this->port_, this->tag_); + if (!this->resolved_.ready()) { + return; + } + struct sockaddr_storage dest; + socklen_t dest_len = + this->resolved_.to_sockaddr(reinterpret_cast(&dest), sizeof(dest), this->port_); + if (dest_len == 0) { + this->note_attempt(); + return; + } + this->sock_ = socket_loop_monitored(dest.ss_family, SOCK_STREAM, IPPROTO_TCP); + if (this->sock_ == nullptr) { + this->drop_(LOG_STR("Connect failed"), errno); + return; + } + set_stream_options(this->sock_.get()); + // Starts the pending-connect clock that poll() times out against. + this->note_attempt(); + // An immediate success is reported by the next poll(); poll_connect() sees it writable. + if (this->sock_->connect(reinterpret_cast(&dest), dest_len) != 0 && errno != EINPROGRESS) { + this->drop_(LOG_STR("Connect failed"), errno); + } +} + +void TcpClientLink::adopt(std::unique_ptr sock) { + this->close(); + set_stream_options(sock.get()); + this->sock_ = std::move(sock); + this->connected_ = true; +} + +ssize_t TcpClientLink::read(uint8_t *buf, size_t len) { + if (!this->connected_) { + return 0; + } + ssize_t count = this->sock_->read(buf, len); + if (count > 0) { + return count; + } + if (count == 0 || (errno != EAGAIN && errno != EWOULDBLOCK)) { + this->drop_(LOG_STR("Connection lost"), count == 0 ? 0 : errno); + return -1; + } + return 0; +} + +ssize_t TcpClientLink::write(const uint8_t *buf, size_t len) { + if (!this->connected_ || len == 0) { + return 0; + } + ssize_t sent = this->sock_->write(buf, len); + if (sent >= 0) { + return sent; + } + if (errno == EAGAIN || errno == EWOULDBLOCK) { + return 0; + } + this->drop_(LOG_STR("Connection lost"), errno); + return -1; +} + +void TcpClientLink::close() { + if (this->sock_ != nullptr) { + this->sock_->shutdown(SHUT_RDWR); + this->sock_->close(); + this->sock_.reset(); + } + this->connected_ = false; + this->resolved_.forget(); +} + +void TcpClientLink::drop_(const LogString *what, int err) { + ESP_LOGW(this->tag_, "%s: %d", LOG_STR_ARG(what), err); + this->close(); + this->note_attempt(); +} + +} // namespace esphome::socket + +#endif diff --git a/esphome/components/socket/tcp_client_link.h b/esphome/components/socket/tcp_client_link.h new file mode 100644 index 0000000000..065c4df562 --- /dev/null +++ b/esphome/components/socket/tcp_client_link.h @@ -0,0 +1,74 @@ +#pragma once + +#include "headers.h" + +#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) + +#include "ipv4_resolve.h" +#include "socket.h" +#include "esphome/core/application.h" +#include "esphome/core/log.h" +#include "esphome/core/string_ref.h" + +#include +#include + +namespace esphome::socket { + +/// A reconnecting TCP stream driven from loop(). Owns the socket, the DNS +/// lookup and the retry backoff. A fatal read/write error closes the link +/// and schedules the next attempt; the caller sees the edge via connected(). +class TcpClientLink { + public: + void set_host(const char *host) { this->host_ = StringRef(host); } + void set_port(uint16_t port) { this->port_ = port; } + void set_reconnect_interval(uint32_t ms) { this->reconnect_interval_ms_ = ms; } + const char *host() const { return this->host_.c_str(); } + uint16_t port() const { return this->port_; } + uint32_t reconnect_interval() const { return this->reconnect_interval_ms_; } + + /// Call from setup(). tag names this link's log lines. + void begin(const char *tag); + /// Connect state machine; call every loop while acting as a client. + /// Inline no-op while connected or waiting out the backoff. + void poll() { + if (this->connected_ || (this->sock_ == nullptr && this->in_backoff())) { + return; + } + this->poll_slow_(); + } + /// Take over an accepted socket (the server side of a bridge). + void adopt(std::unique_ptr sock); + /// Returns bytes moved, 0 when nothing can move now, -1 when the link dropped. + ssize_t read(uint8_t *buf, size_t len); + ssize_t write(const uint8_t *buf, size_t len); + /// Close without scheduling a reconnect (shutdown). + void close(); + + bool connected() const { return this->connected_; } + bool ready() const { return this->sock_ != nullptr && this->sock_->ready(); } + /// Shared retry clock, also usable for a listen socket. + void note_attempt() { this->last_attempt_ms_ = App.get_loop_component_start_time(); } + bool in_backoff() const { + return App.get_loop_component_start_time() - this->last_attempt_ms_ < this->reconnect_interval_ms_; + } + + protected: + void poll_slow_(); + void try_connect_(); + /// Close after a failure, log what and errno, schedule the next attempt. + void drop_(const LogString *what, int err); + + StringRef host_; + std::unique_ptr sock_; + const char *tag_{nullptr}; + uint32_t last_attempt_ms_{0}; + uint32_t reconnect_interval_ms_{5000}; + Ipv4Resolve resolved_; + uint16_t port_{0}; + bool connected_{false}; +}; + +} // namespace esphome::socket + +#endif diff --git a/esphome/components/substitutions/__init__.py b/esphome/components/substitutions/__init__.py index 5ef7a699eb..d59f39b9c2 100644 --- a/esphome/components/substitutions/__init__.py +++ b/esphome/components/substitutions/__init__.py @@ -353,7 +353,7 @@ def resolve_include( strict_undefined: bool = True, errors: ErrList | None = None, ) -> Any: - """Resolve an include, substituting the filename if needed. + """Resolve an include, substituting the condition and filename if needed. Note: no path-traversal validation is performed on the resolved filename. A substitution that resolves to an absolute path will bypass the parent @@ -362,6 +362,19 @@ def resolve_include( values (including command-line substitutions), so path restrictions are an explicit non-goal here. """ + if isinstance(original_condition := include.condition, str): + condition = str( + _expand_substitutions( + original_condition, + path + ["condition"], + context_vars, + strict_undefined, + errors, + ) + ) + if condition != original_condition: + include = include.with_condition(condition) + original = include.file filename = str( _expand_substitutions( @@ -372,8 +385,8 @@ def resolve_include( if substituted: include = include.with_file(filename) try: - return include.load() - except esphome.core.EsphomeError as err: + return include.load() if include.should_load() else {} + except (esphome.core.EsphomeError, cv.Invalid) as err: resolved = f" (expanded from '{original}')" if substituted else "" raise cv.Invalid( f"Error including file '{filename}'{resolved}: {err}" diff --git a/esphome/components/tcp_uart/__init__.py b/esphome/components/tcp_uart/__init__.py new file mode 100644 index 0000000000..20866d1a0c --- /dev/null +++ b/esphome/components/tcp_uart/__init__.py @@ -0,0 +1,67 @@ +import esphome.codegen as cg +from esphome.components import binary_sensor, socket, uart +from esphome.components.const import ( + CONF_DATA_BITS, + CONF_HOST, + CONF_PARITY, + CONF_STOP_BITS, +) +import esphome.config_validation as cv +from esphome.const import ( + CONF_BAUD_RATE, + CONF_ID, + CONF_PORT, + DEVICE_CLASS_CONNECTIVITY, + ENTITY_CATEGORY_DIAGNOSTIC, +) +from esphome.types import ConfigType + +CODEOWNERS = ["@Bascht74"] +DEPENDENCIES = ["network"] +AUTO_LOAD = ["uart", "binary_sensor", "socket"] +MULTI_CONF = True + +tcp_uart_ns = cg.esphome_ns.namespace("tcp_uart") +TcpUart = tcp_uart_ns.class_("TcpUart", uart.UARTComponent, cg.Component) + +CONF_RECONNECT_INTERVAL = "reconnect_interval" +CONF_CONNECTED = "connected" + + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(TcpUart), + cv.Required(CONF_HOST): cv.string, + cv.Required(CONF_PORT): cv.port, + cv.Optional(CONF_BAUD_RATE, default=9600): cv.int_range(min=1), + cv.Optional(CONF_DATA_BITS, default=8): cv.int_range(min=5, max=8), + cv.Optional(CONF_PARITY, default="NONE"): cv.enum( + uart.UART_PARITY_OPTIONS, upper=True + ), + cv.Optional(CONF_STOP_BITS, default=1): cv.one_of(1, 2, int=True), + cv.Optional( + CONF_RECONNECT_INTERVAL, default="5s" + ): cv.positive_time_period_milliseconds, + cv.Optional(CONF_CONNECTED): binary_sensor.binary_sensor_schema( + device_class=DEVICE_CLASS_CONNECTIVITY, + entity_category=ENTITY_CATEGORY_DIAGNOSTIC, + ), + } + ).extend(cv.COMPONENT_SCHEMA), + socket.consume_sockets(1, "tcp_uart"), +) + + +async def to_code(config: ConfigType) -> None: + socket.require_tcp_client_link() + var = cg.new_Pvariable(config[CONF_ID], config[CONF_HOST], config[CONF_PORT]) + await cg.register_component(var, config) + cg.add(var.set_reconnect_interval(config[CONF_RECONNECT_INTERVAL])) + # The socket is not clocked. These only satisfy UARTComponent and a consumer check. + cg.add(var.set_baud_rate(config[CONF_BAUD_RATE])) + cg.add(var.set_data_bits(config[CONF_DATA_BITS])) + cg.add(var.set_stop_bits(config[CONF_STOP_BITS])) + cg.add(var.set_parity(config[CONF_PARITY])) + binary_sensors = binary_sensor.sub_binary_sensors(config) + await binary_sensors(CONF_CONNECTED, var.set_connected_sensor) diff --git a/esphome/components/tcp_uart/tcp_uart.cpp b/esphome/components/tcp_uart/tcp_uart.cpp new file mode 100644 index 0000000000..fc81f6ecfe --- /dev/null +++ b/esphome/components/tcp_uart/tcp_uart.cpp @@ -0,0 +1,131 @@ +#include "tcp_uart.h" + +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +#include +#include + +namespace esphome::tcp_uart { + +static const char *const TAG = "tcp_uart"; + +static constexpr uint32_t DROP_LOG_INTERVAL_MS = 5000; + +void TcpUart::setup() { + this->link_.begin(TAG); + if (this->connected_sensor_ != nullptr) { + this->connected_sensor_->publish_state(false); + } +} + +void TcpUart::dump_config() { + ESP_LOGCONFIG(TAG, + "TCP UART:\n" + " Host: %s:%u\n" + " Reconnect Interval: %" PRIu32 "ms", + this->link_.host(), this->link_.port(), this->link_.reconnect_interval()); + LOG_BINARY_SENSOR(" ", "Connected", this->connected_sensor_); +} + +void TcpUart::sync_link_() { + bool up = this->link_.connected(); + this->link_was_up_ = up; + if (!up) { + this->rx_start_ = this->rx_end_ = 0; + this->tx_len_ = 0; + } + if (this->connected_sensor_ != nullptr) { + this->connected_sensor_->publish_state(up); + } +} + +void TcpUart::read_socket_() { + if (this->rx_start_ != 0) { + this->rx_end_ -= this->rx_start_; + std::memmove(this->rx_, this->rx_ + this->rx_start_, this->rx_end_); + this->rx_start_ = 0; + } + size_t room = RX_BUFFER_SIZE - this->rx_end_; + if (room == 0) { + // Only a read that filled all free space gets here, so rx_pending_ is already set. + return; + } + ssize_t count = this->link_.read(this->rx_ + this->rx_end_, room); + if (count <= 0) { + // A dropped link (-1) is cleaned up by sync_link_() on the next loop. + if (count == 0) { + this->rx_pending_ = false; + } + return; + } + this->rx_end_ += static_cast(count); + this->rx_pending_ = static_cast(count) == room; +} + +void TcpUart::flush_tx_() { + ssize_t sent = this->link_.write(this->tx_, this->tx_len_); + if (sent > 0) { + this->tx_len_ -= static_cast(sent); + std::memmove(this->tx_, this->tx_ + sent, this->tx_len_); + } +} + +void TcpUart::loop() { + this->link_.poll(); + if (this->link_.connected() != this->link_was_up_) { + this->sync_link_(); + } + if (!this->link_was_up_) { + return; + } + if (this->rx_pending_ || this->link_.ready()) { + this->read_socket_(); + } + if (this->tx_len_ != 0) { + this->flush_tx_(); + } +} + +void TcpUart::write_array(const uint8_t *data, size_t len) { + size_t room = this->link_.connected() ? sizeof(this->tx_) - this->tx_len_ : 0; + if (len > room) { + uint32_t now = App.get_loop_component_start_time(); + if (this->last_drop_log_ms_ == 0 || now - this->last_drop_log_ms_ >= DROP_LOG_INTERVAL_MS) { + ESP_LOGW(TAG, "%s, dropped %u bytes", + this->link_.connected() ? LOG_STR_LITERAL("TX buffer full") : LOG_STR_LITERAL("Not connected"), + static_cast(len - room)); + this->last_drop_log_ms_ = now; + } + len = room; + } + std::memcpy(this->tx_ + this->tx_len_, data, len); + this->tx_len_ += static_cast(len); +} + +bool TcpUart::peek_byte(uint8_t *data) { + if (this->rx_start_ == this->rx_end_) { + return false; + } + *data = this->rx_[this->rx_start_]; + return true; +} + +bool TcpUart::read_array(uint8_t *data, size_t len) { + if (this->available() < len) { + return false; + } + std::memcpy(data, this->rx_ + this->rx_start_, len); + this->rx_start_ += static_cast(len); + return true; +} + +uart::UARTFlushResult TcpUart::flush() { + this->flush_tx_(); + if (this->tx_len_ == 0) { + return uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS; + } + return uart::UARTFlushResult::UART_FLUSH_RESULT_TIMEOUT; +} + +} // namespace esphome::tcp_uart diff --git a/esphome/components/tcp_uart/tcp_uart.h b/esphome/components/tcp_uart/tcp_uart.h new file mode 100644 index 0000000000..c13d2ef6f0 --- /dev/null +++ b/esphome/components/tcp_uart/tcp_uart.h @@ -0,0 +1,66 @@ +#pragma once + +#include "esphome/components/binary_sensor/binary_sensor.h" +#include "esphome/components/socket/tcp_client_link.h" +#include "esphome/components/uart/uart_component.h" +#include "esphome/core/component.h" + +#include + +namespace esphome::tcp_uart { + +/// TCP client presented as a UART. Bytes are copied unchanged. +class TcpUart : public uart::UARTComponent, public Component { + public: + TcpUart(const char *host, uint16_t port) { + this->link_.set_host(host); + this->link_.set_port(port); + this->rx_buffer_size_ = RX_BUFFER_SIZE; + } + + void set_reconnect_interval(uint32_t ms) { this->link_.set_reconnect_interval(ms); } + void set_connected_sensor(binary_sensor::BinarySensor *sensor) { this->connected_sensor_ = sensor; } + + void setup() override; + void loop() override; + void dump_config() override; + void on_shutdown() override { this->link_.close(); } + float get_setup_priority() const override { return setup_priority::AFTER_WIFI; } + + void write_array(const uint8_t *data, size_t len) override; + bool peek_byte(uint8_t *data) override; + bool read_array(uint8_t *data, size_t len) override; + size_t available() override { return static_cast(this->rx_end_ - this->rx_start_); } + // Same room write_array() grants, so consumers can apply backpressure. + size_t available_for_write() override { return this->link_.connected() ? sizeof(this->tx_) - this->tx_len_ : 0; } + uart::UARTFlushResult flush() override; + bool is_connected() override { return this->link_.connected(); } +#if defined(USE_ESP8266) || defined(USE_ESP32) + void load_settings(bool dump_config) override {} +#endif + + protected: + void check_logger_conflict() override {} + void sync_link_(); + void read_socket_(); + void flush_tx_(); + + static constexpr size_t RX_BUFFER_SIZE = 1024; + static constexpr size_t TX_BUFFER_SIZE = 1024; + + socket::TcpClientLink link_; + binary_sensor::BinarySensor *connected_sensor_{nullptr}; + uint32_t last_drop_log_ms_{0}; + uint16_t tx_len_{0}; + // rx_[rx_start_, rx_end_) holds unread bytes; read_socket_() compacts to the front. + uint16_t rx_start_{0}; + uint16_t rx_end_{0}; + // The link state loop() saw last; edges clear the buffers and publish the sensor. + bool link_was_up_{false}; + // A read stopped before EAGAIN. ready() stays false until new data arrives. + bool rx_pending_{false}; + uint8_t rx_[RX_BUFFER_SIZE]{}; + uint8_t tx_[TX_BUFFER_SIZE]{}; +}; + +} // namespace esphome::tcp_uart diff --git a/esphome/core/config.py b/esphome/core/config.py index 281d430410..532ca7c36d 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -7,7 +7,10 @@ from pathlib import Path from esphome import automation, core import esphome.codegen as cg -from esphome.config_helpers import filter_source_files_from_platform +from esphome.config_helpers import ( + filter_source_files_from_defines, + filter_source_files_from_platform, +) import esphome.config_validation as cv from esphome.const import ( CONF_AREA, @@ -897,7 +900,7 @@ async def to_code(config: ConfigType) -> None: # Platform-specific source files for core -FILTER_SOURCE_FILES = filter_source_files_from_platform( +_filter_platform_sources = filter_source_files_from_platform( { "static_task.cpp": { PlatformFramework.ESP32_ARDUINO, @@ -950,3 +953,11 @@ FILTER_SOURCE_FILES = filter_source_files_from_platform( # as they are only included when needed by the preprocessor } ) + +# util.cpp holds only the out-of-line MQTT connectivity helpers; without +# USE_MQTT the header provides inline stubs and the file must not compile. +_filter_define_sources = filter_source_files_from_defines({"util.cpp": "USE_MQTT"}) + + +def FILTER_SOURCE_FILES() -> list[str]: + return _filter_platform_sources() + _filter_define_sources() diff --git a/esphome/core/defines.h b/esphome/core/defines.h index aeffba9165..e05a67f9c8 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -295,6 +295,7 @@ #define USE_RUNTIME_IMAGE_QOI #define USE_RUNTIME_STATS #define USE_OTA +#define USE_OTA_DEFLATE #define USE_OTA_ENCRYPTION #define USE_OTA_ENCRYPTION_PROVISIONED #define USE_OTA_ENCRYPTION_REQUIRED @@ -418,6 +419,8 @@ #define USE_SENDSPIN_VISUALIZER #define USE_SENDSPIN_PORT 8928 // NOLINT #define USE_SOCKET_IMPL_BSD_SOCKETS +#define USE_SOCKET_IPV4_RESOLVE +#define USE_SOCKET_TCP_CLIENT_LINK #define USE_LWIP_FAST_SELECT #define USE_SPEAKER diff --git a/esphome/core/util.cpp b/esphome/core/util.cpp index 54a7956163..4bca130e4a 100644 --- a/esphome/core/util.cpp +++ b/esphome/core/util.cpp @@ -1,23 +1,16 @@ #include "esphome/core/util.h" -#include "esphome/core/application.h" -#include "esphome/core/version.h" -#include "esphome/core/log.h" +// Compiled only on MQTT builds (core FILTER_SOURCE_FILES); without USE_MQTT +// the header provides inline stubs and this file must stay empty, so the +// guard protects builds that compile every source, such as clang-tidy. #ifdef USE_MQTT #include "esphome/components/mqtt/mqtt_client.h" -#endif namespace esphome { -bool mqtt_is_connected() { -#ifdef USE_MQTT - if (mqtt::global_mqtt_client != nullptr) { - return mqtt::global_mqtt_client->is_connected(); - } -#endif - return false; -} +bool mqtt_is_connected() { return mqtt::global_mqtt_client != nullptr && mqtt::global_mqtt_client->is_connected(); } bool remote_is_connected() { return api_is_connected() || mqtt_is_connected(); } } // namespace esphome +#endif diff --git a/esphome/core/util.h b/esphome/core/util.h index 8f90aa3411..aa3098ba65 100644 --- a/esphome/core/util.h +++ b/esphome/core/util.h @@ -25,9 +25,17 @@ ESPHOME_ALWAYS_INLINE inline bool api_is_connected() { return false; } #endif /// Return whether the node has an active connection to an MQTT broker +#ifdef USE_MQTT bool mqtt_is_connected(); +#else +inline bool mqtt_is_connected() { return false; } +#endif /// Return whether the node has any form of "remote" connection via the API or to an MQTT broker +#ifdef USE_MQTT bool remote_is_connected(); +#else +inline bool remote_is_connected() { return api_is_connected(); } +#endif } // namespace esphome diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 8c377561ca..fdfc5c9b01 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -2,6 +2,7 @@ from collections.abc import Callable from ctypes.util import find_library +from dataclasses import dataclass, field import json import logging import os @@ -18,7 +19,7 @@ from esphome.build_helpers.ccache import ( ) from esphome.build_helpers.pch import ccache_pch_env from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path -from esphome.core import Version +from esphome.core import CORE, Version from esphome.framework_helpers import ( PathType, create_venv, @@ -42,6 +43,21 @@ _LOGGER = logging.getLogger(__name__) _SCRIPTS_DIR = Path(__file__).parent +DOMAIN = "espidf_framework" + + +@dataclass +class _FrameworkCache: + tool_paths: dict[Path, tuple[list[str], dict[str, str]]] = field( + default_factory=dict + ) + + +def _cache() -> _FrameworkCache: + if DOMAIN not in CORE.data: + CORE.data[DOMAIN] = _FrameworkCache() + return CORE.data[DOMAIN] + ESPHOME_STAMP_FILE = ".esphome.stamp.json" @@ -320,15 +336,42 @@ def _raise_script_failure(what: str, root: PathType, stderr: str | None) -> NoRe ) -def _get_idf_version( - idf_framework_root: PathType, env: dict[str, str] | None = None -) -> str: +# What idf_tools.get_idf_version() matches: ``version.txt`` first, then the +# version header. Both give major.minor only. +_IDF_VERSION_TXT_RE = re.compile(r"^v(\d+\.\d+)") +_IDF_VERSION_HEADER_RE = re.compile( + r"^#define\s+ESP_IDF_VERSION_MAJOR\s+(\d+).+?^#define\s+ESP_IDF_VERSION_MINOR\s+(\d+)", + re.DOTALL | re.MULTILINE, +) + + +def _read_text(path: Path) -> str: + return path.read_text(encoding="utf-8") if path.is_file() else "" + + +def read_idf_version_txt(idf_framework_root: Path) -> str | None: + """major.minor from ``version.txt``, as idf_tools reads it.""" + if match := _IDF_VERSION_TXT_RE.match( + _read_text(idf_framework_root / "version.txt") + ): + return match.group(1) + return None + + +def read_idf_version_header(idf_framework_root: Path) -> str | None: + """major.minor from ``esp_idf_version.h``, as idf_tools reads it.""" + header = idf_framework_root / "components" / "esp_common" / "include" + if match := _IDF_VERSION_HEADER_RE.search(_read_text(header / "esp_idf_version.h")): + return f"{match.group(1)}.{match.group(2)}" + return None + + +def _get_idf_version(idf_framework_root: PathType) -> str: """ Get the ESP-IDF version from the specified framework root. Args: idf_framework_root: Path to the ESP-IDF framework root directory - env: Optional dictionary of environment variables to set Returns: String containing ESP-IDF version @@ -336,9 +379,20 @@ def _get_idf_version( Raises: RuntimeError: If ESP-IDF version cannot be determined """ + root = Path(idf_framework_root) + try: + version = read_idf_version_txt(root) or read_idf_version_header(root) + except (OSError, UnicodeError) as e: + raise RuntimeError(f"Can't get ESP-IDF version of {root}: {e}") from e + if version is None: + raise RuntimeError(f"Can't get ESP-IDF version of {root}") + return version + +def idf_tools_version(idf_framework_root: PathType) -> str: + """The version from the framework's own ``idf_tools``, for the CI drift guard.""" success, stdout, stderr = _run_idf_tools_script( - idf_framework_root, "get_idf_version.py", "ESP-IDF version", env=env + idf_framework_root, "get_idf_version.py", "ESP-IDF version" ) if stdout: stdout = stdout.strip() @@ -362,7 +416,17 @@ def _get_idf_tool_paths( Raises: RuntimeError: If ESP-IDF tool paths cannot be determined + + The install check and the build environment both resolve the same + framework, so the result is cached per run and the helper script runs + once per build instead of once per caller. The script also reads + ``IDF_TOOLS_PATH``; every caller sets it from ``get_idf_tools_path()``, + so the key leaves it out. """ + cache = _cache().tool_paths + key = Path(idf_framework_root) + if (cached := cache.get(key)) is not None: + return cached success, stdout, stderr = _run_idf_tools_script( idf_framework_root, "get_idf_tool_paths.py", "ESP-IDF tool paths", env=env @@ -373,11 +437,13 @@ def _get_idf_tool_paths( # Extract json values try: data = json.loads(stdout) - return data["paths_to_export"], data["export_vars"] + result = (data["paths_to_export"], data["export_vars"]) except Exception as e: raise RuntimeError( f"Can't extract ESP-IDF tool paths of {idf_framework_root}" ) from e + cache[key] = result + return result def _get_python_version( @@ -927,6 +993,7 @@ def _check_esphome_idf_framework_install( # Validate via the managed tool-path resolution, not ``idf_tools.py check``: # ``check`` probes tools on the system PATH and aborts if any fail to run (e.g. a # broken Homebrew openocd), which forced a toolchain reinstall on every build. + # The resolved paths stay cached for get_framework_env. try: _get_idf_tool_paths(framework_path, env) install = False @@ -1041,7 +1108,7 @@ def _check_esp_idf_python_env_install( create_venv(python_env_path, msg=f"ESP-IDF {version}") - esp_idf_version = _get_idf_version(framework_path, env=env) + esp_idf_version = _get_idf_version(framework_path) constraint_file_path = ( get_idf_tools_path() / f"espidf.constraints.v{esp_idf_version}.txt" ) @@ -1282,7 +1349,7 @@ def get_framework_env( # 4. Set framework-specific environment variables env["IDF_PATH"] = str(framework_path) - env["ESP_IDF_VERSION"] = _get_idf_version(framework_path, env) + env["ESP_IDF_VERSION"] = _get_idf_version(framework_path) # 5. Get and add tool paths and environment variables paths_to_export, export_vars = _get_idf_tool_paths(framework_path, env) diff --git a/esphome/espidf/size_summary.py b/esphome/espidf/size_summary.py index ffe97ba618..9fc1a6b3a9 100644 --- a/esphome/espidf/size_summary.py +++ b/esphome/espidf/size_summary.py @@ -1,6 +1,6 @@ """PlatformIO-format RAM/Flash one-liners after a native ESP-IDF build. -The ninja ``size`` target (run after ``all`` in +The ninja ``size`` target (built together with ``all`` in ``toolchain.run_compile``) prints the per-region table inline as part of the build. This module adds two summary lines underneath, byte-identical to PlatformIO's output: diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index 2e8cc0c632..effb817f35 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -366,7 +366,9 @@ def _tool_env() -> dict[str, str]: return env -def run_reconfigure(verbose: bool = False) -> int: +def run_reconfigure( + verbose: bool = False, extra_env: dict[str, str] | None = None +) -> int: """Run the CMake configure, with the arguments idf.py uses.""" build_dir = _build_dir() build_dir.mkdir(parents=True, exist_ok=True) @@ -389,7 +391,7 @@ def run_reconfigure(verbose: bool = False) -> int: rc = run_build_tool( cmd, cwd=build_dir, - env=_tool_env(), + env={**_tool_env(), **(extra_env or {})}, filter_lines=None if verbose else FILTER_IDF_LINES, log_path=log_path, ) @@ -428,21 +430,21 @@ def _build_jobs(config) -> int | None: def _run_ninja( - target: str, - *, + *targets: str, verbose: bool, jobs: int | None, progress: bool = False, extra_env: dict[str, str] | None = None, ) -> int: - """Build one ninja target, with the flags and env idf.py uses.""" + """Build ninja targets in one run, with the flags and env idf.py uses.""" cmd = [_get_idf_tool("ninja")] if jobs is not None: cmd += ["-j", str(jobs)] if verbose: cmd.append("-v") - cmd.append(target) - log_path = _build_dir() / "log" / f"ninja_{Path(target).name}_output.log" + cmd += targets + log_name = "_".join(Path(t).name for t in targets) + log_path = _build_dir() / "log" / f"ninja_{log_name}_output.log" rc = run_build_tool( cmd, cwd=_build_dir(), @@ -452,7 +454,7 @@ def _run_ninja( log_path=log_path, ) if rc != 0: - _LOGGER.error("ninja %s failed with exit code %d", target, rc) + _LOGGER.error("ninja %s failed with exit code %d", " ".join(targets), rc) _print_hints(log_path) return rc @@ -845,10 +847,11 @@ def run_compile(config, verbose: bool) -> int: write_pch_checksum() - # idf.py's ``build size``, minus the second ``ninja all`` it runs first. - rc = _run_ninja("all", verbose=verbose, jobs=jobs, progress=True) - if rc == 0: - rc = _run_ninja("size", verbose=verbose, jobs=jobs, extra_env=_size_env()) + # idf.py's ``build size`` in one ninja run; size needs the map, so it + # runs after the link. + rc = _run_ninja( + "all", "size", verbose=verbose, jobs=jobs, progress=True, extra_env=_size_env() + ) if rc == 0: size_json = CORE.relative_build_path("build", "esp_idf_size.json") partitions = CORE.relative_build_path("partitions.csv") diff --git a/esphome/espota2.py b/esphome/espota2.py index 952f88fc4b..dbdf323366 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -65,9 +65,15 @@ CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01 CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02 CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04 CLIENT_FEATURE_SUPPORTS_NOISE = 0x08 +CLIENT_FEATURE_SUPPORTS_DEFLATE = 0x10 SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01 SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02 SERVER_FEATURE_SUPPORTS_NOISE = 0x04 +# Binding once offered: the device then expects the image size and a deflate stream +SERVER_FEATURE_SUPPORTS_DEFLATE = 0x08 + +# Wire constant: the deflate bit promises a 4 KB window (OTA_INFLATE_WINDOW_SIZE) +DEFLATE_WINDOW_BITS = 12 NOISE_FRAME_INDICATOR = 0x01 NOISE_HANDSHAKE_OK = 0x00 @@ -87,6 +93,9 @@ _SUPPORTED_OTA_TYPES: frozenset[int] = frozenset( ) UPLOAD_BLOCK_SIZE = 8192 +# Sizes on the wire are 4 bytes MSB first +SIZE_FIELD_BYTES = 4 +COMPRESS_LEVEL = 9 UPLOAD_BUFFER_SIZE = UPLOAD_BLOCK_SIZE * 8 # Flaky Wi-Fi links often drop the first OTA attempt, and the device may need time @@ -573,6 +582,7 @@ def perform_ota( CLIENT_FEATURE_SUPPORTS_COMPRESSION | CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + | CLIENT_FEATURE_SUPPORTS_DEFLATE ) if noise_psk: features_to_send |= CLIENT_FEATURE_SUPPORTS_NOISE @@ -675,8 +685,18 @@ def perform_ota( f"retry {flag_name}." ) - if features & SERVER_FEATURE_SUPPORTS_COMPRESSION: - upload_contents = gzip.compress(file_contents, compresslevel=9) + deflate = bool(extended_proto and features & SERVER_FEATURE_SUPPORTS_DEFLATE) + if deflate: + import zlib + + # The device inflates while receiving through a small ring window + upload_contents = zlib.compress( + file_contents, COMPRESS_LEVEL, wbits=-DEFLATE_WINDOW_BITS + ) + _LOGGER.info("Compressed to %s bytes (deflate)", len(upload_contents)) + elif features & SERVER_FEATURE_SUPPORTS_COMPRESSION: + # The device stores the gzip file and inflates it when it reboots + upload_contents = gzip.compress(file_contents, compresslevel=COMPRESS_LEVEL) _LOGGER.info("Compressed to %s bytes", len(upload_contents)) else: upload_contents = file_contents @@ -735,22 +755,20 @@ def perform_ota( send_check(sock, ota_type, "ota type") upload_size = len(upload_contents) - upload_size_encoded = [ - (upload_size >> 24) & 0xFF, - (upload_size >> 16) & 0xFF, - (upload_size >> 8) & 0xFF, - (upload_size >> 0) & 0xFF, - ] # The device erases flash between receiving the size and acking the # prepare, so this window shows the erase cost (near zero when the # device erases lazily during the upload) prepare_start = time.perf_counter() - send_check(sock, upload_size_encoded, "binary size") + send_check(sock, upload_size.to_bytes(SIZE_FIELD_BYTES, "big"), "binary size") + if deflate: + # Own frame: an encrypted session carries one field per frame + send_check(sock, file_size.to_bytes(SIZE_FIELD_BYTES, "big"), "image size") receive_exactly(sock, 1, "update prepare result", RESPONSE_UPDATE_PREPARE_OK) prepare_duration = time.perf_counter() - prepare_start _LOGGER.info("Preparing for upload took %.2f seconds", prepare_duration) - upload_md5 = hashlib.md5(upload_contents).hexdigest() + # The device hashes what it writes: the inflated image, else the received bytes + upload_md5 = hashlib.md5(file_contents if deflate else upload_contents).hexdigest() _LOGGER.debug("MD5 of upload is %s", upload_md5) send_check(sock, upload_md5, "file checksum") diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index 7c6cf691b9..e892664724 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -35,7 +35,7 @@ from esphome.core import ( TimePeriod, ) from esphome.expression import has_substitution_or_expression -from esphome.helpers import add_class_to_obj +from esphome.helpers import FALSY_BOOL_STRINGS, TRUTHY_BOOL_STRINGS, add_class_to_obj from esphome.util import OrderedDict, filter_yaml_files _LOGGER = logging.getLogger(__name__) @@ -75,6 +75,12 @@ def take_dropped_merge_keys() -> list[tuple[str, str]]: return CORE.data.pop(_MERGE_WARNINGS_KEY, []) +def _raise_invalid(message: str) -> None: + from voluptuous import Invalid + + raise Invalid(message) + + class SensitiveStr(str): """Marker subclass for validated strings that should be masked in user-visible YAML output. ``cv.sensitive`` wraps validated values in this @@ -224,16 +230,18 @@ class IncludeFile: Created during YAML parsing instead of loading the file immediately, allowing substitution variables to appear in the filename path - (e.g. ``!include device-${platform}.yaml``). The actual file is - loaded on the first call to ``load()``, and the result is cached. + (e.g. ``!include device-${platform}.yaml``) and in an optional condition. + The actual file is loaded on the first call to ``load()``, and the result + is cached. The client is responsible for testing the condition if needed. """ def __init__( self, parent_file: Path, file: str, - vars: dict[str, Any] | None, yaml_loader: Callable[[Path], Any], + vars: dict[str, Any] | None = None, + condition: bool | str | None = None, ) -> None: self.parent_file = parent_file # The raw include text may be a substitution/Jinja expression, so it @@ -241,8 +249,9 @@ class IncludeFile: # rewrites "/" to "\", which Jinja then decodes as escapes like # "\b" -> backspace (issue #18545). self.file = file - self.vars = vars self.yaml_loader = yaml_loader + self.vars = vars + self.condition = condition self._content: Any = _UNSET def __repr__(self) -> str: @@ -256,23 +265,62 @@ class IncludeFile: """ if self._content is not _UNSET: return self._content - if self.has_unresolved_expressions(): - from voluptuous import Invalid - - raise Invalid( + if self.has_unresolved_file(): + _raise_invalid( f"Cannot load include with unresolved substitutions: {self.file}" ) self._content = self.yaml_loader(self.parent_file.parent / self.file) self._content = add_context(self._content, self.vars) return self._content - def has_unresolved_expressions(self) -> bool: + def has_unresolved_file(self) -> bool: """Check if the filename contains substitution variables or Jinja expressions.""" return has_substitution_or_expression(self.file) + def should_load(self) -> bool: + """Evaluates the condition and returns True if the file should be loaded.""" + if self.condition is None: + return True + if isinstance(self.condition, bool): + return self.condition + if self.has_unresolved_condition(): + _raise_invalid( + f"Cannot evaluate include condition for '{self.file}' with unresolved substitutions: {self.condition}" + ) + value = self.condition.lower() + if value in TRUTHY_BOOL_STRINGS: + return True + if value not in FALSY_BOOL_STRINGS: + _raise_invalid( + f"Cannot convert include condition for '{self.file}' to a boolean, please use 'true' or 'false': {self.condition}" + ) + return False + + def has_unresolved_condition(self) -> bool: + """Check if the condition contains substitution variables or Jinja expressions.""" + return isinstance(self.condition, str) and has_substitution_or_expression( + self.condition + ) + def with_file(self, file: str) -> IncludeFile: """Clone this include with *file* as the filename.""" - return IncludeFile(self.parent_file, file, self.vars, self.yaml_loader) + return IncludeFile( + self.parent_file, + file, + self.yaml_loader, + vars=self.vars, + condition=self.condition, + ) + + def with_condition(self, condition: bool | str | None) -> IncludeFile: + """Clone this include with *condition* as the condition.""" + return IncludeFile( + self.parent_file, + self.file, + self.yaml_loader, + vars=self.vars, + condition=condition, + ) def _is_visible_path(rel: Path) -> bool: @@ -434,7 +482,7 @@ def force_load_include_files( if id(obj) in _seen: return _seen.add(id(obj)) - if obj.has_unresolved_expressions(): + if obj.has_unresolved_file(): _load_include_candidates( obj, warn_on_unresolved=warn_on_unresolved, @@ -578,14 +626,21 @@ def _resolve_merge_include(value: Any, node: yaml.Node, value_node: yaml.Node) - for _ in range(_MAX_MERGE_INCLUDE_DEPTH): if not isinstance(value, IncludeFile): break - if value.has_unresolved_expressions(): + if value.has_unresolved_file(): raise yaml.constructor.ConstructorError( "While constructing a mapping", node.start_mark, "Substitution in include filename with merge keys is not supported yet.", value_node.start_mark, ) - value = value.load() + if value.has_unresolved_condition(): + raise yaml.constructor.ConstructorError( + "While constructing a mapping", + node.start_mark, + "Substitution in include condition with merge keys is not supported yet.", + value_node.start_mark, + ) + value = value.load() if value.should_load() else {} else: raise yaml.constructor.ConstructorError( "While constructing a mapping", @@ -791,11 +846,11 @@ class ESPHomeLoaderMixin: @_add_data_ref def construct_include(self, node: yaml.Node) -> Any: - from esphome.const import CONF_VARS + from esphome.const import CONF_CONDITION, CONF_FILE, CONF_VARS - def extract_file_vars(node): + def extract_fields(node): fields = self.construct_yaml_map(node) - file = fields.get("file") + file = fields.get(CONF_FILE) if file is None: raise yaml.MarkedYAMLError("Must include 'file'", node.start_mark) if not isinstance(file, str): @@ -803,14 +858,21 @@ class ESPHomeLoaderMixin: "Include 'file' must be a string", node.start_mark ) vars = fields.get(CONF_VARS) - return file, vars + condition = fields.get(CONF_CONDITION) + if condition is not None and not isinstance(condition, (bool, str)): + raise yaml.MarkedYAMLError( + "Include 'condition' must be a boolean or string", node.start_mark + ) + return file, vars, condition if isinstance(node, yaml.nodes.MappingNode): - file, vars = extract_file_vars(node) + file, vars, condition = extract_fields(node) else: - file, vars = node.value, None + file, vars, condition = node.value, None, None - return IncludeFile(self.name, file, vars, self.yaml_loader) + return IncludeFile( + self.name, file, self.yaml_loader, vars=vars, condition=condition + ) # Directory includes (!include_dir_*) load eagerly during YAML parsing # because their paths are directory names, not individual files, and @@ -967,7 +1029,7 @@ def _load_yaml_internal(fname: Path, *, track_document_range: bool = True) -> An # Top-level !include returns a deferred IncludeFile; resolve it so # callers always receive the final content. if isinstance(res, IncludeFile): - res = res.load() + res = res.load() if res.should_load() else {} return res @@ -1358,8 +1420,12 @@ class ESPHomeDumper(yaml.SafeDumper): return self.represent_scalar(tag="!remove", value=value.value) def represent_include_file(self, value): - if value.vars: - mapping = {"file": value.file, "vars": value.vars} + if value.vars or value.condition is not None: + mapping = {"file": value.file} + if value.vars: + mapping["vars"] = value.vars + if value.condition is not None: + mapping["condition"] = value.condition return self.represent_mapping( tag="!include", mapping=mapping, flow_style=False ) diff --git a/script/build_helpers.py b/script/build_helpers.py index f0b354a8a7..885a5a1475 100644 --- a/script/build_helpers.py +++ b/script/build_helpers.py @@ -46,6 +46,8 @@ EXIT_SKIPPED = 1 EXIT_COMPILE_ERROR = 2 EXIT_CONFIG_ERROR = 3 EXIT_NO_EXECUTABLE = 4 +# A test folder with this name would be synced into src/esphome and swept away with the core tree +CORE_TREE_DIR = "esphome" # Name of the per-component YAML config file in benchmark directories BENCHMARK_YAML_FILENAME = "benchmark.yaml" @@ -465,8 +467,19 @@ def build_and_run( components = sorted(components) - # Build include list: main entry point + component folders + extra dirs - includes: list[str] = [main_entry] + components + # Build include list: main entry point + component folders + extra dirs. The core tree + # folder is listed file by file, nested files included, since a folder include would + # land in src/esphome (see CORE_TREE_DIR) + includes: list[str] = [main_entry] + for component in components: + if component != CORE_TREE_DIR: + includes.append(component) + continue + includes.extend( + str(path.relative_to(tests_dir)) + for path in sorted((tests_dir / component).rglob("*")) + if path.suffix in (".cpp", ".h") + ) if extra_include_dirs: for d in extra_include_dirs: if d.is_dir() and (any(d.glob("*.cpp")) or any(d.glob("*.h"))): diff --git a/script/check_idf_py_equivalence.py b/script/check_idf_py_equivalence.py index c0aa3c0d5e..cbd55d57cc 100755 --- a/script/check_idf_py_equivalence.py +++ b/script/check_idf_py_equivalence.py @@ -55,6 +55,19 @@ MACRO_CHANGED = ( "IDF changed __build_process_project_includes; update " "IDF_BOOTLOADER_OVERRIDE in esphome/build_gen/espidf.py" ) +VERSION_DRIFT = ( + "ESPHome reads ESP-IDF version {ours!r} from {source} but idf_tools reports " + "{theirs!r}; update read_idf_version_{source} in esphome/espidf/framework.py" +) +LWIP_NOT_EMPTY = ( + "lwip source {source} compiles to a non-empty object with {option} off; " + "drop it from LWIP_EMPTY_SOURCES in esphome/build_gen/espidf.py" +) +LWIP_NOTHING_MATCHED = ( + "no lwip object matched {regex!r} for {option}; the lwip layout or the " + "pattern in esphome/build_gen/espidf.py changed" +) +LWIP_NM_FAILED = "nm failed on lwip object {source}: {error}" WORK_SUFFIXES = (".obj", ".o", ".a", ".elf", ".map", ".bin", ".ld") DEFAULT_GLOB = "tests/test_build_components/build/.esphome/build/*" @@ -104,6 +117,51 @@ def _log_problems( return problems +def _lwip_empty_source_problems(build_path: Path) -> list[str]: + """Compile the lwip sources the generated CMakeLists drops; any with + symbols is a problem. Leaves the tree configured with every source.""" + # pylint: disable=protected-access + from esphome.build_gen.espidf import LWIP_EMPTY_SOURCES, LWIP_FULL_SOURCES_ENV + from esphome.espidf import toolchain + + if (rc := toolchain.run_reconfigure(extra_env={LWIP_FULL_SOURCES_ENV: "1"})) != 0: + return [f"CMake configure with every lwip source failed with exit code {rc}"] + if rc := toolchain._run_ninja("esp-idf/lwip/liblwip.a", verbose=False, jobs=None): + return [f"building every lwip source failed with exit code {rc}"] + build = build_path / "build" + config = json.loads( + (build / "config" / "sdkconfig.json").read_text(encoding="utf-8") + ) + objects = [ + obj.as_posix().removesuffix(".obj") + for obj in (build / "esp-idf" / "lwip").rglob("*.obj") + ] + nm = toolchain._parse_cmakecache(build / "CMakeCache.txt")["CMAKE_NM"] + problems = [] + for option, regex in LWIP_EMPTY_SOURCES: + # Absent means the option is invisible here; the filter keeps those. + if config.get(option.removeprefix("CONFIG_"), True): + continue + matched = [source for source in objects if re.search(regex, source)] + if not matched: + problems.append(LWIP_NOTHING_MATCHED.format(regex=regex, option=option)) + for source in matched: + name = Path(source).name + result = subprocess.run( + [nm, "--defined-only", f"{source}.obj"], + capture_output=True, + text=True, + check=False, + ) + if result.returncode: + problems.append( + LWIP_NM_FAILED.format(source=name, error=result.stderr.strip()) + ) + elif result.stdout.strip(): + problems.append(LWIP_NOT_EMPTY.format(source=name, option=option)) + return problems + + def _setup_core(build_path: Path, description: dict) -> tuple[str, str]: """Point CORE at the tree so ESPHome resolves the same IDF env as the build.""" from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT @@ -127,7 +185,7 @@ def check(build_path: Path) -> list[str]: # pylint: disable=protected-access from esphome.build_gen.espidf import idf_macro_matches from esphome.core import CORE - from esphome.espidf import toolchain + from esphome.espidf import framework, toolchain description = json.loads( (build_path / "build" / "project_description.json").read_text(encoding="utf-8") @@ -139,8 +197,18 @@ def check(build_path: Path) -> list[str]: CORE.skip_bootloader = skip_bootloader # A prior tree's memoized decision must not leak into this one. toolchain._cache().skip_bootloader = None - if not idf_macro_matches(toolchain._get_idf_path(version)): + idf_path = toolchain._get_idf_path(version) + if not idf_macro_matches(idf_path): return [MACRO_CHANGED] + # A managed tree always has version.txt, so the header branch is + # compared on its own or it would never be exercised here. + theirs = framework.idf_tools_version(idf_path) + for source, read in ( + ("txt", framework.read_idf_version_txt), + ("header", framework.read_idf_version_header), + ): + if (ours := read(idf_path)) != theirs: + return [VERSION_DRIFT.format(ours=ours, source=source, theirs=theirs)] # ESP-IDF's openthread stamps the configure time into its compile flags; # pin it before the env is cached so both configures get the same value. os.environ["SOURCE_DATE_EPOCH"] = "0" @@ -188,7 +256,8 @@ def check(build_path: Path) -> list[str]: problems.append(f"idf.py dropped {out} from {log}") elif mtimes_before.get(key) != mtimes_after[key]: problems.append(f"idf.py rebuilt {out}") - return problems + # Last: it reconfigures the tree, which would otherwise relink above. + return problems or _lwip_empty_source_problems(build_path) def main() -> int: diff --git a/script/ci-custom.py b/script/ci-custom.py index aaf177c941..692393b221 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -904,6 +904,8 @@ def lint_relative_py_import(fname: Path, line, col, content): # neither can live in a C++ namespace. "esphome/components/esp32_hosted/esp_now_hosted.cpp", "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", + # C header shared with the vendored decoder + "esphome/components/esphome/ota/ota_esphome_inflate.h", ], ) def lint_namespace(fname: Path, content: str) -> str | None: diff --git a/tests/components/esphome/__init__.py b/tests/components/esphome/__init__.py new file mode 100644 index 0000000000..713c0d5a63 --- /dev/null +++ b/tests/components/esphome/__init__.py @@ -0,0 +1,12 @@ +from esphome.loader import FileResource +from tests.testing_helpers import ComponentManifestOverride + + +def override_manifest(manifest: ComponentManifestOverride) -> None: + # to_code emits the component count the application needs + manifest.enable_codegen() + # Only the decoder is under test; its ota platform is not in this build + manifest.resources = manifest.resources + [ + FileResource("esphome.components.esphome", "ota/ota_esphome_inflate.c"), + FileResource("esphome.components.esphome", "ota/ota_esphome_inflate.h"), + ] diff --git a/tests/components/esphome/test_ota_inflate.cpp b/tests/components/esphome/test_ota_inflate.cpp new file mode 100644 index 0000000000..e18c96440f --- /dev/null +++ b/tests/components/esphome/test_ota_inflate.cpp @@ -0,0 +1,324 @@ +#include + +#include +#include +#include + +#include "esphome/components/esphome/ota/ota_esphome_inflate.h" + +namespace esphome::testing { + +// build_plain() compressed with the CLI's window (espota2.DEFLATE_WINDOW_BITS): +// DEFLATED = zlib.compress(plain, 9, wbits=-12) +// STORED = zlib.compress(plain[:300], 0, wbits=-12) +static const uint8_t DEFLATED[] = { + 0xed, 0xc8, 0xf7, 0x3f, 0xd4, 0x0f, 0x03, 0x00, 0x70, 0x67, 0xaf, 0x4b, 0x67, 0x66, 0x9f, 0x90, 0x91, 0x11, 0xc2, + 0x11, 0x91, 0xb8, 0xb3, 0xf7, 0x3a, 0xd9, 0x5f, 0x4e, 0x99, 0x67, 0x1e, 0xce, 0x8a, 0xac, 0xec, 0x59, 0xb8, 0xc2, + 0x95, 0x5d, 0x56, 0x42, 0x67, 0x73, 0x46, 0xf6, 0xca, 0xce, 0xc8, 0xc8, 0xc8, 0x91, 0x8a, 0x7c, 0x2f, 0x7a, 0xfe, + 0x86, 0xe7, 0x87, 0xe7, 0x87, 0xe7, 0xf5, 0xfa, 0xbc, 0x7f, 0x7c, 0xbb, 0x07, 0xa2, 0x1f, 0xfa, 0xf9, 0xb8, 0x43, + 0xfd, 0x82, 0x5c, 0xa0, 0x6e, 0xee, 0x28, 0x6f, 0x97, 0x20, 0x77, 0xa8, 0x3b, 0x70, 0xc0, 0x01, 0x07, 0x1c, 0x70, + 0xc0, 0x01, 0x07, 0x1c, 0x70, 0xc0, 0x01, 0x07, 0x1c, 0x70, 0xc0, 0x01, 0x07, 0x1c, 0x70, 0xc0, 0x01, 0x07, 0x1c, + 0x70, 0xc0, 0x01, 0xf7, 0x5f, 0xdd, 0x40, 0xd4, 0x63, 0x2f, 0x03, 0xf2, 0x17, 0xb2, 0xe5, 0x69, 0xc7, 0x5a, 0x04, + 0xdf, 0x46, 0x22, 0xa8, 0x4b, 0x6e, 0xd5, 0x3a, 0x80, 0x05, 0xe1, 0x81, 0x4a, 0x44, 0x56, 0x27, 0xa9, 0x21, 0xf7, + 0xad, 0xd9, 0xb0, 0xd3, 0xf1, 0xa5, 0x0b, 0x6a, 0x96, 0x56, 0xb2, 0xca, 0xb5, 0xee, 0x0f, 0x96, 0x28, 0xc3, 0x9e, + 0x8f, 0x6e, 0xf2, 0x84, 0xa4, 0x3b, 0x2d, 0x16, 0x64, 0x08, 0x6a, 0x28, 0x91, 0xee, 0x87, 0x1a, 0x81, 0xff, 0xed, + 0x2c, 0x5f, 0xda, 0x4a, 0x48, 0x5f, 0x91, 0xf0, 0x31, 0xfe, 0x6b, 0xbd, 0x81, 0x86, 0x92, 0x33, 0x1a, 0x6c, 0x69, + 0x32, 0xfb, 0x19, 0x56, 0x2c, 0xc6, 0xaa, 0xef, 0xe8, 0x16, 0x98, 0xcf, 0x98, 0xbf, 0xa0, 0x2d, 0xde, 0x7f, 0xdf, + 0x4b, 0xcb, 0xf6, 0x5c, 0xaf, 0x11, 0x24, 0xf0, 0x46, 0x3e, 0x49, 0x0e, 0x67, 0x0e, 0xcf, 0xf3, 0x57, 0xca, 0xb0, + 0x39, 0x55, 0xe1, 0xe7, 0xfc, 0x37, 0x29, 0xe8, 0xd7, 0xf3, 0x08, 0x2e, 0xfb, 0x7b, 0x6d, 0x3e, 0xfe, 0xe9, 0x2c, + 0x68, 0xe4, 0x20, 0x88, 0x57, 0x56, 0x41, 0x3d, 0xab, 0x9b, 0xbf, 0x0c, 0x0c, 0xae, 0x51, 0x48, 0x8b, 0x72, 0xcc, + 0xb8, 0xbb, 0xf3, 0x30, 0xa8, 0x5c, 0xb5, 0xa1, 0x2f, 0x07, 0x52, 0xe9, 0x36, 0xb8, 0x3c, 0xbc, 0xee, 0xbc, 0xf1, + 0xa3, 0x8b, 0x81, 0xc2, 0x03, 0xbf, 0x29, 0x5a, 0x22, 0x24, 0x97, 0xf8, 0xc9, 0xb5, 0xbf, 0xd2, 0x24, 0x4a, 0x86, + 0xc1, 0x2b, 0xb7, 0xb8, 0xde, 0xa7, 0xea, 0xa5, 0x1d, 0x13, 0x1c, 0x1d, 0xd3, 0x3c, 0x81, 0x60, 0x2e, 0x2f, 0x93, + 0x5c, 0x78, 0x43, 0x2e, 0x39, 0x68, 0x09, 0x13, 0x09, 0x10, 0xce, 0xd6, 0x8d, 0xe9, 0x2f, 0xaf, 0x88, 0x6f, 0x99, + 0x4f, 0xcd, 0xdc, 0xaa, 0xf9, 0x47, 0xdb, 0x1c, 0xb5, 0x89, 0x53, 0x10, 0x3d, 0x77, 0xac, 0x26, 0x04, 0x3a, 0x3b, + 0x18, 0xf8, 0x47, 0x75, 0x56, 0xa5, 0xda, 0x70, 0xfb, 0xf7, 0x0d, 0x3b, 0x6e, 0x4b, 0x2a, 0xbc, 0x49, 0x32, 0x43, + 0x95, 0x62, 0x83, 0x3d, 0xdc, 0x0a, 0x1f, 0x1d, 0xf9, 0x59, 0x6b, 0x95, 0xf0, 0x9b, 0xf5, 0x53, 0x9e, 0xb5, 0x65, + 0xeb, 0x74, 0xfa, 0x81, 0x9b, 0x61, 0xa3, 0x57, 0x2f, 0xda, 0x2c, 0xcd, 0xf8, 0xb0, 0x74, 0xb9, 0x62, 0x39, 0x91, + 0xd9, 0x7d, 0xb3, 0x03, 0x65, 0x2e, 0x66, 0x20, 0x18, 0xac, 0xa2, 0xeb, 0x3b, 0x35, 0x98, 0xa3, 0x28, 0x46, 0x30, + 0xee, 0xd3, 0xeb, 0x47, 0x49, 0x11, 0xc5, 0xcd, 0xa0, 0xa5, 0x1c, 0x2e, 0x9d, 0x14, 0xd6, 0x46, 0x4a, 0x05, 0x7b, + 0xd3, 0xb9, 0x0d, 0xeb, 0xd7, 0x6f, 0xb5, 0xf4, 0xed, 0x3f, 0x27, 0xb8, 0xec, 0x51, 0xb1, 0x6e, 0xb0, 0x14, 0xed, + 0xdf, 0x90, 0x72, 0x59, 0x71, 0xd5, 0xf5, 0xf2, 0x61, 0x5f, 0xa7, 0x8a, 0xd7, 0x0f, 0x80, 0x2f, 0xe5, 0x0f, 0x45, + 0xf2, 0x4d, 0xd1, 0xdc, 0xdd, 0xce, 0x46, 0xdf, 0x77, 0xf2, 0xa6, 0xa2, 0x79, 0x77, 0xf0, 0x0e, 0xaa, 0x68, 0x14, + 0x85, 0x32, 0x05, 0x29, 0x75, 0x2b, 0x6c, 0xb0, 0x7c, 0x84, 0xc9, 0xec, 0x49, 0x70, 0x9b, 0x38, 0x36, 0x4c, 0xe9, + 0xa5, 0xdc, 0xb1, 0xb8, 0x28, 0xd6, 0x20, 0x89, 0x8a, 0x2a, 0x62, 0xc1, 0x90, 0x3a, 0x43, 0x48, 0xaa, 0xfc, 0xec, + 0x52, 0xf4, 0x02, 0xa7, 0xcd, 0x46, 0xf9, 0x78, 0x64, 0x4f, 0xad, 0x6b, 0x17, 0xdb, 0x56, 0xa2, 0xa1, 0x6a, 0xcf, + 0x3b, 0x66, 0x64, 0x01, 0xc2, 0xd6, 0xae, 0x23, 0x34, 0x82, 0x60, 0x0a, 0x3a, 0xe4, 0xdd, 0x7f, 0xaa, 0x97, 0x44, + 0x9a, 0x63, 0x8f, 0xaf, 0xa9, 0x97, 0x5a, 0x55, 0xcc, 0x81, 0x48, 0x83, 0xf8, 0xe6, 0xc5, 0xef, 0xdf, 0xfa, 0x5a, + 0x4f, 0x7f, 0x18, 0x56, 0xc0, 0x66, 0x36, 0xad, 0x8a, 0x79, 0xab, 0xde, 0xf4, 0x7b, 0x70, 0x98, 0xea, 0xfa, 0xf4, + 0xc7, 0x01, 0x31, 0x16, 0xea, 0xf1, 0x70, 0x8d, 0xee, 0x87, 0x52, 0x13, 0x01, 0x9e, 0x0c, 0xbd, 0x5a, 0x14, 0x01, + 0x8e, 0xf7, 0x6e, 0xff, 0xce, 0x7f, 0xa4, 0xd1, 0xa1, 0x6a, 0x9b, 0x95, 0xb2, 0x31, 0x8a, 0x6f, 0xc8, 0xfe, 0x8c, + 0x29, 0x55, 0xbc, 0xfc, 0x61, 0xd0, 0xde, 0xb0, 0xa9, 0x0c, 0x01, 0x0f, 0xba, 0x77, 0x7e, 0x77, 0xd9, 0x76, 0xa4, + 0xfd, 0xab, 0x38, 0x18, 0x92, 0xec, 0xf0, 0xd3, 0x57, 0x7f, 0xf4, 0xbd, 0x65, 0xd4, 0x77, 0x8e, 0xa1, 0x92, 0x82, + 0x0c, 0x7b, 0x34, 0xd3, 0x11, 0xfb, 0xc0, 0x36, 0x23, 0x8e, 0xbb, 0x89, 0xdf, 0x70, 0xdc, 0x9a, 0x76, 0x45, 0x0b, + 0x22, 0xfa, 0x35, 0xdf, 0x8f, 0x45, 0x48, 0x7d, 0xed, 0xdc, 0x06, 0x01, 0x5e, 0xbb, 0xe9, 0xf9, 0x07, 0x93, 0xf6, + 0x69, 0x6f, 0xf1, 0xfc, 0xfd, 0xfa, 0x41, 0xc0, 0x13, 0x9e, 0x74, 0x2d, 0x76, 0xfe, 0xa7, 0xbe, 0x4e, 0xd9, 0xaa, + 0x64, 0xa6, 0xed, 0xd3, 0xad, 0xee, 0x62, 0x9d, 0x39, 0xc9, 0xb8, 0xae, 0x86, 0x31, 0x4f, 0x62, 0x57, 0xea, 0xea, + 0x5a, 0xe3, 0x59, 0xd8, 0x99, 0xb3, 0xcd, 0x9f, 0x3b, 0xea, 0x58, 0x99, 0x11, 0xdc, 0x3d, 0xac, 0x58, 0xd9, 0xa6, + 0xae, 0x2d, 0x07, 0x1d, 0xd7, 0xfa, 0x87, 0x78, 0xa7, 0x7f, 0x2a, 0x4f, 0x25, 0x58, 0xef, 0x53, 0x78, 0x2e, 0x93, + 0xdc, 0x44, 0xcc, 0x53, 0x88, 0x77, 0x1c, 0xda, 0x14, 0xaf, 0xe1, 0x67, 0x92, 0xff, 0x36, 0x96, 0x20, 0x6f, 0x9d, + 0x2c, 0x7f, 0xea, 0x31, 0xf2, 0x34, 0x50, 0xc2, 0x39, 0x84, 0xee, 0x4c, 0xbe, 0xca, 0x06, 0x6f, 0x67, 0x42, 0xea, + 0x13, 0x58, 0xee, 0xdd, 0x8e, 0x29, 0x4f, 0xee, 0xd8, 0x93, 0x0d, 0x45, 0x80, 0x4d, 0xf3, 0x12, 0x79, 0xbb, 0x36, + 0xa3, 0x73, 0x95, 0x95, 0xb6, 0xfc, 0x54, 0x60, 0xb2, 0xcc, 0x71, 0xaa, 0xf1, 0x6b, 0x66, 0xed, 0xba, 0x8b, 0xd6, + 0x6d, 0x61, 0x79, 0x61, 0x1d, 0xb3, 0xba, 0xa6, 0x2f, 0xaa, 0xdc, 0x1d, 0xb8, 0x22, 0xd8, 0x98, 0x58, 0xed, 0x4d, + 0x3c, 0xea, 0xa9, 0x37, 0x5e, 0x5e, 0x7b, 0x47, 0xa1, 0x7a, 0x39, 0x42, 0xe4, 0x3b, 0xbb, 0x69, 0x0a, 0x8b, 0x32, + 0x6e, 0x63, 0xeb, 0x87, 0xf6, 0x5d, 0xaa, 0xbf, 0xbe, 0xc5, 0xb2, 0x85, 0x60, 0xdc, 0x32, 0x07, 0x85, 0x73, 0x3d, + 0x96, 0x8b, 0x89, 0x71, 0x52, 0xb4, 0x93, 0xe6, 0x18, 0xad, 0xbf, 0xce, 0x21, 0x1d, 0x33, 0xb5, 0xb3, 0x35, 0x6a, + 0x5b, 0xe7, 0x47, 0x13, 0x19, 0x8f, 0x53, 0xf4, 0x0c, 0x2a, 0x39, 0x16, 0x37, 0x39, 0x3b, 0x5f, 0x81, 0x51, 0xbc, + 0x23, 0x92, 0x2c, 0x8d, 0xbf, 0x2f, 0xee, 0xbf, 0xed, 0x9d, 0x3f, 0xe0, 0x16, 0x21, 0x5a, 0x57, 0xa6, 0x8c, 0x58, + 0x7a, 0xd5, 0xa1, 0x6d, 0xed, 0xe8, 0x90, 0x97, 0x14, 0xb4, 0x6b, 0xa5, 0x3b, 0xd7, 0x90, 0x84, 0x9e, 0x07, 0xc2, + 0x7f, 0x1e, 0x08, 0xa0, 0x6f, 0x69, 0xf1, 0xcc, 0x4e, 0xca, 0x07, 0x64, 0xa2, 0xb4, 0xbc, 0x5f, 0xe0, 0xa7, 0x0e, + 0x31, 0x77, 0x6f, 0x35, 0xd7, 0x66, 0x83, 0x5d, 0x64, 0x4e, 0xf1, 0x3c, 0x7a, 0xcb, 0xa5, 0xfc, 0xc9, 0x95, 0xab, + 0x27, 0x30, 0x6b, 0x82, 0x57, 0xcd, 0xb0, 0x85, 0x9d, 0xc5, 0xa9, 0x0f, 0xdb, 0xe3, 0x5a, 0xc1, 0xb2, 0x3f, 0xfa, + 0xea, 0xf3, 0x3a, 0xa0, 0xe4, 0xad, 0xd7, 0x14, 0xcf, 0x4f, 0xc9, 0x46, 0x09, 0x6e, 0x70, 0x27, 0xe4, 0x9b, 0x5f, + 0x61, 0x4a, 0xb2, 0xfb, 0xc3, 0xf0, 0x3a, 0x53, 0x38, 0x2a, 0x59, 0x48, 0x2b, 0xab, 0x21, 0x64, 0x1a, 0x1c, 0x90, + 0xb6, 0x69, 0xbb, 0x89, 0xe3, 0x9a, 0x12, 0xb7, 0xfd, 0x47, 0xcc, 0x57, 0xdb, 0x2b, 0xf6, 0x35, 0xe1, 0x79, 0xe0, + 0xbf, 0x6d, 0x7f, 0x71, 0x3c, 0x21, 0x27, 0x82, 0x12, 0x05, 0xae, 0x43, 0x11, 0x0f, 0xc8, 0xe7, 0x3c, 0xf0, 0x0b, + 0xe4, 0x69, 0xff, 0xb2, 0xda, 0x9e, 0x7f, 0xfe, 0xa4, 0x3f, 0xdf, 0x02, 0x58, 0xaa, 0x60, 0xd2, 0x8e, 0xd5, 0x6c, + 0x92, 0x22, 0x38, 0xb1, 0xd2, 0x84, 0x80, 0xea, 0xce, 0xfe, 0x34, 0x79, 0xd6, 0x94, 0xb0, 0xad, 0x51, 0x67, 0x65, + 0xf5, 0xab, 0xd5, 0x18, 0x05, 0x92, 0x8a, 0x24, 0x36, 0xc2, 0xc4, 0x99, 0xad, 0x0d, 0x26, 0xb8, 0x70, 0xea, 0x5b, + 0x74, 0x35, 0x6a, 0xfa, 0xac, 0x97, 0xd8, 0x4c, 0x08, 0x6d, 0xc1, 0xfe, 0x60, 0xe4, 0xd4, 0x8f, 0x93, 0xd4, 0xaf, + 0xa2, 0xec, 0xb8, 0x71, 0xf6, 0xcb, 0x3f, 0xdc, 0x3e, 0x42, 0xca, 0xe2, 0x8d, 0xb9, 0x0a, 0x2c, 0xef, 0xb6, 0x18, + 0x75, 0x46, 0x8a, 0x2b, 0x21, 0xeb, 0xc5, 0x12, 0x4d, 0xd7, 0xb8, 0x8e, 0x6c, 0xec, 0x9d, 0x12, 0xde, 0xa5, 0x45, + 0xab, 0xc4, 0x26, 0x47, 0xfa, 0x57, 0xad, 0x78, 0x4c, 0x5e, 0x87, 0x0b, 0x42, 0x6a, 0xbe, 0xef, 0xbd, 0xbc, 0xe5, + 0xd7, 0x4c, 0x3a, 0x0f, 0x96, 0x29, 0xcb, 0x45, 0x46, 0x3f, 0x8a, 0x31, 0xda, 0x8d, 0x7f, 0x89, 0xa4, 0xd6, 0xb4, + 0xe5, 0x99, 0x27, 0xa7, 0xab, 0x15, 0x82, 0x75, 0x3e, 0xc0, 0x9d, 0xae, 0x8c, 0x7f, 0x19, 0x80, 0xf5, 0xa6, 0x89, + 0xbc, 0xb7, 0x36, 0x50, 0x1f, 0x63, 0xca, 0x40, 0x3e, 0xe1, 0x4c, 0x2d, 0x1e, 0x15, 0x51, 0x93, 0xf2, 0x1a, 0x97, + 0x3e, 0xad, 0x5e, 0xba, 0xe2, 0xd6, 0xa7, 0x7c, 0x51, 0x45, 0xdf, 0x92, 0xbe, 0x35, 0xf3, 0xb4, 0xe8, 0x5c, 0x2e, + 0x6a, 0x5c, 0xda, 0xe8, 0xf7, 0x51, 0xdc, 0xcb, 0x06, 0xd2, 0xd2, 0xa4, 0x7c, 0x7d, 0x5f, 0x20, 0xb8, 0x50, 0x04, + 0xb5, 0xb1, 0x9a, 0x4e, 0xc9, 0x94, 0xe4, 0x40, 0x2b, 0x85, 0xc8, 0xd3, 0x71, 0xfa, 0x83, 0x81, 0x28, 0x53, 0xef, + 0xa0, 0xb0, 0xf0, 0xc8, 0xb9, 0x52, 0xb2, 0x0f, 0x8c, 0x49, 0x93, 0xeb, 0x2a, 0x95, 0x83, 0x4f, 0x47, 0xe8, 0x0c, + 0x83, 0x7b, 0x88, 0x18, 0x63, 0xa9, 0x87, 0x7f, 0xe3, 0x57, 0xa6, 0x57, 0xdd, 0x6d, 0x2a, 0x85, 0xe9, 0x79, 0x16, + 0x3b, 0x07, 0x59, 0x4f, 0xed, 0x9a, 0x17, 0xe7, 0xcd, 0x35, 0xa6, 0x9b, 0x06, 0x1d, 0x18, 0x5e, 0xb2, 0x9b, 0xac, + 0x18, 0xc6, 0x5b, 0xec, 0xbe, 0x29, 0xa3, 0x57, 0xc2, 0x6b, 0x3f, 0x11, 0x89, 0xd1, 0x57, 0xe4, 0xb8, 0xaf, 0x11, + 0xba, 0x90, 0x93, 0x4f, 0x5e, 0x0c, 0x6b, 0x9f, 0x61, 0x5c, 0xdd, 0xb6, 0xcf, 0x18, 0xcc, 0x9a, 0x95, 0x32, 0x5f, + 0x6f, 0xf1, 0x43, 0x17, 0x2d, 0xbc, 0x70, 0xbe, 0xd4, 0x76, 0x38, 0x42, 0xcc, 0xdd, 0xf0, 0xff, 0x5b, 0xe8, 0x19, + 0xc9, 0xf4, 0x25, 0xda, 0xdb, 0x5f, 0xd2, 0x3c, 0xf3, 0xe0, 0xed, 0xb0, 0xa2, 0x4c, 0xdf, 0x05, 0xf2, 0x0c, 0xc4, + 0x21, 0xb9, 0xc4, 0x02, 0xd2, 0x1e, 0x46, 0x2c, 0x57, 0x78, 0xb0, 0xf1, 0x85, 0x33, 0x51, 0xc5, 0x3e, 0xce, 0x69, + 0x68, 0x77, 0x3e, 0xb4, 0x5a, 0x4e, 0x43, 0x89, 0x85, 0x71, 0x36, 0x84, 0x56, 0x5c, 0x6b, 0xce, 0x78, 0x4e, 0x86, + 0x6a, 0x5f, 0xf1, 0x2d, 0x73, 0x4c, 0xba, 0xc1, 0x5e, 0x2b, 0x6e, 0x0d, 0xdd, 0x45, 0x1d, 0x92, 0xb4, 0x31, 0xd9, + 0xce, 0x0b, 0x58, 0x67, 0xc1, 0xa6, 0x47, 0x2e, 0x5e, 0x6d, 0x6d, 0xa3, 0x78, 0xed, 0xc5, 0xcb, 0xac, 0x20, 0xb8, + 0x6a, 0x14, 0x32, 0x55, 0xe8, 0xfb, 0xe0, 0xee, 0x31, 0xfc, 0x47, 0x33, 0xf6, 0xe7, 0x54, 0xd1, 0x61, 0x91, 0x74, + 0x90, 0xfa, 0x44, 0x0a, 0xc8, 0xaf, 0xa2, 0xdb, 0x2c, 0x83, 0xda, 0xe8, 0xd0, 0xb7, 0x34, 0x8a, 0xc8, 0xbb, 0x86, + 0x58, 0x82, 0x1a, 0x2f, 0xd6, 0xc5, 0x92, 0xe9, 0xd0, 0x21, 0x11, 0xf6, 0xeb, 0x9b, 0xed, 0xbf, 0xa9, 0x71, 0x89, + 0xb2, 0x59, 0x76, 0xb3, 0xeb, 0x9c, 0xe1, 0x73, 0x79, 0x7b, 0x11, 0x3d, 0x4e, 0x33, 0xca, 0xf6, 0x7e, 0xd8, 0xcf, + 0x06, 0x41, 0x39, 0xef, 0x78, 0x5a, 0x89, 0x69, 0x0c, 0xff, 0xda, 0x31, 0x16, 0x32, 0xd4, 0xe5, 0x2d, 0x8b, 0x22, + 0xb9, 0x82, 0x38, 0xd8, 0x22, 0xfc, 0xe8, 0xbd, 0xbf, 0x0d, 0x4b, 0xc9, 0x0d, 0x74, 0x5e, 0x2d, 0xb2, 0x1f, 0x23, + 0xa8, 0xf8, 0xa7, 0x27, 0xe1, 0x02, 0x61, 0x83, 0xc4, 0xd0, 0x63, 0x0e, 0x23, 0xf2, 0x34, 0x06, 0x69, 0xd1, 0xa3, + 0xa2, 0x76, 0x76, 0x94, 0xbc, 0x1f, 0x78, 0x73, 0x5a, 0x48, 0x82, 0xb4, 0x68, 0x3d, 0x24, 0x52, 0xe8, 0x92, 0x4e, + 0x58, 0xdf, 0x55, 0x3f, 0xa8, 0x48, 0x59, 0x35, 0x90, 0x9b, 0x55, 0xc9, 0xdc, 0xff, 0xcc, 0x22, 0x7c, 0xd1, 0x29, + 0xe0, 0xd5, 0xd8, 0x31, 0x3d, 0x9b, 0xae, 0x38, 0x52, 0x2a, 0xa1, 0xc5, 0x86, 0x19, 0x93, 0xe7, 0x2d, 0x9b, 0xa9, + 0x09, 0xbf, 0xdd, 0x89, 0x43, 0x35, 0xb2, 0x6f, 0x91, 0xba, 0x28, 0xfa, 0x2f, 0x2c, 0xab, 0xe8, 0x58, 0xe6, 0x4c, + 0xbf, 0xd5, 0xaf, 0x49, 0xcd, 0x7f, 0x18, 0xa4, 0x69, 0xd3, 0xff, 0x99, 0xad, 0x9d, 0xb0, 0x93, 0x3b, 0x04, 0x01, + 0xd5, 0x7d, 0x33, 0xe0, 0x66, 0xe2, 0xd4, 0xeb, 0xae, 0xcd, 0x7f, 0x66, 0xf2, 0x2c, 0xa7, 0xfe, 0x7d, 0x51, 0x8b, + 0x2a, 0x25, 0xa8, 0x90, 0xb2, 0x71, 0xe5, 0x98, 0x40, 0xb8, 0xe9, 0x1f, 0x64, 0xd9, 0x90, 0xf2, 0x39, 0x25, 0x6f, + 0x58, 0x35, 0x9e, 0x40, 0x25, 0x92, 0xbb, 0xa2, 0x4d, 0x45, 0xf6, 0xc3, 0x1a, 0xc6, 0xc0, 0xf6, 0xc2, 0x9b, 0x2b, + 0xb5, 0xc7, 0xe9, 0xc8, 0x75, 0x03, 0x71, 0x1c, 0x76, 0x0e, 0xde, 0x5c, 0xe7, 0xf4, 0x91, 0xc4, 0x1d, 0x9f, 0xcb, + 0x67, 0x1c, 0x1b, 0xc9, 0x25, 0x6d, 0x95, 0x05, 0xd7, 0x3b, 0xbf, 0x2d, 0x90, 0x96, 0xc3, 0x44, 0xa3, 0x0a, 0xee, + 0x7a, 0x4c, 0x7e, 0x7f, 0xc2, 0x4e, 0x4f, 0x38, 0x88, 0xe9, 0x9f, 0x93, 0xd8, 0xd1, 0x10, 0x8c, 0x85, 0x89, 0xb2, + 0xd0, 0x95, 0xc3, 0xe4, 0xd2, 0x8f, 0x2b, 0x0c, 0x82, 0xbd, 0x25, 0x74, 0x39, 0xe7, 0x3e, 0xef, 0xd2, 0x2f, 0x6f, + 0xdc, 0xe0, 0x75, 0x6b, 0xf2, 0x66, 0x7d, 0x65, 0x18, 0x26, 0xb0, 0x7a, 0x7a, 0x45, 0xf0, 0xf2, 0xc4, 0xbe, 0x70, + 0x6e, 0x58, 0x3a, 0x54, 0x31, 0x64, 0xf6, 0x49, 0xb5, 0xdc, 0x1c, 0x9e, 0x87, 0xdb, 0x6a, 0x92, 0x2a, 0x59, 0x73, + 0x35, 0x4d, 0x59, 0xed, 0xd0, 0x8a, 0xd9, 0xac, 0xea, 0x7c, 0xc0, 0x0a, 0xfe, 0x8f, 0x29, 0xb7, 0xda, 0xdc, 0x8a, + 0x55, 0x0f, 0x61, 0xc0, 0x82, 0x49, 0x4c, 0xc2, 0x51, 0x4b, 0x41, 0x7d, 0x1f, 0xf1, 0x07, 0x13, 0x23, 0x16, 0xe4, + 0x88, 0x76, 0x38, 0x92, 0xbd, 0x3d, 0xe2, 0x7b, 0xab, 0x1c, 0x53, 0x35, 0xfb, 0x85, 0x76, 0x35, 0xf7, 0xfc, 0xdc, + 0x3e, 0xc9, 0xae, 0x6a, 0x7c, 0x10, 0x44, 0xa5, 0xfb, 0xc4, 0x81, 0x9a, 0x6d, 0x9b, 0xf2, 0x87, 0x0a, 0x9e, 0xcd, + 0x75, 0xc2, 0xd5, 0xe7, 0x0b, 0xcd, 0x5c, 0x6e, 0xf2, 0x51, 0x96, 0xa3, 0xe8, 0xd5, 0x5a, 0x17, 0xf5, 0x1a, 0x28, + 0xff, 0xc8, 0xf0, 0xfc, 0x7e, 0x3b, 0xd3, 0xf3, 0x12, 0x61, 0xe9, 0x02, 0xfa, 0x7b, 0x8e, 0x6b, 0x34, 0x4a, 0x5a, + 0xb6, 0xb1, 0x71, 0x7d, 0xad, 0xbc, 0x88, 0x22, 0xf6, 0x14, 0xc5, 0x57, 0x1a, 0x0b, 0xb6, 0x8b, 0xbb, 0x9a, 0x09, + 0xf2, 0xe8, 0x8d, 0x32, 0xd1, 0x54, 0xa9, 0xd2, 0x9f, 0xc5, 0xf4, 0x12, 0xad, 0xb3, 0xfd, 0x52, 0xb7, 0xff, 0x0e, + 0x72, 0x59, 0xcb, 0x1d, 0x8d, 0xa4, 0xf2, 0x0f, 0x39, 0x8f, 0xc9, 0x84, 0xe0, 0x5c, 0xdd, 0x9c, 0xe4, 0x42, 0x4f, + 0xf2, 0x09, 0x1e, 0x05, 0xcc, 0x41, 0xdd, 0xbd, 0xc1, 0xed, 0xdc, 0x8e, 0x64, 0xc3, 0x67, 0x51, 0xe5, 0x8a, 0x9a, + 0x0d, 0x09, 0xb8, 0x4b, 0xa1, 0xb2, 0xdb, 0x01, 0xde, 0xf5, 0xd1, 0x51, 0xf7, 0x9d, 0x88, 0x59, 0xfd, 0xd5, 0x13, + 0x0d, 0x05, 0xd5, 0x5e, 0xfa, 0x63, 0x26, 0x25, 0xad, 0x01, 0x83, 0x6f, 0xf3, 0x7f, 0x56, 0x11, 0x89, 0xaa, 0xdb, + 0x49, 0xaa, 0x8f, 0x7d, 0x10, 0x99, 0xbf, 0x8b, 0xcf, 0xf4, 0xf8, 0xc2, 0x4e, 0x36, 0xa7, 0xb1, 0xfe, 0xc5, 0xf0, + 0x07, 0xc3, 0x57, 0xe3, 0xbd, 0x1e, 0xa1, 0x56, 0x46, 0x32, 0x43, 0x8f, 0x8d, 0xc7, 0x73, 0x0a, 0xa5, 0x35, 0xa9, + 0x93, 0x7c, 0x53, 0x3c, 0xf2, 0x58, 0xf3, 0x6a, 0x37, 0x41, 0xb4, 0xc7, 0x9d, 0xcd, 0xb1, 0x67, 0xb7, 0xaa, 0x5f, + 0x25, 0xb9, 0xc5, 0xb6, 0xa2, 0xdb, 0xce, 0x34, 0x2c, 0xea, 0x93, 0x45, 0x6e, 0x7b, 0x6f, 0xb6, 0xf6, 0x89, 0x83, + 0xd1, 0x9e, 0x08, 0x4b, 0x57, 0x4e, 0xd9, 0xf3, 0xe8, 0x81, 0x03, 0x5c, 0xd6, 0x6c, 0x2c, 0xea, 0x27, 0x2d, 0xfb, + 0x39, 0xbb, 0x25, 0x81, 0xbb, 0x1a, 0xc3, 0xcb, 0xf5, 0x5e, 0x69, 0x1c, 0xa4, 0x8a, 0x0e, 0x4a, 0x69, 0xdf, 0xcd, + 0x6c, 0xc7, 0xce, 0x48, 0xdb, 0x7a, 0x0b, 0xcb, 0x40, 0xc8, 0x59, 0x81, 0xde, 0x71, 0x3e, 0x0d, 0x41, 0xc9, 0xa2, + 0x57, 0xae, 0xb7, 0x3f, 0x8b, 0x4e, 0x7c, 0xcb, 0xf9, 0xa5, 0xcb, 0xf8, 0xee, 0x52, 0xbe, 0x54, 0xa8, 0xef, 0x99, + 0xb7, 0xa1, 0x0b, 0x77, 0xa2, 0xa7, 0x89, 0x47, 0x7f, 0x48, 0xaa, 0x02, 0x36, 0xbf, 0x9d, 0x97, 0xba, 0x43, 0x5e, + 0x39, 0xa6, 0x99, 0x81, 0xde, 0xe4, 0x98, 0x06, 0x6a, 0xc8, 0x2c, 0x96, 0xf5, 0x51, 0x8c, 0x95, 0xd3, 0xe4, 0x2b, + 0x94, 0x9a, 0x15, 0x71, 0x9f, 0xa4, 0x24, 0xfc, 0x92, 0x97, 0x18, 0xb7, 0x56, 0xe5, 0x59, 0xe1, 0x1a, 0x79, 0x3a, + 0x47, 0x77, 0x92, 0xf5, 0x75, 0x42, 0xbd, 0x2d, 0x55, 0x05, 0xcf, 0xe8, 0xea, 0x64, 0x8b, 0xff, 0x30, 0xa1, 0x1e, + 0x28, 0x2c, 0xa9, 0xaa, 0xd4, 0x19, 0x1d, 0x24, 0x7a, 0xcf, 0x5d, 0x39, 0xf2, 0xe3, 0xd2, 0xab, 0x26, 0x97, 0x35, + 0xf6, 0x70, 0x8b, 0x09, 0x8b, 0x00, 0xb9, 0xb8, 0xa8, 0xb7, 0x78, 0xf1, 0x96, 0x03, 0xd2, 0x72, 0x46, 0x26, 0x82, + 0x41, 0xce, 0xe3, 0x73, 0xb2, 0x31, 0x4f, 0x49, 0xc8, 0xa8, 0xcf, 0xcc, 0xb8, 0xcc, 0x5d, 0xab, 0xd1, 0x1e, 0xf5, + 0x5a, 0x9a, 0x54, 0xaf, 0x15, 0xed, 0x5f, 0x96, 0x2b, 0x62, 0x0a, 0x4c, 0x67, 0x21, 0x98, 0xfd, 0x29, 0xf8, 0x12, + 0x9d, 0xc2, 0x34, 0x4d, 0x70, 0x0e, 0x3b, 0x5e, 0xe3, 0x91, 0x58, 0x82, 0xa8, 0x9c, 0x7d, 0x11, 0xfb, 0x5d, 0x81, + 0x0c, 0x0f, 0x9d, 0x84, 0xbb, 0xeb, 0x65, 0x1d, 0xeb, 0x6e, 0x18, 0x8e, 0x93, 0x70, 0xbb, 0x9d, 0xed, 0x58, 0xb2, + 0x64, 0xf2, 0x7b, 0x8f, 0x05, 0xd8, 0xfa, 0x55, 0x50, 0xef, 0x7a, 0x49, 0x0d, 0xdf, 0xca, 0x63, 0xff, 0x41, 0x1d, + 0x89, 0x79, 0xca, 0x8c, 0x1b, 0xbe, 0x75, 0x6f, 0x0e, 0x6b, 0x68, 0x8c, 0x5c, 0x93, 0xc8, 0xf3, 0x62, 0x0b, 0x68, + 0xb1, 0x32, 0x2f, 0x1e, 0xda, 0xe5, 0x6d, 0x45, 0xaf, 0x3d, 0xa7, 0xaf, 0x81, 0x0f, 0xda, 0x82, 0x85, 0x94, 0x06, + 0xc5, 0x3f, 0xf5, 0x73, 0x5f, 0xa4, 0x43, 0x5f, 0xb3, 0xac, 0x3d, 0x16, 0xe8, 0x6f, 0x84, 0xb3, 0xb6, 0x75, 0xf7, + 0x13, 0x03, 0x46, 0x67, 0x87, 0x9a, 0xf1, 0x6c, 0x2e, 0x33, 0x33, 0x14, 0x85, 0x18, 0xe9, 0xd6, 0x1b, 0xee, 0x17, + 0xe9, 0xe7, 0x6b, 0xd4, 0x46, 0xaf, 0x26, 0x55, 0x8d, 0xfc, 0x3a, 0xb4, 0x61, 0x1a, 0x9e, 0x2d, 0x85, 0xa2, 0x86, + 0x8c, 0x92, 0xbd, 0x8d, 0x0e, 0x4b, 0x4e, 0x0d, 0xdf, 0x4a, 0xc6, 0x66, 0xcd, 0x28, 0xff, 0xc4, 0x44, 0xba, 0x39, + 0x4a, 0x04, 0x43, 0xce, 0x95, 0x47, 0x99, 0x8c, 0x9b, 0xb7, 0x72, 0x48, 0xdd, 0x9b, 0x98, 0x4f, 0xd5, 0x18, 0x81, + 0xb4, 0xef, 0xfa, 0x96, 0x54, 0x9b, 0xda, 0x73, 0xe2, 0xe0, 0x27, 0x2f, 0x10, 0x0f, 0xf9, 0x0e, 0x24, 0xb4, 0xd2, + 0x39, 0xf7, 0x49, 0x8c, 0x5c, 0x05, 0x37, 0xdd, 0xa7, 0x73, 0xac, 0x16, 0x85, 0x71, 0x46, 0xd1, 0x6f, 0x7a, 0x25, + 0xd2, 0x5c, 0x4e, 0x70, 0x26, 0x93, 0x36, 0x22, 0x5f, 0xe5, 0x42, 0x4e, 0x89, 0xf2, 0x4c, 0x55, 0x9b, 0xd1, 0xa5, + 0xaf, 0x87, 0xd8, 0x7e, 0xe8, 0xdd, 0xb9, 0x47, 0xb3, 0xda, 0xf7, 0x98, 0x52, 0x73, 0x8b, 0x3c, 0x0b, 0xe7, 0xfa, + 0x14, 0x7f, 0x9a, 0xa6, 0x8e, 0xf4, 0x7c, 0xad, 0x58, 0x11, 0x73, 0xe3, 0x3a, 0x56, 0x47, 0x9b, 0x95, 0x24, 0x68, + 0x35, 0xa9, 0xd9, 0x6e, 0x5c, 0x4d, 0xe8, 0xa4, 0xb8, 0x80, 0x0b, 0x73, 0xc4, 0x0f, 0x87, 0x0c, 0xa1, 0xf4, 0x5a, + 0x3c, 0xce, 0x3f, 0x30, 0x3d, 0xb5, 0xc1, 0x0b, 0x43, 0x2a, 0xb1, 0xbe, 0x6b, 0x12, 0xc4, 0x4b, 0xdb, 0xcb, 0x2f, + 0x77, 0x8a, 0x3a, 0xba, 0xf4, 0xa3, 0x9b, 0x05, 0xee, 0x98, 0x50, 0x77, 0xfe, 0x82, 0x5b, 0x29, 0x6d, 0x7c, 0x0c, + 0x4b, 0x41, 0xa2, 0x6a, 0xb2, 0x27, 0x32, 0xf1, 0x0c, 0x1d, 0xd7, 0x6c, 0x02, 0x4d, 0xbd, 0xf2, 0x5a, 0x66, 0xd3, + 0xb5, 0x30, 0xab, 0x4b, 0x7e, 0x72, 0x77, 0x30, 0x42, 0xd7, 0x70, 0x05, 0x9f, 0xeb, 0x2f, 0x22, 0x7f, 0x49, 0x2d, + 0x5d, 0xcf, 0x4b, 0x84, 0x0c, 0xa5, 0xb9, 0xe5, 0x49, 0xae, 0x54, 0x2c, 0x86, 0x53, 0x4c, 0x9c, 0x42, 0x29, 0x78, + 0xf0, 0x0a, 0x8a, 0x36, 0x53, 0x4d, 0x2b, 0xc9, 0xe2, 0x69, 0x0e, 0x9a, 0x37, 0x0c, 0x91, 0x2d, 0xac, 0x2f, 0x43, + 0x6a, 0xf1, 0xe9, 0xfb, 0xb4, 0x68, 0xec, 0x90, 0x00, 0xff, 0x32, 0xf8, 0x58, 0xab, 0xd1, 0x73, 0x9f, 0x77, 0x72, + 0x22, 0xed, 0x64, 0xac, 0x7d, 0xd0, 0xa8, 0x1f, 0x01, 0x99, 0x9f, 0xfd, 0xa6, 0xcb, 0x62, 0xca, 0x65, 0xd8, 0xf8, + 0x31, 0xf0, 0xc1, 0x5e, 0xef, 0xf1, 0x4d, 0x79, 0xa9, 0xfc, 0x8f, 0x57, 0xcb, 0xc1, 0xb7, 0x03, 0x30, 0xbb, 0xad, + 0xdb, 0x88, 0xf7, 0xcc, 0xa1, 0x25, 0xfd, 0x37, 0x03, 0xc5, 0xdb, 0x7f, 0xb4, 0xe0, 0xe6, 0xa3, 0x7e, 0xa4, 0x52, + 0x68, 0xa7, 0x2e, 0xe6, 0x1c, 0xea, 0x91, 0x48, 0x67, 0xbd, 0x3d, 0x6d, 0xcc, 0x09, 0x49, 0x17, 0x50, 0x16, 0xb5, + 0x65, 0x63, 0xd4, 0x84, 0x2c, 0x3a, 0x5f, 0x6a, 0x3c, 0x66, 0xfa, 0x24, 0xaf, 0xd1, 0xfd, 0x18, 0x66, 0xe4, 0xe8, + 0x3c, 0x6c, 0x74, 0xb6, 0xc5, 0x4c, 0xf8, 0xa8, 0x15, 0x3c, 0xd0, 0xb8, 0x4a, 0x53, 0x15, 0x4b, 0x1e, 0x56, 0x7f, + 0xd8, 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xfe, 0xe7, 0xdc, 0x03, 0xd1, 0x0f, 0xfd, 0x7c, 0xdc, 0xa1, 0x7e, 0x41, 0x2e, 0x50, 0x37, 0x77, 0x94, 0xb7, 0x4b, + 0x90, 0x3b, 0x14, 0x38, 0xe0, 0x80, 0x03, 0x0e, 0x38, 0xe0, 0x80, 0xfb, 0xff, 0xba, 0xff, 0x00, +}; + +static const uint8_t STORED[] = { + 0x01, 0x2c, 0x01, 0xd3, 0xfe, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, + 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, + 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, + 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, + 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, + 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, + 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, + 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, + 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, + 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, + 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, + 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, + 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, + 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, + 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, + 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, + 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, +}; + +static constexpr size_t WINDOW = 4096; +static constexpr size_t PLAIN_SIZE = 16000; + +static uint8_t lcg_next(uint32_t &x) { + x = (x * 1103515245u + 12345u) & 0x7fffffffu; + return (x >> 16) & 0xff; +} + +static std::vector build_plain() { + std::vector plain; + const char *text = "esphome ota deflate "; + for (int i = 0; i < 300; i++) + plain.insert(plain.end(), text, text + strlen(text)); + uint32_t x = 1; + for (int i = 0; i < 3000; i++) + plain.push_back(lcg_next(x)); + plain.insert(plain.end(), 5000, 0); + for (int i = 0; i < 100; i++) + plain.insert(plain.end(), text, text + strlen(text)); + return plain; +} + +// Mirrors the OTA session: chunked input through the read callback, window as output +struct Session : OtaInflateState { + const uint8_t *in; + size_t in_len; + size_t in_pos; + size_t chunk; + std::vector out; + uint8_t window[WINDOW]; +}; + +static int read_cb(OtaInflateState *d) { + auto *s = static_cast(d); + if (s->in_pos >= s->in_len) + return -1; + size_t n = std::min(s->chunk, s->in_len - s->in_pos); + d->source = s->in + s->in_pos + 1; + d->source_limit = s->in + s->in_pos + n; + s->in_pos += n; + return s->in[s->in_pos - n]; +} + +// Inflates the whole input; returns the decoder result and fills s.out +static int inflate_all(Session &s, const uint8_t *in, size_t in_len, size_t chunk) { + s.in = in; + s.in_len = in_len; + s.in_pos = 0; + s.chunk = chunk; + s.out.clear(); + memset(s.window, 0, sizeof(s.window)); + ota_inflate_init(&s, s.window, WINDOW); + s.source_read_cb = read_cb; + int res; + do { + s.dest = s.window; + s.dest_limit = s.window + WINDOW; + res = ota_inflate(&s); + if (res < 0 || s.eof) + return res < 0 ? res : OTA_INFLATE_DATA_ERROR; + s.out.insert(s.out.end(), s.window, s.dest); + if (s.out.size() > PLAIN_SIZE) + return OTA_INFLATE_DATA_ERROR; + } while (res != OTA_INFLATE_DONE); + return res; +} + +TEST(OtaInflate, RoundTripThroughWindow) { + auto s = std::make_unique(); + ASSERT_EQ(inflate_all(*s, DEFLATED, sizeof(DEFLATED), 1040), OTA_INFLATE_DONE); + EXPECT_EQ(s->out, build_plain()); + EXPECT_EQ(s->in_pos, sizeof(DEFLATED)); +} + +TEST(OtaInflate, SmallReadChunks) { + auto s = std::make_unique(); + ASSERT_EQ(inflate_all(*s, DEFLATED, sizeof(DEFLATED), 7), OTA_INFLATE_DONE); + EXPECT_EQ(s->out, build_plain()); +} + +TEST(OtaInflate, StoredBlock) { + auto s = std::make_unique(); + ASSERT_EQ(inflate_all(*s, STORED, sizeof(STORED), 64), OTA_INFLATE_DONE); + auto plain = build_plain(); + plain.resize(300); + EXPECT_EQ(s->out, plain); +} + +TEST(OtaInflate, TruncatedStreamFails) { + auto s = std::make_unique(); + for (size_t cut : {size_t{1}, size_t{100}, size_t{1000}, sizeof(DEFLATED) - 1}) { + EXPECT_LT(inflate_all(*s, DEFLATED, cut, 1040), 0) << "cut at " << cut; + EXPECT_LE(s->out.size(), PLAIN_SIZE); + } +} + +TEST(OtaInflate, TruncatedStoredBlockFails) { + auto s = std::make_unique(); + EXPECT_LT(inflate_all(*s, STORED, sizeof(STORED) - 50, 64), 0); +} + +TEST(OtaInflate, CorruptStreamsNeverEscapeTheWindow) { + // Flipped bytes and garbage; the sanitizers check the decoder stays in bounds + auto s = std::make_unique(); + std::vector bad(DEFLATED, DEFLATED + sizeof(DEFLATED)); + // A coarse, non-aligned stride: neighbouring offsets hit the same paths + for (size_t i = 0; i < bad.size(); i += 29) { + bad[i] ^= 0x5a; + inflate_all(*s, bad.data(), bad.size(), 1040); + bad[i] ^= 0x5a; + } + uint32_t x = 99; + std::vector garbage(2000); + for (int round = 0; round < 50; round++) { + for (auto &b : garbage) + b = lcg_next(x); + inflate_all(*s, garbage.data(), garbage.size(), 1040); + } +} + +} // namespace esphome::testing diff --git a/tests/components/main.cpp b/tests/components/main.cpp index 373fde7151..aa0ceb5abe 100644 --- a/tests/components/main.cpp +++ b/tests/components/main.cpp @@ -28,6 +28,11 @@ void setup() { ::testing::InitGoogleTest(); int exit_code = RUN_ALL_TESTS(); + // A test folder that never reached the build would otherwise pass as an empty run + if (::testing::UnitTest::GetInstance()->total_test_count() == 0) { + fprintf(stderr, "No tests were linked into this binary\n"); + exit_code = 1; + } exit(exit_code); } diff --git a/tests/components/ota/test.bk72xx-ard.yaml b/tests/components/ota/test.bk72xx-ard.yaml new file mode 100644 index 0000000000..dade44d145 --- /dev/null +++ b/tests/components/ota/test.bk72xx-ard.yaml @@ -0,0 +1 @@ +<<: !include common.yaml diff --git a/tests/components/ota/test_backend_contract.cpp b/tests/components/ota/test_backend_contract.cpp index 1b4fbbc32d..36b7a793c3 100644 --- a/tests/components/ota/test_backend_contract.cpp +++ b/tests/components/ota/test_backend_contract.cpp @@ -14,7 +14,7 @@ struct MinimalBackend { OTAResponseTypes write(uint8_t *data, size_t len) { return OTA_RESPONSE_OK; } OTAResponseTypes end() { return OTA_RESPONSE_OK; } void abort() {} - bool supports_compression() { return false; } + static constexpr bool supports_compression() { return false; } }; static_assert(OTABackendContract); diff --git a/tests/components/socket/__init__.py b/tests/components/socket/__init__.py index 8a3da97e23..6c1d2b1c08 100644 --- a/tests/components/socket/__init__.py +++ b/tests/components/socket/__init__.py @@ -2,6 +2,7 @@ # shadowing the stdlib module -- no test_*.py (Python-level) tests can live here. # required for cpp unit testing. import esphome.codegen as cg +from esphome.components import socket as socket_component from tests.testing_helpers import ComponentManifestOverride @@ -12,5 +13,7 @@ def override_manifest(manifest: ComponentManifestOverride) -> None: async def to_code_testing(config): await real_to_code(config) cg.add_define("USE_NETWORK_IPV6", True) + # The gtests link against the filtered helper files. + socket_component.require_tcp_client_link() manifest.to_code = to_code_testing diff --git a/tests/components/tcp_uart/common.yaml b/tests/components/tcp_uart/common.yaml new file mode 100644 index 0000000000..f3c1d061aa --- /dev/null +++ b/tests/components/tcp_uart/common.yaml @@ -0,0 +1,20 @@ +wifi: + ssid: MySSID + password: password1 + +tcp_uart: + - id: tcp_uart_1 + host: 192.0.2.10 + port: 502 + reconnect_interval: 10s + connected: + name: TCP UART Connected + +interval: + - interval: 60s + then: + - lambda: |- + uint8_t byte; + if (id(tcp_uart_1).available() && id(tcp_uart_1).read_byte(&byte)) { + id(tcp_uart_1).write_byte(byte); + } diff --git a/tests/components/tcp_uart/test.bk72xx-ard.yaml b/tests/components/tcp_uart/test.bk72xx-ard.yaml new file mode 100644 index 0000000000..2d42c282f7 --- /dev/null +++ b/tests/components/tcp_uart/test.bk72xx-ard.yaml @@ -0,0 +1,2 @@ +packages: + tcp_uart: !include common.yaml diff --git a/tests/components/tcp_uart/test.esp32-idf.yaml b/tests/components/tcp_uart/test.esp32-idf.yaml new file mode 100644 index 0000000000..2d42c282f7 --- /dev/null +++ b/tests/components/tcp_uart/test.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + tcp_uart: !include common.yaml diff --git a/tests/components/tcp_uart/test.esp8266-ard.yaml b/tests/components/tcp_uart/test.esp8266-ard.yaml new file mode 100644 index 0000000000..2d42c282f7 --- /dev/null +++ b/tests/components/tcp_uart/test.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + tcp_uart: !include common.yaml diff --git a/tests/components/tcp_uart/test.host.yaml b/tests/components/tcp_uart/test.host.yaml new file mode 100644 index 0000000000..22d7f7fbaa --- /dev/null +++ b/tests/components/tcp_uart/test.host.yaml @@ -0,0 +1,4 @@ +packages: + tcp_uart: !include common.yaml + +wifi: !remove diff --git a/tests/components/tcp_uart/test.ln882x-ard.yaml b/tests/components/tcp_uart/test.ln882x-ard.yaml new file mode 100644 index 0000000000..2d42c282f7 --- /dev/null +++ b/tests/components/tcp_uart/test.ln882x-ard.yaml @@ -0,0 +1,2 @@ +packages: + tcp_uart: !include common.yaml diff --git a/tests/components/tcp_uart/test.rp2040-ard.yaml b/tests/components/tcp_uart/test.rp2040-ard.yaml new file mode 100644 index 0000000000..2d42c282f7 --- /dev/null +++ b/tests/components/tcp_uart/test.rp2040-ard.yaml @@ -0,0 +1,2 @@ +packages: + tcp_uart: !include common.yaml diff --git a/tests/components/tcp_uart/test.rtl87xx-ard.yaml b/tests/components/tcp_uart/test.rtl87xx-ard.yaml new file mode 100644 index 0000000000..2d42c282f7 --- /dev/null +++ b/tests/components/tcp_uart/test.rtl87xx-ard.yaml @@ -0,0 +1,2 @@ +packages: + tcp_uart: !include common.yaml diff --git a/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py index 3e397d6186..aee9e3ec13 100644 --- a/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py +++ b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py @@ -1,4 +1,5 @@ import esphome.codegen as cg +from esphome.components import socket import esphome.config_validation as cv from esphome.const import CONF_ID from esphome.types import ConfigType @@ -18,5 +19,6 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config: ConfigType) -> None: + socket.require_ipv4_resolve() var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) diff --git a/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py b/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py new file mode 100644 index 0000000000..24637ae586 --- /dev/null +++ b/tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py @@ -0,0 +1,37 @@ +import esphome.codegen as cg +from esphome.components import socket +from esphome.components.const import CONF_HOST +import esphome.config_validation as cv +from esphome.const import CONF_ID, CONF_PORT +from esphome.types import ConfigType + +AUTO_LOAD = ["socket"] + +CONF_RECONNECT_INTERVAL = "reconnect_interval" + +tcp_client_link_test_component_ns = cg.esphome_ns.namespace( + "tcp_client_link_test_component" +) +TcpClientLinkTestComponent = tcp_client_link_test_component_ns.class_( + "TcpClientLinkTestComponent", cg.Component +) + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(): cv.declare_id(TcpClientLinkTestComponent), + cv.Required(CONF_HOST): cv.string, + cv.Required(CONF_PORT): cv.port, + cv.Optional( + CONF_RECONNECT_INTERVAL, default="1s" + ): cv.positive_time_period_milliseconds, + } +).extend(cv.COMPONENT_SCHEMA) + + +async def to_code(config: ConfigType) -> None: + socket.require_tcp_client_link() + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + cg.add(var.set_host(config[CONF_HOST])) + cg.add(var.set_port(config[CONF_PORT])) + cg.add(var.set_reconnect_interval(config[CONF_RECONNECT_INTERVAL])) diff --git a/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp b/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp new file mode 100644 index 0000000000..7f2af8add8 --- /dev/null +++ b/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp @@ -0,0 +1,28 @@ +#include "tcp_client_link_test_component.h" +#include "esphome/core/log.h" + +namespace esphome::tcp_client_link_test_component { + +static const char *const TAG = "tcp_link_test"; + +void TcpClientLinkTestComponent::setup() { this->link_.begin(TAG); } + +void TcpClientLinkTestComponent::loop() { + this->link_.poll(); + bool up = this->link_.connected(); + if (up != this->was_up_) { + this->was_up_ = up; + ESP_LOGI(TAG, "Link %s", up ? LOG_STR_LITERAL("up") : LOG_STR_LITERAL("down")); + } + if (!up || !this->link_.ready()) { + return; + } + uint8_t buf[64]; + ssize_t count = this->link_.read(buf, sizeof(buf)); + if (count > 0) { + ESP_LOGI(TAG, "Echoing %d bytes", static_cast(count)); + this->link_.write(buf, static_cast(count)); + } +} + +} // namespace esphome::tcp_client_link_test_component diff --git a/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.h b/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.h new file mode 100644 index 0000000000..829f15b8a3 --- /dev/null +++ b/tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.h @@ -0,0 +1,24 @@ +#pragma once + +#include "esphome/components/socket/tcp_client_link.h" +#include "esphome/core/component.h" + +namespace esphome::tcp_client_link_test_component { + +/// Echoes every byte the link receives back to the peer and logs link edges. +class TcpClientLinkTestComponent : public Component { + public: + void set_host(const char *host) { this->link_.set_host(host); } + void set_port(uint16_t port) { this->link_.set_port(port); } + void set_reconnect_interval(uint32_t ms) { this->link_.set_reconnect_interval(ms); } + + void setup() override; + void loop() override; + void on_shutdown() override { this->link_.close(); } + + protected: + socket::TcpClientLink link_; + bool was_up_{false}; +}; + +} // namespace esphome::tcp_client_link_test_component diff --git a/tests/integration/fixtures/host_ota_deflate.yaml b/tests/integration/fixtures/host_ota_deflate.yaml new file mode 100644 index 0000000000..ebf7977123 --- /dev/null +++ b/tests/integration/fixtures/host_ota_deflate.yaml @@ -0,0 +1,9 @@ +esphome: + name: host-ota-test +host: +api: +ota: + - platform: esphome + port: __OTA_PORT__ +logger: + level: DEBUG diff --git a/tests/integration/fixtures/socket_tcp_client_link.yaml b/tests/integration/fixtures/socket_tcp_client_link.yaml new file mode 100644 index 0000000000..2ed7f775ff --- /dev/null +++ b/tests/integration/fixtures/socket_tcp_client_link.yaml @@ -0,0 +1,20 @@ +esphome: + name: socket-tcp-client-link-test + +host: + +api: + +logger: + level: INFO + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + components: [tcp_client_link_test_component] + +tcp_client_link_test_component: + host: 127.0.0.1 + port: 18123 + reconnect_interval: 1s diff --git a/tests/integration/test_host_ota.py b/tests/integration/test_host_ota.py index 56a685eac3..17c4dea25f 100644 --- a/tests/integration/test_host_ota.py +++ b/tests/integration/test_host_ota.py @@ -15,6 +15,7 @@ from dataclasses import dataclass import functools from pathlib import Path import socket +import zlib import pytest @@ -123,6 +124,7 @@ class _Device: binary_path: Path proc: asyncio.subprocess.Process | None = None reboots: int = 0 + inflates: int = 0 def __post_init__(self) -> None: self._rebooted = asyncio.Event() @@ -131,6 +133,8 @@ class _Device: if "Rebooting safely" in line: self.reboots += 1 self._rebooted.set() + if "Inflated " in line and " bytes from " in line: + self.inflates += 1 async def wait_reboot(self, count: int, timeout: float = 10.0) -> None: async with asyncio.timeout(timeout): @@ -241,6 +245,80 @@ async def test_host_ota_self_update( await dev.ota(None, None, "second OTA failed -- listener leaked across execv") +@pytest.mark.asyncio +async def test_host_ota_deflate( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Deflate is negotiated by default, an old client gets an uncompressed + upload, and a corrupt stream is rejected without taking the device down.""" + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + errors: list[str] = [] + + def on_log(line: str) -> None: + # A corrupt stream is caught by the decoder, by the size check or by + # the MD5 at the end, depending on where the damage lands + if any( + text in line + for text in ("Inflate err", "Inflate overrun", "End update err") + ): + errors.append(line) + dev.on_log(line) + + real_compress = zlib.compress + + def corrupt_compress(data: bytes, *args: object, **kwargs: object) -> bytes: + # Reserved block type in the first header: rejected by the decoder on + # every build, unlike a flipped data bit that may only fail the MD5 + out = bytearray(real_compress(data, *args, **kwargs)) + out[0] |= 0x06 + return bytes(out) + + def overlong_compress(data: bytes, *args: object, **kwargs: object) -> bytes: + """A stream that inflates past the size the client announced.""" + return real_compress(data + bytes(8192), *args, **kwargs) + + async with run_binary(dev.binary_path, line_callback=on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + + # Default: the host backend cannot store gzip, so the CLI sends deflate + await dev.ota(None, None, "deflate upload failed") + assert dev.inflates == 1, "device did not inflate the upload" + + # A client that does not offer deflate is served uncompressed + with monkeypatch.context() as m: + m.setattr(espota2, "CLIENT_FEATURE_SUPPORTS_DEFLATE", 0) + await dev.ota(None, None, "uncompressed upload failed") + assert dev.inflates == 1, "device inflated without a client offer" + + # A corrupt stream fails the upload and leaves the device running + with monkeypatch.context() as m: + m.setattr(zlib, "compress", corrupt_compress) + await dev.refused_ota(None, None, "corrupt deflate stream was accepted") + assert errors, "device did not report the corrupt stream" + + # So does a stream that inflates past the announced image size + errors.clear() + with monkeypatch.context() as m: + m.setattr(zlib, "compress", overlong_compress) + await dev.refused_ota(None, None, "overlong deflate stream was accepted") + assert any("Inflate overrun" in line for line in errors), ( + "device wrote past the announced size" + ) + + # and it still takes a good upload afterwards + await dev.ota(None, None, "upload after a rejected stream failed") + assert dev.inflates == 2 + + @pytest.mark.asyncio async def test_host_ota_encrypted( yaml_config: str, diff --git a/tests/integration/test_socket_tcp_client_link.py b/tests/integration/test_socket_tcp_client_link.py new file mode 100644 index 0000000000..31cc344f67 --- /dev/null +++ b/tests/integration/test_socket_tcp_client_link.py @@ -0,0 +1,88 @@ +"""Integration test for socket::TcpClientLink on host. + +Pytest runs a real TCP server; the device echoes through the link. +Covers connect, read, write, a server-initiated drop and the reconnect. +""" + +from __future__ import annotations + +import asyncio +import contextlib + +import pytest + +from .types import APIClientConnectedFactory, RunCompiledFunction + +PAYLOAD = b"hello link" + + +@pytest.mark.asyncio +async def test_socket_tcp_client_link( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, + unused_tcp_port_factory, +) -> None: + server_port = unused_tcp_port_factory() + yaml_config = yaml_config.replace("port: 18123", f"port: {server_port}") + + echoed: list[bytes] = [] + echo_done = asyncio.Event() + reconnected = asyncio.Event() + link_down = asyncio.Event() + second_link_up = asyncio.Event() + link_up_count = 0 + + def on_log_line(line: str) -> None: + nonlocal link_up_count + if "Link up" in line: + link_up_count += 1 + if link_up_count >= 2: + second_link_up.set() + elif "Link down" in line: + link_down.set() + + async def handle( + reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + if not echo_done.is_set(): + writer.write(PAYLOAD) + await writer.drain() + with contextlib.suppress(TimeoutError, asyncio.IncompleteReadError): + echoed.append( + await asyncio.wait_for(reader.readexactly(len(PAYLOAD)), 10) + ) + echo_done.set() + # Drop the connection so the link has to reconnect. + writer.close() + return + reconnected.set() + + server = await asyncio.start_server(handle, "127.0.0.1", server_port) + try: + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + device_info = await client.device_info() + assert device_info is not None + assert device_info.name == "socket-tcp-client-link-test" + + try: + await asyncio.wait_for(echo_done.wait(), timeout=15.0) + except TimeoutError: + pytest.fail("Link never connected or echoed") + assert echoed and echoed[0] == PAYLOAD, "Echo payload mismatch" + + try: + await asyncio.wait_for(link_down.wait(), timeout=15.0) + except TimeoutError: + pytest.fail("Link never reported the drop") + try: + await asyncio.wait_for(reconnected.wait(), timeout=15.0) + await asyncio.wait_for(second_link_up.wait(), timeout=15.0) + except TimeoutError: + pytest.fail("Link did not reconnect after the server dropped it") + finally: + server.close() + await server.wait_closed() diff --git a/tests/script/test_check_idf_py_equivalence.py b/tests/script/test_check_idf_py_equivalence.py index 42dbdf4dc7..f2b798ff21 100644 --- a/tests/script/test_check_idf_py_equivalence.py +++ b/tests/script/test_check_idf_py_equivalence.py @@ -16,7 +16,7 @@ import check_idf_py_equivalence as guard # noqa: E402 from esphome.build_gen import espidf as build_gen # noqa: E402 from esphome.core import CORE # noqa: E402 -from esphome.espidf import toolchain # noqa: E402 +from esphome.espidf import framework, toolchain # noqa: E402 @pytest.fixture(autouse=True) @@ -74,10 +74,12 @@ def _run_check( esphome_rcs: tuple[int, int] = (0, 0), macro_matches: bool = True, envs: list[dict[str, str]] | None = None, + versions: tuple[str | None, str | None, str] = ("5.5", "5.5", "5.5"), ) -> tuple[list[str], list[list[str]]]: """Run check() with idf.py replaced by ``side_effect``; return problems, calls. - ``envs`` collects the env each idf.py call receives. + ``envs`` collects the env each idf.py call receives. ``versions`` is what + version.txt, the version header and idf_tools report for the framework. """ calls: list[list[str]] = [] @@ -98,6 +100,10 @@ def _run_check( patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]), patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]), patch.object(build_gen, "idf_macro_matches", return_value=macro_matches), + patch.object(framework, "read_idf_version_txt", return_value=versions[0]), + patch.object(framework, "read_idf_version_header", return_value=versions[1]), + patch.object(framework, "idf_tools_version", return_value=versions[2]), + patch.object(guard, "_lwip_empty_source_problems", return_value=[]), patch.object(guard.subprocess, "run", side_effect=run), patch.dict(os.environ), ): @@ -285,6 +291,24 @@ def test_check_fails_loudly_when_the_idf_macro_changed(tmp_path: Path) -> None: assert calls == [] +@pytest.mark.parametrize( + ("versions", "source"), + [(("5.4", "5.5", "5.5"), "txt"), (("5.5", None, "5.5"), "header")], +) +def test_check_fails_loudly_when_the_version_read_drifts( + tmp_path: Path, versions: tuple[str | None, str | None, str], source: str +) -> None: + """An IDF bump that changes how idf_tools reads its version must fail CI; + both sources are checked since a managed tree never reaches the header.""" + tree = _make_tree(tmp_path) + problems, calls = _run_check(tree, versions=versions) + ours = versions[0] if source == "txt" else versions[1] + assert problems == [ + guard.VERSION_DRIFT.format(ours=ours, source=source, theirs="5.5") + ] + assert calls == [] + + def test_main_without_build_trees( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: @@ -296,6 +320,99 @@ def test_main_without_build_trees( assert "No native ESP-IDF build tree found" in capsys.readouterr().out +def _make_lwip_tree(tmp_path: Path, objects: list[str], config: dict) -> Path: + """A tree with lwip objects, their sdkconfig.json and an nm in the cache.""" + tree = _make_tree(tmp_path) + objdir = tree / "build" / "esp-idf" / "lwip" / "CMakeFiles" / "__idf_lwip.dir" + for name in objects: + (objdir / name).parent.mkdir(parents=True, exist_ok=True) + (objdir / name).write_bytes(b"x") + (tree / "build" / "config").mkdir(parents=True, exist_ok=True) + (tree / "build" / "config" / "sdkconfig.json").write_text(json.dumps(config)) + with (tree / "build" / "CMakeCache.txt").open("a") as cache: + cache.write("CMAKE_NM:FILEPATH=/tools/nm\n") + return tree + + +def _run_lwip_check( + tree: Path, + non_empty: set[str] = frozenset(), + failing: set[str] = frozenset(), + calls: list[list[str]] | None = None, +) -> list[str]: + """Run the lwip check with nm faked; ``calls`` collects the nm commands.""" + + def run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess: + if calls is not None: + calls.append(cmd) + name = Path(cmd[-1]).name + if name in failing: + return subprocess.CompletedProcess(cmd, 1, "", "bad object") + return subprocess.CompletedProcess( + cmd, 0, "symbol\n" if name in non_empty else "", "" + ) + + with ( + patch.object(toolchain, "run_reconfigure", return_value=0) as reconfigure, + patch.object(toolchain, "_run_ninja", return_value=0), + patch.object(guard.subprocess, "run", side_effect=run), + ): + problems = guard._lwip_empty_source_problems(tree) + reconfigure.assert_called_once_with( + extra_env={build_gen.LWIP_FULL_SOURCES_ENV: "1"} + ) + return problems + + +def test_lwip_check_inspects_only_the_dropped_sources(tmp_path: Path) -> None: + """An option that is on, or absent (invisible), keeps its sources unchecked.""" + tree = _make_lwip_tree( + tmp_path, + [ + "lwip/src/netif/ppp/auth.c.obj", + "lwip/src/core/ipv6/ip6.c.obj", + "lwip/src/core/ipv4/autoip.c.obj", + ], + {"LWIP_PPP_SUPPORT": False, "LWIP_IPV6": True}, + ) + calls: list[list[str]] = [] + assert _run_lwip_check(tree, calls=calls) == [] + assert [Path(c[-1]).name for c in calls] == ["auth.c.obj"] + + +def test_lwip_check_flags_a_dropped_source_with_symbols(tmp_path: Path) -> None: + tree = _make_lwip_tree( + tmp_path, ["lwip/src/netif/ppp/auth.c.obj"], {"LWIP_PPP_SUPPORT": False} + ) + assert _run_lwip_check(tree, non_empty={"auth.c.obj"}) == [ + guard.LWIP_NOT_EMPTY.format(source="auth.c", option="CONFIG_LWIP_PPP_SUPPORT") + ] + + +def test_lwip_check_flags_a_failed_nm(tmp_path: Path) -> None: + """A broken nm must not pass as an empty object.""" + tree = _make_lwip_tree( + tmp_path, ["lwip/src/netif/ppp/auth.c.obj"], {"LWIP_PPP_SUPPORT": False} + ) + assert _run_lwip_check(tree, failing={"auth.c.obj"}) == [ + guard.LWIP_NM_FAILED.format(source="auth.c", error="bad object") + ] + + +def test_lwip_check_fails_per_pattern_that_matched_nothing(tmp_path: Path) -> None: + """A stale pattern is reported even while the others still match.""" + tree = _make_lwip_tree( + tmp_path, + ["lwip/src/netif/ppp/auth.c.obj"], + {"LWIP_PPP_SUPPORT": False, "LWIP_STATS": False}, + ) + assert _run_lwip_check(tree) == [ + guard.LWIP_NOTHING_MATCHED.format( + regex="/core/stats[.]c$", option="CONFIG_LWIP_STATS" + ) + ] + + @pytest.mark.parametrize(("problems", "rc"), [([], 0), (["idf.py changed x"], 1)]) def test_main_reports_each_tree( tmp_path: Path, diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index 7ed6201b6a..2c95c61b37 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -222,6 +222,30 @@ def test_get_project_cmakelists_size_command_uses_json2() -> None: assert "--format=json2" in content +def test_get_project_cmakelists_drops_empty_lwip_sources() -> None: + """The filter comes after project(), where the lwip target exists.""" + from esphome.build_gen.espidf import ( + LWIP_EMPTY_SOURCES, + LWIP_FULL_SOURCES_ENV, + lwip_empty_source_gate, + ) + + content = _render() + filter_at = content.index( + "set_property(TARGET ${esphome_lwip_lib} PROPERTY SOURCES" + ) + assert filter_at > content.index("project(") + assert f"NOT DEFINED ENV{{{LWIP_FULL_SOURCES_ENV}}}" in content + for entry in LWIP_EMPTY_SOURCES: + assert lwip_empty_source_gate(*entry) in content + + +def test_get_project_cmakelists_declares_map_as_link_byproduct() -> None: + """The link declares the map so size can build in the same ninja run.""" + content = _render() + assert "BYPRODUCTS ${CMAKE_BINARY_DIR}/${CMAKE_PROJECT_NAME}.map" in content + + def test_get_project_cmakelists_uses_supplied_builtin_components() -> None: """A cached list replaces project_description.json and is still filtered by EXCLUDE_COMPONENTS.""" diff --git a/tests/unit_tests/components/socket/test_socket_source_filter.py b/tests/unit_tests/components/socket/test_socket_source_filter.py new file mode 100644 index 0000000000..3967c9568d --- /dev/null +++ b/tests/unit_tests/components/socket/test_socket_source_filter.py @@ -0,0 +1,38 @@ +"""Tests for the socket component's source filtering and require functions.""" + +from unittest.mock import patch + +from esphome.components import socket +from esphome.core import Define + + +def test_helper_files_filtered_until_required() -> None: + """ipv4_resolve.cpp and tcp_client_link.cpp compile only when required.""" + with patch("esphome.config_helpers.CORE") as mock_core: + mock_core.defines = set() + filtered = socket.FILTER_SOURCE_FILES() + assert "ipv4_resolve.cpp" in filtered + assert "tcp_client_link.cpp" in filtered + + mock_core.defines = {Define("USE_SOCKET_IPV4_RESOLVE")} + filtered = socket.FILTER_SOURCE_FILES() + assert "ipv4_resolve.cpp" not in filtered + assert "tcp_client_link.cpp" in filtered + + mock_core.defines = { + Define("USE_SOCKET_IPV4_RESOLVE"), + Define("USE_SOCKET_TCP_CLIENT_LINK"), + } + filtered = socket.FILTER_SOURCE_FILES() + assert "ipv4_resolve.cpp" not in filtered + assert "tcp_client_link.cpp" not in filtered + + +def test_require_tcp_client_link_pulls_in_the_resolver() -> None: + """require_tcp_client_link() sets both defines; the link reads the resolver.""" + with patch.object(socket.cg, "add_define") as add_define: + socket.require_tcp_client_link() + assert {call.args[0] for call in add_define.call_args_list} == { + "USE_SOCKET_IPV4_RESOLVE", + "USE_SOCKET_TCP_CLIENT_LINK", + } diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index cfb9651e6b..7fbc2d1b39 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -1493,3 +1493,20 @@ async def test_add_platformio_options_native_arduino( assert "board_build.ldscript is ignored" in caplog.text assert "'arduino' toolchain" in caplog.text assert "upload_speed" not in caplog.text + + +def test_filter_source_files_drops_util_cpp_without_mqtt() -> None: + """util.cpp compiles only on MQTT builds; the header stubs it otherwise.""" + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: "esp8266", + KEY_TARGET_FRAMEWORK: "arduino", + } + CORE.defines = set() + + excluded = config.FILTER_SOURCE_FILES() + assert "util.cpp" in excluded + # The platform map still contributes through the composed function. + assert "static_task.cpp" in excluded + + CORE.defines = {core.Define("USE_API"), core.Define("USE_MQTT")} + assert "util.cpp" not in config.FILTER_SOURCE_FILES() diff --git a/tests/unit_tests/fixtures/substitutions/01-include.approved.yaml b/tests/unit_tests/fixtures/substitutions/01-include.approved.yaml index a812fedcfd..e13db65ef7 100644 --- a/tests/unit_tests/fixtures/substitutions/01-include.approved.yaml +++ b/tests/unit_tests/fixtures/substitutions/01-include.approved.yaml @@ -13,3 +13,10 @@ test_list: - a: alpha - b: beta - c: The value of C is $c + - {} + - values: + - var1: '1' + - a: alpha + - b: delta + - c: The value of C is saw + - {} diff --git a/tests/unit_tests/fixtures/substitutions/01-include.input.yaml b/tests/unit_tests/fixtures/substitutions/01-include.input.yaml index d3daa681a4..b30ae5f99e 100644 --- a/tests/unit_tests/fixtures/substitutions/01-include.input.yaml +++ b/tests/unit_tests/fixtures/substitutions/01-include.input.yaml @@ -13,3 +13,22 @@ test_list: file: inc1.yaml vars: b: "beta" + condition: true + - !include + file: inc1.yaml + vars: + b: "gamma" + c: "saw" + condition: false + - !include + file: inc1.yaml + vars: + b: "delta" + c: "saw" + condition: ${var1 == "1"} + - !include + file: inc1.yaml + vars: + b: "epsilon" + c: "saw" + condition: ${var1 == "2"} diff --git a/tests/unit_tests/fixtures/substitutions/12-yaml-merge.input.yaml b/tests/unit_tests/fixtures/substitutions/12-yaml-merge.input.yaml index a03e66e393..3152f8ddf0 100644 --- a/tests/unit_tests/fixtures/substitutions/12-yaml-merge.input.yaml +++ b/tests/unit_tests/fixtures/substitutions/12-yaml-merge.input.yaml @@ -7,4 +7,10 @@ test_list: file: inc3.yaml vars: y: 8 + condition: true + <<: !include + file: inc3.yaml + vars: + y: 10 + condition: false after: Content after diff --git a/tests/unit_tests/fixtures/yaml_util/includes/true.yaml b/tests/unit_tests/fixtures/yaml_util/includes/true.yaml new file mode 100644 index 0000000000..abd17b22f7 --- /dev/null +++ b/tests/unit_tests/fixtures/yaml_util/includes/true.yaml @@ -0,0 +1,2 @@ +--- +true diff --git a/tests/unit_tests/fixtures/yaml_util/includetest.yaml b/tests/unit_tests/fixtures/yaml_util/includetest.yaml index 7b01da6a17..6db5c294f4 100644 --- a/tests/unit_tests/fixtures/yaml_util/includetest.yaml +++ b/tests/unit_tests/fixtures/yaml_util/includetest.yaml @@ -11,8 +11,8 @@ esphome: # should be substituted as 'original', # not overwritten by vars in the !include above name: ${name} - name_add_mac_suffix: true - libraries: !include {file: includes/list.yaml, vars: {var1: Wire}} + name_add_mac_suffix: !include includes/true.yaml + libraries: !include {file: includes/list.yaml, vars: {var1: Wire}, condition: "true"} esp8266: - board: !include {file: includes/scalar.yaml, vars: {var1: nodemcu}} + board: !include {file: includes/scalar.yaml, vars: {var1: nodemcu}, condition: true} diff --git a/tests/unit_tests/fixtures/yaml_util/includetest_invalid_condition_type.yaml b/tests/unit_tests/fixtures/yaml_util/includetest_invalid_condition_type.yaml new file mode 100644 index 0000000000..94eb7d65ab --- /dev/null +++ b/tests/unit_tests/fixtures/yaml_util/includetest_invalid_condition_type.yaml @@ -0,0 +1,3 @@ +--- +esphome: + libraries: !include {file: includes/list.yaml, vars: {var1: Wire}, condition: 1} diff --git a/tests/unit_tests/fixtures/yaml_util/includetest_no_file.yaml b/tests/unit_tests/fixtures/yaml_util/includetest_no_file.yaml new file mode 100644 index 0000000000..7724bb301f --- /dev/null +++ b/tests/unit_tests/fixtures/yaml_util/includetest_no_file.yaml @@ -0,0 +1,3 @@ +--- +esphome: + libraries: !include {vars: {var1: Wire}} diff --git a/tests/unit_tests/test_bundle.py b/tests/unit_tests/test_bundle.py index 1abc7a3ab8..8a2510cf0a 100644 --- a/tests/unit_tests/test_bundle.py +++ b/tests/unit_tests/test_bundle.py @@ -1384,7 +1384,7 @@ def test_force_load_skips_duplicate_include_file() -> None: self.parent_file = Path("root.yaml") self.load_calls = 0 - def has_unresolved_expressions(self) -> bool: + def has_unresolved_file(self) -> bool: return False def load(self) -> dict[str, Any]: diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index c785d56fa4..c3f29459c6 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -44,6 +44,7 @@ from esphome.espidf.framework import ( check_esp_idf_install, get_framework_env, get_idf_tools_path, + idf_tools_version, ) from esphome.framework_helpers import _tar_extract_all, get_python_env_executable_path @@ -1468,19 +1469,59 @@ def test_demote_unused_tools_already_patched_is_noop(tmp_path: Path) -> None: # --------------------------------------------------------------------------- -def test_get_idf_version_parses_stdout(tmp_path: Path) -> None: +def _write_idf_version_header(root: Path, major: int, minor: int) -> None: + include = root / "components" / "esp_common" / "include" + include.mkdir(parents=True) + (include / "esp_idf_version.h").write_text( + f"#define ESP_IDF_VERSION_MAJOR {major}\n" + "/** Minor version number (x.X.x) */\n" + f"#define ESP_IDF_VERSION_MINOR {minor}\n" + "#define ESP_IDF_VERSION_PATCH 0\n", + encoding="utf-8", + ) + + +def test_get_idf_version_reads_version_txt(tmp_path: Path) -> None: + """version.txt wins and gives major.minor, as idf_tools returns it.""" + (tmp_path / "version.txt").write_text("v5.5.5\n", encoding="utf-8") + _write_idf_version_header(tmp_path, 6, 1) + assert _get_idf_version(tmp_path) == "5.5" + + +def test_get_idf_version_falls_back_to_the_header(tmp_path: Path) -> None: + """A version.txt that does not match (a git ref) defers to the header.""" + (tmp_path / "version.txt").write_text("vrelease/v6.1\n", encoding="utf-8") + _write_idf_version_header(tmp_path, 6, 1) + assert _get_idf_version(tmp_path) == "6.1" + + +def test_get_idf_version_raises_without_a_source(tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="Can't get ESP-IDF version"): + _get_idf_version(tmp_path) + + +def test_get_idf_version_wraps_an_unreadable_source(tmp_path: Path) -> None: + """A source that cannot be decoded keeps the RuntimeError contract.""" + (tmp_path / "version.txt").write_bytes(b"\xff\xfev") + with pytest.raises(RuntimeError, match="Can't get ESP-IDF version") as info: + _get_idf_version(tmp_path) + assert isinstance(info.value.__cause__, UnicodeError) + + +def test_idf_tools_version_runs_the_framework_script(tmp_path: Path) -> None: with patch( - "esphome.espidf.framework.run_command", return_value=(True, "5.1.2\n", "") - ): - assert _get_idf_version(tmp_path) == "5.1.2" + "esphome.espidf.framework.run_command", return_value=(True, "5.5\n", "") + ) as run: + assert idf_tools_version(tmp_path) == "5.5" + assert run.call_args.args[0][1].endswith("get_idf_version.py") -def test_get_idf_version_raises_on_failure(tmp_path: Path) -> None: +def test_idf_tools_version_raises_on_failure(tmp_path: Path) -> None: with ( patch("esphome.espidf.framework.run_command", return_value=(False, "", "boom")), pytest.raises(RuntimeError, match="Can't get ESP-IDF version"), ): - _get_idf_version(tmp_path) + idf_tools_version(tmp_path) def test_get_idf_tool_paths_parses_json(tmp_path: Path) -> None: @@ -1511,6 +1552,30 @@ def test_get_idf_tool_paths_raises_on_failure(tmp_path: Path) -> None: _get_idf_tool_paths(tmp_path) +def test_get_idf_tool_paths_runs_the_script_once_per_build(tmp_path: Path) -> None: + payload = json.dumps({"paths_to_export": ["/a"], "export_vars": {"X": "1"}}) + env = {"IDF_TOOLS_PATH": str(tmp_path / "tools")} + with patch( + "esphome.espidf.framework.run_command", return_value=(True, payload, "") + ) as run: + first = _get_idf_tool_paths(tmp_path, env) + second = _get_idf_tool_paths(tmp_path, env) + assert run.call_count == 1 + assert first == second == (["/a"], {"X": "1"}) + + +def test_get_idf_tool_paths_does_not_cache_a_failure(tmp_path: Path) -> None: + payload = json.dumps({"paths_to_export": ["/a"], "export_vars": {}}) + with patch( + "esphome.espidf.framework.run_command", + side_effect=[(False, "", "err"), (True, payload, "")], + ) as run: + with pytest.raises(RuntimeError, match="Can't get ESP-IDF tool paths"): + _get_idf_tool_paths(tmp_path) + assert _get_idf_tool_paths(tmp_path) == (["/a"], {}) + assert run.call_count == 2 + + def test_get_python_version_parses_stdout(tmp_path: Path) -> None: with patch( "esphome.espidf.framework.run_command", return_value=(True, "3.11.0\n", "") diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 1aaff7e9ce..620d08fe3b 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -534,8 +534,8 @@ def _record_compile_calls( def record_save(components: list[str]) -> None: calls.append(("save", components)) - def record_ninja(target: str, **kwargs: object) -> int: - if target == "all": + def record_ninja(*targets: str, **kwargs: object) -> int: + if "all" in targets: calls.append(("build",)) return 0 @@ -886,7 +886,7 @@ def test_run_compile_full_deps_skips_fragment_check( def test_run_compile_passes_compile_process_limit( setup_core: Path, limit: int | None ) -> None: - """compile_process_limit is the job limit for both ninja runs.""" + """compile_process_limit is the job limit of the one ninja run.""" _setup_build(setup_core) esphome = {} if limit is None else {CONF_COMPILE_PROCESS_LIMIT: limit} @@ -894,8 +894,14 @@ def test_run_compile_passes_compile_process_limit( assert toolchain.run_compile({CONF_ESPHOME: esphome}, verbose=False) == 0 assert mock_run.call_args_list == [ - call("all", verbose=False, jobs=limit, progress=True), - call("size", verbose=False, jobs=limit, extra_env=toolchain._size_env()), + call( + "all", + "size", + verbose=False, + jobs=limit, + progress=True, + extra_env=toolchain._size_env(), + ), ] @@ -1185,14 +1191,17 @@ def test_run_ninja_filters_and_reports_failure( patch.object(toolchain, "_print_hints") as mock_hints, ): mock_run.return_value = 1 - assert toolchain._run_ninja("all", verbose=False, jobs=None, progress=True) == 1 + assert ( + toolchain._run_ninja("all", "size", verbose=False, jobs=None, progress=True) + == 1 + ) log_path = mock_run.call_args.kwargs["log_path"] - assert log_path.name == "ninja_all_output.log" + assert log_path.name == "ninja_all_size_output.log" mock_hints.assert_called_once_with(log_path) - assert mock_run.call_args.args[0] == ["/tools/ninja", "all"] + assert mock_run.call_args.args[0] == ["/tools/ninja", "all", "size"] assert mock_run.call_args.kwargs["filter_lines"] is toolchain.FILTER_IDF_LINES assert mock_run.call_args.kwargs["progress"] is True - assert "ninja all failed with exit code 1" in caplog.text + assert "ninja all size failed with exit code 1" in caplog.text @pytest.mark.parametrize("reconfigure_rc", [0, 5]) @@ -1215,17 +1224,12 @@ def test_run_compile_reconfigures_when_cache_entries_change( assert mock_ninja.called is (reconfigure_rc == 0) -@pytest.mark.parametrize("failing", ["all", "size"]) -def test_run_compile_stops_on_ninja_failure(setup_core: Path, failing: str) -> None: - """A failed build skips size; either failure skips the summary.""" +def test_run_compile_stops_on_ninja_failure(setup_core: Path) -> None: + """A failed ninja run skips the summary.""" _setup_build(setup_core) - with _up_to_date_compile(lambda target, **kw: 7 if target == failing else 0) as ( - mock_ninja, - mock_summary, - ): + with _up_to_date_compile(lambda *targets, **kw: 7) as (mock_ninja, mock_summary): assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 7 - targets = [c.args[0] for c in mock_ninja.call_args_list] - assert targets == (["all"] if failing == "all" else ["all", "size"]) + assert [c.args for c in mock_ninja.call_args_list] == [("all", "size")] mock_summary.assert_not_called() @@ -1236,11 +1240,11 @@ def test_run_compile_testing_mode_builds_memory_ld_first( """Testing mode builds and patches memory.ld before the main build.""" _setup_build(setup_core) CORE.testing_mode = True - targets: list[str] = [] + targets: list[tuple[str, ...]] = [] - def record(target: str, **kwargs: object) -> int: - targets.append(target) - return memory_ld_rc if target.endswith("memory.ld") else 0 + def record(*run_targets: str, **kwargs: object) -> int: + targets.append(run_targets) + return memory_ld_rc if run_targets[0].endswith("memory.ld") else 0 with ( _up_to_date_compile(record), @@ -1249,10 +1253,10 @@ def test_run_compile_testing_mode_builds_memory_ld_first( assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == memory_ld_rc memory_ld = str(Path("esp-idf", "esp_system", "ld", "memory.ld")) if memory_ld_rc: - assert targets == [memory_ld] + assert targets == [(memory_ld,)] mock_patch.assert_not_called() else: - assert targets == [memory_ld, "all", "size"] + assert targets == [(memory_ld,), ("all", "size")] mock_patch.assert_called_once() diff --git a/tests/unit_tests/test_espota2.py b/tests/unit_tests/test_espota2.py index 2d65e8e079..f4c5935059 100644 --- a/tests/unit_tests/test_espota2.py +++ b/tests/unit_tests/test_espota2.py @@ -12,6 +12,7 @@ from pathlib import Path import socket import struct from unittest.mock import Mock, call, patch +import zlib import pytest from pytest import CaptureFixture @@ -354,6 +355,7 @@ def test_perform_ota_successful_md5_auth( espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION | espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + | espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE ] ) ) @@ -601,12 +603,16 @@ def test_perform_ota_upload_error(mock_socket: Mock, mock_file: io.BytesIO) -> N espota2.perform_ota(mock_socket, None, mock_file, "test.bin") -def _no_auth_handshake(version: int) -> list[bytes]: +def _no_auth_handshake(version: int, server_features: int | None = None) -> list[bytes]: """Recv responses for a handshake without auth, up to the MD5 check.""" + if server_features is None: + features = [bytes([espota2.RESPONSE_HEADER_OK])] + else: + features = [bytes([espota2.RESPONSE_FEATURE_FLAGS]), bytes([server_features])] return [ bytes([espota2.RESPONSE_OK]), # First byte of version response bytes([version]), # Version number - bytes([espota2.RESPONSE_HEADER_OK]), # Features response + *features, bytes([espota2.RESPONSE_AUTH_OK]), # No auth required bytes([espota2.RESPONSE_UPDATE_PREPARE_OK]), # Binary size OK bytes([espota2.RESPONSE_BIN_MD5_OK]), # MD5 checksum OK @@ -1054,6 +1060,7 @@ def test_perform_ota_successful_sha256_auth( espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION | espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + | espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE ] ) ) @@ -1110,6 +1117,7 @@ def test_perform_ota_sha256_fallback_to_md5( espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION | espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + | espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE ] ) ) @@ -1219,6 +1227,7 @@ def test_perform_ota_extended_protocol_app( espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION | espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + | espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE ] ) ) @@ -1279,6 +1288,7 @@ def test_perform_ota_successful_partition_table( espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION | espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + | espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE ] ) ) @@ -1507,3 +1517,40 @@ def test_check_error_passes_non_error_when_expect_is_none() -> None: espota2.check_error([espota2.RESPONSE_OK], None) espota2.check_error([espota2.RESPONSE_HEADER_OK], None) espota2.check_error([espota2.RESPONSE_FEATURE_FLAGS], None) + + +# Device replies after the MD5 check for a one-chunk upload +_UPLOAD_TAIL = [ + bytes([espota2.RESPONSE_CHUNK_OK]), + bytes([espota2.RESPONSE_RECEIVE_OK]), + bytes([espota2.RESPONSE_UPDATE_END_OK]), +] + + +@pytest.mark.usefixtures("mock_time") +@pytest.mark.parametrize( + "server_features", + [ + espota2.SERVER_FEATURE_SUPPORTS_DEFLATE, + # Binding offer: deflate wins over gzip + espota2.SERVER_FEATURE_SUPPORTS_DEFLATE + | espota2.SERVER_FEATURE_SUPPORTS_COMPRESSION, + ], +) +def test_perform_ota_with_deflate(mock_socket: Mock, server_features: int) -> None: + """The device gets a raw deflate stream, both sizes and the image MD5.""" + original_content = b"firmware" * 100 + mock_socket.recv.side_effect = ( + _no_auth_handshake(espota2.OTA_VERSION_2_0, server_features) + _UPLOAD_TAIL + ) + + espota2.perform_ota(mock_socket, None, io.BytesIO(original_content), "test.bin") + + sent = [c[0][0] for c in mock_socket.sendall.call_args_list] + # magic, features, ota type, size, image size, md5, data, end ack + sent_size = struct.unpack(">I", sent[3])[0] + assert sent[4] == len(original_content).to_bytes(espota2.SIZE_FIELD_BYTES, "big") + payload = sent[6] + assert len(payload) == sent_size < len(original_content) + assert zlib.decompress(payload, -espota2.DEFLATE_WINDOW_BITS) == original_content + assert sent[5] == hashlib.md5(original_content).hexdigest().encode() diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index b94a70443c..198b002d43 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -573,6 +573,65 @@ def test_undefined_variable_warning( assert "'undefined_var' is undefined" in caplog.text +def test_undefined_variable_warning_avoided_by_include_condition( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Compare two different ways to conditionally enable an !include file + that defines and uses new variables. One way generates a undefined + variable warning when disabled and the other does not.""" + include_file = tmp_path / "include.yaml" + include_file.write_text("substitutions:\n var: 1\nresult: ${var}\n") + method_a_file = tmp_path / "method_a.yaml" + method_a_file.write_text( + "packages:\n" + " - !include\n" + " file: include.yaml\n" + " condition: ${enable == 'y'}\n" + ) + method_b_file = tmp_path / "method_b.yaml" + method_b_file.write_text( + "substitutions:\n" + " content: !include include.yaml\n" + "packages:\n" + " - ${content if enable == 'y' else {}}\n" + ) + + def process_file(file, enable): + with caplog.at_level(logging.WARNING): + args = {"enable": enable} + config = yaml_util.load_yaml(file) + config = do_packages_pass(config, command_line_substitutions=args) + return substitutions.do_substitution_pass( + config, command_line_substitutions=args + ) + + # Case: method A with include enabled produces a result and no warnings + config = process_file(method_a_file, "y") + assert caplog.text == "" + assert config["packages"][0]["result"] == 1 + + # Case: method A with include disabled produces no result and no warnings + # because the file was not even loaded + config = process_file(method_a_file, "n") + assert caplog.text == "" + assert config["packages"][0] == {} + + # Case: method B with include enabled produces a result and no warnings (same as method A) + config = process_file(method_b_file, "y") + assert caplog.text == "" + assert config["packages"][0]["result"] == 1 + + # Case: method B with include disabled produces no result and an undefined variable warning + # because the file is loaded into a variable and its own variables cannot be expanded + # (different from method A) + config = process_file(method_b_file, "n") + assert ( + "Could not resolve substitution variable 'content': 'var' is undefined" + in caplog.text + ) + assert config["packages"][0] == {} + + def test_password_field_warnings_suppressed( caplog: pytest.LogCaptureFixture, ) -> None: @@ -717,11 +776,9 @@ def test_resolve_package_max_depth_exceeded(tmp_path: Path) -> None: # Each call to the loader returns a fresh IncludeFile pointing at itself, # so PACKAGE_SCHEMA always sees an IncludeFile and never a dict. def always_returns_include(path: Path) -> yaml_util.IncludeFile: - return yaml_util.IncludeFile(parent, path.name, None, always_returns_include) + return yaml_util.IncludeFile(parent, path.name, always_returns_include) - package_config = yaml_util.IncludeFile( - parent, "test.yaml", None, always_returns_include - ) + package_config = yaml_util.IncludeFile(parent, "test.yaml", always_returns_include) processor = _PackageProcessor({}, None) with pytest.raises( cv.Invalid, @@ -791,6 +848,22 @@ def test_include_filename_jinja_expression_with_path_separator( assert config["result"] == {"value": 42} +def test_include_condition_substitution_undefined_var(tmp_path: Path) -> None: + """!include with an undefined substitution variable raises cv.Invalid. + + The error message must reference the unresolved condition template so the + user knows which include failed, rather than seeing a bare file-not-found. + """ + main_file = tmp_path / "main.yaml" + main_file.write_text( + "result: !include\n file: file.yaml\n condition: ${undefined_var}\n" + ) + + config = yaml_util.load_yaml(main_file) + with pytest.raises(cv.Invalid, match=r"\$\{undefined_var\}"): + substitutions.do_substitution_pass(config) + + def test_raise_first_undefined_logs_extras_at_debug( caplog: pytest.LogCaptureFixture, ) -> None: @@ -838,7 +911,7 @@ def test_do_substitution_pass_included_substitutions_must_be_mapping( def loader(path: Path): return ["not", "a", "mapping"] - include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader) + include = yaml_util.IncludeFile(parent, "subs.yaml", loader) config = OrderedDict({CONF_SUBSTITUTIONS: include}) with pytest.raises( @@ -861,7 +934,7 @@ def test_do_packages_pass_included_substitutions_must_be_mapping( def loader(path: Path): return ["not", "a", "mapping"] - include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader) + include = yaml_util.IncludeFile(parent, "subs.yaml", loader) config = OrderedDict( { CONF_SUBSTITUTIONS: include, @@ -888,9 +961,7 @@ def test_resolve_package_undefined_var_in_include_filename(tmp_path: Path) -> No def loader(path: Path): raise EsphomeError(f"Error reading file {path}: No such file") - package_config = yaml_util.IncludeFile( - parent, "${undefined_var}.yaml", None, loader - ) + package_config = yaml_util.IncludeFile(parent, "${undefined_var}.yaml", loader) processor = _PackageProcessor({}, None) with pytest.raises(cv.Invalid, match="unresolved substitutions"): processor.resolve_package(package_config, substitutions.ContextVars(), []) @@ -906,7 +977,7 @@ def test_resolve_include_error_shows_expanded_from_when_substituted( def failing_loader(_path: Path) -> None: raise EsphomeError("File not found") - include = yaml_util.IncludeFile(parent, "${device}.yaml", None, failing_loader) + include = yaml_util.IncludeFile(parent, "${device}.yaml", failing_loader) context = substitutions.ContextVars({"device": "my_device"}) with pytest.raises(cv.Invalid) as exc_info: @@ -927,7 +998,7 @@ def test_resolve_include_error_no_expanded_from_for_literal_filename( def failing_loader(_path: Path) -> None: raise EsphomeError("File not found") - include = yaml_util.IncludeFile(parent, "literal.yaml", None, failing_loader) + include = yaml_util.IncludeFile(parent, "literal.yaml", failing_loader) with pytest.raises(cv.Invalid) as exc_info: substitutions.resolve_include(include, [], substitutions.ContextVars()) @@ -946,7 +1017,7 @@ def test_include_vars_applied_to_lambda_value(tmp_path: Path) -> None: included.write_text('!lambda |-\n return "${foo}";\n') include = yaml_util.IncludeFile( - tmp_path / "main.yaml", "lambda.yaml", {"foo": "bar"}, yaml_util.load_yaml + tmp_path / "main.yaml", "lambda.yaml", yaml_util.load_yaml, vars={"foo": "bar"} ) config = OrderedDict({"value": include.load()}) result = substitutions.do_substitution_pass(config) diff --git a/tests/unit_tests/test_yaml_util.py b/tests/unit_tests/test_yaml_util.py index f3f73ea4dc..e1c3080e29 100644 --- a/tests/unit_tests/test_yaml_util.py +++ b/tests/unit_tests/test_yaml_util.py @@ -43,17 +43,37 @@ def clear_core_frontmatter() -> None: core.CORE.frontmatter = {} -def test_include_with_vars(fixture_path: Path) -> None: +def test_include(fixture_path: Path) -> None: + """Test !include with and without vars, with and without conditions""" yaml_file = fixture_path / "yaml_util" / "includetest.yaml" actual = yaml_util.load_yaml(yaml_file) actual = substitutions.do_substitution_pass(actual, None) assert actual["esphome"]["name"] == "original" + assert actual["esphome"]["name_add_mac_suffix"] assert actual["esphome"]["libraries"][0] == "Wire" assert actual["esp8266"]["board"] == "nodemcu" assert actual["wifi"]["ssid"] == "my_custom_ssid" +def test_include_with_no_file(fixture_path: Path) -> None: + """Ensure that an error is emitted when the file field is missing.""" + yaml_file = fixture_path / "yaml_util" / "includetest_no_file.yaml" + + with pytest.raises(EsphomeError, match=r"Must include 'file'"): + yaml_util.load_yaml(yaml_file) + + +def test_include_with_invalid_condition_type(fixture_path: Path) -> None: + """Ensure that an error is emitted when the condition field is an invalid type.""" + yaml_file = fixture_path / "yaml_util" / "includetest_invalid_condition_type.yaml" + + with pytest.raises( + EsphomeError, match=r"Include 'condition' must be a boolean or string" + ): + yaml_util.load_yaml(yaml_file) + + def test_loading_a_missing_file(fixture_path): """We throw EsphomeError when loading a missing file.""" yaml_file = fixture_path / "yaml_util" / "missing.yaml" @@ -64,7 +84,7 @@ def test_loading_a_missing_file(fixture_path): def test_parsing_with_custom_loader(fixture_path): """Test custom loader used for vscode connection - Default loader is tested in test_include_with_vars + Default loader is tested in test_include """ yaml_file = fixture_path / "yaml_util" / "includetest.yaml" @@ -78,10 +98,11 @@ def test_parsing_with_custom_loader(fixture_path): # substitute config to expand includes: substitutions.substitute(config, [], substitutions.ContextVars(), False) - assert len(loader_calls) == 3 + assert len(loader_calls) == 4 assert loader_calls[0].parts[-2:] == ("includes", "included.yaml") - assert loader_calls[1].parts[-2:] == ("includes", "list.yaml") - assert loader_calls[2].parts[-2:] == ("includes", "scalar.yaml") + assert loader_calls[1].parts[-2:] == ("includes", "true.yaml") + assert loader_calls[2].parts[-2:] == ("includes", "list.yaml") + assert loader_calls[3].parts[-2:] == ("includes", "scalar.yaml") def test_construct_secret_simple(fixture_path: Path) -> None: @@ -570,7 +591,7 @@ def test_represent_remove() -> None: def test_represent_include_file() -> None: """Test that IncludeFile objects are dumped as !include scalars.""" include = yaml_util.IncludeFile( - Path("/fake/main.yaml"), "path/to/file.yaml", None, lambda _: {} + Path("/fake/main.yaml"), "path/to/file.yaml", lambda _: {} ) assert yaml_util.dump({"key": include}) == "key: !include 'path/to/file.yaml'\n" @@ -580,13 +601,27 @@ def test_represent_include_file_with_vars() -> None: include = yaml_util.IncludeFile( Path("/fake/main.yaml"), "path/to/file.yaml", - {"key": "value"}, lambda _: {}, + vars={"key": "value"}, + ) + assert ( + yaml_util.dump({"key": include}) + == "key: !include\n file: path/to/file.yaml\n vars:\n key: value\n" + ) + + +def test_represent_include_file_with_condition() -> None: + """Test that IncludeFile with condition is dumped as !include mapping form.""" + include = yaml_util.IncludeFile( + Path("/fake/main.yaml"), + "path/to/file.yaml", + lambda _: {}, + condition="true", + ) + assert ( + yaml_util.dump({"key": include}) + == "key: !include\n file: path/to/file.yaml\n condition: 'true'\n" ) - result = yaml_util.dump({"key": include}) - assert "!include" in result - assert "file: path/to/file.yaml" in result - assert "key: value" in result def test_represent_include_file_with_data_base_mixin() -> None: @@ -596,7 +631,7 @@ def test_represent_include_file_with_data_base_mixin() -> None: subclass. add_multi_representer must match this subclass through the MRO. """ include = yaml_util.IncludeFile( - Path("/fake/main.yaml"), "common/spi.yaml", None, lambda _: {} + Path("/fake/main.yaml"), "common/spi.yaml", lambda _: {} ) wrapped = yaml_util.make_data_base(include) assert isinstance(wrapped, yaml_util.ESPHomeDataBase) @@ -609,7 +644,7 @@ def test_represent_include_file_with_data_base_mixin() -> None: def test_include_file_repr(tmp_path: Path) -> None: """repr() includes the filename so it appears usefully in error messages.""" parent = tmp_path / "main.yaml" - include = yaml_util.IncludeFile(parent, "some/nested.yaml", None, lambda _: {}) + include = yaml_util.IncludeFile(parent, "some/nested.yaml", lambda _: {}) assert repr(include) == "IncludeFile(some/nested.yaml)" @@ -624,7 +659,7 @@ def test_include_file_load_caches_result(tmp_path: Path) -> None: call_count += 1 return content - include = yaml_util.IncludeFile(parent, "child.yaml", None, counting_loader) + include = yaml_util.IncludeFile(parent, "child.yaml", counting_loader) first = include.load() second = include.load() @@ -641,7 +676,7 @@ def test_include_file_load_caches_none_result(tmp_path: Path) -> None: nonlocal call_count call_count += 1 - include = yaml_util.IncludeFile(parent, "empty.yaml", None, counting_loader) + include = yaml_util.IncludeFile(parent, "empty.yaml", counting_loader) first = include.load() second = include.load() @@ -653,7 +688,7 @@ def test_include_file_load_caches_none_result(tmp_path: Path) -> None: def test_include_file_load_raises_on_unresolved_expressions(tmp_path: Path) -> None: """load() raises if the filename contains unresolved substitutions or expressions.""" parent = tmp_path / "main.yaml" - include = yaml_util.IncludeFile(parent, "${undefined_var}.yaml", None, lambda _: {}) + include = yaml_util.IncludeFile(parent, "${undefined_var}.yaml", lambda _: {}) with pytest.raises(cv.Invalid, match="unresolved"): include.load() @@ -670,13 +705,83 @@ def test_include_file_load_raises_on_unresolved_expressions(tmp_path: Path) -> N ("price-100$.yaml", False), # $ at end, not followed by valid substitution ], ) -def test_include_file_has_unresolved_expressions( +def test_include_file_has_unresolved_file( tmp_path: Path, filename: str, expected: bool ) -> None: - """has_unresolved_expressions() detects substitution patterns in the filename.""" + """has_unresolved_file() detects substitution patterns in the filename.""" parent = tmp_path / "main.yaml" - include = yaml_util.IncludeFile(parent, filename, None, lambda _: {}) - assert include.has_unresolved_expressions() == expected + include = yaml_util.IncludeFile(parent, filename, lambda _: {}) + assert include.has_unresolved_file() == expected + + +@pytest.mark.parametrize( + ("condition", "expected"), + [ + (None, False), + (True, False), + (False, False), + ("true", False), + ("false", False), + ("$has_feature", True), # whole substitution + ("${has_feature}", True), # whole substitution + ("tr$ue", True), # partial substitution + ("$.", False), # malformed substitution + ("${1 == 1}", True), # Jinja expression + ("${", False), # malformed expression + ], +) +def test_include_file_has_unresolved_condition( + tmp_path: Path, condition: bool | str | None, expected: bool +) -> None: + """has_unresolved_condition() detects substitution patterns in the condition.""" + parent = tmp_path / "main.yaml" + include = yaml_util.IncludeFile( + parent, "device.yaml", lambda _: {}, condition=condition + ) + assert include.has_unresolved_condition() == expected + + +@pytest.mark.parametrize( + ("condition", "expected_result_or_error"), + [ + (None, True), + (True, True), + (False, False), + ("true", True), + ("True", True), + ("TRUE", True), + ("yes", True), + ("on", True), + ("enable", True), + ("false", False), + ("False", False), + ("FALSE", False), + ("no", False), + ("off", False), + ("disable", False), + ( + "$sub", + "Cannot evaluate include condition for 'device.yaml' with unresolved substitutions", + ), + ("", "Cannot convert include condition for 'device.yaml' to a boolean"), + ("trues", "Cannot convert include condition for 'device.yaml' to a boolean"), + ], +) +def test_include_file_should_load( + tmp_path: Path, + condition: bool | str | None, + expected_result_or_error: bool | str, +) -> None: + """should_load() evaluates the condition and raises an error if it is malformed.""" + parent = tmp_path / "main.yaml" + include = yaml_util.IncludeFile( + parent, "device.yaml", lambda _: {}, condition=condition + ) + if isinstance(expected_result_or_error, bool): + assert include.should_load() == expected_result_or_error + else: + with pytest.raises(cv.Invalid, match=expected_result_or_error): + include.should_load() def test_mapping_include_non_string_file_rejected(tmp_path: Path) -> None: @@ -691,16 +796,16 @@ def test_include_file_templated_filename_stays_raw_string(tmp_path: Path) -> Non """A templated filename keeps its verbatim text (issue #18545).""" parent = tmp_path / "main.yaml" expr = '${ "bluetooth/proxy.yaml" if enable_bluetooth_proxy else "../empty.yaml" }' - include = yaml_util.IncludeFile(parent, expr, None, lambda _: {}) + include = yaml_util.IncludeFile(parent, expr, lambda _: {}) assert include.file == expr - assert include.has_unresolved_expressions() + assert include.has_unresolved_file() assert repr(include) == f"IncludeFile({expr})" def test_represent_include_file_templated() -> None: """Dumping a templated IncludeFile emits the raw expression unchanged.""" expr = '${ "a/b.yaml" if flag else "../c.yaml" }' - include = yaml_util.IncludeFile(Path("/fake/main.yaml"), expr, None, lambda _: {}) + include = yaml_util.IncludeFile(Path("/fake/main.yaml"), expr, lambda _: {}) assert yaml_util.dump({"key": include}) == f"key: !include '{expr}'\n" @@ -710,12 +815,12 @@ def test_include_in_list_context() -> None: parent = Path("/fake/main.yaml") # The nested IncludeFile resolves to a plain string value - inner = yaml_util.IncludeFile(parent, "inner.yaml", None, lambda _: "gamma") + inner = yaml_util.IncludeFile(parent, "inner.yaml", lambda _: "gamma") # The outer IncludeFile returns a list whose last element is itself an IncludeFile, # exercising the substitution pass's ability to recurse into loaded content. outer = yaml_util.IncludeFile( - parent, "items.yaml", None, lambda _: ["alpha", "beta", inner] + parent, "items.yaml", lambda _: ["alpha", "beta", inner] ) config = OrderedDict({"values": outer}) @@ -736,15 +841,48 @@ def test_top_level_include_resolved_by_load_yaml(tmp_path: Path) -> None: assert result["key"] == "value" +@pytest.mark.parametrize( + ("condition", "expected_result_or_error"), + [ + ("true", True), + ("false", False), + ('"TRUE"', True), + ('"FALSE"', False), + ('"x"', "Cannot convert include condition for 'child.yaml' to a boolean"), + ( + "$sub", + "Cannot evaluate include condition for 'child.yaml' with unresolved substitutions", + ), + ], +) +def test_top_level_include_with_condition_resolved_by_load_yaml( + tmp_path: Path, condition: bool | str | None, expected_result_or_error: bool | str +) -> None: + """load_yaml evaluates the condition for a top-level !include.""" + child = tmp_path / "child.yaml" + child.write_text("key: value\n") + main = tmp_path / "main.yaml" + main.write_text(f"!include {{ file: child.yaml, condition: {condition} }}\n") + + if isinstance(expected_result_or_error, bool): + result = yaml_util.load_yaml(main) + assert isinstance(result, dict) + if expected_result_or_error: + assert result["key"] == "value" + else: + assert result == {} + else: + with pytest.raises(cv.Invalid, match=expected_result_or_error): + result = yaml_util.load_yaml(main) + + def test_include_plain_filename_loads_after_deferred_refactor() -> None: """!include with a plain filename (no $ expressions) still loads correctly. Regression guard: the deferred-loading refactor must not break the simple case. """ parent = Path("/fake/main.yaml") - include = yaml_util.IncludeFile( - parent, "child.yaml", None, lambda _: {"answer": 42} - ) + include = yaml_util.IncludeFile(parent, "child.yaml", lambda _: {"answer": 42}) config = OrderedDict({"result": include}) config = substitutions.do_substitution_pass(config) @@ -756,7 +894,23 @@ def test_yaml_merge_include_with_filename_substitution_raises() -> None: """<<: !include ${expr} raises a clear error — substitutions in merge-key filenames are not yet supported, and the error message must say so.""" yaml_text = "base:\n existing: value\n <<: !include ${filename}.yaml\n" - with pytest.raises(EsphomeError, match="not supported yet"): + with pytest.raises( + EsphomeError, + match="Substitution in include filename with merge keys is not supported yet", + ): + yaml_util.parse_yaml( + Path("/fake/main.yaml"), io.StringIO(yaml_text), lambda _: {} + ) + + +def test_yaml_merge_include_with_condition_substitution_raises() -> None: + """<<: !include { file: ${expr}, condition: {} } raises a clear error — substitutions in merge-key conditions + are not yet supported, and the error message must say so.""" + yaml_text = "base:\n existing: value\n <<: !include\n file: filename.yaml\n condition: ${expr}\n" + with pytest.raises( + EsphomeError, + match="Substitution in include condition with merge keys is not supported yet", + ): yaml_util.parse_yaml( Path("/fake/main.yaml"), io.StringIO(yaml_text), lambda _: {} ) @@ -775,8 +929,8 @@ def test_yaml_merge_chain_include_resolves() -> None: """Chained includes in merge keys resolve through multiple IncludeFile layers.""" parent = Path("/fake/main.yaml") - inner = yaml_util.IncludeFile(parent, "inner.yaml", None, lambda _: {"x": 1}) - outer = yaml_util.IncludeFile(parent, "outer.yaml", None, lambda _: inner) + inner = yaml_util.IncludeFile(parent, "inner.yaml", lambda _: {"x": 1}) + outer = yaml_util.IncludeFile(parent, "outer.yaml", lambda _: inner) yaml_text = "base:\n existing: value\n <<: !include outer.yaml\n" config = yaml_util.parse_yaml(parent, io.StringIO(yaml_text), lambda _: outer) @@ -791,7 +945,7 @@ def test_yaml_merge_chain_include_depth_exceeded() -> None: parent = Path("/fake/main.yaml") def self_referencing_loader(path: Path) -> yaml_util.IncludeFile: - return yaml_util.IncludeFile(parent, path.name, None, self_referencing_loader) + return yaml_util.IncludeFile(parent, path.name, self_referencing_loader) yaml_text = "base:\n <<: !include loop.yaml\n" with pytest.raises(EsphomeError, match="Maximum include chain depth"): @@ -1061,7 +1215,7 @@ class _StubInclude: self._raise = raise_on_load self.load_calls = 0 - def has_unresolved_expressions(self) -> bool: + def has_unresolved_file(self) -> bool: return self._unresolved def load(self) -> object: