github/esphome
2
0
Fork 0
mirror of https://github.com/esphome/esphome.git synced 2026-10-03 01:35:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into esp32-log-v2-idf61

Browse Source
This commit is contained in:
J. Nick Koston
2026-10-02 08:14:41 -05:00
parent f8208b8fc2 4724aea42e
commit 003698970e
77 changed files with 3210 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CODEOWNERS
+1
View File
@@ -560,6 +560,7 @@ esphome/components/tc74/* @sethgirvan
esphome/components/tca9548a/* @andreashergert1984
esphome/components/tca9555/* @mobrembski
esphome/components/tcl112/* @glmnet
esphome/components/tcp_uart/* @Bascht74
esphome/components/tee501/* @Stock-M
esphome/components/teleinfo/* @0hax
esphome/components/tem3200/* @bakerkj
esphome/__main__.py
+3 -4
View File
@@ -1300,10 +1300,9 @@ def _choose_ota_platform(config: ConfigType, requested: str | None) -> str:
The native API uses challenge-response auth with MD5/SHA256 hashing of a
server-issued nonce, so the password is never sent over the wire; the
``web_server`` path uses HTTP Basic auth which transmits credentials in
cleartext over the LAN. (The native path also supports gzip compression
on ESP8266, where flash space is tight; on ESP32/RP2040/LibreTiny the
backend reports ``supports_compression() == false`` and the firmware is
sent uncompressed regardless of which platform is used.) Falls back to
cleartext over the LAN. (The native path also compresses the upload:
gzip on ESP8266 and RP2040, which inflate it at reboot, and a deflate
stream on ESP32/LibreTiny, which inflate it as it arrives.) Falls back to
``web_server`` only when that is the only available platform.
"""
# Use a dict (insertion-ordered) instead of a list so error messages and
esphome/build_gen/espidf.py
+43
View File
@@ -66,6 +66,43 @@ else()
"app edits will regenerate sections.ld.")
endif()"""
# lwip sources that compile to empty objects with the option off (their own
# #if guard). (option, regex valid for both Python and CMake); a source is
# only dropped when its option is defined and off, so a renamed option
# keeps it.
LWIP_EMPTY_SOURCES: tuple[tuple[str, str], ...] = (
("CONFIG_LWIP_PPP_SUPPORT", "/netif/ppp/"),
("CONFIG_LWIP_IPV6", "/core/ipv6/"),
("CONFIG_LWIP_AUTOIP", "/core/ipv4/autoip[.]c$"),
("CONFIG_LWIP_STATS", "/core/stats[.]c$"),
)
# Drift guard only: keep every lwip source.
LWIP_FULL_SOURCES_ENV = "ESPHOME_LWIP_FULL_SOURCES"
# Drops the empty objects after project(), once the lwip target exists.
_LWIP_EMPTY_SOURCES_FILTER = f"""\
idf_build_get_property(esphome_build_components BUILD_COMPONENTS)
if(lwip IN_LIST esphome_build_components AND NOT DEFINED ENV{{{LWIP_FULL_SOURCES_ENV}}})
idf_component_get_property(esphome_lwip_lib lwip COMPONENT_LIB)
get_target_property(esphome_lwip_srcs ${{esphome_lwip_lib}} SOURCES)
@FILTERS@
set_property(TARGET ${{esphome_lwip_lib}} PROPERTY SOURCES ${{esphome_lwip_srcs}})
endif()"""
def lwip_empty_source_gate(option: str, regex: str) -> str:
return (
f" if(DEFINED {option} AND NOT {option})\n"
f' list(FILTER esphome_lwip_srcs EXCLUDE REGEX "{regex}")\n'
" endif()"
)
def _lwip_empty_sources_filter() -> str:
gates = "\n".join(lwip_empty_source_gate(*entry) for entry in LWIP_EMPTY_SOURCES)
return _LWIP_EMPTY_SOURCES_FILTER.replace("@FILTERS@", gates)
# Runs after project() so the walk has happened; catches the remaining
# silent path where the top-level out-var was renamed.
_LDGEN_OVERRIDE_CHECK = """\
@@ -348,14 +385,20 @@ project({CORE.name})
{ldgen_override_check}
{_lwip_empty_sources_filter()}
# Emit per-memory-type JSON size data for ESPHome to read post-build.
# json2 stays small; raw dumps every symbol (~2s on a large map) and
# this command runs inside the link edge, blocking everything downstream.
# The map is a BYPRODUCT so ninja knows the link writes it; IDF's size
# target depends on the map and can then be built in the same run as all.
# IDF's cmakev2 declares the map itself, so drop this line on that switch.
add_custom_command(
TARGET ${{CMAKE_PROJECT_NAME}}.elf POST_BUILD
COMMAND ${{PYTHON}} -m esp_idf_size {size_ng_flag} --format=json2
-o ${{CMAKE_BINARY_DIR}}/esp_idf_size.json
${{CMAKE_PROJECT_NAME}}.map
BYPRODUCTS ${{CMAKE_BINARY_DIR}}/${{CMAKE_PROJECT_NAME}}.map
WORKING_DIRECTORY ${{CMAKE_BINARY_DIR}}
VERBATIM
)
esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.cpp
+72 -1
View File
@@ -17,6 +17,13 @@
#include "esphome/core/log.h"
#include <cstring>
#include <esp_idf_version.h>
// cancel_open exists from ESP-IDF 5.5.5 and 6.0.1.
#if (ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 5, 5) && ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(6, 0, 0)) || \
ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 1)
#define BLUEDROID_HAS_CANCEL_OPEN
#endif
namespace esphome::bluetooth_connection {
@@ -63,6 +70,12 @@ void BluedroidGattClient::loop() {
// The one teardown safety net: a lost CLOSE_EVT, or a scheduled
// teardown whose OPEN_EVT never arrives.
if (millis() - this->disconnecting_started_ > ble_device_base::GATT_DISCONNECT_TIMEOUT_MS) {
if (this->conn_id_ == UNSET_CONN_ID) {
this->cancel_pending_open_();
} else if (st == ClientState::CONNECTING) {
// CONNECT_EVT came but OPEN_EVT was lost, so our open still holds the ACL link.
this->check_and_log_error_("esp_ble_gap_disconnect", esp_ble_gap_disconnect(this->remote_bda_));
}
ESP_LOGE(TAG, "[%d] Timeout waiting for teardown, forcing IDLE", this->connection_index_);
// Release before idling: a lost completion must not leak the cache.
this->release_services();
@@ -140,6 +153,8 @@ void BluedroidGattClient::tracker_connect_() {
this->services_released_ = false;
this->seen_mtu_ = false;
this->mtu_failed_ = false;
this->cancel_open_sent_ = false;
this->conn_id_ = UNSET_CONN_ID;
this->enable_loop();
this->set_state(ClientState::CONNECTING);
if (this->connection_type_ == ConnectionType::V3_WITHOUT_CACHE) {
@@ -183,12 +198,28 @@ int BluedroidGattClient::gatt_disconnect() {
// Arm the safety window: a lost OPEN_EVT must not leak the teardown.
this->disconnecting_started_ = millis();
this->enable_loop();
if (this->conn_id_ == UNSET_CONN_ID) {
// CANCEL_OPEN_EVT or a racing OPEN_EVT settles the slot.
this->cancel_pending_open_();
}
return 0;
}
this->unconditional_disconnect_();
return 0;
}
void BluedroidGattClient::cancel_pending_open_() {
#ifdef BLUEDROID_HAS_CANCEL_OPEN
esp_ble_gattc_cancel_open_params_t params{};
params.gattc_if = this->gattc_if_;
memcpy(params.remote_bda, this->remote_bda_, sizeof(esp_bd_addr_t));
// A refused cancel schedules no CANCEL_OPEN_EVT, so the teardown stays cancellable.
if (this->check_and_log_error_("esp_ble_gattc_cancel_open", esp_ble_gattc_cancel_open(&params)) == ESP_OK) {
this->cancel_open_sent_ = true;
}
#endif
}
void BluedroidGattClient::unconditional_disconnect_() {
ESP_LOGI(TAG, "[%d] Disconnecting (conn_id: %d)", this->connection_index_, this->conn_id_);
if (this->conn_id_ == UNSET_CONN_ID) {
@@ -214,6 +245,10 @@ bool BluedroidGattClient::cancel_gatt_disconnect() {
if (this->state() != ClientState::CONNECTING || !this->disconnect_pending()) {
return false;
}
if (this->cancel_open_sent_) {
// The cancel already went out, so CANCEL_OPEN_EVT ends it.
return false;
}
this->want_disconnect_ = false;
return true;
}
@@ -574,6 +609,8 @@ void BluedroidGattClient::handle_open_evt_(esp_ble_gattc_cb_param_t *param) {
this->listener_->on_connection_state(false, 0, param->open.status);
return;
}
// ALREADY_OPEN on an existing link sends no CONNECT_EVT, so take the id here.
this->conn_id_ = param->open.conn_id;
if (this->disconnect_pending()) {
// Open resolved with a teardown scheduled: close now (conn_id_ stays set
// so CLOSE_EVT still matches).
@@ -617,6 +654,11 @@ void BluedroidGattClient::handle_disconnect_evt_(esp_ble_gattc_cb_param_t *param
// Active close delivers CLOSE_EVT first; never walk back to DISCONNECTING.
return;
}
if (this->conn_id_ == UNSET_CONN_ID) {
// No CONNECT_EVT for this attempt yet, so the event belongs to an earlier
// link, such as one the teardown timeout forced down.
return;
}
// Passive disconnect: wait for CLOSE_EVT before going IDLE (reconnecting
// earlier makes the controller reject with 133 or assert) and before
// reporting - the wrapper frees the slot on the report, and a freed slot
@@ -645,6 +687,10 @@ bool BluedroidGattClient::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga
case ESP_GATTC_CONNECT_EVT: {
if (!this->check_addr_(param->connect.remote_bda))
return false;
// Every client interface gets CONNECT_EVT for every new link; only an
// attempt in flight owns it. An idle slot must not adopt another slot's link.
if (this->state() != ClientState::CONNECTING)
break;
this->conn_id_ = param->connect.conn_id;
// MTU request here rather than OPEN_EVT, matching the IDF examples.
auto ret = esp_ble_gattc_send_mtu_req(this->gattc_if_, param->connect.conn_id);
@@ -656,8 +702,15 @@ bool BluedroidGattClient::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga
break;
}
case ESP_GATTC_OPEN_EVT: {
if (!this->check_addr_(param->open.remote_bda))
if (!this->check_addr_(param->open.remote_bda)) {
// Our interface (filtered above), stale address, so nobody tracks this link.
if (param->open.status == ESP_GATT_OK || param->open.status == ESP_GATT_ALREADY_OPEN) {
ESP_LOGW(TAG, "[%d] Closing link left by an abandoned open", this->connection_index_);
this->check_and_log_error_("esp_ble_gattc_close", esp_ble_gattc_close(this->gattc_if_, param->open.conn_id));
return true;
}
return false;
}
this->handle_open_evt_(param);
break;
}
@@ -748,6 +801,24 @@ bool BluedroidGattClient::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga
this->listener_->on_notify_data(param->notify.handle, param->notify.value, param->notify.value_len);
break;
}
#ifdef BLUEDROID_HAS_CANCEL_OPEN
case ESP_GATTC_CANCEL_OPEN_EVT: {
if (!this->check_addr_(param->cancel_open.remote_bda))
return false;
if (param->cancel_open.status != ESP_GATT_OK) {
// Too late to cancel: OPEN_EVT follows and settles the slot. The latch
// stays set because the event cannot be tied to this attempt's cancel.
break;
}
if (this->state() == ClientState::CONNECTING && this->disconnect_pending()) {
ESP_LOGD(TAG, "[%d] Pending open cancelled", this->connection_index_);
this->release_services();
this->set_idle_();
this->listener_->on_connection_state(false, 0, ESP_GATT_CONN_CONN_CANCEL);
}
break;
}
#endif
default:
break;
}
esphome/components/bluetooth_connection/bluetooth_connection_bluedroid.h
+4
View File
@@ -100,6 +100,7 @@ class BluedroidGattClient final : public esp32_ble_tracker::ESPBTClient, public
int handle_search_cmpl_(esp_gatt_status_t status);
void deliver_pending_search_();
void unconditional_disconnect_();
void cancel_pending_open_();
void set_idle_();
void set_disconnecting_();
esp_err_t update_conn_params_(uint16_t min_interval, uint16_t max_interval, uint16_t latency, uint16_t timeout,
@@ -133,6 +134,9 @@ class BluedroidGattClient final : public esp32_ble_tracker::ESPBTClient, public
bool seen_mtu_ : 1 {false};
// The MTU request was refused at CONNECT_EVT; OPEN_EVT reports instead.
bool mtu_failed_ : 1 {false};
// esp_ble_gattc_cancel_open() was accepted; CANCEL_OPEN_EVT or OPEN_EVT ends
// the attempt, so the scheduled teardown can no longer be cancelled.
bool cancel_open_sent_ : 1 {false};
// Search issued at OPEN_EVT overlaps the MTU exchange; discover_services()
// completes from it. Reset by set_idle_().
static_assert(static_cast<uint8_t>(SearchState::REPORT_PENDING) < (1 << 4), "search_state_ bitfield too narrow");
esphome/components/esphome/ota/__init__.py
+14 -1
View File
@@ -315,10 +315,19 @@ FINAL_VALIDATE_SCHEMA = ota_esphome_final_validate
FILTER_SOURCE_FILES = filter_source_files_from_defines(
{"ota_esphome_noise.cpp": "USE_OTA_ENCRYPTION"}
{
"ota_esphome_noise.cpp": "USE_OTA_ENCRYPTION",
"ota_esphome_inflate_session.cpp": "USE_OTA_DEFLATE",
"ota_esphome_inflate.c": "USE_OTA_DEFLATE",
}
)
def enable_deflate() -> None:
"""Compile the on-the-fly inflater for compressed uploads."""
cg.add_define("USE_OTA_DEFLATE")
@coroutine_with_priority(CoroPriority.OTA_UPDATES)
async def to_code(config: ConfigType) -> None:
var = cg.new_Pvariable(config[CONF_ID])
@@ -340,6 +349,10 @@ async def to_code(config: ConfigType) -> None:
if config.get(CONF_ALLOW_PARTITION_ACCESS):
cg.add_define("USE_OTA_PARTITIONS")
# ESP8266 and RP2040 inflate gzip at reboot; the rest inflate on the fly
if not (CORE.is_esp8266 or CORE.is_rp2):
enable_deflate()
# One key per device: an api encryption block supplies it (static or
# runtime) and offers; the ota block only adds the requirement
api_conf = CORE.config.get(CONF_API) or {}
esphome/components/esphome/ota/ota_esphome.cpp
+151 -81
View File
@@ -22,8 +22,10 @@
#include "esphome/core/lwip_fast_select.h"
#endif
#include <algorithm>
#include <cerrno>
#include <cstdio>
#include <cstring>
#include <sys/time.h>
namespace esphome {
@@ -47,6 +49,8 @@ static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds
// practice for a lost chunk ack (1.5 + 3 + 6 + 12 + 24 + 48 s); the CLI waits
// longer (espota2.DATA_PHASE_TIMEOUT) so the device is free before it retries
static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 105000;
static constexpr uint32_t OTA_PROGRESS_INTERVAL_MS = 1000;
static constexpr size_t OTA_SIZE_FIELD_BYTES = 4; // sizes on the wire are 4 bytes MSB first
// Single-instance pointer — multi-port configs are rejected in final_validate.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
@@ -194,12 +198,23 @@ static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01;
static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02;
static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04;
static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_NOISE = 0x08;
static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_DEFLATE = 0x10;
// Noise needs the extended protocol: the prologue binds the 2-byte feature ack
static constexpr uint8_t CLIENT_NOISE_FEATURES =
CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL;
static constexpr uint8_t SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01;
static constexpr uint8_t SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02;
static constexpr uint8_t SERVER_FEATURE_SUPPORTS_NOISE = 0x04;
// Raw deflate, window <= OTA_INFLATE_WINDOW_SIZE. Binding once offered: the
// client must then send the image size frame and a deflate stream.
static constexpr uint8_t SERVER_FEATURE_SUPPORTS_DEFLATE = 0x08;
#ifdef USE_OTA_ENCRYPTION
inline bool ESPHomeOTAComponent::noise_offered_() const {
return (this->handshake_buf_[1] & SERVER_FEATURE_SUPPORTS_NOISE) != 0 &&
(this->ota_features_ & CLIENT_NOISE_FEATURES) == CLIENT_NOISE_FEATURES;
}
#endif
inline bool ESPHomeOTAComponent::extended_proto_() const {
#ifdef USE_OTA_ENCRYPTION_REQUIRED
@@ -305,7 +320,7 @@ void ESPHomeOTAComponent::handle_handshake_() {
this->transition_ota_state_(OTAState::FEATURE_ACK);
const bool supports_compression =
(this->ota_features_ & CLIENT_FEATURE_SUPPORTS_COMPRESSION) != 0 && this->backend_->supports_compression();
(this->ota_features_ & CLIENT_FEATURE_SUPPORTS_COMPRESSION) != 0 && ota::OTABackend::supports_compression();
// Compose the feature-ack response. When the client negotiates the extended protocol we emit
// a 2-byte response (marker + server feature flags); otherwise we emit the single-byte
@@ -325,6 +340,28 @@ void ESPHomeOTAComponent::handle_handshake_() {
#elif defined(USE_OTA_ENCRYPTION)
// A yaml key always exists: validation rejects the all-zeros key
this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE;
#endif
#ifdef USE_OTA_ENCRYPTION
// Reserve the noise session before the optional inflate buffer, so the
// required allocation is not starved by the compression window
if (this->noise_offered_()) {
this->noise_reserve_session_();
}
#endif
#ifdef USE_OTA_DEFLATE
// Offered only once the session memory is in hand; else uncompressed
if ((this->ota_features_ & CLIENT_FEATURE_SUPPORTS_DEFLATE) != 0) {
// Value initialized: a corrupt stream that back references the
// window before it is filled then copies zeros, never stale memory.
// Default placement, PSRAM first where present: the session lives for one
// upload and keeps 4.9 KB of internal heap free while it runs
this->inflate_ = RAMAllocator<InflateSession>().make_unique();
if (this->inflate_ != nullptr) {
this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_DEFLATE;
} else {
ESP_LOGW(TAG, "No memory to inflate");
}
}
#endif
} else {
this->handshake_buf_[0] =
@@ -343,8 +380,7 @@ void ESPHomeOTAComponent::handle_handshake_() {
#ifdef USE_OTA_ENCRYPTION
// Latch the offer actually sent: a key activating between the two
// states must not start a session the client never expects
if ((this->handshake_buf_[1] & SERVER_FEATURE_SUPPORTS_NOISE) != 0 &&
(this->ota_features_ & CLIENT_NOISE_FEATURES) == CLIENT_NOISE_FEATURES) {
if (this->noise_offered_()) {
// handshake_buf_ still holds the feature ack composed above; a
// would-block re-entry lands here without rebuilding it
if (!this->noise_start_session_(this->handshake_buf_[1])) {
@@ -442,16 +478,11 @@ void ESPHomeOTAComponent::handle_data_() {
// Backend calls overwrite this with OK; reset to UNKNOWN before any
// goto error that follows a successful begin()/write()
ota::OTAResponseTypes error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN;
size_t total = 0;
uint32_t last_progress = 0;
uint32_t last_data_ms = 0;
DataTransfer xfer;
uint8_t buf[OTA_BUFFER_SIZE];
char *sbuf = reinterpret_cast<char *>(buf);
size_t ota_size;
size_t image_size;
ota::OTAType ota_type = ota::OTA_TYPE_UPDATE_APP;
#if USE_OTA_VERSION == 2
size_t size_acknowledged = 0;
#endif
// Set socket timeouts and blocking mode (see strategy table above)
struct timeval tv;
@@ -480,16 +511,20 @@ void ESPHomeOTAComponent::handle_data_() {
ESP_LOGV(TAG, "OTA type is 0x%02x", ota_type);
// Read size, 4 bytes MSB first
if (!this->data_readall_(buf, 4)) {
if (!this->read_size_(buf, xfer.ota_size, LOG_STR("size"))) {
// The first request byte is the type on the extended protocol; a close after it was a cut-off request
if (!this->extended_proto_() && this->client_left_before_start_())
return;
this->log_read_error_(LOG_STR("size"));
goto error; // NOLINT(cppcoreguidelines-avoid-goto)
}
ota_size = (static_cast<size_t>(buf[0]) << 24) | (static_cast<size_t>(buf[1]) << 16) |
(static_cast<size_t>(buf[2]) << 8) | buf[3];
ESP_LOGV(TAG, "Size is %zu bytes", ota_size);
image_size = xfer.ota_size;
#ifdef USE_OTA_DEFLATE
if (this->inflate_ != nullptr && !this->read_size_(buf, image_size, LOG_STR("image size"))) {
this->log_read_error_(LOG_STR("image size"));
goto error; // NOLINT(cppcoreguidelines-avoid-goto)
}
#endif
#ifndef USE_OTA_PARTITIONS
if (ota_type != ota::OTA_TYPE_UPDATE_APP) {
@@ -509,7 +544,7 @@ void ESPHomeOTAComponent::handle_data_() {
#endif
// begin() returns quickly; flash sectors are erased incrementally during write().
error_code = this->backend_->begin(ota_size, ota_type);
error_code = this->backend_->begin(image_size, ota_type);
if (error_code != ota::OTA_RESPONSE_OK)
goto error; // NOLINT(cppcoreguidelines-avoid-goto)
@@ -529,77 +564,25 @@ void ESPHomeOTAComponent::handle_data_() {
// Acknowledge MD5 OK - 1 byte
this->data_write_byte_(ota::OTA_RESPONSE_BIN_MD5_OK);
// Track when we last received data so a silently-vanished peer (no FIN/RST
// delivered, e.g. uploader killed mid-transfer or NAT/router dropped state)
// can't wedge the device indefinitely. Without this, the loop only exits
// on actual data, EOF, or a non-EWOULDBLOCK error from read(), and lwIP
// TCP keepalive isn't enabled here.
last_data_ms = millis();
while (total < ota_size) {
if (millis() - last_data_ms > OTA_SOCKET_TIMEOUT_DATA) {
ESP_LOGW(TAG, "No data received for %u ms", (unsigned) OTA_SOCKET_TIMEOUT_DATA);
error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN;
xfer.last_data_ms = millis();
#ifdef USE_OTA_DEFLATE
if (this->inflate_ != nullptr) {
error_code = this->inflate_data_(buf, image_size, xfer);
if (error_code != ota::OTA_RESPONSE_OK)
goto error; // NOLINT(cppcoreguidelines-avoid-goto)
}
size_t remaining = ota_size - total;
size_t requested = remaining < OTA_BUFFER_SIZE ? remaining : OTA_BUFFER_SIZE;
ssize_t read;
#ifdef USE_OTA_ENCRYPTION
if (this->noise_ != nullptr) {
// One frame per call; noise_read_data_ waits internally (readall_), so
// there is no would-block retry here and failures are already logged.
read = this->noise_read_data_(buf, requested);
if (read <= 0) {
if (this->remote_closed_)
this->log_remote_closed_(LOG_STR("data"));
} else
#endif
{
while (xfer.total < xfer.ota_size) {
ssize_t read = this->receive_data_(buf, xfer);
if (read < 0) {
error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN;
goto error; // NOLINT(cppcoreguidelines-avoid-goto)
}
} else
#endif
{
read = this->client_->read(buf, requested);
if (read == -1) {
const int err = errno;
if (this->would_block_(err)) {
// read() already waited up to SO_RCVTIMEO for data, just feed WDT
App.feed_wdt();
continue;
}
ESP_LOGW(TAG, "Read err %d", err);
error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN;
error_code = this->write_flash_(buf, read);
if (error_code != ota::OTA_RESPONSE_OK)
goto error; // NOLINT(cppcoreguidelines-avoid-goto)
} else if (read == 0) {
ESP_LOGW(TAG, "Remote closed");
error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN;
goto error; // NOLINT(cppcoreguidelines-avoid-goto)
}
}
last_data_ms = millis();
error_code = this->backend_->write(buf, read);
if (error_code != ota::OTA_RESPONSE_OK) {
ESP_LOGW(TAG, "Flash write err %d", error_code);
goto error; // NOLINT(cppcoreguidelines-avoid-goto)
}
total += read;
#if USE_OTA_VERSION == 2
while (size_acknowledged + OTA_BLOCK_SIZE <= total || (total == ota_size && size_acknowledged < ota_size)) {
this->data_write_byte_(ota::OTA_RESPONSE_CHUNK_OK);
size_acknowledged += OTA_BLOCK_SIZE;
}
#endif
uint32_t now = millis();
if (now - last_progress > 1000) {
last_progress = now;
float percentage = (total * 100.0f) / ota_size;
ESP_LOGD(TAG, "Progress: %0.1f%%", percentage);
#ifdef USE_OTA_STATE_LISTENER
this->notify_state_(ota::OTA_IN_PROGRESS, percentage, 0);
#endif
// feed watchdog and give other tasks a chance to run
this->yield_and_feed_watchdog_();
this->ack_written_(xfer);
}
}
@@ -815,6 +798,90 @@ bool ESPHomeOTAComponent::try_write_(size_t to_write, const LogString *desc) {
return this->handshake_buf_pos_ >= to_write;
}
bool ESPHomeOTAComponent::read_size_(uint8_t *buf, size_t &size, const LogString *desc) {
if (!this->data_readall_(buf, OTA_SIZE_FIELD_BYTES))
return false;
size = encode_uint32(buf[0], buf[1], buf[2], buf[3]);
ESP_LOGV(TAG, "%s is %zu bytes", LOG_STR_ARG(desc), size);
return true;
}
ota::OTAResponseTypes ESPHomeOTAComponent::write_flash_(uint8_t *data, size_t len) {
ota::OTAResponseTypes result = this->backend_->write(data, len);
if (result != ota::OTA_RESPONSE_OK) {
ESP_LOGW(TAG, "Flash write err %d", result);
}
return result;
}
ssize_t ESPHomeOTAComponent::receive_data_(uint8_t *buf, DataTransfer &xfer) {
const size_t remaining = xfer.ota_size - xfer.total;
const size_t requested = std::min(remaining, OTA_BUFFER_SIZE);
ssize_t read;
for (;;) {
// A silently-vanished peer (no FIN/RST delivered, e.g. uploader killed
// mid-transfer or NAT/router dropped state) must not wedge the device:
// read() only fails on EOF or a real error, and lwIP TCP keepalive isn't
// enabled here.
if (millis() - xfer.last_data_ms > OTA_SOCKET_TIMEOUT_DATA) {
ESP_LOGW(TAG, "No data received for %u ms", (unsigned) OTA_SOCKET_TIMEOUT_DATA);
return -1;
}
#ifdef USE_OTA_ENCRYPTION
if (this->noise_ != nullptr) {
// One frame per call; noise_read_data_ waits internally (readall_), so
// there is no would-block retry here and failures are already logged.
read = this->noise_read_data_(buf, requested);
if (read <= 0) {
if (this->remote_closed_)
this->log_remote_closed_(LOG_STR("data"));
return -1;
}
break;
}
#endif
read = this->client_->read(buf, requested);
if (read > 0)
break;
if (read == 0) {
this->log_remote_closed_(LOG_STR("data"));
return -1;
}
if (!this->would_block_(errno)) {
this->log_socket_error_(LOG_STR("data"));
return -1;
}
// read() already waited up to SO_RCVTIMEO for data, just feed WDT
App.feed_wdt();
}
const uint32_t now = millis();
xfer.last_data_ms = now;
xfer.total += read;
this->ack_received_(xfer);
if (now - xfer.last_progress > OTA_PROGRESS_INTERVAL_MS) {
xfer.last_progress = now;
float percentage = (xfer.total * 100.0f) / xfer.ota_size;
ESP_LOGD(TAG, "Progress: %0.1f%%", percentage);
#ifdef USE_OTA_STATE_LISTENER
this->notify_state_(ota::OTA_IN_PROGRESS, percentage, 0);
#endif
// feed watchdog and give other tasks a chance to run
this->yield_and_feed_watchdog_();
}
return read;
}
void ESPHomeOTAComponent::send_chunk_acks_(DataTransfer &xfer) {
#if USE_OTA_VERSION == 2
while (xfer.acknowledged + OTA_BLOCK_SIZE <= xfer.total ||
(xfer.total == xfer.ota_size && xfer.acknowledged < xfer.ota_size)) {
this->data_write_byte_(ota::OTA_RESPONSE_CHUNK_OK);
xfer.acknowledged += OTA_BLOCK_SIZE;
}
#endif
}
void ESPHomeOTAComponent::cleanup_connection_() {
this->client_->close();
this->client_ = nullptr;
@@ -829,6 +896,9 @@ void ESPHomeOTAComponent::cleanup_connection_() {
#endif
#ifdef USE_OTA_ENCRYPTION
this->noise_ = nullptr;
#endif
#ifdef USE_OTA_DEFLATE
this->inflate_ = nullptr;
#endif
// Intentionally no disable_loop() — letting loop() run one more iteration catches
// any connection that queued on the listener mid-session (otherwise the wake flag,
esphome/components/esphome/ota/ota_esphome.h
+66
View File
@@ -7,6 +7,9 @@
#ifdef USE_OTA_ENCRYPTION
#include "esphome/components/noise/noise_handshake.h"
#endif
#ifdef USE_OTA_DEFLATE
#include "ota_esphome_inflate.h"
#endif
#include "esphome/core/helpers.h"
#include "esphome/core/log.h"
#include "esphome/core/preferences.h"
@@ -89,6 +92,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent {
// The api server's live context when it exists, otherwise our own (a build
// time key, or the saved key loaded in safe mode)
const noise::NoiseContext &noise_context_() const;
// True once the feature ack offers noise and the client asked for it
bool noise_offered_() const;
void noise_reserve_session_();
bool noise_start_session_(uint8_t server_feature_flags);
bool handle_noise_handshake_();
bool noise_try_read_frame_();
@@ -120,6 +126,38 @@ class ESPHomeOTAComponent final : public ota::OTAComponent {
return this->readall_(buf, len);
}
// Upload accounting shared by the data loop and the inflate read callback
struct DataTransfer {
size_t ota_size{0}; // bytes the client sends
size_t total{0}; // bytes received so far
#if USE_OTA_VERSION == 2
size_t acknowledged{0};
#endif
uint32_t last_data_ms{0};
uint32_t last_progress{0};
};
// Up to OTA_BUFFER_SIZE bytes into buf; returns bytes read, -1 on failure (logged)
ssize_t receive_data_(uint8_t *buf, DataTransfer &xfer);
// Raw lwIP cannot service the radio during a sector write, so the ack waits
// for the write there; a socket task lets the next block arrive meanwhile
#ifdef USE_SOCKET_IMPL_LWIP_TCP
static constexpr bool ACK_AFTER_WRITE = true;
#else
static constexpr bool ACK_AFTER_WRITE = false;
#endif
void send_chunk_acks_(DataTransfer &xfer);
inline void ack_received_(DataTransfer &xfer) {
if (!ACK_AFTER_WRITE)
this->send_chunk_acks_(xfer);
}
inline void ack_written_(DataTransfer &xfer) {
if (ACK_AFTER_WRITE)
this->send_chunk_acks_(xfer);
}
inline bool read_size_(uint8_t *buf, size_t &size, const LogString *desc);
// Writes to the backend and logs a failure
ota::OTAResponseTypes write_flash_(uint8_t *data, size_t len);
bool try_read_(size_t to_read, const LogString *desc);
bool try_write_(size_t to_write, const LogString *desc);
@@ -175,6 +213,34 @@ class ESPHomeOTAComponent final : public ota::OTAComponent {
static_assert(OTA_BUFFER_SIZE >= NOISE_CLIENT_MAX_PLAINTEXT + noise::MAC_SIZE,
"OTA_BUFFER_SIZE must fit a full encrypted data frame");
#endif
#ifdef USE_OTA_DEFLATE
// At least 1 << espota2.DEFLATE_WINDOW_BITS; also the inflate output buffer
static constexpr size_t OTA_INFLATE_WINDOW_SIZE = 4096;
// Heap-allocated only while a deflate upload is negotiated; the decoder
// state is the base so the read callback can recover the session
struct InflateSession : OtaInflateState {
// The session outlives the upload it serves, but these three are borrowed
// from inflate_data_'s caller and dangle once that call returns; only that
// call, and the flush and read callback it drives, may read them
ESPHomeOTAComponent *self;
DataTransfer *xfer;
uint8_t *in; // caller's buffer for the compressed input
size_t image_size;
size_t written; // inflated bytes in flash
size_t flushed; // bytes of the current window already in flash
ota::OTAResponseTypes error; // first failure inside the read callback
uint8_t window[OTA_INFLATE_WINDOW_SIZE];
};
#ifndef CLANG_TIDY // static analysis sets every define at once
static_assert(!ota::OTABackend::supports_compression(),
"USE_OTA_DEFLATE is for backends that cannot store a gzip image");
#endif
// Writes the decoded bytes not yet in flash without moving dest
ota::OTAResponseTypes inflate_flush_(InflateSession &session);
ota::OTAResponseTypes inflate_data_(uint8_t *in, size_t image_size, DataTransfer &xfer);
RAMUniquePtr<InflateSession> inflate_;
#endif
static constexpr uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45};
// Derived from the feature byte; storing it would pad the trailing bytes
bool extended_proto_() const;
esphome/components/esphome/ota/ota_esphome_inflate.c
+498
View File
@@ -0,0 +1,498 @@
/*
* uzlib - tiny deflate/inflate library (deflate, gzip, zlib)
*
* Copyright (c) 2003 by Joergen Ibsen / Jibz
* All Rights Reserved
* http://www.ibsensoftware.com/
*
* Copyright (c) 2014-2018 by Paul Sokolovsky
*
* This software is provided 'as-is', without any express
* or implied warranty. In no event will the authors be
* held liable for any damages arising from the use of
* this software.
*
* Permission is granted to anyone to use this software
* for any purpose, including commercial applications,
* and to alter it and redistribute it freely, subject to
* the following restrictions:
*
* 1. The origin of this software must not be
* misrepresented; you must not claim that you
* wrote the original software. If you use this
* software in a product, an acknowledgment in
* the product documentation would be appreciated
* but is not required.
*
* 2. Altered source versions must be plainly marked
* as such, and must not be misrepresented as
* being the original software.
*
* 3. This notice may not be removed or altered from
* any source distribution.
*/
/*
* Altered for ESPHome: this is the raw deflate decoder from uzlib's
* tinflate.c (v2.9.5) with the gzip/zlib header parsers, checksums,
* runtime table builder and in-memory (non ring window) output path
* removed, and the public names prefixed with ota_inflate.
*/
#include "ota_esphome_inflate.h"
#include <stddef.h>
#define TINF_OK OTA_INFLATE_OK
#define TINF_DONE OTA_INFLATE_DONE
#define TINF_DATA_ERROR OTA_INFLATE_DATA_ERROR
#define TINF_DICT_ERROR OTA_INFLATE_DICT_ERROR
#define TINF_DATA struct OtaInflateState
#define TINF_TREE struct OtaInflateTree
#define TINF_ARRAY_SIZE(arr) (sizeof(arr) / sizeof(*(arr)))
/* every output byte also goes into the ring window */
#define TINF_PUT(d, c) \
{ \
*d->dest++ = c; \
d->dict_ring[d->dict_idx++] = c; \
if (d->dict_idx == d->dict_size) \
d->dict_idx = 0; \
}
/* --------------------------------------------------- *
* -- constant tables (upstream builds them at runtime) -- *
* --------------------------------------------------- */
static const unsigned char LENGTH_BITS[30] = {0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2,
2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5};
static const unsigned short LENGTH_BASE[30] = {3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27,
31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258};
static const unsigned char DIST_BITS[30] = {0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6,
6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13};
static const unsigned short DIST_BASE[30] = {1, 2, 3, 4, 5, 7, 9, 13, 17, 25,
33, 49, 65, 97, 129, 193, 257, 385, 513, 769,
1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577};
/* special ordering of code length codes */
static const unsigned char CLCIDX[] = {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15};
/* ----------------------- *
* -- utility functions -- *
* ----------------------- */
/* given an array of code lengths, build a tree */
static void tinf_build_tree(TINF_TREE *t, const unsigned char *lengths, unsigned int num) {
unsigned short offs[16];
unsigned int i, sum;
/* clear code length count table */
for (i = 0; i < 16; ++i)
t->table[i] = 0;
/* scan symbol lengths, and sum code length counts */
for (i = 0; i < num; ++i)
t->table[lengths[i]]++;
/* In the lengths array, 0 means unused code. So, t->table[0] now contains
number of unused codes. But table's purpose is to contain # of codes of
particular length, and there're 0 codes of length 0. */
t->table[0] = 0;
/* compute offset table for distribution sort */
for (sum = 0, i = 0; i < 16; ++i) {
offs[i] = sum;
sum += t->table[i];
}
/* create code->symbol translation table (symbols sorted by code) */
for (i = 0; i < num; ++i) {
if (lengths[i])
t->trans[offs[lengths[i]]++] = i;
}
}
/* ---------------------- *
* -- decode functions -- *
* ---------------------- */
static unsigned char uzlib_get_byte(TINF_DATA *d) {
/* If end of source buffer is not reached, return next byte from source
buffer. */
if (d->source < d->source_limit) {
return *d->source++;
}
/* Otherwise if there's callback and we haven't seen EOF yet, try to
read next byte using it. (Note: the callback can also update ->source
and ->source_limit). */
if (!d->eof) {
int val = d->source_read_cb(d);
if (val >= 0) {
return (unsigned char) val;
}
}
/* Otherwise, we hit EOF (either from ->source_read_cb() or from exhaustion
of the buffer), and it will be "sticky", i.e. further calls to this
function will end up here too. */
d->eof = true;
return 0;
}
/* get one bit from source stream */
static int tinf_getbit(TINF_DATA *d) {
unsigned int bit;
/* check if tag is empty */
if (!d->bitcount--) {
/* load next tag */
d->tag = uzlib_get_byte(d);
d->bitcount = 7;
}
/* shift bit out of tag */
bit = d->tag & 0x01;
d->tag >>= 1;
return bit;
}
/* read a num bit value from a stream and add base */
static unsigned int tinf_read_bits(TINF_DATA *d, int num, int base) {
unsigned int val = 0;
/* read num bits */
if (num) {
unsigned int limit = 1 << (num);
unsigned int mask;
for (mask = 1; mask < limit; mask *= 2)
if (tinf_getbit(d))
val += mask;
}
return val + base;
}
/* given a data stream and a tree, decode a symbol */
static int tinf_decode_symbol(TINF_DATA *d, TINF_TREE *t) {
int sum = 0, cur = 0, len = 0;
/* get more bits while code value is above sum */
do {
cur = 2 * cur + tinf_getbit(d);
if (++len == TINF_ARRAY_SIZE(t->table)) {
return TINF_DATA_ERROR;
}
sum += t->table[len];
cur -= t->table[len];
} while (cur >= 0);
sum += cur;
if (sum < 0 || sum >= t->size) {
return TINF_DATA_ERROR;
}
return t->trans[sum];
}
/* given a data stream, decode dynamic trees from it */
static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) {
/* code lengths for 288 literal/len symbols and 32 dist symbols */
unsigned char lengths[288 + 32];
unsigned int hlit, hdist, hclen, hlimit;
unsigned int i, num, length;
/* get 5 bits HLIT (257-286) */
hlit = tinf_read_bits(d, 5, 257);
/* get 5 bits HDIST (1-32) */
hdist = tinf_read_bits(d, 5, 1);
/* get 4 bits HCLEN (4-19) */
hclen = tinf_read_bits(d, 4, 4);
for (i = 0; i < 19; ++i)
lengths[i] = 0;
/* read code lengths for code length alphabet */
for (i = 0; i < hclen; ++i) {
/* get 3 bits code length (0-7) */
unsigned int clen = tinf_read_bits(d, 3, 0);
lengths[CLCIDX[i]] = clen;
}
/* build code length tree, temporarily use length tree */
tinf_build_tree(lt, lengths, 19);
/* decode code lengths for the dynamic trees */
hlimit = hlit + hdist;
for (num = 0; num < hlimit;) {
int sym = tinf_decode_symbol(d, lt);
unsigned char fill_value = 0;
int lbits, lbase = 3;
/* error decoding */
if (sym < 0)
return sym;
switch (sym) {
case 16:
/* copy previous code length 3-6 times (read 2 bits) */
if (num == 0)
return TINF_DATA_ERROR;
fill_value = lengths[num - 1];
lbits = 2;
break;
case 17:
/* repeat code length 0 for 3-10 times (read 3 bits) */
lbits = 3;
break;
case 18:
/* repeat code length 0 for 11-138 times (read 7 bits) */
lbits = 7;
lbase = 11;
break;
default:
/* values 0-15 represent the actual code lengths */
lengths[num++] = sym;
/* continue the for loop */
continue;
}
/* special code length 16-18 are handled here */
length = tinf_read_bits(d, lbits, lbase);
if (num + length > hlimit)
return TINF_DATA_ERROR;
for (; length; --length) {
lengths[num++] = fill_value;
}
}
/* Check that there's "end of block" symbol */
if (lengths[256] == 0) {
return TINF_DATA_ERROR;
}
/* build dynamic trees */
tinf_build_tree(lt, lengths, hlit);
tinf_build_tree(dt, lengths + hlit, hdist);
return TINF_OK;
}
/* build the fixed huffman trees (RFC 1951 3.2.6) through the generic tree
builder; altered from upstream, which unrolls them by hand */
static void tinf_build_fixed_trees(TINF_TREE *lt, TINF_TREE *dt) {
unsigned char lengths[288];
unsigned int i;
for (i = 0; i < 144; ++i)
lengths[i] = 8;
for (; i < 256; ++i)
lengths[i] = 9;
for (; i < 280; ++i)
lengths[i] = 7;
for (; i < 288; ++i)
lengths[i] = 8;
tinf_build_tree(lt, lengths, 288);
for (i = 0; i < 32; ++i)
lengths[i] = 5;
tinf_build_tree(dt, lengths, 32);
}
/* ----------------------------- *
* -- block inflate functions -- *
* ----------------------------- */
/* given a stream and two trees, inflate next chunk of output (a byte or more) */
static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) {
if (d->curlen == 0) {
unsigned int offs;
int dist;
int sym = tinf_decode_symbol(d, lt);
if (d->eof) {
return TINF_DATA_ERROR;
}
if (sym < 0) {
return sym;
}
/* literal byte */
if (sym < 256) {
TINF_PUT(d, sym);
return TINF_OK;
}
/* end of block */
if (sym == 256) {
return TINF_DONE;
}
/* substring from sliding dictionary */
sym -= 257;
if (sym >= 29) {
return TINF_DATA_ERROR;
}
/* possibly get more bits from length code */
d->curlen = tinf_read_bits(d, LENGTH_BITS[sym], LENGTH_BASE[sym]);
dist = tinf_decode_symbol(d, dt);
if (dist < 0 || dist >= 30) {
return TINF_DATA_ERROR;
}
/* possibly get more bits from distance code */
offs = tinf_read_bits(d, DIST_BITS[dist], DIST_BASE[dist]);
/* calculate and validate actual LZ offset to use */
if (offs > d->dict_size) {
return TINF_DICT_ERROR;
}
/* Note: we don't try to catch offset which points to not yet filled
part of the dictionary here. Doing so would require keeping another
variable to track "filled in" size of the dictionary. Appearance of
such an offset cannot lead to accessing memory outside of the
dictionary buffer, and clients which don't want to leak unrelated
information, should explicitly initialize dictionary buffer passed
to uzlib. */
d->lz_off = d->dict_idx - offs;
if (d->lz_off < 0) {
d->lz_off += d->dict_size;
}
}
/* copy next byte from dict substring */
TINF_PUT(d, d->dict_ring[d->lz_off]);
if ((unsigned) ++d->lz_off == d->dict_size) {
d->lz_off = 0;
}
d->curlen--;
return TINF_OK;
}
/* inflate next byte from uncompressed block of data */
static int tinf_inflate_uncompressed_block(TINF_DATA *d) {
if (d->curlen == 0) {
unsigned int length, invlength;
/* get length */
length = uzlib_get_byte(d);
length += 256 * uzlib_get_byte(d);
/* get one's complement of length */
invlength = uzlib_get_byte(d);
invlength += 256 * uzlib_get_byte(d);
/* check length */
if (length != (~invlength & 0x0000ffff))
return TINF_DATA_ERROR;
/* increment length to properly return TINF_DONE below, without
producing data at the same time */
d->curlen = length + 1;
/* make sure we start next block on a byte boundary */
d->bitcount = 0;
}
if (--d->curlen == 0) {
return TINF_DONE;
}
unsigned char c = uzlib_get_byte(d);
TINF_PUT(d, c);
return TINF_OK;
}
/* ---------------------- *
* -- public functions -- *
* ---------------------- */
/* initialize decompression structure */
void ota_inflate_init(TINF_DATA *d, unsigned char *dict, unsigned int dict_len) {
d->source = NULL;
d->source_limit = NULL;
d->tag = 0;
d->eof = 0;
d->bitcount = 0;
d->lz_off = 0;
d->bfinal = 0;
d->btype = -1;
d->dict_size = dict_len;
d->dict_ring = dict;
d->dict_idx = 0;
d->curlen = 0;
d->ltree.trans = d->ltrans;
d->ltree.size = TINF_ARRAY_SIZE(d->ltrans);
d->dtree.trans = d->dtrans;
d->dtree.size = TINF_ARRAY_SIZE(d->dtrans);
}
/* inflate next output bytes from compressed stream */
int ota_inflate(TINF_DATA *d) {
do {
int res;
/* start a new block */
if (d->btype == -1) {
int old_btype;
next_blk:
old_btype = d->btype;
/* read final block flag */
d->bfinal = tinf_getbit(d);
/* read block type (2 bits) */
d->btype = tinf_read_bits(d, 2, 0);
if (d->btype == 1 && old_btype != 1) {
/* build fixed huffman trees */
tinf_build_fixed_trees(&d->ltree, &d->dtree);
} else if (d->btype == 2) {
/* decode trees from stream */
res = tinf_decode_trees(d, &d->ltree, &d->dtree);
if (res != TINF_OK) {
return res;
}
}
}
/* process current block */
switch (d->btype) {
case 0:
/* decompress uncompressed block */
res = tinf_inflate_uncompressed_block(d);
break;
case 1:
case 2:
/* decompress block with fixed/dynamic huffman trees */
/* trees were decoded previously, so it's the same routine for both */
res = tinf_inflate_block_data(d, &d->ltree, &d->dtree);
break;
default:
return TINF_DATA_ERROR;
}
if (res == TINF_DONE && !d->bfinal) {
/* the block has ended (without producing more data), but we
can't return without data, so start procesing next block */
goto next_blk;
}
if (res != TINF_OK) {
return res;
}
} while (d->dest < d->dest_limit);
return TINF_OK;
}
esphome/components/esphome/ota/ota_esphome_inflate.h
+65
View File
@@ -0,0 +1,65 @@
#pragma once
// Raw deflate decoder cut down from uzlib (https://github.com/pfalcon/uzlib,
// zlib licence, see the .c file); output goes through a ring window.
#include <stdbool.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
enum OtaInflateResult {
OTA_INFLATE_OK = 0, /* more data produced, call again */
OTA_INFLATE_DONE = 1, /* end of compressed stream reached */
OTA_INFLATE_DATA_ERROR = -3,
OTA_INFLATE_DICT_ERROR = -5,
};
struct OtaInflateTree {
uint16_t table[16]; /* table of code length counts */
uint16_t *trans; /* code -> symbol translation table, size entries */
uint16_t size;
};
struct OtaInflateState {
/* Next byte in the input buffer and one past its end */
const unsigned char *source;
const unsigned char *source_limit;
/* Called when source is exhausted; returns the next byte or -1 at EOF.
It may refill source/source_limit for buffered operation. */
int (*source_read_cb)(struct OtaInflateState *d);
/* Output cursor and one past the end of the output buffer */
unsigned char *dest;
unsigned char *dest_limit;
/* Ring window holding the last dict_size output bytes for back references */
unsigned char *dict_ring;
unsigned int tag;
unsigned int curlen;
int lz_off;
unsigned int dict_size;
unsigned int dict_idx;
/* One word: btype is -1 between blocks, bitcount never exceeds 7 */
int8_t btype;
uint8_t bfinal;
uint8_t bitcount;
bool eof;
struct OtaInflateTree ltree; /* dynamic length/symbol tree */
struct OtaInflateTree dtree; /* dynamic distance tree */
uint16_t ltrans[288];
uint16_t dtrans[32]; /* the distance alphabet has 30 symbols, so the tree is kept small */
};
/* dict must cover the encoder's window (its largest back reference) */
void ota_inflate_init(struct OtaInflateState *d, unsigned char *dict, unsigned int dict_len);
/* Fills dest up to dest_limit (OK) or to the end of the stream (DONE). dest may
alias dict only if dest_limit - dest == dict_len and dest is reset to dict
exactly when a call returns OK, so the ring index and dest stay in lockstep */
int ota_inflate(struct OtaInflateState *d);
#ifdef __cplusplus
}
#endif
esphome/components/esphome/ota/ota_esphome_inflate_session.cpp
+94
View File
@@ -0,0 +1,94 @@
#include "ota_esphome.h"
#ifdef USE_OTA
#ifdef USE_OTA_DEFLATE
#include "esphome/components/ota/ota_backend.h"
#include "esphome/core/application.h"
#include "esphome/core/log.h"
namespace esphome {
static const char *const TAG = "esphome.ota";
// The window doubles as the output buffer; flushed bytes stay as back
// reference history for the next windowful.
ota::OTAResponseTypes ESPHomeOTAComponent::inflate_flush_(InflateSession &session) {
const size_t produced = session.dest - session.window;
const size_t pending = produced - session.flushed;
if (pending != 0) {
if (pending > session.image_size - session.written) {
ESP_LOGW(TAG, "Inflate overrun");
return ota::OTA_RESPONSE_ERROR_UNKNOWN;
}
ota::OTAResponseTypes result = this->write_flash_(session.window + session.flushed, pending);
if (result != ota::OTA_RESPONSE_OK)
return result;
session.flushed = produced;
session.written += pending;
// A compressible region yields many windows per socket read
App.feed_wdt();
}
// Even with nothing new written: a block boundary can fall inside a header
this->ack_written_(*session.xfer);
return ota::OTA_RESPONSE_OK;
}
ota::OTAResponseTypes ESPHomeOTAComponent::inflate_data_(uint8_t *in, size_t image_size, DataTransfer &xfer) {
InflateSession &session = *this->inflate_;
session.self = this;
session.xfer = &xfer;
session.in = in;
session.image_size = image_size;
session.written = 0;
session.error = ota::OTA_RESPONSE_OK;
ota_inflate_init(&session, session.window, OTA_INFLATE_WINDOW_SIZE);
// Where the ack must follow the write, flush and ack before waiting for
// input, or the client waits for an ack while the decoder waits for data
session.source_read_cb = [](OtaInflateState *d) -> int {
auto *s = static_cast<InflateSession *>(d);
if (ACK_AFTER_WRITE) {
s->error = s->self->inflate_flush_(*s);
if (s->error != ota::OTA_RESPONSE_OK)
return -1;
}
// More input than announced; reported by the size check below
if (s->xfer->total >= s->xfer->ota_size)
return -1;
ssize_t read = s->self->receive_data_(s->in, *s->xfer);
if (read <= 0) {
// Already logged by receive_data_
s->error = ota::OTA_RESPONSE_ERROR_UNKNOWN;
return -1;
}
d->source = s->in + 1;
d->source_limit = s->in + read;
return s->in[0];
};
int res;
do {
// The ring index wrapped to 0 exactly when the window filled
session.dest = session.window;
session.dest_limit = session.window + OTA_INFLATE_WINDOW_SIZE;
session.flushed = 0;
res = ota_inflate(&session);
// A stored block keeps emitting zeros after a failed read, hence eof
if (res < 0 || session.eof)
break;
session.error = this->inflate_flush_(session);
} while (res != OTA_INFLATE_DONE && session.error == ota::OTA_RESPONSE_OK);
// Transport and flash failures are logged where they happen
if (session.error != ota::OTA_RESPONSE_OK)
return session.error;
if (res != OTA_INFLATE_DONE || session.written != image_size || xfer.total != xfer.ota_size) {
ESP_LOGW(TAG, "Inflate err %d, %zu of %zu B from %zu of %zu", res, session.written, image_size, xfer.total,
xfer.ota_size);
return ota::OTA_RESPONSE_ERROR_UNKNOWN;
}
ESP_LOGD(TAG, "Inflated %zu bytes from %zu", session.written, xfer.total);
return ota::OTA_RESPONSE_OK;
}
} // namespace esphome
#endif // USE_OTA_DEFLATE
#endif // USE_OTA
esphome/components/esphome/ota/ota_esphome_noise.cpp
+7 -4
View File
@@ -32,7 +32,12 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() {
}
}
/** Allocate the session and start the responder handshake.
void ESPHomeOTAComponent::noise_reserve_session_() {
// Default placement, PSRAM first where present: the session lives for one upload
this->noise_ = RAMAllocator<NoiseSession>().make_unique();
}
/** Start the responder handshake, on the session reserved at offer time.
*
* The prologue binds the whole plaintext preamble, so any tampering with the
* negotiation (a stripped feature flag, a changed version) breaks the first
@@ -41,9 +46,7 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() {
*/
bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) {
// A provisioned key cleared between the offer and here is not guarded: the
// session runs on the zero key load_psk fills in and fails the client's MAC.
// Default placement, PSRAM first where present: the session only lives for one upload
this->noise_ = RAMAllocator<NoiseSession>().make_unique();
// session runs on the zero key load_psk fills in and fails the client's MAC
static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version
static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1;
static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags
esphome/components/ota/ota_backend.h
+6 -1
View File
@@ -7,6 +7,7 @@
#include <concepts>
#include <cstddef>
#include <cstdint>
#include <type_traits>
#ifdef USE_OTA_STATE_LISTENER
#include <vector>
@@ -102,6 +103,8 @@ enum OTAType : uint8_t {
// - set_update_md5: expected digest of the incoming image, hex string.
// - write: consume the next chunk; end: finalize and mark bootable.
// - abort: safe to call in any state, including after end().
// - supports_compression: constexpr, whether a gzip image is stored as is and
// inflated at reboot.
template<typename T>
concept OTABackendContract = requires(T backend, size_t image_size, uint8_t *data, size_t len, const char *md5) {
{ backend.begin(image_size, OTA_TYPE_UPDATE_APP) } -> std::same_as<OTAResponseTypes>;
@@ -110,7 +113,9 @@ concept OTABackendContract = requires(T backend, size_t image_size, uint8_t *dat
{ backend.write(data, len) } -> std::same_as<OTAResponseTypes>;
{ backend.end() } -> std::same_as<OTAResponseTypes>;
backend.abort();
{ backend.supports_compression() } -> std::same_as<bool>;
{ T::supports_compression() } -> std::same_as<bool>;
// The value must be a constant expression
typename std::bool_constant<T::supports_compression()>;
};
/** Listener interface for OTA state changes.
esphome/components/ota/ota_backend_arduino_libretiny.h
+1 -1
View File
@@ -13,7 +13,7 @@ class ArduinoLibreTinyOTABackend final {
OTAResponseTypes write(uint8_t *data, size_t len);
OTAResponseTypes end();
void abort();
bool supports_compression() { return false; }
static constexpr bool supports_compression() { return false; }
private:
bool md5_set_{false};
esphome/components/ota/ota_backend_arduino_rp2.h
+4 -1
View File
@@ -15,7 +15,10 @@ class ArduinoRP2OTABackend final {
OTAResponseTypes write(uint8_t *data, size_t len);
OTAResponseTypes end();
void abort();
bool supports_compression() { return false; }
// The core's OTA stub inflates a staged gzip image at reboot, on every chip
// from 4.0.3 (ESPHome pins 6.0.0). begin() only sees the gzip size; the
// inflated size is known when the stub reads the trailer.
static constexpr bool supports_compression() { return USE_ARDUINO_VERSION_CODE >= VERSION_CODE(4, 0, 3); }
private:
bool md5_set_{false};
esphome/components/ota/ota_backend_esp8266.h
+1 -1
View File
@@ -20,7 +20,7 @@ class ESP8266OTABackend final {
OTAResponseTypes end();
void abort();
// Compression supported in all ESP8266 Arduino versions ESPHome supports (>= 2.7.0)
bool supports_compression() { return true; }
static constexpr bool supports_compression() { return true; }
protected:
/// Erase flash sector if current address is at sector boundary
esphome/components/ota/ota_backend_esp_idf.h
+1 -1
View File
@@ -33,7 +33,7 @@ class IDFOTABackend final {
OTAResponseTypes write(uint8_t *data, size_t len);
OTAResponseTypes end();
void abort();
bool supports_compression() { return false; }
static constexpr bool supports_compression() { return false; }
protected:
#ifdef USE_OTA_PARTITIONS
esphome/components/ota/ota_backend_factory.h
+3 -2
View File
@@ -25,7 +25,7 @@ struct StubOTABackend {
OTAResponseTypes write(uint8_t *data, size_t len) { return OTA_RESPONSE_ERROR_UNKNOWN; }
OTAResponseTypes end() { return OTA_RESPONSE_ERROR_UNKNOWN; }
void abort() {}
bool supports_compression() { return false; }
static constexpr bool supports_compression() { return false; }
};
std::unique_ptr<StubOTABackend> make_ota_backend();
} // namespace esphome::ota
@@ -33,6 +33,7 @@ std::unique_ptr<StubOTABackend> make_ota_backend();
namespace esphome::ota {
using OTABackendPtr = decltype(make_ota_backend());
static_assert(OTABackendContract<OTABackendPtr::element_type>,
using OTABackend = OTABackendPtr::element_type;
static_assert(OTABackendContract<OTABackend>,
"The platform's OTA backend is missing part of the backend surface (ota_backend.h)");
} // namespace esphome::ota
esphome/components/ota/ota_backend_host.h
+1 -1
View File
@@ -19,7 +19,7 @@ class HostOTABackend final {
OTAResponseTypes write(uint8_t *data, size_t len);
OTAResponseTypes end();
void abort();
bool supports_compression() { return false; }
static constexpr bool supports_compression() { return false; }
protected:
md5::MD5Digest md5_{};
esphome/components/socket/__init__.py
+15 -1
View File
@@ -140,6 +140,17 @@ def require_wake_loop_threadsafe() -> None:
cg.add_define("USE_SOCKET_SELECT_SUPPORT")
def require_ipv4_resolve() -> None:
"""Compile the shared IPv4 lookup; call from a consumer's to_code."""
cg.add_define("USE_SOCKET_IPV4_RESOLVE")
def require_tcp_client_link() -> None:
"""Compile the reconnecting TCP client link; call from a consumer's to_code."""
require_ipv4_resolve()
cg.add_define("USE_SOCKET_TCP_CLIENT_LINK")
CONFIG_SCHEMA = cv.Schema(
{
cv.SplitDefault(
@@ -184,11 +195,14 @@ async def to_code(config: ConfigType) -> None:
# Each implementation file is fully #ifdef'd on the define set in to_code
# for the selected implementation.
# for the selected implementation. The helper files compile only for
# consumers that called the matching require_ function.
FILTER_SOURCE_FILES = filter_source_files_from_defines(
{
"lwip_raw_tcp_impl.cpp": "USE_SOCKET_IMPL_LWIP_TCP",
"bsd_sockets_impl.cpp": "USE_SOCKET_IMPL_BSD_SOCKETS",
"lwip_sockets_impl.cpp": "USE_SOCKET_IMPL_LWIP_SOCKETS",
"ipv4_resolve.cpp": "USE_SOCKET_IPV4_RESOLVE",
"tcp_client_link.cpp": "USE_SOCKET_TCP_CLIENT_LINK",
}
)
esphome/components/socket/tcp_client_link.cpp
+150
View File
@@ -0,0 +1,150 @@
#include "tcp_client_link.h"
#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS)
#include "esphome/core/application.h"
#include "esphome/core/log.h"
#include <algorithm>
#include <cerrno>
namespace esphome::socket {
// After this long in SYN, the stack's own retries are cut short.
static constexpr uint32_t CONNECT_TIMEOUT_MS = 10000;
// Non-blocking options and TCP keepalive for a bridged stream socket.
// Keepalive is best-effort: the raw lwIP implementation (ESP8266, RP2040)
// rejects it, so a half-open link there is only detected by a failed write.
static void set_stream_options(Socket *sock) {
int yes = 1;
sock->setblocking(false);
sock->setsockopt(IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes));
sock->setsockopt(SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes));
#ifdef TCP_KEEPIDLE
int idle = 30;
int interval = 10;
int count = 3;
sock->setsockopt(IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle));
sock->setsockopt(IPPROTO_TCP, TCP_KEEPINTVL, &interval, sizeof(interval));
sock->setsockopt(IPPROTO_TCP, TCP_KEEPCNT, &count, sizeof(count));
#endif
}
void TcpClientLink::begin(const char *tag) {
this->tag_ = tag;
// The first attempt must not wait out a full interval.
this->last_attempt_ms_ = App.get_loop_component_start_time() - this->reconnect_interval_ms_;
}
void TcpClientLink::poll_slow_() {
if (this->sock_ == nullptr) {
this->try_connect_();
return;
}
int err = 0;
switch (poll_connect(*this->sock_, err)) {
case ConnectPollResult::CONNECT_POLL_RESULT_PENDING:
// Give up before the stack's SYN retries do, so the interval stays honest
// and the next attempt resolves the host again.
if (App.get_loop_component_start_time() - this->last_attempt_ms_ >=
std::max(this->reconnect_interval_ms_, CONNECT_TIMEOUT_MS)) {
this->drop_(LOG_STR("Connect failed"), ETIMEDOUT);
}
return;
case ConnectPollResult::CONNECT_POLL_RESULT_ERROR:
this->drop_(LOG_STR("Connect failed"), err);
return;
default:
break;
}
this->connected_ = true;
ESP_LOGI(this->tag_, "Connected to %s:%u", this->host_.c_str(), this->port_);
}
void TcpClientLink::try_connect_() {
if (this->resolved_.consume_failure()) {
this->note_attempt();
return;
}
this->resolved_.start(this->host_.c_str(), this->port_, this->tag_);
if (!this->resolved_.ready()) {
return;
}
struct sockaddr_storage dest;
socklen_t dest_len =
this->resolved_.to_sockaddr(reinterpret_cast<struct sockaddr *>(&dest), sizeof(dest), this->port_);
if (dest_len == 0) {
this->note_attempt();
return;
}
this->sock_ = socket_loop_monitored(dest.ss_family, SOCK_STREAM, IPPROTO_TCP);
if (this->sock_ == nullptr) {
this->drop_(LOG_STR("Connect failed"), errno);
return;
}
set_stream_options(this->sock_.get());
// Starts the pending-connect clock that poll() times out against.
this->note_attempt();
// An immediate success is reported by the next poll(); poll_connect() sees it writable.
if (this->sock_->connect(reinterpret_cast<struct sockaddr *>(&dest), dest_len) != 0 && errno != EINPROGRESS) {
this->drop_(LOG_STR("Connect failed"), errno);
}
}
void TcpClientLink::adopt(std::unique_ptr<Socket> sock) {
this->close();
set_stream_options(sock.get());
this->sock_ = std::move(sock);
this->connected_ = true;
}
ssize_t TcpClientLink::read(uint8_t *buf, size_t len) {
if (!this->connected_) {
return 0;
}
ssize_t count = this->sock_->read(buf, len);
if (count > 0) {
return count;
}
if (count == 0 || (errno != EAGAIN && errno != EWOULDBLOCK)) {
this->drop_(LOG_STR("Connection lost"), count == 0 ? 0 : errno);
return -1;
}
return 0;
}
ssize_t TcpClientLink::write(const uint8_t *buf, size_t len) {
if (!this->connected_ || len == 0) {
return 0;
}
ssize_t sent = this->sock_->write(buf, len);
if (sent >= 0) {
return sent;
}
if (errno == EAGAIN || errno == EWOULDBLOCK) {
return 0;
}
this->drop_(LOG_STR("Connection lost"), errno);
return -1;
}
void TcpClientLink::close() {
if (this->sock_ != nullptr) {
this->sock_->shutdown(SHUT_RDWR);
this->sock_->close();
this->sock_.reset();
}
this->connected_ = false;
this->resolved_.forget();
}
void TcpClientLink::drop_(const LogString *what, int err) {
ESP_LOGW(this->tag_, "%s: %d", LOG_STR_ARG(what), err);
this->close();
this->note_attempt();
}
} // namespace esphome::socket
#endif
esphome/components/socket/tcp_client_link.h
+74
View File
@@ -0,0 +1,74 @@
#pragma once
#include "headers.h"
#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS)
#include "ipv4_resolve.h"
#include "socket.h"
#include "esphome/core/application.h"
#include "esphome/core/log.h"
#include "esphome/core/string_ref.h"
#include <cstdint>
#include <memory>
namespace esphome::socket {
/// A reconnecting TCP stream driven from loop(). Owns the socket, the DNS
/// lookup and the retry backoff. A fatal read/write error closes the link
/// and schedules the next attempt; the caller sees the edge via connected().
class TcpClientLink {
public:
void set_host(const char *host) { this->host_ = StringRef(host); }
void set_port(uint16_t port) { this->port_ = port; }
void set_reconnect_interval(uint32_t ms) { this->reconnect_interval_ms_ = ms; }
const char *host() const { return this->host_.c_str(); }
uint16_t port() const { return this->port_; }
uint32_t reconnect_interval() const { return this->reconnect_interval_ms_; }
/// Call from setup(). tag names this link's log lines.
void begin(const char *tag);
/// Connect state machine; call every loop while acting as a client.
/// Inline no-op while connected or waiting out the backoff.
void poll() {
if (this->connected_ || (this->sock_ == nullptr && this->in_backoff())) {
return;
}
this->poll_slow_();
}
/// Take over an accepted socket (the server side of a bridge).
void adopt(std::unique_ptr<Socket> sock);
/// Returns bytes moved, 0 when nothing can move now, -1 when the link dropped.
ssize_t read(uint8_t *buf, size_t len);
ssize_t write(const uint8_t *buf, size_t len);
/// Close without scheduling a reconnect (shutdown).
void close();
bool connected() const { return this->connected_; }
bool ready() const { return this->sock_ != nullptr && this->sock_->ready(); }
/// Shared retry clock, also usable for a listen socket.
void note_attempt() { this->last_attempt_ms_ = App.get_loop_component_start_time(); }
bool in_backoff() const {
return App.get_loop_component_start_time() - this->last_attempt_ms_ < this->reconnect_interval_ms_;
}
protected:
void poll_slow_();
void try_connect_();
/// Close after a failure, log what and errno, schedule the next attempt.
void drop_(const LogString *what, int err);
StringRef host_;
std::unique_ptr<Socket> sock_;
const char *tag_{nullptr};
uint32_t last_attempt_ms_{0};
uint32_t reconnect_interval_ms_{5000};
Ipv4Resolve resolved_;
uint16_t port_{0};
bool connected_{false};
};
} // namespace esphome::socket
#endif
esphome/components/substitutions/__init__.py
+16 -3
View File
@@ -353,7 +353,7 @@ def resolve_include(
strict_undefined: bool = True,
errors: ErrList | None = None,
) -> Any:
"""Resolve an include, substituting the filename if needed.
"""Resolve an include, substituting the condition and filename if needed.
Note: no path-traversal validation is performed on the resolved filename.
A substitution that resolves to an absolute path will bypass the parent
@@ -362,6 +362,19 @@ def resolve_include(
values (including command-line substitutions), so path restrictions are
an explicit non-goal here.
"""
if isinstance(original_condition := include.condition, str):
condition = str(
_expand_substitutions(
original_condition,
path + ["condition"],
context_vars,
strict_undefined,
errors,
)
)
if condition != original_condition:
include = include.with_condition(condition)
original = include.file
filename = str(
_expand_substitutions(
@@ -372,8 +385,8 @@ def resolve_include(
if substituted:
include = include.with_file(filename)
try:
return include.load()
except esphome.core.EsphomeError as err:
return include.load() if include.should_load() else {}
except (esphome.core.EsphomeError, cv.Invalid) as err:
resolved = f" (expanded from '{original}')" if substituted else ""
raise cv.Invalid(
f"Error including file '{filename}'{resolved}: {err}"
esphome/components/tcp_uart/__init__.py
+67
View File
@@ -0,0 +1,67 @@
import esphome.codegen as cg
from esphome.components import binary_sensor, socket, uart
from esphome.components.const import (
CONF_DATA_BITS,
CONF_HOST,
CONF_PARITY,
CONF_STOP_BITS,
)
import esphome.config_validation as cv
from esphome.const import (
CONF_BAUD_RATE,
CONF_ID,
CONF_PORT,
DEVICE_CLASS_CONNECTIVITY,
ENTITY_CATEGORY_DIAGNOSTIC,
)
from esphome.types import ConfigType
CODEOWNERS = ["@Bascht74"]
DEPENDENCIES = ["network"]
AUTO_LOAD = ["uart", "binary_sensor", "socket"]
MULTI_CONF = True
tcp_uart_ns = cg.esphome_ns.namespace("tcp_uart")
TcpUart = tcp_uart_ns.class_("TcpUart", uart.UARTComponent, cg.Component)
CONF_RECONNECT_INTERVAL = "reconnect_interval"
CONF_CONNECTED = "connected"
CONFIG_SCHEMA = cv.All(
cv.Schema(
{
cv.GenerateID(): cv.declare_id(TcpUart),
cv.Required(CONF_HOST): cv.string,
cv.Required(CONF_PORT): cv.port,
cv.Optional(CONF_BAUD_RATE, default=9600): cv.int_range(min=1),
cv.Optional(CONF_DATA_BITS, default=8): cv.int_range(min=5, max=8),
cv.Optional(CONF_PARITY, default="NONE"): cv.enum(
uart.UART_PARITY_OPTIONS, upper=True
),
cv.Optional(CONF_STOP_BITS, default=1): cv.one_of(1, 2, int=True),
cv.Optional(
CONF_RECONNECT_INTERVAL, default="5s"
): cv.positive_time_period_milliseconds,
cv.Optional(CONF_CONNECTED): binary_sensor.binary_sensor_schema(
device_class=DEVICE_CLASS_CONNECTIVITY,
entity_category=ENTITY_CATEGORY_DIAGNOSTIC,
),
}
).extend(cv.COMPONENT_SCHEMA),
socket.consume_sockets(1, "tcp_uart"),
)
async def to_code(config: ConfigType) -> None:
socket.require_tcp_client_link()
var = cg.new_Pvariable(config[CONF_ID], config[CONF_HOST], config[CONF_PORT])
await cg.register_component(var, config)
cg.add(var.set_reconnect_interval(config[CONF_RECONNECT_INTERVAL]))
# The socket is not clocked. These only satisfy UARTComponent and a consumer check.
cg.add(var.set_baud_rate(config[CONF_BAUD_RATE]))
cg.add(var.set_data_bits(config[CONF_DATA_BITS]))
cg.add(var.set_stop_bits(config[CONF_STOP_BITS]))
cg.add(var.set_parity(config[CONF_PARITY]))
binary_sensors = binary_sensor.sub_binary_sensors(config)
await binary_sensors(CONF_CONNECTED, var.set_connected_sensor)
esphome/components/tcp_uart/tcp_uart.cpp
+131
View File
@@ -0,0 +1,131 @@
#include "tcp_uart.h"
#include "esphome/core/application.h"
#include "esphome/core/log.h"
#include <cinttypes>
#include <cstring>
namespace esphome::tcp_uart {
static const char *const TAG = "tcp_uart";
static constexpr uint32_t DROP_LOG_INTERVAL_MS = 5000;
void TcpUart::setup() {
this->link_.begin(TAG);
if (this->connected_sensor_ != nullptr) {
this->connected_sensor_->publish_state(false);
}
}
void TcpUart::dump_config() {
ESP_LOGCONFIG(TAG,
"TCP UART:\n"
" Host: %s:%u\n"
" Reconnect Interval: %" PRIu32 "ms",
this->link_.host(), this->link_.port(), this->link_.reconnect_interval());
LOG_BINARY_SENSOR(" ", "Connected", this->connected_sensor_);
}
void TcpUart::sync_link_() {
bool up = this->link_.connected();
this->link_was_up_ = up;
if (!up) {
this->rx_start_ = this->rx_end_ = 0;
this->tx_len_ = 0;
}
if (this->connected_sensor_ != nullptr) {
this->connected_sensor_->publish_state(up);
}
}
void TcpUart::read_socket_() {
if (this->rx_start_ != 0) {
this->rx_end_ -= this->rx_start_;
std::memmove(this->rx_, this->rx_ + this->rx_start_, this->rx_end_);
this->rx_start_ = 0;
}
size_t room = RX_BUFFER_SIZE - this->rx_end_;
if (room == 0) {
// Only a read that filled all free space gets here, so rx_pending_ is already set.
return;
}
ssize_t count = this->link_.read(this->rx_ + this->rx_end_, room);
if (count <= 0) {
// A dropped link (-1) is cleaned up by sync_link_() on the next loop.
if (count == 0) {
this->rx_pending_ = false;
}
return;
}
this->rx_end_ += static_cast<uint16_t>(count);
this->rx_pending_ = static_cast<size_t>(count) == room;
}
void TcpUart::flush_tx_() {
ssize_t sent = this->link_.write(this->tx_, this->tx_len_);
if (sent > 0) {
this->tx_len_ -= static_cast<uint16_t>(sent);
std::memmove(this->tx_, this->tx_ + sent, this->tx_len_);
}
}
void TcpUart::loop() {
this->link_.poll();
if (this->link_.connected() != this->link_was_up_) {
this->sync_link_();
}
if (!this->link_was_up_) {
return;
}
if (this->rx_pending_ || this->link_.ready()) {
this->read_socket_();
}
if (this->tx_len_ != 0) {
this->flush_tx_();
}
}
void TcpUart::write_array(const uint8_t *data, size_t len) {
size_t room = this->link_.connected() ? sizeof(this->tx_) - this->tx_len_ : 0;
if (len > room) {
uint32_t now = App.get_loop_component_start_time();
if (this->last_drop_log_ms_ == 0 || now - this->last_drop_log_ms_ >= DROP_LOG_INTERVAL_MS) {
ESP_LOGW(TAG, "%s, dropped %u bytes",
this->link_.connected() ? LOG_STR_LITERAL("TX buffer full") : LOG_STR_LITERAL("Not connected"),
static_cast<unsigned>(len - room));
this->last_drop_log_ms_ = now;
}
len = room;
}
std::memcpy(this->tx_ + this->tx_len_, data, len);
this->tx_len_ += static_cast<uint16_t>(len);
}
bool TcpUart::peek_byte(uint8_t *data) {
if (this->rx_start_ == this->rx_end_) {
return false;
}
*data = this->rx_[this->rx_start_];
return true;
}
bool TcpUart::read_array(uint8_t *data, size_t len) {
if (this->available() < len) {
return false;
}
std::memcpy(data, this->rx_ + this->rx_start_, len);
this->rx_start_ += static_cast<uint16_t>(len);
return true;
}
uart::UARTFlushResult TcpUart::flush() {
this->flush_tx_();
if (this->tx_len_ == 0) {
return uart::UARTFlushResult::UART_FLUSH_RESULT_SUCCESS;
}
return uart::UARTFlushResult::UART_FLUSH_RESULT_TIMEOUT;
}
} // namespace esphome::tcp_uart
esphome/components/tcp_uart/tcp_uart.h
+66
View File
@@ -0,0 +1,66 @@
#pragma once
#include "esphome/components/binary_sensor/binary_sensor.h"
#include "esphome/components/socket/tcp_client_link.h"
#include "esphome/components/uart/uart_component.h"
#include "esphome/core/component.h"
#include <cstdint>
namespace esphome::tcp_uart {
/// TCP client presented as a UART. Bytes are copied unchanged.
class TcpUart : public uart::UARTComponent, public Component {
public:
TcpUart(const char *host, uint16_t port) {
this->link_.set_host(host);
this->link_.set_port(port);
this->rx_buffer_size_ = RX_BUFFER_SIZE;
}
void set_reconnect_interval(uint32_t ms) { this->link_.set_reconnect_interval(ms); }
void set_connected_sensor(binary_sensor::BinarySensor *sensor) { this->connected_sensor_ = sensor; }
void setup() override;
void loop() override;
void dump_config() override;
void on_shutdown() override { this->link_.close(); }
float get_setup_priority() const override { return setup_priority::AFTER_WIFI; }
void write_array(const uint8_t *data, size_t len) override;
bool peek_byte(uint8_t *data) override;
bool read_array(uint8_t *data, size_t len) override;
size_t available() override { return static_cast<size_t>(this->rx_end_ - this->rx_start_); }
// Same room write_array() grants, so consumers can apply backpressure.
size_t available_for_write() override { return this->link_.connected() ? sizeof(this->tx_) - this->tx_len_ : 0; }
uart::UARTFlushResult flush() override;
bool is_connected() override { return this->link_.connected(); }
#if defined(USE_ESP8266) || defined(USE_ESP32)
void load_settings(bool dump_config) override {}
#endif
protected:
void check_logger_conflict() override {}
void sync_link_();
void read_socket_();
void flush_tx_();
static constexpr size_t RX_BUFFER_SIZE = 1024;
static constexpr size_t TX_BUFFER_SIZE = 1024;
socket::TcpClientLink link_;
binary_sensor::BinarySensor *connected_sensor_{nullptr};
uint32_t last_drop_log_ms_{0};
uint16_t tx_len_{0};
// rx_[rx_start_, rx_end_) holds unread bytes; read_socket_() compacts to the front.
uint16_t rx_start_{0};
uint16_t rx_end_{0};
// The link state loop() saw last; edges clear the buffers and publish the sensor.
bool link_was_up_{false};
// A read stopped before EAGAIN. ready() stays false until new data arrives.
bool rx_pending_{false};
uint8_t rx_[RX_BUFFER_SIZE]{};
uint8_t tx_[TX_BUFFER_SIZE]{};
};
} // namespace esphome::tcp_uart
esphome/core/config.py
+13 -2
View File
@@ -7,7 +7,10 @@ from pathlib import Path
from esphome import automation, core
import esphome.codegen as cg
from esphome.config_helpers import filter_source_files_from_platform
from esphome.config_helpers import (
filter_source_files_from_defines,
filter_source_files_from_platform,
)
import esphome.config_validation as cv
from esphome.const import (
CONF_AREA,
@@ -897,7 +900,7 @@ async def to_code(config: ConfigType) -> None:
# Platform-specific source files for core
FILTER_SOURCE_FILES = filter_source_files_from_platform(
_filter_platform_sources = filter_source_files_from_platform(
{
"static_task.cpp": {
PlatformFramework.ESP32_ARDUINO,
@@ -950,3 +953,11 @@ FILTER_SOURCE_FILES = filter_source_files_from_platform(
# as they are only included when needed by the preprocessor
}
)
# util.cpp holds only the out-of-line MQTT connectivity helpers; without
# USE_MQTT the header provides inline stubs and the file must not compile.
_filter_define_sources = filter_source_files_from_defines({"util.cpp": "USE_MQTT"})
def FILTER_SOURCE_FILES() -> list[str]:
return _filter_platform_sources() + _filter_define_sources()
esphome/core/defines.h
+3
View File
@@ -295,6 +295,7 @@
#define USE_RUNTIME_IMAGE_QOI
#define USE_RUNTIME_STATS
#define USE_OTA
#define USE_OTA_DEFLATE
#define USE_OTA_ENCRYPTION
#define USE_OTA_ENCRYPTION_PROVISIONED
#define USE_OTA_ENCRYPTION_REQUIRED
@@ -418,6 +419,8 @@
#define USE_SENDSPIN_VISUALIZER
#define USE_SENDSPIN_PORT 8928 // NOLINT
#define USE_SOCKET_IMPL_BSD_SOCKETS
#define USE_SOCKET_IPV4_RESOLVE
#define USE_SOCKET_TCP_CLIENT_LINK
#define USE_LWIP_FAST_SELECT
#define USE_SPEAKER
esphome/core/util.cpp
+5 -12
View File
@@ -1,23 +1,16 @@
#include "esphome/core/util.h"
#include "esphome/core/application.h"
#include "esphome/core/version.h"
#include "esphome/core/log.h"
// Compiled only on MQTT builds (core FILTER_SOURCE_FILES); without USE_MQTT
// the header provides inline stubs and this file must stay empty, so the
// guard protects builds that compile every source, such as clang-tidy.
#ifdef USE_MQTT
#include "esphome/components/mqtt/mqtt_client.h"
#endif
namespace esphome {
bool mqtt_is_connected() {
#ifdef USE_MQTT
if (mqtt::global_mqtt_client != nullptr) {
return mqtt::global_mqtt_client->is_connected();
}
#endif
return false;
}
bool mqtt_is_connected() { return mqtt::global_mqtt_client != nullptr && mqtt::global_mqtt_client->is_connected(); }
bool remote_is_connected() { return api_is_connected() || mqtt_is_connected(); }
} // namespace esphome
#endif
esphome/core/util.h
+8
View File
@@ -25,9 +25,17 @@ ESPHOME_ALWAYS_INLINE inline bool api_is_connected() { return false; }
#endif
/// Return whether the node has an active connection to an MQTT broker
#ifdef USE_MQTT
bool mqtt_is_connected();
#else
inline bool mqtt_is_connected() { return false; }
#endif
/// Return whether the node has any form of "remote" connection via the API or to an MQTT broker
#ifdef USE_MQTT
bool remote_is_connected();
#else
inline bool remote_is_connected() { return api_is_connected(); }
#endif
} // namespace esphome
esphome/espidf/framework.py
+76 -9
View File
@@ -2,6 +2,7 @@
from collections.abc import Callable
from ctypes.util import find_library
from dataclasses import dataclass, field
import json
import logging
import os
@@ -18,7 +19,7 @@ from esphome.build_helpers.ccache import (
)
from esphome.build_helpers.pch import ccache_pch_env
from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path
from esphome.core import Version
from esphome.core import CORE, Version
from esphome.framework_helpers import (
PathType,
create_venv,
@@ -42,6 +43,21 @@ _LOGGER = logging.getLogger(__name__)
_SCRIPTS_DIR = Path(__file__).parent
DOMAIN = "espidf_framework"
@dataclass
class _FrameworkCache:
tool_paths: dict[Path, tuple[list[str], dict[str, str]]] = field(
default_factory=dict
)
def _cache() -> _FrameworkCache:
if DOMAIN not in CORE.data:
CORE.data[DOMAIN] = _FrameworkCache()
return CORE.data[DOMAIN]
ESPHOME_STAMP_FILE = ".esphome.stamp.json"
@@ -320,15 +336,42 @@ def _raise_script_failure(what: str, root: PathType, stderr: str | None) -> NoRe
)
def _get_idf_version(
idf_framework_root: PathType, env: dict[str, str] | None = None
) -> str:
# What idf_tools.get_idf_version() matches: ``version.txt`` first, then the
# version header. Both give major.minor only.
_IDF_VERSION_TXT_RE = re.compile(r"^v(\d+\.\d+)")
_IDF_VERSION_HEADER_RE = re.compile(
r"^#define\s+ESP_IDF_VERSION_MAJOR\s+(\d+).+?^#define\s+ESP_IDF_VERSION_MINOR\s+(\d+)",
re.DOTALL | re.MULTILINE,
)
def _read_text(path: Path) -> str:
return path.read_text(encoding="utf-8") if path.is_file() else ""
def read_idf_version_txt(idf_framework_root: Path) -> str | None:
"""major.minor from ``version.txt``, as idf_tools reads it."""
if match := _IDF_VERSION_TXT_RE.match(
_read_text(idf_framework_root / "version.txt")
):
return match.group(1)
return None
def read_idf_version_header(idf_framework_root: Path) -> str | None:
"""major.minor from ``esp_idf_version.h``, as idf_tools reads it."""
header = idf_framework_root / "components" / "esp_common" / "include"
if match := _IDF_VERSION_HEADER_RE.search(_read_text(header / "esp_idf_version.h")):
return f"{match.group(1)}.{match.group(2)}"
return None
def _get_idf_version(idf_framework_root: PathType) -> str:
"""
Get the ESP-IDF version from the specified framework root.
Args:
idf_framework_root: Path to the ESP-IDF framework root directory
env: Optional dictionary of environment variables to set
Returns:
String containing ESP-IDF version
@@ -336,9 +379,20 @@ def _get_idf_version(
Raises:
RuntimeError: If ESP-IDF version cannot be determined
"""
root = Path(idf_framework_root)
try:
version = read_idf_version_txt(root) or read_idf_version_header(root)
except (OSError, UnicodeError) as e:
raise RuntimeError(f"Can't get ESP-IDF version of {root}: {e}") from e
if version is None:
raise RuntimeError(f"Can't get ESP-IDF version of {root}")
return version
def idf_tools_version(idf_framework_root: PathType) -> str:
"""The version from the framework's own ``idf_tools``, for the CI drift guard."""
success, stdout, stderr = _run_idf_tools_script(
idf_framework_root, "get_idf_version.py", "ESP-IDF version", env=env
idf_framework_root, "get_idf_version.py", "ESP-IDF version"
)
if stdout:
stdout = stdout.strip()
@@ -362,7 +416,17 @@ def _get_idf_tool_paths(
Raises:
RuntimeError: If ESP-IDF tool paths cannot be determined
The install check and the build environment both resolve the same
framework, so the result is cached per run and the helper script runs
once per build instead of once per caller. The script also reads
``IDF_TOOLS_PATH``; every caller sets it from ``get_idf_tools_path()``,
so the key leaves it out.
"""
cache = _cache().tool_paths
key = Path(idf_framework_root)
if (cached := cache.get(key)) is not None:
return cached
success, stdout, stderr = _run_idf_tools_script(
idf_framework_root, "get_idf_tool_paths.py", "ESP-IDF tool paths", env=env
@@ -373,11 +437,13 @@ def _get_idf_tool_paths(
# Extract json values
try:
data = json.loads(stdout)
return data["paths_to_export"], data["export_vars"]
result = (data["paths_to_export"], data["export_vars"])
except Exception as e:
raise RuntimeError(
f"Can't extract ESP-IDF tool paths of {idf_framework_root}"
) from e
cache[key] = result
return result
def _get_python_version(
@@ -927,6 +993,7 @@ def _check_esphome_idf_framework_install(
# Validate via the managed tool-path resolution, not ``idf_tools.py check``:
# ``check`` probes tools on the system PATH and aborts if any fail to run (e.g. a
# broken Homebrew openocd), which forced a toolchain reinstall on every build.
# The resolved paths stay cached for get_framework_env.
try:
_get_idf_tool_paths(framework_path, env)
install = False
@@ -1041,7 +1108,7 @@ def _check_esp_idf_python_env_install(
create_venv(python_env_path, msg=f"ESP-IDF {version}")
esp_idf_version = _get_idf_version(framework_path, env=env)
esp_idf_version = _get_idf_version(framework_path)
constraint_file_path = (
get_idf_tools_path() / f"espidf.constraints.v{esp_idf_version}.txt"
)
@@ -1282,7 +1349,7 @@ def get_framework_env(
# 4. Set framework-specific environment variables
env["IDF_PATH"] = str(framework_path)
env["ESP_IDF_VERSION"] = _get_idf_version(framework_path, env)
env["ESP_IDF_VERSION"] = _get_idf_version(framework_path)
# 5. Get and add tool paths and environment variables
paths_to_export, export_vars = _get_idf_tool_paths(framework_path, env)
esphome/espidf/size_summary.py
+1 -1
View File
@@ -1,6 +1,6 @@
"""PlatformIO-format RAM/Flash one-liners after a native ESP-IDF build.
The ninja ``size`` target (run after ``all`` in
The ninja ``size`` target (built together with ``all`` in
``toolchain.run_compile``) prints the per-region table inline as part
of the build. This module adds two summary lines underneath,
byte-identical to PlatformIO's output:
esphome/espidf/toolchain.py
+15 -12
View File
@@ -366,7 +366,9 @@ def _tool_env() -> dict[str, str]:
return env
def run_reconfigure(verbose: bool = False) -> int:
def run_reconfigure(
verbose: bool = False, extra_env: dict[str, str] | None = None
) -> int:
"""Run the CMake configure, with the arguments idf.py uses."""
build_dir = _build_dir()
build_dir.mkdir(parents=True, exist_ok=True)
@@ -389,7 +391,7 @@ def run_reconfigure(verbose: bool = False) -> int:
rc = run_build_tool(
cmd,
cwd=build_dir,
env=_tool_env(),
env={**_tool_env(), **(extra_env or {})},
filter_lines=None if verbose else FILTER_IDF_LINES,
log_path=log_path,
)
@@ -428,21 +430,21 @@ def _build_jobs(config) -> int | None:
def _run_ninja(
target: str,
*,
*targets: str,
verbose: bool,
jobs: int | None,
progress: bool = False,
extra_env: dict[str, str] | None = None,
) -> int:
"""Build one ninja target, with the flags and env idf.py uses."""
"""Build ninja targets in one run, with the flags and env idf.py uses."""
cmd = [_get_idf_tool("ninja")]
if jobs is not None:
cmd += ["-j", str(jobs)]
if verbose:
cmd.append("-v")
cmd.append(target)
log_path = _build_dir() / "log" / f"ninja_{Path(target).name}_output.log"
cmd += targets
log_name = "_".join(Path(t).name for t in targets)
log_path = _build_dir() / "log" / f"ninja_{log_name}_output.log"
rc = run_build_tool(
cmd,
cwd=_build_dir(),
@@ -452,7 +454,7 @@ def _run_ninja(
log_path=log_path,
)
if rc != 0:
_LOGGER.error("ninja %s failed with exit code %d", target, rc)
_LOGGER.error("ninja %s failed with exit code %d", " ".join(targets), rc)
_print_hints(log_path)
return rc
@@ -845,10 +847,11 @@ def run_compile(config, verbose: bool) -> int:
write_pch_checksum()
# idf.py's ``build size``, minus the second ``ninja all`` it runs first.
rc = _run_ninja("all", verbose=verbose, jobs=jobs, progress=True)
if rc == 0:
rc = _run_ninja("size", verbose=verbose, jobs=jobs, extra_env=_size_env())
# idf.py's ``build size`` in one ninja run; size needs the map, so it
# runs after the link.
rc = _run_ninja(
"all", "size", verbose=verbose, jobs=jobs, progress=True, extra_env=_size_env()
)
if rc == 0:
size_json = CORE.relative_build_path("build", "esp_idf_size.json")
partitions = CORE.relative_build_path("partitions.csv")
esphome/espota2.py
+28 -10
View File
@@ -65,9 +65,15 @@ CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01
CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02
CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04
CLIENT_FEATURE_SUPPORTS_NOISE = 0x08
CLIENT_FEATURE_SUPPORTS_DEFLATE = 0x10
SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01
SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02
SERVER_FEATURE_SUPPORTS_NOISE = 0x04
# Binding once offered: the device then expects the image size and a deflate stream
SERVER_FEATURE_SUPPORTS_DEFLATE = 0x08
# Wire constant: the deflate bit promises a 4 KB window (OTA_INFLATE_WINDOW_SIZE)
DEFLATE_WINDOW_BITS = 12
NOISE_FRAME_INDICATOR = 0x01
NOISE_HANDSHAKE_OK = 0x00
@@ -87,6 +93,9 @@ _SUPPORTED_OTA_TYPES: frozenset[int] = frozenset(
)
UPLOAD_BLOCK_SIZE = 8192
# Sizes on the wire are 4 bytes MSB first
SIZE_FIELD_BYTES = 4
COMPRESS_LEVEL = 9
UPLOAD_BUFFER_SIZE = UPLOAD_BLOCK_SIZE * 8
# Flaky Wi-Fi links often drop the first OTA attempt, and the device may need time
@@ -573,6 +582,7 @@ def perform_ota(
CLIENT_FEATURE_SUPPORTS_COMPRESSION
| CLIENT_FEATURE_SUPPORTS_SHA256_AUTH
| CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL
| CLIENT_FEATURE_SUPPORTS_DEFLATE
)
if noise_psk:
features_to_send |= CLIENT_FEATURE_SUPPORTS_NOISE
@@ -675,8 +685,18 @@ def perform_ota(
f"retry {flag_name}."
)
if features & SERVER_FEATURE_SUPPORTS_COMPRESSION:
upload_contents = gzip.compress(file_contents, compresslevel=9)
deflate = bool(extended_proto and features & SERVER_FEATURE_SUPPORTS_DEFLATE)
if deflate:
import zlib
# The device inflates while receiving through a small ring window
upload_contents = zlib.compress(
file_contents, COMPRESS_LEVEL, wbits=-DEFLATE_WINDOW_BITS
)
_LOGGER.info("Compressed to %s bytes (deflate)", len(upload_contents))
elif features & SERVER_FEATURE_SUPPORTS_COMPRESSION:
# The device stores the gzip file and inflates it when it reboots
upload_contents = gzip.compress(file_contents, compresslevel=COMPRESS_LEVEL)
_LOGGER.info("Compressed to %s bytes", len(upload_contents))
else:
upload_contents = file_contents
@@ -735,22 +755,20 @@ def perform_ota(
send_check(sock, ota_type, "ota type")
upload_size = len(upload_contents)
upload_size_encoded = [
(upload_size >> 24) & 0xFF,
(upload_size >> 16) & 0xFF,
(upload_size >> 8) & 0xFF,
(upload_size >> 0) & 0xFF,
]
# The device erases flash between receiving the size and acking the
# prepare, so this window shows the erase cost (near zero when the
# device erases lazily during the upload)
prepare_start = time.perf_counter()
send_check(sock, upload_size_encoded, "binary size")
send_check(sock, upload_size.to_bytes(SIZE_FIELD_BYTES, "big"), "binary size")
if deflate:
# Own frame: an encrypted session carries one field per frame
send_check(sock, file_size.to_bytes(SIZE_FIELD_BYTES, "big"), "image size")
receive_exactly(sock, 1, "update prepare result", RESPONSE_UPDATE_PREPARE_OK)
prepare_duration = time.perf_counter() - prepare_start
_LOGGER.info("Preparing for upload took %.2f seconds", prepare_duration)
upload_md5 = hashlib.md5(upload_contents).hexdigest()
# The device hashes what it writes: the inflated image, else the received bytes
upload_md5 = hashlib.md5(file_contents if deflate else upload_contents).hexdigest()
_LOGGER.debug("MD5 of upload is %s", upload_md5)
send_check(sock, upload_md5, "file checksum")
esphome/yaml_util.py
+90 -24
View File
@@ -35,7 +35,7 @@ from esphome.core import (
TimePeriod,
)
from esphome.expression import has_substitution_or_expression
from esphome.helpers import add_class_to_obj
from esphome.helpers import FALSY_BOOL_STRINGS, TRUTHY_BOOL_STRINGS, add_class_to_obj
from esphome.util import OrderedDict, filter_yaml_files
_LOGGER = logging.getLogger(__name__)
@@ -75,6 +75,12 @@ def take_dropped_merge_keys() -> list[tuple[str, str]]:
return CORE.data.pop(_MERGE_WARNINGS_KEY, [])
def _raise_invalid(message: str) -> None:
from voluptuous import Invalid
raise Invalid(message)
class SensitiveStr(str):
"""Marker subclass for validated strings that should be masked in
user-visible YAML output. ``cv.sensitive`` wraps validated values in this
@@ -224,16 +230,18 @@ class IncludeFile:
Created during YAML parsing instead of loading the file immediately,
allowing substitution variables to appear in the filename path
(e.g. ``!include device-${platform}.yaml``). The actual file is
loaded on the first call to ``load()``, and the result is cached.
(e.g. ``!include device-${platform}.yaml``) and in an optional condition.
The actual file is loaded on the first call to ``load()``, and the result
is cached. The client is responsible for testing the condition if needed.
"""
def __init__(
self,
parent_file: Path,
file: str,
vars: dict[str, Any] | None,
yaml_loader: Callable[[Path], Any],
vars: dict[str, Any] | None = None,
condition: bool | str | None = None,
) -> None:
self.parent_file = parent_file
# The raw include text may be a substitution/Jinja expression, so it
@@ -241,8 +249,9 @@ class IncludeFile:
# rewrites "/" to "\", which Jinja then decodes as escapes like
# "\b" -> backspace (issue #18545).
self.file = file
self.vars = vars
self.yaml_loader = yaml_loader
self.vars = vars
self.condition = condition
self._content: Any = _UNSET
def __repr__(self) -> str:
@@ -256,23 +265,62 @@ class IncludeFile:
"""
if self._content is not _UNSET:
return self._content
if self.has_unresolved_expressions():
from voluptuous import Invalid
raise Invalid(
if self.has_unresolved_file():
_raise_invalid(
f"Cannot load include with unresolved substitutions: {self.file}"
)
self._content = self.yaml_loader(self.parent_file.parent / self.file)
self._content = add_context(self._content, self.vars)
return self._content
def has_unresolved_expressions(self) -> bool:
def has_unresolved_file(self) -> bool:
"""Check if the filename contains substitution variables or Jinja expressions."""
return has_substitution_or_expression(self.file)
def should_load(self) -> bool:
"""Evaluates the condition and returns True if the file should be loaded."""
if self.condition is None:
return True
if isinstance(self.condition, bool):
return self.condition
if self.has_unresolved_condition():
_raise_invalid(
f"Cannot evaluate include condition for '{self.file}' with unresolved substitutions: {self.condition}"
)
value = self.condition.lower()
if value in TRUTHY_BOOL_STRINGS:
return True
if value not in FALSY_BOOL_STRINGS:
_raise_invalid(
f"Cannot convert include condition for '{self.file}' to a boolean, please use 'true' or 'false': {self.condition}"
)
return False
def has_unresolved_condition(self) -> bool:
"""Check if the condition contains substitution variables or Jinja expressions."""
return isinstance(self.condition, str) and has_substitution_or_expression(
self.condition
)
def with_file(self, file: str) -> IncludeFile:
"""Clone this include with *file* as the filename."""
return IncludeFile(self.parent_file, file, self.vars, self.yaml_loader)
return IncludeFile(
self.parent_file,
file,
self.yaml_loader,
vars=self.vars,
condition=self.condition,
)
def with_condition(self, condition: bool | str | None) -> IncludeFile:
"""Clone this include with *condition* as the condition."""
return IncludeFile(
self.parent_file,
self.file,
self.yaml_loader,
vars=self.vars,
condition=condition,
)
def _is_visible_path(rel: Path) -> bool:
@@ -434,7 +482,7 @@ def force_load_include_files(
if id(obj) in _seen:
return
_seen.add(id(obj))
if obj.has_unresolved_expressions():
if obj.has_unresolved_file():
_load_include_candidates(
obj,
warn_on_unresolved=warn_on_unresolved,
@@ -578,14 +626,21 @@ def _resolve_merge_include(value: Any, node: yaml.Node, value_node: yaml.Node) -
for _ in range(_MAX_MERGE_INCLUDE_DEPTH):
if not isinstance(value, IncludeFile):
break
if value.has_unresolved_expressions():
if value.has_unresolved_file():
raise yaml.constructor.ConstructorError(
"While constructing a mapping",
node.start_mark,
"Substitution in include filename with merge keys is not supported yet.",
value_node.start_mark,
)
value = value.load()
if value.has_unresolved_condition():
raise yaml.constructor.ConstructorError(
"While constructing a mapping",
node.start_mark,
"Substitution in include condition with merge keys is not supported yet.",
value_node.start_mark,
)
value = value.load() if value.should_load() else {}
else:
raise yaml.constructor.ConstructorError(
"While constructing a mapping",
@@ -791,11 +846,11 @@ class ESPHomeLoaderMixin:
@_add_data_ref
def construct_include(self, node: yaml.Node) -> Any:
from esphome.const import CONF_VARS
from esphome.const import CONF_CONDITION, CONF_FILE, CONF_VARS
def extract_file_vars(node):
def extract_fields(node):
fields = self.construct_yaml_map(node)
file = fields.get("file")
file = fields.get(CONF_FILE)
if file is None:
raise yaml.MarkedYAMLError("Must include 'file'", node.start_mark)
if not isinstance(file, str):
@@ -803,14 +858,21 @@ class ESPHomeLoaderMixin:
"Include 'file' must be a string", node.start_mark
)
vars = fields.get(CONF_VARS)
return file, vars
condition = fields.get(CONF_CONDITION)
if condition is not None and not isinstance(condition, (bool, str)):
raise yaml.MarkedYAMLError(
"Include 'condition' must be a boolean or string", node.start_mark
)
return file, vars, condition
if isinstance(node, yaml.nodes.MappingNode):
file, vars = extract_file_vars(node)
file, vars, condition = extract_fields(node)
else:
file, vars = node.value, None
file, vars, condition = node.value, None, None
return IncludeFile(self.name, file, vars, self.yaml_loader)
return IncludeFile(
self.name, file, self.yaml_loader, vars=vars, condition=condition
)
# Directory includes (!include_dir_*) load eagerly during YAML parsing
# because their paths are directory names, not individual files, and
@@ -967,7 +1029,7 @@ def _load_yaml_internal(fname: Path, *, track_document_range: bool = True) -> An
# Top-level !include returns a deferred IncludeFile; resolve it so
# callers always receive the final content.
if isinstance(res, IncludeFile):
res = res.load()
res = res.load() if res.should_load() else {}
return res
@@ -1358,8 +1420,12 @@ class ESPHomeDumper(yaml.SafeDumper):
return self.represent_scalar(tag="!remove", value=value.value)
def represent_include_file(self, value):
if value.vars:
mapping = {"file": value.file, "vars": value.vars}
if value.vars or value.condition is not None:
mapping = {"file": value.file}
if value.vars:
mapping["vars"] = value.vars
if value.condition is not None:
mapping["condition"] = value.condition
return self.represent_mapping(
tag="!include", mapping=mapping, flow_style=False
)
script/build_helpers.py
+15 -2
View File
@@ -46,6 +46,8 @@ EXIT_SKIPPED = 1
EXIT_COMPILE_ERROR = 2
EXIT_CONFIG_ERROR = 3
EXIT_NO_EXECUTABLE = 4
# A test folder with this name would be synced into src/esphome and swept away with the core tree
CORE_TREE_DIR = "esphome"
# Name of the per-component YAML config file in benchmark directories
BENCHMARK_YAML_FILENAME = "benchmark.yaml"
@@ -465,8 +467,19 @@ def build_and_run(
components = sorted(components)
# Build include list: main entry point + component folders + extra dirs
includes: list[str] = [main_entry] + components
# Build include list: main entry point + component folders + extra dirs. The core tree
# folder is listed file by file, nested files included, since a folder include would
# land in src/esphome (see CORE_TREE_DIR)
includes: list[str] = [main_entry]
for component in components:
if component != CORE_TREE_DIR:
includes.append(component)
continue
includes.extend(
str(path.relative_to(tests_dir))
for path in sorted((tests_dir / component).rglob("*"))
if path.suffix in (".cpp", ".h")
)
if extra_include_dirs:
for d in extra_include_dirs:
if d.is_dir() and (any(d.glob("*.cpp")) or any(d.glob("*.h"))):
script/check_idf_py_equivalence.py
+72 -3
View File
@@ -55,6 +55,19 @@ MACRO_CHANGED = (
"IDF changed __build_process_project_includes; update "
"IDF_BOOTLOADER_OVERRIDE in esphome/build_gen/espidf.py"
)
VERSION_DRIFT = (
"ESPHome reads ESP-IDF version {ours!r} from {source} but idf_tools reports "
"{theirs!r}; update read_idf_version_{source} in esphome/espidf/framework.py"
)
LWIP_NOT_EMPTY = (
"lwip source {source} compiles to a non-empty object with {option} off; "
"drop it from LWIP_EMPTY_SOURCES in esphome/build_gen/espidf.py"
)
LWIP_NOTHING_MATCHED = (
"no lwip object matched {regex!r} for {option}; the lwip layout or the "
"pattern in esphome/build_gen/espidf.py changed"
)
LWIP_NM_FAILED = "nm failed on lwip object {source}: {error}"
WORK_SUFFIXES = (".obj", ".o", ".a", ".elf", ".map", ".bin", ".ld")
DEFAULT_GLOB = "tests/test_build_components/build/.esphome/build/*"
@@ -104,6 +117,51 @@ def _log_problems(
return problems
def _lwip_empty_source_problems(build_path: Path) -> list[str]:
"""Compile the lwip sources the generated CMakeLists drops; any with
symbols is a problem. Leaves the tree configured with every source."""
# pylint: disable=protected-access
from esphome.build_gen.espidf import LWIP_EMPTY_SOURCES, LWIP_FULL_SOURCES_ENV
from esphome.espidf import toolchain
if (rc := toolchain.run_reconfigure(extra_env={LWIP_FULL_SOURCES_ENV: "1"})) != 0:
return [f"CMake configure with every lwip source failed with exit code {rc}"]
if rc := toolchain._run_ninja("esp-idf/lwip/liblwip.a", verbose=False, jobs=None):
return [f"building every lwip source failed with exit code {rc}"]
build = build_path / "build"
config = json.loads(
(build / "config" / "sdkconfig.json").read_text(encoding="utf-8")
)
objects = [
obj.as_posix().removesuffix(".obj")
for obj in (build / "esp-idf" / "lwip").rglob("*.obj")
]
nm = toolchain._parse_cmakecache(build / "CMakeCache.txt")["CMAKE_NM"]
problems = []
for option, regex in LWIP_EMPTY_SOURCES:
# Absent means the option is invisible here; the filter keeps those.
if config.get(option.removeprefix("CONFIG_"), True):
continue
matched = [source for source in objects if re.search(regex, source)]
if not matched:
problems.append(LWIP_NOTHING_MATCHED.format(regex=regex, option=option))
for source in matched:
name = Path(source).name
result = subprocess.run(
[nm, "--defined-only", f"{source}.obj"],
capture_output=True,
text=True,
check=False,
)
if result.returncode:
problems.append(
LWIP_NM_FAILED.format(source=name, error=result.stderr.strip())
)
elif result.stdout.strip():
problems.append(LWIP_NOT_EMPTY.format(source=name, option=option))
return problems
def _setup_core(build_path: Path, description: dict) -> tuple[str, str]:
"""Point CORE at the tree so ESPHome resolves the same IDF env as the build."""
from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT
@@ -127,7 +185,7 @@ def check(build_path: Path) -> list[str]:
# pylint: disable=protected-access
from esphome.build_gen.espidf import idf_macro_matches
from esphome.core import CORE
from esphome.espidf import toolchain
from esphome.espidf import framework, toolchain
description = json.loads(
(build_path / "build" / "project_description.json").read_text(encoding="utf-8")
@@ -139,8 +197,18 @@ def check(build_path: Path) -> list[str]:
CORE.skip_bootloader = skip_bootloader
# A prior tree's memoized decision must not leak into this one.
toolchain._cache().skip_bootloader = None
if not idf_macro_matches(toolchain._get_idf_path(version)):
idf_path = toolchain._get_idf_path(version)
if not idf_macro_matches(idf_path):
return [MACRO_CHANGED]
# A managed tree always has version.txt, so the header branch is
# compared on its own or it would never be exercised here.
theirs = framework.idf_tools_version(idf_path)
for source, read in (
("txt", framework.read_idf_version_txt),
("header", framework.read_idf_version_header),
):
if (ours := read(idf_path)) != theirs:
return [VERSION_DRIFT.format(ours=ours, source=source, theirs=theirs)]
# ESP-IDF's openthread stamps the configure time into its compile flags;
# pin it before the env is cached so both configures get the same value.
os.environ["SOURCE_DATE_EPOCH"] = "0"
@@ -188,7 +256,8 @@ def check(build_path: Path) -> list[str]:
problems.append(f"idf.py dropped {out} from {log}")
elif mtimes_before.get(key) != mtimes_after[key]:
problems.append(f"idf.py rebuilt {out}")
return problems
# Last: it reconfigures the tree, which would otherwise relink above.
return problems or _lwip_empty_source_problems(build_path)
def main() -> int:
script/ci-custom.py
+2
View File
@@ -904,6 +904,8 @@ def lint_relative_py_import(fname: Path, line, col, content):
# neither can live in a C++ namespace.
"esphome/components/esp32_hosted/esp_now_hosted.cpp",
"esphome/components/esp32_hosted/esp_now_hosted_rpc.h",
# C header shared with the vendored decoder
"esphome/components/esphome/ota/ota_esphome_inflate.h",
],
)
def lint_namespace(fname: Path, content: str) -> str | None:
tests/components/esphome/__init__.py
+12
View File
@@ -0,0 +1,12 @@
from esphome.loader import FileResource
from tests.testing_helpers import ComponentManifestOverride
def override_manifest(manifest: ComponentManifestOverride) -> None:
# to_code emits the component count the application needs
manifest.enable_codegen()
# Only the decoder is under test; its ota platform is not in this build
manifest.resources = manifest.resources + [
FileResource("esphome.components.esphome", "ota/ota_esphome_inflate.c"),
FileResource("esphome.components.esphome", "ota/ota_esphome_inflate.h"),
]
tests/components/esphome/test_ota_inflate.cpp
+324
View File
@@ -0,0 +1,324 @@
#include <gtest/gtest.h>
#include <cstdint>
#include <cstring>
#include <vector>
#include "esphome/components/esphome/ota/ota_esphome_inflate.h"
namespace esphome::testing {
// build_plain() compressed with the CLI's window (espota2.DEFLATE_WINDOW_BITS):
// DEFLATED = zlib.compress(plain, 9, wbits=-12)
// STORED = zlib.compress(plain[:300], 0, wbits=-12)
static const uint8_t DEFLATED[] = {
0xed, 0xc8, 0xf7, 0x3f, 0xd4, 0x0f, 0x03, 0x00, 0x70, 0x67, 0xaf, 0x4b, 0x67, 0x66, 0x9f, 0x90, 0x91, 0x11, 0xc2,
0x11, 0x91, 0xb8, 0xb3, 0xf7, 0x3a, 0xd9, 0x5f, 0x4e, 0x99, 0x67, 0x1e, 0xce, 0x8a, 0xac, 0xec, 0x59, 0xb8, 0xc2,
0x95, 0x5d, 0x56, 0x42, 0x67, 0x73, 0x46, 0xf6, 0xca, 0xce, 0xc8, 0xc8, 0xc8, 0x91, 0x8a, 0x7c, 0x2f, 0x7a, 0xfe,
0x86, 0xe7, 0x87, 0xe7, 0x87, 0xe7, 0xf5, 0xfa, 0xbc, 0x7f, 0x7c, 0xbb, 0x07, 0xa2, 0x1f, 0xfa, 0xf9, 0xb8, 0x43,
0xfd, 0x82, 0x5c, 0xa0, 0x6e, 0xee, 0x28, 0x6f, 0x97, 0x20, 0x77, 0xa8, 0x3b, 0x70, 0xc0, 0x01, 0x07, 0x1c, 0x70,
0xc0, 0x01, 0x07, 0x1c, 0x70, 0xc0, 0x01, 0x07, 0x1c, 0x70, 0xc0, 0x01, 0x07, 0x1c, 0x70, 0xc0, 0x01, 0x07, 0x1c,
0x70, 0xc0, 0x01, 0xf7, 0x5f, 0xdd, 0x40, 0xd4, 0x63, 0x2f, 0x03, 0xf2, 0x17, 0xb2, 0xe5, 0x69, 0xc7, 0x5a, 0x04,
0xdf, 0x46, 0x22, 0xa8, 0x4b, 0x6e, 0xd5, 0x3a, 0x80, 0x05, 0xe1, 0x81, 0x4a, 0x44, 0x56, 0x27, 0xa9, 0x21, 0xf7,
0xad, 0xd9, 0xb0, 0xd3, 0xf1, 0xa5, 0x0b, 0x6a, 0x96, 0x56, 0xb2, 0xca, 0xb5, 0xee, 0x0f, 0x96, 0x28, 0xc3, 0x9e,
0x8f, 0x6e, 0xf2, 0x84, 0xa4, 0x3b, 0x2d, 0x16, 0x64, 0x08, 0x6a, 0x28, 0x91, 0xee, 0x87, 0x1a, 0x81, 0xff, 0xed,
0x2c, 0x5f, 0xda, 0x4a, 0x48, 0x5f, 0x91, 0xf0, 0x31, 0xfe, 0x6b, 0xbd, 0x81, 0x86, 0x92, 0x33, 0x1a, 0x6c, 0x69,
0x32, 0xfb, 0x19, 0x56, 0x2c, 0xc6, 0xaa, 0xef, 0xe8, 0x16, 0x98, 0xcf, 0x98, 0xbf, 0xa0, 0x2d, 0xde, 0x7f, 0xdf,
0x4b, 0xcb, 0xf6, 0x5c, 0xaf, 0x11, 0x24, 0xf0, 0x46, 0x3e, 0x49, 0x0e, 0x67, 0x0e, 0xcf, 0xf3, 0x57, 0xca, 0xb0,
0x39, 0x55, 0xe1, 0xe7, 0xfc, 0x37, 0x29, 0xe8, 0xd7, 0xf3, 0x08, 0x2e, 0xfb, 0x7b, 0x6d, 0x3e, 0xfe, 0xe9, 0x2c,
0x68, 0xe4, 0x20, 0x88, 0x57, 0x56, 0x41, 0x3d, 0xab, 0x9b, 0xbf, 0x0c, 0x0c, 0xae, 0x51, 0x48, 0x8b, 0x72, 0xcc,
0xb8, 0xbb, 0xf3, 0x30, 0xa8, 0x5c, 0xb5, 0xa1, 0x2f, 0x07, 0x52, 0xe9, 0x36, 0xb8, 0x3c, 0xbc, 0xee, 0xbc, 0xf1,
0xa3, 0x8b, 0x81, 0xc2, 0x03, 0xbf, 0x29, 0x5a, 0x22, 0x24, 0x97, 0xf8, 0xc9, 0xb5, 0xbf, 0xd2, 0x24, 0x4a, 0x86,
0xc1, 0x2b, 0xb7, 0xb8, 0xde, 0xa7, 0xea, 0xa5, 0x1d, 0x13, 0x1c, 0x1d, 0xd3, 0x3c, 0x81, 0x60, 0x2e, 0x2f, 0x93,
0x5c, 0x78, 0x43, 0x2e, 0x39, 0x68, 0x09, 0x13, 0x09, 0x10, 0xce, 0xd6, 0x8d, 0xe9, 0x2f, 0xaf, 0x88, 0x6f, 0x99,
0x4f, 0xcd, 0xdc, 0xaa, 0xf9, 0x47, 0xdb, 0x1c, 0xb5, 0x89, 0x53, 0x10, 0x3d, 0x77, 0xac, 0x26, 0x04, 0x3a, 0x3b,
0x18, 0xf8, 0x47, 0x75, 0x56, 0xa5, 0xda, 0x70, 0xfb, 0xf7, 0x0d, 0x3b, 0x6e, 0x4b, 0x2a, 0xbc, 0x49, 0x32, 0x43,
0x95, 0x62, 0x83, 0x3d, 0xdc, 0x0a, 0x1f, 0x1d, 0xf9, 0x59, 0x6b, 0x95, 0xf0, 0x9b, 0xf5, 0x53, 0x9e, 0xb5, 0x65,
0xeb, 0x74, 0xfa, 0x81, 0x9b, 0x61, 0xa3, 0x57, 0x2f, 0xda, 0x2c, 0xcd, 0xf8, 0xb0, 0x74, 0xb9, 0x62, 0x39, 0x91,
0xd9, 0x7d, 0xb3, 0x03, 0x65, 0x2e, 0x66, 0x20, 0x18, 0xac, 0xa2, 0xeb, 0x3b, 0x35, 0x98, 0xa3, 0x28, 0x46, 0x30,
0xee, 0xd3, 0xeb, 0x47, 0x49, 0x11, 0xc5, 0xcd, 0xa0, 0xa5, 0x1c, 0x2e, 0x9d, 0x14, 0xd6, 0x46, 0x4a, 0x05, 0x7b,
0xd3, 0xb9, 0x0d, 0xeb, 0xd7, 0x6f, 0xb5, 0xf4, 0xed, 0x3f, 0x27, 0xb8, 0xec, 0x51, 0xb1, 0x6e, 0xb0, 0x14, 0xed,
0xdf, 0x90, 0x72, 0x59, 0x71, 0xd5, 0xf5, 0xf2, 0x61, 0x5f, 0xa7, 0x8a, 0xd7, 0x0f, 0x80, 0x2f, 0xe5, 0x0f, 0x45,
0xf2, 0x4d, 0xd1, 0xdc, 0xdd, 0xce, 0x46, 0xdf, 0x77, 0xf2, 0xa6, 0xa2, 0x79, 0x77, 0xf0, 0x0e, 0xaa, 0x68, 0x14,
0x85, 0x32, 0x05, 0x29, 0x75, 0x2b, 0x6c, 0xb0, 0x7c, 0x84, 0xc9, 0xec, 0x49, 0x70, 0x9b, 0x38, 0x36, 0x4c, 0xe9,
0xa5, 0xdc, 0xb1, 0xb8, 0x28, 0xd6, 0x20, 0x89, 0x8a, 0x2a, 0x62, 0xc1, 0x90, 0x3a, 0x43, 0x48, 0xaa, 0xfc, 0xec,
0x52, 0xf4, 0x02, 0xa7, 0xcd, 0x46, 0xf9, 0x78, 0x64, 0x4f, 0xad, 0x6b, 0x17, 0xdb, 0x56, 0xa2, 0xa1, 0x6a, 0xcf,
0x3b, 0x66, 0x64, 0x01, 0xc2, 0xd6, 0xae, 0x23, 0x34, 0x82, 0x60, 0x0a, 0x3a, 0xe4, 0xdd, 0x7f, 0xaa, 0x97, 0x44,
0x9a, 0x63, 0x8f, 0xaf, 0xa9, 0x97, 0x5a, 0x55, 0xcc, 0x81, 0x48, 0x83, 0xf8, 0xe6, 0xc5, 0xef, 0xdf, 0xfa, 0x5a,
0x4f, 0x7f, 0x18, 0x56, 0xc0, 0x66, 0x36, 0xad, 0x8a, 0x79, 0xab, 0xde, 0xf4, 0x7b, 0x70, 0x98, 0xea, 0xfa, 0xf4,
0xc7, 0x01, 0x31, 0x16, 0xea, 0xf1, 0x70, 0x8d, 0xee, 0x87, 0x52, 0x13, 0x01, 0x9e, 0x0c, 0xbd, 0x5a, 0x14, 0x01,
0x8e, 0xf7, 0x6e, 0xff, 0xce, 0x7f, 0xa4, 0xd1, 0xa1, 0x6a, 0x9b, 0x95, 0xb2, 0x31, 0x8a, 0x6f, 0xc8, 0xfe, 0x8c,
0x29, 0x55, 0xbc, 0xfc, 0x61, 0xd0, 0xde, 0xb0, 0xa9, 0x0c, 0x01, 0x0f, 0xba, 0x77, 0x7e, 0x77, 0xd9, 0x76, 0xa4,
0xfd, 0xab, 0x38, 0x18, 0x92, 0xec, 0xf0, 0xd3, 0x57, 0x7f, 0xf4, 0xbd, 0x65, 0xd4, 0x77, 0x8e, 0xa1, 0x92, 0x82,
0x0c, 0x7b, 0x34, 0xd3, 0x11, 0xfb, 0xc0, 0x36, 0x23, 0x8e, 0xbb, 0x89, 0xdf, 0x70, 0xdc, 0x9a, 0x76, 0x45, 0x0b,
0x22, 0xfa, 0x35, 0xdf, 0x8f, 0x45, 0x48, 0x7d, 0xed, 0xdc, 0x06, 0x01, 0x5e, 0xbb, 0xe9, 0xf9, 0x07, 0x93, 0xf6,
0x69, 0x6f, 0xf1, 0xfc, 0xfd, 0xfa, 0x41, 0xc0, 0x13, 0x9e, 0x74, 0x2d, 0x76, 0xfe, 0xa7, 0xbe, 0x4e, 0xd9, 0xaa,
0x64, 0xa6, 0xed, 0xd3, 0xad, 0xee, 0x62, 0x9d, 0x39, 0xc9, 0xb8, 0xae, 0x86, 0x31, 0x4f, 0x62, 0x57, 0xea, 0xea,
0x5a, 0xe3, 0x59, 0xd8, 0x99, 0xb3, 0xcd, 0x9f, 0x3b, 0xea, 0x58, 0x99, 0x11, 0xdc, 0x3d, 0xac, 0x58, 0xd9, 0xa6,
0xae, 0x2d, 0x07, 0x1d, 0xd7, 0xfa, 0x87, 0x78, 0xa7, 0x7f, 0x2a, 0x4f, 0x25, 0x58, 0xef, 0x53, 0x78, 0x2e, 0x93,
0xdc, 0x44, 0xcc, 0x53, 0x88, 0x77, 0x1c, 0xda, 0x14, 0xaf, 0xe1, 0x67, 0x92, 0xff, 0x36, 0x96, 0x20, 0x6f, 0x9d,
0x2c, 0x7f, 0xea, 0x31, 0xf2, 0x34, 0x50, 0xc2, 0x39, 0x84, 0xee, 0x4c, 0xbe, 0xca, 0x06, 0x6f, 0x67, 0x42, 0xea,
0x13, 0x58, 0xee, 0xdd, 0x8e, 0x29, 0x4f, 0xee, 0xd8, 0x93, 0x0d, 0x45, 0x80, 0x4d, 0xf3, 0x12, 0x79, 0xbb, 0x36,
0xa3, 0x73, 0x95, 0x95, 0xb6, 0xfc, 0x54, 0x60, 0xb2, 0xcc, 0x71, 0xaa, 0xf1, 0x6b, 0x66, 0xed, 0xba, 0x8b, 0xd6,
0x6d, 0x61, 0x79, 0x61, 0x1d, 0xb3, 0xba, 0xa6, 0x2f, 0xaa, 0xdc, 0x1d, 0xb8, 0x22, 0xd8, 0x98, 0x58, 0xed, 0x4d,
0x3c, 0xea, 0xa9, 0x37, 0x5e, 0x5e, 0x7b, 0x47, 0xa1, 0x7a, 0x39, 0x42, 0xe4, 0x3b, 0xbb, 0x69, 0x0a, 0x8b, 0x32,
0x6e, 0x63, 0xeb, 0x87, 0xf6, 0x5d, 0xaa, 0xbf, 0xbe, 0xc5, 0xb2, 0x85, 0x60, 0xdc, 0x32, 0x07, 0x85, 0x73, 0x3d,
0x96, 0x8b, 0x89, 0x71, 0x52, 0xb4, 0x93, 0xe6, 0x18, 0xad, 0xbf, 0xce, 0x21, 0x1d, 0x33, 0xb5, 0xb3, 0x35, 0x6a,
0x5b, 0xe7, 0x47, 0x13, 0x19, 0x8f, 0x53, 0xf4, 0x0c, 0x2a, 0x39, 0x16, 0x37, 0x39, 0x3b, 0x5f, 0x81, 0x51, 0xbc,
0x23, 0x92, 0x2c, 0x8d, 0xbf, 0x2f, 0xee, 0xbf, 0xed, 0x9d, 0x3f, 0xe0, 0x16, 0x21, 0x5a, 0x57, 0xa6, 0x8c, 0x58,
0x7a, 0xd5, 0xa1, 0x6d, 0xed, 0xe8, 0x90, 0x97, 0x14, 0xb4, 0x6b, 0xa5, 0x3b, 0xd7, 0x90, 0x84, 0x9e, 0x07, 0xc2,
0x7f, 0x1e, 0x08, 0xa0, 0x6f, 0x69, 0xf1, 0xcc, 0x4e, 0xca, 0x07, 0x64, 0xa2, 0xb4, 0xbc, 0x5f, 0xe0, 0xa7, 0x0e,
0x31, 0x77, 0x6f, 0x35, 0xd7, 0x66, 0x83, 0x5d, 0x64, 0x4e, 0xf1, 0x3c, 0x7a, 0xcb, 0xa5, 0xfc, 0xc9, 0x95, 0xab,
0x27, 0x30, 0x6b, 0x82, 0x57, 0xcd, 0xb0, 0x85, 0x9d, 0xc5, 0xa9, 0x0f, 0xdb, 0xe3, 0x5a, 0xc1, 0xb2, 0x3f, 0xfa,
0xea, 0xf3, 0x3a, 0xa0, 0xe4, 0xad, 0xd7, 0x14, 0xcf, 0x4f, 0xc9, 0x46, 0x09, 0x6e, 0x70, 0x27, 0xe4, 0x9b, 0x5f,
0x61, 0x4a, 0xb2, 0xfb, 0xc3, 0xf0, 0x3a, 0x53, 0x38, 0x2a, 0x59, 0x48, 0x2b, 0xab, 0x21, 0x64, 0x1a, 0x1c, 0x90,
0xb6, 0x69, 0xbb, 0x89, 0xe3, 0x9a, 0x12, 0xb7, 0xfd, 0x47, 0xcc, 0x57, 0xdb, 0x2b, 0xf6, 0x35, 0xe1, 0x79, 0xe0,
0xbf, 0x6d, 0x7f, 0x71, 0x3c, 0x21, 0x27, 0x82, 0x12, 0x05, 0xae, 0x43, 0x11, 0x0f, 0xc8, 0xe7, 0x3c, 0xf0, 0x0b,
0xe4, 0x69, 0xff, 0xb2, 0xda, 0x9e, 0x7f, 0xfe, 0xa4, 0x3f, 0xdf, 0x02, 0x58, 0xaa, 0x60, 0xd2, 0x8e, 0xd5, 0x6c,
0x92, 0x22, 0x38, 0xb1, 0xd2, 0x84, 0x80, 0xea, 0xce, 0xfe, 0x34, 0x79, 0xd6, 0x94, 0xb0, 0xad, 0x51, 0x67, 0x65,
0xf5, 0xab, 0xd5, 0x18, 0x05, 0x92, 0x8a, 0x24, 0x36, 0xc2, 0xc4, 0x99, 0xad, 0x0d, 0x26, 0xb8, 0x70, 0xea, 0x5b,
0x74, 0x35, 0x6a, 0xfa, 0xac, 0x97, 0xd8, 0x4c, 0x08, 0x6d, 0xc1, 0xfe, 0x60, 0xe4, 0xd4, 0x8f, 0x93, 0xd4, 0xaf,
0xa2, 0xec, 0xb8, 0x71, 0xf6, 0xcb, 0x3f, 0xdc, 0x3e, 0x42, 0xca, 0xe2, 0x8d, 0xb9, 0x0a, 0x2c, 0xef, 0xb6, 0x18,
0x75, 0x46, 0x8a, 0x2b, 0x21, 0xeb, 0xc5, 0x12, 0x4d, 0xd7, 0xb8, 0x8e, 0x6c, 0xec, 0x9d, 0x12, 0xde, 0xa5, 0x45,
0xab, 0xc4, 0x26, 0x47, 0xfa, 0x57, 0xad, 0x78, 0x4c, 0x5e, 0x87, 0x0b, 0x42, 0x6a, 0xbe, 0xef, 0xbd, 0xbc, 0xe5,
0xd7, 0x4c, 0x3a, 0x0f, 0x96, 0x29, 0xcb, 0x45, 0x46, 0x3f, 0x8a, 0x31, 0xda, 0x8d, 0x7f, 0x89, 0xa4, 0xd6, 0xb4,
0xe5, 0x99, 0x27, 0xa7, 0xab, 0x15, 0x82, 0x75, 0x3e, 0xc0, 0x9d, 0xae, 0x8c, 0x7f, 0x19, 0x80, 0xf5, 0xa6, 0x89,
0xbc, 0xb7, 0x36, 0x50, 0x1f, 0x63, 0xca, 0x40, 0x3e, 0xe1, 0x4c, 0x2d, 0x1e, 0x15, 0x51, 0x93, 0xf2, 0x1a, 0x97,
0x3e, 0xad, 0x5e, 0xba, 0xe2, 0xd6, 0xa7, 0x7c, 0x51, 0x45, 0xdf, 0x92, 0xbe, 0x35, 0xf3, 0xb4, 0xe8, 0x5c, 0x2e,
0x6a, 0x5c, 0xda, 0xe8, 0xf7, 0x51, 0xdc, 0xcb, 0x06, 0xd2, 0xd2, 0xa4, 0x7c, 0x7d, 0x5f, 0x20, 0xb8, 0x50, 0x04,
0xb5, 0xb1, 0x9a, 0x4e, 0xc9, 0x94, 0xe4, 0x40, 0x2b, 0x85, 0xc8, 0xd3, 0x71, 0xfa, 0x83, 0x81, 0x28, 0x53, 0xef,
0xa0, 0xb0, 0xf0, 0xc8, 0xb9, 0x52, 0xb2, 0x0f, 0x8c, 0x49, 0x93, 0xeb, 0x2a, 0x95, 0x83, 0x4f, 0x47, 0xe8, 0x0c,
0x83, 0x7b, 0x88, 0x18, 0x63, 0xa9, 0x87, 0x7f, 0xe3, 0x57, 0xa6, 0x57, 0xdd, 0x6d, 0x2a, 0x85, 0xe9, 0x79, 0x16,
0x3b, 0x07, 0x59, 0x4f, 0xed, 0x9a, 0x17, 0xe7, 0xcd, 0x35, 0xa6, 0x9b, 0x06, 0x1d, 0x18, 0x5e, 0xb2, 0x9b, 0xac,
0x18, 0xc6, 0x5b, 0xec, 0xbe, 0x29, 0xa3, 0x57, 0xc2, 0x6b, 0x3f, 0x11, 0x89, 0xd1, 0x57, 0xe4, 0xb8, 0xaf, 0x11,
0xba, 0x90, 0x93, 0x4f, 0x5e, 0x0c, 0x6b, 0x9f, 0x61, 0x5c, 0xdd, 0xb6, 0xcf, 0x18, 0xcc, 0x9a, 0x95, 0x32, 0x5f,
0x6f, 0xf1, 0x43, 0x17, 0x2d, 0xbc, 0x70, 0xbe, 0xd4, 0x76, 0x38, 0x42, 0xcc, 0xdd, 0xf0, 0xff, 0x5b, 0xe8, 0x19,
0xc9, 0xf4, 0x25, 0xda, 0xdb, 0x5f, 0xd2, 0x3c, 0xf3, 0xe0, 0xed, 0xb0, 0xa2, 0x4c, 0xdf, 0x05, 0xf2, 0x0c, 0xc4,
0x21, 0xb9, 0xc4, 0x02, 0xd2, 0x1e, 0x46, 0x2c, 0x57, 0x78, 0xb0, 0xf1, 0x85, 0x33, 0x51, 0xc5, 0x3e, 0xce, 0x69,
0x68, 0x77, 0x3e, 0xb4, 0x5a, 0x4e, 0x43, 0x89, 0x85, 0x71, 0x36, 0x84, 0x56, 0x5c, 0x6b, 0xce, 0x78, 0x4e, 0x86,
0x6a, 0x5f, 0xf1, 0x2d, 0x73, 0x4c, 0xba, 0xc1, 0x5e, 0x2b, 0x6e, 0x0d, 0xdd, 0x45, 0x1d, 0x92, 0xb4, 0x31, 0xd9,
0xce, 0x0b, 0x58, 0x67, 0xc1, 0xa6, 0x47, 0x2e, 0x5e, 0x6d, 0x6d, 0xa3, 0x78, 0xed, 0xc5, 0xcb, 0xac, 0x20, 0xb8,
0x6a, 0x14, 0x32, 0x55, 0xe8, 0xfb, 0xe0, 0xee, 0x31, 0xfc, 0x47, 0x33, 0xf6, 0xe7, 0x54, 0xd1, 0x61, 0x91, 0x74,
0x90, 0xfa, 0x44, 0x0a, 0xc8, 0xaf, 0xa2, 0xdb, 0x2c, 0x83, 0xda, 0xe8, 0xd0, 0xb7, 0x34, 0x8a, 0xc8, 0xbb, 0x86,
0x58, 0x82, 0x1a, 0x2f, 0xd6, 0xc5, 0x92, 0xe9, 0xd0, 0x21, 0x11, 0xf6, 0xeb, 0x9b, 0xed, 0xbf, 0xa9, 0x71, 0x89,
0xb2, 0x59, 0x76, 0xb3, 0xeb, 0x9c, 0xe1, 0x73, 0x79, 0x7b, 0x11, 0x3d, 0x4e, 0x33, 0xca, 0xf6, 0x7e, 0xd8, 0xcf,
0x06, 0x41, 0x39, 0xef, 0x78, 0x5a, 0x89, 0x69, 0x0c, 0xff, 0xda, 0x31, 0x16, 0x32, 0xd4, 0xe5, 0x2d, 0x8b, 0x22,
0xb9, 0x82, 0x38, 0xd8, 0x22, 0xfc, 0xe8, 0xbd, 0xbf, 0x0d, 0x4b, 0xc9, 0x0d, 0x74, 0x5e, 0x2d, 0xb2, 0x1f, 0x23,
0xa8, 0xf8, 0xa7, 0x27, 0xe1, 0x02, 0x61, 0x83, 0xc4, 0xd0, 0x63, 0x0e, 0x23, 0xf2, 0x34, 0x06, 0x69, 0xd1, 0xa3,
0xa2, 0x76, 0x76, 0x94, 0xbc, 0x1f, 0x78, 0x73, 0x5a, 0x48, 0x82, 0xb4, 0x68, 0x3d, 0x24, 0x52, 0xe8, 0x92, 0x4e,
0x58, 0xdf, 0x55, 0x3f, 0xa8, 0x48, 0x59, 0x35, 0x90, 0x9b, 0x55, 0xc9, 0xdc, 0xff, 0xcc, 0x22, 0x7c, 0xd1, 0x29,
0xe0, 0xd5, 0xd8, 0x31, 0x3d, 0x9b, 0xae, 0x38, 0x52, 0x2a, 0xa1, 0xc5, 0x86, 0x19, 0x93, 0xe7, 0x2d, 0x9b, 0xa9,
0x09, 0xbf, 0xdd, 0x89, 0x43, 0x35, 0xb2, 0x6f, 0x91, 0xba, 0x28, 0xfa, 0x2f, 0x2c, 0xab, 0xe8, 0x58, 0xe6, 0x4c,
0xbf, 0xd5, 0xaf, 0x49, 0xcd, 0x7f, 0x18, 0xa4, 0x69, 0xd3, 0xff, 0x99, 0xad, 0x9d, 0xb0, 0x93, 0x3b, 0x04, 0x01,
0xd5, 0x7d, 0x33, 0xe0, 0x66, 0xe2, 0xd4, 0xeb, 0xae, 0xcd, 0x7f, 0x66, 0xf2, 0x2c, 0xa7, 0xfe, 0x7d, 0x51, 0x8b,
0x2a, 0x25, 0xa8, 0x90, 0xb2, 0x71, 0xe5, 0x98, 0x40, 0xb8, 0xe9, 0x1f, 0x64, 0xd9, 0x90, 0xf2, 0x39, 0x25, 0x6f,
0x58, 0x35, 0x9e, 0x40, 0x25, 0x92, 0xbb, 0xa2, 0x4d, 0x45, 0xf6, 0xc3, 0x1a, 0xc6, 0xc0, 0xf6, 0xc2, 0x9b, 0x2b,
0xb5, 0xc7, 0xe9, 0xc8, 0x75, 0x03, 0x71, 0x1c, 0x76, 0x0e, 0xde, 0x5c, 0xe7, 0xf4, 0x91, 0xc4, 0x1d, 0x9f, 0xcb,
0x67, 0x1c, 0x1b, 0xc9, 0x25, 0x6d, 0x95, 0x05, 0xd7, 0x3b, 0xbf, 0x2d, 0x90, 0x96, 0xc3, 0x44, 0xa3, 0x0a, 0xee,
0x7a, 0x4c, 0x7e, 0x7f, 0xc2, 0x4e, 0x4f, 0x38, 0x88, 0xe9, 0x9f, 0x93, 0xd8, 0xd1, 0x10, 0x8c, 0x85, 0x89, 0xb2,
0xd0, 0x95, 0xc3, 0xe4, 0xd2, 0x8f, 0x2b, 0x0c, 0x82, 0xbd, 0x25, 0x74, 0x39, 0xe7, 0x3e, 0xef, 0xd2, 0x2f, 0x6f,
0xdc, 0xe0, 0x75, 0x6b, 0xf2, 0x66, 0x7d, 0x65, 0x18, 0x26, 0xb0, 0x7a, 0x7a, 0x45, 0xf0, 0xf2, 0xc4, 0xbe, 0x70,
0x6e, 0x58, 0x3a, 0x54, 0x31, 0x64, 0xf6, 0x49, 0xb5, 0xdc, 0x1c, 0x9e, 0x87, 0xdb, 0x6a, 0x92, 0x2a, 0x59, 0x73,
0x35, 0x4d, 0x59, 0xed, 0xd0, 0x8a, 0xd9, 0xac, 0xea, 0x7c, 0xc0, 0x0a, 0xfe, 0x8f, 0x29, 0xb7, 0xda, 0xdc, 0x8a,
0x55, 0x0f, 0x61, 0xc0, 0x82, 0x49, 0x4c, 0xc2, 0x51, 0x4b, 0x41, 0x7d, 0x1f, 0xf1, 0x07, 0x13, 0x23, 0x16, 0xe4,
0x88, 0x76, 0x38, 0x92, 0xbd, 0x3d, 0xe2, 0x7b, 0xab, 0x1c, 0x53, 0x35, 0xfb, 0x85, 0x76, 0x35, 0xf7, 0xfc, 0xdc,
0x3e, 0xc9, 0xae, 0x6a, 0x7c, 0x10, 0x44, 0xa5, 0xfb, 0xc4, 0x81, 0x9a, 0x6d, 0x9b, 0xf2, 0x87, 0x0a, 0x9e, 0xcd,
0x75, 0xc2, 0xd5, 0xe7, 0x0b, 0xcd, 0x5c, 0x6e, 0xf2, 0x51, 0x96, 0xa3, 0xe8, 0xd5, 0x5a, 0x17, 0xf5, 0x1a, 0x28,
0xff, 0xc8, 0xf0, 0xfc, 0x7e, 0x3b, 0xd3, 0xf3, 0x12, 0x61, 0xe9, 0x02, 0xfa, 0x7b, 0x8e, 0x6b, 0x34, 0x4a, 0x5a,
0xb6, 0xb1, 0x71, 0x7d, 0xad, 0xbc, 0x88, 0x22, 0xf6, 0x14, 0xc5, 0x57, 0x1a, 0x0b, 0xb6, 0x8b, 0xbb, 0x9a, 0x09,
0xf2, 0xe8, 0x8d, 0x32, 0xd1, 0x54, 0xa9, 0xd2, 0x9f, 0xc5, 0xf4, 0x12, 0xad, 0xb3, 0xfd, 0x52, 0xb7, 0xff, 0x0e,
0x72, 0x59, 0xcb, 0x1d, 0x8d, 0xa4, 0xf2, 0x0f, 0x39, 0x8f, 0xc9, 0x84, 0xe0, 0x5c, 0xdd, 0x9c, 0xe4, 0x42, 0x4f,
0xf2, 0x09, 0x1e, 0x05, 0xcc, 0x41, 0xdd, 0xbd, 0xc1, 0xed, 0xdc, 0x8e, 0x64, 0xc3, 0x67, 0x51, 0xe5, 0x8a, 0x9a,
0x0d, 0x09, 0xb8, 0x4b, 0xa1, 0xb2, 0xdb, 0x01, 0xde, 0xf5, 0xd1, 0x51, 0xf7, 0x9d, 0x88, 0x59, 0xfd, 0xd5, 0x13,
0x0d, 0x05, 0xd5, 0x5e, 0xfa, 0x63, 0x26, 0x25, 0xad, 0x01, 0x83, 0x6f, 0xf3, 0x7f, 0x56, 0x11, 0x89, 0xaa, 0xdb,
0x49, 0xaa, 0x8f, 0x7d, 0x10, 0x99, 0xbf, 0x8b, 0xcf, 0xf4, 0xf8, 0xc2, 0x4e, 0x36, 0xa7, 0xb1, 0xfe, 0xc5, 0xf0,
0x07, 0xc3, 0x57, 0xe3, 0xbd, 0x1e, 0xa1, 0x56, 0x46, 0x32, 0x43, 0x8f, 0x8d, 0xc7, 0x73, 0x0a, 0xa5, 0x35, 0xa9,
0x93, 0x7c, 0x53, 0x3c, 0xf2, 0x58, 0xf3, 0x6a, 0x37, 0x41, 0xb4, 0xc7, 0x9d, 0xcd, 0xb1, 0x67, 0xb7, 0xaa, 0x5f,
0x25, 0xb9, 0xc5, 0xb6, 0xa2, 0xdb, 0xce, 0x34, 0x2c, 0xea, 0x93, 0x45, 0x6e, 0x7b, 0x6f, 0xb6, 0xf6, 0x89, 0x83,
0xd1, 0x9e, 0x08, 0x4b, 0x57, 0x4e, 0xd9, 0xf3, 0xe8, 0x81, 0x03, 0x5c, 0xd6, 0x6c, 0x2c, 0xea, 0x27, 0x2d, 0xfb,
0x39, 0xbb, 0x25, 0x81, 0xbb, 0x1a, 0xc3, 0xcb, 0xf5, 0x5e, 0x69, 0x1c, 0xa4, 0x8a, 0x0e, 0x4a, 0x69, 0xdf, 0xcd,
0x6c, 0xc7, 0xce, 0x48, 0xdb, 0x7a, 0x0b, 0xcb, 0x40, 0xc8, 0x59, 0x81, 0xde, 0x71, 0x3e, 0x0d, 0x41, 0xc9, 0xa2,
0x57, 0xae, 0xb7, 0x3f, 0x8b, 0x4e, 0x7c, 0xcb, 0xf9, 0xa5, 0xcb, 0xf8, 0xee, 0x52, 0xbe, 0x54, 0xa8, 0xef, 0x99,
0xb7, 0xa1, 0x0b, 0x77, 0xa2, 0xa7, 0x89, 0x47, 0x7f, 0x48, 0xaa, 0x02, 0x36, 0xbf, 0x9d, 0x97, 0xba, 0x43, 0x5e,
0x39, 0xa6, 0x99, 0x81, 0xde, 0xe4, 0x98, 0x06, 0x6a, 0xc8, 0x2c, 0x96, 0xf5, 0x51, 0x8c, 0x95, 0xd3, 0xe4, 0x2b,
0x94, 0x9a, 0x15, 0x71, 0x9f, 0xa4, 0x24, 0xfc, 0x92, 0x97, 0x18, 0xb7, 0x56, 0xe5, 0x59, 0xe1, 0x1a, 0x79, 0x3a,
0x47, 0x77, 0x92, 0xf5, 0x75, 0x42, 0xbd, 0x2d, 0x55, 0x05, 0xcf, 0xe8, 0xea, 0x64, 0x8b, 0xff, 0x30, 0xa1, 0x1e,
0x28, 0x2c, 0xa9, 0xaa, 0xd4, 0x19, 0x1d, 0x24, 0x7a, 0xcf, 0x5d, 0x39, 0xf2, 0xe3, 0xd2, 0xab, 0x26, 0x97, 0x35,
0xf6, 0x70, 0x8b, 0x09, 0x8b, 0x00, 0xb9, 0xb8, 0xa8, 0xb7, 0x78, 0xf1, 0x96, 0x03, 0xd2, 0x72, 0x46, 0x26, 0x82,
0x41, 0xce, 0xe3, 0x73, 0xb2, 0x31, 0x4f, 0x49, 0xc8, 0xa8, 0xcf, 0xcc, 0xb8, 0xcc, 0x5d, 0xab, 0xd1, 0x1e, 0xf5,
0x5a, 0x9a, 0x54, 0xaf, 0x15, 0xed, 0x5f, 0x96, 0x2b, 0x62, 0x0a, 0x4c, 0x67, 0x21, 0x98, 0xfd, 0x29, 0xf8, 0x12,
0x9d, 0xc2, 0x34, 0x4d, 0x70, 0x0e, 0x3b, 0x5e, 0xe3, 0x91, 0x58, 0x82, 0xa8, 0x9c, 0x7d, 0x11, 0xfb, 0x5d, 0x81,
0x0c, 0x0f, 0x9d, 0x84, 0xbb, 0xeb, 0x65, 0x1d, 0xeb, 0x6e, 0x18, 0x8e, 0x93, 0x70, 0xbb, 0x9d, 0xed, 0x58, 0xb2,
0x64, 0xf2, 0x7b, 0x8f, 0x05, 0xd8, 0xfa, 0x55, 0x50, 0xef, 0x7a, 0x49, 0x0d, 0xdf, 0xca, 0x63, 0xff, 0x41, 0x1d,
0x89, 0x79, 0xca, 0x8c, 0x1b, 0xbe, 0x75, 0x6f, 0x0e, 0x6b, 0x68, 0x8c, 0x5c, 0x93, 0xc8, 0xf3, 0x62, 0x0b, 0x68,
0xb1, 0x32, 0x2f, 0x1e, 0xda, 0xe5, 0x6d, 0x45, 0xaf, 0x3d, 0xa7, 0xaf, 0x81, 0x0f, 0xda, 0x82, 0x85, 0x94, 0x06,
0xc5, 0x3f, 0xf5, 0x73, 0x5f, 0xa4, 0x43, 0x5f, 0xb3, 0xac, 0x3d, 0x16, 0xe8, 0x6f, 0x84, 0xb3, 0xb6, 0x75, 0xf7,
0x13, 0x03, 0x46, 0x67, 0x87, 0x9a, 0xf1, 0x6c, 0x2e, 0x33, 0x33, 0x14, 0x85, 0x18, 0xe9, 0xd6, 0x1b, 0xee, 0x17,
0xe9, 0xe7, 0x6b, 0xd4, 0x46, 0xaf, 0x26, 0x55, 0x8d, 0xfc, 0x3a, 0xb4, 0x61, 0x1a, 0x9e, 0x2d, 0x85, 0xa2, 0x86,
0x8c, 0x92, 0xbd, 0x8d, 0x0e, 0x4b, 0x4e, 0x0d, 0xdf, 0x4a, 0xc6, 0x66, 0xcd, 0x28, 0xff, 0xc4, 0x44, 0xba, 0x39,
0x4a, 0x04, 0x43, 0xce, 0x95, 0x47, 0x99, 0x8c, 0x9b, 0xb7, 0x72, 0x48, 0xdd, 0x9b, 0x98, 0x4f, 0xd5, 0x18, 0x81,
0xb4, 0xef, 0xfa, 0x96, 0x54, 0x9b, 0xda, 0x73, 0xe2, 0xe0, 0x27, 0x2f, 0x10, 0x0f, 0xf9, 0x0e, 0x24, 0xb4, 0xd2,
0x39, 0xf7, 0x49, 0x8c, 0x5c, 0x05, 0x37, 0xdd, 0xa7, 0x73, 0xac, 0x16, 0x85, 0x71, 0x46, 0xd1, 0x6f, 0x7a, 0x25,
0xd2, 0x5c, 0x4e, 0x70, 0x26, 0x93, 0x36, 0x22, 0x5f, 0xe5, 0x42, 0x4e, 0x89, 0xf2, 0x4c, 0x55, 0x9b, 0xd1, 0xa5,
0xaf, 0x87, 0xd8, 0x7e, 0xe8, 0xdd, 0xb9, 0x47, 0xb3, 0xda, 0xf7, 0x98, 0x52, 0x73, 0x8b, 0x3c, 0x0b, 0xe7, 0xfa,
0x14, 0x7f, 0x9a, 0xa6, 0x8e, 0xf4, 0x7c, 0xad, 0x58, 0x11, 0x73, 0xe3, 0x3a, 0x56, 0x47, 0x9b, 0x95, 0x24, 0x68,
0x35, 0xa9, 0xd9, 0x6e, 0x5c, 0x4d, 0xe8, 0xa4, 0xb8, 0x80, 0x0b, 0x73, 0xc4, 0x0f, 0x87, 0x0c, 0xa1, 0xf4, 0x5a,
0x3c, 0xce, 0x3f, 0x30, 0x3d, 0xb5, 0xc1, 0x0b, 0x43, 0x2a, 0xb1, 0xbe, 0x6b, 0x12, 0xc4, 0x4b, 0xdb, 0xcb, 0x2f,
0x77, 0x8a, 0x3a, 0xba, 0xf4, 0xa3, 0x9b, 0x05, 0xee, 0x98, 0x50, 0x77, 0xfe, 0x82, 0x5b, 0x29, 0x6d, 0x7c, 0x0c,
0x4b, 0x41, 0xa2, 0x6a, 0xb2, 0x27, 0x32, 0xf1, 0x0c, 0x1d, 0xd7, 0x6c, 0x02, 0x4d, 0xbd, 0xf2, 0x5a, 0x66, 0xd3,
0xb5, 0x30, 0xab, 0x4b, 0x7e, 0x72, 0x77, 0x30, 0x42, 0xd7, 0x70, 0x05, 0x9f, 0xeb, 0x2f, 0x22, 0x7f, 0x49, 0x2d,
0x5d, 0xcf, 0x4b, 0x84, 0x0c, 0xa5, 0xb9, 0xe5, 0x49, 0xae, 0x54, 0x2c, 0x86, 0x53, 0x4c, 0x9c, 0x42, 0x29, 0x78,
0xf0, 0x0a, 0x8a, 0x36, 0x53, 0x4d, 0x2b, 0xc9, 0xe2, 0x69, 0x0e, 0x9a, 0x37, 0x0c, 0x91, 0x2d, 0xac, 0x2f, 0x43,
0x6a, 0xf1, 0xe9, 0xfb, 0xb4, 0x68, 0xec, 0x90, 0x00, 0xff, 0x32, 0xf8, 0x58, 0xab, 0xd1, 0x73, 0x9f, 0x77, 0x72,
0x22, 0xed, 0x64, 0xac, 0x7d, 0xd0, 0xa8, 0x1f, 0x01, 0x99, 0x9f, 0xfd, 0xa6, 0xcb, 0x62, 0xca, 0x65, 0xd8, 0xf8,
0x31, 0xf0, 0xc1, 0x5e, 0xef, 0xf1, 0x4d, 0x79, 0xa9, 0xfc, 0x8f, 0x57, 0xcb, 0xc1, 0xb7, 0x03, 0x30, 0xbb, 0xad,
0xdb, 0x88, 0xf7, 0xcc, 0xa1, 0x25, 0xfd, 0x37, 0x03, 0xc5, 0xdb, 0x7f, 0xb4, 0xe0, 0xe6, 0xa3, 0x7e, 0xa4, 0x52,
0x68, 0xa7, 0x2e, 0xe6, 0x1c, 0xea, 0x91, 0x48, 0x67, 0xbd, 0x3d, 0x6d, 0xcc, 0x09, 0x49, 0x17, 0x50, 0x16, 0xb5,
0x65, 0x63, 0xd4, 0x84, 0x2c, 0x3a, 0x5f, 0x6a, 0x3c, 0x66, 0xfa, 0x24, 0xaf, 0xd1, 0xfd, 0x18, 0x66, 0xe4, 0xe8,
0x3c, 0x6c, 0x74, 0xb6, 0xc5, 0x4c, 0xf8, 0xa8, 0x15, 0x3c, 0xd0, 0xb8, 0x4a, 0x53, 0x15, 0x4b, 0x1e, 0x56, 0x7f,
0xd8, 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0xfe, 0xe7, 0xdc, 0x03, 0xd1, 0x0f, 0xfd, 0x7c, 0xdc, 0xa1, 0x7e, 0x41, 0x2e, 0x50, 0x37, 0x77, 0x94, 0xb7, 0x4b,
0x90, 0x3b, 0x14, 0x38, 0xe0, 0x80, 0x03, 0x0e, 0x38, 0xe0, 0x80, 0xfb, 0xff, 0xba, 0xff, 0x00,
};
static const uint8_t STORED[] = {
0x01, 0x2c, 0x01, 0xd3, 0xfe, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64,
0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61,
0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f,
0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65,
0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f,
0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70,
0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65,
0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65,
0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61,
0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66,
0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64,
0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61,
0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f,
0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f, 0x6d, 0x65,
0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70, 0x68, 0x6f,
0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20, 0x65, 0x73, 0x70,
0x68, 0x6f, 0x6d, 0x65, 0x20, 0x6f, 0x74, 0x61, 0x20, 0x64, 0x65, 0x66, 0x6c, 0x61, 0x74, 0x65, 0x20,
};
static constexpr size_t WINDOW = 4096;
static constexpr size_t PLAIN_SIZE = 16000;
static uint8_t lcg_next(uint32_t &x) {
x = (x * 1103515245u + 12345u) & 0x7fffffffu;
return (x >> 16) & 0xff;
}
static std::vector<uint8_t> build_plain() {
std::vector<uint8_t> plain;
const char *text = "esphome ota deflate ";
for (int i = 0; i < 300; i++)
plain.insert(plain.end(), text, text + strlen(text));
uint32_t x = 1;
for (int i = 0; i < 3000; i++)
plain.push_back(lcg_next(x));
plain.insert(plain.end(), 5000, 0);
for (int i = 0; i < 100; i++)
plain.insert(plain.end(), text, text + strlen(text));
return plain;
}
// Mirrors the OTA session: chunked input through the read callback, window as output
struct Session : OtaInflateState {
const uint8_t *in;
size_t in_len;
size_t in_pos;
size_t chunk;
std::vector<uint8_t> out;
uint8_t window[WINDOW];
};
static int read_cb(OtaInflateState *d) {
auto *s = static_cast<Session *>(d);
if (s->in_pos >= s->in_len)
return -1;
size_t n = std::min(s->chunk, s->in_len - s->in_pos);
d->source = s->in + s->in_pos + 1;
d->source_limit = s->in + s->in_pos + n;
s->in_pos += n;
return s->in[s->in_pos - n];
}
// Inflates the whole input; returns the decoder result and fills s.out
static int inflate_all(Session &s, const uint8_t *in, size_t in_len, size_t chunk) {
s.in = in;
s.in_len = in_len;
s.in_pos = 0;
s.chunk = chunk;
s.out.clear();
memset(s.window, 0, sizeof(s.window));
ota_inflate_init(&s, s.window, WINDOW);
s.source_read_cb = read_cb;
int res;
do {
s.dest = s.window;
s.dest_limit = s.window + WINDOW;
res = ota_inflate(&s);
if (res < 0 || s.eof)
return res < 0 ? res : OTA_INFLATE_DATA_ERROR;
s.out.insert(s.out.end(), s.window, s.dest);
if (s.out.size() > PLAIN_SIZE)
return OTA_INFLATE_DATA_ERROR;
} while (res != OTA_INFLATE_DONE);
return res;
}
TEST(OtaInflate, RoundTripThroughWindow) {
auto s = std::make_unique<Session>();
ASSERT_EQ(inflate_all(*s, DEFLATED, sizeof(DEFLATED), 1040), OTA_INFLATE_DONE);
EXPECT_EQ(s->out, build_plain());
EXPECT_EQ(s->in_pos, sizeof(DEFLATED));
}
TEST(OtaInflate, SmallReadChunks) {
auto s = std::make_unique<Session>();
ASSERT_EQ(inflate_all(*s, DEFLATED, sizeof(DEFLATED), 7), OTA_INFLATE_DONE);
EXPECT_EQ(s->out, build_plain());
}
TEST(OtaInflate, StoredBlock) {
auto s = std::make_unique<Session>();
ASSERT_EQ(inflate_all(*s, STORED, sizeof(STORED), 64), OTA_INFLATE_DONE);
auto plain = build_plain();
plain.resize(300);
EXPECT_EQ(s->out, plain);
}
TEST(OtaInflate, TruncatedStreamFails) {
auto s = std::make_unique<Session>();
for (size_t cut : {size_t{1}, size_t{100}, size_t{1000}, sizeof(DEFLATED) - 1}) {
EXPECT_LT(inflate_all(*s, DEFLATED, cut, 1040), 0) << "cut at " << cut;
EXPECT_LE(s->out.size(), PLAIN_SIZE);
}
}
TEST(OtaInflate, TruncatedStoredBlockFails) {
auto s = std::make_unique<Session>();
EXPECT_LT(inflate_all(*s, STORED, sizeof(STORED) - 50, 64), 0);
}
TEST(OtaInflate, CorruptStreamsNeverEscapeTheWindow) {
// Flipped bytes and garbage; the sanitizers check the decoder stays in bounds
auto s = std::make_unique<Session>();
std::vector<uint8_t> bad(DEFLATED, DEFLATED + sizeof(DEFLATED));
// A coarse, non-aligned stride: neighbouring offsets hit the same paths
for (size_t i = 0; i < bad.size(); i += 29) {
bad[i] ^= 0x5a;
inflate_all(*s, bad.data(), bad.size(), 1040);
bad[i] ^= 0x5a;
}
uint32_t x = 99;
std::vector<uint8_t> garbage(2000);
for (int round = 0; round < 50; round++) {
for (auto &b : garbage)
b = lcg_next(x);
inflate_all(*s, garbage.data(), garbage.size(), 1040);
}
}
} // namespace esphome::testing
tests/components/main.cpp
+5
View File
@@ -28,6 +28,11 @@ void setup() {
::testing::InitGoogleTest();
int exit_code = RUN_ALL_TESTS();
// A test folder that never reached the build would otherwise pass as an empty run
if (::testing::UnitTest::GetInstance()->total_test_count() == 0) {
fprintf(stderr, "No tests were linked into this binary\n");
exit_code = 1;
}
exit(exit_code);
}
tests/components/ota/test.bk72xx-ard.yaml
+1
View File
@@ -0,0 +1 @@
<<: !include common.yaml
tests/components/ota/test_backend_contract.cpp
+1 -1
View File
@@ -14,7 +14,7 @@ struct MinimalBackend {
OTAResponseTypes write(uint8_t *data, size_t len) { return OTA_RESPONSE_OK; }
OTAResponseTypes end() { return OTA_RESPONSE_OK; }
void abort() {}
bool supports_compression() { return false; }
static constexpr bool supports_compression() { return false; }
};
static_assert(OTABackendContract<MinimalBackend>);
tests/components/socket/__init__.py
+3
View File
@@ -2,6 +2,7 @@
# shadowing the stdlib module -- no test_*.py (Python-level) tests can live here.
# required for cpp unit testing.
import esphome.codegen as cg
from esphome.components import socket as socket_component
from tests.testing_helpers import ComponentManifestOverride
@@ -12,5 +13,7 @@ def override_manifest(manifest: ComponentManifestOverride) -> None:
async def to_code_testing(config):
await real_to_code(config)
cg.add_define("USE_NETWORK_IPV6", True)
# The gtests link against the filtered helper files.
socket_component.require_tcp_client_link()
manifest.to_code = to_code_testing
tests/components/tcp_uart/common.yaml
+20
View File
@@ -0,0 +1,20 @@
wifi:
ssid: MySSID
password: password1
tcp_uart:
- id: tcp_uart_1
host: 192.0.2.10
port: 502
reconnect_interval: 10s
connected:
name: TCP UART Connected
interval:
- interval: 60s
then:
- lambda: |-
uint8_t byte;
if (id(tcp_uart_1).available() && id(tcp_uart_1).read_byte(&byte)) {
id(tcp_uart_1).write_byte(byte);
}
tests/components/tcp_uart/test.bk72xx-ard.yaml
+2
View File
@@ -0,0 +1,2 @@
packages:
tcp_uart: !include common.yaml
tests/components/tcp_uart/test.esp32-idf.yaml
+2
View File
@@ -0,0 +1,2 @@
packages:
tcp_uart: !include common.yaml
tests/components/tcp_uart/test.esp8266-ard.yaml
+2
View File
@@ -0,0 +1,2 @@
packages:
tcp_uart: !include common.yaml
tests/components/tcp_uart/test.host.yaml
+4
View File
@@ -0,0 +1,4 @@
packages:
tcp_uart: !include common.yaml
wifi: !remove
tests/components/tcp_uart/test.ln882x-ard.yaml
+2
View File
@@ -0,0 +1,2 @@
packages:
tcp_uart: !include common.yaml
tests/components/tcp_uart/test.rp2040-ard.yaml
+2
View File
@@ -0,0 +1,2 @@
packages:
tcp_uart: !include common.yaml
tests/components/tcp_uart/test.rtl87xx-ard.yaml
+2
View File
@@ -0,0 +1,2 @@
packages:
tcp_uart: !include common.yaml
tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py
+2
View File
@@ -1,4 +1,5 @@
import esphome.codegen as cg
from esphome.components import socket
import esphome.config_validation as cv
from esphome.const import CONF_ID
from esphome.types import ConfigType
@@ -18,5 +19,6 @@ CONFIG_SCHEMA = cv.Schema(
async def to_code(config: ConfigType) -> None:
socket.require_ipv4_resolve()
var = cg.new_Pvariable(config[CONF_ID])
await cg.register_component(var, config)
tests/integration/fixtures/external_components/tcp_client_link_test_component/__init__.py
+37
View File
@@ -0,0 +1,37 @@
import esphome.codegen as cg
from esphome.components import socket
from esphome.components.const import CONF_HOST
import esphome.config_validation as cv
from esphome.const import CONF_ID, CONF_PORT
from esphome.types import ConfigType
AUTO_LOAD = ["socket"]
CONF_RECONNECT_INTERVAL = "reconnect_interval"
tcp_client_link_test_component_ns = cg.esphome_ns.namespace(
"tcp_client_link_test_component"
)
TcpClientLinkTestComponent = tcp_client_link_test_component_ns.class_(
"TcpClientLinkTestComponent", cg.Component
)
CONFIG_SCHEMA = cv.Schema(
{
cv.GenerateID(): cv.declare_id(TcpClientLinkTestComponent),
cv.Required(CONF_HOST): cv.string,
cv.Required(CONF_PORT): cv.port,
cv.Optional(
CONF_RECONNECT_INTERVAL, default="1s"
): cv.positive_time_period_milliseconds,
}
).extend(cv.COMPONENT_SCHEMA)
async def to_code(config: ConfigType) -> None:
socket.require_tcp_client_link()
var = cg.new_Pvariable(config[CONF_ID])
await cg.register_component(var, config)
cg.add(var.set_host(config[CONF_HOST]))
cg.add(var.set_port(config[CONF_PORT]))
cg.add(var.set_reconnect_interval(config[CONF_RECONNECT_INTERVAL]))
tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.cpp
+28
View File
@@ -0,0 +1,28 @@
#include "tcp_client_link_test_component.h"
#include "esphome/core/log.h"
namespace esphome::tcp_client_link_test_component {
static const char *const TAG = "tcp_link_test";
void TcpClientLinkTestComponent::setup() { this->link_.begin(TAG); }
void TcpClientLinkTestComponent::loop() {
this->link_.poll();
bool up = this->link_.connected();
if (up != this->was_up_) {
this->was_up_ = up;
ESP_LOGI(TAG, "Link %s", up ? LOG_STR_LITERAL("up") : LOG_STR_LITERAL("down"));
}
if (!up || !this->link_.ready()) {
return;
}
uint8_t buf[64];
ssize_t count = this->link_.read(buf, sizeof(buf));
if (count > 0) {
ESP_LOGI(TAG, "Echoing %d bytes", static_cast<int>(count));
this->link_.write(buf, static_cast<size_t>(count));
}
}
} // namespace esphome::tcp_client_link_test_component
tests/integration/fixtures/external_components/tcp_client_link_test_component/tcp_client_link_test_component.h
+24
View File
@@ -0,0 +1,24 @@
#pragma once
#include "esphome/components/socket/tcp_client_link.h"
#include "esphome/core/component.h"
namespace esphome::tcp_client_link_test_component {
/// Echoes every byte the link receives back to the peer and logs link edges.
class TcpClientLinkTestComponent : public Component {
public:
void set_host(const char *host) { this->link_.set_host(host); }
void set_port(uint16_t port) { this->link_.set_port(port); }
void set_reconnect_interval(uint32_t ms) { this->link_.set_reconnect_interval(ms); }
void setup() override;
void loop() override;
void on_shutdown() override { this->link_.close(); }
protected:
socket::TcpClientLink link_;
bool was_up_{false};
};
} // namespace esphome::tcp_client_link_test_component
tests/integration/fixtures/host_ota_deflate.yaml
+9
View File
@@ -0,0 +1,9 @@
esphome:
name: host-ota-test
host:
api:
ota:
- platform: esphome
port: __OTA_PORT__
logger:
level: DEBUG
tests/integration/fixtures/socket_tcp_client_link.yaml
+20
View File
@@ -0,0 +1,20 @@
esphome:
name: socket-tcp-client-link-test
host:
api:
logger:
level: INFO
external_components:
- source:
type: local
path: EXTERNAL_COMPONENT_PATH
components: [tcp_client_link_test_component]
tcp_client_link_test_component:
host: 127.0.0.1
port: 18123
reconnect_interval: 1s
tests/integration/test_host_ota.py
+78
View File
@@ -15,6 +15,7 @@ from dataclasses import dataclass
import functools
from pathlib import Path
import socket
import zlib
import pytest
@@ -123,6 +124,7 @@ class _Device:
binary_path: Path
proc: asyncio.subprocess.Process | None = None
reboots: int = 0
inflates: int = 0
def __post_init__(self) -> None:
self._rebooted = asyncio.Event()
@@ -131,6 +133,8 @@ class _Device:
if "Rebooting safely" in line:
self.reboots += 1
self._rebooted.set()
if "Inflated " in line and " bytes from " in line:
self.inflates += 1
async def wait_reboot(self, count: int, timeout: float = 10.0) -> None:
async with asyncio.timeout(timeout):
@@ -241,6 +245,80 @@ async def test_host_ota_self_update(
await dev.ota(None, None, "second OTA failed -- listener leaked across execv")
@pytest.mark.asyncio
async def test_host_ota_deflate(
yaml_config: str,
write_yaml_config: ConfigWriter,
compile_esphome: CompileFunction,
reserved_tcp_port: tuple[int, socket.socket],
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Deflate is negotiated by default, an old client gets an uncompressed
upload, and a corrupt stream is rejected without taking the device down."""
dev = _Device(
*await _build(
yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port
)
)
errors: list[str] = []
def on_log(line: str) -> None:
# A corrupt stream is caught by the decoder, by the size check or by
# the MD5 at the end, depending on where the damage lands
if any(
text in line
for text in ("Inflate err", "Inflate overrun", "End update err")
):
errors.append(line)
dev.on_log(line)
real_compress = zlib.compress
def corrupt_compress(data: bytes, *args: object, **kwargs: object) -> bytes:
# Reserved block type in the first header: rejected by the decoder on
# every build, unlike a flipped data bit that may only fail the MD5
out = bytearray(real_compress(data, *args, **kwargs))
out[0] |= 0x06
return bytes(out)
def overlong_compress(data: bytes, *args: object, **kwargs: object) -> bytes:
"""A stream that inflates past the size the client announced."""
return real_compress(data + bytes(8192), *args, **kwargs)
async with run_binary(dev.binary_path, line_callback=on_log) as (proc, _lines):
dev.proc = proc
await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT)
# Default: the host backend cannot store gzip, so the CLI sends deflate
await dev.ota(None, None, "deflate upload failed")
assert dev.inflates == 1, "device did not inflate the upload"
# A client that does not offer deflate is served uncompressed
with monkeypatch.context() as m:
m.setattr(espota2, "CLIENT_FEATURE_SUPPORTS_DEFLATE", 0)
await dev.ota(None, None, "uncompressed upload failed")
assert dev.inflates == 1, "device inflated without a client offer"
# A corrupt stream fails the upload and leaves the device running
with monkeypatch.context() as m:
m.setattr(zlib, "compress", corrupt_compress)
await dev.refused_ota(None, None, "corrupt deflate stream was accepted")
assert errors, "device did not report the corrupt stream"
# So does a stream that inflates past the announced image size
errors.clear()
with monkeypatch.context() as m:
m.setattr(zlib, "compress", overlong_compress)
await dev.refused_ota(None, None, "overlong deflate stream was accepted")
assert any("Inflate overrun" in line for line in errors), (
"device wrote past the announced size"
)
# and it still takes a good upload afterwards
await dev.ota(None, None, "upload after a rejected stream failed")
assert dev.inflates == 2
@pytest.mark.asyncio
async def test_host_ota_encrypted(
yaml_config: str,
tests/integration/test_socket_tcp_client_link.py
+88
View File
@@ -0,0 +1,88 @@
"""Integration test for socket::TcpClientLink on host.
Pytest runs a real TCP server; the device echoes through the link.
Covers connect, read, write, a server-initiated drop and the reconnect.
"""
from __future__ import annotations
import asyncio
import contextlib
import pytest
from .types import APIClientConnectedFactory, RunCompiledFunction
PAYLOAD = b"hello link"
@pytest.mark.asyncio
async def test_socket_tcp_client_link(
yaml_config: str,
run_compiled: RunCompiledFunction,
api_client_connected: APIClientConnectedFactory,
unused_tcp_port_factory,
) -> None:
server_port = unused_tcp_port_factory()
yaml_config = yaml_config.replace("port: 18123", f"port: {server_port}")
echoed: list[bytes] = []
echo_done = asyncio.Event()
reconnected = asyncio.Event()
link_down = asyncio.Event()
second_link_up = asyncio.Event()
link_up_count = 0
def on_log_line(line: str) -> None:
nonlocal link_up_count
if "Link up" in line:
link_up_count += 1
if link_up_count >= 2:
second_link_up.set()
elif "Link down" in line:
link_down.set()
async def handle(
reader: asyncio.StreamReader, writer: asyncio.StreamWriter
) -> None:
if not echo_done.is_set():
writer.write(PAYLOAD)
await writer.drain()
with contextlib.suppress(TimeoutError, asyncio.IncompleteReadError):
echoed.append(
await asyncio.wait_for(reader.readexactly(len(PAYLOAD)), 10)
)
echo_done.set()
# Drop the connection so the link has to reconnect.
writer.close()
return
reconnected.set()
server = await asyncio.start_server(handle, "127.0.0.1", server_port)
try:
async with (
run_compiled(yaml_config, line_callback=on_log_line),
api_client_connected() as client,
):
device_info = await client.device_info()
assert device_info is not None
assert device_info.name == "socket-tcp-client-link-test"
try:
await asyncio.wait_for(echo_done.wait(), timeout=15.0)
except TimeoutError:
pytest.fail("Link never connected or echoed")
assert echoed and echoed[0] == PAYLOAD, "Echo payload mismatch"
try:
await asyncio.wait_for(link_down.wait(), timeout=15.0)
except TimeoutError:
pytest.fail("Link never reported the drop")
try:
await asyncio.wait_for(reconnected.wait(), timeout=15.0)
await asyncio.wait_for(second_link_up.wait(), timeout=15.0)
except TimeoutError:
pytest.fail("Link did not reconnect after the server dropped it")
finally:
server.close()
await server.wait_closed()
tests/script/test_check_idf_py_equivalence.py
+119 -2
View File
@@ -16,7 +16,7 @@ import check_idf_py_equivalence as guard # noqa: E402
from esphome.build_gen import espidf as build_gen # noqa: E402
from esphome.core import CORE # noqa: E402
from esphome.espidf import toolchain # noqa: E402
from esphome.espidf import framework, toolchain # noqa: E402
@pytest.fixture(autouse=True)
@@ -74,10 +74,12 @@ def _run_check(
esphome_rcs: tuple[int, int] = (0, 0),
macro_matches: bool = True,
envs: list[dict[str, str]] | None = None,
versions: tuple[str | None, str | None, str] = ("5.5", "5.5", "5.5"),
) -> tuple[list[str], list[list[str]]]:
"""Run check() with idf.py replaced by ``side_effect``; return problems, calls.
``envs`` collects the env each idf.py call receives.
``envs`` collects the env each idf.py call receives. ``versions`` is what
version.txt, the version header and idf_tools report for the framework.
"""
calls: list[list[str]] = []
@@ -98,6 +100,10 @@ def _run_check(
patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]),
patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]),
patch.object(build_gen, "idf_macro_matches", return_value=macro_matches),
patch.object(framework, "read_idf_version_txt", return_value=versions[0]),
patch.object(framework, "read_idf_version_header", return_value=versions[1]),
patch.object(framework, "idf_tools_version", return_value=versions[2]),
patch.object(guard, "_lwip_empty_source_problems", return_value=[]),
patch.object(guard.subprocess, "run", side_effect=run),
patch.dict(os.environ),
):
@@ -285,6 +291,24 @@ def test_check_fails_loudly_when_the_idf_macro_changed(tmp_path: Path) -> None:
assert calls == []
@pytest.mark.parametrize(
("versions", "source"),
[(("5.4", "5.5", "5.5"), "txt"), (("5.5", None, "5.5"), "header")],
)
def test_check_fails_loudly_when_the_version_read_drifts(
tmp_path: Path, versions: tuple[str | None, str | None, str], source: str
) -> None:
"""An IDF bump that changes how idf_tools reads its version must fail CI;
both sources are checked since a managed tree never reaches the header."""
tree = _make_tree(tmp_path)
problems, calls = _run_check(tree, versions=versions)
ours = versions[0] if source == "txt" else versions[1]
assert problems == [
guard.VERSION_DRIFT.format(ours=ours, source=source, theirs="5.5")
]
assert calls == []
def test_main_without_build_trees(
tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
@@ -296,6 +320,99 @@ def test_main_without_build_trees(
assert "No native ESP-IDF build tree found" in capsys.readouterr().out
def _make_lwip_tree(tmp_path: Path, objects: list[str], config: dict) -> Path:
"""A tree with lwip objects, their sdkconfig.json and an nm in the cache."""
tree = _make_tree(tmp_path)
objdir = tree / "build" / "esp-idf" / "lwip" / "CMakeFiles" / "__idf_lwip.dir"
for name in objects:
(objdir / name).parent.mkdir(parents=True, exist_ok=True)
(objdir / name).write_bytes(b"x")
(tree / "build" / "config").mkdir(parents=True, exist_ok=True)
(tree / "build" / "config" / "sdkconfig.json").write_text(json.dumps(config))
with (tree / "build" / "CMakeCache.txt").open("a") as cache:
cache.write("CMAKE_NM:FILEPATH=/tools/nm\n")
return tree
def _run_lwip_check(
tree: Path,
non_empty: set[str] = frozenset(),
failing: set[str] = frozenset(),
calls: list[list[str]] | None = None,
) -> list[str]:
"""Run the lwip check with nm faked; ``calls`` collects the nm commands."""
def run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess:
if calls is not None:
calls.append(cmd)
name = Path(cmd[-1]).name
if name in failing:
return subprocess.CompletedProcess(cmd, 1, "", "bad object")
return subprocess.CompletedProcess(
cmd, 0, "symbol\n" if name in non_empty else "", ""
)
with (
patch.object(toolchain, "run_reconfigure", return_value=0) as reconfigure,
patch.object(toolchain, "_run_ninja", return_value=0),
patch.object(guard.subprocess, "run", side_effect=run),
):
problems = guard._lwip_empty_source_problems(tree)
reconfigure.assert_called_once_with(
extra_env={build_gen.LWIP_FULL_SOURCES_ENV: "1"}
)
return problems
def test_lwip_check_inspects_only_the_dropped_sources(tmp_path: Path) -> None:
"""An option that is on, or absent (invisible), keeps its sources unchecked."""
tree = _make_lwip_tree(
tmp_path,
[
"lwip/src/netif/ppp/auth.c.obj",
"lwip/src/core/ipv6/ip6.c.obj",
"lwip/src/core/ipv4/autoip.c.obj",
],
{"LWIP_PPP_SUPPORT": False, "LWIP_IPV6": True},
)
calls: list[list[str]] = []
assert _run_lwip_check(tree, calls=calls) == []
assert [Path(c[-1]).name for c in calls] == ["auth.c.obj"]
def test_lwip_check_flags_a_dropped_source_with_symbols(tmp_path: Path) -> None:
tree = _make_lwip_tree(
tmp_path, ["lwip/src/netif/ppp/auth.c.obj"], {"LWIP_PPP_SUPPORT": False}
)
assert _run_lwip_check(tree, non_empty={"auth.c.obj"}) == [
guard.LWIP_NOT_EMPTY.format(source="auth.c", option="CONFIG_LWIP_PPP_SUPPORT")
]
def test_lwip_check_flags_a_failed_nm(tmp_path: Path) -> None:
"""A broken nm must not pass as an empty object."""
tree = _make_lwip_tree(
tmp_path, ["lwip/src/netif/ppp/auth.c.obj"], {"LWIP_PPP_SUPPORT": False}
)
assert _run_lwip_check(tree, failing={"auth.c.obj"}) == [
guard.LWIP_NM_FAILED.format(source="auth.c", error="bad object")
]
def test_lwip_check_fails_per_pattern_that_matched_nothing(tmp_path: Path) -> None:
"""A stale pattern is reported even while the others still match."""
tree = _make_lwip_tree(
tmp_path,
["lwip/src/netif/ppp/auth.c.obj"],
{"LWIP_PPP_SUPPORT": False, "LWIP_STATS": False},
)
assert _run_lwip_check(tree) == [
guard.LWIP_NOTHING_MATCHED.format(
regex="/core/stats[.]c$", option="CONFIG_LWIP_STATS"
)
]
@pytest.mark.parametrize(("problems", "rc"), [([], 0), (["idf.py changed x"], 1)])
def test_main_reports_each_tree(
tmp_path: Path,
tests/unit_tests/build_gen/test_espidf.py
+24
View File
@@ -222,6 +222,30 @@ def test_get_project_cmakelists_size_command_uses_json2() -> None:
assert "--format=json2" in content
def test_get_project_cmakelists_drops_empty_lwip_sources() -> None:
"""The filter comes after project(), where the lwip target exists."""
from esphome.build_gen.espidf import (
LWIP_EMPTY_SOURCES,
LWIP_FULL_SOURCES_ENV,
lwip_empty_source_gate,
)
content = _render()
filter_at = content.index(
"set_property(TARGET ${esphome_lwip_lib} PROPERTY SOURCES"
)
assert filter_at > content.index("project(")
assert f"NOT DEFINED ENV{{{LWIP_FULL_SOURCES_ENV}}}" in content
for entry in LWIP_EMPTY_SOURCES:
assert lwip_empty_source_gate(*entry) in content
def test_get_project_cmakelists_declares_map_as_link_byproduct() -> None:
"""The link declares the map so size can build in the same ninja run."""
content = _render()
assert "BYPRODUCTS ${CMAKE_BINARY_DIR}/${CMAKE_PROJECT_NAME}.map" in content
def test_get_project_cmakelists_uses_supplied_builtin_components() -> None:
"""A cached list replaces project_description.json and is still filtered
by EXCLUDE_COMPONENTS."""
tests/unit_tests/components/socket/test_socket_source_filter.py
+38
View File
@@ -0,0 +1,38 @@
"""Tests for the socket component's source filtering and require functions."""
from unittest.mock import patch
from esphome.components import socket
from esphome.core import Define
def test_helper_files_filtered_until_required() -> None:
"""ipv4_resolve.cpp and tcp_client_link.cpp compile only when required."""
with patch("esphome.config_helpers.CORE") as mock_core:
mock_core.defines = set()
filtered = socket.FILTER_SOURCE_FILES()
assert "ipv4_resolve.cpp" in filtered
assert "tcp_client_link.cpp" in filtered
mock_core.defines = {Define("USE_SOCKET_IPV4_RESOLVE")}
filtered = socket.FILTER_SOURCE_FILES()
assert "ipv4_resolve.cpp" not in filtered
assert "tcp_client_link.cpp" in filtered
mock_core.defines = {
Define("USE_SOCKET_IPV4_RESOLVE"),
Define("USE_SOCKET_TCP_CLIENT_LINK"),
}
filtered = socket.FILTER_SOURCE_FILES()
assert "ipv4_resolve.cpp" not in filtered
assert "tcp_client_link.cpp" not in filtered
def test_require_tcp_client_link_pulls_in_the_resolver() -> None:
"""require_tcp_client_link() sets both defines; the link reads the resolver."""
with patch.object(socket.cg, "add_define") as add_define:
socket.require_tcp_client_link()
assert {call.args[0] for call in add_define.call_args_list} == {
"USE_SOCKET_IPV4_RESOLVE",
"USE_SOCKET_TCP_CLIENT_LINK",
}
tests/unit_tests/core/test_config.py
+17
View File
@@ -1493,3 +1493,20 @@ async def test_add_platformio_options_native_arduino(
assert "board_build.ldscript is ignored" in caplog.text
assert "'arduino' toolchain" in caplog.text
assert "upload_speed" not in caplog.text
def test_filter_source_files_drops_util_cpp_without_mqtt() -> None:
"""util.cpp compiles only on MQTT builds; the header stubs it otherwise."""
CORE.data[KEY_CORE] = {
KEY_TARGET_PLATFORM: "esp8266",
KEY_TARGET_FRAMEWORK: "arduino",
}
CORE.defines = set()
excluded = config.FILTER_SOURCE_FILES()
assert "util.cpp" in excluded
# The platform map still contributes through the composed function.
assert "static_task.cpp" in excluded
CORE.defines = {core.Define("USE_API"), core.Define("USE_MQTT")}
assert "util.cpp" not in config.FILTER_SOURCE_FILES()
tests/unit_tests/fixtures/substitutions/01-include.approved.yaml
+7
View File
@@ -13,3 +13,10 @@ test_list:
- a: alpha
- b: beta
- c: The value of C is $c
- {}
- values:
- var1: '1'
- a: alpha
- b: delta
- c: The value of C is saw
- {}
tests/unit_tests/fixtures/substitutions/01-include.input.yaml
+19
View File
@@ -13,3 +13,22 @@ test_list:
file: inc1.yaml
vars:
b: "beta"
condition: true
- !include
file: inc1.yaml
vars:
b: "gamma"
c: "saw"
condition: false
- !include
file: inc1.yaml
vars:
b: "delta"
c: "saw"
condition: ${var1 == "1"}
- !include
file: inc1.yaml
vars:
b: "epsilon"
c: "saw"
condition: ${var1 == "2"}
tests/unit_tests/fixtures/substitutions/12-yaml-merge.input.yaml
+6
View File
@@ -7,4 +7,10 @@ test_list:
file: inc3.yaml
vars:
y: 8
condition: true
<<: !include
file: inc3.yaml
vars:
y: 10
condition: false
after: Content after
tests/unit_tests/fixtures/yaml_util/includes/true.yaml
+2
View File
@@ -0,0 +1,2 @@
---
true
tests/unit_tests/fixtures/yaml_util/includetest.yaml
+3 -3
View File
@@ -11,8 +11,8 @@ esphome:
# should be substituted as 'original',
# not overwritten by vars in the !include above
name: ${name}
name_add_mac_suffix: true
libraries: !include {file: includes/list.yaml, vars: {var1: Wire}}
name_add_mac_suffix: !include includes/true.yaml
libraries: !include {file: includes/list.yaml, vars: {var1: Wire}, condition: "true"}
esp8266:
board: !include {file: includes/scalar.yaml, vars: {var1: nodemcu}}
board: !include {file: includes/scalar.yaml, vars: {var1: nodemcu}, condition: true}
tests/unit_tests/fixtures/yaml_util/includetest_invalid_condition_type.yaml
+3
View File
@@ -0,0 +1,3 @@
---
esphome:
libraries: !include {file: includes/list.yaml, vars: {var1: Wire}, condition: 1}
tests/unit_tests/fixtures/yaml_util/includetest_no_file.yaml
+3
View File
@@ -0,0 +1,3 @@
---
esphome:
libraries: !include {vars: {var1: Wire}}
tests/unit_tests/test_bundle.py
+1 -1
View File
@@ -1384,7 +1384,7 @@ def test_force_load_skips_duplicate_include_file() -> None:
self.parent_file = Path("root.yaml")
self.load_calls = 0
def has_unresolved_expressions(self) -> bool:
def has_unresolved_file(self) -> bool:
return False
def load(self) -> dict[str, Any]:
tests/unit_tests/test_espidf_framework.py
+71 -6
View File
@@ -44,6 +44,7 @@ from esphome.espidf.framework import (
check_esp_idf_install,
get_framework_env,
get_idf_tools_path,
idf_tools_version,
)
from esphome.framework_helpers import _tar_extract_all, get_python_env_executable_path
@@ -1468,19 +1469,59 @@ def test_demote_unused_tools_already_patched_is_noop(tmp_path: Path) -> None:
# ---------------------------------------------------------------------------
def test_get_idf_version_parses_stdout(tmp_path: Path) -> None:
def _write_idf_version_header(root: Path, major: int, minor: int) -> None:
include = root / "components" / "esp_common" / "include"
include.mkdir(parents=True)
(include / "esp_idf_version.h").write_text(
f"#define ESP_IDF_VERSION_MAJOR {major}\n"
"/** Minor version number (x.X.x) */\n"
f"#define ESP_IDF_VERSION_MINOR {minor}\n"
"#define ESP_IDF_VERSION_PATCH 0\n",
encoding="utf-8",
)
def test_get_idf_version_reads_version_txt(tmp_path: Path) -> None:
"""version.txt wins and gives major.minor, as idf_tools returns it."""
(tmp_path / "version.txt").write_text("v5.5.5\n", encoding="utf-8")
_write_idf_version_header(tmp_path, 6, 1)
assert _get_idf_version(tmp_path) == "5.5"
def test_get_idf_version_falls_back_to_the_header(tmp_path: Path) -> None:
"""A version.txt that does not match (a git ref) defers to the header."""
(tmp_path / "version.txt").write_text("vrelease/v6.1\n", encoding="utf-8")
_write_idf_version_header(tmp_path, 6, 1)
assert _get_idf_version(tmp_path) == "6.1"
def test_get_idf_version_raises_without_a_source(tmp_path: Path) -> None:
with pytest.raises(RuntimeError, match="Can't get ESP-IDF version"):
_get_idf_version(tmp_path)
def test_get_idf_version_wraps_an_unreadable_source(tmp_path: Path) -> None:
"""A source that cannot be decoded keeps the RuntimeError contract."""
(tmp_path / "version.txt").write_bytes(b"\xff\xfev")
with pytest.raises(RuntimeError, match="Can't get ESP-IDF version") as info:
_get_idf_version(tmp_path)
assert isinstance(info.value.__cause__, UnicodeError)
def test_idf_tools_version_runs_the_framework_script(tmp_path: Path) -> None:
with patch(
"esphome.espidf.framework.run_command", return_value=(True, "5.1.2\n", "")
):
assert _get_idf_version(tmp_path) == "5.1.2"
"esphome.espidf.framework.run_command", return_value=(True, "5.5\n", "")
) as run:
assert idf_tools_version(tmp_path) == "5.5"
assert run.call_args.args[0][1].endswith("get_idf_version.py")
def test_get_idf_version_raises_on_failure(tmp_path: Path) -> None:
def test_idf_tools_version_raises_on_failure(tmp_path: Path) -> None:
with (
patch("esphome.espidf.framework.run_command", return_value=(False, "", "boom")),
pytest.raises(RuntimeError, match="Can't get ESP-IDF version"),
):
_get_idf_version(tmp_path)
idf_tools_version(tmp_path)
def test_get_idf_tool_paths_parses_json(tmp_path: Path) -> None:
@@ -1511,6 +1552,30 @@ def test_get_idf_tool_paths_raises_on_failure(tmp_path: Path) -> None:
_get_idf_tool_paths(tmp_path)
def test_get_idf_tool_paths_runs_the_script_once_per_build(tmp_path: Path) -> None:
payload = json.dumps({"paths_to_export": ["/a"], "export_vars": {"X": "1"}})
env = {"IDF_TOOLS_PATH": str(tmp_path / "tools")}
with patch(
"esphome.espidf.framework.run_command", return_value=(True, payload, "")
) as run:
first = _get_idf_tool_paths(tmp_path, env)
second = _get_idf_tool_paths(tmp_path, env)
assert run.call_count == 1
assert first == second == (["/a"], {"X": "1"})
def test_get_idf_tool_paths_does_not_cache_a_failure(tmp_path: Path) -> None:
payload = json.dumps({"paths_to_export": ["/a"], "export_vars": {}})
with patch(
"esphome.espidf.framework.run_command",
side_effect=[(False, "", "err"), (True, payload, "")],
) as run:
with pytest.raises(RuntimeError, match="Can't get ESP-IDF tool paths"):
_get_idf_tool_paths(tmp_path)
assert _get_idf_tool_paths(tmp_path) == (["/a"], {})
assert run.call_count == 2
def test_get_python_version_parses_stdout(tmp_path: Path) -> None:
with patch(
"esphome.espidf.framework.run_command", return_value=(True, "3.11.0\n", "")
tests/unit_tests/test_espidf_toolchain.py
+28 -24
View File
@@ -534,8 +534,8 @@ def _record_compile_calls(
def record_save(components: list[str]) -> None:
calls.append(("save", components))
def record_ninja(target: str, **kwargs: object) -> int:
if target == "all":
def record_ninja(*targets: str, **kwargs: object) -> int:
if "all" in targets:
calls.append(("build",))
return 0
@@ -886,7 +886,7 @@ def test_run_compile_full_deps_skips_fragment_check(
def test_run_compile_passes_compile_process_limit(
setup_core: Path, limit: int | None
) -> None:
"""compile_process_limit is the job limit for both ninja runs."""
"""compile_process_limit is the job limit of the one ninja run."""
_setup_build(setup_core)
esphome = {} if limit is None else {CONF_COMPILE_PROCESS_LIMIT: limit}
@@ -894,8 +894,14 @@ def test_run_compile_passes_compile_process_limit(
assert toolchain.run_compile({CONF_ESPHOME: esphome}, verbose=False) == 0
assert mock_run.call_args_list == [
call("all", verbose=False, jobs=limit, progress=True),
call("size", verbose=False, jobs=limit, extra_env=toolchain._size_env()),
call(
"all",
"size",
verbose=False,
jobs=limit,
progress=True,
extra_env=toolchain._size_env(),
),
]
@@ -1185,14 +1191,17 @@ def test_run_ninja_filters_and_reports_failure(
patch.object(toolchain, "_print_hints") as mock_hints,
):
mock_run.return_value = 1
assert toolchain._run_ninja("all", verbose=False, jobs=None, progress=True) == 1
assert (
toolchain._run_ninja("all", "size", verbose=False, jobs=None, progress=True)
== 1
)
log_path = mock_run.call_args.kwargs["log_path"]
assert log_path.name == "ninja_all_output.log"
assert log_path.name == "ninja_all_size_output.log"
mock_hints.assert_called_once_with(log_path)
assert mock_run.call_args.args[0] == ["/tools/ninja", "all"]
assert mock_run.call_args.args[0] == ["/tools/ninja", "all", "size"]
assert mock_run.call_args.kwargs["filter_lines"] is toolchain.FILTER_IDF_LINES
assert mock_run.call_args.kwargs["progress"] is True
assert "ninja all failed with exit code 1" in caplog.text
assert "ninja all size failed with exit code 1" in caplog.text
@pytest.mark.parametrize("reconfigure_rc", [0, 5])
@@ -1215,17 +1224,12 @@ def test_run_compile_reconfigures_when_cache_entries_change(
assert mock_ninja.called is (reconfigure_rc == 0)
@pytest.mark.parametrize("failing", ["all", "size"])
def test_run_compile_stops_on_ninja_failure(setup_core: Path, failing: str) -> None:
"""A failed build skips size; either failure skips the summary."""
def test_run_compile_stops_on_ninja_failure(setup_core: Path) -> None:
"""A failed ninja run skips the summary."""
_setup_build(setup_core)
with _up_to_date_compile(lambda target, **kw: 7 if target == failing else 0) as (
mock_ninja,
mock_summary,
):
with _up_to_date_compile(lambda *targets, **kw: 7) as (mock_ninja, mock_summary):
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 7
targets = [c.args[0] for c in mock_ninja.call_args_list]
assert targets == (["all"] if failing == "all" else ["all", "size"])
assert [c.args for c in mock_ninja.call_args_list] == [("all", "size")]
mock_summary.assert_not_called()
@@ -1236,11 +1240,11 @@ def test_run_compile_testing_mode_builds_memory_ld_first(
"""Testing mode builds and patches memory.ld before the main build."""
_setup_build(setup_core)
CORE.testing_mode = True
targets: list[str] = []
targets: list[tuple[str, ...]] = []
def record(target: str, **kwargs: object) -> int:
targets.append(target)
return memory_ld_rc if target.endswith("memory.ld") else 0
def record(*run_targets: str, **kwargs: object) -> int:
targets.append(run_targets)
return memory_ld_rc if run_targets[0].endswith("memory.ld") else 0
with (
_up_to_date_compile(record),
@@ -1249,10 +1253,10 @@ def test_run_compile_testing_mode_builds_memory_ld_first(
assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == memory_ld_rc
memory_ld = str(Path("esp-idf", "esp_system", "ld", "memory.ld"))
if memory_ld_rc:
assert targets == [memory_ld]
assert targets == [(memory_ld,)]
mock_patch.assert_not_called()
else:
assert targets == [memory_ld, "all", "size"]
assert targets == [(memory_ld,), ("all", "size")]
mock_patch.assert_called_once()
tests/unit_tests/test_espota2.py
+49 -2
View File
@@ -12,6 +12,7 @@ from pathlib import Path
import socket
import struct
from unittest.mock import Mock, call, patch
import zlib
import pytest
from pytest import CaptureFixture
@@ -354,6 +355,7 @@ def test_perform_ota_successful_md5_auth(
espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION
| espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH
| espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL
| espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE
]
)
)
@@ -601,12 +603,16 @@ def test_perform_ota_upload_error(mock_socket: Mock, mock_file: io.BytesIO) -> N
espota2.perform_ota(mock_socket, None, mock_file, "test.bin")
def _no_auth_handshake(version: int) -> list[bytes]:
def _no_auth_handshake(version: int, server_features: int | None = None) -> list[bytes]:
"""Recv responses for a handshake without auth, up to the MD5 check."""
if server_features is None:
features = [bytes([espota2.RESPONSE_HEADER_OK])]
else:
features = [bytes([espota2.RESPONSE_FEATURE_FLAGS]), bytes([server_features])]
return [
bytes([espota2.RESPONSE_OK]), # First byte of version response
bytes([version]), # Version number
bytes([espota2.RESPONSE_HEADER_OK]), # Features response
*features,
bytes([espota2.RESPONSE_AUTH_OK]), # No auth required
bytes([espota2.RESPONSE_UPDATE_PREPARE_OK]), # Binary size OK
bytes([espota2.RESPONSE_BIN_MD5_OK]), # MD5 checksum OK
@@ -1054,6 +1060,7 @@ def test_perform_ota_successful_sha256_auth(
espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION
| espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH
| espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL
| espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE
]
)
)
@@ -1110,6 +1117,7 @@ def test_perform_ota_sha256_fallback_to_md5(
espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION
| espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH
| espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL
| espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE
]
)
)
@@ -1219,6 +1227,7 @@ def test_perform_ota_extended_protocol_app(
espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION
| espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH
| espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL
| espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE
]
)
)
@@ -1279,6 +1288,7 @@ def test_perform_ota_successful_partition_table(
espota2.CLIENT_FEATURE_SUPPORTS_COMPRESSION
| espota2.CLIENT_FEATURE_SUPPORTS_SHA256_AUTH
| espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL
| espota2.CLIENT_FEATURE_SUPPORTS_DEFLATE
]
)
)
@@ -1507,3 +1517,40 @@ def test_check_error_passes_non_error_when_expect_is_none() -> None:
espota2.check_error([espota2.RESPONSE_OK], None)
espota2.check_error([espota2.RESPONSE_HEADER_OK], None)
espota2.check_error([espota2.RESPONSE_FEATURE_FLAGS], None)
# Device replies after the MD5 check for a one-chunk upload
_UPLOAD_TAIL = [
bytes([espota2.RESPONSE_CHUNK_OK]),
bytes([espota2.RESPONSE_RECEIVE_OK]),
bytes([espota2.RESPONSE_UPDATE_END_OK]),
]
@pytest.mark.usefixtures("mock_time")
@pytest.mark.parametrize(
"server_features",
[
espota2.SERVER_FEATURE_SUPPORTS_DEFLATE,
# Binding offer: deflate wins over gzip
espota2.SERVER_FEATURE_SUPPORTS_DEFLATE
| espota2.SERVER_FEATURE_SUPPORTS_COMPRESSION,
],
)
def test_perform_ota_with_deflate(mock_socket: Mock, server_features: int) -> None:
"""The device gets a raw deflate stream, both sizes and the image MD5."""
original_content = b"firmware" * 100
mock_socket.recv.side_effect = (
_no_auth_handshake(espota2.OTA_VERSION_2_0, server_features) + _UPLOAD_TAIL
)
espota2.perform_ota(mock_socket, None, io.BytesIO(original_content), "test.bin")
sent = [c[0][0] for c in mock_socket.sendall.call_args_list]
# magic, features, ota type, size, image size, md5, data, end ack
sent_size = struct.unpack(">I", sent[3])[0]
assert sent[4] == len(original_content).to_bytes(espota2.SIZE_FIELD_BYTES, "big")
payload = sent[6]
assert len(payload) == sent_size < len(original_content)
assert zlib.decompress(payload, -espota2.DEFLATE_WINDOW_BITS) == original_content
assert sent[5] == hashlib.md5(original_content).hexdigest().encode()
tests/unit_tests/test_substitutions.py
+83 -12
View File
@@ -573,6 +573,65 @@ def test_undefined_variable_warning(
assert "'undefined_var' is undefined" in caplog.text
def test_undefined_variable_warning_avoided_by_include_condition(
tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""Compare two different ways to conditionally enable an !include file
that defines and uses new variables. One way generates a undefined
variable warning when disabled and the other does not."""
include_file = tmp_path / "include.yaml"
include_file.write_text("substitutions:\n var: 1\nresult: ${var}\n")
method_a_file = tmp_path / "method_a.yaml"
method_a_file.write_text(
"packages:\n"
" - !include\n"
" file: include.yaml\n"
" condition: ${enable == 'y'}\n"
)
method_b_file = tmp_path / "method_b.yaml"
method_b_file.write_text(
"substitutions:\n"
" content: !include include.yaml\n"
"packages:\n"
" - ${content if enable == 'y' else {}}\n"
)
def process_file(file, enable):
with caplog.at_level(logging.WARNING):
args = {"enable": enable}
config = yaml_util.load_yaml(file)
config = do_packages_pass(config, command_line_substitutions=args)
return substitutions.do_substitution_pass(
config, command_line_substitutions=args
)
# Case: method A with include enabled produces a result and no warnings
config = process_file(method_a_file, "y")
assert caplog.text == ""
assert config["packages"][0]["result"] == 1
# Case: method A with include disabled produces no result and no warnings
# because the file was not even loaded
config = process_file(method_a_file, "n")
assert caplog.text == ""
assert config["packages"][0] == {}
# Case: method B with include enabled produces a result and no warnings (same as method A)
config = process_file(method_b_file, "y")
assert caplog.text == ""
assert config["packages"][0]["result"] == 1
# Case: method B with include disabled produces no result and an undefined variable warning
# because the file is loaded into a variable and its own variables cannot be expanded
# (different from method A)
config = process_file(method_b_file, "n")
assert (
"Could not resolve substitution variable 'content': 'var' is undefined"
in caplog.text
)
assert config["packages"][0] == {}
def test_password_field_warnings_suppressed(
caplog: pytest.LogCaptureFixture,
) -> None:
@@ -717,11 +776,9 @@ def test_resolve_package_max_depth_exceeded(tmp_path: Path) -> None:
# Each call to the loader returns a fresh IncludeFile pointing at itself,
# so PACKAGE_SCHEMA always sees an IncludeFile and never a dict.
def always_returns_include(path: Path) -> yaml_util.IncludeFile:
return yaml_util.IncludeFile(parent, path.name, None, always_returns_include)
return yaml_util.IncludeFile(parent, path.name, always_returns_include)
package_config = yaml_util.IncludeFile(
parent, "test.yaml", None, always_returns_include
)
package_config = yaml_util.IncludeFile(parent, "test.yaml", always_returns_include)
processor = _PackageProcessor({}, None)
with pytest.raises(
cv.Invalid,
@@ -791,6 +848,22 @@ def test_include_filename_jinja_expression_with_path_separator(
assert config["result"] == {"value": 42}
def test_include_condition_substitution_undefined_var(tmp_path: Path) -> None:
"""!include with an undefined substitution variable raises cv.Invalid.
The error message must reference the unresolved condition template so the
user knows which include failed, rather than seeing a bare file-not-found.
"""
main_file = tmp_path / "main.yaml"
main_file.write_text(
"result: !include\n file: file.yaml\n condition: ${undefined_var}\n"
)
config = yaml_util.load_yaml(main_file)
with pytest.raises(cv.Invalid, match=r"\$\{undefined_var\}"):
substitutions.do_substitution_pass(config)
def test_raise_first_undefined_logs_extras_at_debug(
caplog: pytest.LogCaptureFixture,
) -> None:
@@ -838,7 +911,7 @@ def test_do_substitution_pass_included_substitutions_must_be_mapping(
def loader(path: Path):
return ["not", "a", "mapping"]
include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader)
include = yaml_util.IncludeFile(parent, "subs.yaml", loader)
config = OrderedDict({CONF_SUBSTITUTIONS: include})
with pytest.raises(
@@ -861,7 +934,7 @@ def test_do_packages_pass_included_substitutions_must_be_mapping(
def loader(path: Path):
return ["not", "a", "mapping"]
include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader)
include = yaml_util.IncludeFile(parent, "subs.yaml", loader)
config = OrderedDict(
{
CONF_SUBSTITUTIONS: include,
@@ -888,9 +961,7 @@ def test_resolve_package_undefined_var_in_include_filename(tmp_path: Path) -> No
def loader(path: Path):
raise EsphomeError(f"Error reading file {path}: No such file")
package_config = yaml_util.IncludeFile(
parent, "${undefined_var}.yaml", None, loader
)
package_config = yaml_util.IncludeFile(parent, "${undefined_var}.yaml", loader)
processor = _PackageProcessor({}, None)
with pytest.raises(cv.Invalid, match="unresolved substitutions"):
processor.resolve_package(package_config, substitutions.ContextVars(), [])
@@ -906,7 +977,7 @@ def test_resolve_include_error_shows_expanded_from_when_substituted(
def failing_loader(_path: Path) -> None:
raise EsphomeError("File not found")
include = yaml_util.IncludeFile(parent, "${device}.yaml", None, failing_loader)
include = yaml_util.IncludeFile(parent, "${device}.yaml", failing_loader)
context = substitutions.ContextVars({"device": "my_device"})
with pytest.raises(cv.Invalid) as exc_info:
@@ -927,7 +998,7 @@ def test_resolve_include_error_no_expanded_from_for_literal_filename(
def failing_loader(_path: Path) -> None:
raise EsphomeError("File not found")
include = yaml_util.IncludeFile(parent, "literal.yaml", None, failing_loader)
include = yaml_util.IncludeFile(parent, "literal.yaml", failing_loader)
with pytest.raises(cv.Invalid) as exc_info:
substitutions.resolve_include(include, [], substitutions.ContextVars())
@@ -946,7 +1017,7 @@ def test_include_vars_applied_to_lambda_value(tmp_path: Path) -> None:
included.write_text('!lambda |-\n return "${foo}";\n')
include = yaml_util.IncludeFile(
tmp_path / "main.yaml", "lambda.yaml", {"foo": "bar"}, yaml_util.load_yaml
tmp_path / "main.yaml", "lambda.yaml", yaml_util.load_yaml, vars={"foo": "bar"}
)
config = OrderedDict({"value": include.load()})
result = substitutions.do_substitution_pass(config)
tests/unit_tests/test_yaml_util.py
+187 -33
View File
@@ -43,17 +43,37 @@ def clear_core_frontmatter() -> None:
core.CORE.frontmatter = {}
def test_include_with_vars(fixture_path: Path) -> None:
def test_include(fixture_path: Path) -> None:
"""Test !include with and without vars, with and without conditions"""
yaml_file = fixture_path / "yaml_util" / "includetest.yaml"
actual = yaml_util.load_yaml(yaml_file)
actual = substitutions.do_substitution_pass(actual, None)
assert actual["esphome"]["name"] == "original"
assert actual["esphome"]["name_add_mac_suffix"]
assert actual["esphome"]["libraries"][0] == "Wire"
assert actual["esp8266"]["board"] == "nodemcu"
assert actual["wifi"]["ssid"] == "my_custom_ssid"
def test_include_with_no_file(fixture_path: Path) -> None:
"""Ensure that an error is emitted when the file field is missing."""
yaml_file = fixture_path / "yaml_util" / "includetest_no_file.yaml"
with pytest.raises(EsphomeError, match=r"Must include 'file'"):
yaml_util.load_yaml(yaml_file)
def test_include_with_invalid_condition_type(fixture_path: Path) -> None:
"""Ensure that an error is emitted when the condition field is an invalid type."""
yaml_file = fixture_path / "yaml_util" / "includetest_invalid_condition_type.yaml"
with pytest.raises(
EsphomeError, match=r"Include 'condition' must be a boolean or string"
):
yaml_util.load_yaml(yaml_file)
def test_loading_a_missing_file(fixture_path):
"""We throw EsphomeError when loading a missing file."""
yaml_file = fixture_path / "yaml_util" / "missing.yaml"
@@ -64,7 +84,7 @@ def test_loading_a_missing_file(fixture_path):
def test_parsing_with_custom_loader(fixture_path):
"""Test custom loader used for vscode connection
Default loader is tested in test_include_with_vars
Default loader is tested in test_include
"""
yaml_file = fixture_path / "yaml_util" / "includetest.yaml"
@@ -78,10 +98,11 @@ def test_parsing_with_custom_loader(fixture_path):
# substitute config to expand includes:
substitutions.substitute(config, [], substitutions.ContextVars(), False)
assert len(loader_calls) == 3
assert len(loader_calls) == 4
assert loader_calls[0].parts[-2:] == ("includes", "included.yaml")
assert loader_calls[1].parts[-2:] == ("includes", "list.yaml")
assert loader_calls[2].parts[-2:] == ("includes", "scalar.yaml")
assert loader_calls[1].parts[-2:] == ("includes", "true.yaml")
assert loader_calls[2].parts[-2:] == ("includes", "list.yaml")
assert loader_calls[3].parts[-2:] == ("includes", "scalar.yaml")
def test_construct_secret_simple(fixture_path: Path) -> None:
@@ -570,7 +591,7 @@ def test_represent_remove() -> None:
def test_represent_include_file() -> None:
"""Test that IncludeFile objects are dumped as !include scalars."""
include = yaml_util.IncludeFile(
Path("/fake/main.yaml"), "path/to/file.yaml", None, lambda _: {}
Path("/fake/main.yaml"), "path/to/file.yaml", lambda _: {}
)
assert yaml_util.dump({"key": include}) == "key: !include 'path/to/file.yaml'\n"
@@ -580,13 +601,27 @@ def test_represent_include_file_with_vars() -> None:
include = yaml_util.IncludeFile(
Path("/fake/main.yaml"),
"path/to/file.yaml",
{"key": "value"},
lambda _: {},
vars={"key": "value"},
)
assert (
yaml_util.dump({"key": include})
== "key: !include\n file: path/to/file.yaml\n vars:\n key: value\n"
)
def test_represent_include_file_with_condition() -> None:
"""Test that IncludeFile with condition is dumped as !include mapping form."""
include = yaml_util.IncludeFile(
Path("/fake/main.yaml"),
"path/to/file.yaml",
lambda _: {},
condition="true",
)
assert (
yaml_util.dump({"key": include})
== "key: !include\n file: path/to/file.yaml\n condition: 'true'\n"
)
result = yaml_util.dump({"key": include})
assert "!include" in result
assert "file: path/to/file.yaml" in result
assert "key: value" in result
def test_represent_include_file_with_data_base_mixin() -> None:
@@ -596,7 +631,7 @@ def test_represent_include_file_with_data_base_mixin() -> None:
subclass. add_multi_representer must match this subclass through the MRO.
"""
include = yaml_util.IncludeFile(
Path("/fake/main.yaml"), "common/spi.yaml", None, lambda _: {}
Path("/fake/main.yaml"), "common/spi.yaml", lambda _: {}
)
wrapped = yaml_util.make_data_base(include)
assert isinstance(wrapped, yaml_util.ESPHomeDataBase)
@@ -609,7 +644,7 @@ def test_represent_include_file_with_data_base_mixin() -> None:
def test_include_file_repr(tmp_path: Path) -> None:
"""repr() includes the filename so it appears usefully in error messages."""
parent = tmp_path / "main.yaml"
include = yaml_util.IncludeFile(parent, "some/nested.yaml", None, lambda _: {})
include = yaml_util.IncludeFile(parent, "some/nested.yaml", lambda _: {})
assert repr(include) == "IncludeFile(some/nested.yaml)"
@@ -624,7 +659,7 @@ def test_include_file_load_caches_result(tmp_path: Path) -> None:
call_count += 1
return content
include = yaml_util.IncludeFile(parent, "child.yaml", None, counting_loader)
include = yaml_util.IncludeFile(parent, "child.yaml", counting_loader)
first = include.load()
second = include.load()
@@ -641,7 +676,7 @@ def test_include_file_load_caches_none_result(tmp_path: Path) -> None:
nonlocal call_count
call_count += 1
include = yaml_util.IncludeFile(parent, "empty.yaml", None, counting_loader)
include = yaml_util.IncludeFile(parent, "empty.yaml", counting_loader)
first = include.load()
second = include.load()
@@ -653,7 +688,7 @@ def test_include_file_load_caches_none_result(tmp_path: Path) -> None:
def test_include_file_load_raises_on_unresolved_expressions(tmp_path: Path) -> None:
"""load() raises if the filename contains unresolved substitutions or expressions."""
parent = tmp_path / "main.yaml"
include = yaml_util.IncludeFile(parent, "${undefined_var}.yaml", None, lambda _: {})
include = yaml_util.IncludeFile(parent, "${undefined_var}.yaml", lambda _: {})
with pytest.raises(cv.Invalid, match="unresolved"):
include.load()
@@ -670,13 +705,83 @@ def test_include_file_load_raises_on_unresolved_expressions(tmp_path: Path) -> N
("price-100$.yaml", False), # $ at end, not followed by valid substitution
],
)
def test_include_file_has_unresolved_expressions(
def test_include_file_has_unresolved_file(
tmp_path: Path, filename: str, expected: bool
) -> None:
"""has_unresolved_expressions() detects substitution patterns in the filename."""
"""has_unresolved_file() detects substitution patterns in the filename."""
parent = tmp_path / "main.yaml"
include = yaml_util.IncludeFile(parent, filename, None, lambda _: {})
assert include.has_unresolved_expressions() == expected
include = yaml_util.IncludeFile(parent, filename, lambda _: {})
assert include.has_unresolved_file() == expected
@pytest.mark.parametrize(
("condition", "expected"),
[
(None, False),
(True, False),
(False, False),
("true", False),
("false", False),
("$has_feature", True), # whole substitution
("${has_feature}", True), # whole substitution
("tr$ue", True), # partial substitution
("$.", False), # malformed substitution
("${1 == 1}", True), # Jinja expression
("${", False), # malformed expression
],
)
def test_include_file_has_unresolved_condition(
tmp_path: Path, condition: bool | str | None, expected: bool
) -> None:
"""has_unresolved_condition() detects substitution patterns in the condition."""
parent = tmp_path / "main.yaml"
include = yaml_util.IncludeFile(
parent, "device.yaml", lambda _: {}, condition=condition
)
assert include.has_unresolved_condition() == expected
@pytest.mark.parametrize(
("condition", "expected_result_or_error"),
[
(None, True),
(True, True),
(False, False),
("true", True),
("True", True),
("TRUE", True),
("yes", True),
("on", True),
("enable", True),
("false", False),
("False", False),
("FALSE", False),
("no", False),
("off", False),
("disable", False),
(
"$sub",
"Cannot evaluate include condition for 'device.yaml' with unresolved substitutions",
),
("", "Cannot convert include condition for 'device.yaml' to a boolean"),
("trues", "Cannot convert include condition for 'device.yaml' to a boolean"),
],
)
def test_include_file_should_load(
tmp_path: Path,
condition: bool | str | None,
expected_result_or_error: bool | str,
) -> None:
"""should_load() evaluates the condition and raises an error if it is malformed."""
parent = tmp_path / "main.yaml"
include = yaml_util.IncludeFile(
parent, "device.yaml", lambda _: {}, condition=condition
)
if isinstance(expected_result_or_error, bool):
assert include.should_load() == expected_result_or_error
else:
with pytest.raises(cv.Invalid, match=expected_result_or_error):
include.should_load()
def test_mapping_include_non_string_file_rejected(tmp_path: Path) -> None:
@@ -691,16 +796,16 @@ def test_include_file_templated_filename_stays_raw_string(tmp_path: Path) -> Non
"""A templated filename keeps its verbatim text (issue #18545)."""
parent = tmp_path / "main.yaml"
expr = '${ "bluetooth/proxy.yaml" if enable_bluetooth_proxy else "../empty.yaml" }'
include = yaml_util.IncludeFile(parent, expr, None, lambda _: {})
include = yaml_util.IncludeFile(parent, expr, lambda _: {})
assert include.file == expr
assert include.has_unresolved_expressions()
assert include.has_unresolved_file()
assert repr(include) == f"IncludeFile({expr})"
def test_represent_include_file_templated() -> None:
"""Dumping a templated IncludeFile emits the raw expression unchanged."""
expr = '${ "a/b.yaml" if flag else "../c.yaml" }'
include = yaml_util.IncludeFile(Path("/fake/main.yaml"), expr, None, lambda _: {})
include = yaml_util.IncludeFile(Path("/fake/main.yaml"), expr, lambda _: {})
assert yaml_util.dump({"key": include}) == f"key: !include '{expr}'\n"
@@ -710,12 +815,12 @@ def test_include_in_list_context() -> None:
parent = Path("/fake/main.yaml")
# The nested IncludeFile resolves to a plain string value
inner = yaml_util.IncludeFile(parent, "inner.yaml", None, lambda _: "gamma")
inner = yaml_util.IncludeFile(parent, "inner.yaml", lambda _: "gamma")
# The outer IncludeFile returns a list whose last element is itself an IncludeFile,
# exercising the substitution pass's ability to recurse into loaded content.
outer = yaml_util.IncludeFile(
parent, "items.yaml", None, lambda _: ["alpha", "beta", inner]
parent, "items.yaml", lambda _: ["alpha", "beta", inner]
)
config = OrderedDict({"values": outer})
@@ -736,15 +841,48 @@ def test_top_level_include_resolved_by_load_yaml(tmp_path: Path) -> None:
assert result["key"] == "value"
@pytest.mark.parametrize(
("condition", "expected_result_or_error"),
[
("true", True),
("false", False),
('"TRUE"', True),
('"FALSE"', False),
('"x"', "Cannot convert include condition for 'child.yaml' to a boolean"),
(
"$sub",
"Cannot evaluate include condition for 'child.yaml' with unresolved substitutions",
),
],
)
def test_top_level_include_with_condition_resolved_by_load_yaml(
tmp_path: Path, condition: bool | str | None, expected_result_or_error: bool | str
) -> None:
"""load_yaml evaluates the condition for a top-level !include."""
child = tmp_path / "child.yaml"
child.write_text("key: value\n")
main = tmp_path / "main.yaml"
main.write_text(f"!include {{ file: child.yaml, condition: {condition} }}\n")
if isinstance(expected_result_or_error, bool):
result = yaml_util.load_yaml(main)
assert isinstance(result, dict)
if expected_result_or_error:
assert result["key"] == "value"
else:
assert result == {}
else:
with pytest.raises(cv.Invalid, match=expected_result_or_error):
result = yaml_util.load_yaml(main)
def test_include_plain_filename_loads_after_deferred_refactor() -> None:
"""!include with a plain filename (no $ expressions) still loads correctly.
Regression guard: the deferred-loading refactor must not break the simple case.
"""
parent = Path("/fake/main.yaml")
include = yaml_util.IncludeFile(
parent, "child.yaml", None, lambda _: {"answer": 42}
)
include = yaml_util.IncludeFile(parent, "child.yaml", lambda _: {"answer": 42})
config = OrderedDict({"result": include})
config = substitutions.do_substitution_pass(config)
@@ -756,7 +894,23 @@ def test_yaml_merge_include_with_filename_substitution_raises() -> None:
"""<<: !include ${expr} raises a clear error — substitutions in merge-key filenames
are not yet supported, and the error message must say so."""
yaml_text = "base:\n existing: value\n <<: !include ${filename}.yaml\n"
with pytest.raises(EsphomeError, match="not supported yet"):
with pytest.raises(
EsphomeError,
match="Substitution in include filename with merge keys is not supported yet",
):
yaml_util.parse_yaml(
Path("/fake/main.yaml"), io.StringIO(yaml_text), lambda _: {}
)
def test_yaml_merge_include_with_condition_substitution_raises() -> None:
"""<<: !include { file: ${expr}, condition: {} } raises a clear error — substitutions in merge-key conditions
are not yet supported, and the error message must say so."""
yaml_text = "base:\n existing: value\n <<: !include\n file: filename.yaml\n condition: ${expr}\n"
with pytest.raises(
EsphomeError,
match="Substitution in include condition with merge keys is not supported yet",
):
yaml_util.parse_yaml(
Path("/fake/main.yaml"), io.StringIO(yaml_text), lambda _: {}
)
@@ -775,8 +929,8 @@ def test_yaml_merge_chain_include_resolves() -> None:
"""Chained includes in merge keys resolve through multiple IncludeFile layers."""
parent = Path("/fake/main.yaml")
inner = yaml_util.IncludeFile(parent, "inner.yaml", None, lambda _: {"x": 1})
outer = yaml_util.IncludeFile(parent, "outer.yaml", None, lambda _: inner)
inner = yaml_util.IncludeFile(parent, "inner.yaml", lambda _: {"x": 1})
outer = yaml_util.IncludeFile(parent, "outer.yaml", lambda _: inner)
yaml_text = "base:\n existing: value\n <<: !include outer.yaml\n"
config = yaml_util.parse_yaml(parent, io.StringIO(yaml_text), lambda _: outer)
@@ -791,7 +945,7 @@ def test_yaml_merge_chain_include_depth_exceeded() -> None:
parent = Path("/fake/main.yaml")
def self_referencing_loader(path: Path) -> yaml_util.IncludeFile:
return yaml_util.IncludeFile(parent, path.name, None, self_referencing_loader)
return yaml_util.IncludeFile(parent, path.name, self_referencing_loader)
yaml_text = "base:\n <<: !include loop.yaml\n"
with pytest.raises(EsphomeError, match="Maximum include chain depth"):
@@ -1061,7 +1215,7 @@ class _StubInclude:
self._raise = raise_on_load
self.load_calls = 0
def has_unresolved_expressions(self) -> bool:
def has_unresolved_file(self) -> bool:
return self._unresolved
def load(self) -> object: