From a1515ec66252a6ba0114dc6508fb9ff804ad45d1 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 09:54:06 -0700 Subject: [PATCH 001/147] [modbus_controller] Replace register_count/force_new_range with reuse_previous_range (#18085) Co-authored-by: Claude Fable 5 Co-authored-by: J. Nick Koston --- esphome/components/modbus/modbus_helpers.h | 20 ++ .../components/modbus_controller/__init__.py | 99 ++++++- .../binary_sensor/__init__.py | 9 +- .../binary_sensor/modbus_binarysensor.h | 15 +- esphome/components/modbus_controller/const.py | 1 + .../modbus_controller/modbus_controller.cpp | 276 ++++++++++-------- .../modbus_controller/modbus_controller.h | 45 ++- .../modbus_controller/number/__init__.py | 10 +- .../number/modbus_number.cpp | 4 +- .../modbus_controller/number/modbus_number.h | 5 +- .../modbus_controller/output/__init__.py | 90 ++++-- .../output/modbus_output.cpp | 25 +- .../modbus_controller/output/modbus_output.h | 4 +- .../modbus_controller/select/__init__.py | 41 +-- .../select/modbus_select.cpp | 14 +- .../modbus_controller/select/modbus_select.h | 7 +- .../modbus_controller/sensor/__init__.py | 15 +- .../modbus_controller/sensor/modbus_sensor.h | 5 +- .../modbus_controller/switch/__init__.py | 9 +- .../modbus_controller/switch/modbus_switch.h | 5 +- .../modbus_controller/text_sensor/__init__.py | 18 +- .../text_sensor/modbus_textsensor.h | 7 +- .../components/modbus_controller/common.yaml | 18 +- .../fixtures/uart_mock_modbus_grouping.yaml | 17 +- .../fixtures/uart_mock_modbus_ranges.yaml | 227 ++++++++++++++ .../uart_mock_modbus_shared_address.yaml | 10 +- tests/integration/test_uart_mock_modbus.py | 77 ++++- 27 files changed, 777 insertions(+), 296 deletions(-) create mode 100644 tests/integration/fixtures/uart_mock_modbus_ranges.yaml diff --git a/esphome/components/modbus/modbus_helpers.h b/esphome/components/modbus/modbus_helpers.h index b04df1923f..76056ed3e8 100644 --- a/esphome/components/modbus/modbus_helpers.h +++ b/esphome/components/modbus/modbus_helpers.h @@ -220,6 +220,26 @@ inline bool value_type_is_float(SensorValueType v) { return v == SensorValueType::FP32 || v == SensorValueType::FP32_R; } +/// Number of 16-bit registers a value of this type occupies (RAW counts as one register). +inline uint16_t register_width_for(SensorValueType v) { + switch (v) { + case SensorValueType::U_DWORD: + case SensorValueType::S_DWORD: + case SensorValueType::U_DWORD_R: + case SensorValueType::S_DWORD_R: + case SensorValueType::FP32: + case SensorValueType::FP32_R: + return 2; + case SensorValueType::U_QWORD: + case SensorValueType::S_QWORD: + case SensorValueType::U_QWORD_R: + case SensorValueType::S_QWORD_R: + return 4; + default: + return 1; + } +} + /// Coils and discrete inputs are the bit-addressed entity tables; the other types are 16-bit registers. inline bool is_entity_type_binary(EntityType type) { return type == EntityType::COIL || type == EntityType::DISCRETE_INPUT; diff --git a/esphome/components/modbus_controller/__init__.py b/esphome/components/modbus_controller/__init__.py index c390d8ab79..f888cc060e 100644 --- a/esphome/components/modbus_controller/__init__.py +++ b/esphome/components/modbus_controller/__init__.py @@ -41,6 +41,7 @@ from .const import ( CONF_REGISTER_COUNT, CONF_REGISTER_TYPE, CONF_RESPONSE_SIZE, + CONF_REUSE_PREVIOUS_RANGE, CONF_SERVER_COURTESY_RESPONSE, CONF_SERVER_REGISTERS, CONF_SKIP_UPDATES, @@ -60,6 +61,13 @@ ModbusController = modbus_controller_ns.class_("ModbusController", cg.PollingCom SensorItem = modbus_controller_ns.struct("SensorItem") +RangeReuse = modbus_controller_ns.enum("RangeReuse", is_class=True) +RANGE_REUSE = { + "auto": RangeReuse.AUTO, + True: RangeReuse.ALWAYS, + False: RangeReuse.NEVER, +} + _LOGGER = logging.getLogger(__name__) @@ -184,13 +192,88 @@ ModbusItemBaseSchema = cv.Schema( ): cv.positive_int, cv.Optional(CONF_BITMASK, default=0xFFFFFFFF): cv.hex_uint32_t, cv.Optional(CONF_SKIP_UPDATES): validate_skip_updates_deprecated, - cv.Optional(CONF_FORCE_NEW_RANGE, default=False): cv.boolean, + cv.Optional(CONF_REUSE_PREVIOUS_RANGE, default="auto"): cv.Any( + cv.boolean, cv.one_of("auto", lower=True) + ), + # Deprecated options, migrated by validate_range_reuse_migration(). Remove before 2027.3.0 + cv.Optional(CONF_FORCE_NEW_RANGE): cv.boolean, + cv.Optional(CONF_REGISTER_COUNT): cv.positive_int, cv.Optional(CONF_LAMBDA): cv.returning_lambda, - cv.Optional(CONF_RESPONSE_SIZE, default=0): cv.positive_int, + cv.Optional(CONF_RESPONSE_SIZE, default=0): cv.int_range(min=0, max=250), }, ) +def _derived_register_widths(config: ConfigType) -> set[int]: + """Register widths an item derives on its own; a matching register_count is redundant.""" + response_size = config.get(CONF_RESPONSE_SIZE, 0) + if (value_type := config.get(CONF_VALUE_TYPE)) is not None: + widths = {TYPE_REGISTER_MAP[value_type]} + if value_type == "RAW" and response_size > 0: + widths.add((response_size + 1) // 2) + return widths + if response_size > 0: + # text sensors: the old default was floor(response_size / 2); the derived width is now ceil + return {response_size // 2, (response_size + 1) // 2} + return {1} + + +def entity_label(config: ConfigType) -> str: + """The entity's name or id, so migration messages say which entry to edit.""" + label = config.get(CONF_NAME) or config.get(CONF_ID) + return str(label) if label is not None else "" + + +# Remove before 2027.3.0 +def validate_range_reuse_migration(config: ConfigType) -> ConfigType: + """Migrate the removed force_new_range/register_count options to reuse_previous_range.""" + if (force_new_range := config.pop(CONF_FORCE_NEW_RANGE, None)) is not None: + if config[CONF_REUSE_PREVIOUS_RANGE] != "auto": + raise cv.Invalid( + f"'{CONF_FORCE_NEW_RANGE}' and '{CONF_REUSE_PREVIOUS_RANGE}' can't be used together; " + f"remove '{CONF_FORCE_NEW_RANGE}'" + ) + if force_new_range: + _LOGGER.warning( + "%s: '%s' is deprecated; '%s: false' replaces it but only stops this entity joining " + "the PREVIOUS range - set it on the following entity too if the range must stay " + "isolated. Removed in 2027.3.0", + entity_label(config), + CONF_FORCE_NEW_RANGE, + CONF_REUSE_PREVIOUS_RANGE, + ) + config[CONF_REUSE_PREVIOUS_RANGE] = False + else: + _LOGGER.warning( + "%s: '%s: false' has no effect; remove it. Removed in 2027.3.0", + entity_label(config), + CONF_FORCE_NEW_RANGE, + ) + if (register_count := config.pop(CONF_REGISTER_COUNT, None)) is not None: + if ( + register_count not in _derived_register_widths(config) + and register_count != 0 + ): + raise cv.Invalid( + f"'{CONF_REGISTER_COUNT}' has been removed; the number of registers to read is now " + f"derived from '{CONF_VALUE_TYPE}' (or '{CONF_RESPONSE_SIZE}' for RAW values and text " + f"sensors). To make one request span extra registers up to the next sensor, set " + f"'{CONF_REUSE_PREVIOUS_RANGE}: true' on the NEXT sensor instead; for RAW or text block " + f"reads set '{CONF_RESPONSE_SIZE}' to the byte count; to force multi-register writes set " + f"'use_write_multiple: true'. See " + "https://esphome.io/components/modbus_controller/" + ) + _LOGGER.warning( + "%s: '%s' is now derived from '%s' (or '%s' for RAW values and text sensors) and has no " + "effect; remove it. Removed in 2027.3.0", + entity_label(config), + CONF_REGISTER_COUNT, + CONF_VALUE_TYPE, + CONF_RESPONSE_SIZE, + ) + return config + + def validate_modbus_register(config: ConfigType) -> ConfigType: # custom_command is the deprecated alias for custom_pdu (migrated later in final validate); treat # either as "a custom frame is configured" so the address/register_type rules match. @@ -293,20 +376,13 @@ def reject_odd_holding_write_offset(config: ConfigType) -> ConfigType: return config -def modbus_calc_properties(config: ConfigType) -> tuple[int, int]: +def modbus_calc_properties(config: ConfigType) -> int: byte_offset = 0 - reg_count = 0 if CONF_OFFSET in config: byte_offset = config[CONF_OFFSET] # A CONF_BYTE_OFFSET setting overrides CONF_OFFSET if CONF_BYTE_OFFSET in config: byte_offset = config[CONF_BYTE_OFFSET] - if CONF_REGISTER_COUNT in config: - reg_count = config[CONF_REGISTER_COUNT] - if CONF_VALUE_TYPE in config: - value_type = config[CONF_VALUE_TYPE] - if reg_count == 0: - reg_count = TYPE_REGISTER_MAP[value_type] if CONF_CUSTOM_PDU in config: if CONF_ADDRESS not in config: # generate a unique modbus address using the hash of the name @@ -317,8 +393,7 @@ def modbus_calc_properties(config: ConfigType) -> tuple[int, int]: value = value.encode() config[CONF_ADDRESS] = binascii.crc_hqx(value, 0) config[CONF_REGISTER_TYPE] = cv.enum(MODBUS_REGISTER_TYPE)("custom") - config[CONF_FORCE_NEW_RANGE] = True - return byte_offset, reg_count + return byte_offset async def add_modbus_base_properties( diff --git a/esphome/components/modbus_controller/binary_sensor/__init__.py b/esphome/components/modbus_controller/binary_sensor/__init__.py index 366dab6062..32247b4cec 100644 --- a/esphome/components/modbus_controller/binary_sensor/__init__.py +++ b/esphome/components/modbus_controller/binary_sensor/__init__.py @@ -5,6 +5,7 @@ import esphome.config_validation as cv from esphome.const import CONF_ADDRESS, CONF_ID from .. import ( + RANGE_REUSE, ModbusItemBaseSchema, SensorItem, add_modbus_base_properties, @@ -12,12 +13,13 @@ from .. import ( modbus_controller_ns, validate_custom_pdu_item, validate_modbus_register, + validate_range_reuse_migration, ) from ..const import ( CONF_BITMASK, - CONF_FORCE_NEW_RANGE, CONF_MODBUS_CONTROLLER_ID, CONF_REGISTER_TYPE, + CONF_REUSE_PREVIOUS_RANGE, ) DEPENDENCIES = ["modbus_controller"] @@ -38,20 +40,21 @@ CONFIG_SCHEMA = cv.All( } ), validate_modbus_register, + validate_range_reuse_migration, ) FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item async def to_code(config): - byte_offset, _ = modbus_calc_properties(config) + byte_offset = modbus_calc_properties(config) var = cg.new_Pvariable( config[CONF_ID], config[CONF_REGISTER_TYPE], config[CONF_ADDRESS], byte_offset, config[CONF_BITMASK], - config[CONF_FORCE_NEW_RANGE], + RANGE_REUSE[config[CONF_REUSE_PREVIOUS_RANGE]], ) await cg.register_component(var, config) await binary_sensor.register_binary_sensor(var, config) diff --git a/esphome/components/modbus_controller/binary_sensor/modbus_binarysensor.h b/esphome/components/modbus_controller/binary_sensor/modbus_binarysensor.h index f5ddbd82cc..a6b5bc4ef9 100644 --- a/esphome/components/modbus_controller/binary_sensor/modbus_binarysensor.h +++ b/esphome/components/modbus_controller/binary_sensor/modbus_binarysensor.h @@ -11,19 +11,22 @@ namespace esphome::modbus_controller { class ModbusBinarySensor final : public Component, public binary_sensor::BinarySensor, public SensorItem { public: ModbusBinarySensor(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint32_t bitmask, - bool force_new_range) { + RangeReuse reuse_previous_range) { this->register_type = register_type; this->set_address(start_address); this->set_offset_from_start_address(offset); this->bitmask = bitmask; this->sensor_value_type = SensorValueType::BIT; - this->force_new_range = force_new_range; + this->reuse_previous_range = reuse_previous_range; + } - if (modbus::helpers::is_entity_type_binary(register_type)) { - this->register_count = offset + 1; - } else { - this->register_count = 1; + /// On the bit-addressed tables the bit sits at start_address + offset, so the read must span offset + 1 + /// bits. Uses the offset as configured: `offset` itself is overwritten with the position in the range. + uint16_t entity_count() const override { + if (modbus::helpers::is_entity_type_binary(this->register_type)) { + return this->offset_from_start_address + 1; } + return 1; } void parse_and_publish(std::span data) override; diff --git a/esphome/components/modbus_controller/const.py b/esphome/components/modbus_controller/const.py index 8412a651b8..364a0a510e 100644 --- a/esphome/components/modbus_controller/const.py +++ b/esphome/components/modbus_controller/const.py @@ -18,6 +18,7 @@ CONF_REGISTER_LAST_ADDRESS = "register_last_address" CONF_REGISTER_TYPE = "register_type" CONF_REGISTER_VALUE = "register_value" CONF_RESPONSE_SIZE = "response_size" +CONF_REUSE_PREVIOUS_RANGE = "reuse_previous_range" CONF_SERVER_COURTESY_RESPONSE = "server_courtesy_response" CONF_SERVER_REGISTERS = "server_registers" CONF_SKIP_UPDATES = "skip_updates" diff --git a/esphome/components/modbus_controller/modbus_controller.cpp b/esphome/components/modbus_controller/modbus_controller.cpp index 8801c33d8c..c7fc10a0bb 100644 --- a/esphome/components/modbus_controller/modbus_controller.cpp +++ b/esphome/components/modbus_controller/modbus_controller.cpp @@ -3,6 +3,7 @@ #include "esphome/core/log.h" #include +#include namespace esphome::modbus_controller { @@ -137,7 +138,7 @@ ModbusCommandItem::ModbusCommandItem(ModbusController &controller, modbus::Modbu SensorItem *sensor) : modbus::ModbusClientDevice(parent, address), start_address_(sensor->start_address), - register_count_(sensor->register_count), + register_count_(sensor->entity_count()), custom_pdu_(&sensor->custom_pdu), controller_(&controller) { // The PDU's first byte is its real function code; carry it so dump_config, the on_command_sent @@ -350,129 +351,176 @@ void ModbusController::update() { } // walk through the sensors and determine the register ranges to read +namespace { + +class RangeBuilder { + public: + explicit RangeBuilder(FixedVector &ranges) : ranges_(ranges) {} + + bool can_join(const SensorItem *curr) const { + return this->have_range_ && curr->reuse_previous_range != RangeReuse::NEVER && + this->r_.register_type == curr->register_type && curr->register_type != modbus::EntityType::CUSTOM; + } + + // A sensor that joined mid-range must never anchor this - hence both address tests. + bool try_reuse_register(SensorItem *curr) { + const uint32_t range_end = this->range_end_(); + if (curr->start_address != range_end - this->prev_->entity_count() || + this->prev_->start_address + this->prev_->entity_count() != range_end || + curr->entity_count() != this->prev_->entity_count() || + curr->get_register_size() != this->prev_->get_register_size()) { + return false; + } + if (!place_offset(curr, static_cast(this->prev_->offset) + curr->offset_from_start_address)) + return false; + ESP_LOGV(TAG, "Re-use previous register 0x%X", curr->start_address); + return true; + } + + bool try_extend(SensorItem *curr) { + const uint32_t range_end = this->range_end_(); + const bool reachable = + curr->reuse_previous_range == RangeReuse::ALWAYS + ? curr->start_address >= range_end + : curr->start_address == range_end && (curr->addresses_bits() || !this->range_custom_size_); + if (!reachable) + return false; + const uint16_t gap = static_cast(curr->start_address - range_end); + const uint32_t new_count = this->r_.register_count + gap + curr->entity_count(); + const uint16_t max_quantity = + curr->addresses_bits() ? modbus::MAX_NUM_OF_COILS_TO_READ : modbus::MAX_NUM_OF_REGISTERS_TO_READ; + const uint32_t prospective_offset = + (curr->addresses_bits() ? static_cast(curr->start_address - this->r_.start_address) + : static_cast(this->range_bytes_) + gap * 2) + + curr->offset_from_start_address; + if (new_count > max_quantity || !place_offset(curr, prospective_offset)) { + return false; + } + if (!curr->addresses_bits()) + this->range_bytes_ += static_cast(gap) * 2; + this->range_bytes_ += curr->get_register_size(); + this->range_custom_size_ = this->range_custom_size_ || has_custom_size(curr); + this->r_.register_count = static_cast(new_count); + ESP_LOGV(TAG, "Extend range to include 0x%X", curr->start_address); + return true; + } + + bool try_cover(SensorItem *curr) { + if (!this->range_shared_ || this->range_forced_ || curr->start_address < this->r_.start_address || + curr->start_address + curr->entity_count() > this->range_end_() || this->range_custom_size_ || + has_custom_size(curr)) { + return false; + } + const uint32_t addr_delta = curr->start_address - this->r_.start_address; + if (!place_offset(curr, (curr->addresses_bits() ? addr_delta : addr_delta * 2) + curr->offset_from_start_address)) + return false; + ESP_LOGV(TAG, "Register 0x%X already covered by range 0x%X", curr->start_address, this->r_.start_address); + return true; + } + + // A response dispatches to a single range per (start address, register type), so same-address items + // must share - even reuse_previous_range: false and custom entities. + bool try_share(SensorItem *curr) { + if (!this->have_range_ || this->r_.register_type != curr->register_type || + this->r_.start_address != curr->start_address) { + return false; + } + curr->offset = curr->offset_from_start_address; + this->r_.register_count = std::max(this->r_.register_count, curr->entity_count()); + this->range_bytes_ = std::max(this->range_bytes_, curr->get_register_size()); + this->range_custom_size_ = this->range_custom_size_ || has_custom_size(curr); + this->range_shared_ = true; + this->range_forced_ = this->range_forced_ || curr->reuse_previous_range == RangeReuse::NEVER; + ESP_LOGV(TAG, "Share range start 0x%X", curr->start_address); + return true; + } + + bool always_declined(const SensorItem *curr) const { + return this->have_range_ && curr->reuse_previous_range == RangeReuse::ALWAYS && + this->r_.register_type == curr->register_type && curr->start_address != this->r_.start_address; + } + + void open(SensorItem *curr) { + this->close(); + this->r_ = {}; + this->range_bytes_ = curr->get_register_size(); + this->range_custom_size_ = has_custom_size(curr); + this->range_forced_ = curr->reuse_previous_range == RangeReuse::NEVER; + this->range_shared_ = false; + curr->offset = curr->offset_from_start_address; + this->r_.start_address = curr->start_address; + this->r_.register_count = curr->entity_count(); + this->r_.register_type = curr->register_type; + if (curr->register_type == modbus::EntityType::CUSTOM) + this->r_.custom_pdu = &curr->custom_pdu; + this->have_range_ = true; + } + + void record(SensorItem *curr) { + curr->range_start_address = this->r_.start_address; + this->r_.sensors.insert(curr); + this->prev_ = curr; + } + + void close() { + if (!this->have_range_) + return; + ESP_LOGV(TAG, "Add range 0x%X %d", this->r_.start_address, this->r_.register_count); + this->ranges_.push_back(std::move(this->r_)); + this->have_range_ = false; + } + + private: + uint32_t range_end_() const { return this->r_.start_address + this->r_.register_count; } + // The resolved offset must fit its uint8_t field or the sensor would parse the wrong slice. + static bool place_offset(SensorItem *curr, uint32_t offset) { + if (offset > std::numeric_limits::max()) + return false; + curr->offset = static_cast(offset); + return true; + } + static bool has_custom_size(const SensorItem *item) { + return item->get_register_size() != static_cast(item->entity_count()) * 2; + } + FixedVector &ranges_; + RegisterRange r_ = {}; + bool have_range_ = false; + bool range_forced_ = false; // a reuse: false member blocks the coverage join + bool range_shared_ = false; // only a share-widened range absorbs by coverage + size_t range_bytes_ = 0; + bool range_custom_size_ = false; + SensorItem *prev_ = nullptr; +}; + +} // namespace + void ModbusController::create_polling_commands_() { if (this->sensorset_.empty()) { ESP_LOGW(TAG, "No sensors registered"); return; } - // Sensors are walked in the sensor set's order (see SensorItemsComparator): register type, then - // force_new_range ahead of the rest, then address - so the walk is not purely address-ordered. - // Each keeps the address it was configured with; what is resolved here is its `offset`, the position - // of its data within the response of whichever range it ends up in. - // One range per sensor is a strict upper bound: each walk step closes at most one range, plus one - // closed after the walk. Sized to that bound so no push is ever silently dropped, then handed on by move. + // At most one range closes per sensor plus one final close, so sensorset_.size() bounds the pushes + // (FixedVector silently drops past capacity). FixedVector ranges; ranges.init(this->sensorset_.size()); - RegisterRange r = {}; - bool have_range = false; - // Set while the open range belongs to a force_new_range sensor: a range the user asked to keep - // separate must not quietly absorb other sensors. - bool range_forced = false; - // Set once a sensor has joined by sharing the range's start address, which widens the read. Only a - // widened range can absorb a later sensor by coverage: ranges that were kept apart before stay apart, - // so their frames and polling rates are untouched. - bool range_shared = false; - // Bytes the range's registers have consumed so far. An extending sensor starts after them, so a - // register that returns more bytes than its count implies pushes the sensors after it along. - // range_custom_size records whether any of them returns something other than two bytes per register, - // which is what makes a position inside the range impossible to work out from addresses alone. Coils - // count as such: they carry one bit per address, so bit ranges never take the coverage join. - size_t range_bytes = 0; - bool range_custom_size = false; - SensorItem *prev = nullptr; + RangeBuilder builder(ranges); for (SensorItem *curr : this->sensorset_) { - ESP_LOGV(TAG, "Register: 0x%X count=%d size=%zu offset=%u addr=%p", curr->start_address, curr->register_count, + ESP_LOGV(TAG, "Register: 0x%X width=%u size=%zu offset=%u addr=%p", curr->start_address, curr->entity_count(), curr->get_register_size(), curr->offset, curr); - - const bool custom_size = curr->get_register_size() != static_cast(curr->register_count) * 2; - - bool join = false; - if (have_range && !curr->force_new_range && r.register_type == curr->register_type && - curr->register_type != modbus::EntityType::CUSTOM) { - if (curr->start_address == (r.start_address + r.register_count - prev->register_count) && - prev->start_address + prev->register_count == r.start_address + r.register_count && - curr->register_count == prev->register_count && curr->get_register_size() == prev->get_register_size()) { - // A second sensor on the register(s) the previous one covers: it reads those same bytes, - // starting where that sensor's offset pointed, so a chain configured 0/2/4 resolves to 0/2/6. - // Both address tests matter. The first identifies the previous sensor's register by working back - // from the range's end, which only describes it while it actually sits there - hence the second. - // A sensor that joined mid-range must never anchor this, or the next one inherits its offset. - curr->offset = static_cast(prev->offset + curr->offset_from_start_address); - join = true; - ESP_LOGV(TAG, "Re-use previous register 0x%X", curr->start_address); - } else if (curr->start_address == (r.start_address + r.register_count)) { - // The next contiguous register(s): the data begins after what the range has consumed so far - - // the byte cursor for registers, the distance in bits for coils. - curr->offset = - static_cast((curr->addresses_bits() ? curr->start_address - r.start_address : range_bytes) + - curr->offset_from_start_address); - range_bytes += curr->get_register_size(); - range_custom_size = range_custom_size || custom_size; - r.register_count += curr->register_count; - join = true; - ESP_LOGV(TAG, "Extend range to include 0x%X", curr->start_address); - } else if (range_shared && !range_forced && curr->start_address >= r.start_address && - curr->start_address + curr->register_count <= r.start_address + r.register_count && - !range_custom_size && !custom_size) { - // The registers already fall inside a range that a shared-address join widened, so this sensor - // reads its slice of that response instead of adding an overlapping second poll. The guards keep - // it narrow: only a widened range, never a force-isolated one; only where every register in the - // range returns two bytes, so interior positions follow from the addresses; only sensors genuinely - // inside it, which is why the lower bound is needed given the walk is not address-ordered. - const uint16_t addr_delta = curr->start_address - r.start_address; - curr->offset = static_cast((curr->addresses_bits() ? addr_delta : addr_delta * 2) + - curr->offset_from_start_address); - join = true; - ESP_LOGV(TAG, "Register 0x%X already covered by range 0x%X", curr->start_address, r.start_address); - } + bool join = builder.can_join(curr) && + (builder.try_reuse_register(curr) || builder.try_extend(curr) || builder.try_cover(curr)); + if (!join && builder.always_declined(curr)) { + ESP_LOGW(TAG, "reuse_previous_range on 0x%X cannot join the previous range; starting a new range", + curr->start_address); } - - // Sensors on the same start address have to share one range: a response is dispatched to a single - // range per (start_address, register_type), so a second range with that key would never receive - // data. This holds for force_new_range and custom entities too. The read widens to cover whichever - // sensor needs the most registers, which also fixes a short read for coils that use offset. - if (!join && have_range && r.register_type == curr->register_type && r.start_address == curr->start_address) { - curr->offset = curr->offset_from_start_address; // shares the range start - r.register_count = std::max(r.register_count, curr->register_count); - range_bytes = std::max(range_bytes, curr->get_register_size()); - range_custom_size = range_custom_size || custom_size; - range_shared = true; - range_forced = range_forced || curr->force_new_range; - join = true; - ESP_LOGV(TAG, "Share range start 0x%X", curr->start_address); - } - - if (!join) { - if (have_range) { - ESP_LOGV(TAG, "Add range 0x%X %d", r.start_address, r.register_count); - ranges.push_back(std::move(r)); - } - r = {}; - range_bytes = curr->get_register_size(); - range_custom_size = custom_size; - range_forced = curr->force_new_range; - range_shared = false; - curr->offset = curr->offset_from_start_address; - r.start_address = curr->start_address; - r.register_count = curr->register_count; - r.register_type = curr->register_type; - if (curr->register_type == modbus::EntityType::CUSTOM) - r.custom_pdu = &curr->custom_pdu; - have_range = true; - } - - // Every member records its range's first register. The resolved offset is relative to it, so the - // two together give the sensor's real position, and the address a write entity targets. - curr->range_start_address = r.start_address; - r.sensors.insert(curr); - prev = curr; + join = join || builder.try_share(curr); + if (!join) + builder.open(curr); + builder.record(curr); } - if (have_range) { - ESP_LOGV(TAG, "Add last range 0x%X %d", r.start_address, r.register_count); - ranges.push_back(std::move(r)); - } - // Staged in a setup-time vector so the device storage can be sized exactly (see polling_devices_). + builder.close(); + this->polling_devices_.init(ranges.size()); for (auto &range : ranges) { this->polling_devices_.emplace_back(*this, std::move(range)); @@ -490,8 +538,8 @@ void ModbusController::dump_config() { #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE ESP_LOGCONFIG(TAG, "sensormap"); for (auto &it : this->sensorset_) { - ESP_LOGCONFIG(TAG, " Sensor type=%u start=0x%X offset=0x%X count=%d size=%zu", - static_cast(it->register_type), it->start_address, it->offset, it->register_count, + ESP_LOGCONFIG(TAG, " Sensor type=%u start=0x%X offset=0x%X width=%u size=%zu", + static_cast(it->register_type), it->start_address, it->offset, it->entity_count(), it->get_register_size()); } ESP_LOGCONFIG(TAG, "ranges"); diff --git a/esphome/components/modbus_controller/modbus_controller.h b/esphome/components/modbus_controller/modbus_controller.h index 490efbde0b..821c500a31 100644 --- a/esphome/components/modbus_controller/modbus_controller.h +++ b/esphome/components/modbus_controller/modbus_controller.h @@ -126,6 +126,16 @@ inline std::vector float_to_payload(float value, SensorValueType value class ModbusController; +/// How an item relates to the register range built just before it (same register type, address order). +/// The numeric order doubles as the comparator tiebreak for items at the same address (see +/// SensorItemsComparator): AUTO items form the shared range first, so a NEVER item comes last and +/// shares a range it did not start (items on one address must share, see create_polling_commands_()). +enum class RangeReuse : uint8_t { + AUTO = 0, // join when adjacent and the position in the reply is exact (no non-standard response_size ahead) + ALWAYS = 1, // join unconditionally, reading across any address gap + NEVER = 2, // never join backward (later items may still extend this item's range) +}; + class SensorItem { public: /// Parse this sensor's slice out of its range's response and publish it. The span points into the @@ -159,11 +169,26 @@ class SensorItem { } void set_custom_pdu(std::initializer_list pdu) { this->custom_pdu.set(pdu.begin(), pdu.size()); } + + /// Entities this item spans: one bit for bit-addressed types, ceil(bytes / 2) registers for RAW + /// with a response_size, else the value type's register width. + virtual uint16_t entity_count() const { + if (modbus::helpers::is_entity_type_binary(this->register_type)) { + return 1; + } + if (this->sensor_value_type == SensorValueType::RAW && this->response_bytes > 0) { + return (this->response_bytes + 1) / 2; + } + return modbus::helpers::register_width_for(this->sensor_value_type); + } + + /// Bytes this item's registers occupy in a response: one per bit for bit-addressed types; response_size + /// when set (devices that answer more bytes per register than the standard two); else two per register. size_t virtual get_register_size() const { if (this->addresses_bits()) { return 1; } else { // if CONF_RESPONSE_BYTES is used override the default - return response_bytes > 0 ? response_bytes : register_count * 2; + return response_bytes > 0 ? response_bytes : this->entity_count() * 2; } } // Override register size for modbus devices not using 1 register for one dword @@ -177,7 +202,6 @@ class SensorItem { /// for the registers ahead of it (including wide response_size ones) and for any offset inherited /// from an earlier sensor sharing the same register. uint8_t offset{0}; - uint8_t register_count{0}; uint8_t response_bytes{0}; /// The offset exactly as configured: measured from this sensor's own start_address, where `offset` /// is measured from the first register of the range it ends up polled in. Same units as `offset` - @@ -188,7 +212,7 @@ class SensorItem { /// First register of the range this sensor is polled in; equals start_address for an unpolled item. uint16_t range_start_address{0}; SmallInlineBuffer<8> custom_pdu{}; - bool force_new_range{false}; + RangeReuse reuse_previous_range{RangeReuse::AUTO}; }; // ModbusController::create_polling_commands_ tries to optimize register range @@ -201,16 +225,17 @@ class SensorItemsComparator { return lhs->register_type < rhs->register_type; } - // ensure that sensor with force_new_range set are before the others - if (lhs->force_new_range != rhs->force_new_range) { - return lhs->force_new_range > rhs->force_new_range; - } - // sort by start address if (lhs->start_address != rhs->start_address) { return lhs->start_address < rhs->start_address; } + // at the same address: AUTO before ALWAYS before NEVER, so a NEVER item never starts the range + // the others at that address are then forced to share (see RangeReuse) + if (lhs->reuse_previous_range != rhs->reuse_previous_range) { + return lhs->reuse_previous_range < rhs->reuse_previous_range; + } + // sort by the offset as configured (ensures update of sensors in ascending order). The resolved // `offset` is deliberately not used: ranges are built while iterating this set and assign it, and // a sort key that changed under the iteration would corrupt the set's ordering. @@ -229,8 +254,8 @@ using SensorSet = std::set; struct RegisterRange { uint16_t start_address; modbus::EntityType register_type; - uint8_t register_count; - SensorSet sensors; // all sensors of this range + uint16_t register_count; // registers (or bits) the poll command reads; joins across gaps can exceed 255 + SensorSet sensors; // all sensors of this range /// A custom range polls this PDU, referenced from the sensor that opened the range. const SmallInlineBuffer<8> *custom_pdu{nullptr}; }; diff --git a/esphome/components/modbus_controller/number/__init__.py b/esphome/components/modbus_controller/number/__init__.py index 6a5b7041b8..6f7bf588af 100644 --- a/esphome/components/modbus_controller/number/__init__.py +++ b/esphome/components/modbus_controller/number/__init__.py @@ -17,20 +17,22 @@ from esphome.const import ( from esphome.types import ConfigType from .. import ( + RANGE_REUSE, ModbusItemBaseSchema, SensorItem, add_modbus_base_properties, modbus_calc_properties, modbus_controller_ns, validate_custom_pdu_item, + validate_range_reuse_migration, ) from ..const import ( CONF_BITMASK, CONF_CUSTOM_COMMAND, CONF_CUSTOM_PDU, - CONF_FORCE_NEW_RANGE, CONF_MODBUS_CONTROLLER_ID, CONF_REGISTER_TYPE, + CONF_REUSE_PREVIOUS_RANGE, CONF_USE_WRITE_MULTIPLE, CONF_VALUE_TYPE, CONF_WRITE_LAMBDA, @@ -86,13 +88,14 @@ CONFIG_SCHEMA = cv.All( ), validate_min_max, validate_modbus_number, + validate_range_reuse_migration, ) FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item async def to_code(config: ConfigType) -> None: - byte_offset, reg_count = modbus_calc_properties(config) + byte_offset = modbus_calc_properties(config) var = cg.new_Pvariable( config[CONF_ID], config[CONF_REGISTER_TYPE], @@ -100,8 +103,7 @@ async def to_code(config: ConfigType) -> None: byte_offset, config[CONF_BITMASK], config[CONF_VALUE_TYPE], - reg_count, - config[CONF_FORCE_NEW_RANGE], + RANGE_REUSE[config[CONF_REUSE_PREVIOUS_RANGE]], ) await cg.register_component(var, config) diff --git a/esphome/components/modbus_controller/number/modbus_number.cpp b/esphome/components/modbus_controller/number/modbus_number.cpp index e890a2a9ac..aff05cd517 100644 --- a/esphome/components/modbus_controller/number/modbus_number.cpp +++ b/esphome/components/modbus_controller/number/modbus_number.cpp @@ -83,10 +83,10 @@ void ModbusNumber::control(float value) { ESP_LOGD(TAG, "Updating register: connected Sensor=%s start address=0x%X register count=%d new value=%.02f (val=%.02f)", - this->get_name().c_str(), this->start_address, this->register_count, value, write_value); + this->get_name().c_str(), this->start_address, this->entity_count(), value, write_value); bool queued; - if (this->register_count == 1 && !this->use_write_multiple_) { + if (this->entity_count() == 1 && !this->use_write_multiple_) { queued = this->write_single_register(this->write_address(), data[0]); } else { queued = this->write_multiple_registers(this->write_address(), data); diff --git a/esphome/components/modbus_controller/number/modbus_number.h b/esphome/components/modbus_controller/number/modbus_number.h index 59c76e18f2..a61840cf5b 100644 --- a/esphome/components/modbus_controller/number/modbus_number.h +++ b/esphome/components/modbus_controller/number/modbus_number.h @@ -13,14 +13,13 @@ using value_to_data_t = std::function(float); class ModbusNumber final : public number::Number, public Component, public SensorItem, public WriterEntity { public: ModbusNumber(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint32_t bitmask, - SensorValueType value_type, int register_count, bool force_new_range) { + SensorValueType value_type, RangeReuse reuse_previous_range) { this->register_type = register_type; this->set_address(start_address); this->set_offset_from_start_address(offset); this->bitmask = bitmask; this->sensor_value_type = value_type; - this->register_count = register_count; - this->force_new_range = force_new_range; + this->reuse_previous_range = reuse_previous_range; }; void dump_config() override; diff --git a/esphome/components/modbus_controller/output/__init__.py b/esphome/components/modbus_controller/output/__init__.py index c2055fa690..0e8d5363d7 100644 --- a/esphome/components/modbus_controller/output/__init__.py +++ b/esphome/components/modbus_controller/output/__init__.py @@ -1,3 +1,5 @@ +import logging + import esphome.codegen as cg from esphome.components import output from esphome.components.modbus.helpers import ( @@ -12,6 +14,7 @@ from esphome.types import ConfigType from .. import ( ModbusItemBaseSchema, SensorItem, + entity_label, modbus_calc_properties, modbus_controller_ns, reject_odd_holding_write_offset, @@ -19,13 +22,18 @@ from .. import ( from ..const import ( CONF_CUSTOM_COMMAND, CONF_CUSTOM_PDU, + CONF_FORCE_NEW_RANGE, CONF_MODBUS_CONTROLLER_ID, + CONF_REGISTER_COUNT, CONF_REGISTER_TYPE, + CONF_REUSE_PREVIOUS_RANGE, CONF_USE_WRITE_MULTIPLE, CONF_VALUE_TYPE, CONF_WRITE_LAMBDA, ) +_LOGGER = logging.getLogger(__name__) + DEPENDENCIES = ["modbus_controller"] CODEOWNERS = ["@martgras"] @@ -38,26 +46,30 @@ ModbusBinaryOutput = modbus_controller_ns.class_( ) -CONFIG_SCHEMA = cv.typed_schema( - { - "coil": output.BINARY_OUTPUT_SCHEMA.extend(ModbusItemBaseSchema).extend( - { - cv.GenerateID(): cv.declare_id(ModbusBinaryOutput), - cv.Required(CONF_ADDRESS): cv.positive_int, - cv.Optional(CONF_CUSTOM_PDU): cv.invalid( - "custom_pdu is not supported for outputs; use a write_lambda instead" - ), - cv.Optional(CONF_CUSTOM_COMMAND): cv.invalid( - "custom_command is not supported for outputs; use a write_lambda instead" - ), - cv.Optional(CONF_WRITE_LAMBDA): cv.returning_lambda, - cv.Optional(CONF_USE_WRITE_MULTIPLE, default=False): cv.boolean, - } - ), - "holding": cv.All( - output.FLOAT_OUTPUT_SCHEMA.extend(ModbusItemBaseSchema).extend( +def _warn_unused_range_options(config: ConfigType) -> ConfigType: + # Outputs are write-only and never polled, so nothing here builds a range for them. The write + # spans whatever the payload holds, so register_count no longer bounds it either. + for key in (CONF_FORCE_NEW_RANGE, CONF_REGISTER_COUNT): + if config.pop(key, None) is not None: + _LOGGER.warning( + "%s: '%s' has no effect on outputs; remove it. Removed in 2027.3.0", + entity_label(config), + key, + ) + if config.pop(CONF_REUSE_PREVIOUS_RANGE, None) not in (None, "auto"): + raise cv.Invalid( + f"'{CONF_REUSE_PREVIOUS_RANGE}' has no effect on outputs: they are write-only and are " + f"never part of a polled range. Remove it." + ) + return config + + +CONFIG_SCHEMA = cv.All( + cv.typed_schema( + { + "coil": output.BINARY_OUTPUT_SCHEMA.extend(ModbusItemBaseSchema).extend( { - cv.GenerateID(): cv.declare_id(ModbusFloatOutput), + cv.GenerateID(): cv.declare_id(ModbusBinaryOutput), cv.Required(CONF_ADDRESS): cv.positive_int, cv.Optional(CONF_CUSTOM_PDU): cv.invalid( "custom_pdu is not supported for outputs; use a write_lambda instead" @@ -65,25 +77,42 @@ CONFIG_SCHEMA = cv.typed_schema( cv.Optional(CONF_CUSTOM_COMMAND): cv.invalid( "custom_command is not supported for outputs; use a write_lambda instead" ), - cv.Optional(CONF_VALUE_TYPE, default="U_WORD"): cv.enum( - SENSOR_VALUE_TYPE - ), cv.Optional(CONF_WRITE_LAMBDA): cv.returning_lambda, - cv.Optional(CONF_MULTIPLY, default=1.0): cv.float_, cv.Optional(CONF_USE_WRITE_MULTIPLE, default=False): cv.boolean, } ), - reject_odd_holding_write_offset, - ), - }, - lower=True, - key=CONF_REGISTER_TYPE, - default_type="holding", + "holding": cv.All( + output.FLOAT_OUTPUT_SCHEMA.extend(ModbusItemBaseSchema).extend( + { + cv.GenerateID(): cv.declare_id(ModbusFloatOutput), + cv.Required(CONF_ADDRESS): cv.positive_int, + cv.Optional(CONF_CUSTOM_PDU): cv.invalid( + "custom_pdu is not supported for outputs; use a write_lambda instead" + ), + cv.Optional(CONF_CUSTOM_COMMAND): cv.invalid( + "custom_command is not supported for outputs; use a write_lambda instead" + ), + cv.Optional(CONF_VALUE_TYPE, default="U_WORD"): cv.enum( + SENSOR_VALUE_TYPE + ), + cv.Optional(CONF_WRITE_LAMBDA): cv.returning_lambda, + cv.Optional(CONF_MULTIPLY, default=1.0): cv.float_, + cv.Optional(CONF_USE_WRITE_MULTIPLE, default=False): cv.boolean, + } + ), + reject_odd_holding_write_offset, + ), + }, + lower=True, + key=CONF_REGISTER_TYPE, + default_type="holding", + ), + _warn_unused_range_options, ) async def to_code(config: ConfigType) -> None: - byte_offset, reg_count = modbus_calc_properties(config) + byte_offset = modbus_calc_properties(config) # Binary Output write_template = None if config[CONF_REGISTER_TYPE] == "coil": @@ -109,7 +138,6 @@ async def to_code(config: ConfigType) -> None: config[CONF_ADDRESS], byte_offset, config[CONF_VALUE_TYPE], - reg_count, ) cg.add(var.set_write_multiply(config[CONF_MULTIPLY])) if CONF_WRITE_LAMBDA in config: diff --git a/esphome/components/modbus_controller/output/modbus_output.cpp b/esphome/components/modbus_controller/output/modbus_output.cpp index b05d3889fd..ad29015d32 100644 --- a/esphome/components/modbus_controller/output/modbus_output.cpp +++ b/esphome/components/modbus_controller/output/modbus_output.cpp @@ -46,29 +46,24 @@ void ModbusFloatOutput::write_state(float value) { modbus::helpers::float_to_payload(data, value, this->sensor_value_type); } - ESP_LOGD(TAG, "Updating register: start address=0x%X register count=%d new value=%.02f (val=%.02f)", - this->start_address, this->register_count, value, original_value); + ESP_LOGD(TAG, "Updating register: start address=0x%X register count=%u new value=%.02f (val=%.02f)", + this->start_address, this->entity_count(), value, original_value); - // The command declares register_count registers, so the payload must be exactly that many words; - // anything else would put a byte count on the wire that disagrees with the quantity field. - // number_to_payload() appends nothing for RAW, so an empty payload must be caught before data[0]. + // float_to_payload() appends nothing for RAW, so an empty payload must be caught before data[0]. if (data.empty()) { ESP_LOGW(TAG, "No payload was created for updating output"); return; } - // register_count declares the READ range width - it may pull neighboring registers into one poll - - // so a write covers exactly the registers the value occupies: the quantity comes from the payload, - // never from register_count (padding to it would zero registers the user only declared for reading). - // A payload wider than the declared range means the config and the lambda disagree - drop it. - if (data.size() > this->register_count) { - ESP_LOGE(TAG, "Payload has %zu registers but register_count is %u; dropping write", data.size(), - this->register_count); + // The value type sets the write width, so a wider payload means the config and the lambda disagree. + if (data.size() > this->entity_count()) { + ESP_LOGE(TAG, "Payload has %zu registers but the value type only spans %u; dropping write", data.size(), + this->entity_count()); return; } bool queued; - if (this->register_count == 1 && !this->use_write_multiple_) { + if (this->entity_count() == 1 && !this->use_write_multiple_) { queued = this->write_single_register(this->write_address(), data[0]); } else { queued = this->write_multiple_registers(this->write_address(), data); @@ -85,7 +80,7 @@ void ModbusFloatOutput::dump_config() { " Device start address: 0x%X\n" " Register count: %d\n" " Value type: %d", - this->start_address, this->register_count, static_cast(this->sensor_value_type)); + this->start_address, this->entity_count(), static_cast(this->sensor_value_type)); } // ModbusBinaryOutput @@ -145,7 +140,7 @@ void ModbusBinaryOutput::dump_config() { " Device start address: 0x%X\n" " Register count: %d\n" " Value type: %d", - this->start_address, this->register_count, static_cast(this->sensor_value_type)); + this->start_address, this->entity_count(), static_cast(this->sensor_value_type)); } } // namespace esphome::modbus_controller diff --git a/esphome/components/modbus_controller/output/modbus_output.h b/esphome/components/modbus_controller/output/modbus_output.h index 48153dc0b7..f76c7eada7 100644 --- a/esphome/components/modbus_controller/output/modbus_output.h +++ b/esphome/components/modbus_controller/output/modbus_output.h @@ -10,13 +10,12 @@ namespace esphome::modbus_controller { class ModbusFloatOutput final : public output::FloatOutput, public Component, public SensorItem, public WriterEntity { public: - ModbusFloatOutput(uint16_t start_address, uint8_t offset, SensorValueType value_type, int register_count) { + ModbusFloatOutput(uint16_t start_address, uint8_t offset, SensorValueType value_type) { this->register_type = modbus::EntityType::HOLDING; // A byte offset folds into the address as whole registers; odd offsets are rejected at validation. this->set_address(start_address + offset / 2); this->set_offset_from_start_address(0); this->bitmask = 0xFFFFFFFF; - this->register_count = register_count; this->sensor_value_type = value_type; } void dump_config() override; @@ -46,7 +45,6 @@ class ModbusBinaryOutput final : public output::BinaryOutput, public Component, this->set_address(start_address + offset); this->bitmask = 0xFFFFFFFF; this->sensor_value_type = SensorValueType::BIT; - this->register_count = 1; this->set_offset_from_start_address(0); } void dump_config() override; diff --git a/esphome/components/modbus_controller/select/__init__.py b/esphome/components/modbus_controller/select/__init__.py index 07893e3303..d8319932ab 100644 --- a/esphome/components/modbus_controller/select/__init__.py +++ b/esphome/components/modbus_controller/select/__init__.py @@ -3,25 +3,24 @@ from typing import Any import esphome.codegen as cg from esphome.components import select -from esphome.components.modbus.helpers import ( - SENSOR_VALUE_TYPE, - TYPE_REGISTER_MAP, - RegisterValues, -) +from esphome.components.modbus.helpers import SENSOR_VALUE_TYPE, RegisterValues import esphome.config_validation as cv from esphome.const import CONF_ADDRESS, CONF_ID, CONF_LAMBDA, CONF_OPTIMISTIC from esphome.types import ConfigType from .. import ( + RANGE_REUSE, ModbusController, SensorItem, modbus_controller_ns, + validate_range_reuse_migration, validate_skip_updates_deprecated, ) from ..const import ( CONF_FORCE_NEW_RANGE, CONF_MODBUS_CONTROLLER_ID, CONF_REGISTER_COUNT, + CONF_REUSE_PREVIOUS_RANGE, CONF_SKIP_UPDATES, CONF_USE_WRITE_MULTIPLE, CONF_VALUE_TYPE, @@ -55,18 +54,6 @@ def ensure_option_map() -> Callable[[Any], dict[str, int]]: return validator -def register_count_value_type_min(value: ConfigType) -> ConfigType: - reg_count = value.get(CONF_REGISTER_COUNT) - if reg_count is not None: - value_type = value[CONF_VALUE_TYPE] - min_register_count = TYPE_REGISTER_MAP[value_type] - if min_register_count > reg_count: - raise cv.Invalid( - f"Value type {value_type} needs at least {min_register_count} registers" - ) - return value - - INTEGER_SENSOR_VALUE_TYPE = { key: value for key, value in SENSOR_VALUE_TYPE.items() if not key.startswith("FP") } @@ -81,9 +68,13 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_VALUE_TYPE, default="U_WORD"): cv.enum( INTEGER_SENSOR_VALUE_TYPE ), - cv.Optional(CONF_REGISTER_COUNT): cv.positive_int, cv.Optional(CONF_SKIP_UPDATES): validate_skip_updates_deprecated, - cv.Optional(CONF_FORCE_NEW_RANGE, default=False): cv.boolean, + cv.Optional(CONF_REUSE_PREVIOUS_RANGE, default="auto"): cv.Any( + cv.boolean, cv.one_of("auto", lower=True) + ), + # Deprecated options, migrated by validate_range_reuse_migration(). Remove before 2027.3.0 + cv.Optional(CONF_FORCE_NEW_RANGE): cv.boolean, + cv.Optional(CONF_REGISTER_COUNT): cv.positive_int, cv.Required(CONF_OPTIONSMAP): ensure_option_map(), cv.Optional(CONF_USE_WRITE_MULTIPLE, default=False): cv.boolean, cv.Optional(CONF_OPTIMISTIC, default=False): cv.boolean, @@ -91,24 +82,18 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_WRITE_LAMBDA): cv.returning_lambda, }, ), - register_count_value_type_min, + validate_range_reuse_migration, ) async def to_code(config: ConfigType) -> None: - value_type = config[CONF_VALUE_TYPE] - reg_count = config.get(CONF_REGISTER_COUNT) - if reg_count is None: - reg_count = TYPE_REGISTER_MAP[value_type] - options_map = config[CONF_OPTIONSMAP] var = cg.new_Pvariable( config[CONF_ID], - value_type, + config[CONF_VALUE_TYPE], config[CONF_ADDRESS], - reg_count, - config[CONF_FORCE_NEW_RANGE], + RANGE_REUSE[config[CONF_REUSE_PREVIOUS_RANGE]], list(options_map.values()), ) diff --git a/esphome/components/modbus_controller/select/modbus_select.cpp b/esphome/components/modbus_controller/select/modbus_select.cpp index c1cc241d6b..a2f15d54f6 100644 --- a/esphome/components/modbus_controller/select/modbus_select.cpp +++ b/esphome/components/modbus_controller/select/modbus_select.cpp @@ -83,19 +83,17 @@ void ModbusSelect::control(size_t index) { } } - // register_count declares the READ range width - it may pull neighboring registers into one poll - - // so a write covers exactly the registers the value occupies: the quantity comes from the payload, - // never from register_count (padding to it would zero registers the user only declared for reading). - // A payload wider than the declared range means the config and the lambda disagree - drop it. - if (data.size() > this->register_count) { - ESP_LOGE(TAG, "Payload has %zu registers but register_count is %u; dropping write", data.size(), - this->register_count); + // A write covers exactly the registers the value occupies: the quantity comes from the payload. A + // payload wider than the value type's register width means the config and the lambda disagree - drop it. + if (data.size() > this->entity_count()) { + ESP_LOGE(TAG, "Payload has %zu registers but the value type only spans %u; dropping write", data.size(), + this->entity_count()); return; } const uint16_t write_address = this->write_address(); bool queued; - if ((this->register_count == 1) && (!this->use_write_multiple_)) { + if ((this->entity_count() == 1) && (!this->use_write_multiple_)) { queued = this->write_single_register(write_address, data[0]); } else { queued = this->write_multiple_registers(write_address, data); diff --git a/esphome/components/modbus_controller/select/modbus_select.h b/esphome/components/modbus_controller/select/modbus_select.h index c6ac76a45b..3827d38755 100644 --- a/esphome/components/modbus_controller/select/modbus_select.h +++ b/esphome/components/modbus_controller/select/modbus_select.h @@ -11,16 +11,15 @@ namespace esphome::modbus_controller { class ModbusSelect final : public Component, public select::Select, public SensorItem, public WriterEntity { public: - ModbusSelect(SensorValueType sensor_value_type, uint16_t start_address, uint8_t register_count, bool force_new_range, + ModbusSelect(SensorValueType sensor_value_type, uint16_t start_address, RangeReuse reuse_previous_range, std::vector mapping) { this->register_type = modbus::EntityType::HOLDING; // not configurable this->sensor_value_type = sensor_value_type; this->set_address(start_address); this->set_offset_from_start_address(0); // not configurable this->bitmask = 0xFFFFFFFF; // not configurable - this->register_count = register_count; - this->response_bytes = 0; // not configurable - this->force_new_range = force_new_range; + this->response_bytes = 0; // not configurable + this->reuse_previous_range = reuse_previous_range; this->mapping_ = std::move(mapping); } diff --git a/esphome/components/modbus_controller/sensor/__init__.py b/esphome/components/modbus_controller/sensor/__init__.py index 2c34ef04b4..bd51b9a8a3 100644 --- a/esphome/components/modbus_controller/sensor/__init__.py +++ b/esphome/components/modbus_controller/sensor/__init__.py @@ -5,6 +5,7 @@ import esphome.config_validation as cv from esphome.const import CONF_ADDRESS, CONF_ID from .. import ( + RANGE_REUSE, ModbusItemBaseSchema, SensorItem, add_modbus_base_properties, @@ -12,13 +13,13 @@ from .. import ( modbus_controller_ns, validate_custom_pdu_item, validate_modbus_register, + validate_range_reuse_migration, ) from ..const import ( CONF_BITMASK, - CONF_FORCE_NEW_RANGE, CONF_MODBUS_CONTROLLER_ID, - CONF_REGISTER_COUNT, CONF_REGISTER_TYPE, + CONF_REUSE_PREVIOUS_RANGE, CONF_VALUE_TYPE, ) @@ -38,27 +39,25 @@ CONFIG_SCHEMA = cv.All( { cv.Optional(CONF_REGISTER_TYPE): cv.enum(MODBUS_REGISTER_TYPE), cv.Optional(CONF_VALUE_TYPE, default="U_WORD"): cv.enum(SENSOR_VALUE_TYPE), - cv.Optional(CONF_REGISTER_COUNT, default=0): cv.positive_int, } ), validate_modbus_register, + validate_range_reuse_migration, ) FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item async def to_code(config): - byte_offset, reg_count = modbus_calc_properties(config) - value_type = config[CONF_VALUE_TYPE] + byte_offset = modbus_calc_properties(config) var = cg.new_Pvariable( config[CONF_ID], config[CONF_REGISTER_TYPE], config[CONF_ADDRESS], byte_offset, config[CONF_BITMASK], - value_type, - reg_count, - config[CONF_FORCE_NEW_RANGE], + config[CONF_VALUE_TYPE], + RANGE_REUSE[config[CONF_REUSE_PREVIOUS_RANGE]], ) await cg.register_component(var, config) await sensor.register_sensor(var, config) diff --git a/esphome/components/modbus_controller/sensor/modbus_sensor.h b/esphome/components/modbus_controller/sensor/modbus_sensor.h index 68dc9e6fcc..12c29bf584 100644 --- a/esphome/components/modbus_controller/sensor/modbus_sensor.h +++ b/esphome/components/modbus_controller/sensor/modbus_sensor.h @@ -11,14 +11,13 @@ namespace esphome::modbus_controller { class ModbusSensor final : public Component, public sensor::Sensor, public SensorItem { public: ModbusSensor(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint32_t bitmask, - SensorValueType value_type, int register_count, bool force_new_range) { + SensorValueType value_type, RangeReuse reuse_previous_range) { this->register_type = register_type; this->set_address(start_address); this->set_offset_from_start_address(offset); this->bitmask = bitmask; this->sensor_value_type = value_type; - this->register_count = register_count; - this->force_new_range = force_new_range; + this->reuse_previous_range = reuse_previous_range; } void parse_and_publish(std::span data) override; diff --git a/esphome/components/modbus_controller/switch/__init__.py b/esphome/components/modbus_controller/switch/__init__.py index 49dc0bb222..00b67446a3 100644 --- a/esphome/components/modbus_controller/switch/__init__.py +++ b/esphome/components/modbus_controller/switch/__init__.py @@ -6,6 +6,7 @@ from esphome.const import CONF_ADDRESS, CONF_ASSUMED_STATE, CONF_ID from esphome.types import ConfigType from .. import ( + RANGE_REUSE, ModbusItemBaseSchema, SensorItem, add_modbus_base_properties, @@ -14,12 +15,13 @@ from .. import ( reject_odd_holding_write_offset, validate_custom_pdu_item, validate_modbus_register, + validate_range_reuse_migration, ) from ..const import ( CONF_BITMASK, - CONF_FORCE_NEW_RANGE, CONF_MODBUS_CONTROLLER_ID, CONF_REGISTER_TYPE, + CONF_REUSE_PREVIOUS_RANGE, CONF_USE_WRITE_MULTIPLE, CONF_WRITE_LAMBDA, ) @@ -54,20 +56,21 @@ CONFIG_SCHEMA = cv.All( ), validate_modbus_register, _validate_holding_offset, + validate_range_reuse_migration, ) FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item async def to_code(config: ConfigType) -> None: - byte_offset, _ = modbus_calc_properties(config) + byte_offset = modbus_calc_properties(config) var = cg.new_Pvariable( config[CONF_ID], config[CONF_REGISTER_TYPE], config[CONF_ADDRESS], byte_offset, config[CONF_BITMASK], - config[CONF_FORCE_NEW_RANGE], + RANGE_REUSE[config[CONF_REUSE_PREVIOUS_RANGE]], ) await cg.register_component(var, config) await switch.register_switch(var, config) diff --git a/esphome/components/modbus_controller/switch/modbus_switch.h b/esphome/components/modbus_controller/switch/modbus_switch.h index bd1c837080..688a620bac 100644 --- a/esphome/components/modbus_controller/switch/modbus_switch.h +++ b/esphome/components/modbus_controller/switch/modbus_switch.h @@ -11,13 +11,12 @@ namespace esphome::modbus_controller { class ModbusSwitch final : public Component, public switch_::Switch, public SensorItem, public WriterEntity { public: ModbusSwitch(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint32_t bitmask, - bool force_new_range) { + RangeReuse reuse_previous_range) { this->register_type = register_type; this->set_address(start_address); this->set_offset_from_start_address(offset); this->bitmask = bitmask; this->sensor_value_type = SensorValueType::BIT; - this->register_count = 1; // A holding byte offset folds into the address as whole registers (odd offsets are rejected at // validation: a 16-bit register write cannot target half a register); a coil offset is a coil count. if (register_type == modbus::EntityType::HOLDING) { @@ -27,7 +26,7 @@ class ModbusSwitch final : public Component, public switch_::Switch, public Sens this->set_address(start_address + offset); this->set_offset_from_start_address(0); } - this->force_new_range = force_new_range; + this->reuse_previous_range = reuse_previous_range; }; void setup() override; void write_state(bool state) override; diff --git a/esphome/components/modbus_controller/text_sensor/__init__.py b/esphome/components/modbus_controller/text_sensor/__init__.py index 31f5f87a98..7ab77700ca 100644 --- a/esphome/components/modbus_controller/text_sensor/__init__.py +++ b/esphome/components/modbus_controller/text_sensor/__init__.py @@ -5,6 +5,7 @@ import esphome.config_validation as cv from esphome.const import CONF_ADDRESS, CONF_ID from .. import ( + RANGE_REUSE, ModbusItemBaseSchema, SensorItem, add_modbus_base_properties, @@ -12,14 +13,14 @@ from .. import ( modbus_controller_ns, validate_custom_pdu_item, validate_modbus_register, + validate_range_reuse_migration, ) from ..const import ( - CONF_FORCE_NEW_RANGE, CONF_MODBUS_CONTROLLER_ID, CONF_RAW_ENCODE, - CONF_REGISTER_COUNT, CONF_REGISTER_TYPE, CONF_RESPONSE_SIZE, + CONF_REUSE_PREVIOUS_RANGE, ) DEPENDENCIES = ["modbus_controller"] @@ -47,32 +48,27 @@ CONFIG_SCHEMA = cv.All( { cv.GenerateID(): cv.declare_id(ModbusTextSensor), cv.Optional(CONF_REGISTER_TYPE): cv.enum(MODBUS_REGISTER_TYPE), - cv.Optional(CONF_REGISTER_COUNT, default=0): cv.positive_int, - cv.Optional(CONF_RESPONSE_SIZE, default=2): cv.positive_int, + cv.Optional(CONF_RESPONSE_SIZE, default=2): cv.int_range(min=1, max=250), cv.Optional(CONF_RAW_ENCODE, default="ANSI"): cv.enum(RAW_ENCODING), } ), validate_modbus_register, + validate_range_reuse_migration, ) FINAL_VALIDATE_SCHEMA = validate_custom_pdu_item async def to_code(config): - byte_offset, reg_count = modbus_calc_properties(config) - response_size = config[CONF_RESPONSE_SIZE] - reg_count = config[CONF_REGISTER_COUNT] - if reg_count == 0: - reg_count = response_size // 2 + byte_offset = modbus_calc_properties(config) var = cg.new_Pvariable( config[CONF_ID], config[CONF_REGISTER_TYPE], config[CONF_ADDRESS], byte_offset, - reg_count, config[CONF_RESPONSE_SIZE], config[CONF_RAW_ENCODE], - config[CONF_FORCE_NEW_RANGE], + RANGE_REUSE[config[CONF_REUSE_PREVIOUS_RANGE]], ) await cg.register_component(var, config) diff --git a/esphome/components/modbus_controller/text_sensor/modbus_textsensor.h b/esphome/components/modbus_controller/text_sensor/modbus_textsensor.h index 9e8dce57e7..6657967786 100644 --- a/esphome/components/modbus_controller/text_sensor/modbus_textsensor.h +++ b/esphome/components/modbus_controller/text_sensor/modbus_textsensor.h @@ -12,17 +12,16 @@ enum class RawEncoding { NONE = 0, HEXBYTES = 1, COMMA = 2, ANSI = 3 }; class ModbusTextSensor final : public Component, public text_sensor::TextSensor, public SensorItem { public: - ModbusTextSensor(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint8_t register_count, - uint16_t response_bytes, RawEncoding encode, bool force_new_range) { + ModbusTextSensor(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint16_t response_bytes, + RawEncoding encode, RangeReuse reuse_previous_range) { this->register_type = register_type; this->set_address(start_address); this->set_offset_from_start_address(offset); this->response_bytes = response_bytes; - this->register_count = register_count; this->encode_ = encode; this->bitmask = 0xFFFFFFFF; this->sensor_value_type = SensorValueType::RAW; - this->force_new_range = force_new_range; + this->reuse_previous_range = reuse_previous_range; } void dump_config() override; diff --git a/tests/components/modbus_controller/common.yaml b/tests/components/modbus_controller/common.yaml index 78bec522cf..b9a7610cb7 100644 --- a/tests/components/modbus_controller/common.yaml +++ b/tests/components/modbus_controller/common.yaml @@ -21,6 +21,7 @@ binary_sensor: name: Test Binary Sensor with Lambda register_type: input address: 0x3201 + reuse_previous_range: false lambda: |- return x; @@ -85,6 +86,7 @@ select: name: Test Select with Lambda address: 1001 value_type: U_WORD + reuse_previous_range: auto optionsmap: "Off": 0 "On": 1 @@ -140,9 +142,10 @@ sensor: register_type: holding address: 0x9002 value_type: U_WORD + reuse_previous_range: true lambda: |- return x / 10.0; - # Non-mergeable sensor sharing the start address of modbus_sensor1 (different register_count): + # Non-mergeable sensor sharing the start address of modbus_sensor1 (different value type width): # must join the same range, never open a second range keyed on the same (address, type). - platform: modbus_controller modbus_controller_id: modbus_controller1 @@ -187,8 +190,8 @@ sensor: value_type: U_WORD lambda: |- return modbus_controller::get_data(data, item->offset) * 0.1f; - # force_new_range sensors sort before plain ones, so this high-address forced sensor is grouped - # first and the lower-address plain sensors above must still get their own ranges. + # The deprecated force_new_range migrates to reuse_previous_range: false, so this sensor never + # joins a range built before it and the lower-address sensors above keep their own ranges. - platform: modbus_controller modbus_controller_id: modbus_controller1 id: modbus_sensor_forced_high @@ -224,7 +227,6 @@ text_sensor: name: Test Text Sensor register_type: holding address: 0x9013 - register_count: 3 raw_encode: HEXBYTES response_size: 6 - platform: modbus_controller @@ -233,12 +235,13 @@ text_sensor: name: Test Text Sensor with Lambda register_type: holding address: 0x9014 - register_count: 2 response_size: 4 lambda: |- return "Modified: " + x; - # A register reporting FEWER bytes than 2*register_count (response_size: 3 for 2 registers), followed - # by a contiguous sensor: the follower's byte position must track the actual 3 bytes, not underflow. + # A register reporting FEWER bytes than two per register (response_size: 3 over 2 registers), followed + # by a contiguous reuse:true sensor (auto never joins past a response_size register): the follower's + # byte position must track the actual 3 bytes, not underflow. + # register_count matches the derived width, so it migrates with a deprecation warning. - platform: modbus_controller modbus_controller_id: modbus_controller1 id: modbus_text_sensor_narrow @@ -255,4 +258,5 @@ text_sensor: register_type: holding address: 0x9032 register_count: 1 + reuse_previous_range: true raw_encode: HEXBYTES diff --git a/tests/integration/fixtures/uart_mock_modbus_grouping.yaml b/tests/integration/fixtures/uart_mock_modbus_grouping.yaml index d580f5c2e2..2c4c39e7a5 100644 --- a/tests/integration/fixtures/uart_mock_modbus_grouping.yaml +++ b/tests/integration/fixtures/uart_mock_modbus_grouping.yaml @@ -27,8 +27,8 @@ uart_mock: # so these also pin the grouping: an extra or differently shaped read fails the test. - expect_tx: [0x01, 0x01, 0x00, 0x10, 0x00, 0x02, 0xBC, 0x0E] # coils 0x10 count 2 inject_rx: [0x01, 0x01, 0x01, 0x01, 0x90, 0x48] # bit0 set, bit1 clear - - expect_tx: [0x01, 0x03, 0x01, 0x60, 0x00, 0x01, 0x85, 0xE8] # holding 0x160 count 1 - inject_rx: [0x01, 0x03, 0x02, 0x01, 0x60, 0xB9, 0xFC] # 352 + - expect_tx: [0x01, 0x03, 0x01, 0x60, 0x00, 0x02, 0xC5, 0xE9] # holding 0x160 count 2 + inject_rx: [0x01, 0x03, 0x04, 0x01, 0x60, 0x01, 0x61, 0x3B, 0xA9] # 352, 353 - expect_tx: [0x01, 0x03, 0x01, 0x00, 0x00, 0x01, 0x85, 0xF6] # holding 0x100 count 1 inject_rx: [0x01, 0x03, 0x04, 0x01, 0x11, 0x02, 0x22, 0x2A, 0xB3] # 4 bytes: 273 then 546 - expect_tx: [0x01, 0x03, 0x01, 0x20, 0x00, 0x04, 0x44, 0x3F] # holding 0x120 count 4 @@ -49,8 +49,6 @@ uart_mock: inject_rx: [0x01, 0x03, 0x02, 0x33, 0x33, 0xEC, 0xA1] # 13107 - expect_tx: [0x01, 0x03, 0x01, 0x70, 0x00, 0x03, 0x05, 0xEC] # holding 0x170 count 3 inject_rx: [0x01, 0x03, 0x06, 0x00, 0x2A, 0x1B, 0x2C, 0x03, 0x0D, 0x3E, 0xAB] # 6 bytes - - expect_tx: [0x01, 0x03, 0x01, 0x61, 0x00, 0x01, 0xD4, 0x28] # holding 0x161 count 1 - inject_rx: [0x01, 0x03, 0x02, 0x01, 0x61, 0x78, 0x3C] # 353 modbus: uart_id: virtual_uart_dev @@ -104,8 +102,9 @@ sensor: value_type: U_DWORD modbus_controller_id: modbus_controller_ok - # D - a wide (response_size) register followed by a contiguous one: the follower must start after the - # bytes the wide register actually returned, not after 2 * register_count. + # D - a wide (response_size) register followed by a contiguous reuse:true one (auto never joins past + # a response_size register): the follower must start after the bytes the wide register actually + # returned, not after two per register. - platform: modbus_controller name: "wide_first" address: 0x130 @@ -118,6 +117,7 @@ sensor: address: 0x131 register_type: holding value_type: U_WORD + reuse_previous_range: true modbus_controller_id: modbus_controller_ok # E - a gap: these must never share a range. @@ -195,13 +195,14 @@ sensor: value_type: U_WORD modbus_controller_id: modbus_controller_ok - # H - a sensor pinned to its own range, followed by a contiguous one. + # H - a sensor that never joins the range built before it (reuse_previous_range: false), followed + # by a contiguous plain item that extends the new range it started. - platform: modbus_controller name: "forced_first" address: 0x160 register_type: holding value_type: U_WORD - force_new_range: true + reuse_previous_range: false modbus_controller_id: modbus_controller_ok - platform: modbus_controller name: "forced_next" diff --git a/tests/integration/fixtures/uart_mock_modbus_ranges.yaml b/tests/integration/fixtures/uart_mock_modbus_ranges.yaml new file mode 100644 index 0000000000..09e5a20241 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_ranges.yaml @@ -0,0 +1,227 @@ +esphome: + name: uart-mock-modbus-ranges-test + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +# Each expect_tx below pins the exact read request the controller's range builder emits, so this +# fixture is a wire-level test of reuse_previous_range (auto/yes/no), gap joins, same-register reuse, +# response_size surplus accounting, and RAW/text block reads. +uart_mock: + - id: virtual_uart_dev + baud_rate: 9600 + rx_full_threshold: 120 + rx_timeout: 2 + # auto_start must be false to avoid races: the test presses the + # "Start Scenario" button only after subscribing to states. + auto_start: false + debug: + responses: + - expect_tx: [0x01, 0x03, 0x00, 0x00, 0x00, 0x03, 0x05, 0xCB] # auto adjacency: one read covers 0x00-0x02 + inject_rx: [0x01, 0x03, 0x06, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0xFD, 0x74] + - expect_tx: [0x01, 0x03, 0x00, 0x10, 0x00, 0x01, 0x85, 0xCF] # auto gap: 0x10 alone + inject_rx: [0x01, 0x03, 0x02, 0x00, 0x04, 0xB9, 0x87] + - expect_tx: [0x01, 0x03, 0x00, 0x13, 0x00, 0x01, 0x75, 0xCF] # auto gap: 0x13 alone + inject_rx: [0x01, 0x03, 0x02, 0x00, 0x05, 0x78, 0x47] + - expect_tx: [0x01, 0x03, 0x00, 0x20, 0x00, 0x04, 0x45, 0xC3] # yes across gap: one read 0x20-0x23, gap registers ignored + inject_rx: [0x01, 0x03, 0x08, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x33, 0xD1] + - expect_tx: [0x01, 0x03, 0x00, 0x30, 0x00, 0x01, 0x84, 0x05] # no isolation: 0x30 alone despite adjacency + inject_rx: [0x01, 0x03, 0x02, 0x00, 0x0A, 0x38, 0x43] + - expect_tx: [0x01, 0x03, 0x00, 0x31, 0x00, 0x01, 0xD5, 0xC5] # no isolation: 0x31 alone + inject_rx: [0x01, 0x03, 0x02, 0x00, 0x0B, 0xF9, 0x83] + - expect_tx: [0x01, 0x03, 0x00, 0x3F, 0x00, 0x01, 0xB4, 0x06] # open NEVER: 0x3F alone (the reuse:false item split off) + inject_rx: [0x01, 0x03, 0x02, 0x00, 0x0C, 0xB8, 0x41] + - expect_tx: [0x01, 0x03, 0x00, 0x40, 0x00, 0x02, 0xC5, 0xDF] # open NEVER: 0x40 (reuse: false) still extended by the auto item at 0x41 + inject_rx: [0x01, 0x03, 0x04, 0x00, 0x0D, 0x00, 0x0E, 0xEA, 0x34] + - expect_tx: [0x01, 0x03, 0x00, 0x50, 0x00, 0x01, 0x84, 0x1B] # same-address reuse: one read, two sensors on 0x50 + inject_rx: [0x01, 0x03, 0x02, 0x12, 0x34, 0xB5, 0x33] + - expect_tx: [0x01, 0x03, 0x00, 0x60, 0x00, 0x04, 0x44, 0x17] # text block + adjacent word: one read 0x60-0x63 + inject_rx: [0x01, 0x03, 0x08, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x00, 0x0F, 0x78, 0x0E] + - expect_tx: [0x01, 0x03, 0x00, 0x70, 0x00, 0x02, 0xC5, 0xD0] # response_size surplus: 0x70 answers 4 bytes, reuse:true word at 0x71 shifted along + inject_rx: [0x01, 0x03, 0x06, 0x00, 0x10, 0xAA, 0xBB, 0x00, 0x11, 0x71, 0x47] + - expect_tx: [0x01, 0x03, 0x00, 0x90, 0x00, 0x01, 0x84, 0x27] # auto after surplus: 0x90 alone (auto never joins past response_size) + inject_rx: [0x01, 0x03, 0x04, 0x00, 0x18, 0xCC, 0xDD, 0xEF, 0x6D] + - expect_tx: [0x01, 0x03, 0x00, 0x91, 0x00, 0x01, 0xD5, 0xE7] # auto after surplus: 0x91 alone + inject_rx: [0x01, 0x03, 0x02, 0x00, 0x19, 0x79, 0x8E] + - expect_tx: [0x01, 0x03, 0x00, 0x80, 0x00, 0x04, 0x45, 0xE1] # RAW block via response_size: 8 bytes = 4 registers in one read + inject_rx: [0x01, 0x03, 0x08, 0x00, 0x14, 0x00, 0x15, 0x00, 0x16, 0x00, 0x17, 0x6D, 0xDF] + +modbus: + uart_id: virtual_uart_dev + send_wait_time: 200ms + turnaround_time: 10ms + +modbus_controller: + - address: 1 + id: ranges_controller + max_cmd_retries: 0 + # The test triggers a single poll by pressing the "Start Scenario" button + update_interval: never + +sensor: + # Case 1: three adjacent registers merge into one read (auto default) + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "adjacent_a" + register_type: holding + address: 0x00 + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "adjacent_b" + register_type: holding + address: 0x01 + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "adjacent_c" + register_type: holding + address: 0x02 + + # Case 2: a gap keeps auto items apart + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "gap_a" + register_type: holding + address: 0x10 + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "gap_b" + register_type: holding + address: 0x13 + + # Case 3: reuse_previous_range: true bridges the gap into one read + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "bridge_a" + register_type: holding + address: 0x20 + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "bridge_b" + register_type: holding + address: 0x23 + reuse_previous_range: true + + # Case 4: reuse_previous_range: false splits adjacent registers + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "split_a" + register_type: holding + address: 0x30 + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "split_b" + register_type: holding + address: 0x31 + reuse_previous_range: false + + # Case 5: a reuse:false item starts its own range but stays open for later auto items + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "open_prev" + register_type: holding + address: 0x3F + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "open_never" + register_type: holding + address: 0x40 + reuse_previous_range: false + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "open_tagalong" + register_type: holding + address: 0x41 + + # Case 6: two sensors on the same register share one read + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "shared_lo" + register_type: holding + address: 0x50 + bitmask: 0x00FF + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "shared_hi" + register_type: holding + address: 0x50 + bitmask: 0xFF00 + + # Case 10 (text block, see text_sensor below) shares the range with this word at 0x63 + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "after_text" + register_type: holding + address: 0x63 + + # Case 11: response_size surplus — the device answers 4 bytes for this single register, so the + # following sensor's data sits 2 bytes later than its address alone implies. Joining past a + # non-standard response_size takes an explicit reuse_previous_range: true. + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "surplus" + register_type: holding + address: 0x70 + response_size: 4 + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "after_surplus" + register_type: holding + address: 0x71 + reuse_previous_range: true + + # Case 13: auto never joins past a response_size register — despite adjacency these poll separately + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "surplus_split" + register_type: holding + address: 0x90 + response_size: 4 + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "after_surplus_split" + register_type: holding + address: 0x91 + + # Case 12: RAW + response_size reads a block of ceil(8/2) = 4 registers + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "raw_block" + register_type: holding + address: 0x80 + value_type: RAW + response_size: 8 + lambda: |- + return (float) data.size(); + +text_sensor: + # Case 10: text sensor reads 3 registers (response_size 6) + - platform: modbus_controller + modbus_controller_id: ranges_controller + name: "text_block" + register_type: holding + address: 0x60 + response_size: 6 + raw_encode: NONE + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + on_press: + - lambda: |- + id(virtual_uart_dev).start_scenario(); + id(ranges_controller).set_update_interval(1000); + id(ranges_controller).start_poller(); diff --git a/tests/integration/fixtures/uart_mock_modbus_shared_address.yaml b/tests/integration/fixtures/uart_mock_modbus_shared_address.yaml index 109603f3b6..7a94082ed8 100644 --- a/tests/integration/fixtures/uart_mock_modbus_shared_address.yaml +++ b/tests/integration/fixtures/uart_mock_modbus_shared_address.yaml @@ -32,9 +32,9 @@ uart_mock: # duplicate or overlapping range would put an extra frame on the bus and fail to match. - expect_tx: [0x01, 0x03, 0x90, 0x01, 0x00, 0x02, 0xB8, 0xCB] # Read holding 0x9001 count 2 on device 1 inject_rx: [0x01, 0x03, 0x04, 0x03, 0x97, 0x02, 0x91, 0x8B, 0x57] # 0x9001=0x0397, 0x9002=0x0291 - # A force_new_range sensor at a HIGH address (0x30) sorts before the plain sensor at a LOW address - # (0x10). The two must poll as separate ranges: the covered branch's lower-bound check prevents the - # 0x10 sensor from being absorbed into the forced 0x30 range with a wrapped byte offset. + # A sensor at 0x30 with the deprecated force_new_range (migrates to reuse_previous_range: false) + # and a plain sensor at 0x10. The two must poll as separate ranges: the 0x10 sensor must not be + # absorbed into the isolated 0x30 range. - expect_tx: [0x01, 0x03, 0x00, 0x30, 0x00, 0x01, 0x84, 0x05] # Read holding 0x30 count 1 (forced range) inject_rx: [0x01, 0x03, 0x02, 0x01, 0x11, 0x79, 0xD8] # 0x30 = 0x0111 = 273 - expect_tx: [0x01, 0x03, 0x00, 0x10, 0x00, 0x01, 0x85, 0xCF] # Read holding 0x10 count 1 (own range) @@ -87,7 +87,7 @@ sensor: register_type: holding value_type: U_WORD modbus_controller_id: modbus_controller_ok - # Forced sensor at a high address: sorts first, opens its own isolated range + # Isolated sensor (deprecated spelling, migrates to reuse_previous_range: false): own range - platform: modbus_controller name: "forced_high" address: 0x30 @@ -95,7 +95,7 @@ sensor: value_type: U_WORD force_new_range: true modbus_controller_id: modbus_controller_ok - # Plain sensor at a lower address: must get its own range, never absorbed into the forced one + # Plain sensor at a lower address: must get its own range, never absorbed into the isolated one - platform: modbus_controller name: "plain_low" address: 0x10 diff --git a/tests/integration/test_uart_mock_modbus.py b/tests/integration/test_uart_mock_modbus.py index f88febabf5..864275f5ed 100644 --- a/tests/integration/test_uart_mock_modbus.py +++ b/tests/integration/test_uart_mock_modbus.py @@ -21,7 +21,7 @@ import asyncio from collections.abc import Callable from dataclasses import dataclass -from aioesphomeapi import ButtonInfo, NumberInfo, SwitchInfo +from aioesphomeapi import ButtonInfo, NumberInfo, SwitchInfo, TextSensorState import pytest from .state_utils import SensorTracker, find_entity, wait_for_state @@ -1158,3 +1158,78 @@ async def test_uart_mock_modbus_deprecated_write_buffer( assert warn_count == 1, ( f"deprecation warning should fire exactly once per entity, got {warn_count}" ) + + +@pytest.mark.asyncio +async def test_uart_mock_modbus_ranges( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Wire-level test of the range builder's reuse_previous_range semantics. + + Every expect_tx in the fixture pins the exact read request the controller emits, so a + wrongly merged or split range fails on the mock before any value arrives. Covers: auto + adjacency merging, auto gap splitting, reuse:true bridging a gap (with correct data + offsets past the gap), reuse:false splitting adjacent registers while staying open for + later auto items, two sensors sharing one register, a text block read sized by + response_size with a following word, response_size surplus shifting a later reuse:true + sensor's bytes while an auto sensor refuses to join past the surplus, and a RAW block + read of ceil(response_size / 2) registers. + """ + + line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback() + + expected_values = { + "adjacent_a": 1, + "adjacent_b": 2, + "adjacent_c": 3, + "gap_a": 4, + "gap_b": 5, + "bridge_a": 6, + "bridge_b": 9, + "split_a": 10, + "split_b": 11, + "open_prev": 12, + "open_never": 13, + "open_tagalong": 14, + "shared_lo": 0x34, + "shared_hi": 0x12, + "after_text": 15, + "surplus": 16, + "after_surplus": 17, + "surplus_split": 24, + "after_surplus_split": 25, + "raw_block": 8, # the RAW lambda publishes data.size(): 4 registers = 8 bytes + } + tracker = SensorTracker(list(expected_values.keys())) + futures = tracker.expect_all(expected_values) + + # The tracker only handles numeric sensors; capture the text block separately. + text_future: asyncio.Future = asyncio.get_running_loop().create_future() + tracker_on_state = tracker.on_state + + def on_state(state) -> None: + if ( + isinstance(state, TextSensorState) + and not state.missing_state + and state.state == "ABCDEF" + and not text_future.done() + ): + text_future.set_result(True) + tracker_on_state(state) + + tracker.on_state = on_state + + async with ( + run_compiled(yaml_config, line_callback=line_callback), + api_client_connected() as client, + ): + await tracker.setup_and_start_scenario(client) + await tracker.await_all(futures) + # text_block is not tracker-registered (non-numeric), so time out explicitly. + try: + await asyncio.wait_for(text_future, timeout=5.0) + except TimeoutError: + pytest.fail("text_block never published 'ABCDEF'") + _assert_no_modbus_errors(error_log_lines, warning_log_lines) From 8db07d0de5a912ece03dae8136d97fa175f37fb4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 28 Aug 2026 12:57:45 -0500 Subject: [PATCH 002/147] [mdns] Bump espressif/mdns to 1.12.0 (#18861) --- esphome/components/mdns/__init__.py | 6 ++++-- esphome/idf_component.yml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index c9334ea97a..64d7b9adc5 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -1,5 +1,5 @@ import esphome.codegen as cg -from esphome.components.esp32 import add_idf_component +from esphome.components.esp32 import add_idf_component, add_idf_sdkconfig_option from esphome.config_helpers import filter_source_files_from_platform, get_logger_level import esphome.config_validation as cv from esphome.const import ( @@ -208,7 +208,9 @@ async def to_code(config: ConfigType) -> None: ethernet.request_ethernet_ip_state_listener() if CORE.is_esp32: - add_idf_component(name="espressif/mdns", ref="1.11.3") + add_idf_component(name="espressif/mdns", ref="1.12.0") + # ESPHome only advertises; the browse APIs are unused + add_idf_sdkconfig_option("CONFIG_MDNS_ENABLE_BROWSE", False) cg.add_define("USE_MDNS") diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 62fd597845..e817a253d9 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -24,7 +24,7 @@ dependencies: espressif/esp32-camera: version: 2.1.7 espressif/mdns: - version: 1.11.3 + version: 1.12.0 espressif/esp_wifi_remote: version: 1.6.3 rules: From 768ab5b672bceed3c81d09db60c3d710bcbc93f1 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 11:16:41 -0700 Subject: [PATCH 003/147] [modbus] Hub and helpers cleanup; tighten queue_pdu validation (#18847) --- esphome/components/modbus/__init__.py | 5 +- esphome/components/modbus/modbus.cpp | 91 ++------ esphome/components/modbus/modbus.h | 214 ++++++------------ .../components/modbus/modbus_definitions.h | 1 - esphome/components/modbus/modbus_helpers.cpp | 13 +- esphome/components/modbus/modbus_helpers.h | 39 ++-- .../modbus/modbus_client_hub_test.cpp | 81 +++++-- .../components/modbus/modbus_helpers_test.cpp | 6 + .../modbus/modbus_unknown_function_test.cpp | 32 ++- 9 files changed, 207 insertions(+), 275 deletions(-) diff --git a/esphome/components/modbus/__init__.py b/esphome/components/modbus/__init__.py index 769858e72a..76cfdbed70 100644 --- a/esphome/components/modbus/__init__.py +++ b/esphome/components/modbus/__init__.py @@ -89,9 +89,8 @@ _WRITE_FUNCTION_CODES = frozenset({0x05, 0x06, 0x0F, 0x10, 0x16, 0x17}) def is_function_code_write(function_code: int) -> bool: """True if the Modbus function code writes (mutates). The exception bit (0x80) is masked off first, - so an exception-flagged code still classifies by its base code - stricter than the runtime hub, - whose classify() treats an exception-flagged code as a read. Keep in sync with - modbus::helpers::is_function_code_write().""" + so an exception-flagged code still classifies by its base code (the runtime hub never queues one: + queue_pdu() refuses them). Keep in sync with modbus::helpers::is_function_code_write().""" return function_code & 0x7F in _WRITE_FUNCTION_CODES diff --git a/esphome/components/modbus/modbus.cpp b/esphome/components/modbus/modbus.cpp index e83ffb2708..aa998d283a 100644 --- a/esphome/components/modbus/modbus.cpp +++ b/esphome/components/modbus/modbus.cpp @@ -10,17 +10,12 @@ namespace esphome::modbus { static const char *const TAG = "modbus"; -// Maximum bytes to log for Modbus frames (truncated if larger) static constexpr size_t MODBUS_MAX_LOG_BYTES = 64; // Approximate bits per character on the wire (depends on parity/stop bit config) static constexpr uint32_t MODBUS_BITS_PER_CHAR = 11; -// Milliseconds per second static constexpr uint32_t MS_PER_SEC = 1000; -// Shortest gap between two "no device accepted broadcast" warnings -static constexpr uint32_t UNACCEPTED_BROADCAST_WARN_INTERVAL_MS = 60 * MS_PER_SEC; - void Modbus::setup() { if (this->flow_control_pin_ != nullptr) { this->flow_control_pin_->setup(); @@ -43,10 +38,7 @@ void Modbus::setup() { } void Modbus::loop() { - // Receive any available bytes from UART this->receive_bytes_(); - - // Parse bytes into frames and process them this->parse_modbus_frames(); } @@ -55,7 +47,7 @@ void ModbusClientHub::loop() { // never times out an entry whose pending count has not been drained. No-op when nothing is owed. this->sweep_(); - this->Modbus::loop(); // receive bytes and parse frames + this->Modbus::loop(); // Send-wait watchdog: only the cheap time check runs at loop rate; expire_waiting_() looks the // entry up and holds off if the response has started arriving. @@ -104,11 +96,8 @@ bool Modbus::timeout_() { } int32_t Modbus::tx_delay_remaining() { - // We use millis() here and elsewhere instead of App.get_loop_component_start_time() to avoid stale timestamps - // It's critical in all timestamp comparisons that the left timestamp comes before the right one in time - // If we use a cached value in place of millis() and last_modbus_byte_ is updated inside our loop - // then the comparison is backwards (small negative which wraps to large positive) and will cause a false timeout - // So in this component we don't use any cached timestamp values to avoid these annoying bugs + // millis() here and everywhere in this component, never a cached loop timestamp: a cached "now" can + // predate last_modbus_byte_, and the unsigned subtraction then wraps huge and forces a false timeout. const uint32_t now = millis(); return std::max({(int32_t) 0, (int32_t) (this->last_send_tx_offset_ + this->frame_delay_ms_ - (now - this->last_send_)), @@ -124,22 +113,13 @@ int32_t ModbusClientHub::tx_delay_remaining() { } bool Modbus::tx_blocked() { - // We block transmission in any of these cases: - // 1. There are bytes in the UART Rx buffer - // 2. There are bytes in our Rx buffer - // 3. The last sent byte isn't more than tx_delay ms ago (i.e. wait to tell receivers that our previous Tx is done) - // 4. The last received byte isn't more than tx_delay ms ago (i.e. wait to be sure there isn't more Rx coming) - // N.B. We allow a small delay (MODBUS_TX_MAX_DELAY_MS) to avoid looping on small delays. This gets handled by - // send_frame_. + // Blocked while any rx bytes are pending, or within tx_delay of the last byte in either direction + // (receivers must see our previous tx as done, and more rx may be coming). A remaining delay up to + // MODBUS_TX_MAX_DELAY_MS doesn't block - send_frame_ absorbs it instead of looping on small waits. return this->available() || !this->rx_buffer_.empty() || this->tx_delay_remaining() > MODBUS_TX_MAX_DELAY_MS; } -bool ModbusClientHub::tx_blocked() { - // We block transmission in any of these case: - // 1. We're waiting for a response (a waiting entry: WAITING/INTERRUPTED/WAITING_RETIRED/INTERRUPTED_RETIRED) - // 2. Any of the base class tx_blocked conditions - return this->waiting_for_response_ || this->Modbus::tx_blocked(); -} +bool ModbusClientHub::tx_blocked() { return this->waiting_for_response_ || this->Modbus::tx_blocked(); } bool ModbusClientHub::tx_buffer_empty() { // "Empty" for ready_for_immediate_send(): no one-shot is queued ahead of the caller. Entries in @@ -219,10 +199,9 @@ void ModbusServerHub::parse_modbus_frames() { this->clear_rx_buffer_(LOG_STR("timeout after partial response"), true); } +// Scans forward from min_length to find a frame boundary by CRC match for unknown-length function codes. +// Returns the matched frame length, or 0 if no valid CRC was found within MAX_FRAME_SIZE. uint16_t Modbus::find_frame_end_by_crc_(uint16_t min_length) const { - // Unknown-length functions (user-defined codes, unimplemented management codes, unassigned values) - // could be any length - we have to rely on the CRC to determine completeness. - // If a CRC match is never found, the buffer will eventually overflow and be cleared. const uint8_t *raw = &this->rx_buffer_[0]; const size_t size = this->rx_buffer_.size(); const auto max_len = static_cast(std::min(size, size_t(MAX_FRAME_SIZE))); @@ -531,8 +510,7 @@ void ModbusServerHub::process_broadcast_frame_(uint8_t function_code, std::span< return; } // A broadcast is never answered, so a rejecting device has no other feedback channel: report the - // per-device outcome at V, and warn if the write reached nobody at all. - bool accepted = false; + // per-device outcome at V. for (auto *device : this->devices_) { // Same handlers as an addressed write - a device cannot tell a broadcast apart, and does not need // to: the hub owns the difference, which is only that no reply is ever sent. @@ -542,24 +520,6 @@ void ModbusServerHub::process_broadcast_frame_(uint8_t function_code, std::span< if (device_status.has_value()) { ESP_LOGV(TAG, "Device %" PRIu8 " rejected broadcast write with exception %" PRIu8, device->get_address(), static_cast(device_status.value())); - } else { - accepted = true; - } - } - if (!accepted && !this->devices_.empty()) { - const uint16_t entity_count = coils ? coil_count : static_cast(registers.size()); - const LogString *const entity_name = coils ? LOG_STR("coils") : LOG_STR("registers"); - // Warn at most once per interval, then drop to VERBOSE: on a shared bus a broadcast aimed at other nodes - // repeats forever, so warning per frame would flood the log. - const uint32_t now = millis(); - if (this->last_unaccepted_broadcast_warn_ == 0 || - now - this->last_unaccepted_broadcast_warn_ > UNACCEPTED_BROADCAST_WARN_INTERVAL_MS) { - this->last_unaccepted_broadcast_warn_ = now; - ESP_LOGW(TAG, "No device accepted broadcast write of %" PRIu16 " %s at 0x%04X", entity_count, - LOG_STR_ARG(entity_name), start_address); - } else { - ESP_LOGV(TAG, "No device accepted broadcast write of %" PRIu16 " %s at 0x%04X", entity_count, - LOG_STR_ARG(entity_name), start_address); } } } @@ -783,8 +743,6 @@ bool Modbus::send_frame_(const ModbusFrame &frame) { delay(tx_delay_remaining); } - // The delay above can span several ms; a byte arriving in that window blocks transmission after the - // caller's gate already passed. Don't collide with the incoming frame - leave the entry to retry. if (this->tx_blocked()) { return false; } @@ -831,7 +789,7 @@ void ModbusClientHub::send_next_frame_() { // reports the transmission, and the entry then retires with no terminal callback instead of // occupying the waiting slot until the send-wait timeout expires. The turnaround delay already // spaces the next frame; the following sweep erases the entry. - ESP_LOGV(TAG, "Broadcast to address 0 sent; no reply expected (fire-and-forget)"); + ESP_LOGV(TAG, "Broadcast to address 0 sent; no reply expected"); cmd->complete_broadcast(); this->sweep_needed_ = true; return; @@ -983,6 +941,8 @@ bool ModbusDeviceCommand::timed_out() { this->decrement_pending(); // resolve this request (WAITING-origin, so pending >= 1) if (this->device == nullptr) return false; // resolved, no one to tell + // A cleared frame that timed out still honors a retry: the clear is address-scoped (any device may + // call it) while the retry is the owning device's call via on_no_response - the bus obeys the owner. if (this->device->on_no_response(this->frame.pdu())) this->increment_pending(); // granted retry = re-request (capped) return true; @@ -1054,18 +1014,14 @@ bool ModbusClientHub::queue_pdu(uint8_t address, std::span pdu, M ESP_LOGE(TAG, "Frame too large, refused: %" PRIu8 ":%zu bytes", address, pdu.size()); return false; } - // classify() drives both the broadcast guard and the continuous check below; compute it once. - const CommandPriority priority = ModbusDeviceCommand::classify(pdu[0]); - // A broadcast (address 0) is never answered (Modbus 4.1), so it is only meaningful for a command that - // changes state. Refuse a broadcast that expects a reply - anything but a write or a custom/vendor code - - // as it could never deliver a result, so the caller learns via the false return (and on_not_sent). - // 0x17 (read/write multiple) is a knowing inclusion: classify() treats it as a write, so its write half - // lands on every server and its unanswerable read half is simply discarded. An exception-flagged custom - // code (0x80 bit set) is refused: is_function_code_custom() masks that bit away, so exclude it explicitly - // here to match classify()'s exception-first handling of the write side. - if (address == BROADCAST_ADDRESS && priority != CommandPriority::WRITE && - (!helpers::is_function_code_custom(pdu[0]) || helpers::is_function_code_exception(pdu[0]))) { + if (helpers::is_function_code_exception(pdu[0])) { + ESP_LOGW(TAG, "Exception PDU refused for address %" PRIu8 ": function code 0x%X has the exception bit set", address, + pdu[0]); + return false; + } + + if (address == BROADCAST_ADDRESS && !helpers::is_function_code_broadcastable(pdu[0])) { ESP_LOGW(TAG, "Broadcast refused for function 0x%X: a broadcast (address 0) is never answered", pdu[0]); return false; } @@ -1073,7 +1029,7 @@ bool ModbusClientHub::queue_pdu(uint8_t address, std::span pdu, M // Normalize the caller's options in place (the param is a by-value copy) so everything stored or // merged below carries effective options, never the raw request. // continuous is ignored for every mutating code (re-writing a value forever is never intended). - if (options.continuous && priority == CommandPriority::WRITE) { + if (options.continuous && helpers::is_function_code_write(pdu[0])) { ESP_LOGW(TAG, "continuous is ignored for a mutating function (0x%X, address %" PRIu8 ")", pdu[0], address); options.continuous = false; } @@ -1089,9 +1045,7 @@ bool ModbusClientHub::queue_pdu(uint8_t address, std::span pdu, M continue; if (device == nullptr) { // A dropped read is routine (DEBUG); a dropped write/custom warns (unobservable without a device). - const bool requeueable = - !helpers::is_function_code_exception(pdu[0]) && helpers::is_function_code_read_only(pdu[0]); - if (requeueable) { + if (helpers::is_function_code_read_only(pdu[0])) { ESP_LOGD(TAG, "Anonymous duplicate of active frame for %" PRIu8 " (function 0x%X), dropped", address, pdu[0]); } else { ESP_LOGW(TAG, @@ -1364,7 +1318,6 @@ void ModbusClientDevice::dispatch_response_(std::span request_pdu } } -// Default on_custom_response handler to warn when responses unexpectedly trigger on_custom_response void ModbusClientDevice::on_custom_response(std::span request_pdu, std::span response_pdu, ResponseStatus status) { // The dispatcher never calls this with an empty request, but this is a public virtual - stay safe. diff --git a/esphome/components/modbus/modbus.h b/esphome/components/modbus/modbus.h index e766ff04cc..3ddaafb9fc 100644 --- a/esphome/components/modbus/modbus.h +++ b/esphome/components/modbus/modbus.h @@ -16,26 +16,21 @@ namespace esphome::modbus { -// Tx queue backstop. Duplicate frames dedup into one entry, so reads can never approach this in a -// sane config - it exists to stop a runaway generator of distinct frames (e.g. a loop writing a -// changing value) from growing the heap unboundedly. The deque grows on demand; this reserves nothing. -// Worst case the cap permits: 128 distinct max-size frames = ~32 kB of spilled frame data plus -// ~3 kB of deque node storage (typical 8-byte frames stay inline; large PDUs spill to one -// allocation each) - pathological configs only, but the numbers matter when tuning for ESP8266. +// Tx queue backstop: duplicates dedup into one entry, so only a runaway generator of distinct frames +// (e.g. a loop writing a changing value) could grow the heap unboundedly. static constexpr uint16_t MODBUS_TX_BUFFER_SIZE = 128; static constexpr uint16_t MODBUS_TX_MAX_DELAY_MS = 5; // Typical frames -- reads and single-register/coil writes -- are exactly 8 bytes -// (address + 5-byte PDU + 2-byte CRC) and fit inline with no heap allocation. +// (address + 5-byte PDU + 2-byte CRC). static constexpr uint16_t MODBUS_FRAME_INLINE_SIZE = 8; struct ModbusFrame { - // Frame held in a small-buffer-optimized buffer. Typical frames fit inline; only larger - // multi-register or custom frames spill to a single heap allocation. This keeps the common, - // high-frequency tx traffic off the heap entirely, avoiding per-frame alloc/free churn. - // The buffer tracks its own length, so no separate size field is needed. - SmallInlineBuffer data; // Modbus RTU max is 256 bytes + // Small-buffer-optimized: typical frames fit inline, keeping high-frequency tx traffic off the + // heap; only large multi-register or custom frames spill to a single heap allocation. + SmallInlineBuffer data; + // A frame is [address][PDU...][CRC lo][CRC hi]. These are the only places that need to know that layout ModbusFrame(uint8_t address, const uint8_t *pdu, uint16_t pdu_len) { uint8_t *buf = this->data.init(pdu_len + 3); buf[0] = address; @@ -46,12 +41,9 @@ struct ModbusFrame { } uint16_t size() const { return static_cast(this->data.size()); } - - // A frame is [address][PDU...][CRC lo][CRC hi]. These are the only places that need to know that layout uint8_t address() const { return this->data.data()[0]; } - /// The PDU: function code + data, without address or CRC. Only valid while the frame is alive. - /// Requires a complete frame (size() >= MIN_FRAME_SIZE, guaranteed by the constructors) - the - /// subtraction would wrap on anything shorter. + /// A PDU is [function code][data...] without address or CRC. Only valid while the frame is alive. + /// Requires a complete frame (size() >= MIN_FRAME_SIZE, guaranteed by the constructors) std::span pdu() const { return std::span(this->data.data() + 1, this->size() - 3u); } }; @@ -73,15 +65,9 @@ class Modbus : public uart::UARTDevice, public Component { virtual int32_t tx_delay_remaining(); virtual void parse_modbus_frames() = 0; bool parse_modbus_server_frame_(); - // pdu is the whole PDU (function code + payload, no address/CRC); pdu[0] is the (standard or custom) function code. virtual void process_modbus_server_frame(uint8_t address, std::span pdu) = 0; void clear_rx_buffer_(const LogString *reason, bool warn = false, size_t bytes_to_clear = 0); - // Transmit a frame. Callers gate on tx_blocked() first, but the pre-send delay can span several ms, - // so this re-checks after the delay and returns false without transmitting if a byte arrived in that - // window (the caller then leaves its entry to retry). Returns true once the frame has been transmitted. bool send_frame_(const ModbusFrame &frame); - // Scans forward from min_length to find a frame boundary by CRC match for custom function codes. - // Returns the matched frame length, or 0 if no valid CRC was found within MAX_FRAME_SIZE. uint16_t find_frame_end_by_crc_(uint16_t min_length) const; uint32_t last_modbus_byte_{0}; @@ -99,8 +85,7 @@ class Modbus : public uart::UARTDevice, public Component { class ModbusClientDevice; class ModbusServerDevice; -// Transmit ordering, highest first: writes before one-shot reads before continuous polls. Derived -// at selection time, never caller-chosen or stored. +// Transmit ordering, highest first: writes before one-shot reads before continuous polls. enum class CommandPriority : uint8_t { CONTINUOUS = 0, READ, WRITE }; // Per-entry lifecycle state. Waiting states (see waiting_state()) hold the bus; the sweep delivers owed @@ -112,20 +97,15 @@ enum class FrameState : uint8_t { RECEIVED_EXCEPTION, TIMED_OUT, // on_no_response delivered at the send-wait timeout; awaiting reschedule/erase INTERRUPTED, // unexpected frame arrived; ignores this transaction, waits out the timeout - WAITING_RETIRED, // cleared while WAITING: a late response is still delivered as its usual terminal - INTERRUPTED_RETIRED, // cleared while INTERRUPTED: still distrusts late frames, ends in on_no_response - RETIRED, // cleared, off the wire + WAITING_RETIRED, // retired while WAITING: a late response is still delivered as its usual terminal + INTERRUPTED_RETIRED, // retired while INTERRUPTED: still distrusts late frames, ends in on_no_response + RETIRED, // retired, off the wire }; // Per-command send options. Append-only; pass via designated initializers ({.continuous = true}). -// The queue entry stores this struct whole, so a new field arrives at the queue with no plumbing - -// but it arrives inert. Every new field must define three rules before it does anything: -// 1. normalization in queue_pdu() (is it valid for this function code? e.g. continuous is -// stripped for mutating codes), -// 2. a merge rule for when a duplicate send absorbs into a live entry (continuous -// upgrades/downgrades via make_continuous(); a new field needs its own answer), -// 3. teardown: retire() resets the whole struct; silent_retire() leaves it, relying on the sweep -// to erase the entry. +// A new field reaches the queue with no plumbing but arrives inert until it defines three rules: +// normalization in queue_pdu(), a merge rule for duplicate absorption, and teardown in +// retire()/silent_retire(). struct CommandOptions { // A continuous poll lives in the queue until cancelled or failed; ignored for mutating codes. bool continuous{false}; @@ -135,17 +115,13 @@ struct ModbusDeviceCommand { ModbusClientDevice *device; ModbusFrame frame; // Place-in-line stamp (hub's free-running counter); selection takes the oldest for round-robin - // fairness within a class. Meant to wrap. Declared ahead of the byte fields so the tail packs - // densely and a growing CommandOptions eats trailing padding before enlarging the struct. + // fairness within a class. Meant to wrap. uint16_t seq{0}; FrameState state{FrameState::READY}; // Accepted requests this entry stands for, capped at max_pending(); drains one terminal each. // A continuous poll is a subscription: pending fixed at 1, removed only by cancellation or failure. uint8_t pending{1}; - // The entry's LIVE effective options, not a record of the caller's request: queue_pdu() normalizes - // before storing, duplicate absorption mutates continuous via make_continuous(), and retire() resets - // the struct (silent_retire() leaves it, relying on the sweep to erase the entry). See the - // CommandOptions comment for the rules a new field must define. + // The entry's LIVE effective options, not a record of the caller's request CommandOptions options; // Build a command from a PDU span (caller bounds it to MAX_PDU_SIZE) and pre-normalized options; @@ -154,28 +130,22 @@ struct ModbusDeviceCommand { CommandOptions options = {}, uint16_t seq = 0) : device(device), frame(address, pdu.data(), static_cast(pdu.size())), seq(seq), options(options) {} - // Transmit ordering class, derived (never stored): a continuous poll ranks below every one-shot. CommandPriority priority() const { - return this->options.continuous ? CommandPriority::CONTINUOUS : classify(this->frame.pdu()[0]); - } - // Wire-derived class: mutating codes rank WRITE; exception-flagged codes are excluded. - static CommandPriority classify(uint8_t function_code) { - if (helpers::is_function_code_exception(function_code)) - return CommandPriority::READ; - if (helpers::is_function_code_write(function_code)) { + if (this->options.continuous) + return CommandPriority::CONTINUOUS; + if (helpers::is_function_code_write(this->frame.pdu()[0])) { return CommandPriority::WRITE; } return CommandPriority::READ; } - // Requests this entry can serve: a standard read twice (run plus one re-run), everything else once. + // Requests this entry can serve uint8_t max_pending() const { const uint8_t fc = this->frame.pdu()[0]; - const bool requeueable = !helpers::is_function_code_exception(fc) && helpers::is_function_code_read_only(fc); - return (requeueable && !this->options.continuous) ? 2 : 1; + return (helpers::is_function_code_read_only(fc) && !this->options.continuous) ? 2 : 1; } - // Device-scoped clear: detach with no callback (device-less, pending 0). An entry still waiting for - // a response keeps its state as a reply-ignoring shell that resolves silently; any other goes RETIRED. + // Device-scoped clear: detach with no callback. An entry still waiting for a response keeps its state as a + // reply-ignoring shell that resolves silently; any other goes RETIRED. void silent_retire() { if (!this->waiting_state()) this->state = FrameState::RETIRED; @@ -183,28 +153,18 @@ struct ModbusDeviceCommand { this->device = nullptr; } // Fire-and-forget completion for a broadcast (address 0): the frame was transmitted (on_sent already - // fired), but a broadcast is never answered (Modbus 4.1), so the entry retires with NO terminal - // callback and the sweep erases it. Unlike response()/error()/timed_out(), it delivers nothing. - // A broadcast only carries a write or a custom code (reads are refused at queue_pdu()), and every such - // code caps pending at 1, so pending is always 1 here - clear it. + // fired), but a broadcast is never answered (Modbus 4.1), so the entry retires with no terminal callback. void complete_broadcast() { this->state = FrameState::RETIRED; this->pending = 0; } - // Re-ready for another transmission, restamped to the tail of its class (hub passes next_seq_++). + // Re-ready for another transmission, restamped to the tail of its class void requeue(uint16_t seq) { this->state = FrameState::READY; this->seq = seq; } // Re-task a frame that lives on: upgrade a one-shot to a continuous poll, or downgrade a poll back to - // a one-shot. Either way the entry keeps running and owes a request, so this is not a plain setter - - // to tear an entry down instead, use retire()/silent_retire(), which leave pending as the count owed. - // On: the entry becomes a continuous poll, superseding any absorbed requests (pending resets to the - // single subscription). Off: a one-shot duplicate has cancelled the poll, but the entry must still run - // once to serve that request - so restore one first. While the flag is still set max_pending() is 1, - // so the restore lifts a terminated poll (pending 0, after an error/timeout) back to 1 and is a no-op - // on a live poll already at 1; the flag drops afterwards, when a read's cap can widen to 2 without - // retroactively inflating that no-op. + // a one-shot. void make_continuous(bool continuous) { if (continuous) { this->options.continuous = true; @@ -214,13 +174,9 @@ struct ModbusDeviceCommand { this->options.continuous = false; } } - // Address-scoped clear: keep pending and device so the sweep delivers one on_not_sent() per un-run + // Address-scoped clear: keep pending and device so the sweep delivers one on_not_sent() per un-delivered // request. An entry still waiting for a response keeps its in-flight request (whose usual terminal is - // still coming) and drains only its duplicates: WAITING -> WAITING_RETIRED, and INTERRUPTED -> - // INTERRUPTED_RETIRED which keeps distrusting late frames (they were already interrupted). Any other - // state -> RETIRED, draining everything. A cleared frame that then times out still honors a retry: - // the clear is address-scoped (any device may call it) while the retry is the owning device's call - // via on_no_response - the bus obeys the owner. + // still coming) and drains only its duplicates. void retire() { if (this->state == FrameState::WAITING) { this->state = FrameState::WAITING_RETIRED; @@ -229,10 +185,10 @@ struct ModbusDeviceCommand { } else if (!this->waiting_state()) { // an already-retired shell stays put; off the wire -> RETIRED this->state = FrameState::RETIRED; } - this->options = {}; // reset every option so a future field is torn down without editing here + this->options = {}; // reset every option } - // True while the entry is still waiting for a response; the erase pass exempts these even at pending 0. + // True while the entry is still waiting for a response bool waiting_state() const { return this->state == FrameState::WAITING || this->state == FrameState::INTERRUPTED || this->state == FrameState::WAITING_RETIRED || this->state == FrameState::INTERRUPTED_RETIRED; @@ -245,7 +201,7 @@ struct ModbusDeviceCommand { } return false; } - // Add one request, honouring the cap; false = already at cap (absorb a duplicate, restore a retry). + bool increment_pending() { if (this->pending < this->max_pending()) { this->pending++; @@ -255,7 +211,7 @@ struct ModbusDeviceCommand { } // Terminal/lifecycle methods: each owns its transition, callback, and pending accounting and - // returns whether a callback ran. Out-of-line: ModbusClientDevice is incomplete here. + // returns whether a callback ran. bool sent(); bool response(std::span response_pdu); bool error(ExceptionCode exception_code); @@ -264,9 +220,6 @@ struct ModbusDeviceCommand { bool notify_retired(); /// True if this command carries the same wire frame (address + PDU) as the given one. - /// Cancellation matches the exact frame, not the action instance: a continuous poll whose - /// start_address (or other field) is templated produces one poll per distinct frame, and a later - /// cancel built from different argument values will not reach the polls it does not byte-match. bool same_frame(uint8_t address, std::span pdu) const { const auto own_pdu = this->frame.pdu(); return own_pdu.size() == pdu.size() && this->frame.address() == address && @@ -291,17 +244,13 @@ class ModbusClientHub : public Modbus { payload_len), device); }; - /// Queue a request. The name says queue, not send: the frame is appended to the transmit queue and - /// goes out later from loop(), so a true return means accepted into the machine (it will resolve in - /// exactly one terminal callback - except a broadcast (address 0), which is never answered and so gets - /// only on_sent()), NOT that anything reached the wire - that is on_sent(). False means - /// it never entered the machine at all (empty or oversize PDU, full queue, anonymous or over-cap - /// duplicate) and no callback of any kind will follow; the false return is the whole story. + /// Queue a request. True = accepted: it resolves in exactly one terminal callback (a broadcast, + /// address 0, gets only on_sent()). False = refused, and no callback of any kind follows. + /// Neither means anything reached the wire - on_sent() reports that. bool queue_pdu(uint8_t address, std::span pdu, ModbusClientDevice *device = nullptr, CommandOptions options = {}); - // Remove before 2027.2.0. Deliberately the signature 2026.7.4 shipped - void, and no CommandOptions: - // the bool return and the options argument arrived after that release, so nothing external can be - // relying on them under this name. Callers who want the queued/refused answer move to queue_pdu(). + // Remove before 2027.2.0. Deliberately the void, no-options signature 2026.7.4 shipped: nothing + // external can rely on the later additions under this name. ESPDEPRECATED("Use queue_pdu() instead - the call queues a request, it does not send one, and it " "reports whether the request was accepted. Removed in 2027.2.0", "2026.8.0") @@ -310,9 +259,10 @@ class ModbusClientHub : public Modbus { } ESPDEPRECATED("Use queue_pdu(payload[0], , device) instead. Removed in 2027.2.0", "2026.8.0") void send_raw(const std::vector &payload, ModbusClientDevice *device = nullptr); - // Clear an address's commands; each un-run request resolves via on_not_sent(), but a frame on the - // wire still runs to its usual terminal. clear_tx_queue_for_device() instead discards silently. + // Clear all commands matching the given address; each unsent request resolves via on_not_sent(), but a + // frame on the wire still runs to its usual terminal. void clear_tx_queue_for_address(uint8_t address); + // Clear all commands for a given device; no callbacks are delivered. void clear_tx_queue_for_device(ModbusClientDevice *device); protected: @@ -322,8 +272,7 @@ class ModbusClientHub : public Modbus { void send_next_frame_(); // Deliver owed callbacks from a quiescent hub and apply lifecycle bookkeeping; see FrameState. void sweep_(); - // The selection function: best READY entry (WRITE class first, then one-shot reads, then the - // least-recently-served continuous; FIFO by seq within each group), or nullptr. + // The selection function: best READY entry (ordered by priority; FIFO by seq within each group), or nullptr. ModbusDeviceCommand *select_next_ready_(); // Locate the single entry waiting for a response (WAITING/INTERRUPTED/WAITING_RETIRED/INTERRUPTED_RETIRED). ModbusDeviceCommand *find_waiting_(); @@ -349,13 +298,10 @@ class ModbusClientHub : public Modbus { // Transaction status: std::nullopt on success, otherwise a Modbus exception code using ResponseStatus = std::optional; -/// True when a transaction carried no exception. The optional holds the exception, so has_value() means -/// the request FAILED - the inverse of how "status" usually reads. Prefer this at the call site; the -/// bare !status.has_value() has already been mistaken for a failure check more than once. Where the code -/// is going to unwrap the exception anyway, status.has_value() followed by status.value() stays clearer. +/// True when a transaction carried no exception. inline bool succeeded(ResponseStatus status) { return !status.has_value(); } -// Register values exchanged with server handlers, in host byte order. Sized at the larger of the two protocol +// Register values exchanged with server handlers, in address order. Sized at the larger of the two protocol // maxima (read = 125 / 0x7D, write = 123 / 0x7B); the per-direction count limit is enforced by the hub, not by // the capacity of this type. using RegisterValues = StaticVector; @@ -373,59 +319,46 @@ class ModbusServerHub : public Modbus { void process_modbus_client_frame_(uint8_t address, uint8_t function_code, std::span data); // Dispatches a broadcast (address 0) write to every registered device; broadcasts are never answered. void process_broadcast_frame_(uint8_t function_code, std::span data); - // Parses a WRITE_SINGLE_REGISTER / WRITE_MULTIPLE_REGISTERS PDU into start_address and the host-order register - // values, validating the register count and address range. Returns std::nullopt on success, otherwise the Modbus - // exception code describing the failure. Shared by unicast writes (which reply with the exception) and broadcast - // writes (which silently drop invalid frames). + // Parses a WRITE_SINGLE_REGISTER / WRITE_MULTIPLE_REGISTERS PDU into start_address and the address order register + // values, validating the register count and address range. Shared by unicast and broadcast writes. ResponseStatus parse_write_single_(std::span data, uint16_t &start_address, RegisterValues ®isters); ResponseStatus parse_write_multiple_(std::span data, uint16_t &start_address, RegisterValues ®isters); - // Appends the big-endian register values in values to registers, in host byte order. + // Assembles host-order registers from the big-endian bytes in values and appends them to registers. void assemble_registers_(std::span values, RegisterValues ®isters); ModbusServerDevice *find_device_(uint8_t address); - // Returns std::nullopt if [start_address, start_address + count) fits in the 16-bit address space, - // otherwise ILLEGAL_DATA_ADDRESS. The caller sends the exception reply if one is required - a broadcast - // write is never answered, so the check cannot send it itself. Shared by the register and - // coil/discrete-input handlers, which all address the same 16-bit space. + // Returns std::nullopt if [start_address, start_address + count) fits in a 16-bit address space, otherwise + // ILLEGAL_DATA_ADDRESS. The caller sends the exception reply if one is required. Shared by the + // register/coil/discrete-input handlers, which all use a 16-bit address space. ResponseStatus check_address_range_(uint16_t start_address, uint16_t count); - // Parses a read request PDU (start address(2) + quantity(2)), shared by the register and - // coil/discrete-input reads so the two cannot drift apart. max_entities is the protocol ceiling for the - // function code; entity_name only labels the rejection log. + // Parses read request data. max_entities is the protocol ceiling for the function code; entity_name labels + // the rejection log. ResponseStatus parse_read_request_(std::span data, uint16_t max_entities, const LogString *entity_name, uint16_t &start_address, uint16_t &count); - // Parses a single-coil write PDU (FC 0x05), which carries a 2-byte on/off value rather than packed - // bytes. The caller packs value into a byte it owns to build the PackedBits view the handlers take. + // Parses single-coil write data ResponseStatus parse_write_single_coil_(std::span data, uint16_t &start_address, bool &value); - // Parses a multiple-coil write PDU (FC 0x0F) into a packed-bit view pointing straight into the receive - // buffer, so the coil values are never copied. Both coil parsers are shared by the addressed and - // broadcast paths so the two validate identically. + // Parses write-multiple-coil data into a packed-bit view pointing straight into the receive buffer, so the + // coil values are never copied. ResponseStatus parse_write_multiple_coils_(std::span data, uint16_t &start_address, uint16_t &count, std::span &packed_bytes); - // Builds the body of a register read response (byte count followed by the big-endian register values) into - // response_buffer. Shared by every function code that answers with register values, so the read reply stays - // identical across them. Returns false once an exception has been sent: the one the handler reported via - // status, or SERVICE_DEVICE_FAILURE if it returned the wrong number of registers, the count exceeds the - // protocol read limit, or the body does not fit. + // Builds the body of a register read response into response_buffer. Returns false once an exception has + // been sent: the one the handler reported via status, or SERVICE_DEVICE_FAILURE if it returned the wrong + // number of registers, the count exceeds the protocol read limit, or the body does not fit. bool build_or_reject_read_response_(uint8_t address, uint8_t function_code, ResponseStatus status, uint16_t number_of_registers, const RegisterValues ®isters, std::span response_buffer, uint16_t &response_len); void send_raw_(const uint8_t *payload, uint16_t len); // Sends and logs the exception reply when status holds one; returns true if the request was rejected. - // Every parse and handler rejection funnels through here, so the reply and its log cannot drift apart. bool rejected_(uint8_t address, uint8_t function_code, ResponseStatus status); void send_exception_(uint8_t address, uint8_t function_code, ExceptionCode exception_code); void send_response_(uint8_t address, uint8_t function_code, const uint8_t *payload, uint16_t payload_len); uint8_t expecting_peer_response_{0}; std::vector devices_; - // Stamp of the last "broadcast reached no device" warning, 0 until the first one is logged. Rate limiting - // on time rather than on address keeps the log bounded no matter how many addresses a shared bus carries. - uint32_t last_unaccepted_broadcast_warn_{0}; - // Holds the raw payload of a single reply deferred for sending when tx was blocked at send time. // Only one server reply can be waiting at once, so a single fixed buffer avoids heap allocation. std::array deferred_payload_; @@ -555,10 +488,7 @@ class ModbusClientDevice { helpers::create_client_pdu((FunctionCode) function, start_address, number_of_entities, payload, payload_len), this); } - /// See ModbusClientHub::queue_pdu(): true = accepted into the queue and a terminal callback will - /// follow (except a broadcast (address 0), which is never answered and so gets only on_sent()), - /// false = refused at the door and nothing further happens. Neither means the frame is on the wire; - /// on_sent() reports that. + /// See ModbusClientHub::queue_pdu() for the return contract. bool queue_pdu(std::span pdu, CommandOptions options = {}) { return this->parent_->queue_pdu(this->address_, pdu, this, options); } @@ -573,11 +503,8 @@ class ModbusClientDevice { return; // too short to contain a PDU; refused at the door like any invalid send this->parent_->queue_pdu(payload[0], std::span(payload).subspan(1), this); } - // The typed request builders below all queue through queue_pdu(), so they share its contract: true - // means the request is queued and will resolve in exactly one terminal callback (except a broadcast - // (address 0), which is never answered and so gets only on_sent()), false means it was refused outright - // with no callback. Neither says the frame has been transmitted - on_sent() does. - // Reads via the table-appropriate function code; an unreadable entity type maps to INVALID, which + // The typed request builders below all queue through queue_pdu() and share its return contract. + // Reads use the table-appropriate function code; an unreadable entity type maps to INVALID, which // create_read_pdu() rejects into an empty PDU and queue_pdu() refuses with a false return. bool read_entities(EntityType entity_type, uint16_t start_address, uint16_t number_of_entities, CommandOptions options = {}) { @@ -619,11 +546,9 @@ class ModbusClientDevice { bool write_multiple_coils(uint16_t start_address, PackedBits bits) { return this->queue_pdu(helpers::create_write_coils_pdu(start_address, bits)); } - /// FC 0x17: the read-back is delivered through on_read_holding_registers() (the response carries only the - /// read registers, the same wire shape as a holding-register read). A device exception - typically a - /// rejected write half - arrives at that same on_read_holding_registers() with the error in its status, - /// exactly as success does, so a subclass overriding that one callback handles both outcomes and never - /// needs to also override on_error(). + /// FC 0x17: the read-back is delivered through on_read_holding_registers(), and a device exception + /// (typically a rejected write half) arrives there too via its status - one callback handles both + /// outcomes with no on_error() override needed. bool read_write_multiple_registers(uint16_t read_start_address, uint16_t read_count, uint16_t write_start_address, std::span write_values) { return this->queue_pdu(helpers::create_read_write_multiple_registers_pdu(read_start_address, read_count, @@ -644,12 +569,9 @@ class ModbusClientDevice { bool custom_response_warned_{false}; // first unhandled custom response warns; repeats log at VERBOSE }; -// Compatibility shim for external components written against the pre-2026.8 API, which subclassed -// ModbusDevice and overrode on_modbus_data()/on_modbus_error(). The name is free (nothing in-tree -// uses it), so instead of a plain alias it adapts the new span-based hooks back to the old -// signatures: on_modbus_data() receives the response payload as an owning vector (the heap copy -// exists only on this deprecated path) and on_modbus_error() the function code and exception code. -// Remove before 2027.2.0 (window restarted when the plain alias became a behavior shim in 2026.8.0) +// Compatibility shim adapting the span-based hooks back to the pre-2026.8 on_modbus_data()/ +// on_modbus_error() signatures (the owning-vector heap copy exists only on this deprecated path). +// Remove before 2027.2.0 (window restarted when the plain alias became a behavior shim in 2026.8.0). class ESPDEPRECATED("Subclass ModbusClientDevice and override on_response()/on_error() instead. Removed in 2027.2.0", "2026.8.0") ModbusDevice : public ModbusClientDevice { public: diff --git a/esphome/components/modbus/modbus_definitions.h b/esphome/components/modbus/modbus_definitions.h index 83f314352f..089fc3d1ae 100644 --- a/esphome/components/modbus/modbus_definitions.h +++ b/esphome/components/modbus/modbus_definitions.h @@ -47,7 +47,6 @@ enum class FunctionCode : uint8_t { using ModbusFunctionCode ESPDEPRECATED("Use modbus::FunctionCode instead. Removed in 2027.2.0", "2026.8.0") = FunctionCode; -/*Allow direct comparison operators between FunctionCode and uint8_t*/ inline bool operator==(FunctionCode lhs, uint8_t rhs) { return static_cast(lhs) == rhs; } inline bool operator==(uint8_t lhs, FunctionCode rhs) { return lhs == static_cast(rhs); } inline bool operator!=(FunctionCode lhs, uint8_t rhs) { return !(static_cast(lhs) == rhs); } diff --git a/esphome/components/modbus/modbus_helpers.cpp b/esphome/components/modbus/modbus_helpers.cpp index db21b6e6fd..836d9b2d38 100644 --- a/esphome/components/modbus/modbus_helpers.cpp +++ b/esphome/components/modbus/modbus_helpers.cpp @@ -30,9 +30,11 @@ uint16_t server_pdu_length(const uint8_t *frame, size_t size) { switch (static_cast(frame[0])) { case FunctionCode::READ_COILS: case FunctionCode::READ_DISCRETE_INPUTS: + // function(1) + byte count(1) + packed coil bytes + return 2 + (size > 1 ? std::min(frame[1], uint8_t(packed_bit_bytes(MAX_NUM_OF_COILS_TO_READ))) : 0); case FunctionCode::READ_HOLDING_REGISTERS: case FunctionCode::READ_INPUT_REGISTERS: - // function(1) + byte count(1) + data + // function(1) + byte count(1) + register data return 2 + (size > 1 ? std::min(frame[1], uint8_t(MAX_NUM_OF_REGISTERS_TO_READ * 2)) : 0); case FunctionCode::WRITE_SINGLE_COIL: case FunctionCode::WRITE_SINGLE_REGISTER: @@ -60,6 +62,9 @@ uint16_t server_pdu_length(const uint8_t *frame, size_t size) { uint16_t client_pdu_length(const uint8_t *frame, size_t size) { if (size < MIN_PDU_SIZE) return MIN_PDU_SIZE; + if (is_function_code_exception(frame[0])) { + return 2; // never a valid request; sized like the exception reply so the CRC fails at once + } switch (static_cast(frame[0])) { case FunctionCode::READ_COILS: case FunctionCode::READ_DISCRETE_INPUTS: @@ -381,8 +386,6 @@ ReadPdu create_read_pdu(FunctionCode function_code, uint16_t start_address, uint PduBuffer create_client_pdu(FunctionCode function_code, uint16_t start_address, uint16_t number_of_entities, const uint8_t *values, size_t values_len) { PduBuffer pdu; // declared before every return so NRVO fires (all paths return the same object) - // Generic entry point; prefer the direction- and type-specific builders (create_read_pdu(), - // create_write_registers_pdu(), etc.) which bound their inputs per spec. if (is_function_code_read_only(static_cast(function_code))) { if (values != nullptr || values_len > 0) { ESP_LOGW(TAG, "Values provided for read function code %02X, but will be ignored", @@ -445,9 +448,7 @@ PduBuffer create_client_pdu(FunctionCode function_code, uint16_t start_address, return pdu; } // The quantity is spec-bounded above, so the data length just has to agree with it exactly - // (registers are 2 bytes each, coils pack 8 per byte). This is the same consistency the response - // dispatch enforces via is_client_pdu_standard(), so a frame built here can never be classified - // non-standard on reply, and the spec bound keeps the PDU within capacity by construction. + // (registers are 2 bytes each, coils pack 8 per byte). // Checked before the header append: a failed check must return an empty PDU, not a 5-byte partial one. const bool bits = function_code == FunctionCode::WRITE_MULTIPLE_COILS; const size_t expected_len = bits ? packed_bit_bytes(number_of_entities) : static_cast(number_of_entities) * 2; diff --git a/esphome/components/modbus/modbus_helpers.h b/esphome/components/modbus/modbus_helpers.h index 76056ed3e8..c3bccc4cba 100644 --- a/esphome/components/modbus/modbus_helpers.h +++ b/esphome/components/modbus/modbus_helpers.h @@ -60,14 +60,11 @@ inline bool is_function_code_custom(uint8_t function_code) { /// in step with those switches). Deliberately wider than is_function_code_custom(): the user-defined /// ranges are unknown to the parser too, but so are the assigned-but-unimplemented codes /// (READ_EXCEPTION_STATUS, DIAGNOSTICS, GET_COMM_EVENT_*, REPORT_SERVER_ID) and every unassigned value. -/// The 0x80 exception flag is masked off first, so a frame with it set classifies by its base code - -/// even though a spec exception reply has a known 2-byte PDU. That is deliberate, matching what -/// is_function_code_custom() has always done: some vendors use codes with the 0x80 bit set as ordinary -/// codes with longer payloads, so the response parser CRC-scans these rather than assuming the spec -/// length. For an intact spec exception the scan matches at its first candidate, so only a corrupt one -/// pays (recovery by timeout instead of an immediate CRC failure). +/// Exception-flagged codes (0x80 set) are always the 2-byte spec exception shape, so never unknown. inline bool is_function_code_unknown_length(uint8_t function_code) { - switch (static_cast(function_code & FUNCTION_CODE_MASK)) { + if (is_function_code_exception(function_code)) + return false; + switch (static_cast(function_code)) { case FunctionCode::READ_COILS: case FunctionCode::READ_DISCRETE_INPUTS: case FunctionCode::READ_HOLDING_REGISTERS: @@ -87,6 +84,17 @@ inline bool is_function_code_unknown_length(uint8_t function_code) { } } +/// True when the underlying function code (exception bit masked off) may be broadcast (address 0). +/// Refused: the reads (including read-write), plus every other code whose response length the parser +/// knows (file record, FIFO). Allowed: the writes, and any code the parser does not know, since the +/// hub cannot tell one of those apart from a vendor write. +inline bool is_function_code_broadcastable(uint8_t function_code) { + uint8_t masked_function_code = function_code & FUNCTION_CODE_MASK; + if (is_function_code_read(masked_function_code)) + return false; + return is_function_code_write(masked_function_code) || is_function_code_unknown_length(masked_function_code); +} + // Returns the expected length of a server response PDU based on the function code. // If too few bytes have arrived to determine the length, returns the minimum length. `size` is the // number of bytes available so far, which may exceed the eventual PDU (e.g. include the frame's CRC @@ -205,7 +213,7 @@ enum class SensorValueType : uint8_t { S_DWORD = 0x4, // 2 Registers signed BIT = 0x5, U_DWORD_R = 0x6, // 2 Registers unsigned - S_DWORD_R = 0x7, // 2 Registers unsigned + S_DWORD_R = 0x7, // 2 Registers signed U_QWORD = 0x8, S_QWORD = 0x9, U_QWORD_R = 0xA, @@ -280,7 +288,7 @@ inline uint8_t c_to_hex(char c) { return (c >= 'A') ? (c >= 'a') ? (c - 'a' + 10 * byte_from_hex_str("1122", 1) returns uint_8 value 0x22 == 34 * byte_from_hex_str("1122", 0) returns 0x11 * @param value string containing hex encoding - * @param position offset in bytes. Because each byte is encoded in 2 hex digits the position of the original byte in + * @param pos offset in bytes. Because each byte is encoded in 2 hex digits the position of the original byte in * the hex string is byte_pos * 2 * @return byte value */ @@ -292,8 +300,7 @@ inline uint8_t byte_from_hex_str(const std::string &value, uint8_t pos) { /** Get a word from a hex string * @param value string containing hex encoding - * @param position offset in bytes. Because each byte is encoded in 2 hex digits the position of the original byte in - * the hex string is byte_pos * 2 + * @param pos offset in bytes (see byte_from_hex_str) * @return word value */ inline uint16_t word_from_hex_str(const std::string &value, uint8_t pos) { @@ -302,8 +309,7 @@ inline uint16_t word_from_hex_str(const std::string &value, uint8_t pos) { /** Get a dword from a hex string * @param value string containing hex encoding - * @param position offset in bytes. Because each byte is encoded in 2 hex digits the position of the original byte in - * the hex string is byte_pos * 2 + * @param pos offset in bytes (see byte_from_hex_str) * @return dword value */ inline uint32_t dword_from_hex_str(const std::string &value, uint8_t pos) { @@ -312,8 +318,7 @@ inline uint32_t dword_from_hex_str(const std::string &value, uint8_t pos) { /** Get a qword from a hex string * @param value string containing hex encoding - * @param position offset in bytes. Because each byte is encoded in 2 hex digits the position of the original byte in - * the hex string is byte_pos * 2 + * @param pos offset in bytes (see byte_from_hex_str) * @return qword value */ inline uint64_t qword_from_hex_str(const std::string &value, uint8_t pos) { @@ -328,9 +333,9 @@ template T get_data(const std::vector &data, size_t buffer_ * Responses for coil are packed into bytes . * coil 3 is bit 3 of the first response byte * coil 9 is bit 2 of the second response byte - * @param coil number of the cil + * @param bit index of the bit to extract * @param data modbus response buffer (uint8_t) - * @return content of coil register + * @return value of the requested bit */ inline bool bit_from_packed(int bit, std::span data) { auto data_byte = bit / 8; diff --git a/tests/components/modbus/modbus_client_hub_test.cpp b/tests/components/modbus/modbus_client_hub_test.cpp index 026df34bcc..18c04f32d5 100644 --- a/tests/components/modbus/modbus_client_hub_test.cpp +++ b/tests/components/modbus/modbus_client_hub_test.cpp @@ -775,7 +775,7 @@ TEST(ModbusClientHubBroadcast, DeliversNoTerminalToTypedDevice) { // A broadcast is only meaningful for a command that changes state; a broadcast READ could never be // answered, so the hub refuses it at the door (false return, no entry queued) rather than silently -// retiring it. Writes, 0x17, and custom codes still go through (covered above). +// retiring it. Writes and custom/unknown codes still go through (covered in the neighboring tests). TEST(ModbusClientHubBroadcast, RefusesReadBroadcast) { NullUART uart; NoResponseProbeHub hub; @@ -814,9 +814,8 @@ TEST(ModbusClientHubBroadcast, AcceptsCustomBroadcast) { EXPECT_EQ(hub.entries(), 0u); // the entry is gone } -// An exception-flagged custom code (0x80 bit set) is not a real request: is_function_code_custom() masks -// the bit away and would accept it, but the broadcast guard excludes it, matching classify()'s handling -// of an exception-flagged write. +// An exception-flagged code (0x80 bit set) is never a valid request - that bit is response-only - so +// queue_pdu refuses it up front, before the broadcast guard, whatever its base code. TEST(ModbusClientHubBroadcast, RefusesExceptionFlaggedCustomBroadcast) { NullUART uart; NoResponseProbeHub hub; @@ -833,6 +832,50 @@ TEST(ModbusClientHubBroadcast, RefusesExceptionFlaggedCustomBroadcast) { EXPECT_EQ(device.sent_count_, 0); // never transmitted } +// FC23 (read/write multiple) has a read half that expects a reply, so the Modbus spec does not allow it +// as a broadcast. is_function_code_read() covers it, so the broadcast guard refuses it despite its write +// half. +TEST(ModbusClientHubBroadcast, RefusesReadWriteMultipleBroadcast) { + NullUART uart; + NoResponseProbeHub hub; + hub.set_uart_parent(&uart); + hub.setup(); + BroadcastProbeDevice device(&hub, BROADCAST_ADDRESS); + + // fc, read start+qty, write start+qty, byte count, one data word. + const uint8_t read_write_multiple[] = {0x17, 0x00, 0x00, 0x00, 0x01, 0x00, 0x10, 0x00, 0x01, 0x02, 0xBE, 0xEF}; + EXPECT_FALSE(device.queue_pdu(read_write_multiple)); // its read half could never be answered + EXPECT_EQ(hub.entries(), 0u); +} + +// FC 0x18 (read FIFO queue) is not a "read" by is_function_code_read(), but the hub has an explicit +// response-length rule for it - it demonstrably expects a reply, so it cannot broadcast. +TEST(ModbusClientHubBroadcast, RefusesKnownLengthNonWriteBroadcast) { + NullUART uart; + NoResponseProbeHub hub; + hub.set_uart_parent(&uart); + hub.setup(); + BroadcastProbeDevice device(&hub, BROADCAST_ADDRESS); + + const uint8_t read_fifo[] = {0x18, 0x00, 0x10}; // fc, FIFO pointer address + EXPECT_FALSE(device.queue_pdu(read_fifo)); + EXPECT_EQ(hub.entries(), 0u); +} + +// A code that is neither a read nor exception-flagged (here 0x63, unassigned) is fire-and-forget on a +// broadcast: the hub can't know it isn't a vendor write, so it is accepted and delivered to all devices. +TEST(ModbusClientHubBroadcast, AcceptsNonReadUnknownBroadcast) { + NullUART uart; + NoResponseProbeHub hub; + hub.set_uart_parent(&uart); + hub.setup(); + BroadcastProbeDevice device(&hub, BROADCAST_ADDRESS); + + const uint8_t unknown[] = {0x63, 0x00, 0x01}; + EXPECT_TRUE(device.queue_pdu(unknown)); // not a read, so not refused + EXPECT_EQ(hub.entries(), 1u); +} + namespace { // tx_blocked() clear for send_next_frame_'s gate, then blocked for send_frame_'s post-delay re-check. class RejectPostDelayHub : public NoResponseProbeHub { @@ -1882,30 +1925,20 @@ TEST(ModbusClientHubPriority, ResendFromOnResponseAbsorbsIntoCompletingCommand) EXPECT_FALSE(hub.queued(0).options.continuous); // the one-shot re-send downgraded the poll } -// An exception-flagged function code is never silently re-sendable, even though the read check -// masks the exception bit: its duplicate takes the drop path like any other non-read. -TEST(ModbusClientHubPriority, ExceptionFlaggedDuplicateDroppedNotPromoted) { +// The exception bit marks a response, so a request carrying it is refused outright. +TEST(ModbusClientHubPriority, ExceptionFlaggedPduRefused) { NoResponseProbeHub hub; SentCountingDevice device(&hub, 0x02); - const uint8_t weird[] = {0x83, 0x01, 0x00, 0x00, 0x02}; // read-shaped but exception-flagged - EXPECT_TRUE(device.queue_pdu(weird)); - EXPECT_FALSE(device.queue_pdu(weird)); // non-requeueable: cap of one, so the duplicate is refused + // The 0x80 exception flag is a response-only bit; a request must never set it. queue_pdu refuses an + // exception-flagged PDU up front - nothing is queued - whether its base code reads (0x83 = 0x03 | 0x80) + // or writes (0x86 = 0x06 | 0x80). + const uint8_t read_shaped[] = {0x83, 0x01, 0x00, 0x00, 0x02}; + const uint8_t write_shaped[] = {0x86, 0x00, 0x10, 0xBE, 0xEF}; + EXPECT_FALSE(device.queue_pdu(read_shaped)); + EXPECT_FALSE(device.queue_pdu(write_shaped)); hub.sweep_for_test(); - - ASSERT_EQ(hub.queued_frames(), 1u); - EXPECT_EQ(hub.queued(0).pending, 1u); - EXPECT_EQ(device.not_sent_count_, 0); - - // The write-shaped twin (0x86 masks to WRITE_SINGLE_REGISTER) must not take WRITE-class - // ordering either: exception-flagged codes are excluded from the mutates classification. - const uint8_t weird_write[] = {0x86, 0x00, 0x10, 0xBE, 0xEF}; - device.queue_pdu(weird_write); - ASSERT_EQ(hub.queued_frames(), 2u); - EXPECT_EQ(hub.queued(1).priority(), CommandPriority::READ); // not WRITE - const ModbusDeviceCommand *next = hub.next_ready(); - ASSERT_NE(next, nullptr); - EXPECT_EQ(next->frame.pdu()[0], 0x83); // FIFO by age: it did not jump the older entry + EXPECT_EQ(hub.queued_frames(), 0u); } namespace { diff --git a/tests/components/modbus/modbus_helpers_test.cpp b/tests/components/modbus/modbus_helpers_test.cpp index 53f51b016b..28573dc8a6 100644 --- a/tests/components/modbus/modbus_helpers_test.cpp +++ b/tests/components/modbus/modbus_helpers_test.cpp @@ -63,6 +63,12 @@ TEST(ModbusClientFrameLength, TooShortReturnsMinimum) { EXPECT_EQ(client_frame_length(frame, 1), MIN_FRAME_SIZE); } +TEST(ModbusClientFrameLength, ExceptionFlaggedIsTheExceptionShape) { + // Sized at 2 so an exception-flagged request fails its CRC at once instead of being scanned for. + const uint8_t exception_request[] = {0x83, 0x02}; + EXPECT_EQ(client_pdu_length(exception_request, sizeof(exception_request)), 2); +} + TEST(ModbusClientFrameLength, ReadAndWriteSingleAreFixed) { // basic_register request fixture is a read-holding request -> 8 bytes const uint8_t read[] = {0x01, 0x03, 0x00, 0x03, 0x00, 0x01, 0x74, 0x0A}; diff --git a/tests/components/modbus/modbus_unknown_function_test.cpp b/tests/components/modbus/modbus_unknown_function_test.cpp index 8b91d088b8..33f0cd24df 100644 --- a/tests/components/modbus/modbus_unknown_function_test.cpp +++ b/tests/components/modbus/modbus_unknown_function_test.cpp @@ -54,7 +54,8 @@ class TestServerHub : public ModbusServerHub { // The frame-length parsers have explicit cases for exactly these 13 codes; every other value - the // assigned-but-unimplemented management codes, both user-defined ranges, and all unassigned codes - -// must classify as unknown length. The exception flag masks off first. +// must classify as unknown length. Exception replies are always the 2-byte spec shape, so every +// 0x80-set code is known length. TEST(ModbusUnknownFunction, HelperMatchesParserCoverage) { for (uint8_t fc : {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x0F, 0x10, 0x14, 0x15, 0x16, 0x17, 0x18}) { EXPECT_FALSE(helpers::is_function_code_unknown_length(fc)) << "fc 0x" << std::hex << int(fc); @@ -62,11 +63,13 @@ TEST(ModbusUnknownFunction, HelperMatchesParserCoverage) { for (uint8_t fc : {0x07, 0x08, 0x0B, 0x0C, 0x11, 0x2A, 0x41, 0x48, 0x49, 0x64, 0x6E, 0x00, 0x7F}) { EXPECT_TRUE(helpers::is_function_code_unknown_length(fc)) << "fc 0x" << std::hex << int(fc); } - // Exception replies classify by their base code. + // Every exception-flagged code is known length (the 2-byte spec exception shape), whatever its base. EXPECT_FALSE(helpers::is_function_code_unknown_length(0x83)); - EXPECT_TRUE(helpers::is_function_code_unknown_length(0x87)); - // Strictly wider than the user-defined ranges: every custom code is unknown-length, but not vice versa. - for (int fc = 0; fc <= 0xFF; fc++) { + EXPECT_FALSE(helpers::is_function_code_unknown_length(0x87)); + EXPECT_FALSE(helpers::is_function_code_unknown_length(0xC9)); + // Strictly wider than the user-defined ranges below 0x80: every non-exception custom code is + // unknown-length, but not vice versa. + for (int fc = 0; fc <= 0x7F; fc++) { if (helpers::is_function_code_custom(fc)) EXPECT_TRUE(helpers::is_function_code_unknown_length(fc)) << "fc 0x" << std::hex << fc; } @@ -75,10 +78,10 @@ TEST(ModbusUnknownFunction, HelperMatchesParserCoverage) { // Derived contract check: the helper must say "unknown" exactly when both length parsers fall // through to default. With a zero-filled max-size PDU every explicit case returns at least 2 // (file records bottom out at 2, FIFO at 3) and only default returns MIN_PDU_SIZE, so comparing - // against MIN_PDU_SIZE detects a case added to either switch without updating the helper. The - // loop stops at 0x7F: above it the helper masks the exception flag off while client_pdu_length() - // switches on the unmasked byte and server_pdu_length() early-returns the exception length. - for (int fc = 0; fc <= 0x7F; fc++) { + // against MIN_PDU_SIZE detects a case added to either switch without updating the helper. Both + // parsers early-return the 2-byte exception shape above 0x7F, which the helper's own exception + // early-return mirrors, so the whole byte range is covered. + for (int fc = 0; fc <= 0xFF; fc++) { const uint8_t pdu[MAX_PDU_SIZE] = {static_cast(fc)}; // zero header fields EXPECT_EQ(helpers::is_function_code_unknown_length(fc), helpers::client_pdu_length(pdu, sizeof(pdu)) == MIN_PDU_SIZE) @@ -89,6 +92,17 @@ TEST(ModbusUnknownFunction, HelperMatchesParserCoverage) { } } +// Broadcastable = writes plus unknown codes (possible vendor writes); everything known to expect a +// reply is not. Classifies the underlying code: the exception bit masks off first (0x85 as 0x05). +TEST(ModbusUnknownFunction, BroadcastableClassification) { + for (uint8_t fc : {0x05, 0x06, 0x0F, 0x10, 0x16, 0x49, 0x63, 0x6E, 0x85, 0xC9}) { + EXPECT_TRUE(helpers::is_function_code_broadcastable(fc)) << "fc 0x" << std::hex << int(fc); + } + for (uint8_t fc : {0x01, 0x02, 0x03, 0x04, 0x14, 0x15, 0x17, 0x18, 0x83, 0x97}) { + EXPECT_FALSE(helpers::is_function_code_broadcastable(fc)) << "fc 0x" << std::hex << int(fc); + } +} + // A response with a function code outside the user-defined ranges (0x49) has no length case in // server_pdu_length(), so the parser must find the frame end by CRC scan - the same way it already // handles user-defined codes. Frame: address + FC 0x49 + 3 data bytes + CRC = 7 bytes. Without the From 59397b4e28e2b7a7faec74969dc35bd1a0a0bc79 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 11:36:24 -0700 Subject: [PATCH 004/147] [modbus] Build single-value register writes on a right-sized stack buffer (#18844) Co-authored-by: J. Nick Koston --- esphome/components/modbus/modbus.h | 3 +++ .../components/modbus/modbus_definitions.h | 3 +++ esphome/components/modbus/modbus_helpers.cpp | 22 ++++++++++++++++--- esphome/components/modbus/modbus_helpers.h | 13 +++++++++++ esphome/core/helpers.h | 1 + .../components/modbus/modbus_helpers_test.cpp | 22 +++++++++++++++++++ 6 files changed, 61 insertions(+), 3 deletions(-) diff --git a/esphome/components/modbus/modbus.h b/esphome/components/modbus/modbus.h index 3ddaafb9fc..69a7eb82e3 100644 --- a/esphome/components/modbus/modbus.h +++ b/esphome/components/modbus/modbus.h @@ -534,6 +534,9 @@ class ModbusClientDevice { return this->queue_pdu(helpers::create_write_single_coil_pdu(address, value)); } bool write_multiple_registers(uint16_t start_address, std::span values) { + // Empty goes to the full-size builder so the rejection log names this method's limit, not the small one's. + if (!values.empty() && values.size() <= helpers::MAX_FEW_REGISTERS) + return this->queue_pdu(helpers::create_write_few_registers_pdu(start_address, values)); return this->queue_pdu(helpers::create_write_registers_pdu(start_address, values)); } /// Note: std::vector cannot bind to std::span; use a contiguous bool container or the packed diff --git a/esphome/components/modbus/modbus_definitions.h b/esphome/components/modbus/modbus_definitions.h index 089fc3d1ae..0939f9e76c 100644 --- a/esphome/components/modbus/modbus_definitions.h +++ b/esphome/components/modbus/modbus_definitions.h @@ -116,6 +116,9 @@ static constexpr uint16_t MAX_RAW_SIZE = 254; // Max RAW size is 256 - CRC(2) = static constexpr uint16_t READ_PDU_SIZE = 5; // A single-write PDU is always function code(1) + address(2) + value(2) static constexpr uint16_t WRITE_SINGLE_PDU_SIZE = 5; +// A multiple-write PDU starts with function code(1) + start address(2) + quantity(2) + byte count(1), +// followed by two bytes per register. +static constexpr uint16_t WRITE_MULTIPLE_HEADER_SIZE = 6; static constexpr uint16_t MAX_FRAME_SIZE = 256; // 4.1 Address 0 is the broadcast address: the request is processed by every device and never answered. diff --git a/esphome/components/modbus/modbus_helpers.cpp b/esphome/components/modbus/modbus_helpers.cpp index 836d9b2d38..92bd06cdf5 100644 --- a/esphome/components/modbus/modbus_helpers.cpp +++ b/esphome/components/modbus/modbus_helpers.cpp @@ -485,9 +485,12 @@ static bool register_block_in_range(const LogString *role, uint16_t start_addres return true; } -PduBuffer create_write_registers_pdu(uint16_t start_address, std::span values) { - PduBuffer pdu; // declared before every return so NRVO fires (all paths return the same object) - if (!register_block_in_range(LOG_STR("Write"), start_address, values.size(), MAX_NUM_OF_REGISTERS_TO_WRITE)) { +// The ceiling comes from the buffer itself: push_back() drops silently, so a bound wider than the buffer +// would put a truncated frame on the wire. +template static Pdu build_write_registers_pdu(uint16_t start_address, std::span values) { + constexpr auto max_registers = static_cast((Pdu::capacity() - WRITE_MULTIPLE_HEADER_SIZE) / 2); + Pdu pdu; // declared before every return so NRVO fires (all paths return the same object) + if (!register_block_in_range(LOG_STR("Write"), start_address, values.size(), max_registers)) { return pdu; } append_pdu_header(pdu, FunctionCode::WRITE_MULTIPLE_REGISTERS, start_address, values.size()); @@ -498,6 +501,19 @@ PduBuffer create_write_registers_pdu(uint16_t start_address, std::span values) { + return build_write_registers_pdu(start_address, values); +} + +WriteFewRegistersPdu create_write_few_registers_pdu(uint16_t start_address, std::span values) { + return build_write_registers_pdu(start_address, values); +} + PduBuffer create_read_write_multiple_registers_pdu(uint16_t read_start_address, uint16_t read_count, uint16_t write_start_address, std::span write_values) { diff --git a/esphome/components/modbus/modbus_helpers.h b/esphome/components/modbus/modbus_helpers.h index c3bccc4cba..a070ce250c 100644 --- a/esphome/components/modbus/modbus_helpers.h +++ b/esphome/components/modbus/modbus_helpers.h @@ -473,11 +473,15 @@ inline int64_t payload_to_number(const std::vector &data, SensorValueTy */ std::optional registers_to_number(const uint16_t *registers, size_t count, SensorValueType sensor_value_type); +/// The widest standard numeric value (a QWORD) spans 4 registers, so one entity value never writes more. +static constexpr uint16_t MAX_FEW_REGISTERS = 4; + // Named PDU buffer types: the builders' storage strategy (currently stack-allocated StaticVector, // right-sized per shape) can be swapped in one place without touching every signature. using PduBuffer = StaticVector; using ReadPdu = StaticVector; using WriteSinglePdu = StaticVector; +using WriteFewRegistersPdu = StaticVector; /// Scratch space for packing coils into wire layout: one bit per coil, sized for the spec maximum. using CoilPackBuffer = StaticVector; @@ -521,6 +525,15 @@ PduBuffer create_client_pdu(FunctionCode function_code, uint16_t start_address, */ PduBuffer create_write_registers_pdu(uint16_t start_address, std::span values); +/** Create modbus write multiple registers command (function 0x10) on a right-sized stack buffer. + * Identical wire bytes to create_write_registers_pdu() for any accepted input. + * @param start_address modbus address of the first register to write + * @param values register values to write, at most MAX_FEW_REGISTERS (an over-long or empty set is + * rejected and an empty PDU is returned) + * @return PDU (function code + data, no address, no CRC) + */ +WriteFewRegistersPdu create_write_few_registers_pdu(uint16_t start_address, std::span values); + /** Create modbus read/write multiple registers command * Function 0x17 Read/Write Multiple Registers * Writes write_values then reads read_count registers in one transaction (write first, per Modbus 6.17); diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 1ccc833048..a0afb03124 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -290,6 +290,7 @@ template class StaticVector { } size_t size() const { return count_; } + static constexpr size_t capacity() { return N; } bool empty() const { return count_ == 0; } // Direct access to underlying data diff --git a/tests/components/modbus/modbus_helpers_test.cpp b/tests/components/modbus/modbus_helpers_test.cpp index 28573dc8a6..87af49710f 100644 --- a/tests/components/modbus/modbus_helpers_test.cpp +++ b/tests/components/modbus/modbus_helpers_test.cpp @@ -489,6 +489,28 @@ TEST(ModbusTypedBuilders, WriteRegistersPduRejectsOverLimit) { EXPECT_FALSE(create_write_registers_pdu(0x0000, values).empty()); } +TEST(ModbusTypedBuilders, WriteFewRegistersPduMatchesFullSizeBuilder) { + static_assert(sizeof(WriteFewRegistersPdu) < sizeof(PduBuffer) / 4, + "WriteFewRegistersPdu must be meaningfully smaller"); + const uint16_t values[] = {0x000B, 0x0016, 0xABCD, 0xFF00}; + for (size_t count = 1; count <= MAX_FEW_REGISTERS; count++) { + auto small = create_write_few_registers_pdu(0x0102, std::span(values, count)); + auto full = create_write_registers_pdu(0x0102, std::span(values, count)); + EXPECT_EQ(std::vector(small.begin(), small.end()), std::vector(full.begin(), full.end())) + << count << " registers"; + EXPECT_EQ(small.size(), 6u + 2 * count); + EXPECT_TRUE(is_client_pdu_standard(small.data(), small.size())); + } +} + +TEST(ModbusTypedBuilders, WriteFewRegistersPduRejectsInvalidInput) { + const uint16_t values[MAX_FEW_REGISTERS + 1] = {0xAAAA, 0xAAAA, 0xAAAA, 0xAAAA, 0xAAAA}; + EXPECT_TRUE(create_write_few_registers_pdu(0x0000, values).empty()); + EXPECT_FALSE(create_write_few_registers_pdu(0x0000, std::span(values, MAX_FEW_REGISTERS)).empty()); + EXPECT_TRUE(create_write_few_registers_pdu(0x0000, std::span()).empty()); + EXPECT_TRUE(create_write_few_registers_pdu(0xFFFF, std::span(values, 2)).empty()); +} + TEST(ModbusTypedBuilders, ReadWriteMultipleRegistersPduWireBytes) { const uint16_t write_values[] = {0x000B, 0x0016}; // Read 2 registers at 0x0010, write 2 registers at 0x0020. From cc41fd0beb6b6651ca2f46185409a98e75e694a4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 28 Aug 2026 13:42:42 -0500 Subject: [PATCH 005/147] [core] Make rmtree tolerate missing paths and concurrent directory changes (#18846) --- esphome/helpers.py | 48 +++++++++++++++++---- tests/unit_tests/test_helpers.py | 74 +++++++++++++++++++++++++++++++- 2 files changed, 113 insertions(+), 9 deletions(-) diff --git a/esphome/helpers.py b/esphome/helpers.py index a38fcaf821..3ccf0fe65a 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Iterable, MutableMapping +from collections.abc import Callable, Iterable, MutableMapping from contextlib import suppress import ipaddress import logging @@ -456,23 +456,55 @@ def add_git_ceiling_directory(env: MutableMapping[str, str], directory: Path) -> env["GIT_CEILING_DIRECTORIES"] = os.pathsep.join(parts) -def rmtree(path: Path | str) -> None: - """Remove a directory tree, handling read-only files on Windows. +# Deletion attempts when a directory keeps being repopulated mid-delete +RMTREE_MAX_ATTEMPTS = 3 - On Windows, git pack files and other files may be marked read-only, - causing shutil.rmtree to fail. This handles that by removing the - read-only flag and retrying. + +def rmtree(path: Path | str) -> None: + """Remove a directory tree, tolerating common filesystem races. + + Read-only files (e.g. git pack files on Windows) get the read-only flag + removed and are retried. Paths that are already gone, whether the target + itself or entries vanishing mid-delete, are treated as removed. + Directories repopulated mid-delete (e.g. Finder recreating .DS_Store on + macOS) are retried a few times. """ + import errno import shutil + import time - def _onexc(func, path, exc): + def _onexc(func: Callable[..., object], path: str | Path, exc: OSError) -> None: + if isinstance(exc, FileNotFoundError): + _LOGGER.debug("rmtree: %s already gone", path) + return if os.access(path, os.W_OK): raise exc Path(path).chmod(stat.S_IWUSR | stat.S_IRUSR) func(path) - shutil.rmtree(path, onexc=_onexc) + last_err: OSError | None = None + for attempt in range(RMTREE_MAX_ATTEMPTS - 1): + try: + shutil.rmtree(path, onexc=_onexc) + return + except OSError as err: + if err.errno not in (errno.ENOTEMPTY, errno.EEXIST): + raise + _LOGGER.debug( + "rmtree: %s repopulated mid-delete (attempt %d): %s", + path, + attempt + 1, + err, + ) + last_err = err + # Give the racing writer (e.g. Finder) time to settle + time.sleep(0.05 * (attempt + 1)) + try: + shutil.rmtree(path, onexc=_onexc) + except OSError as err: + # Keep the earlier races visible in the traceback + raise err from last_err def walk_files(path: Path): diff --git a/tests/unit_tests/test_helpers.py b/tests/unit_tests/test_helpers.py index 53c326e0d0..ff82fa3c80 100644 --- a/tests/unit_tests/test_helpers.py +++ b/tests/unit_tests/test_helpers.py @@ -1,3 +1,4 @@ +import errno import io import logging import os @@ -5,7 +6,7 @@ from pathlib import Path import socket import stat import types -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch from aioesphomeapi.host_resolver import AddrInfo, IPv4Sockaddr, IPv6Sockaddr from hypothesis import given, settings @@ -966,6 +967,77 @@ def test_copy_file_if_changed_nonexistent_source(tmp_path: Path) -> None: helpers.copy_file_if_changed(src, dst) +def test_rmtree_removes_tree(tmp_path: Path) -> None: + """Test rmtree removes a populated directory tree.""" + target = tmp_path / "target" + (target / "sub").mkdir(parents=True) + (target / "sub" / "file.txt").write_text("content") + + helpers.rmtree(target) + assert not target.exists() + + +def test_rmtree_nonexistent_path(tmp_path: Path) -> None: + """Test rmtree on an already-removed path is a no-op.""" + helpers.rmtree(tmp_path / "gone") + + +def test_rmtree_retries_when_directory_repopulated(tmp_path: Path) -> None: + """Test rmtree retries when a file appears mid-delete (Finder .DS_Store race).""" + target = tmp_path / "target" + (target / "sub").mkdir(parents=True) + real_rmdir = os.rmdir + repopulated = False + + def racy_rmdir(path, **kwargs): + nonlocal repopulated + if not repopulated and Path(path).name == "target": + repopulated = True + (target / ".DS_Store").write_text("x") # Finder wins the race + real_rmdir(path, **kwargs) + + with patch("os.rmdir", side_effect=racy_rmdir), patch("time.sleep"): + helpers.rmtree(target) + assert repopulated + assert not target.exists() + + +def test_rmtree_raises_after_retries_exhausted(tmp_path: Path) -> None: + """Test rmtree gives up on a persistent ENOTEMPTY once attempts run out.""" + target = tmp_path / "target" + target.mkdir() + errs = [ + OSError(errno.ENOTEMPTY, "Directory not empty", str(target)) + for _ in range(helpers.RMTREE_MAX_ATTEMPTS) + ] + + with ( + patch("shutil.rmtree", side_effect=errs) as mock_rmtree, + patch("time.sleep") as mock_sleep, + pytest.raises(OSError, match="Directory not empty") as excinfo, + ): + helpers.rmtree(target) + assert mock_rmtree.call_count == helpers.RMTREE_MAX_ATTEMPTS + assert mock_sleep.call_args_list == [call(0.05), call(0.1)] + # Final failure chains to the last retried race + assert excinfo.value is errs[-1] + assert excinfo.value.__cause__ is errs[-2] + + +def test_rmtree_does_not_retry_other_oserror(tmp_path: Path) -> None: + """Test rmtree raises non-ENOTEMPTY errors immediately.""" + target = tmp_path / "target" + target.mkdir() + err = OSError(errno.EACCES, "Permission denied", str(target)) + + with ( + patch("shutil.rmtree", side_effect=err) as mock_rmtree, + pytest.raises(OSError, match="Permission denied"), + ): + helpers.rmtree(target) + assert mock_rmtree.call_count == 1 + + def test_resolve_ip_address_sorting() -> None: """Test that results are sorted by preference.""" # Create multiple address infos with different preferences From 4333870590953b001e3a60340c2d6e4b1d7b675f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 28 Aug 2026 13:43:48 -0500 Subject: [PATCH 006/147] [core] Use uv for the ESP-IDF Python environment when available (#18838) --- esphome/espidf/framework.py | 33 ++++++++++++++++------- tests/unit_tests/test_espidf_framework.py | 27 +++++++++++++++++++ 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 6c2a285360..9373b5f569 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -1053,15 +1053,28 @@ def _check_esp_idf_python_env_install( constraint_file_path, ) - cmd_pip_install = [ - str(env_python_path), - "-m", - "pip", - "install", - "--upgrade", - "--constraint", - constraint_file_path, - ] + # uv (much faster than pip) when available, e.g. in the docker image + if uv_path := shutil.which("uv"): + cmd_pip_install = [ + uv_path, + "pip", + "install", + "--python", + str(env_python_path), + "--upgrade", + "--constraint", + str(constraint_file_path), + ] + else: + cmd_pip_install = [ + str(env_python_path), + "-m", + "pip", + "install", + "--upgrade", + "--constraint", + str(constraint_file_path), + ] _LOGGER.info("Installing ESP-IDF %s Python dependencies ...", version) cmd = cmd_pip_install + [ @@ -1135,6 +1148,8 @@ def check_esp_idf_install( env = {} env["IDF_TOOLS_PATH"] = str(get_idf_tools_path()) env["IDF_PATH"] = "" + # uv defaults to 3 HTTP retries; match the pioarduino penv's bump to 10 + env["UV_HTTP_RETRIES"] = os.environ.get("UV_HTTP_RETRIES", "10") # An explicit ESPHOME_IDF_DEFAULT_TARGETS wins over the caller's # per-variant request (builder-image pre-warm); otherwise the caller's diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index afa4433aa1..3eeace9914 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -520,6 +520,33 @@ def test_check_esp_idf_install_feature_failure(espidf_mocks: SimpleNamespace) -> check_esp_idf_install(_IDF_VERSION, force=True, features=["fb"]) +def test_python_deps_use_uv_when_available( + espidf_mocks: SimpleNamespace, monkeypatch: pytest.MonkeyPatch +) -> None: + """The python env installs go through uv when on the PATH, pip otherwise.""" + monkeypatch.delenv("UV_HTTP_RETRIES", raising=False) + with patch( + "esphome.espidf.framework.shutil.which", + # Keyed on the name: the same which() also probes the default tools + side_effect=lambda name: "/usr/bin/uv" if name == "uv" else None, + ): + check_esp_idf_install(_IDF_VERSION, force=True, features=["fb"]) + upgrade_call, feature_call = espidf_mocks.run_ok.call_args_list[1:3] + upgrade_cmd, feature_cmd = upgrade_call.args[0], feature_call.args[0] + assert upgrade_cmd[:3] == ["/usr/bin/uv", "pip", "install"] + assert "--python" in upgrade_cmd + assert feature_cmd[:3] == ["/usr/bin/uv", "pip", "install"] + assert upgrade_call.kwargs["env"]["UV_HTTP_RETRIES"] == "10" + + espidf_mocks.run_ok.reset_mock() + monkeypatch.setenv("UV_HTTP_RETRIES", "3") # an explicit user value wins + with patch("esphome.espidf.framework.shutil.which", return_value=None): + check_esp_idf_install(_IDF_VERSION, force=True, features=["fb"]) + upgrade_call = espidf_mocks.run_ok.call_args_list[1] + assert upgrade_call.args[0][1:4] == ["-m", "pip", "install"] + assert upgrade_call.kwargs["env"]["UV_HTTP_RETRIES"] == "3" + + def _mark_installed() -> None: """Create the extracted marker and python-env interpreter so the install check takes the already-installed path rather than force-installing.""" From 1623fe0852722b9c93e767aa2af5557389486d52 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 28 Aug 2026 13:49:21 -0500 Subject: [PATCH 007/147] [esp32] Exclude the WiFi and Bluetooth stacks from builds that do not use them (#18599) --- esphome/components/esp32/__init__.py | 15 +++++++++ esphome/components/espnow/__init__.py | 5 +++ esphome/components/mdns/__init__.py | 8 +++++ esphome/components/zigbee/zigbee_esp32.py | 5 +++ .../config/exclusion_reincludes_espnow.yaml | 11 +++++++ .../config/exclusion_reincludes_wifi_ble.yaml | 13 ++++++++ .../esp32/config/network_ethernet_only.yaml | 2 ++ .../esp32/config/network_wifi_only.yaml | 2 ++ tests/component_tests/esp32/test_esp32.py | 33 ++++++++++++++++--- 9 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 tests/component_tests/esp32/config/exclusion_reincludes_espnow.yaml create mode 100644 tests/component_tests/esp32/config/exclusion_reincludes_wifi_ble.yaml diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index bc91f29a42..48bb7bf6a1 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -214,11 +214,13 @@ COMPILER_OPTIMIZATIONS = { # builds that need them. DEFAULT_EXCLUDED_IDF_COMPONENTS = ( "app_trace", # CPU trace/SystemView support - unused by ESPHome + "bt", # Bluetooth stack - re-included by request_bluetooth(); its REQUIRES pulls the WiFi stack back "cmock", # Unit testing mock framework - ESPHome doesn't use IDF's testing "console", # Console REPL - unused by ESPHome; espressif/mdns pulls it back when configured "driver", # Legacy driver shim - only needed by esp32_touch, esp32_can for legacy headers "esp-tls", # TLS wrapper - re-included by http_request, mqtt, web_server_idf "esp_adc", # ADC driver - only needed by adc component + "esp_coex", # WiFi/BT coexistence - re-included by esp32_ble_tracker, zigbee; esp_wifi/bt pull it back "esp_driver_cam", # Camera driver - the esp32-camera managed component pulls it back "esp_driver_dac", # DAC driver - only needed by esp32_dac component "esp_driver_gptimer", # General purpose timer - re-included by ac_dimmer, opentherm, Arduino BLE libs @@ -236,6 +238,7 @@ DEFAULT_EXCLUDED_IDF_COMPONENTS = ( "esp_driver_twai", # TWAI/CAN driver - only needed by esp32_can component "esp_eth", # Ethernet driver - only needed by ethernet component "esp_gdbstub", # GDB stub panic handler - unused by ESPHome; bt pulls it back + "esp_hal_ieee802154", # 802.15.4 HAL - ieee802154 pulls it back "esp_hid", # HID host/device support - ESPHome doesn't implement HID functionality "esp_http_client", # HTTP client - only needed by http_request component "esp_http_server", # HTTP server - re-included by web_server_idf, esp32_camera_web_server @@ -243,8 +246,11 @@ DEFAULT_EXCLUDED_IDF_COMPONENTS = ( "esp_https_server", # HTTPS server - ESPHome has its own web server "esp_lcd", # LCD controller drivers - only needed by display component "esp_local_ctrl", # Local control over HTTPS/BLE - ESPHome has native API + "esp_phy", # RF PHY - esp_wifi/bt/ieee802154 pull it back when they are in the build + "esp_wifi", # WiFi stack - re-included by request_wifi(), espnow; bt pulls it back for BLE builds "espcoredump", # Core dump support - ESPHome has its own debug component "fatfs", # FAT filesystem - ESPHome doesn't use filesystem storage + "ieee802154", # 802.15.4 radio - IDF openthread and the Zigbee libs pull it back "json", # cJSON library - ESPHome uses ArduinoJson instead "mqtt", # ESP-IDF MQTT library - ESPHome has its own MQTT implementation "nvs_sec_provider", # NVS encryption key provider - re-included when CONFIG_NVS_ENCRYPTION is set @@ -260,6 +266,7 @@ DEFAULT_EXCLUDED_IDF_COMPONENTS = ( "unity", # Unit testing framework - ESPHome doesn't use IDF's testing "wear_levelling", # Flash wear levelling for fatfs - unused since fatfs unused "wifi_provisioning", # WiFi provisioning - ESPHome uses its own improv implementation + "wpa_supplicant", # WPA supplicant - re-included by request_wifi() for esp_eap_client.h ) # Additional IDF managed components to exclude for Arduino framework builds @@ -709,6 +716,9 @@ def request_wifi(ap: bool = False) -> None: net.wifi = True if ap: net.wifi_ap = True + include_builtin_idf_component("esp_wifi") + # wifi_component.cpp includes esp_eap_client.h/esp_wpa2.h + include_builtin_idf_component("wpa_supplicant") def request_ethernet() -> None: @@ -720,11 +730,14 @@ def request_bluetooth() -> None: """Request the Bluetooth controller.""" net = _network_sdkconfig() net.bluetooth = True + include_builtin_idf_component("bt") def request_software_coexistence() -> None: """Request WiFi/BT software coexistence (only valid alongside WiFi).""" _network_sdkconfig().software_coexistence = True + # Callers include esp_coexist.h directly. + include_builtin_idf_component("esp_coex") def add_idf_component( @@ -2304,6 +2317,8 @@ async def _reconcile_network_sdkconfig() -> None: # WiFi stack: disable only when Ethernet is present and WiFi is not. WiFi # relies on the IDF default (enabled), so it is never written True here. + # esp_wifi is excluded by default on IDF, so this only matters for Arduino + # or when bt pulls it back. wifi_disabled = net.ethernet and not net.wifi if wifi_disabled: set_idf_sdkconfig_default("CONFIG_ESP_WIFI_ENABLED", False) diff --git a/esphome/components/espnow/__init__.py b/esphome/components/espnow/__init__.py index ee3732c406..5541a6ee97 100644 --- a/esphome/components/espnow/__init__.py +++ b/esphome/components/espnow/__init__.py @@ -155,6 +155,11 @@ async def to_code(config: ConfigType) -> None: cg.add_define("USE_ESPNOW") cg.add_define("USE_ESPNOW_MAX_PAYLOAD_SIZE", config[CONF_MAX_PAYLOAD_SIZE]) + if CORE.is_esp32: + from esphome.components.esp32 import include_builtin_idf_component + + include_builtin_idf_component("esp_wifi") + if CONF_WIFI in CORE.config: # Track the Wi-Fi channel via connect events instead of polling every loop wifi.request_wifi_connect_state_listener() diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index 64d7b9adc5..f039bb69f0 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -9,6 +9,7 @@ from esphome.const import ( CONF_PROTOCOL, CONF_SERVICE, CONF_SERVICES, + CONF_WIFI, PlatformFramework, ) from esphome.core import CORE, Lambda, coroutine_with_priority @@ -211,6 +212,13 @@ async def to_code(config: ConfigType) -> None: add_idf_component(name="espressif/mdns", ref="1.12.0") # ESPHome only advertises; the browse APIs are unused add_idf_sdkconfig_option("CONFIG_MDNS_ENABLE_BROWSE", False) + # The mdns console CLI is never used by ESPHome + add_idf_sdkconfig_option("CONFIG_MDNS_ENABLE_CONSOLE_CLI", False) + if CONF_WIFI not in CORE.config: + # Without WiFi the predefined STA/AP interface handlers are dead + # code; disabling them lets mdns build without the WiFi stack. + add_idf_sdkconfig_option("CONFIG_MDNS_PREDEF_NETIF_STA", False) + add_idf_sdkconfig_option("CONFIG_MDNS_PREDEF_NETIF_AP", False) cg.add_define("USE_MDNS") diff --git a/esphome/components/zigbee/zigbee_esp32.py b/esphome/components/zigbee/zigbee_esp32.py index ade45e8cc3..57fa3b2a00 100644 --- a/esphome/components/zigbee/zigbee_esp32.py +++ b/esphome/components/zigbee/zigbee_esp32.py @@ -9,6 +9,7 @@ from esphome.components.esp32 import ( add_idf_component, add_idf_sdkconfig_option, add_partition, + include_builtin_idf_component, require_vfs_select, ) import esphome.config_validation as cv @@ -288,6 +289,10 @@ async def esp32_to_code(config: ConfigType) -> "MockObj": ref="2.0.4", ) + if CONF_WIFI in CORE.config: + # zigbee_esp32.cpp uses esp_coexist.h when WiFi is present + include_builtin_idf_component("esp_coex") + # add sdkconfigs later so they can overwrite esp32 defaults CORE.add_job(_zigbee_add_sdkconfigs, config) diff --git a/tests/component_tests/esp32/config/exclusion_reincludes_espnow.yaml b/tests/component_tests/esp32/config/exclusion_reincludes_espnow.yaml new file mode 100644 index 0000000000..2ede6c42df --- /dev/null +++ b/tests/component_tests/esp32/config/exclusion_reincludes_espnow.yaml @@ -0,0 +1,11 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +espnow: + channel: 1 + auto_add_peer: true diff --git a/tests/component_tests/esp32/config/exclusion_reincludes_wifi_ble.yaml b/tests/component_tests/esp32/config/exclusion_reincludes_wifi_ble.yaml new file mode 100644 index 0000000000..883abdb5fa --- /dev/null +++ b/tests/component_tests/esp32/config/exclusion_reincludes_wifi_ble.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: MySSID + password: password1 + +esp32_ble_tracker: diff --git a/tests/component_tests/esp32/config/network_ethernet_only.yaml b/tests/component_tests/esp32/config/network_ethernet_only.yaml index 73d11e0a13..4f357e40e6 100644 --- a/tests/component_tests/esp32/config/network_ethernet_only.yaml +++ b/tests/component_tests/esp32/config/network_ethernet_only.yaml @@ -6,6 +6,8 @@ esp32: framework: type: esp-idf +mdns: + ethernet: type: W5500 clk_pin: 19 diff --git a/tests/component_tests/esp32/config/network_wifi_only.yaml b/tests/component_tests/esp32/config/network_wifi_only.yaml index 61dfde3e03..3abc17e324 100644 --- a/tests/component_tests/esp32/config/network_wifi_only.yaml +++ b/tests/component_tests/esp32/config/network_wifi_only.yaml @@ -6,6 +6,8 @@ esp32: framework: type: esp-idf +mdns: + wifi: ssid: "test_ssid" password: "test_password" diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index 297844b4e6..c72c4c3a6b 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -24,6 +24,7 @@ from esphome.components.esp32 import ( ) from esphome.components.esp32.const import ( KEY_ESP32, + KEY_EXCLUDE_COMPONENTS, KEY_NETWORK_SDKCONFIG, KEY_SDKCONFIG_OPTIONS, KEY_VARIANT, @@ -298,6 +299,20 @@ def test_esp32_configuration_errors( ("esp-tls", "esp_http_client"), id="nextion", ), + pytest.param( + # esp_wifi/wpa_supplicant from request_wifi(), bt from + # request_bluetooth(), esp_coex from esp32_ble_tracker's software + # coexistence (defaults on with wifi). esp_phy stays excluded; + # IDF requirement expansion pulls it back via esp_wifi. + "exclusion_reincludes_wifi_ble.yaml", + ("esp_wifi", "wpa_supplicant", "bt", "esp_coex"), + id="wifi_ble", + ), + pytest.param( + "exclusion_reincludes_espnow.yaml", + ("esp_wifi",), + id="espnow", + ), ], ) def test_default_exclusions_reincluded_by_owning_components( @@ -309,8 +324,6 @@ def test_default_exclusions_reincluded_by_owning_components( """Components whose IDF driver is excluded by default must re-include it during codegen; a dropped include_builtin_idf_component() call would only surface as a missing-header failure in a full compile job.""" - from esphome.components.esp32.const import KEY_EXCLUDE_COMPONENTS - generate_main(component_config_path(config_file)) excluded = CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] @@ -329,8 +342,6 @@ def test_nvs_sec_provider_stays_excluded_when_encryption_is_off( component_config_path: Callable[[str], Path], ) -> None: """An explicit CONFIG_NVS_ENCRYPTION=n keeps nvs_sec_provider excluded.""" - from esphome.components.esp32.const import KEY_EXCLUDE_COMPONENTS - generate_main(component_config_path("exclusion_stays_nvs_sdkconfig_off.yaml")) assert "nvs_sec_provider" in CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] @@ -939,6 +950,14 @@ def test_network_wifi_only_reconciles_end_to_end( sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] assert sdkconfig.get("CONFIG_ESP_WIFI_SOFTAP_SUPPORT") is False assert sdkconfig.get("CONFIG_LWIP_DHCPS") is False + # request_wifi() also puts the WiFi components back in the build set; + # esp_phy stays excluded, IDF requirement expansion pulls it back. + excluded = CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] + assert "esp_wifi" not in excluded + assert "wpa_supplicant" not in excluded + assert "esp_phy" in excluded + # With wifi present mdns keeps its predefined interfaces. + assert "CONFIG_MDNS_PREDEF_NETIF_STA" not in sdkconfig # WiFi stack stays enabled (no ethernet) and no Bluetooth requested. assert "CONFIG_ESP_WIFI_ENABLED" not in sdkconfig assert "CONFIG_BT_ENABLED" not in sdkconfig @@ -954,6 +973,12 @@ def test_network_ethernet_only_reconciles_end_to_end( sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] assert sdkconfig.get("CONFIG_ESP_WIFI_ENABLED") is False assert sdkconfig.get("CONFIG_SW_COEXIST_ENABLE") is False + # The whole radio stack stays out of the build set as well. + excluded = CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] + assert {"esp_wifi", "wpa_supplicant", "esp_phy", "esp_coex", "bt"} <= excluded + # Without wifi, mdns drops its predefined STA/AP interfaces. + assert sdkconfig.get("CONFIG_MDNS_PREDEF_NETIF_STA") is False + assert sdkconfig.get("CONFIG_MDNS_PREDEF_NETIF_AP") is False def test_network_wifi_ble_coexistence_reconciles_end_to_end( From 0dce7f48459fd1a3f0b954b83126bcb4407ea34c Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 28 Aug 2026 13:50:52 -0500 Subject: [PATCH 008/147] [esp8266] Add the native library backend (#18558) --- esphome/arduino/__init__.py | 0 esphome/arduino/library.py | 531 ++++++++ esphome/build_gen/build_tool.py | 108 ++ esphome/platformio/library.py | 109 +- tests/unit_tests/build_gen/test_build_tool.py | 246 ++++ tests/unit_tests/test_arduino_library.py | 1162 +++++++++++++++++ tests/unit_tests/test_platformio_library.py | 209 ++- 7 files changed, 2355 insertions(+), 10 deletions(-) create mode 100644 esphome/arduino/__init__.py create mode 100644 esphome/arduino/library.py create mode 100644 esphome/build_gen/build_tool.py create mode 100644 tests/unit_tests/build_gen/test_build_tool.py create mode 100644 tests/unit_tests/test_arduino_library.py diff --git a/esphome/arduino/__init__.py b/esphome/arduino/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esphome/arduino/library.py b/esphome/arduino/library.py new file mode 100644 index 0000000000..e224e62589 --- /dev/null +++ b/esphome/arduino/library.py @@ -0,0 +1,531 @@ +"""Arduino-core backend for the shared PlatformIO library converter. + +Bundled names build straight from the framework tree; everything else goes +through ``esphome.platformio.library``. Mirrors ``lib_ldf_mode=off``: each +library builds its own archive; all include dirs join one global path. + +Deviations from PlatformIO: flat-layout libraries get the recursive default +source filter; ``dot_a_linkage`` is honored; bundled libraries never run a +manifest ``extraScript``; manifest ``-I`` flags join the global include path; +``precompiled``/``ldflags`` properties are refused by name. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import logging +from pathlib import Path +import re + +from esphome.core import CORE, EsphomeError, Library +from esphome.helpers import walk_files +from esphome.platformio.extra_script import apply_extra_script +from esphome.platformio.library import ( + DEFAULT_BUILD_INCLUDE_DIR, + DEFAULT_BUILD_SRC_FILTER, + ESPHOME_DATA_KEY, + ESPHOME_DATA_LINK_FLAGS_KEY, + LIBRARY_HEADER_SUFFIXES, + SRC_FILE_EXTENSIONS, + ConvertedLibrary, + IncompatiblePlatform, + InvalidLibrary, + LibraryBackend, + _url_or_none, + check_library_data, + collect_filtered_files, + convert_libraries, + ensure_list, + is_lib_ignored, + lex_build_flags, + lib_ignore_set, + normalize_dependencies, + parse_library_json, + parse_library_properties, + warn_properties_depends, +) + +_LOGGER = logging.getLogger(__name__) + + +@dataclass +class ArduinoLibrary: + """One resolved library, ready for the ninja generator.""" + + name: str + sources: list[Path] = field(default_factory=list) + include_dirs: list[Path] = field(default_factory=list) + # Extra compile flags private to this library's own sources + flags: list[str] = field(default_factory=list) + # PlatformIO's build.libArchive / Arduino's dot_a_linkage: when False the + # objects go to the linker directly (symbols nothing references survive) + lib_archive: bool = True + # Link inputs the library contributes (-L dirs / -l libs, e.g. from + # precompiled vendor blobs) and -Wl, options for the firmware link + link_dirs: list[Path] = field(default_factory=list) + link_libs: list[str] = field(default_factory=list) + link_flags: list[str] = field(default_factory=list) + + +# Source-like suffixes the case-sensitive suffix map rejects +_UNMAPPED_SOURCE_SUFFIXES = frozenset( + {s.lower() for s in SRC_FILE_EXTENSIONS} | {".ino"} +) + +# Filename-plain names: an allowlist excludes separators, drive colons, +# and dot-only names by shape +_SAFE_LIBRARY_NAME_RE = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_. +-]*\Z") + + +def _is_safe_library_name(name: object) -> bool: + """Whether a name may be joined under the framework's libraries dir.""" + return isinstance(name, str) and _SAFE_LIBRARY_NAME_RE.fullmatch(name) is not None + + +def _manifest_build(name: str, data: object) -> dict: + """The manifest's ``build`` section; malformed manifests fail by name.""" + build = data.get("build", {}) if isinstance(data, dict) else None + if not isinstance(build, dict): + raise EsphomeError(f"Library {name} has a malformed manifest") + return build + + +def _resolve_src_dir(name: str, read_path: Path, build: dict) -> str: + """Resolve PIO's source dir: manifest srcDir, else src/Src, else the root.""" + if "srcDir" not in build: + return next((d for d in ("src", "Src") if (read_path / d).is_dir()), ".") + # A declared srcDir (falsy included) that does not resolve is a manifest error + src_dir = build["srcDir"] + if not (isinstance(src_dir, str) and src_dir and (read_path / src_dir).is_dir()): + raise EsphomeError( + f"Library {name} declares srcDir {src_dir!r} which does not exist" + ) + return src_dir + + +def _reject_unsupported_link_fields(name: str, data: dict) -> None: + # PIO honors these; ignoring them would fail at link with no stated + # cause. Property values are strings, so "false" is not a declaration. + precompiled = data.get("precompiled") + if precompiled and str(precompiled).strip().lower() != "false": + raise EsphomeError( + f"Library {name} declares precompiled, which this backend does not support" + ) + if data.get("ldflags"): + raise EsphomeError( + f"Library {name} declares ldflags, which this backend does not support" + ) + + +def _resolve_lib_archive(name: str, data: dict, build: dict) -> bool: + """build.libArchive, else dot_a_linkage (an Arduino IDE property PIO + ignores; a deliberate extra), else archive.""" + + # Strict parse: bool("false") is True + def _parse(key: str, raw: object) -> bool: + if isinstance(raw, bool): + return raw + value = str(raw).strip().lower() + if value in ("true", "false"): + return value == "true" + raise EsphomeError(f"Library {name} has a malformed {key} value {raw!r}") + + if "libArchive" in build: + return _parse("libArchive", build["libArchive"]) + if "dot_a_linkage" in data: + return _parse("dot_a_linkage", data["dot_a_linkage"]) + return True + + +def _classify_build_flags( + name: str, read_path: Path, lib: ArduinoLibrary, flag_tokens: list[str] +) -> list[str]: + """Route the lexed build.flags into the library's flag lists. + + Returns the ``-I`` arguments for the include-dir resolution. + """ + include_flags: list[str] = [] + for tok in flag_tokens: + if tok.startswith("-I"): + include_flags.append(tok[2:]) + elif tok.startswith("-L"): + link_dir = (read_path / tok[2:]).resolve() + if not link_dir.is_dir(): + # Kept (the linker ignores missing -L dirs); the warning + # names the culprit before a bare "cannot find -lfoo" + _LOGGER.warning( + "Library %s declares library dir %s which does not exist", + name, + tok[2:], + ) + lib.link_dirs.append(link_dir) + elif tok.startswith("-l"): + lib.link_libs.append(tok[2:]) + elif tok.startswith("-Wl,"): + lib.link_flags.append(tok) + else: + lib.flags.append(tok) + return include_flags + + +def _resolve_include_dirs( + name: str, + read_path: Path, + lib: ArduinoLibrary, + build: dict, + src_dir: str, + include_flags: list[str], +) -> None: + include_dir = build.get("includeDir", DEFAULT_BUILD_INCLUDE_DIR) + if not isinstance(include_dir, str): + raise EsphomeError(f"Library {name} has a malformed includeDir") + for d, explicit in [ + (include_dir, "includeDir" in build), + (src_dir, False), # _resolve_src_dir already validated it + *((flag, True) for flag in include_flags), + ]: + if (path := (read_path / d)).is_dir(): + lib.include_dirs.append(path.resolve()) + elif explicit: + # Warn-and-drop (unlike srcDir): a missing include dir is + # harmless until a header is needed, and the compile names it + _LOGGER.warning( + "Library %s declares include dir %s which does not exist", name, d + ) + + +def _collect_lib_sources( + name: str, + read_path: Path, + lib: ArduinoLibrary, + src_dir: str, + src_filter: list[str], +) -> None: + sources: list[Path] = [] + dropped: list[str] = [] + saw_header = False + for f in collect_filtered_files(read_path / src_dir, src_filter): + path = Path(f) + suffix = path.suffix + if suffix in SRC_FILE_EXTENSIONS: + # resolve() per file: srcFilter patterns may escape src_dir + sources.append(path.resolve()) + elif suffix.lower() in _UNMAPPED_SOURCE_SUFFIXES: + # A source-like suffix the case-sensitive map rejects (.CPP, + # .ino) is a dropped compilation unit; headers fall through + dropped.append(path.name) + elif suffix.lower() in LIBRARY_HEADER_SUFFIXES: + saw_header = True + lib.sources = sorted(sources) + if dropped: + _LOGGER.warning( + "Library %s: %d file(s) with unmapped source suffixes are not compiled: %s", + name, + len(dropped), + ", ".join(sorted(dropped)), + ) + if not lib.sources and not saw_header: + # Matched headers mean header-only; a filter matching nothing is + # a manifest/tree problem (a truly empty tree raises elsewhere) + _LOGGER.warning("Library %s: no source files matched", name) + + +def _library_info(name: str, read_path: Path, data: dict) -> ArduinoLibrary: + """Resolve one library's sources, include dirs, and flags (PIO semantics).""" + build = _manifest_build(name, data) + _reject_unsupported_link_fields(name, data) + src_dir = _resolve_src_dir(name, read_path, build) + src_filter = ensure_list(build.get("srcFilter", DEFAULT_BUILD_SRC_FILTER)) + if not all(isinstance(entry, str) for entry in src_filter): + raise EsphomeError(f"Library {name} has a malformed srcFilter") + lib = ArduinoLibrary(name=name, lib_archive=_resolve_lib_archive(name, data, build)) + # PlatformIO shell-lexes each build.flags entry + include_flags = _classify_build_flags( + name, read_path, lib, lex_build_flags(build.get("flags", []), f"library {name}") + ) + _resolve_include_dirs(name, read_path, lib, build, src_dir, include_flags) + _collect_lib_sources(name, read_path, lib, src_dir, src_filter) + return lib + + +def _bundled_library(framework_path: Path, name: str) -> ArduinoLibrary: + """A library bundled with the Arduino core, read from the framework tree. + + ``library.json`` wins over ``library.properties`` when both exist, as in + PlatformIO's LibBuilderFactory; only the JSON manifest can carry a + ``build`` section (srcDir, srcFilter, flags). + """ + lib_dir = framework_path / "libraries" / name + manifest_json = lib_dir / "library.json" + if manifest_json.is_file(): + try: + data = parse_library_json(manifest_json) + except ValueError as err: # JSONDecodeError + raise EsphomeError( + f"Bundled library {name} has a corrupt library.json ({err}); " + "the framework install may be incomplete (run 'esphome clean-all')" + ) from err + elif (manifest := lib_dir / "library.properties").is_file(): + data = parse_library_properties(manifest) + else: + # Debug, not warning: the legacy manifest-less layout is legal and + # the 3.1.2 core ships one such library (FSTools), so a warning + # would be unactionable noise on every build using it + _LOGGER.debug("Bundled library %s has no manifest; using defaults", name) + data = {} + if isinstance(data, dict): + # Bundled manifest deps are never walked; make the skip visible + if data.get("dependencies"): + _LOGGER.warning( + "Bundled library %s declares dependencies, which are not " + "resolved automatically; add them with add_library() if needed", + name, + ) + warn_properties_depends(name, data) + build = data.get("build") + if isinstance(build, dict) and build.get("extraScript"): + # Scripts only run on the converted path; building without + # the script's flags would miscompile + raise EsphomeError( + f"Bundled library {name} declares an extraScript, which is " + "not run for bundled libraries" + ) + lib = _library_info(name, lib_dir, data) + _assert_tree_has_code( + name, + lib_dir, + "the framework install may be incomplete (run 'esphome clean-all')", + ) + return lib + + +def _assert_tree_has_code(name: str, root: Path, hint: str) -> None: + """An empty or half-extracted tree can never link; fail by name (a + warning would scroll away and resurface as undefined symbols).""" + if not any( + Path(p).suffix in SRC_FILE_EXTENSIONS + or Path(p).suffix.lower() in LIBRARY_HEADER_SUFFIXES + for p in walk_files(root) + ): + raise EsphomeError(f"Library {name} has no sources or headers; {hint}") + + +def _external_short_name(name: str) -> str: + """The short library name of a requested spec. + + "owner/Name" and plain names take the last path segment; "Name=" + takes the declared name. Git tails (".git", "#ref") are stripped like + the walk's URL normalization; the comparand is a manifest dependency + name, never a spec. + """ + head, sep, tail = name.partition("=") + if sep and "://" in tail: + return head + short = name.rsplit("/", maxsplit=1)[-1] + return short.partition("#")[0].removesuffix(".git") + + +def _check_unfulfilled_provides( + provided_requests: set[str], satisfied: set[str], still_requested: set[str] +) -> None: + """Fail by name when a walk-skipped dependency was never added. + + An unfulfilled provides() promise only surfaces as undefined symbols + at link. The walk records across re-resolutions, so a name no final + manifest still requests is stale state, never a failure. + """ + if missing := sorted((provided_requests & still_requested) - satisfied): + raise EsphomeError( + "provides() skipped these dependencies but nothing added them: " + f"{', '.join(missing)}; the build is missing libraries" + ) + + +def resolve_libraries( + framework_path: Path, *, pio_platform: str, board_mcu: str, cache_key: str +) -> list[ArduinoLibrary]: + """Resolve every ``cg.add_library()`` entry into an :class:`ArduinoLibrary`. + + ``pio_platform``/``board_mcu`` filter manifests the way PlatformIO would + for that core (e.g. ``espressif8266``/``esp8266``); ``cache_key`` keys the + shared converter's download cache. + + The returned list is not topologically sorted, so the caller must link + the archives inside one ``--start-group``/``--end-group`` pair (the + bundled-first grouping is incidental). + """ + bundled: list[ArduinoLibrary] = [] + external: list[Library] = [] + # PlatformIO's lib_ignore covers framework-bundled libraries too; the + # shared converter only filters the registry/git ones. + lib_ignore = lib_ignore_set() + # Exact directory names keep membership case-sensitive everywhere + # (an is_dir() probe would match "wire" on macOS/Windows and build + # the bundled Wire twice) + libraries_dir = framework_path / "libraries" + if not libraries_dir.is_dir(): + # A registry fallback would fail later with a misleading + # package-not-found error per bundled name + raise EsphomeError( + f"{libraries_dir} is missing; the framework install may be " + "incomplete (run 'esphome clean-all')" + ) + bundled_dir_names = frozenset(p.name for p in libraries_dir.iterdir() if p.is_dir()) + + def _provided(name: object) -> bool: + return _is_safe_library_name(name) and name in bundled_dir_names + + for library in CORE.platformio_libraries.values(): + if is_lib_ignored(library.name, lib_ignore): + continue + # Bundled only for a bare name with a matching framework dir; pinned + # or unmatched names resolve from the registry, as under PlatformIO. + if not library.repository and not library.version and _provided(library.name): + # Bundled manifest deps are not walked; _bundled_library warns + bundled.append(_bundled_library(framework_path, library.name)) + else: + external.append(library) + + converted: list[ArduinoLibrary] = [] + bundled_names = {lib.name for lib in bundled} + converted_manifest_names: set[str] = set() + # Bundled candidates skipped on purpose (platform filter); the + # provides() reconciliation must count them as satisfied + knowingly_skipped: set[str] = set() + # Dependency names of the manifests actually emitted; a walk recording + # for a since-re-resolved manifest must not fail the reconciliation + final_dep_names: set[str] = set() + # Ordered set of bundled dependency names to add once conversion is done + pending_bundled: dict[str, None] = {} + # Deps matching a separately-requested external are already in the build + # (a duplicate archive means duplicate-symbol link errors) + external_short_names = { + _external_short_name(lib.name) for lib in external if lib.name + } + + def _add_bundled_dependencies(component: ConvertedLibrary) -> None: + # A version-less bare name ("Hash") is a core-bundled library the + # shared converter cannot resolve from the registry + for dep in normalize_dependencies( + component.data.get("dependencies"), component.name + ): + # normalize_dependencies guarantees a non-empty str name + name = dep["name"] + final_dep_names.add(name) + if "/" in name: + owner, _, pkg = name.partition("/") + if _is_safe_library_name(owner) and _is_safe_library_name(pkg): + # Owner-qualified; the converter resolves it from the registry + continue + if not _is_safe_library_name(name): + # The name becomes a path component; never join a traversal + _LOGGER.warning( + "Ignoring malformed dependency entry %r of library %s", + dep, + component.name, + ) + continue + if name in external_short_names: + if _provided(name): + # A bundled copy is suppressed; a coincidental name + # collision would surface as link errors + _LOGGER.warning( + "Dependency %s of %s is assumed satisfied by a " + "requested external library; the bundled copy is " + "not added", + name, + component.name, + ) + else: + _LOGGER.debug( + "Dependency %s of %s assumed satisfied by a requested " + "external library", + name, + component.name, + ) + continue + if name in bundled_names or is_lib_ignored(name, lib_ignore): + continue + if _url_or_none(dep.get("version")) is not None: + # A URL names one specific source; never add the bundled copy + continue + if dep.get("owner") or not _provided(name): + # Only owner-less framework-tree names take the bundled + # copy (PIO's process_dependencies); the walk reports drops + continue + try: + # framework=None: the walk already warned for non-platform + # causes; debug keeps one fault from warning twice (pinned + # by test_nonplatform_rejection_warns_once_through_real_converter) + check_library_data(dep, pio_platform, None) + except IncompatiblePlatform as err: + # A knowing skip (platform filter), not a broken promise + knowingly_skipped.add(name) + _LOGGER.debug("Skip bundled candidate %s: %s", name, err) + continue + except InvalidLibrary as err: + # Malformed manifest data never counts as satisfied; the + # walk owns the warning (see the warns-once test above) + _LOGGER.debug("Skip malformed bundled candidate %s: %s", name, err) + continue + # Deferred: a later manifest name may satisfy this + pending_bundled.setdefault(name) + + def _emit(component: ConvertedLibrary) -> None: + apply_extra_script( + component, board_mcu=lambda: board_mcu, pio_platform=pio_platform + ) + _assert_tree_has_code( + component.get_require_name(), + component.source_dir, + "the download may be incomplete (run 'esphome clean-all')", + ) + if isinstance(manifest_name := component.data.get("name"), str): + converted_manifest_names.add(manifest_name) + lib = _library_info( + component.get_require_name(), component.source_dir, component.data + ) + # Extra-script LINKFLAGS travel outside build.flags; dropping + # them would link wrong with no stated cause + lib.link_flags.extend( + component.data.get(ESPHOME_DATA_KEY, {}).get( + ESPHOME_DATA_LINK_FLAGS_KEY, [] + ) + ) + converted.append(lib) + _add_bundled_dependencies(component) + + backend = LibraryBackend( + platform=pio_platform, + framework="arduino", + emit=_emit, + cache_key=cache_key, + # The walk must not resolve bundled names from the registry; + # _add_bundled_dependencies adds them after emit + provides=_provided, + ) + if external: + convert_libraries(external, backend) + for name in pending_bundled: + if name in converted_manifest_names: + # The converted library is this one; the bundled copy would + # double the archive. Warn like the external_short_names twin. + _LOGGER.warning( + "Dependency %s is assumed satisfied by a converted library's " + "manifest name; the bundled copy is not added", + name, + ) + continue + bundled_names.add(name) + bundled.append(_bundled_library(framework_path, name)) + + _check_unfulfilled_provides( + backend.provided_requests, + bundled_names + | converted_manifest_names + | external_short_names + | knowingly_skipped, + final_dep_names, + ) + + return bundled + converted diff --git a/esphome/build_gen/build_tool.py b/esphome/build_gen/build_tool.py new file mode 100644 index 0000000000..00aa1ec69d --- /dev/null +++ b/esphome/build_gen/build_tool.py @@ -0,0 +1,108 @@ +"""Tiny cross-platform build steps invoked from the generated ninja file. + +Plain script (not ``python -m``): it runs from ninja with whatever Python +started esphome and must not depend on the package being importable. + +Subcommands: + ar remove stale archive, then ``ar rcs`` + copy copy a file + +The ar rspfile carries one object path per line (the generating rule must +use ``$in_newline``, never ``$in``). +""" + +from pathlib import Path +import shutil +import subprocess +import sys + + +def _read_rspfile(rspfile: str) -> list[str]: + r"""The object paths listed in ``rspfile``, unquoted. + + GNU ar treats backslashes in response files as escapes (corrupts + Windows paths), so the caller expands the list into argv; strip the + simple surrounding quote ninja adds to special paths, then undo + ninja's POSIX escape for an embedded quote ('a'\\''b.o' -> a'b.o). + """ + return [ + line[1:-1].replace("'\\''", "'") + if len(line) >= 2 and line[0] == line[-1] and line[0] in "'\"" + else line + for line in Path(rspfile).read_text(encoding="utf-8").splitlines() + if line + ] + + +def _run_ar(ar: str, archive: str, rspfile: str) -> int: + # Remove first: ``ar rcs`` replaces members but never drops ones whose + # source was removed from the build, which would leak stale objects. + Path(archive).unlink(missing_ok=True) + objects = _read_rspfile(rspfile) + if not objects: + # An empty archive would "succeed" here and fail far away at link + print(f"ar: no objects listed in {rspfile} for {archive}", file=sys.stderr) + return 1 + # Batch by argv length: expanding the rspfile gives back the Windows + # 32767-char command-line limit it existed to avoid. "rcs" creates, + # "qs" appends; the s keeps the symbol index explicit on every ar. + op = "rcs" + ok = False + try: + while objects: + batch = [objects.pop(0)] + batch_len = len(batch[0]) + while objects and batch_len + len(objects[0]) < 25000: + batch_len += len(objects[0]) + 1 + batch.append(objects.pop(0)) + rc = subprocess.run( + [ar, op, archive, *batch], check=False, close_fds=False + ).returncode + if rc != 0: + return rc + op = "qs" + ok = True + return 0 + finally: + if not ok: + # Any failure (bad exit, missing ar binary, interrupt) must not + # leave a truncated archive behind + Path(archive).unlink(missing_ok=True) + + +def _run_copy(src: str, dst: str) -> int: + try: + shutil.copyfile(src, dst) + except OSError as err: + # Never leave a partially written output (e.g. a firmware image); + # SameFileError means dst IS src, where unlinking destroys the input + if not isinstance(err, shutil.SameFileError): + Path(dst).unlink(missing_ok=True) + print(f"copy: {src} -> {dst} failed: {err}", file=sys.stderr) + return 1 + return 0 + + +# mode -> (handler, expected operand count); surplus argv means a +# mis-specified ninja rule and must error, not silently drop operands +_MODES = {"ar": (_run_ar, 3), "copy": (_run_copy, 2)} + + +def main() -> int: + mode = sys.argv[1] if len(sys.argv) > 1 else "" + if entry := _MODES.get(mode): + handler, argc = entry + args = sys.argv[2:] + if len(args) != argc: + print( + f"build_tool {mode}: expected {argc} arguments, got {len(args)}", + file=sys.stderr, + ) + return 1 + return handler(*args) + print(f"unknown build_tool mode: {mode}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 306f07854e..0402311a9a 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -74,6 +74,11 @@ SOURCE_KIND_FOR_SUFFIX: dict[str, str] = { ".ASM": "asm", } SRC_FILE_EXTENSIONS = list(SOURCE_KIND_FOR_SUFFIX) +# Suffixes that count as headers when probing whether a library has any +# usable files at all (compare against Path.suffix.lower()) +LIBRARY_HEADER_SUFFIXES = frozenset( + {".h", ".hpp", ".hh", ".hxx", ".inc", ".ipp", ".tcc"} +) DOMAIN = "pio_components" @@ -329,6 +334,11 @@ class LibraryBackend: framework: str emit: Callable[["ConvertedLibrary"], None] cache_key: str + # Owner-less names this returns True for are skipped by the walk; + # the backend supplies them itself (e.g. core-bundled libraries) and + # reconciles provided_requests after resolving + provides: Callable[[str], bool] | None = None + provided_requests: set[str] = field(default_factory=set) def ensure_list[T](obj: T | list[T]) -> list[T]: @@ -469,7 +479,7 @@ def _valid_manifest_shape(data: Any) -> bool: ) -def check_library_data(data: dict, platform: str | None, framework: str): +def check_library_data(data: dict, platform: str | None, framework: str | None): """ Check whether a library manifest is compatible with the target toolchain. @@ -486,7 +496,8 @@ def check_library_data(data: dict, platform: str | None, framework: str): for targets (e.g. Zephyr) where PIO manifests rarely declare the platform yet portable libraries still build. framework: The active framework name (e.g. ``espidf``, ``arduino``, - ``zephyr``) the manifest is expected to declare. + ``zephyr``) the manifest is expected to declare. ``None`` skips + the framework check (and its warning), mirroring ``platform``. Raises: InvalidLibrary: If the library does not support the target platform. @@ -517,7 +528,7 @@ def check_library_data(data: dict, platform: str | None, framework: str): # under the target framework, and there's no way to opt out of the check at # this layer. Warn instead of failing so the user isn't forced to fork the # library to fix the manifest. - valid_framework = "*" in frameworks or framework in frameworks + valid_framework = framework is None or "*" in frameworks or framework in frameworks if not valid_framework: _LOGGER.warning( @@ -914,6 +925,56 @@ def is_lib_ignored(name: str | None, lib_ignore: set[str]) -> bool: ) +def _reconcile_versionless_skips( + skipped_versionless: list[tuple[Any, Any, str]], + components: dict[str, ConvertedLibrary], + backend: LibraryBackend, +) -> None: + """Warn for version-less deps nothing satisfied, and record the + backend-provided ones in ``backend.provided_requests`` for its + post-emit reconciliation; a silent drop surfaces as link errors far + from the cause.""" + resolved_manifest_names = {c.data.get("name") for c in components.values()} + # A treeless backend can never supply a bundled name; noise for it + log = _LOGGER.warning if backend.provides is not None else _LOGGER.debug + warned: set[str] = set() + for dep_name, dep_owner, requester in skipped_versionless: + if not isinstance(dep_name, str) or not dep_name or dep_name in warned: + continue + if dep_name in components: + # A version-less dep's request key is the name itself + continue + if ( + not dep_owner + and backend.provides is not None + and backend.provides(dep_name) + ): + # provides() only satisfies owner-less names (same guard as + # the walk's skip); record for the post-emit reconciliation. + # Checked before the manifest-name evidence so the overlap + # case warns once, in the backend's own suppression loop + backend.provided_requests.add(dep_name) + continue + if dep_name in resolved_manifest_names: + # Name-only evidence: a coincidental collision must stay + # visible where the user could pin it + warned.add(dep_name) + log( + "Version-less dependency %s of %s assumed satisfied by a " + "resolved library's manifest name only", + dep_name, + requester, + ) + continue + warned.add(dep_name) + log( + "Dependency %s of %s has no version to resolve and nothing " + "provides it; skipping", + dep_name, + requester, + ) + + def _fetch_source( component: ConvertedLibrary, salt: str, @@ -1083,6 +1144,8 @@ def convert_libraries( components: dict[str, ConvertedLibrary] = {} resolved_requirements: dict[str, frozenset[str]] = {} top_level_keys = set(top_level) + # (name, owner, requester) reconciled against the final resolution set + skipped_versionless: list[tuple[Any, Any, str]] = [] worklist = deque(dict.fromkeys(top_level)) while worklist: # Drain the frontier sequentially (spec resolution mutates shared @@ -1187,13 +1250,23 @@ def convert_libraries( component.data.get("dependencies"), component.name ): if "version" not in dependency: - # Cannot resolve from the registry; common for bundled - # names (Wire, SPI) -- unactionable noise above debug + # Cannot resolve from the registry; the post-emit + # reconciliation owns the drop warning + dep_name = dependency.get("name") _LOGGER.debug( "Skip version-less dependency %r of %s", - dependency.get("name"), + dep_name, component.name, ) + if not is_lib_ignored( + dep_name, lib_ignore + ) and dependency_is_usable( + dependency, backend.platform, backend.framework, component.name + ): + # Filtered or ignored deps are deliberately absent + skipped_versionless.append( + (dep_name, dependency.get("owner"), component.name) + ) continue if not dependency_is_usable( dependency, backend.platform, backend.framework, component.name @@ -1205,11 +1278,31 @@ def convert_libraries( if is_lib_ignored(dep_name, lib_ignore): _LOGGER.debug("Skip ignored dependency %s", dep_name) continue - # The version field may actually be a URL (git/archive dependency). + # The version may be a URL (git/archive), which names one + # specific source; never substitute a bundled library for it dep_version = dependency["version"] dep_url = _url_or_none(dep_version) if dep_url is not None: dep_version = None + elif ( + backend.provides is not None + and not dependency.get("owner") + and backend.provides(dep_name) + ): + # The backend adds it from its own tree; resolving here + # would fetch a same-named registry package + if dep_version and dep_version != "*": + # The pin is discarded; make the substitution visible + _LOGGER.warning( + "Dependency %s pins version %s; using the library " + "bundled with the framework instead", + dep_name, + dep_version, + ) + else: + _LOGGER.debug("Skip backend-provided dependency %s", dep_name) + backend.provided_requests.add(dep_name) + continue dep_key = add_spec(dep_name, dep_version, dep_url) node.edges.add(dep_key) worklist.append(dep_key) @@ -1263,4 +1356,6 @@ def convert_libraries( for component in components.values(): backend.emit(component) + _reconcile_versionless_skips(skipped_versionless, components, backend) + return [components[key] for key in top_level if key in components] diff --git a/tests/unit_tests/build_gen/test_build_tool.py b/tests/unit_tests/build_gen/test_build_tool.py new file mode 100644 index 0000000000..b029c647ab --- /dev/null +++ b/tests/unit_tests/build_gen/test_build_tool.py @@ -0,0 +1,246 @@ +"""Tests for the ninja build-tool helper script.""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +import sys +from unittest.mock import MagicMock, patch + +import pytest + +from esphome.build_gen import build_tool + + +def test_ar_removes_stale_archive(tmp_path: Path) -> None: + archive = tmp_path / "lib.a" + archive.write_text("stale") + rsp = tmp_path / "lib.a.rsp" + rsp.write_text("a.o\n") + with ( + patch.object( + build_tool.sys, + "argv", + ["build_tool", "ar", "ar-bin", str(archive), str(rsp)], + ), + patch.object( + build_tool.subprocess, "run", return_value=MagicMock(returncode=0) + ) as mock_run, + ): + assert build_tool.main() == 0 + assert not archive.exists() + # The rspfile is expanded by the shim (GNU ar would escape backslashes) + assert mock_run.call_args[0][0] == ["ar-bin", "rcs", str(archive), "a.o"] + + +def test_copy(tmp_path: Path) -> None: + src = tmp_path / "firmware.bin" + src.write_text("data") + dst = tmp_path / "firmware.factory.bin" + with patch.object( + build_tool.sys, "argv", ["build_tool", "copy", str(src), str(dst)] + ): + assert build_tool.main() == 0 + assert dst.read_text() == "data" + + +def test_unknown_mode(capsys: pytest.CaptureFixture[str]) -> None: + with patch.object(build_tool.sys, "argv", ["build_tool", "bogus"]): + assert build_tool.main() == 1 + assert "unknown build_tool mode" in capsys.readouterr().err + + +def test_runs_as_script(tmp_path: Path) -> None: + """The ninja rules invoke the file as a plain script.""" + + src = tmp_path / "a.bin" + src.write_text("x") + dst = tmp_path / "b.bin" + result = subprocess.run( + [sys.executable, build_tool.__file__, "copy", str(src), str(dst)], + check=False, + ) + assert result.returncode == 0 + assert dst.read_text() == "x" + + +def test_ar_expands_rspfile_without_escaping(tmp_path) -> None: + """Backslash paths survive: the shim expands the rspfile itself instead + of letting GNU ar treat backslashes as escapes.""" + rsp = tmp_path / "objs.rsp" + rsp.write_text("obj/a.o\nsub\\b.o\n") + with ( + patch.object( + build_tool.sys, + "argv", + ["build_tool", "ar", "ar-bin", str(tmp_path / "lib.a"), str(rsp)], + ), + patch.object( + build_tool.subprocess, "run", return_value=MagicMock(returncode=0) + ) as mock_run, + ): + assert build_tool.main() == 0 + assert mock_run.call_args[0][0] == [ + "ar-bin", + "rcs", + str(tmp_path / "lib.a"), + "obj/a.o", + "sub\\b.o", + ] + + +def test_ar_unquotes_ninja_escaped_paths(tmp_path: Path) -> None: + """The shim strips a simple surrounding quote, since ninja shell- + quotes special rsp paths, so ar sees the real filename.""" + rsp = tmp_path / "t.rsp" + rsp.write_text("'obj/a b.o'\nobj/c.o\n") + with ( + patch.object( + build_tool.sys, "argv", ["bt", "ar", "/usr/bin/ar", "lib.a", str(rsp)] + ), + patch.object(build_tool.subprocess, "run") as mock_run, + ): + mock_run.return_value.returncode = 0 + rc = build_tool.main() + assert rc == 0 + assert mock_run.call_args.args[0] == [ + "/usr/bin/ar", + "rcs", + "lib.a", + "obj/a b.o", + "obj/c.o", + ] + + +def test_ar_empty_object_list_fails( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A lost object list is an error here, not undefined symbols at link.""" + rsp = tmp_path / "t.rsp" + rsp.write_text("\n\n") + with patch.object( + build_tool.sys, "argv", ["bt", "ar", "/usr/bin/ar", "lib.a", str(rsp)] + ): + rc = build_tool.main() + assert rc == 1 + assert "no objects listed" in capsys.readouterr().err + + +def test_ar_batches_long_object_lists(tmp_path: Path) -> None: + """The expanded argv must stay under the Windows 32767-char limit: a + long object list creates with rcs, then appends with qs.""" + archive = tmp_path / "lib.a" + rsp = tmp_path / "lib.a.rsp" + objects = [f"dir/{'x' * 120}_{i}.o" for i in range(400)] + rsp.write_text("\n".join(objects) + "\n") + with ( + patch.object( + build_tool.sys, + "argv", + ["build_tool", "ar", "ar-bin", str(archive), str(rsp)], + ), + patch.object( + build_tool.subprocess, "run", return_value=MagicMock(returncode=0) + ) as mock_run, + ): + assert build_tool.main() == 0 + calls = [c[0][0] for c in mock_run.call_args_list] + assert len(calls) > 1 + assert calls[0][1] == "rcs" + assert all(c[1] == "qs" for c in calls[1:]) + assert [o for c in calls for o in c[3:]] == objects + assert all(sum(len(a) + 1 for a in c) < 32000 for c in calls) + + +def test_ar_batch_failure_stops(tmp_path: Path) -> None: + """A failing batch propagates its exit code without running the rest.""" + archive = tmp_path / "lib.a" + rsp = tmp_path / "lib.a.rsp" + rsp.write_text("\n".join(f"{'y' * 200}_{i}.o" for i in range(300)) + "\n") + with ( + patch.object( + build_tool.sys, + "argv", + ["build_tool", "ar", "ar-bin", str(archive), str(rsp)], + ), + patch.object( + build_tool.subprocess, + "run", + side_effect=lambda cmd, **kw: ( + archive.write_text("partial"), + MagicMock(returncode=3), + )[1], + ) as mock_run, + ): + assert build_tool.main() == 3 + assert mock_run.call_count == 1 + # The failed batch must not leave a truncated archive behind + assert not archive.exists() + + +def test_ar_exception_leaves_no_partial_archive(tmp_path: Path) -> None: + """A missing ar binary mid-loop must not leave a truncated archive from + earlier successful batches.""" + archive = tmp_path / "lib.a" + rsp = tmp_path / "lib.a.rsp" + rsp.write_text("a.o\n") + with ( + patch.object( + build_tool.sys, + "argv", + ["build_tool", "ar", "ar-bin", str(archive), str(rsp)], + ), + patch.object( + build_tool.subprocess, + "run", + side_effect=lambda cmd, **kw: ( + archive.write_text("partial"), + (_ for _ in ()).throw(FileNotFoundError("no ar")), + ), + ), + pytest.raises(FileNotFoundError), + ): + build_tool.main() + assert not archive.exists() + + +def test_surplus_arguments_error(capsys: pytest.CaptureFixture[str]) -> None: + """A mis-specified ninja rule passing extra operands errors instead of + silently dropping them.""" + with patch.object( + build_tool.sys, "argv", ["build_tool", "copy", "a", "b", "extra"] + ): + assert build_tool.main() == 1 + assert "expected 2 arguments, got 3" in capsys.readouterr().err + + +def test_copy_same_file_keeps_the_input( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A same-file copy (dst IS src) must not unlink the input, and fails + with a message and exit code like the other shim paths.""" + src = tmp_path / "firmware.bin" + src.write_bytes(b"image") + with patch.object( + build_tool.sys, "argv", ["build_tool", "copy", str(src), str(src)] + ): + assert build_tool.main() == 1 + assert src.read_bytes() == b"image" + assert "failed" in capsys.readouterr().err + + +def test_copy_failure_leaves_no_partial_output(tmp_path: Path) -> None: + """A failed copy unlinks the destination; a partial firmware image must + never be left on disk.""" + dst = tmp_path / "firmware.factory.bin" + dst.write_text("stale") + with ( + patch.object(build_tool.shutil, "copyfile", side_effect=OSError("disk full")), + patch.object( + build_tool.sys, + "argv", + ["build_tool", "copy", str(tmp_path / "src.bin"), str(dst)], + ), + ): + assert build_tool.main() == 1 + assert not dst.exists() diff --git a/tests/unit_tests/test_arduino_library.py b/tests/unit_tests/test_arduino_library.py new file mode 100644 index 0000000000..87de28cf32 --- /dev/null +++ b/tests/unit_tests/test_arduino_library.py @@ -0,0 +1,1162 @@ +"""Tests for esphome.arduino.library (Arduino-core library resolution).""" + +from __future__ import annotations + +from contextlib import contextmanager +import json +import logging +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.arduino import library as component +from esphome.const import KEY_CORE, KEY_TARGET_PLATFORM, PLATFORM_ESP8266 +from esphome.core import CORE, EsphomeError, Library +import esphome.platformio.library as pio_library +from esphome.platformio.library import ( + ConvertedLibrary, + IncompatiblePlatform, + InvalidLibrary, + LibraryBackend, +) + + +@pytest.fixture(autouse=True) +def _reset_libraries() -> None: + # conftest's reset_core fixture clears platformio_libraries after each test + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_ESP8266} + + +def _add_library(name: str, version: str | None, repository: str | None = None) -> None: + CORE.add_library(Library(name=name, version=version, repository=repository)) + + +def _make_framework(tmp_path: Path) -> Path: + framework = tmp_path / "framework" + lib = framework / "libraries" / "ESP8266WiFi" / "src" + lib.mkdir(parents=True) + (lib / "ESP8266WiFi.cpp").write_text("") + (lib / "ESP8266WiFi.h").write_text("") + (lib.parent / "library.properties").write_text("name=ESP8266WiFi\nversion=1.0\n") + root_lib = framework / "libraries" / "Wire" + root_lib.mkdir(parents=True) + (root_lib / "Wire.cpp").write_text("") + (root_lib / "examples").mkdir() + (root_lib / "examples" / "scan.ino").write_text("") + return framework + + +@contextmanager +def _emitting_converter(*converted): + """Patch convert_libraries to emit the given components via the backend.""" + + def fake_convert(libraries: list, backend: LibraryBackend) -> list: + assert backend.platform == "espressif8266" + assert backend.framework == "arduino" + assert backend.cache_key == "arduino8266" + for c in converted: + backend.emit(c) + return list(converted) + + with ( + patch.object(component, "convert_libraries", side_effect=fake_convert), + patch.object(component, "apply_extra_script") as mock_extra, + ): + yield mock_extra + + +def _converted(name: str, source_dir: Path, data: dict) -> ConvertedLibrary: + converted = ConvertedLibrary(name, "1.0.0", source=None) + converted.path = source_dir + converted.data = data + return converted + + +def _resolve(framework: Path) -> list[component.ArduinoLibrary]: + return component.resolve_libraries( + framework, + pio_platform="espressif8266", + board_mcu="esp8266", + cache_key="arduino8266", + ) + + +def _webserver(tmp_path: Path, data: dict) -> ConvertedLibrary: + """Register ESPAsyncWebServer and return its converted stand-in.""" + _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6") + lib_dir = tmp_path / "converted" / "webserver" + (lib_dir / "src").mkdir(parents=True) + (lib_dir / "src" / "server.cpp").write_text("") + return _converted("esp32async__ESPAsyncWebServer", lib_dir, data) + + +def _local_lib(tmp_path: Path, dependencies: dict | list) -> None: + """Register a local file:// library declaring the given dependencies.""" + local_lib = tmp_path / "locallib" + (local_lib / "src").mkdir(parents=True) + (local_lib / "src" / "local.cpp").write_text("") + (local_lib / "library.json").write_text( + json.dumps( + {"name": "LocalLib", "version": "1.0.0", "dependencies": dependencies} + ) + ) + # as_uri() forms a valid file:// URL on every platform (file:///C:/... + # on Windows; a bare f-string would embed backslashes) + _add_library(local_lib.as_uri(), None) + + +def _ws_tcp_pair(tmp_path: Path) -> tuple[ConvertedLibrary, ConvertedLibrary]: + """Build ESPAsyncWebServer (depending on ESPAsyncTCP) plus resolved TCP.""" + ws_dir = tmp_path / "converted" / "webserver" + (ws_dir / "src").mkdir(parents=True) + (ws_dir / "src" / "server.cpp").write_text("") + tcp_dir = tmp_path / "converted" / "tcp" + (tcp_dir / "src").mkdir(parents=True) + (tcp_dir / "src" / "tcp.cpp").write_text("") + ws = _converted( + "esp32async__ESPAsyncWebServer", + ws_dir, + {"build": {}, "dependencies": [{"name": "ESPAsyncTCP"}]}, + ) + tcp = _converted("esp32async__ESPAsyncTCP", tcp_dir, {"build": {}}) + return ws, tcp + + +def test_library_info_src_layout(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + lib = component._bundled_library(framework, "ESP8266WiFi") + assert lib.name == "ESP8266WiFi" + assert [p.name for p in lib.sources] == ["ESP8266WiFi.cpp"] + assert lib.include_dirs == [(framework / "libraries/ESP8266WiFi/src").resolve()] + + +def test_library_info_root_layout_excludes_examples(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + lib = component._bundled_library(framework, "Wire") + assert [p.name for p in lib.sources] == ["Wire.cpp"] + assert lib.include_dirs == [(framework / "libraries/Wire").resolve()] + + +def test_library_info_flags_parsing(tmp_path: Path) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "a.cpp").write_text("") + (read_path / "inc").mkdir() + (read_path / "blobs").mkdir() + data = { + "build": { + "flags": [ + "-DFOO=1 -I inc", + "-lalgobsec", + "-fno-lto", + "-Wl,--wrap=malloc", + # Bare flags join their argument within one entry only, as + # ParseFlags lexes each entry independently + "-l m", + "-L blobs", + ], + } + } + lib = component._library_info("x", read_path, data) + assert lib.flags == ["-DFOO=1", "-fno-lto"] + assert lib.include_dirs == [ + (read_path / "src").resolve(), + (read_path / "inc").resolve(), + ] + assert lib.link_dirs == [(read_path / "blobs").resolve()] + assert lib.link_libs == ["algobsec", "m"] + assert lib.link_flags == ["-Wl,--wrap=malloc"] + + +def test_library_info_missing_link_dir_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + read_path.mkdir() + data = {"build": {"flags": ["-Lmissing_blobs"]}} + lib = component._library_info("x", read_path, data) + assert "declares library dir missing_blobs which does not exist" in caplog.text + # Kept anyway: the linker ignores missing -L dirs + assert lib.link_dirs == [(read_path / "missing_blobs").resolve()] + + +def test_library_info_declared_filter_matches_nothing_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + data = {"build": {"srcFilter": ["+"]}} + lib = component._library_info("x", read_path, data) + assert not lib.sources + assert "no source files matched" in caplog.text + + +def test_empty_converted_tree_raises_at_emit(tmp_path: Path) -> None: + """A converted tree with no sources and no headers is a broken download; + fail by name like the bundled case.""" + framework = _make_framework(tmp_path) + _add_library("Some/Empty", "1.0.0") + lib_dir = tmp_path / "converted" / "empty" + (lib_dir / "src").mkdir(parents=True) + converted = _converted("some__Empty", lib_dir, {"build": {}}) + with ( + _emitting_converter(converted), + pytest.raises(EsphomeError, match="no sources or headers; the download"), + ): + _resolve(framework) + + +def test_library_info_no_src_dir(tmp_path: Path) -> None: + read_path = tmp_path / "empty" + read_path.mkdir() + lib = component._library_info("x", read_path, {}) + # With no manifest hints the source dir falls back to the library root + assert lib.sources == [] + assert lib.include_dirs == [read_path.resolve()] + + +def test_resolve_libraries_bundled(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + _add_library("ESP8266WiFi", None) + libs = _resolve(framework) + assert [lib.name for lib in libs] == ["ESP8266WiFi"] + + +@pytest.mark.parametrize("version", [None, "1.1.0"]) +def test_resolve_libraries_registry_name_is_external( + tmp_path: Path, version: str | None +) -> None: + """A name that is not bundled reaches the converter, bare or pinned.""" + framework = _make_framework(tmp_path) + _add_library("pngle", version) + with patch.object(component, "convert_libraries", return_value=[]) as mock_convert: + _resolve(framework) + (libraries, _backend), _ = mock_convert.call_args + assert [lib.name for lib in libraries] == ["pngle"] + + +def test_resolve_libraries_external_and_bundled_deps(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6") + + lib_dir = tmp_path / "converted" / "webserver" + (lib_dir / "src").mkdir(parents=True) + (lib_dir / "src" / "server.cpp").write_text("") + converted = _converted( + "esp32async__ESPAsyncWebServer", + lib_dir, + { + "build": {}, + "dependencies": [ + # Version-less bundled dependency: resolved from the framework + {"name": "Wire", "platforms": "espressif8266"}, + # Wrong platform: skipped + {"name": "ESP8266WiFi", "platforms": "espressif32"}, + # Registry dependency with a version: handled by the converter + {"name": "ESPAsyncTCP", "owner": "ESP32Async", "version": "^2.0.0"}, + # Not bundled: skipped + {"name": "NotBundled"}, + ], + }, + ) + + with _emitting_converter(converted) as mock_extra: + libs = _resolve(framework) + + mock_extra.assert_called_once() + assert mock_extra.call_args.args == (converted,) + assert mock_extra.call_args.kwargs["pio_platform"] == "espressif8266" + # board_mcu is passed lazily, as the shared helper requires + assert mock_extra.call_args.kwargs["board_mcu"]() == "esp8266" + assert [lib.name for lib in libs] == [ + "Wire", + "esp32async__ESPAsyncWebServer", + ] + + +def test_resolve_libraries_bundled_dep_already_present(tmp_path: Path) -> None: + framework = _make_framework(tmp_path) + _add_library("Wire", None) + _add_library("Some/External", "1.0.0") + + lib_dir = tmp_path / "converted" / "external" + lib_dir.mkdir(parents=True) + (lib_dir / "main.cpp").write_text("") + converted = _converted( + "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]} + ) + + with _emitting_converter(converted): + libs = _resolve(framework) + + # Wire appears once (from the explicit registration), not twice + assert [lib.name for lib in libs] == ["Wire", "some__External"] + + +def test_library_info_trailing_bare_flag_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + lib = component._library_info("x", read_path, {"build": {"flags": ["-DA=1 -l"]}}) + assert lib.flags == ["-DA=1"] + assert lib.link_libs == [] + assert "Ignoring trailing '-l'" in caplog.text + + +def test_library_info_missing_explicit_include_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + lib = component._library_info("x", read_path, {"build": {"flags": ["-Inope"]}}) + assert lib.include_dirs == [(read_path / "src").resolve()] + assert "include dir nope which does not exist" in caplog.text + + +def test_library_info_missing_declared_src_dir_raises(tmp_path: Path) -> None: + """An explicitly declared srcDir that does not exist is a manifest error.""" + read_path = tmp_path / "lib" + read_path.mkdir() + with pytest.raises(EsphomeError, match="srcDir 'nosrc' which does not exist"): + component._library_info("x", read_path, {"build": {"srcDir": "nosrc"}}) + + +def test_library_info_missing_declared_include_dir_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + read_path = tmp_path / "lib" + read_path.mkdir() + component._library_info("x", read_path, {"build": {"includeDir": "noinc"}}) + assert "include dir noinc which does not exist" in caplog.text + + +def test_resolve_libraries_lib_ignore_covers_bundled(tmp_path: Path) -> None: + """lib_ignore applies to framework-bundled libraries, as under PlatformIO.""" + framework = _make_framework(tmp_path) + _add_library("ESP8266WiFi", None) + _add_library("Wire", None) + CORE.platformio_options = {"lib_ignore": ["Wire"]} + libs = _resolve(framework) + assert [lib.name for lib in libs] == ["ESP8266WiFi"] + + +def test_resolve_libraries_lib_ignore_covers_bundled_dependencies( + tmp_path: Path, +) -> None: + framework = _make_framework(tmp_path) + _add_library("Some/External", "1.0.0") + CORE.platformio_options = {"lib_ignore": ["Wire"]} + + lib_dir = tmp_path / "converted" / "external" + lib_dir.mkdir(parents=True) + (lib_dir / "main.cpp").write_text("") + converted = _converted( + "some__External", lib_dir, {"dependencies": [{"name": "Wire"}]} + ) + + with _emitting_converter(converted): + libs = _resolve(framework) + + assert [lib.name for lib in libs] == ["some__External"] + + +def test_bundled_library_prefers_library_json(tmp_path: Path) -> None: + """A bundled library.json wins over library.properties (PIO semantics); + its build section is honored.""" + framework = _make_framework(tmp_path) + lib_dir = framework / "libraries" / "GDBStub" + (lib_dir / "custom").mkdir(parents=True) + (lib_dir / "custom" / "gdb.cpp").write_text("") + (lib_dir / "library.properties").write_text("name=GDBStub\n") + (lib_dir / "library.json").write_text( + '{"name": "GDBStub", "build": {"srcDir": "custom"}}' + ) + lib = component._bundled_library(framework, "GDBStub") + assert [s.name for s in lib.sources] == ["gdb.cpp"] + + +def test_library_info_lib_archive_flag(tmp_path: Path) -> None: + """Both libArchive (library.json) and dot_a_linkage (properties) reach + the generator's contract; default is archive.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + assert component._library_info("x", read_path, {}).lib_archive is True + assert ( + component._library_info( + "x", read_path, {"build": {"libArchive": False}} + ).lib_archive + is False + ) + assert ( + component._library_info("x", read_path, {"dot_a_linkage": "false"}).lib_archive + is False + ) + assert ( + component._library_info("x", read_path, {"dot_a_linkage": "true"}).lib_archive + is True + ) + + +def test_resolve_libraries_dep_warnings( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A nameless dependency entry warns in the shared normalizer; an + owner-without-version entry is left to the walk's reconciliation.""" + framework = _make_framework(tmp_path) + converted = _webserver( + tmp_path, + { + "build": {}, + "dependencies": [ + {"owner": "someone"}, + {"name": "Orphan", "owner": "someone"}, + ], + }, + ) + with _emitting_converter(converted): + _resolve(framework) + assert "Ignoring unrecognized dependency entry" in caplog.text + assert "Orphan" not in caplog.text + + +def test_bundled_dependency_nonplatform_rejection_is_silent_here( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The shared walk owns the rejection warning; the backend-side filter + stays at debug so one manifest fault never warns twice.""" + framework = _make_framework(tmp_path) + converted = _webserver(tmp_path, {"build": {}, "dependencies": [{"name": "Wire"}]}) + with ( + _emitting_converter(converted), + patch.object( + component, + "check_library_data", + side_effect=InvalidLibrary("manifest is corrupt"), + ), + ): + libs = _resolve(framework) + assert "Wire" not in [lib.name for lib in libs] + assert "manifest is corrupt" not in caplog.text + + +def test_nonplatform_rejection_warns_once_through_real_converter( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """One manifest fault produces exactly one warning across the walk and + the backend-side bundled filter.""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, [{"name": "Wire"}]) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + real = pio_library.check_library_data + + def flaky(data, platform, framework_name): + if data.get("name") == "Wire": + raise InvalidLibrary("manifest is corrupt") + return real(data, platform, framework_name) + + monkeypatch.setattr(pio_library, "check_library_data", flaky) + monkeypatch.setattr(component, "check_library_data", flaky) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + _resolve(framework) + assert caplog.text.count("manifest is corrupt") == 1 + + +def test_url_pinned_bundled_name_not_doubled(tmp_path: Path) -> None: + """A URL-pinned dependency names one specific source; the bundled copy + of the same short name must never be added on top of the fork.""" + framework = _make_framework(tmp_path) + converted = _webserver( + tmp_path, + { + "build": {}, + "dependencies": [ + {"name": "Wire", "version": "https://github.com/x/wire-fork.git"} + ], + }, + ) + with _emitting_converter(converted): + libs = _resolve(framework) + assert "Wire" not in [lib.name for lib in libs] + + +def test_versioned_bundled_candidate_fault_warns_once( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A versioned bundled-name dependency with a manifest fault warns once, + from the walk's usability filter; the backend-side re-check stays quiet.""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, [{"name": "Wire", "version": "*"}]) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + real = pio_library.check_library_data + + def flaky(data, platform, framework_name): + if data.get("name") == "Wire": + raise InvalidLibrary("manifest is corrupt") + return real(data, platform, framework_name) + + monkeypatch.setattr(pio_library, "check_library_data", flaky) + monkeypatch.setattr(component, "check_library_data", flaky) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + libs = _resolve(framework) + assert "Wire" not in [lib.name for lib in libs] + assert caplog.text.count("manifest is corrupt") == 1 + + +def test_short_name_collision_with_bundled_name_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Suppressing a genuinely bundled name on a short-name match warns; + an accidental collision would otherwise surface at link.""" + framework = _make_framework(tmp_path) + _add_library("Someone/Wire", "1.0.0") + converted = _converted( + "someone__Wire", + tmp_path / "conv", + {"build": {}, "dependencies": [{"name": "Wire"}]}, + ) + (tmp_path / "conv" / "src").mkdir(parents=True) + (tmp_path / "conv" / "src" / "a.cpp").write_text("") + with _emitting_converter(converted): + libs = _resolve(framework) + assert "Wire" not in [lib.name for lib in libs] + assert "assumed satisfied by a requested external library" in caplog.text + assert any(r.levelname == "WARNING" for r in caplog.records) + + +def test_missing_libraries_dir_is_a_broken_install(tmp_path: Path) -> None: + """A framework tree without libraries/ must fail by name, not silently + reroute every bundled name to the registry.""" + framework = tmp_path / "framework" + framework.mkdir() + _add_library("Wire", None) + with pytest.raises(EsphomeError, match="framework install may be incomplete"): + _resolve(framework) + + +def test_provided_is_case_sensitive(tmp_path: Path) -> None: + """Membership uses the exact on-disk names, so a case-insensitive + filesystem cannot add the same bundled library twice.""" + framework = _make_framework(tmp_path) + converted = _webserver(tmp_path, {"build": {}, "dependencies": [{"name": "wire"}]}) + with _emitting_converter(converted): + libs = _resolve(framework) + assert "wire" not in [lib.name for lib in libs] + assert "Wire" not in [lib.name for lib in libs] + + +@pytest.mark.parametrize("declared", ["", None]) +def test_library_info_falsy_declared_src_dir_raises( + tmp_path: Path, declared: str | None +) -> None: + """A declared-but-falsy srcDir must not silently fall back to the probe.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + with pytest.raises(EsphomeError, match="does not exist"): + component._library_info("x", read_path, {"build": {"srcDir": declared}}) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (False, False), + ("false", False), + ("False", False), + ("true", True), + ], +) +def test_library_info_lib_archive_parse( + tmp_path: Path, + value: object, + expected: bool, +) -> None: + """bool("false") is True; the string forms must parse, not coerce.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + lib = component._library_info("x", read_path, {"build": {"libArchive": value}}) + assert lib.lib_archive is expected + + +def test_library_info_unsupported_link_fields_raise(tmp_path: Path) -> None: + """precompiled/ldflags properties are not supported; refuse by name.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + with pytest.raises(EsphomeError, match="declares precompiled"): + component._library_info("x", read_path, {"precompiled": "true", "build": {}}) + with pytest.raises(EsphomeError, match="declares ldflags"): + component._library_info("x", read_path, {"ldflags": "-lfoo", "build": {}}) + + +@pytest.mark.parametrize("value", ["false", "False", " false ", "", False, None]) +def test_library_info_precompiled_opt_out_accepted( + tmp_path: Path, value: object +) -> None: + """Manifest values are strings; precompiled=false is the spec's + explicit opt-out, not a declaration.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + data = {"build": {}} + if value is not None: + data["precompiled"] = value + component._library_info("x", read_path, data) + + +@pytest.mark.parametrize("value", ["full", True, "weird"]) +def test_library_info_precompiled_set_raises(tmp_path: Path, value: object) -> None: + """Both full (Arduino's other legal value) and unknown spellings fail safe.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + with pytest.raises(EsphomeError, match="declares precompiled"): + component._library_info("x", read_path, {"precompiled": value, "build": {}}) + + +def test_library_info_default_filter_matching_nothing_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The empty-match warning is not gated on a declared srcFilter/srcDir; + a default-filter src/ holding only inert files warns too.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "keywords.txt").write_text("") + lib = component._library_info("x", read_path, {"build": {}}) + assert not lib.sources + assert "no source files matched" in caplog.text + + +def test_library_info_unmapped_sources_warn( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Source-like files the case-sensitive suffix map rejects are named, + even when other sources compiled (a partial drop links with undefined + symbols far from the cause).""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "impl.CPP").write_text("") + (read_path / "src" / "sketch.ino").write_text("") + (read_path / "src" / "ok.cpp").write_text("") + lib = component._library_info("x", read_path, {"build": {}}) + assert [s.name for s in lib.sources] == ["ok.cpp"] + assert "not compiled: impl.CPP, sketch.ino" in caplog.text + + +def test_library_info_inert_only_filter_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A declared srcFilter matching only inert files (no sources, no + headers) warns like one matching nothing at all.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "keywords.txt").write_text("") + component._library_info("x", read_path, {"build": {"srcFilter": ["+<*>"]}}) + assert "no source files matched" in caplog.text + + +def test_library_info_declared_filter_matching_headers_stays_quiet( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A declared filter matching real headers is a header-only library.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "api.h").write_text("") + component._library_info("x", read_path, {"build": {"srcFilter": ["+<*>"]}}) + assert "no source files matched" not in caplog.text + + +def test_library_info_header_only_src_stays_quiet( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A header-only library (real headers in src/) is routine, not a + warning (the default +<*> filter matches the headers too).""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "ArduinoJson.h").write_text("") + (read_path / "keywords.txt").write_text("") + lib = component._library_info("x", read_path, {"build": {}}) + assert lib.sources == [] + assert "not compiled" not in caplog.text + assert "srcFilter" not in caplog.text + + +def test_library_info_lib_archive_malformed_raises(tmp_path: Path) -> None: + """A typo'd libArchive fails by name like the other build fields.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + with pytest.raises(EsphomeError, match="malformed libArchive value 'archive-me'"): + component._library_info("x", read_path, {"build": {"libArchive": "archive-me"}}) + + +def test_bundled_dependency_dict_shorthand_prefers_bundled(tmp_path: Path) -> None: + """The {"Wire": "*"} dict shorthand resolves to the bundled library.""" + framework = _make_framework(tmp_path) + converted = _webserver(tmp_path, {"build": {}, "dependencies": {"Wire": "*"}}) + with _emitting_converter(converted): + libs = _resolve(framework) + assert "Wire" in [lib.name for lib in libs] + + +def test_unfulfilled_provides_promise_raises(tmp_path: Path) -> None: + """A provides()-skipped dependency nothing added can only surface as + undefined symbols at link, so it fails here by name; satisfied ones + pass silently.""" + with pytest.raises(EsphomeError, match="Wire") as err: + component._check_unfulfilled_provides( + {"Wire", "Hash"}, {"Hash"}, {"Wire", "Hash"} + ) + assert str(err.value).count("Wire") == 1 + assert "Hash" not in str(err.value) + component._check_unfulfilled_provides({"Hash"}, {"Hash"}, {"Hash"}) + # A recording for a since-re-resolved manifest is stale walk state, + # never a failure: no final manifest still requests Wire + component._check_unfulfilled_provides({"Wire"}, set(), set()) + + +def test_extra_script_link_flags_reach_the_library(tmp_path: Path) -> None: + """LINKFLAGS captured by an extra script travel outside build.flags and + must reach the library's link flags, matching the ESP-IDF backend.""" + framework = _make_framework(tmp_path) + converted = _webserver( + tmp_path, + { + "build": {}, + component.ESPHOME_DATA_KEY: { + component.ESPHOME_DATA_LINK_FLAGS_KEY: ["-Wl,--wrap=foo"] + }, + }, + ) + with _emitting_converter(converted): + libs = _resolve(framework) + (webserver,) = (lib for lib in libs if "ESPAsyncWebServer" in lib.name) + assert "-Wl,--wrap=foo" in webserver.link_flags + + +def test_bundled_dependency_platform_rejection_is_debug( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The typed IncompatiblePlatform (the routine cross-platform skip) + stays at debug regardless of message wording.""" + framework = _make_framework(tmp_path) + converted = _webserver(tmp_path, {"build": {}, "dependencies": [{"name": "Wire"}]}) + with ( + _emitting_converter(converted), + patch.object( + component, + "check_library_data", + side_effect=IncompatiblePlatform("nothing about the p-word here"), + ), + ): + libs = _resolve(framework) + assert "Wire" not in [lib.name for lib in libs] + assert "Skipping dependency Wire" not in caplog.text + + +@pytest.mark.parametrize("data", [{"build": "src"}, [], "nope"]) +def test_library_info_malformed_manifest_is_named(tmp_path: Path, data: object) -> None: + """A malformed manifest names the library, never an AttributeError.""" + read_path = tmp_path / "lib" + read_path.mkdir() + with pytest.raises(EsphomeError, match="Library x has a malformed manifest"): + component._library_info("x", read_path, data) + + +def test_bundled_library_with_declared_dependencies_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A bundled manifest that declares dependencies is visible, not + silently skipped (a no-op for the ESP8266 core, not for every core).""" + framework = _make_framework(tmp_path) + wire = framework / "libraries" / "Wire" + (wire / "library.json").write_text( + '{"name": "Wire", "dependencies": [{"name": "SPI"}]}' + ) + _add_library("Wire", None) + _resolve(framework) + assert "Bundled library Wire declares dependencies" in caplog.text + + +@pytest.mark.parametrize( + ("build", "match"), + [ + ({"includeDir": ["a", "b"]}, "malformed includeDir"), + ({"srcFilter": [123]}, "malformed srcFilter"), + ], +) +def test_library_info_malformed_build_fields_are_named( + tmp_path: Path, build: dict, match: str +) -> None: + """Malformed includeDir/srcFilter fail naming the library like srcDir.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + with pytest.raises(EsphomeError, match=match): + component._library_info("x", read_path, {"build": build}) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("true", True), + ("False", False), + ], +) +def test_library_info_dot_a_linkage_parses_strictly( + tmp_path: Path, + value: str, + expected: bool, +) -> None: + """The dot_a_linkage property uses the same strict table as libArchive.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + lib = component._library_info("x", read_path, {"dot_a_linkage": value, "build": {}}) + assert lib.lib_archive is expected + + +def test_library_info_dot_a_linkage_malformed_raises(tmp_path: Path) -> None: + """A typo'd dot_a_linkage must not silently flip link semantics.""" + read_path = tmp_path / "lib" + (read_path / "src").mkdir(parents=True) + (read_path / "src" / "stub.cpp").write_text("") + with pytest.raises(EsphomeError, match="malformed dot_a_linkage value 'yes'"): + component._library_info("x", read_path, {"dot_a_linkage": "yes", "build": {}}) + + +def test_bundled_library_properties_depends_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The library.properties depends= spelling reaches the visibility + warning too; the shared parser returns it raw.""" + framework = _make_framework(tmp_path) + wire = framework / "libraries" / "Wire" + (wire / "library.properties").write_text("name=Wire\nversion=1.0\ndepends=SPI\n") + _add_library("Wire", None) + caplog.set_level("INFO") + _resolve(framework) + assert "Library Wire declares dependencies via library.properties" in caplog.text + + +def test_bundled_library_extra_script_raises(tmp_path: Path) -> None: + """A bundled manifest relying on an extraScript would miscompile; + refuse by name.""" + framework = _make_framework(tmp_path) + wire = framework / "libraries" / "Wire" + (wire / "library.json").write_text( + '{"name": "Wire", "build": {"extraScript": "extra.py"}}' + ) + _add_library("Wire", None) + with pytest.raises(EsphomeError, match="Wire declares an extraScript"): + _resolve(framework) + + +def test_dependency_requested_top_level_is_not_a_drop( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A version-less manifest dependency the config separately requests is + already in the build; it is not probed as a bundled library.""" + framework = _make_framework(tmp_path) + _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6") + _add_library("ESP32Async/ESPAsyncTCP", "2.0.0") + ws, tcp = _ws_tcp_pair(tmp_path) + with _emitting_converter(ws, tcp): + libs = _resolve(framework) + # Exactly the two converted libraries; no bundled stand-in was added + assert [lib.name for lib in libs] == [ + "esp32async__ESPAsyncWebServer", + "esp32async__ESPAsyncTCP", + ] + assert "Skipping" not in caplog.text + + +def test_bundled_library_non_dict_manifest_skips_probes_and_raises( + tmp_path: Path, +) -> None: + """A bundled library.json that is a JSON array skips the dependency and + extraScript probes and fails in _library_info naming the library.""" + framework = _make_framework(tmp_path) + wire = framework / "libraries" / "Wire" + (wire / "library.json").write_text('["not", "a", "manifest"]') + with pytest.raises(EsphomeError, match="Library Wire has a malformed manifest"): + component._bundled_library(framework, "Wire") + + +def test_bundled_missing_manifest_is_debug_only( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The legacy manifest-less layout is legal (the core ships FSTools + without one), so the diagnostic must stay below warning level.""" + framework = _make_framework(tmp_path) + with caplog.at_level(logging.DEBUG): + component._bundled_library(framework, "Wire") + record = next(r for r in caplog.records if "has no manifest" in r.message) + assert record.levelno == logging.DEBUG + + +def test_bundled_corrupt_library_json_fails_by_name(tmp_path: Path) -> None: + """A truncated bundled library.json fails with the library name and the + clean-all hint, not a raw JSONDecodeError.""" + framework = _make_framework(tmp_path) + (framework / "libraries" / "Wire" / "library.json").write_text("{truncated") + with pytest.raises(EsphomeError, match="Wire has a corrupt library.json"): + component._bundled_library(framework, "Wire") + + +def test_dict_shorthand_dependency_skips_registry_through_real_converter( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """{"Wire": "*"} resolves to the bundled copy without touching the + registry (real converter).""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, {"Wire": "*"}) + # Pin the component cache to tmp_path (data_dir honors an ambient + # ESPHOME_DATA_DIR otherwise) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + libs = _resolve(framework) + names = [lib.name for lib in libs] + assert "Wire" in names + assert any("locallib" in n.lower() for n in names) + # The walk populated provided_requests for the skip; the backend added + # the bundled copy, so the reconciliation passed without raising + + +def test_versionless_provides_skip_is_reconciled_through_real_converter( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A truly version-less bare-name dependency the walk skips on the + backend's promise is recorded and fulfilled by the bundled copy.""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, ["Wire"]) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + libs = _resolve(framework) + assert "Wire" in [lib.name for lib in libs] + + +def test_platform_filtered_bundled_candidate_does_not_break_reconciliation( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A bundled candidate the backend knowingly skips (platform filter) + counts as satisfied; the promise reconciliation must not raise.""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, [{"name": "Wire", "platforms": ["espressif32"]}]) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + libs = _resolve(framework) + assert "Wire" not in [lib.name for lib in libs] + + +@pytest.mark.parametrize( + ("bad_name", "message"), + [ + # A non-string name never leaves the shared normalizer + (1, "Ignoring unrecognized dependency entry"), + ("../escape", "Ignoring malformed dependency entry"), + ("..", "Ignoring malformed dependency entry"), + ], +) +def test_bundled_dependency_bad_name_is_malformed( + tmp_path: Path, bad_name: object, message: str, caplog: pytest.LogCaptureFixture +) -> None: + """A dependency name becomes a path component; a traversal or a + non-string is a malformed entry, never joined.""" + framework = _make_framework(tmp_path) + converted = _webserver( + tmp_path, {"build": {}, "dependencies": [{"name": bad_name}]} + ) + with _emitting_converter(converted): + _resolve(framework) + assert message in caplog.text + + +def test_owner_qualified_dependency_is_silent( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The owner-qualified dependency spelling (PIO's Owner/Pkg) resolves via + the converter; it must not draw the malformed-entry warning.""" + framework = _make_framework(tmp_path) + converted = _webserver( + tmp_path, + { + "build": {}, + "dependencies": [{"name": "ESP32Async/AsyncTCP", "version": "^3.0"}], + }, + ) + with _emitting_converter(converted): + _resolve(framework) + assert "malformed" not in caplog.text + + +def test_bundled_dependency_string_list_form(tmp_path: Path) -> None: + """The bare string-list dependency form (PIO-legal) resolves to the + bundled library instead of vanishing in normalization.""" + framework = _make_framework(tmp_path) + converted = _webserver(tmp_path, {"build": {}, "dependencies": ["Wire"]}) + with _emitting_converter(converted): + libs = _resolve(framework) + assert "Wire" in [lib.name for lib in libs] + + +def test_pinned_bundled_dependency_substitution_warns( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A non-* version pin on a backend-provided dependency is discarded + for the bundled copy; the substitution must be visible.""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, {"Wire": "^2.0.0"}) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + libs = _resolve(framework) + assert "Wire" in [lib.name for lib in libs] + assert "pins version ^2.0.0; using the library bundled" in caplog.text + + +def test_transitively_resolved_dependency_does_not_warn( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A dependency the walk already resolved does not warn.""" + framework = _make_framework(tmp_path) + _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6") + ws, tcp = _ws_tcp_pair(tmp_path) + with _emitting_converter(ws, tcp): + libs = _resolve(framework) + assert [lib.name for lib in libs] == [ + "esp32async__ESPAsyncWebServer", + "esp32async__ESPAsyncTCP", + ] + assert "Skipping" not in caplog.text + + +@pytest.mark.parametrize( + ("spec", "expected"), + [ + ("owner/Name", "Name"), + ("Name", "Name"), + ("Foo=file:///srv/Wire", "Foo"), + ("Foo=https://github.com/x/Wire", "Foo"), + # An "=" without a URL is a registry name, not the custom-name form + ("FOO=BAR", "FOO=BAR"), + ("https://github.com/x/Wire", "Wire"), + # Git tails are stripped like the walk's URL normalization + ("https://github.com/x/Wire.git", "Wire"), + ("git+https://github.com/x/Wire.git#v1", "Wire"), + ], +) +def test_external_short_name(spec: str, expected: str) -> None: + assert component._external_short_name(spec) == expected + + +def test_converted_manifest_name_suppresses_bundled_dependency( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A name a converted library's manifest provides is not also added + from the framework tree, even when the provider emits later; the + suppression warns like its external_short_names twin.""" + framework = _make_framework(tmp_path) + _add_library("ESP32Async/ESPAsyncWebServer", "3.9.6") + # Requested under a different short name; only the manifest says "Wire" + _add_library("Someone/WireLib", "9.9.9") + ws_dir = tmp_path / "converted" / "webserver" + (ws_dir / "src").mkdir(parents=True) + (ws_dir / "src" / "stub.cpp").write_text("") + wire_dir = tmp_path / "converted" / "wire" + (wire_dir / "src").mkdir(parents=True) + (wire_dir / "src" / "wire.cpp").write_text("") + ws = _converted( + "esp32async__ESPAsyncWebServer", + ws_dir, + {"build": {}, "dependencies": [{"name": "Wire"}]}, + ) + registry_wire = _converted( + "someone__WireLib", wire_dir, {"name": "Wire", "build": {}} + ) + with _emitting_converter(ws, registry_wire): + libs = _resolve(framework) + # The bundled Wire is not added alongside the registry-resolved one + assert [lib.name for lib in libs] == [ + "esp32async__ESPAsyncWebServer", + "someone__WireLib", + ] + assert "Dependency Wire is assumed satisfied by a converted" in caplog.text + + +def test_bundled_library_root_headers_pass_the_probe(tmp_path: Path) -> None: + """Headers anywhere in the bundled tree (uncommon suffixes and case + included) prove the install is intact, even with an empty src dir.""" + framework = _make_framework(tmp_path) + lib_dir = framework / "libraries" / "HeaderOnly" + (lib_dir / "src").mkdir(parents=True) + (lib_dir / "impl.HXX").write_text("") + lib = component._bundled_library(framework, "HeaderOnly") + assert lib.sources == [] + + +def test_empty_bundled_library_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A bundled directory with no sources or headers is a broken install + that can never link; fail by name instead of warning into it.""" + framework = _make_framework(tmp_path) + (framework / "libraries" / "Empty").mkdir() + _add_library("Empty", None) + with pytest.raises(EsphomeError, match="Library Empty has no sources or headers"): + _resolve(framework) + + +def test_versionless_dependency_with_provider_stays_quiet( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """With a provides backend the version-less skip is routine (debug) and + the bundled copy is picked up after emit.""" + framework = _make_framework(tmp_path) + _local_lib(tmp_path, [{"name": "Wire"}]) + monkeypatch.setenv("ESPHOME_DATA_DIR", str(tmp_path / ".esphome")) + with patch.object( + pio_library, + "_resolve_registry_version", + side_effect=AssertionError("registry touched"), + ): + libs = _resolve(framework) + assert "Wire" in [lib.name for lib in libs] + assert "has no version to resolve" not in caplog.text diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 0c873dc3fe..0a16b118fc 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -10,7 +10,7 @@ from pathlib import Path import pytest -from esphome.core import EsphomeError, Library +from esphome.core import CORE, EsphomeError, Library import esphome.platformio.library as lib from esphome.platformio.library import ( SOURCE_KIND_FOR_SUFFIX, @@ -29,9 +29,13 @@ from esphome.platformio.library import ( ) -def _backend(emit=lambda component: None) -> LibraryBackend: +def _backend(emit=lambda component: None, provides=None) -> LibraryBackend: return LibraryBackend( - platform="espressif32", framework="espidf", emit=emit, cache_key="idf" + platform="espressif32", + framework="espidf", + emit=emit, + cache_key="idf", + provides=provides, ) @@ -952,3 +956,202 @@ def test_source_kind_map_shape() -> None: assert SOURCE_KIND_FOR_SUFFIX[".S"] == "aspp" assert SOURCE_KIND_FOR_SUFFIX[".c"] == "c" assert SOURCE_KIND_FOR_SUFFIX[".cpp"] == "cxx" + # SCons's case-sensitive C++ suffixes: PIO compiles .C as C++ + assert SOURCE_KIND_FOR_SUFFIX[".C"] == "cxx" + assert SOURCE_KIND_FOR_SUFFIX[".C++"] == "cxx" + + +def test_versionless_platform_filtered_dependency_stays_quiet( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A version-less dependency the platform filter excludes is + deliberately absent, not a drop to warn about.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + { + "esphome/A": { + "name": "A", + "dependencies": [{"name": "Hash", "platforms": "espressif8266"}], + } + }, + ) + convert_libraries([Library("esphome/A", None, None)], _backend()) + assert "has no version to resolve" not in caplog.text + + +def test_versionless_ignored_dependency_stays_quiet( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A lib_ignore'd version-less dependency is deliberately excluded, not + a drop; no reconciliation warning.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + {"esphome/A": {"name": "A", "dependencies": [{"name": "Hash"}]}}, + ) + CORE.platformio_options = {"lib_ignore": ["Hash"]} + convert_libraries([Library("esphome/A", None, None)], _backend()) + assert "has no version to resolve" not in caplog.text + + +def test_versionless_dependency_without_provider_warns( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A backend whose tree could supply the name warns on the drop; one + without provides() can never act on it, so it stays at debug.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + { + "esphome/A": { + "name": "A", + # The duplicate entry warns once (reconciliation dedup) + "dependencies": [{"name": "Hash"}, {"name": "Hash"}], + } + }, + ) + convert_libraries( + [Library("esphome/A", None, None)], _backend(provides=lambda name: False) + ) + assert ( + caplog.text.count( + "Hash of esphome/A has no version to resolve and nothing provides it" + ) + == 1 + ) + caplog.clear() + with caplog.at_level(logging.DEBUG): + convert_libraries([Library("esphome/A", None, None)], _backend()) + records = [ + r + for r in caplog.records + if "has no version to resolve and nothing provides it" in r.message + ] + assert records and all(r.levelno == logging.DEBUG for r in records) + + +def test_url_version_dependency_is_not_substituted_by_provides( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A URL-valued version names one specific source; the backend-provided + skip must not replace it with the bundled copy.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + { + "esphome/A": { + "name": "A", + "dependencies": [ + {"name": "Hash", "version": "https://github.com/o/Hash.git"} + ], + }, + "o/Hash": {"name": "Hash"}, + }, + ) + emitted: list[str] = [] + convert_libraries( + [Library("esphome/A", "1.0.0", None)], + _backend(emit=lambda c: emitted.append(c.name), provides=lambda name: True), + ) + assert "Skip backend-provided" not in caplog.text + assert "using the library bundled" not in caplog.text + assert any("o/hash" in n.lower() for n in emitted) + + +def test_versionless_owner_qualified_dependency_warns_despite_provides( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """An owner-qualified version-less dependency is not satisfied by + provides(); it must still warn.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + { + "esphome/A": { + "name": "A", + "dependencies": [{"name": "Wire", "owner": "Foo"}], + } + }, + ) + convert_libraries( + [Library("esphome/A", None, None)], + _backend(provides=lambda name: name == "Wire"), + ) + assert "Wire of esphome/A has no version to resolve" in caplog.text + + +def test_versionless_provided_dependency_stays_quiet( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """An owner-less version-less dependency the backend provides is added + by the backend after emit; no reconciliation warning.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + {"esphome/A": {"name": "A", "dependencies": [{"name": "Wire"}]}}, + ) + convert_libraries( + [Library("esphome/A", None, None)], + _backend(provides=lambda name: name == "Wire"), + ) + assert "has no version to resolve" not in caplog.text + + +def test_versionless_dependency_requested_top_level_stays_quiet( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A version-less dependency the config also requests top-level is in + the build; no drop warning even without a provides backend.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + { + "esphome/A": {"name": "A", "dependencies": [{"name": "Hash"}]}, + "Hash": {"name": "Hash"}, + }, + ) + convert_libraries( + [Library("esphome/A", None, None), Library("Hash", None, None)], + _backend(), + ) + assert "has no version to resolve" not in caplog.text + + +def test_versionless_url_ish_dependency_name_warns_cleanly( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A malformed URL-ish dependency name falls to the drop warning, never + a RuntimeError out of the key parser.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + {"esphome/A": {"name": "A", "dependencies": [{"name": "file://"}]}}, + ) + convert_libraries( + [Library("esphome/A", None, None)], _backend(provides=lambda name: False) + ) + assert ( + "file:// of esphome/A has no version to resolve and nothing provides it" + in caplog.text + ) + + +def test_versionless_dependency_matching_resolved_manifest_name_stays_quiet( + tmp_path, monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """A bare name satisfied by an owner-qualified component's manifest + name is not a drop.""" + _patch_download_with_manifests( + monkeypatch, + tmp_path, + { + "esphome/A": {"name": "A", "dependencies": [{"name": "B"}]}, + "esphome/B": {"name": "B"}, + }, + ) + convert_libraries( + [Library("esphome/A", None, None), Library("esphome/B", None, None)], + _backend(), + ) + assert "has no version to resolve" not in caplog.text From 246670e22b5d124e809f48158fe07525390e530f Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 12:54:31 -0700 Subject: [PATCH 009/147] [modbus] Add a compile-time register value decoder (#18863) --- esphome/components/modbus/modbus_helpers.h | 45 +++++++++++++++++++ .../components/modbus/modbus_helpers_test.cpp | 40 +++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/esphome/components/modbus/modbus_helpers.h b/esphome/components/modbus/modbus_helpers.h index a070ce250c..486064da01 100644 --- a/esphome/components/modbus/modbus_helpers.h +++ b/esphome/components/modbus/modbus_helpers.h @@ -473,6 +473,51 @@ inline int64_t payload_to_number(const std::vector &data, SensorValueTy */ std::optional registers_to_number(const uint16_t *registers, size_t count, SensorValueType sensor_value_type); +/// Combine two register words into a 32-bit value. +constexpr uint32_t registers_to_uint32(uint16_t high_word, uint16_t low_word) { + return (static_cast(high_word) << 16) | low_word; +} + +// Always false, whatever the type: it exists only to make the static_assert below depend on the +// template argument. Not a queryable trait. +template inline constexpr bool VALUE_TYPE_SUPPORTED = false; + +/** Decode one value whose type is known at compile time, from registers in host byte order. + * Unlike registers_to_number(), the type is a template argument, so only the one decode is compiled + * and the caller gets the value's natural type back rather than an int64_t. The "_R" types take the + * low word first; the rest take the high word first. + * Supports the WORD, DWORD and FP32 types, including their _S and _R forms; the QWORD types are + * out of scope and fail to compile, so use registers_to_number() for those. + * Use register_width_for() for the number of registers the caller must supply. + * Note that the FP32 branches are only usable in a constant expression where std::bit_cast is + * available; elsewhere bit_cast falls back to a non-constexpr memcpy (see core/helpers.h). + */ +template constexpr auto registers_to_value(const uint16_t *registers) { + if constexpr (VALUE_TYPE == SensorValueType::U_WORD) { + return registers[0]; + } else if constexpr (VALUE_TYPE == SensorValueType::S_WORD) { + return static_cast(registers[0]); + } else if constexpr (VALUE_TYPE == SensorValueType::U_WORD_S) { + return byteswap(registers[0]); + } else if constexpr (VALUE_TYPE == SensorValueType::S_WORD_S) { + return static_cast(byteswap(registers[0])); + } else if constexpr (VALUE_TYPE == SensorValueType::U_DWORD) { + return registers_to_uint32(registers[0], registers[1]); + } else if constexpr (VALUE_TYPE == SensorValueType::U_DWORD_R) { + return registers_to_uint32(registers[1], registers[0]); + } else if constexpr (VALUE_TYPE == SensorValueType::S_DWORD) { + return static_cast(registers_to_uint32(registers[0], registers[1])); + } else if constexpr (VALUE_TYPE == SensorValueType::S_DWORD_R) { + return static_cast(registers_to_uint32(registers[1], registers[0])); + } else if constexpr (VALUE_TYPE == SensorValueType::FP32) { + return bit_cast(registers_to_uint32(registers[0], registers[1])); + } else if constexpr (VALUE_TYPE == SensorValueType::FP32_R) { + return bit_cast(registers_to_uint32(registers[1], registers[0])); + } else { + static_assert(VALUE_TYPE_SUPPORTED, "registers_to_value() does not support this value type"); + } +} + /// The widest standard numeric value (a QWORD) spans 4 registers, so one entity value never writes more. static constexpr uint16_t MAX_FEW_REGISTERS = 4; diff --git a/tests/components/modbus/modbus_helpers_test.cpp b/tests/components/modbus/modbus_helpers_test.cpp index 87af49710f..21c264ea69 100644 --- a/tests/components/modbus/modbus_helpers_test.cpp +++ b/tests/components/modbus/modbus_helpers_test.cpp @@ -432,6 +432,46 @@ TEST(ModbusHelpersTest, RegistersToNumberRejectsTruncatedMultiRegisterValue) { EXPECT_FALSE(registers_to_number(registers, 1, SensorValueType::U_DWORD).has_value()); } +// --- registers_to_value ---------------------------------------------------- +// The compile-time decoder must agree with the runtime one for every type it supports, +// so the two implementations cannot drift apart. + +template void expect_matches_registers_to_number(const uint16_t *registers) { + const auto expected = registers_to_number(registers, register_width_for(VALUE_TYPE), VALUE_TYPE); + // Plain control flow rather than ASSERT_TRUE: the optional analysis does not see through the macro. + if (!expected.has_value()) { + ADD_FAILURE() << "registers_to_number() returned no value for value_type=" << static_cast(VALUE_TYPE); + return; + } + const int64_t number = expected.value(); + if constexpr (VALUE_TYPE == SensorValueType::FP32 || VALUE_TYPE == SensorValueType::FP32_R) { + EXPECT_FLOAT_EQ(registers_to_value(registers), bit_cast(static_cast(number))) + << "value_type=" << static_cast(VALUE_TYPE); + } else { + EXPECT_EQ(static_cast(registers_to_value(registers)), number) + << "value_type=" << static_cast(VALUE_TYPE); + } +} + +TEST(ModbusHelpersTest, RegistersToValueMatchesRegistersToNumber) { + // A high bit in each word exercises sign handling and word order together. + const uint16_t registers[] = {0x8001, 0xFE02}; + expect_matches_registers_to_number(registers); + expect_matches_registers_to_number(registers); + expect_matches_registers_to_number(registers); + expect_matches_registers_to_number(registers); + expect_matches_registers_to_number(registers); + expect_matches_registers_to_number(registers); + expect_matches_registers_to_number(registers); + expect_matches_registers_to_number(registers); + expect_matches_registers_to_number(registers); + expect_matches_registers_to_number(registers); +} + +TEST(ModbusHelpersTest, RegistersToUint32CombinesWordsHighFirst) { + EXPECT_EQ(registers_to_uint32(0x1234, 0x5678), 0x12345678u); +} + // --- packed bit helpers ------------------------------------------------------ TEST(ModbusHelpersTest, PackBitsAppendsToContainer) { From d4348335dd3aac23d1644660e89fbf149ac29be4 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 13:11:42 -0700 Subject: [PATCH 010/147] [growatt_solar] Use the typed modbus read callback and drop the send-pacing state machine (#18850) --- .../growatt_solar/growatt_solar.cpp | 119 ++++++------------ .../components/growatt_solar/growatt_solar.h | 7 +- 2 files changed, 40 insertions(+), 86 deletions(-) diff --git a/esphome/components/growatt_solar/growatt_solar.cpp b/esphome/components/growatt_solar/growatt_solar.cpp index d2102496a2..bc3c3d52db 100644 --- a/esphome/components/growatt_solar/growatt_solar.cpp +++ b/esphome/components/growatt_solar/growatt_solar.cpp @@ -1,6 +1,4 @@ #include "growatt_solar.h" -#include "esphome/core/application.h" -#include "esphome/core/helpers.h" #include "esphome/core/log.h" namespace esphome::growatt_solar { @@ -9,97 +7,65 @@ static const char *const TAG = "growatt_solar"; static const uint8_t MODBUS_REGISTER_COUNT[] = {33, 95}; // indexed with enum GrowattProtocolVersion -void GrowattSolar::loop() { - // If update() was unable to send we retry until we can send. - if (!this->waiting_to_update_) - return; - update(); -} +void GrowattSolar::update() { this->read_input_registers(0, MODBUS_REGISTER_COUNT[this->protocol_version_]); } -void GrowattSolar::update() { - // If our last send has had no reply yet, and it wasn't that long ago, do nothing. - const uint32_t now = App.get_loop_component_start_time(); - if (now - this->last_send_ < this->get_update_interval() / 2) { - return; - } - - // The bus might be slow, or there might be other devices, or other components might be talking to our device. - if (!this->ready_for_immediate_send()) { - this->waiting_to_update_ = true; - return; - } - - this->waiting_to_update_ = false; - this->read_input_registers(0, MODBUS_REGISTER_COUNT[this->protocol_version_]); - this->last_send_ = millis(); -} - -void GrowattSolar::on_response(std::span request_pdu, std::span response_pdu) { - auto data = modbus::helpers::server_pdu_payload(response_pdu); - // Other components might be sending commands to our device. But we don't get called with enough - // context to know what is what. So if we didn't do a send, we ignore the data. - if (!this->last_send_) - return; - this->last_send_ = 0; - - // Also ignore the data if the message is too short. Otherwise we will publish invalid values. - if (data.size() < MODBUS_REGISTER_COUNT[this->protocol_version_] * 2) +void GrowattSolar::on_read_input_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) { + if (!modbus::succeeded(status)) return; - auto publish_1_reg_sensor_state = [&](sensor::Sensor *sensor, size_t i, float unit) -> void { - if (sensor == nullptr) + // Publish a sensor if its register(s) are in this response; skipping absent registers keeps this + // correct for any read range, so the poll may be split into multiple requests. + auto publish_1_reg_sensor_state = [&](sensor::Sensor *sensor, size_t reg, float unit) -> void { + if (sensor == nullptr || reg < start_address) return; - float value = encode_uint16(data[i * 2], data[i * 2 + 1]) * unit; - sensor->publish_state(value); + size_t offset = reg - start_address; + if (offset >= registers.size()) + return; + sensor->publish_state(registers[offset] * unit); }; - auto publish_2_reg_sensor_state = [&](sensor::Sensor *sensor, size_t reg1, size_t reg2, float unit) -> void { - float value = ((encode_uint16(data[reg1 * 2], data[reg1 * 2 + 1]) << 16) + - encode_uint16(data[reg2 * 2], data[reg2 * 2 + 1])) * - unit; - if (sensor != nullptr) - sensor->publish_state(value); + auto publish_2_reg_sensor_state = [&](sensor::Sensor *sensor, size_t reg, float unit) -> void { + constexpr auto value_type = modbus::helpers::SensorValueType::U_DWORD; + if (sensor == nullptr || reg < start_address) + return; + size_t offset = reg - start_address; + if (offset + modbus::helpers::register_width_for(value_type) > registers.size()) + return; + sensor->publish_state(modbus::helpers::registers_to_value(registers.data() + offset) * unit); }; switch (this->protocol_version_) { case RTU: { publish_1_reg_sensor_state(this->inverter_status_, RTU_INVERTER_STATUS, 1); - publish_2_reg_sensor_state(this->pv_active_power_sensor_, RTU_PV_ACTIVE_POWER, RTU_PV_ACTIVE_POWER + 1, - ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->pv_active_power_sensor_, RTU_PV_ACTIVE_POWER, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->pvs_[0].voltage_sensor_, RTU_PV1_VOLTAGE, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->pvs_[0].current_sensor_, RTU_PV1_CURRENT, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->pvs_[0].active_power_sensor_, RTU_PV1_ACTIVE_POWER, RTU_PV1_ACTIVE_POWER + 1, - ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->pvs_[0].active_power_sensor_, RTU_PV1_ACTIVE_POWER, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->pvs_[1].voltage_sensor_, RTU_PV2_VOLTAGE, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->pvs_[1].current_sensor_, RTU_PV2_CURRENT, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->pvs_[1].active_power_sensor_, RTU_PV2_ACTIVE_POWER, RTU_PV2_ACTIVE_POWER + 1, - ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->pvs_[1].active_power_sensor_, RTU_PV2_ACTIVE_POWER, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->grid_active_power_sensor_, RTU_GRID_ACTIVE_POWER, RTU_GRID_ACTIVE_POWER + 1, - ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->grid_active_power_sensor_, RTU_GRID_ACTIVE_POWER, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->grid_frequency_sensor_, RTU_GRID_FREQUENCY, TWO_DEC_UNIT); publish_1_reg_sensor_state(this->phases_[0].voltage_sensor_, RTU_PHASE1_VOLTAGE, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->phases_[0].current_sensor_, RTU_PHASE1_CURRENT, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->phases_[0].active_power_sensor_, RTU_PHASE1_ACTIVE_POWER, - RTU_PHASE1_ACTIVE_POWER + 1, ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->phases_[0].active_power_sensor_, RTU_PHASE1_ACTIVE_POWER, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->phases_[1].voltage_sensor_, RTU_PHASE2_VOLTAGE, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->phases_[1].current_sensor_, RTU_PHASE2_CURRENT, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->phases_[1].active_power_sensor_, RTU_PHASE2_ACTIVE_POWER, - RTU_PHASE2_ACTIVE_POWER + 1, ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->phases_[1].active_power_sensor_, RTU_PHASE2_ACTIVE_POWER, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->phases_[2].voltage_sensor_, RTU_PHASE3_VOLTAGE, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->phases_[2].current_sensor_, RTU_PHASE3_CURRENT, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->phases_[2].active_power_sensor_, RTU_PHASE3_ACTIVE_POWER, - RTU_PHASE3_ACTIVE_POWER + 1, ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->phases_[2].active_power_sensor_, RTU_PHASE3_ACTIVE_POWER, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->today_production_, RTU_TODAY_PRODUCTION, RTU_TODAY_PRODUCTION + 1, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->total_energy_production_, RTU_TOTAL_ENERGY_PRODUCTION, - RTU_TOTAL_ENERGY_PRODUCTION + 1, ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->today_production_, RTU_TODAY_PRODUCTION, ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->total_energy_production_, RTU_TOTAL_ENERGY_PRODUCTION, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->inverter_module_temp_, RTU_INVERTER_MODULE_TEMP, ONE_DEC_UNIT); break; @@ -107,42 +73,33 @@ void GrowattSolar::on_response(std::span request_pdu, std::spaninverter_status_, RTU2_INVERTER_STATUS, 1); - publish_2_reg_sensor_state(this->pv_active_power_sensor_, RTU2_PV_ACTIVE_POWER, RTU2_PV_ACTIVE_POWER + 1, - ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->pv_active_power_sensor_, RTU2_PV_ACTIVE_POWER, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->pvs_[0].voltage_sensor_, RTU2_PV1_VOLTAGE, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->pvs_[0].current_sensor_, RTU2_PV1_CURRENT, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->pvs_[0].active_power_sensor_, RTU2_PV1_ACTIVE_POWER, RTU2_PV1_ACTIVE_POWER + 1, - ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->pvs_[0].active_power_sensor_, RTU2_PV1_ACTIVE_POWER, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->pvs_[1].voltage_sensor_, RTU2_PV2_VOLTAGE, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->pvs_[1].current_sensor_, RTU2_PV2_CURRENT, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->pvs_[1].active_power_sensor_, RTU2_PV2_ACTIVE_POWER, RTU2_PV2_ACTIVE_POWER + 1, - ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->pvs_[1].active_power_sensor_, RTU2_PV2_ACTIVE_POWER, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->grid_active_power_sensor_, RTU2_GRID_ACTIVE_POWER, RTU2_GRID_ACTIVE_POWER + 1, - ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->grid_active_power_sensor_, RTU2_GRID_ACTIVE_POWER, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->grid_frequency_sensor_, RTU2_GRID_FREQUENCY, TWO_DEC_UNIT); publish_1_reg_sensor_state(this->phases_[0].voltage_sensor_, RTU2_PHASE1_VOLTAGE, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->phases_[0].current_sensor_, RTU2_PHASE1_CURRENT, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->phases_[0].active_power_sensor_, RTU2_PHASE1_ACTIVE_POWER, - RTU2_PHASE1_ACTIVE_POWER + 1, ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->phases_[0].active_power_sensor_, RTU2_PHASE1_ACTIVE_POWER, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->phases_[1].voltage_sensor_, RTU2_PHASE2_VOLTAGE, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->phases_[1].current_sensor_, RTU2_PHASE2_CURRENT, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->phases_[1].active_power_sensor_, RTU2_PHASE2_ACTIVE_POWER, - RTU2_PHASE2_ACTIVE_POWER + 1, ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->phases_[1].active_power_sensor_, RTU2_PHASE2_ACTIVE_POWER, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->phases_[2].voltage_sensor_, RTU2_PHASE3_VOLTAGE, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->phases_[2].current_sensor_, RTU2_PHASE3_CURRENT, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->phases_[2].active_power_sensor_, RTU2_PHASE3_ACTIVE_POWER, - RTU2_PHASE3_ACTIVE_POWER + 1, ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->phases_[2].active_power_sensor_, RTU2_PHASE3_ACTIVE_POWER, ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->today_production_, RTU2_TODAY_PRODUCTION, RTU2_TODAY_PRODUCTION + 1, - ONE_DEC_UNIT); - publish_2_reg_sensor_state(this->total_energy_production_, RTU2_TOTAL_ENERGY_PRODUCTION, - RTU2_TOTAL_ENERGY_PRODUCTION + 1, ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->today_production_, RTU2_TODAY_PRODUCTION, ONE_DEC_UNIT); + publish_2_reg_sensor_state(this->total_energy_production_, RTU2_TOTAL_ENERGY_PRODUCTION, ONE_DEC_UNIT); publish_1_reg_sensor_state(this->inverter_module_temp_, RTU2_INVERTER_MODULE_TEMP, ONE_DEC_UNIT); break; diff --git a/esphome/components/growatt_solar/growatt_solar.h b/esphome/components/growatt_solar/growatt_solar.h index a172f49001..60706930c7 100644 --- a/esphome/components/growatt_solar/growatt_solar.h +++ b/esphome/components/growatt_solar/growatt_solar.h @@ -67,9 +67,9 @@ constexpr size_t RTU2_INVERTER_MODULE_TEMP = 93; // length = 1 class GrowattSolar final : public PollingComponent, public modbus::ModbusClientDevice { public: - void loop() override; void update() override; - void on_response(std::span request_pdu, std::span response_pdu) override; + void on_read_input_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) override; void dump_config() override; void set_protocol_version(GrowattProtocolVersion protocol_version) { this->protocol_version_ = protocol_version; } @@ -104,9 +104,6 @@ class GrowattSolar final : public PollingComponent, public modbus::ModbusClientD } protected: - bool waiting_to_update_{false}; - uint32_t last_send_{0}; - struct GrowattPhase { sensor::Sensor *voltage_sensor_{nullptr}; sensor::Sensor *current_sensor_{nullptr}; From 03147bc3b1d59e58b8eab22e4b5679a17f3d8af6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 28 Aug 2026 15:12:50 -0500 Subject: [PATCH 011/147] [core] Avoid double promotion in update interval and step formatting (#18825) --- esphome/core/component.cpp | 4 +-- esphome/core/helpers.cpp | 34 ++++++++++++++++++-------- tests/components/core/test_helpers.cpp | 6 ++--- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/esphome/core/component.cpp b/esphome/core/component.cpp index e5fbb8ba07..41dd32ea66 100644 --- a/esphome/core/component.cpp +++ b/esphome/core/component.cpp @@ -336,10 +336,8 @@ void log_update_interval(const char *tag, PollingComponent *component) { uint32_t update_interval = component->get_update_interval(); if (update_interval == SCHEDULER_DONT_RUN) { ESP_LOGCONFIG(tag, " Update Interval: never"); - } else if (update_interval < 100) { - ESP_LOGCONFIG(tag, " Update Interval: %.3fs", update_interval / 1000.0f); } else { - ESP_LOGCONFIG(tag, " Update Interval: %.1fs", update_interval / 1000.0f); + ESP_LOGCONFIG(tag, " Update Interval: %" PRIu32 ".%03" PRIu32 "s", update_interval / 1000, update_interval % 1000); } } float Component::get_actual_setup_priority() const { diff --git a/esphome/core/helpers.cpp b/esphome/core/helpers.cpp index 6bfe5c9e3c..433d2547b0 100644 --- a/esphome/core/helpers.cpp +++ b/esphome/core/helpers.cpp @@ -568,7 +568,7 @@ size_t value_accuracy_to_buf(std::span buf, float } // Fallback for NaN/Inf/high accuracy/out-of-range - int len = snprintf(buf.data(), buf.size(), "%.*f", accuracy_decimals, value); + int len = snprintf(buf.data(), buf.size(), "%.*f", accuracy_decimals, static_cast(value)); if (len < 0) return 0; return static_cast(len) >= buf.size() ? buf.size() - 1 : static_cast(len); @@ -586,16 +586,30 @@ size_t value_accuracy_with_uom_to_buf(std::span bu } int8_t step_to_accuracy_decimals(float step) { - // use printf %g to find number of digits based on temperature step - char buf[32]; - snprintf(buf, sizeof buf, "%.5g", step); - - std::string str{buf}; - size_t dot_pos = str.find('.'); - if (dot_pos == std::string::npos) + // Decimals needed to show the step at five significant digits, trailing zeros dropped. + if (!std::isfinite(step) || step == 0.0f) return 0; - - return str.length() - dot_pos - 1; + float mantissa = std::fabs(step); + int8_t decimals = 4; // decimals needed for five significant digits when mantissa is in [1, 10) + while (mantissa >= 10.0f) { + mantissa /= 10.0f; + decimals--; + } + while (mantissa < 1.0f) { + mantissa *= 10.0f; + decimals++; + } + if (decimals <= 0) + return 0; + float scaled = mantissa * 10000.0f; + auto digits = static_cast(scaled); + if (scaled - static_cast(digits) >= 0.5f) + digits++; + while (decimals > 0 && digits % 10 == 0) { + digits /= 10; + decimals--; + } + return decimals; } // Map a base64/base64url character to its 6-bit value (0-63) arithmetically. diff --git a/tests/components/core/test_helpers.cpp b/tests/components/core/test_helpers.cpp index a031dcb36f..baf688fc8a 100644 --- a/tests/components/core/test_helpers.cpp +++ b/tests/components/core/test_helpers.cpp @@ -328,10 +328,10 @@ TEST(StepToAccuracyDecimals, RoundsUpToWholeNumber) { } TEST(StepToAccuracyDecimals, OutsideFixedNotationRange) { - // %.5g prints these in exponent form, so the count comes from parsing "1e-05" or "1.2346e+05". - EXPECT_EQ(step_to_accuracy_decimals(0.00001f), 0); + // %.5g would print these in exponent form; the count is now the real one rather than a parse of "1e-05". + EXPECT_EQ(step_to_accuracy_decimals(0.00001f), 5); EXPECT_EQ(step_to_accuracy_decimals(0.000125f), 6); - EXPECT_EQ(step_to_accuracy_decimals(123456.0f), 8); + EXPECT_EQ(step_to_accuracy_decimals(123456.0f), 0); EXPECT_EQ(step_to_accuracy_decimals(1000000.0f), 0); } From e8d76b735b38e03c8a22fb25ebe68fcabce1a3ed Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 13:21:52 -0700 Subject: [PATCH 012/147] [havells_solar] Use the typed modbus read callback with address-based extraction (#18851) Co-authored-by: J. Nick Koston --- .../havells_solar/havells_solar.cpp | 142 ++++++------------ .../components/havells_solar/havells_solar.h | 3 +- 2 files changed, 48 insertions(+), 97 deletions(-) diff --git a/esphome/components/havells_solar/havells_solar.cpp b/esphome/components/havells_solar/havells_solar.cpp index 6af72c352b..c98dc0de2f 100644 --- a/esphome/components/havells_solar/havells_solar.cpp +++ b/esphome/components/havells_solar/havells_solar.cpp @@ -1,6 +1,5 @@ #include "havells_solar.h" #include "havells_solar_registers.h" -#include "esphome/core/helpers.h" #include "esphome/core/log.h" namespace esphome::havells_solar { @@ -9,116 +8,67 @@ static const char *const TAG = "havells_solar"; static const uint8_t MODBUS_REGISTER_COUNT = 48; // 48 x 16-bit registers -void HavellsSolar::on_response(std::span request_pdu, std::span response_pdu) { - auto data = modbus::helpers::server_pdu_payload(response_pdu); - if (data.size() < MODBUS_REGISTER_COUNT * 2) { - ESP_LOGW(TAG, "Invalid size for HavellsSolar!"); - return; - } +void HavellsSolar::on_read_holding_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) { + if (!modbus::succeeded(status)) + return; // the hub already logs exception responses - /* Usage: returns the float value of 1 register read by modbus - Arg1: Register address * number of bytes per register - Arg2: Multiplier for final register value - */ - auto havells_solar_get_2_registers = [&](size_t i, float unit) -> float { - uint32_t temp = encode_uint32(data[i], data[i + 1], data[i + 2], data[i + 3]); - return temp * unit; + // Publish a sensor if its register(s) are in this response; skipping absent registers keeps this + // correct for any read range, so the poll may be split into multiple requests. + auto publish_1_register = [&](sensor::Sensor *sensor, uint16_t reg, float unit) -> void { + if (sensor == nullptr || reg < start_address) + return; + size_t offset = reg - start_address; + if (offset >= registers.size()) + return; + sensor->publish_state(registers[offset] * unit); }; - /* Usage: returns the float value of 2 registers read by modbus - Arg1: Register address * number of bytes per register - Arg2: Multiplier for final register value - */ - auto havells_solar_get_1_register = [&](size_t i, float unit) -> float { - uint16_t temp = encode_uint16(data[i], data[i + 1]); - return temp * unit; + auto publish_2_registers = [&](sensor::Sensor *sensor, uint16_t reg, float unit) -> void { + constexpr auto value_type = modbus::helpers::SensorValueType::U_DWORD; + if (sensor == nullptr || reg < start_address) + return; + size_t offset = reg - start_address; + if (offset + modbus::helpers::register_width_for(value_type) > registers.size()) + return; + sensor->publish_state(modbus::helpers::registers_to_value(registers.data() + offset) * unit); }; for (uint8_t i = 0; i < 3; i++) { - auto phase = this->phases_[i]; + auto &phase = this->phases_[i]; if (!phase.setup) continue; - - float voltage = havells_solar_get_1_register(HAVELLS_PHASE_1_VOLTAGE * 2 + (i * 4), ONE_DEC_UNIT); - float current = havells_solar_get_1_register(HAVELLS_PHASE_1_CURRENT * 2 + (i * 4), TWO_DEC_UNIT); - - if (phase.voltage_sensor_ != nullptr) - phase.voltage_sensor_->publish_state(voltage); - if (phase.current_sensor_ != nullptr) - phase.current_sensor_->publish_state(current); + publish_1_register(phase.voltage_sensor_, HAVELLS_PHASE_1_VOLTAGE + i * 2, ONE_DEC_UNIT); + publish_1_register(phase.current_sensor_, HAVELLS_PHASE_1_CURRENT + i * 2, TWO_DEC_UNIT); } for (uint8_t i = 0; i < 2; i++) { - auto pv = this->pvs_[i]; + auto &pv = this->pvs_[i]; if (!pv.setup) continue; - - float voltage = havells_solar_get_1_register(HAVELLS_PV_1_VOLTAGE * 2 + (i * 4), ONE_DEC_UNIT); - float current = havells_solar_get_1_register(HAVELLS_PV_1_CURRENT * 2 + (i * 4), TWO_DEC_UNIT); - float active_power = havells_solar_get_1_register(HAVELLS_PV_1_POWER * 2 + (i * 2), MULTIPLY_TEN_UNIT); - float voltage_sampled_by_secondary_cpu = - havells_solar_get_1_register(HAVELLS_PV1_VOLTAGE_SAMPLED_BY_SECONDARY_CPU * 2 + (i * 2), ONE_DEC_UNIT); - float insulation_of_p_to_ground = - havells_solar_get_1_register(HAVELLS_PV1_INSULATION_OF_P_TO_GROUND * 2 + (i * 2), NO_DEC_UNIT); - - if (pv.voltage_sensor_ != nullptr) - pv.voltage_sensor_->publish_state(voltage); - if (pv.current_sensor_ != nullptr) - pv.current_sensor_->publish_state(current); - if (pv.active_power_sensor_ != nullptr) - pv.active_power_sensor_->publish_state(active_power); - if (pv.voltage_sampled_by_secondary_cpu_sensor_ != nullptr) - pv.voltage_sampled_by_secondary_cpu_sensor_->publish_state(voltage_sampled_by_secondary_cpu); - if (pv.insulation_of_p_to_ground_sensor_ != nullptr) - pv.insulation_of_p_to_ground_sensor_->publish_state(insulation_of_p_to_ground); + publish_1_register(pv.voltage_sensor_, HAVELLS_PV_1_VOLTAGE + i * 2, ONE_DEC_UNIT); + publish_1_register(pv.current_sensor_, HAVELLS_PV_1_CURRENT + i * 2, TWO_DEC_UNIT); + publish_1_register(pv.active_power_sensor_, HAVELLS_PV_1_POWER + i, MULTIPLY_TEN_UNIT); + publish_1_register(pv.voltage_sampled_by_secondary_cpu_sensor_, HAVELLS_PV1_VOLTAGE_SAMPLED_BY_SECONDARY_CPU + i, + ONE_DEC_UNIT); + publish_1_register(pv.insulation_of_p_to_ground_sensor_, HAVELLS_PV1_INSULATION_OF_P_TO_GROUND + i, NO_DEC_UNIT); } - float frequency = havells_solar_get_1_register(HAVELLS_GRID_FREQUENCY * 2, TWO_DEC_UNIT); - float active_power = havells_solar_get_1_register(HAVELLS_SYSTEM_ACTIVE_POWER * 2, MULTIPLY_TEN_UNIT); - float reactive_power = havells_solar_get_1_register(HAVELLS_SYSTEM_REACTIVE_POWER * 2, TWO_DEC_UNIT); - float today_production = havells_solar_get_1_register(HAVELLS_TODAY_PRODUCTION * 2, TWO_DEC_UNIT); - float total_energy_production = havells_solar_get_2_registers(HAVELLS_TOTAL_ENERGY_PRODUCTION * 2, NO_DEC_UNIT); - float total_generation_time = havells_solar_get_2_registers(HAVELLS_TOTAL_GENERATION_TIME * 2, NO_DEC_UNIT); - float today_generation_time = havells_solar_get_1_register(HAVELLS_TODAY_GENERATION_TIME * 2, NO_DEC_UNIT); - float inverter_module_temp = havells_solar_get_1_register(HAVELLS_INVERTER_MODULE_TEMP * 2, NO_DEC_UNIT); - float inverter_inner_temp = havells_solar_get_1_register(HAVELLS_INVERTER_INNER_TEMP * 2, NO_DEC_UNIT); - float inverter_bus_voltage = havells_solar_get_1_register(HAVELLS_INVERTER_BUS_VOLTAGE * 2, NO_DEC_UNIT); - float insulation_pv_n_to_ground = havells_solar_get_1_register(HAVELLS_INSULATION_OF_PV_N_TO_GROUND * 2, NO_DEC_UNIT); - float gfci_value = havells_solar_get_1_register(HAVELLS_GFCI_VALUE * 2, NO_DEC_UNIT); - float dci_of_r = havells_solar_get_1_register(HAVELLS_DCI_OF_R * 2, NO_DEC_UNIT); - float dci_of_s = havells_solar_get_1_register(HAVELLS_DCI_OF_S * 2, NO_DEC_UNIT); - float dci_of_t = havells_solar_get_1_register(HAVELLS_DCI_OF_T * 2, NO_DEC_UNIT); - - if (this->frequency_sensor_ != nullptr) - this->frequency_sensor_->publish_state(frequency); - if (this->active_power_sensor_ != nullptr) - this->active_power_sensor_->publish_state(active_power); - if (this->reactive_power_sensor_ != nullptr) - this->reactive_power_sensor_->publish_state(reactive_power); - if (this->today_production_sensor_ != nullptr) - this->today_production_sensor_->publish_state(today_production); - if (this->total_energy_production_sensor_ != nullptr) - this->total_energy_production_sensor_->publish_state(total_energy_production); - if (this->total_generation_time_sensor_ != nullptr) - this->total_generation_time_sensor_->publish_state(total_generation_time); - if (this->today_generation_time_sensor_ != nullptr) - this->today_generation_time_sensor_->publish_state(today_generation_time); - if (this->inverter_module_temp_sensor_ != nullptr) - this->inverter_module_temp_sensor_->publish_state(inverter_module_temp); - if (this->inverter_inner_temp_sensor_ != nullptr) - this->inverter_inner_temp_sensor_->publish_state(inverter_inner_temp); - if (this->inverter_bus_voltage_sensor_ != nullptr) - this->inverter_bus_voltage_sensor_->publish_state(inverter_bus_voltage); - if (this->insulation_pv_n_to_ground_sensor_ != nullptr) - this->insulation_pv_n_to_ground_sensor_->publish_state(insulation_pv_n_to_ground); - if (this->gfci_value_sensor_ != nullptr) - this->gfci_value_sensor_->publish_state(gfci_value); - if (this->dci_of_r_sensor_ != nullptr) - this->dci_of_r_sensor_->publish_state(dci_of_r); - if (this->dci_of_s_sensor_ != nullptr) - this->dci_of_s_sensor_->publish_state(dci_of_s); - if (this->dci_of_t_sensor_ != nullptr) - this->dci_of_t_sensor_->publish_state(dci_of_t); + publish_1_register(this->frequency_sensor_, HAVELLS_GRID_FREQUENCY, TWO_DEC_UNIT); + publish_1_register(this->active_power_sensor_, HAVELLS_SYSTEM_ACTIVE_POWER, MULTIPLY_TEN_UNIT); + publish_1_register(this->reactive_power_sensor_, HAVELLS_SYSTEM_REACTIVE_POWER, TWO_DEC_UNIT); + publish_1_register(this->today_production_sensor_, HAVELLS_TODAY_PRODUCTION, TWO_DEC_UNIT); + publish_2_registers(this->total_energy_production_sensor_, HAVELLS_TOTAL_ENERGY_PRODUCTION, NO_DEC_UNIT); + publish_2_registers(this->total_generation_time_sensor_, HAVELLS_TOTAL_GENERATION_TIME, NO_DEC_UNIT); + publish_1_register(this->today_generation_time_sensor_, HAVELLS_TODAY_GENERATION_TIME, NO_DEC_UNIT); + publish_1_register(this->inverter_module_temp_sensor_, HAVELLS_INVERTER_MODULE_TEMP, NO_DEC_UNIT); + publish_1_register(this->inverter_inner_temp_sensor_, HAVELLS_INVERTER_INNER_TEMP, NO_DEC_UNIT); + publish_1_register(this->inverter_bus_voltage_sensor_, HAVELLS_INVERTER_BUS_VOLTAGE, NO_DEC_UNIT); + publish_1_register(this->insulation_pv_n_to_ground_sensor_, HAVELLS_INSULATION_OF_PV_N_TO_GROUND, NO_DEC_UNIT); + publish_1_register(this->gfci_value_sensor_, HAVELLS_GFCI_VALUE, NO_DEC_UNIT); + publish_1_register(this->dci_of_r_sensor_, HAVELLS_DCI_OF_R, NO_DEC_UNIT); + publish_1_register(this->dci_of_s_sensor_, HAVELLS_DCI_OF_S, NO_DEC_UNIT); + publish_1_register(this->dci_of_t_sensor_, HAVELLS_DCI_OF_T, NO_DEC_UNIT); } void HavellsSolar::update() { this->read_holding_registers(0, MODBUS_REGISTER_COUNT); } diff --git a/esphome/components/havells_solar/havells_solar.h b/esphome/components/havells_solar/havells_solar.h index ed5d13b8b6..a77b8bf977 100644 --- a/esphome/components/havells_solar/havells_solar.h +++ b/esphome/components/havells_solar/havells_solar.h @@ -77,7 +77,8 @@ class HavellsSolar final : public PollingComponent, public modbus::ModbusClientD void update() override; - void on_response(std::span request_pdu, std::span response_pdu) override; + void on_read_holding_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) override; void dump_config() override; From 97f643574c32d4d348bd935992c1d35295f95f6a Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 13:27:01 -0700 Subject: [PATCH 013/147] [kuntze] Use the typed modbus read callback and queue all reads at once (#18852) Co-authored-by: J. Nick Koston --- esphome/components/kuntze/kuntze.cpp | 69 +++++++++++----------------- esphome/components/kuntze/kuntze.h | 8 +--- 2 files changed, 30 insertions(+), 47 deletions(-) diff --git a/esphome/components/kuntze/kuntze.cpp b/esphome/components/kuntze/kuntze.cpp index c47a80777c..cb04afe437 100644 --- a/esphome/components/kuntze/kuntze.cpp +++ b/esphome/components/kuntze/kuntze.cpp @@ -1,87 +1,74 @@ #include "kuntze.h" -#include "esphome/core/helpers.h" #include "esphome/core/log.h" -#include "esphome/core/application.h" namespace esphome::kuntze { static const char *const TAG = "kuntze"; -static const uint16_t REGISTER[] = {4136, 4160, 4680, 6000, 4688, 4728, 5832}; +static constexpr uint16_t REGISTER_PH = 4136; +static constexpr uint16_t REGISTER_TEMPERATURE = 4160; +static constexpr uint16_t REGISTER_DIS1 = 4680; +static constexpr uint16_t REGISTER_DIS2 = 6000; +static constexpr uint16_t REGISTER_REDOX = 4688; +static constexpr uint16_t REGISTER_EC = 4728; +static constexpr uint16_t REGISTER_OCI = 5832; +static constexpr uint16_t REGISTER[] = {REGISTER_PH, REGISTER_TEMPERATURE, REGISTER_DIS1, REGISTER_DIS2, + REGISTER_REDOX, REGISTER_EC, REGISTER_OCI}; -// Maximum bytes to log for Modbus responses (2 registers = 4, plus count = 5) -static constexpr size_t KUNTZE_MAX_LOG_BYTES = 8; +void Kuntze::on_read_holding_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) { + if (!modbus::succeeded(status) || registers.size() < 2) + return; -void Kuntze::on_response(std::span request_pdu, std::span response_pdu) { - auto data = modbus::helpers::server_pdu_payload(response_pdu); - auto get_16bit = [&](int i) -> uint16_t { return (uint16_t(data[i * 2]) << 8) | uint16_t(data[i * 2 + 1]); }; + // Each value is a register pair: the reading, then the number of decimal places in its low byte. + float value = registers[0]; + for (uint16_t i = 0; i < (registers[1] & 0xFF); i++) + value /= 10.0f; - this->waiting_ = false; -#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE - char hex_buf[format_hex_pretty_size(KUNTZE_MAX_LOG_BYTES)]; -#endif - ESP_LOGV(TAG, "Data: %s", format_hex_pretty_to(hex_buf, data.data(), data.size())); - - float value = (float) get_16bit(0); - for (int i = 0; i < data[3]; i++) - value /= 10.0; - switch (this->state_) { - case 1: + switch (start_address) { + case REGISTER_PH: ESP_LOGD(TAG, "pH=%.1f", value); if (this->ph_sensor_ != nullptr) this->ph_sensor_->publish_state(value); break; - case 2: + case REGISTER_TEMPERATURE: ESP_LOGD(TAG, "temperature=%.1f", value); if (this->temperature_sensor_ != nullptr) this->temperature_sensor_->publish_state(value); break; - case 3: + case REGISTER_DIS1: ESP_LOGD(TAG, "DIS1=%.1f", value); if (this->dis1_sensor_ != nullptr) this->dis1_sensor_->publish_state(value); break; - case 4: + case REGISTER_DIS2: ESP_LOGD(TAG, "DIS2=%.1f", value); if (this->dis2_sensor_ != nullptr) this->dis2_sensor_->publish_state(value); break; - case 5: + case REGISTER_REDOX: ESP_LOGD(TAG, "REDOX=%.1f", value); if (this->redox_sensor_ != nullptr) this->redox_sensor_->publish_state(value); break; - case 6: + case REGISTER_EC: ESP_LOGD(TAG, "EC=%.1f", value); if (this->ec_sensor_ != nullptr) this->ec_sensor_->publish_state(value); break; - case 7: + case REGISTER_OCI: ESP_LOGD(TAG, "OCI=%.1f", value); if (this->oci_sensor_ != nullptr) this->oci_sensor_->publish_state(value); break; } - if (++this->state_ > 7) - this->state_ = 0; } -void Kuntze::loop() { - uint32_t now = App.get_loop_component_start_time(); - // timeout after 15 seconds - if (this->waiting_ && (now - this->last_send_ > 15000)) { - ESP_LOGW(TAG, "timed out waiting for response"); - this->waiting_ = false; - } - if (this->waiting_ || (this->state_ == 0)) - return; - this->last_send_ = now; - this->read_holding_registers(REGISTER[this->state_ - 1], 2); - this->waiting_ = true; +void Kuntze::update() { + for (uint16_t reg : REGISTER) + this->read_holding_registers(reg, 2); } -void Kuntze::update() { this->state_ = 1; } - void Kuntze::dump_config() { ESP_LOGCONFIG(TAG, "Kuntze:\n" diff --git a/esphome/components/kuntze/kuntze.h b/esphome/components/kuntze/kuntze.h index 28c8089748..84197b379d 100644 --- a/esphome/components/kuntze/kuntze.h +++ b/esphome/components/kuntze/kuntze.h @@ -18,18 +18,14 @@ class Kuntze final : public PollingComponent, public modbus::ModbusClientDevice void set_ec_sensor(sensor::Sensor *ec_sensor) { ec_sensor_ = ec_sensor; } void set_oci_sensor(sensor::Sensor *oci_sensor) { oci_sensor_ = oci_sensor; } - void loop() override; void update() override; - void on_response(std::span request_pdu, std::span response_pdu) override; + void on_read_holding_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) override; void dump_config() override; protected: - int state_{0}; - bool waiting_{false}; - uint32_t last_send_{0}; - sensor::Sensor *ph_sensor_{nullptr}; sensor::Sensor *temperature_sensor_{nullptr}; sensor::Sensor *dis1_sensor_{nullptr}; From 5fc9bff371c0783317059e359d4a6b5c8b184864 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 13:27:45 -0700 Subject: [PATCH 014/147] [pzemdc] Use the typed modbus read callback with address-based extraction (#18853) Co-authored-by: J. Nick Koston --- esphome/components/pzemdc/pzemdc.cpp | 87 ++++++++++++++++------------ esphome/components/pzemdc/pzemdc.h | 5 +- 2 files changed, 53 insertions(+), 39 deletions(-) diff --git a/esphome/components/pzemdc/pzemdc.cpp b/esphome/components/pzemdc/pzemdc.cpp index 926ad83f09..546e4225de 100644 --- a/esphome/components/pzemdc/pzemdc.cpp +++ b/esphome/components/pzemdc/pzemdc.cpp @@ -6,52 +6,63 @@ namespace esphome::pzemdc { static const char *const TAG = "pzemdc"; static const uint8_t PZEM_CMD_RESET_ENERGY = 0x42; -static const uint8_t PZEM_REGISTER_COUNT = 10; // 10x 16-bit registers +static const uint8_t PZEM_REGISTER_COUNT = 8; // 8x 16-bit registers -void PZEMDC::on_response(std::span request_pdu, std::span response_pdu) { - auto data = modbus::helpers::server_pdu_payload(response_pdu); - if (data.size() < 16) { - ESP_LOGW(TAG, "Invalid size for PZEM DC!"); - return; - } +// Register map, see https://github.com/esphome/feature-requests/issues/49#issuecomment-538636809 +// 32-bit values are two registers, low word first. +static const uint16_t PZEM_REGISTER_VOLTAGE = 0; // 1 register, 0.01 V +static const uint16_t PZEM_REGISTER_CURRENT = 1; // 1 register, 0.01 A +static const uint16_t PZEM_REGISTER_POWER = 2; // 2 registers, 0.1 W +static const uint16_t PZEM_REGISTER_ENERGY = 4; // 2 registers, 1 Wh - // See https://github.com/esphome/feature-requests/issues/49#issuecomment-538636809 - // 0 1 2 3 4 5 6 7 = ModBus register - // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 = Buffer index - // 01 04 10 05 40 00 0A 00 0D 00 00 00 02 00 00 00 00 00 00 D6 29 - // Id Cc Sz Volt- Curre Power------ Energy----- HiAlm LoAlm Crc-- +void PZEMDC::on_read_input_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) { + if (!modbus::succeeded(status)) + return; // the hub already logs exception responses - auto pzem_get_16bit = [&](size_t i) -> uint16_t { - return (uint16_t(data[i + 0]) << 8) | (uint16_t(data[i + 1]) << 0); - }; - auto pzem_get_32bit = [&](size_t i) -> uint32_t { - return (uint32_t(pzem_get_16bit(i + 2)) << 16) | (uint32_t(pzem_get_16bit(i + 0)) << 0); + // Publish a sensor if its register(s) are in this response; skipping absent registers keeps this + // correct for any read range, so the poll may be split into multiple requests. + auto publish_1_register = [&](sensor::Sensor *sensor, uint16_t reg, float divisor) -> void { + if (sensor == nullptr || reg < start_address) + return; + size_t offset = reg - start_address; + if (offset >= registers.size()) + return; + sensor->publish_state(registers[offset] / divisor); }; - uint16_t raw_voltage = pzem_get_16bit(0); - float voltage = raw_voltage / 100.0f; // max 655.35 V + auto publish_2_registers = [&](sensor::Sensor *sensor, uint16_t reg, float divisor) -> void { + constexpr auto value_type = modbus::helpers::SensorValueType::U_DWORD_R; + if (sensor == nullptr || reg < start_address) + return; + size_t offset = reg - start_address; + if (offset + modbus::helpers::register_width_for(value_type) > registers.size()) + return; + sensor->publish_state(modbus::helpers::registers_to_value(registers.data() + offset) / divisor); + }; - uint16_t raw_current = pzem_get_16bit(2); - float current = raw_current / 100.0f; // max 655.35 A - - uint32_t raw_power = pzem_get_32bit(4); - float power = raw_power / 10.0f; // max 429496729.5 W - - uint32_t raw_energy = pzem_get_32bit(8); - float energy = raw_energy / 1000.0f; // max 4294967.295 kWh - - ESP_LOGD(TAG, "PZEM DC: V=%.1f V, I=%.3f A, P=%.1f W", voltage, current, power); - if (this->voltage_sensor_ != nullptr) - this->voltage_sensor_->publish_state(voltage); - if (this->current_sensor_ != nullptr) - this->current_sensor_->publish_state(current); - if (this->power_sensor_ != nullptr) - this->power_sensor_->publish_state(power); - if (this->energy_sensor_ != nullptr) - this->energy_sensor_->publish_state(energy); + publish_1_register(this->voltage_sensor_, PZEM_REGISTER_VOLTAGE, 100.0f); + publish_1_register(this->current_sensor_, PZEM_REGISTER_CURRENT, 100.0f); + publish_2_registers(this->power_sensor_, PZEM_REGISTER_POWER, 10.0f); + publish_2_registers(this->energy_sensor_, PZEM_REGISTER_ENERGY, 1000.0f); } -void PZEMDC::update() { this->read_input_registers(0, 8); } +void PZEMDC::on_custom_response(std::span request_pdu, std::span response_pdu, + modbus::ResponseStatus status) { + // The only custom request this component sends is the energy reset; acknowledge its echo here so + // the default unhandled-response warning stays meaningful. + if (!request_pdu.empty() && request_pdu[0] == PZEM_CMD_RESET_ENERGY) { + if (modbus::succeeded(status)) { + ESP_LOGD(TAG, "Energy reset acknowledged"); + } else { + ESP_LOGW(TAG, "Energy reset rejected"); + } + return; + } + modbus::ModbusClientDevice::on_custom_response(request_pdu, response_pdu, status); +} + +void PZEMDC::update() { this->read_input_registers(0, PZEM_REGISTER_COUNT); } void PZEMDC::dump_config() { ESP_LOGCONFIG(TAG, "PZEMDC:\n" diff --git a/esphome/components/pzemdc/pzemdc.h b/esphome/components/pzemdc/pzemdc.h index b7657608e6..69c8a9dd6c 100644 --- a/esphome/components/pzemdc/pzemdc.h +++ b/esphome/components/pzemdc/pzemdc.h @@ -18,7 +18,10 @@ class PZEMDC final : public PollingComponent, public modbus::ModbusClientDevice void update() override; - void on_response(std::span request_pdu, std::span response_pdu) override; + void on_read_input_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) override; + void on_custom_response(std::span request_pdu, std::span response_pdu, + modbus::ResponseStatus status) override; void dump_config() override; From 346c2509017cc6ef47492e00721652ceef85a673 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 13:34:03 -0700 Subject: [PATCH 015/147] [selec_meter] Use the typed modbus read callback with address-based extraction (#18854) Co-authored-by: J. Nick Koston --- .../components/selec_meter/selec_meter.cpp | 100 ++++++------------ esphome/components/selec_meter/selec_meter.h | 3 +- 2 files changed, 34 insertions(+), 69 deletions(-) diff --git a/esphome/components/selec_meter/selec_meter.cpp b/esphome/components/selec_meter/selec_meter.cpp index 688923d8e6..97831e8354 100644 --- a/esphome/components/selec_meter/selec_meter.cpp +++ b/esphome/components/selec_meter/selec_meter.cpp @@ -1,6 +1,5 @@ #include "selec_meter.h" #include "selec_meter_registers.h" -#include "esphome/core/helpers.h" #include "esphome/core/log.h" namespace esphome::selec_meter { @@ -9,76 +8,41 @@ static const char *const TAG = "selec_meter"; static const uint8_t MODBUS_REGISTER_COUNT = 34; // 34 x 16-bit registers -void SelecMeter::on_response(std::span request_pdu, std::span response_pdu) { - auto data = modbus::helpers::server_pdu_payload(response_pdu); - if (data.size() < MODBUS_REGISTER_COUNT * 2) { - ESP_LOGW(TAG, "Invalid size for SelecMeter!"); - return; - } +void SelecMeter::on_read_input_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) { + if (!modbus::succeeded(status)) + return; // the hub already logs exception responses - auto selec_meter_get_float = [&](size_t i, float unit) -> float { - uint32_t temp = encode_uint32(data[i + 2], data[i + 3], data[i], data[i + 1]); - - float f; - memcpy(&f, &temp, sizeof(f)); - return (f * unit); + // Publish a sensor if both of its registers are in this response; skipping absent registers keeps + // this correct for any read range, so the poll may be split into multiple requests. + // Values are 32-bit floats, low word first. + auto publish = [&](sensor::Sensor *sensor, uint16_t reg, float unit) -> void { + constexpr auto value_type = modbus::helpers::SensorValueType::FP32_R; + if (sensor == nullptr || reg < start_address) + return; + size_t offset = reg - start_address; + if (offset + modbus::helpers::register_width_for(value_type) > registers.size()) + return; + sensor->publish_state(modbus::helpers::registers_to_value(registers.data() + offset) * unit); }; - float total_active_energy = selec_meter_get_float(SELEC_TOTAL_ACTIVE_ENERGY * 2, NO_DEC_UNIT); - float import_active_energy = selec_meter_get_float(SELEC_IMPORT_ACTIVE_ENERGY * 2, NO_DEC_UNIT); - float export_active_energy = selec_meter_get_float(SELEC_EXPORT_ACTIVE_ENERGY * 2, NO_DEC_UNIT); - float total_reactive_energy = selec_meter_get_float(SELEC_TOTAL_REACTIVE_ENERGY * 2, NO_DEC_UNIT); - float import_reactive_energy = selec_meter_get_float(SELEC_IMPORT_REACTIVE_ENERGY * 2, NO_DEC_UNIT); - float export_reactive_energy = selec_meter_get_float(SELEC_EXPORT_REACTIVE_ENERGY * 2, NO_DEC_UNIT); - float apparent_energy = selec_meter_get_float(SELEC_APPARENT_ENERGY * 2, NO_DEC_UNIT); - float active_power = selec_meter_get_float(SELEC_ACTIVE_POWER * 2, MULTIPLY_THOUSAND_UNIT); - float reactive_power = selec_meter_get_float(SELEC_REACTIVE_POWER * 2, MULTIPLY_THOUSAND_UNIT); - float apparent_power = selec_meter_get_float(SELEC_APPARENT_POWER * 2, MULTIPLY_THOUSAND_UNIT); - float voltage = selec_meter_get_float(SELEC_VOLTAGE * 2, NO_DEC_UNIT); - float current = selec_meter_get_float(SELEC_CURRENT * 2, NO_DEC_UNIT); - float power_factor = selec_meter_get_float(SELEC_POWER_FACTOR * 2, NO_DEC_UNIT); - float frequency = selec_meter_get_float(SELEC_FREQUENCY * 2, NO_DEC_UNIT); - float maximum_demand_active_power = - selec_meter_get_float(SELEC_MAXIMUM_DEMAND_ACTIVE_POWER * 2, MULTIPLY_THOUSAND_UNIT); - float maximum_demand_reactive_power = - selec_meter_get_float(SELEC_MAXIMUM_DEMAND_REACTIVE_POWER * 2, MULTIPLY_THOUSAND_UNIT); - float maximum_demand_apparent_power = - selec_meter_get_float(SELEC_MAXIMUM_DEMAND_APPARENT_POWER * 2, MULTIPLY_THOUSAND_UNIT); - - if (this->total_active_energy_sensor_ != nullptr) - this->total_active_energy_sensor_->publish_state(total_active_energy); - if (this->import_active_energy_sensor_ != nullptr) - this->import_active_energy_sensor_->publish_state(import_active_energy); - if (this->export_active_energy_sensor_ != nullptr) - this->export_active_energy_sensor_->publish_state(export_active_energy); - if (this->total_reactive_energy_sensor_ != nullptr) - this->total_reactive_energy_sensor_->publish_state(total_reactive_energy); - if (this->import_reactive_energy_sensor_ != nullptr) - this->import_reactive_energy_sensor_->publish_state(import_reactive_energy); - if (this->export_reactive_energy_sensor_ != nullptr) - this->export_reactive_energy_sensor_->publish_state(export_reactive_energy); - if (this->apparent_energy_sensor_ != nullptr) - this->apparent_energy_sensor_->publish_state(apparent_energy); - if (this->active_power_sensor_ != nullptr) - this->active_power_sensor_->publish_state(active_power); - if (this->reactive_power_sensor_ != nullptr) - this->reactive_power_sensor_->publish_state(reactive_power); - if (this->apparent_power_sensor_ != nullptr) - this->apparent_power_sensor_->publish_state(apparent_power); - if (this->voltage_sensor_ != nullptr) - this->voltage_sensor_->publish_state(voltage); - if (this->current_sensor_ != nullptr) - this->current_sensor_->publish_state(current); - if (this->power_factor_sensor_ != nullptr) - this->power_factor_sensor_->publish_state(power_factor); - if (this->frequency_sensor_ != nullptr) - this->frequency_sensor_->publish_state(frequency); - if (this->maximum_demand_active_power_sensor_ != nullptr) - this->maximum_demand_active_power_sensor_->publish_state(maximum_demand_active_power); - if (this->maximum_demand_reactive_power_sensor_ != nullptr) - this->maximum_demand_reactive_power_sensor_->publish_state(maximum_demand_reactive_power); - if (this->maximum_demand_apparent_power_sensor_ != nullptr) - this->maximum_demand_apparent_power_sensor_->publish_state(maximum_demand_apparent_power); + publish(this->total_active_energy_sensor_, SELEC_TOTAL_ACTIVE_ENERGY, NO_DEC_UNIT); + publish(this->import_active_energy_sensor_, SELEC_IMPORT_ACTIVE_ENERGY, NO_DEC_UNIT); + publish(this->export_active_energy_sensor_, SELEC_EXPORT_ACTIVE_ENERGY, NO_DEC_UNIT); + publish(this->total_reactive_energy_sensor_, SELEC_TOTAL_REACTIVE_ENERGY, NO_DEC_UNIT); + publish(this->import_reactive_energy_sensor_, SELEC_IMPORT_REACTIVE_ENERGY, NO_DEC_UNIT); + publish(this->export_reactive_energy_sensor_, SELEC_EXPORT_REACTIVE_ENERGY, NO_DEC_UNIT); + publish(this->apparent_energy_sensor_, SELEC_APPARENT_ENERGY, NO_DEC_UNIT); + publish(this->active_power_sensor_, SELEC_ACTIVE_POWER, MULTIPLY_THOUSAND_UNIT); + publish(this->reactive_power_sensor_, SELEC_REACTIVE_POWER, MULTIPLY_THOUSAND_UNIT); + publish(this->apparent_power_sensor_, SELEC_APPARENT_POWER, MULTIPLY_THOUSAND_UNIT); + publish(this->voltage_sensor_, SELEC_VOLTAGE, NO_DEC_UNIT); + publish(this->current_sensor_, SELEC_CURRENT, NO_DEC_UNIT); + publish(this->power_factor_sensor_, SELEC_POWER_FACTOR, NO_DEC_UNIT); + publish(this->frequency_sensor_, SELEC_FREQUENCY, NO_DEC_UNIT); + publish(this->maximum_demand_active_power_sensor_, SELEC_MAXIMUM_DEMAND_ACTIVE_POWER, MULTIPLY_THOUSAND_UNIT); + publish(this->maximum_demand_reactive_power_sensor_, SELEC_MAXIMUM_DEMAND_REACTIVE_POWER, MULTIPLY_THOUSAND_UNIT); + publish(this->maximum_demand_apparent_power_sensor_, SELEC_MAXIMUM_DEMAND_APPARENT_POWER, MULTIPLY_THOUSAND_UNIT); } void SelecMeter::update() { this->read_input_registers(0, MODBUS_REGISTER_COUNT); } diff --git a/esphome/components/selec_meter/selec_meter.h b/esphome/components/selec_meter/selec_meter.h index 5ae1f9bf99..470242c918 100644 --- a/esphome/components/selec_meter/selec_meter.h +++ b/esphome/components/selec_meter/selec_meter.h @@ -37,7 +37,8 @@ class SelecMeter final : public PollingComponent, public modbus::ModbusClientDev void update() override; - void on_response(std::span request_pdu, std::span response_pdu) override; + void on_read_input_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) override; void dump_config() override; }; From 7255315ce25dcdd25202726e570fd013d59e0336 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 13:34:23 -0700 Subject: [PATCH 016/147] [sdm_meter] Use the typed modbus read callback with address-based extraction (#18849) Co-authored-by: J. Nick Koston --- esphome/components/sdm_meter/sdm_meter.cpp | 94 +++++++--------------- esphome/components/sdm_meter/sdm_meter.h | 3 +- 2 files changed, 31 insertions(+), 66 deletions(-) diff --git a/esphome/components/sdm_meter/sdm_meter.cpp b/esphome/components/sdm_meter/sdm_meter.cpp index 1ebc7fa3d8..f242b6b36d 100644 --- a/esphome/components/sdm_meter/sdm_meter.cpp +++ b/esphome/components/sdm_meter/sdm_meter.cpp @@ -1,85 +1,49 @@ #include "sdm_meter.h" #include "sdm_meter_registers.h" -#include "esphome/core/helpers.h" #include "esphome/core/log.h" namespace esphome::sdm_meter { static const char *const TAG = "sdm_meter"; -static const uint8_t MODBUS_REGISTER_COUNT = 80; // 74 x 16-bit registers +static const uint8_t MODBUS_REGISTER_COUNT = 80; // 80 x 16-bit registers (40 float values) -void SDMMeter::on_response(std::span request_pdu, std::span response_pdu) { - auto data = modbus::helpers::server_pdu_payload(response_pdu); - if (data.size() < MODBUS_REGISTER_COUNT * 2) { - ESP_LOGW(TAG, "Invalid size for SDMMeter!"); - return; - } +void SDMMeter::on_read_input_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) { + if (!modbus::succeeded(status)) + return; // the hub already logs exception responses - auto sdm_meter_get_float = [&](size_t i) -> float { - uint32_t temp = encode_uint32(data[i], data[i + 1], data[i + 2], data[i + 3]); - float f; - memcpy(&f, &temp, sizeof(f)); - return f; + // Publish a sensor if both of its registers are in this response; skipping absent registers keeps + // this correct for any read range, so the poll may be split into multiple requests. + auto publish = [&](uint16_t reg, sensor::Sensor *sensor) { + constexpr auto value_type = modbus::helpers::SensorValueType::FP32; + if (sensor == nullptr || reg < start_address) + return; + size_t offset = reg - start_address; + if (offset + modbus::helpers::register_width_for(value_type) > registers.size()) + return; + sensor->publish_state(modbus::helpers::registers_to_value(registers.data() + offset)); }; for (uint8_t i = 0; i < 3; i++) { - auto phase = this->phases_[i]; + auto &phase = this->phases_[i]; if (!phase.setup) continue; - - float voltage = sdm_meter_get_float(SDM_PHASE_1_VOLTAGE * 2 + (i * 4)); - float current = sdm_meter_get_float(SDM_PHASE_1_CURRENT * 2 + (i * 4)); - float active_power = sdm_meter_get_float(SDM_PHASE_1_ACTIVE_POWER * 2 + (i * 4)); - float apparent_power = sdm_meter_get_float(SDM_PHASE_1_APPARENT_POWER * 2 + (i * 4)); - float reactive_power = sdm_meter_get_float(SDM_PHASE_1_REACTIVE_POWER * 2 + (i * 4)); - float power_factor = sdm_meter_get_float(SDM_PHASE_1_POWER_FACTOR * 2 + (i * 4)); - float phase_angle = sdm_meter_get_float(SDM_PHASE_1_ANGLE * 2 + (i * 4)); - - ESP_LOGD( - TAG, - "SDMMeter Phase %c: V=%.3f V, I=%.3f A, Active P=%.3f W, Apparent P=%.3f VA, Reactive P=%.3f var, PF=%.3f, " - "PA=%.3f °", - i + 'A', voltage, current, active_power, apparent_power, reactive_power, power_factor, phase_angle); - if (phase.voltage_sensor_ != nullptr) - phase.voltage_sensor_->publish_state(voltage); - if (phase.current_sensor_ != nullptr) - phase.current_sensor_->publish_state(current); - if (phase.active_power_sensor_ != nullptr) - phase.active_power_sensor_->publish_state(active_power); - if (phase.apparent_power_sensor_ != nullptr) - phase.apparent_power_sensor_->publish_state(apparent_power); - if (phase.reactive_power_sensor_ != nullptr) - phase.reactive_power_sensor_->publish_state(reactive_power); - if (phase.power_factor_sensor_ != nullptr) - phase.power_factor_sensor_->publish_state(power_factor); - if (phase.phase_angle_sensor_ != nullptr) - phase.phase_angle_sensor_->publish_state(phase_angle); + publish(SDM_PHASE_1_VOLTAGE + i * 2, phase.voltage_sensor_); + publish(SDM_PHASE_1_CURRENT + i * 2, phase.current_sensor_); + publish(SDM_PHASE_1_ACTIVE_POWER + i * 2, phase.active_power_sensor_); + publish(SDM_PHASE_1_APPARENT_POWER + i * 2, phase.apparent_power_sensor_); + publish(SDM_PHASE_1_REACTIVE_POWER + i * 2, phase.reactive_power_sensor_); + publish(SDM_PHASE_1_POWER_FACTOR + i * 2, phase.power_factor_sensor_); + publish(SDM_PHASE_1_ANGLE + i * 2, phase.phase_angle_sensor_); } - float total_power = sdm_meter_get_float(SDM_TOTAL_SYSTEM_POWER * 2); - float frequency = sdm_meter_get_float(SDM_FREQUENCY * 2); - float import_active_energy = sdm_meter_get_float(SDM_IMPORT_ACTIVE_ENERGY * 2); - float export_active_energy = sdm_meter_get_float(SDM_EXPORT_ACTIVE_ENERGY * 2); - float import_reactive_energy = sdm_meter_get_float(SDM_IMPORT_REACTIVE_ENERGY * 2); - float export_reactive_energy = sdm_meter_get_float(SDM_EXPORT_REACTIVE_ENERGY * 2); - - ESP_LOGD(TAG, "SDMMeter: F=%.3f Hz, Im.A.E=%.3f Wh, Ex.A.E=%.3f Wh, Im.R.E=%.3f VARh, Ex.R.E=%.3f VARh, T.P=%.3f W", - frequency, import_active_energy, export_active_energy, import_reactive_energy, export_reactive_energy, - total_power); - - if (this->total_power_sensor_ != nullptr) - this->total_power_sensor_->publish_state(total_power); - if (this->frequency_sensor_ != nullptr) - this->frequency_sensor_->publish_state(frequency); - if (this->import_active_energy_sensor_ != nullptr) - this->import_active_energy_sensor_->publish_state(import_active_energy); - if (this->export_active_energy_sensor_ != nullptr) - this->export_active_energy_sensor_->publish_state(export_active_energy); - if (this->import_reactive_energy_sensor_ != nullptr) - this->import_reactive_energy_sensor_->publish_state(import_reactive_energy); - if (this->export_reactive_energy_sensor_ != nullptr) - this->export_reactive_energy_sensor_->publish_state(export_reactive_energy); + publish(SDM_TOTAL_SYSTEM_POWER, this->total_power_sensor_); + publish(SDM_FREQUENCY, this->frequency_sensor_); + publish(SDM_IMPORT_ACTIVE_ENERGY, this->import_active_energy_sensor_); + publish(SDM_EXPORT_ACTIVE_ENERGY, this->export_active_energy_sensor_); + publish(SDM_IMPORT_REACTIVE_ENERGY, this->import_reactive_energy_sensor_); + publish(SDM_EXPORT_REACTIVE_ENERGY, this->export_reactive_energy_sensor_); } void SDMMeter::update() { this->read_input_registers(0, MODBUS_REGISTER_COUNT); } diff --git a/esphome/components/sdm_meter/sdm_meter.h b/esphome/components/sdm_meter/sdm_meter.h index e09b74bbc0..80370010bd 100644 --- a/esphome/components/sdm_meter/sdm_meter.h +++ b/esphome/components/sdm_meter/sdm_meter.h @@ -55,7 +55,8 @@ class SDMMeter final : public PollingComponent, public modbus::ModbusClientDevic void update() override; - void on_response(std::span request_pdu, std::span response_pdu) override; + void on_read_input_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) override; void dump_config() override; From dfac9e1f11c291e1127fdc7b7a5b2dd537a995d1 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 13:35:10 -0700 Subject: [PATCH 017/147] [pzemac] Use the typed modbus read callback with address-based extraction (#18855) Co-authored-by: J. Nick Koston --- esphome/components/pzemac/pzemac.cpp | 103 ++++++++++++++------------- esphome/components/pzemac/pzemac.h | 5 +- 2 files changed, 58 insertions(+), 50 deletions(-) diff --git a/esphome/components/pzemac/pzemac.cpp b/esphome/components/pzemac/pzemac.cpp index d817888922..50c626ec7f 100644 --- a/esphome/components/pzemac/pzemac.cpp +++ b/esphome/components/pzemac/pzemac.cpp @@ -8,57 +8,62 @@ static const char *const TAG = "pzemac"; static const uint8_t PZEM_CMD_RESET_ENERGY = 0x42; static const uint8_t PZEM_REGISTER_COUNT = 10; // 10x 16-bit registers -void PZEMAC::on_response(std::span request_pdu, std::span response_pdu) { - auto data = modbus::helpers::server_pdu_payload(response_pdu); - if (data.size() < 20) { - ESP_LOGW(TAG, "Invalid size for PZEM AC!"); +// Register map, see https://github.com/esphome/feature-requests/issues/49#issuecomment-538636809 +// 32-bit values are two registers, low word first. +static const uint16_t PZEM_REGISTER_VOLTAGE = 0; // 1 register, 0.1 V +static const uint16_t PZEM_REGISTER_CURRENT = 1; // 2 registers, 0.001 A +static const uint16_t PZEM_REGISTER_ACTIVE_POWER = 3; // 2 registers, 0.1 W +static const uint16_t PZEM_REGISTER_ACTIVE_ENERGY = 5; // 2 registers, 1 Wh +static const uint16_t PZEM_REGISTER_FREQUENCY = 7; // 1 register, 0.1 Hz +static const uint16_t PZEM_REGISTER_POWER_FACTOR = 8; // 1 register, 0.01 + +void PZEMAC::on_read_input_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) { + if (!modbus::succeeded(status)) + return; // the hub already logs exception responses + + // Publish a sensor if its register(s) are in this response; skipping absent registers keeps this + // correct for any read range, so the poll may be split into multiple requests. + auto publish_1_register = [&](sensor::Sensor *sensor, uint16_t reg, float divisor) -> void { + if (sensor == nullptr || reg < start_address) + return; + size_t offset = reg - start_address; + if (offset >= registers.size()) + return; + sensor->publish_state(registers[offset] / divisor); + }; + + auto publish_2_registers = [&](sensor::Sensor *sensor, uint16_t reg, float divisor) -> void { + constexpr auto value_type = modbus::helpers::SensorValueType::U_DWORD_R; + if (sensor == nullptr || reg < start_address) + return; + size_t offset = reg - start_address; + if (offset + modbus::helpers::register_width_for(value_type) > registers.size()) + return; + sensor->publish_state(modbus::helpers::registers_to_value(registers.data() + offset) / divisor); + }; + + publish_1_register(this->voltage_sensor_, PZEM_REGISTER_VOLTAGE, 10.0f); + publish_2_registers(this->current_sensor_, PZEM_REGISTER_CURRENT, 1000.0f); + publish_2_registers(this->power_sensor_, PZEM_REGISTER_ACTIVE_POWER, 10.0f); + publish_2_registers(this->energy_sensor_, PZEM_REGISTER_ACTIVE_ENERGY, 1.0f); + publish_1_register(this->frequency_sensor_, PZEM_REGISTER_FREQUENCY, 10.0f); + publish_1_register(this->power_factor_sensor_, PZEM_REGISTER_POWER_FACTOR, 100.0f); +} + +void PZEMAC::on_custom_response(std::span request_pdu, std::span response_pdu, + modbus::ResponseStatus status) { + // The only custom request this component sends is the energy reset; acknowledge its echo here so + // the default unhandled-response warning stays meaningful. + if (!request_pdu.empty() && request_pdu[0] == PZEM_CMD_RESET_ENERGY) { + if (modbus::succeeded(status)) { + ESP_LOGD(TAG, "Energy reset acknowledged"); + } else { + ESP_LOGW(TAG, "Energy reset rejected"); + } return; } - - // See https://github.com/esphome/feature-requests/issues/49#issuecomment-538636809 - // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 - // 01 04 14 08 D1 00 6C 00 00 00 F4 00 00 00 26 00 00 01 F4 00 64 00 00 51 34 - // Id Cc Sz Volt- Current---- Power------ Energy----- Frequ PFact Alarm Crc-- - // 0 2 6 10 14 16 - - auto pzem_get_16bit = [&](size_t i) -> uint16_t { - return (uint16_t(data[i + 0]) << 8) | (uint16_t(data[i + 1]) << 0); - }; - auto pzem_get_32bit = [&](size_t i) -> uint32_t { - return (uint32_t(pzem_get_16bit(i + 2)) << 16) | (uint32_t(pzem_get_16bit(i + 0)) << 0); - }; - - uint16_t raw_voltage = pzem_get_16bit(0); - float voltage = raw_voltage / 10.0f; // max 6553.5 V - - uint32_t raw_current = pzem_get_32bit(2); - float current = raw_current / 1000.0f; // max 4294967.295 A - - uint32_t raw_active_power = pzem_get_32bit(6); - float active_power = raw_active_power / 10.0f; // max 429496729.5 W - - float active_energy = static_cast(pzem_get_32bit(10)); - - uint16_t raw_frequency = pzem_get_16bit(14); - float frequency = raw_frequency / 10.0f; - - uint16_t raw_power_factor = pzem_get_16bit(16); - float power_factor = raw_power_factor / 100.0f; - - ESP_LOGD(TAG, "PZEM AC: V=%.1f V, I=%.3f A, P=%.1f W, E=%.1f Wh, F=%.1f Hz, PF=%.2f", voltage, current, active_power, - active_energy, frequency, power_factor); - if (this->voltage_sensor_ != nullptr) - this->voltage_sensor_->publish_state(voltage); - if (this->current_sensor_ != nullptr) - this->current_sensor_->publish_state(current); - if (this->power_sensor_ != nullptr) - this->power_sensor_->publish_state(active_power); - if (this->energy_sensor_ != nullptr) - this->energy_sensor_->publish_state(active_energy); - if (this->frequency_sensor_ != nullptr) - this->frequency_sensor_->publish_state(frequency); - if (this->power_factor_sensor_ != nullptr) - this->power_factor_sensor_->publish_state(power_factor); + modbus::ModbusClientDevice::on_custom_response(request_pdu, response_pdu, status); } void PZEMAC::update() { this->read_input_registers(0, PZEM_REGISTER_COUNT); } diff --git a/esphome/components/pzemac/pzemac.h b/esphome/components/pzemac/pzemac.h index 171212d3ee..723b21e0b0 100644 --- a/esphome/components/pzemac/pzemac.h +++ b/esphome/components/pzemac/pzemac.h @@ -22,7 +22,10 @@ class PZEMAC final : public PollingComponent, public modbus::ModbusClientDevice void update() override; - void on_response(std::span request_pdu, std::span response_pdu) override; + void on_read_input_registers(uint16_t start_address, std::span registers, + modbus::ResponseStatus status) override; + void on_custom_response(std::span request_pdu, std::span response_pdu, + modbus::ResponseStatus status) override; void dump_config() override; From ed3429d3722c4a4dcd4a1f0b2d0729d5ddd83741 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:43:52 -0500 Subject: [PATCH 018/147] Bump resvg-py from 0.4.0 to 0.5.0 (#18864) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index da100ad0cd..63abc9c645 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import esphome-glyphsets==0.2.0 pillow==12.3.0 -resvg-py==0.4.0 +resvg-py==0.5.0 freetype-py==2.5.1 jinja2==3.1.6 bleak==3.0.2 From 0dc0cf83de8bf777a853ce68bc13043aebe5ee47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Metrich?= <45318189+FredM67@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:02:32 +0200 Subject: [PATCH 019/147] [mk2pvrouter] Add Mk2PVRouter component with sensor support (#8487) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.5 Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: J. Nick Koston --- CODEOWNERS | 1 + esphome/components/mk2pvrouter/__init__.py | 69 +++++++ .../components/mk2pvrouter/mk2pvrouter.cpp | 177 ++++++++++++++++++ esphome/components/mk2pvrouter/mk2pvrouter.h | 69 +++++++ .../components/mk2pvrouter/sensor/__init__.py | 27 +++ .../mk2pvrouter/sensor/mk2pvrouter_sensor.cpp | 24 +++ .../mk2pvrouter/sensor/mk2pvrouter_sensor.h | 15 ++ esphome/core/defines.h | 1 + tests/components/mk2pvrouter/common.yaml | 46 +++++ .../mk2pvrouter/test.esp32-idf.yaml | 3 + .../mk2pvrouter/test.esp8266-ard.yaml | 3 + .../mk2pvrouter/test.rp2040-ard.yaml | 3 + .../uart_9600_even_7bits/esp32-ard.yaml | 14 ++ .../uart_9600_even_7bits/esp32-idf.yaml | 14 ++ .../uart_9600_even_7bits/esp8266-ard.yaml | 14 ++ .../uart_9600_even_7bits/rp2040-ard.yaml | 14 ++ 16 files changed, 494 insertions(+) create mode 100644 esphome/components/mk2pvrouter/__init__.py create mode 100644 esphome/components/mk2pvrouter/mk2pvrouter.cpp create mode 100644 esphome/components/mk2pvrouter/mk2pvrouter.h create mode 100644 esphome/components/mk2pvrouter/sensor/__init__.py create mode 100644 esphome/components/mk2pvrouter/sensor/mk2pvrouter_sensor.cpp create mode 100644 esphome/components/mk2pvrouter/sensor/mk2pvrouter_sensor.h create mode 100644 tests/components/mk2pvrouter/common.yaml create mode 100644 tests/components/mk2pvrouter/test.esp32-idf.yaml create mode 100644 tests/components/mk2pvrouter/test.esp8266-ard.yaml create mode 100644 tests/components/mk2pvrouter/test.rp2040-ard.yaml create mode 100644 tests/test_build_components/common/uart_9600_even_7bits/esp32-ard.yaml create mode 100644 tests/test_build_components/common/uart_9600_even_7bits/esp32-idf.yaml create mode 100644 tests/test_build_components/common/uart_9600_even_7bits/esp8266-ard.yaml create mode 100644 tests/test_build_components/common/uart_9600_even_7bits/rp2040-ard.yaml diff --git a/CODEOWNERS b/CODEOWNERS index e1287ca275..13fae0664b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -350,6 +350,7 @@ esphome/components/mipi_spi/* @clydebarrow esphome/components/mitsubishi/* @RubyBailey esphome/components/mitsubishi_cn105/* @crnjan esphome/components/mixer/speaker/* @kahrendt +esphome/components/mk2pvrouter/* @FredM67 esphome/components/mlx90393/* @functionpointer esphome/components/mlx90614/* @jesserockz esphome/components/mmc5603/* @benhoff diff --git a/esphome/components/mk2pvrouter/__init__.py b/esphome/components/mk2pvrouter/__init__.py new file mode 100644 index 0000000000..d00b4ce8d0 --- /dev/null +++ b/esphome/components/mk2pvrouter/__init__.py @@ -0,0 +1,69 @@ +import esphome.codegen as cg +from esphome.components import uart +import esphome.config_validation as cv +from esphome.const import CONF_ID, CONF_TAG +from esphome.cpp_generator import MockObj +from esphome.types import ConfigType + +CODEOWNERS = ["@FredM67"] +DEPENDENCIES = ["uart"] + +mk2pvrouter_ns = cg.esphome_ns.namespace("mk2pvrouter") +Mk2PVRouter = mk2pvrouter_ns.class_("Mk2PVRouter", cg.Component, uart.UARTDevice) + +CONF_MK2PVROUTER_ID = "mk2pvrouter_id" + +# Tags are copied into a fixed-size buffer (MAX_TAG_SIZE = 8 in mk2pvrouter.h), +# which needs room for a trailing null terminator. +MAX_TAG_LEN = 7 + +MK2PVROUTER_LISTENER_SCHEMA = cv.Schema( + { + cv.GenerateID(CONF_MK2PVROUTER_ID): cv.use_id(Mk2PVRouter), + cv.Required(CONF_TAG): cv.All( + cv.string_strict, cv.Length(min=1, max=MAX_TAG_LEN), lambda x: x.upper() + ), + } +) + +CONFIG_SCHEMA = ( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(Mk2PVRouter), + } + ) + .extend(cv.COMPONENT_SCHEMA) + .extend(uart.UART_DEVICE_SCHEMA) +) + + +def final_validate(config: ConfigType) -> None: + # Validate UART settings + schema = uart.final_validate_device_schema( + "mk2pvrouter", + baud_rate=9600, + parity="EVEN", + data_bits=7, + stop_bits=1, + require_rx=True, + require_tx=False, + ) + schema(config) + + +FINAL_VALIDATE_SCHEMA = final_validate + + +_request_listener_slot = cg.slot_counter("MK2PVROUTER_LISTENER_COUNT") + + +async def register_mk2pvrouter_listener(mk2pvrouter: MockObj, var: MockObj) -> None: + """Register a listener with its hub and count it for the compile-time buffer size.""" + _request_listener_slot() + cg.add(mk2pvrouter.register_mk2pvrouter_listener(var)) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await uart.register_uart_device(var, config) diff --git a/esphome/components/mk2pvrouter/mk2pvrouter.cpp b/esphome/components/mk2pvrouter/mk2pvrouter.cpp new file mode 100644 index 0000000000..a9c922602b --- /dev/null +++ b/esphome/components/mk2pvrouter/mk2pvrouter.cpp @@ -0,0 +1,177 @@ +#include "mk2pvrouter.h" +#include "esphome/core/log.h" +#include + +namespace esphome::mk2pvrouter { + +static const char *const TAG = "mk2pvrouter"; + +constexpr uint8_t START_FRAME = 0x2; +constexpr uint8_t END_FRAME = 0x3; +constexpr uint8_t LINE_FEED = 0xa; +constexpr uint8_t CARRIAGE_RETURN = 0xd; +constexpr uint8_t TAB = 0x9; +constexpr uint8_t MAX_ITERATIONS = 128; +constexpr uint8_t CRC_MASK = 0x3F; +constexpr uint8_t CRC_OFFSET = 0x20; + +// Extracts a TAB-delimited field from [buf_start, buf_end) into dest. +// Returns the field length, or 0 if no TAB was found, or the (uncopied) field +// length if it's >= max_len. +static size_t get_field(char *dest, const char *buf_start, const char *buf_end, size_t max_len) { + const auto *const field_end = static_cast(memchr(buf_start, TAB, buf_end - buf_start)); + if (!field_end) + return 0; + const size_t len = field_end - buf_start; + if (len >= max_len) { + ESP_LOGE(TAG, "Field too long: %zu bytes (max %zu)", len, max_len); + return len; + } + + memcpy(dest, buf_start, len); + dest[len] = '\0'; // Null-terminate + return len; +} + +// Calculates the CRC (checksum) for a given group of characters. +uint8_t Mk2PVRouter::calculate_crc_(const char *grp, size_t grp_len) { + uint8_t crc_tmp{0}; + const auto effective_len = grp_len - CRC_SUFFIX_LEN; + for (size_t i = 0; i < effective_len; i++) { + crc_tmp += grp[i]; + } + crc_tmp &= CRC_MASK; + crc_tmp += CRC_OFFSET; + return crc_tmp; +} + +// Verifies the CRC of a group against its trailing CRC byte. +bool Mk2PVRouter::check_crc_(const char *grp, const char *grp_end) { + const auto grp_len = grp_end - grp; + if (grp_len < static_cast(CRC_SUFFIX_LEN)) { + ESP_LOGE(TAG, "Empty or too short group"); + return false; + } + const auto raw_crc = grp[grp_len - 1]; + + const auto calculated_crc = this->calculate_crc_(grp, grp_len); + + if (raw_crc != calculated_crc) { + ESP_LOGE(TAG, "CRC mismatch: expected %d, got %d", calculated_crc, raw_crc); + return false; + } + return true; +} + +// Validates, parses, and publishes a single tag/value group. +void Mk2PVRouter::process_group_(const char *grp, const char *grp_end) { + if (!this->check_crc_(grp, grp_end)) + return; + + size_t field_len = get_field(this->tag_, grp, grp_end, MAX_TAG_SIZE); + if (!field_len || field_len >= MAX_TAG_SIZE) { + ESP_LOGE(TAG, "Invalid tag"); + return; + } + const auto *val_start = grp + field_len + 1; // Skip tag + TAB. + + field_len = get_field(this->val_, val_start, grp_end, MAX_VAL_SIZE); + if (!field_len || field_len >= MAX_VAL_SIZE) { + ESP_LOGE(TAG, "Invalid value for tag %s", this->tag_); + return; + } + + this->publish_value_(this->tag_, this->val_); +} + +// Reads characters until `c` is found or the internal buffer is full. +bool Mk2PVRouter::read_chars_until_(bool drop, uint8_t c) { + size_t j{0}; + + while (this->available() > 0 && j++ < MAX_ITERATIONS) { + const auto received = this->read(); + if (received < 0) + continue; + if (received == c) + return true; + if (drop) + continue; + if (this->buf_index_ >= (sizeof(this->buf_) - 1)) { + ESP_LOGW(TAG, "Internal buffer full"); + this->buf_index_ = 0; + this->state_ = State::WAITING_FOR_START; + return false; + } + this->buf_[this->buf_index_++] = received; + } + + return false; +} + +void Mk2PVRouter::loop() { + switch (this->state_) { + case State::WAITING_FOR_START: + ESP_LOGVV(TAG, "State: WAITING_FOR_START"); + if (this->read_chars_until_(true, START_FRAME)) + this->state_ = State::START_FRAME_RECEIVED; + break; + case State::START_FRAME_RECEIVED: + ESP_LOGVV(TAG, "State: START_FRAME_RECEIVED"); + if (this->read_chars_until_(false, END_FRAME)) + this->state_ = State::END_FRAME_RECEIVED; + break; + case State::END_FRAME_RECEIVED: { + ESP_LOGVV(TAG, "State: END_FRAME_RECEIVED -> processing"); + + if (this->buf_index_ == 0) { + this->state_ = State::WAITING_FOR_START; + break; + } + + auto *buf_finger = this->buf_; + auto *buf_end = this->buf_ + this->buf_index_; + + // Each group: 0xa(LF) | Tag | 0x9(TAB) | Data | 0x9(TAB) | CRC | 0xd(CR) + // CRC is computed over "Tag | TAB | Data | TAB". + while ((buf_finger = static_cast(memchr(buf_finger, LINE_FEED, buf_end - buf_finger))) != nullptr) { + ++buf_finger; // Skip LF to the start of the group. + + auto *const grp_end = static_cast(memchr(buf_finger, CARRIAGE_RETURN, buf_end - buf_finger)); + if (!grp_end) { + ESP_LOGE(TAG, "No group found"); + break; + } + + this->process_group_(buf_finger, grp_end); + + buf_finger = grp_end; // grp_end is always < buf_end, so this stays in bounds. + } + this->buf_index_ = 0; + this->state_ = State::WAITING_FOR_START; + break; + } + } +} + +void Mk2PVRouter::publish_value_(const char *tag, const char *val) { +#ifdef MK2PVROUTER_LISTENER_COUNT + for (auto *element : this->mk2pvrouter_listeners_) { + if (strcmp(tag, element->get_tag()) != 0) + continue; + element->publish_val(val); + } +#endif +} + +void Mk2PVRouter::dump_config() { + ESP_LOGCONFIG(TAG, "Mk2PVRouter:"); + this->check_uart_settings(BAUD_RATE, 1, uart::UART_CONFIG_PARITY_EVEN, 7); +} + +#ifdef MK2PVROUTER_LISTENER_COUNT +void Mk2PVRouter::register_mk2pvrouter_listener(Mk2PVRouterListener *listener) { + this->mk2pvrouter_listeners_.push_back(listener); +} +#endif + +} // namespace esphome::mk2pvrouter diff --git a/esphome/components/mk2pvrouter/mk2pvrouter.h b/esphome/components/mk2pvrouter/mk2pvrouter.h new file mode 100644 index 0000000000..f542436f1d --- /dev/null +++ b/esphome/components/mk2pvrouter/mk2pvrouter.h @@ -0,0 +1,69 @@ +#pragma once + +#include "esphome/components/uart/uart.h" +#include "esphome/core/component.h" +#include "esphome/core/defines.h" +#include "esphome/core/helpers.h" + +namespace esphome::mk2pvrouter { +/* + * Buffer sizes based on the mk2pvrouter telemetry protocol, as implemented by the + * firmware's teleinfo.h (see github.com/FredM67/PVRouter-{1,3}-phase): + * - Tags: max 4 chars (S_MC is longest), most are 1-2 chars (P, V1, R2, etc.) + * - Values: max 6 digits signed (-10000), typical 1-5 digits. Energy (E) is a daily + * counter reset at midnight, so it stays well within 6 digits. + * - Frame: STX + multiple lines (LF+tag+TAB+value+TAB+crc+CR) + ETX + * - Line format: \n\t\t\r (8-15 bytes per line) + * - Multi-phase with all features: ~150-200 bytes + */ +static constexpr uint8_t MAX_TAG_SIZE = 8; // S_MC (4) + digit (1) + null (1) + margin (2) +static constexpr uint8_t MAX_VAL_SIZE = 8; // -10000 (6) + null (1) + margin (1) +static constexpr uint16_t MAX_BUF_SIZE = 256; // Full frame with all features enabled + +// Listener interface for entities that want updates for a specific tag. +class Mk2PVRouterListener { + public: + explicit Mk2PVRouterListener(const char *tag) : tag_(tag) {} + virtual ~Mk2PVRouterListener() = default; + const char *get_tag() const { return this->tag_; } + virtual void publish_val(const char *val) = 0; + + protected: + const char *tag_; +}; + +// Reads frames via UART, validates their CRC, and publishes tag/value pairs to listeners. +class Mk2PVRouter final : public Component, public uart::UARTDevice { + public: +#ifdef MK2PVROUTER_LISTENER_COUNT + void register_mk2pvrouter_listener(Mk2PVRouterListener *listener); +#endif + void loop() override; + void dump_config() override; + + protected: + static constexpr size_t CRC_SUFFIX_LEN = 1; + static constexpr uint32_t BAUD_RATE = 9600; + + enum class State : uint8_t { + WAITING_FOR_START, + START_FRAME_RECEIVED, + END_FRAME_RECEIVED, + }; + +#ifdef MK2PVROUTER_LISTENER_COUNT + StaticVector mk2pvrouter_listeners_; +#endif + uint16_t buf_index_{0}; + State state_{State::WAITING_FOR_START}; + char tag_[MAX_TAG_SIZE]; + char val_[MAX_VAL_SIZE]; + char buf_[MAX_BUF_SIZE]; // Large buffer last to reduce padding + + bool read_chars_until_(bool drop, uint8_t c); + uint8_t calculate_crc_(const char *grp, size_t grp_len); + bool check_crc_(const char *grp, const char *grp_end); + void process_group_(const char *grp, const char *grp_end); + void publish_value_(const char *tag, const char *val); +}; +} // namespace esphome::mk2pvrouter diff --git a/esphome/components/mk2pvrouter/sensor/__init__.py b/esphome/components/mk2pvrouter/sensor/__init__.py new file mode 100644 index 0000000000..14fc48a626 --- /dev/null +++ b/esphome/components/mk2pvrouter/sensor/__init__.py @@ -0,0 +1,27 @@ +import esphome.codegen as cg +from esphome.components import sensor +from esphome.const import CONF_ID, CONF_TAG +from esphome.types import ConfigType + +from .. import ( + CONF_MK2PVROUTER_ID, + MK2PVROUTER_LISTENER_SCHEMA, + mk2pvrouter_ns, + register_mk2pvrouter_listener, +) + +Mk2PVRouterSensor = mk2pvrouter_ns.class_( + "Mk2PVRouterSensor", sensor.Sensor, cg.Component +) + +CONFIG_SCHEMA = sensor.sensor_schema(Mk2PVRouterSensor).extend( + MK2PVROUTER_LISTENER_SCHEMA +) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID], config[CONF_TAG]) + await cg.register_component(var, config) + await sensor.register_sensor(var, config) + mk2pvrouter = await cg.get_variable(config[CONF_MK2PVROUTER_ID]) + await register_mk2pvrouter_listener(mk2pvrouter, var) diff --git a/esphome/components/mk2pvrouter/sensor/mk2pvrouter_sensor.cpp b/esphome/components/mk2pvrouter/sensor/mk2pvrouter_sensor.cpp new file mode 100644 index 0000000000..96f1ff5954 --- /dev/null +++ b/esphome/components/mk2pvrouter/sensor/mk2pvrouter_sensor.cpp @@ -0,0 +1,24 @@ +#include "mk2pvrouter_sensor.h" +#include "esphome/core/log.h" + +namespace esphome::mk2pvrouter { + +static const char *const TAG = "mk2pvrouter_sensor"; + +Mk2PVRouterSensor::Mk2PVRouterSensor(const char *tag) : Mk2PVRouterListener(tag) {} + +void Mk2PVRouterSensor::publish_val(const char *val) { + auto result = parse_number(val); + if (!result.has_value()) { + ESP_LOGW(TAG, "Failed to parse value '%s' for tag '%s'", val, this->get_tag()); + return; + } + this->publish_state(result.value()); +} + +void Mk2PVRouterSensor::dump_config() { + LOG_SENSOR(" ", "Mk2PVRouter Sensor", this); + ESP_LOGCONFIG(TAG, " Tag: %s", this->get_tag()); +} + +} // namespace esphome::mk2pvrouter diff --git a/esphome/components/mk2pvrouter/sensor/mk2pvrouter_sensor.h b/esphome/components/mk2pvrouter/sensor/mk2pvrouter_sensor.h new file mode 100644 index 0000000000..e4da41e384 --- /dev/null +++ b/esphome/components/mk2pvrouter/sensor/mk2pvrouter_sensor.h @@ -0,0 +1,15 @@ +#pragma once + +#include "esphome/components/mk2pvrouter/mk2pvrouter.h" +#include "esphome/components/sensor/sensor.h" + +namespace esphome::mk2pvrouter { + +class Mk2PVRouterSensor final : public Mk2PVRouterListener, public sensor::Sensor, public Component { + public: + explicit Mk2PVRouterSensor(const char *tag); + void publish_val(const char *val) override; + void dump_config() override; +}; + +} // namespace esphome::mk2pvrouter diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 90ecfea72a..625d4879f5 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -134,6 +134,7 @@ #define MDNS_DYNAMIC_TXT_COUNT 2 #define MICRONOVA_LISTENER_COUNT 1 #define USE_MICRONOVA_WRITER +#define MK2PVROUTER_LISTENER_COUNT 1 #define SERIAL_PROXY_COUNT 2 #define SNTP_SERVER_COUNT 3 #define USE_MEDIA_PLAYER diff --git a/tests/components/mk2pvrouter/common.yaml b/tests/components/mk2pvrouter/common.yaml new file mode 100644 index 0000000000..4421c09854 --- /dev/null +++ b/tests/components/mk2pvrouter/common.yaml @@ -0,0 +1,46 @@ +mk2pvrouter: + id: test_mk2pvrouter + uart_id: uart_bus + +sensor: + - platform: mk2pvrouter + name: Power + tag: P + mk2pvrouter_id: test_mk2pvrouter + unit_of_measurement: W + device_class: power + state_class: measurement + accuracy_decimals: 0 + + - platform: mk2pvrouter + name: Voltage + tag: V + mk2pvrouter_id: test_mk2pvrouter + unit_of_measurement: V + device_class: voltage + state_class: measurement + accuracy_decimals: 2 + filters: + # Device sends voltage * 100 + - multiply: 0.01 + + - platform: mk2pvrouter + name: Energy + tag: E + mk2pvrouter_id: test_mk2pvrouter + unit_of_measurement: Wh + device_class: energy + state_class: total_increasing + accuracy_decimals: 0 + + - platform: mk2pvrouter + name: Temperature + tag: T1 + mk2pvrouter_id: test_mk2pvrouter + unit_of_measurement: "°C" + device_class: temperature + state_class: measurement + accuracy_decimals: 2 + filters: + # Device sends temperature * 100 + - multiply: 0.01 diff --git a/tests/components/mk2pvrouter/test.esp32-idf.yaml b/tests/components/mk2pvrouter/test.esp32-idf.yaml new file mode 100644 index 0000000000..66539a4dd7 --- /dev/null +++ b/tests/components/mk2pvrouter/test.esp32-idf.yaml @@ -0,0 +1,3 @@ +packages: + uart_9600_even_7bits: !include ../../test_build_components/common/uart_9600_even_7bits/esp32-idf.yaml + mk2pvrouter: !include common.yaml diff --git a/tests/components/mk2pvrouter/test.esp8266-ard.yaml b/tests/components/mk2pvrouter/test.esp8266-ard.yaml new file mode 100644 index 0000000000..50a45a6ca5 --- /dev/null +++ b/tests/components/mk2pvrouter/test.esp8266-ard.yaml @@ -0,0 +1,3 @@ +packages: + uart_9600_even_7bits: !include ../../test_build_components/common/uart_9600_even_7bits/esp8266-ard.yaml + mk2pvrouter: !include common.yaml diff --git a/tests/components/mk2pvrouter/test.rp2040-ard.yaml b/tests/components/mk2pvrouter/test.rp2040-ard.yaml new file mode 100644 index 0000000000..f8a5a620b3 --- /dev/null +++ b/tests/components/mk2pvrouter/test.rp2040-ard.yaml @@ -0,0 +1,3 @@ +packages: + uart_9600_even_7bits: !include ../../test_build_components/common/uart_9600_even_7bits/rp2040-ard.yaml + mk2pvrouter: !include common.yaml diff --git a/tests/test_build_components/common/uart_9600_even_7bits/esp32-ard.yaml b/tests/test_build_components/common/uart_9600_even_7bits/esp32-ard.yaml new file mode 100644 index 0000000000..f0d24b9a18 --- /dev/null +++ b/tests/test_build_components/common/uart_9600_even_7bits/esp32-ard.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP32 Arduino tests - 9600 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO17 + rx_pin: GPIO16 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 9600 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_9600_even_7bits/esp32-idf.yaml b/tests/test_build_components/common/uart_9600_even_7bits/esp32-idf.yaml new file mode 100644 index 0000000000..e85fa7fc71 --- /dev/null +++ b/tests/test_build_components/common/uart_9600_even_7bits/esp32-idf.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP32 IDF tests - 9600 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO17 + rx_pin: GPIO16 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 9600 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_9600_even_7bits/esp8266-ard.yaml b/tests/test_build_components/common/uart_9600_even_7bits/esp8266-ard.yaml new file mode 100644 index 0000000000..488bfdbeab --- /dev/null +++ b/tests/test_build_components/common/uart_9600_even_7bits/esp8266-ard.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP8266 Arduino tests - 9600 baud even parity, 7 data bits + +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 9600 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_9600_even_7bits/rp2040-ard.yaml b/tests/test_build_components/common/uart_9600_even_7bits/rp2040-ard.yaml new file mode 100644 index 0000000000..08bec00820 --- /dev/null +++ b/tests/test_build_components/common/uart_9600_even_7bits/rp2040-ard.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for RP2040 Arduino tests - 9600 baud even parity, 7 data bits + +substitutions: + tx_pin: GPIO0 + rx_pin: GPIO1 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 9600 + parity: EVEN + data_bits: 7 + stop_bits: 1 From c9848d8fa66fced271b8ceb815fb7750d275d258 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 29 Aug 2026 08:37:20 +1000 Subject: [PATCH 020/147] [light] Fix gamma table dead zone collapsing to 0 (#18845) --- .../components/light/esp_color_correction.cpp | 6 +- .../light/test_gamma_correction.cpp | 92 +++++++++++++++++++ .../components/light/test_gamma_table.py | 17 +++- 3 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 tests/components/light/test_gamma_correction.cpp diff --git a/esphome/components/light/esp_color_correction.cpp b/esphome/components/light/esp_color_correction.cpp index e793226bb1..12eb6a3008 100644 --- a/esphome/components/light/esp_color_correction.cpp +++ b/esphome/components/light/esp_color_correction.cpp @@ -5,7 +5,11 @@ namespace esphome::light { uint8_t ESPColorCorrection::gamma_correct_(uint8_t value) const { if (this->gamma_table_ == nullptr) return value; - return static_cast((progmem_read_uint16(&this->gamma_table_[value]) + 128) / 257); + uint16_t table_value = progmem_read_uint16(&this->gamma_table_[value]); + uint8_t result = (table_value + 128) / 257; + if (result == 0 && table_value != 0) + return 1; + return result; } uint8_t ESPColorCorrection::gamma_uncorrect_(uint8_t value) const { diff --git a/tests/components/light/test_gamma_correction.cpp b/tests/components/light/test_gamma_correction.cpp new file mode 100644 index 0000000000..4b8d83c544 --- /dev/null +++ b/tests/components/light/test_gamma_correction.cpp @@ -0,0 +1,92 @@ +#include + +#include +#include +#include +#include + +#include "esphome/components/light/esp_color_correction.h" + +namespace esphome::light::testing { + +namespace { + +// A representative fixture for ESPColorCorrection/gamma_table_reverse_search tests below -- +// not a spec for generate_gamma_table() itself, which the Python tests own. +std::array build_gamma_table(double gamma) { + std::array table{}; + table[0] = 0; + for (int i = 1; i < 256; i++) { + double raw = std::round(std::pow(i / 255.0, gamma) * 65535.0); + table[i] = static_cast(std::max(1.0, std::min(65535.0, raw))); + } + return table; +} + +// Bundles a table with an ESPColorCorrection pointing at it, since the correction only holds +// a raw pointer into the table and doesn't own it. +struct GammaFixture { + explicit GammaFixture(double gamma) : table(build_gamma_table(gamma)) { correction.set_gamma_table(table.data()); } + std::array table; + ESPColorCorrection correction; +}; + +} // namespace + +// Regression test for esphome/esphome#18842: ESPColorCorrection's own 16-bit -> 8-bit +// conversion must never round a non-zero table entry down to a zero 8-bit output. +TEST(GammaCorrection, NonZeroInputsSurviveConversion) { + for (double gamma : {1.0, 1.8, 2.0, 2.2, 2.8, 3.0, 4.0}) { + GammaFixture fixture(gamma); + for (int i = 1; i < 256; i++) { + EXPECT_GE(fixture.correction.color_correct_red(i), 1) << "gamma=" << gamma << " index=" << i; + } + } +} + +TEST(GammaCorrection, ZeroInputStaysZero) { + for (double gamma : {1.0, 2.2, 2.8, 4.0}) { + GammaFixture fixture(gamma); + EXPECT_EQ(fixture.correction.color_correct_red(0), 0) << "gamma=" << gamma; + } +} + +TEST(GammaCorrection, FullBrightnessStaysFull) { + for (double gamma : {1.0, 2.2, 2.8, 4.0}) { + GammaFixture fixture(gamma); + EXPECT_EQ(fixture.correction.color_correct_red(255), 255) << "gamma=" << gamma; + } +} + +// Reproduces the reporter's own numbers from esphome/esphome#18842 at gamma=2.8: codes +// 1-27 previously collapsed to an 8-bit output of 0 and must now be non-zero. +TEST(GammaCorrection, DeadZoneFixedAtGamma28) { + GammaFixture fixture(2.8); + for (int i = 1; i < 28; i++) { + EXPECT_GE(fixture.correction.color_correct_red(i), 1) << "index=" << i << " still collapses to 0"; + } +} + +TEST(GammaCorrection, ReverseSearchFindsLargestIndexLessEqualTarget) { + auto table = build_gamma_table(2.8); + for (uint16_t target : {0, 128, 129, 135, 1000, 32768, 65535}) { + uint8_t lo = gamma_table_reverse_search(table.data(), target); + EXPECT_LE(table[lo], target) << "target=" << target; + if (lo < 255) { + EXPECT_GT(table[lo + 1], target) << "target=" << target; + } + } +} + +// color_uncorrect_* binary-searches the table via gamma_table_reverse_search(). +TEST(GammaCorrection, UncorrectStaysMonotonic) { + GammaFixture fixture(2.8); + uint8_t prev = 0; + for (int i = 1; i < 256; i++) { + uint8_t result = fixture.correction.color_uncorrect_red(i); + EXPECT_GE(result, prev) << "index=" << i; + prev = result; + } +} + +} // namespace esphome::light::testing diff --git a/tests/unit_tests/components/light/test_gamma_table.py b/tests/unit_tests/components/light/test_gamma_table.py index a302a355dc..75c3f18e42 100644 --- a/tests/unit_tests/components/light/test_gamma_table.py +++ b/tests/unit_tests/components/light/test_gamma_table.py @@ -53,9 +53,12 @@ def test_nonzero_indices_are_nonzero(gamma: float) -> None: assert table[i] >= 1, f"gamma={gamma}, index {i}: got {table[i]}" -@pytest.mark.parametrize("gamma", [1.0, 2.0, 2.2, 2.8, 3.0]) +@pytest.mark.parametrize("gamma", [1.0, 1.8, 2.0, 2.2, 2.8, 3.0, 4.0]) def test_table_monotonically_nondecreasing(gamma: float) -> None: - """The gamma table must be monotonically non-decreasing.""" + """The gamma table must be monotonically non-decreasing. + + gamma_table_reverse_search()'s binary search depends on this. + """ table = generate_gamma_table(gamma) for i in range(1, 256): assert table[i] >= table[i - 1], ( @@ -115,3 +118,13 @@ def test_lut_output_monotonically_nondecreasing() -> None: result = _simulate_gamma_correct_lut(table, value) assert result >= prev, f"value={value}: result {result} < previous {prev}" prev = result + + +def test_table_matches_raw_power_curve() -> None: + """Check the gamma table against known good values for gamma=2.8.""" + table = generate_gamma_table(2.8) + golden = {1: 1, 5: 1, 15: 24, 27: 122, 28: 135, 100: 4766, 200: 33193, 254: 64818} + for i, expected in golden.items(): + assert table[i] == expected, ( + f"index {i}: table[{i}]={table[i]} expected {expected}" + ) From 1c44cec343e997d15fa70796ba8d7b074a872622 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 15:44:00 -0700 Subject: [PATCH 021/147] [modbus] Add value_at() and decode registers without the byte round-trip (#18873) --- esphome/components/modbus/modbus_helpers.cpp | 55 +++++++++--- esphome/components/modbus/modbus_helpers.h | 42 ++++++++- .../components/modbus/modbus_helpers_test.cpp | 85 ++++++++++++++++++- 3 files changed, 163 insertions(+), 19 deletions(-) diff --git a/esphome/components/modbus/modbus_helpers.cpp b/esphome/components/modbus/modbus_helpers.cpp index 92bd06cdf5..d80e6c86ad 100644 --- a/esphome/components/modbus/modbus_helpers.cpp +++ b/esphome/components/modbus/modbus_helpers.cpp @@ -292,25 +292,52 @@ std::optional payload_to_number(const uint8_t *data, size_t size, Senso } std::optional registers_to_number(const uint16_t *registers, size_t count, SensorValueType sensor_value_type) { - const size_t required_size = required_payload_size(sensor_value_type); - if (required_size == 0) { - return 0; // RAW/unsupported: nothing to read + // RAW and BIT carry no fixed-width number, so there is nothing to decode whatever the span holds. + // register_width_for() reports 1 for them, so this must be checked before the width test below. + if (sensor_value_type == SensorValueType::RAW || sensor_value_type == SensorValueType::BIT) { + return 0; } - const size_t required_words = required_size / 2; + const uint16_t required_words = register_width_for(sensor_value_type); if (required_words > count) { - ESP_LOGE(TAG, "not enough registers for value type=%u count=%zu required=%zu", - static_cast(sensor_value_type), count, required_words); + ESP_LOGE(TAG, "not enough registers for value type=%u count=%zu required=%u", + static_cast(sensor_value_type), count, static_cast(required_words)); return std::nullopt; } - // Serialize the needed words back to big-endian bytes and reuse the audited byte decoder so the - // sign-extension behaviour stays identical to the wire path. - uint8_t bytes[8]; // at most 4 registers (QWORD) - for (size_t i = 0; i < required_words; i++) { - uint16_t reg = registers[i]; - bytes[i * 2] = static_cast(reg >> 8); - bytes[i * 2 + 1] = static_cast(reg & 0xFF); + // Registers are the wire's own unit, so decode them directly rather than serializing back to bytes. + // Each case defers to registers_to_value() so the word order and sign rules have one definition, with + // two deliberate exceptions matching what the byte decoder returned: the float types yield their bit + // pattern rather than a float, and U_QWORD shares the signed branch because the return type is int64_t. + switch (sensor_value_type) { + case SensorValueType::U_WORD: + return registers_to_value(registers); + case SensorValueType::U_WORD_S: + return registers_to_value(registers); + case SensorValueType::S_WORD: + return registers_to_value(registers); + case SensorValueType::S_WORD_S: + return registers_to_value(registers); + case SensorValueType::U_DWORD: + return registers_to_value(registers); + case SensorValueType::U_DWORD_R: + return registers_to_value(registers); + case SensorValueType::S_DWORD: + return registers_to_value(registers); + case SensorValueType::S_DWORD_R: + return registers_to_value(registers); + case SensorValueType::FP32: + return registers_to_uint32(registers[0], registers[1]); + case SensorValueType::FP32_R: + return registers_to_uint32(registers[1], registers[0]); + // Signed for both: an unsigned QWORD above INT64_MAX has to come back as a negative int64_t. + case SensorValueType::U_QWORD: + case SensorValueType::S_QWORD: + return registers_to_value(registers); + case SensorValueType::U_QWORD_R: + case SensorValueType::S_QWORD_R: + return registers_to_value(registers); + default: + return 0; } - return payload_to_number(bytes, required_size, sensor_value_type, 0, 0xFFFFFFFF); } // Append a 16-bit value to a PDU in big-endian (wire) byte order. diff --git a/esphome/components/modbus/modbus_helpers.h b/esphome/components/modbus/modbus_helpers.h index 486064da01..9488a88088 100644 --- a/esphome/components/modbus/modbus_helpers.h +++ b/esphome/components/modbus/modbus_helpers.h @@ -229,7 +229,7 @@ inline bool value_type_is_float(SensorValueType v) { } /// Number of 16-bit registers a value of this type occupies (RAW counts as one register). -inline uint16_t register_width_for(SensorValueType v) { +constexpr uint16_t register_width_for(SensorValueType v) { switch (v) { case SensorValueType::U_DWORD: case SensorValueType::S_DWORD: @@ -478,6 +478,11 @@ constexpr uint32_t registers_to_uint32(uint16_t high_word, uint16_t low_word) { return (static_cast(high_word) << 16) | low_word; } +/// Combine four register words into a 64-bit value, most significant word first. +constexpr uint64_t registers_to_uint64(uint16_t word0, uint16_t word1, uint16_t word2, uint16_t word3) { + return (static_cast(registers_to_uint32(word0, word1)) << 32) | registers_to_uint32(word2, word3); +} + // Always false, whatever the type: it exists only to make the static_assert below depend on the // template argument. Not a queryable trait. template inline constexpr bool VALUE_TYPE_SUPPORTED = false; @@ -486,8 +491,8 @@ template inline constexpr bool VALUE_TYPE_SUPPORTED = false; * Unlike registers_to_number(), the type is a template argument, so only the one decode is compiled * and the caller gets the value's natural type back rather than an int64_t. The "_R" types take the * low word first; the rest take the high word first. - * Supports the WORD, DWORD and FP32 types, including their _S and _R forms; the QWORD types are - * out of scope and fail to compile, so use registers_to_number() for those. + * Supports every fixed-width type: the WORD, DWORD, QWORD and FP32 families, including their _S and + * _R forms. RAW and BIT have no fixed width and fail to compile. * Use register_width_for() for the number of registers the caller must supply. * Note that the FP32 branches are only usable in a constant expression where std::bit_cast is * available; elsewhere bit_cast falls back to a non-constexpr memcpy (see core/helpers.h). @@ -513,11 +518,42 @@ template constexpr auto registers_to_value(const uin return bit_cast(registers_to_uint32(registers[0], registers[1])); } else if constexpr (VALUE_TYPE == SensorValueType::FP32_R) { return bit_cast(registers_to_uint32(registers[1], registers[0])); + } else if constexpr (VALUE_TYPE == SensorValueType::U_QWORD) { + return registers_to_uint64(registers[0], registers[1], registers[2], registers[3]); + } else if constexpr (VALUE_TYPE == SensorValueType::U_QWORD_R) { + return registers_to_uint64(registers[3], registers[2], registers[1], registers[0]); + } else if constexpr (VALUE_TYPE == SensorValueType::S_QWORD) { + return static_cast(registers_to_uint64(registers[0], registers[1], registers[2], registers[3])); + } else if constexpr (VALUE_TYPE == SensorValueType::S_QWORD_R) { + return static_cast(registers_to_uint64(registers[3], registers[2], registers[1], registers[0])); } else { static_assert(VALUE_TYPE_SUPPORTED, "registers_to_value() does not support this value type"); } } +/// The type registers_to_value() yields for a given value type. Distinct from modbus::RegisterValues, +/// which is a container of raw words. +template +using RegisterValueType = decltype(registers_to_value(static_cast(nullptr))); + +/** The value stored at an absolute register address, or nullopt when it is not wholly inside this + * response. Lets a device decode by address rather than by offset, so a poll split across several + * requests needs no extra bookkeeping: a value outside the response simply yields nullopt. + * @param registers the response registers, in host byte order + * @param start_address the address the response begins at + * @param address the address of the wanted value + */ +template +constexpr std::optional> value_at(std::span registers, + uint16_t start_address, uint16_t address) { + if (address < start_address) + return std::nullopt; + const size_t offset = static_cast(address) - start_address; + if (offset + register_width_for(VALUE_TYPE) > registers.size()) + return std::nullopt; + return registers_to_value(registers.data() + offset); +} + /// The widest standard numeric value (a QWORD) spans 4 registers, so one entity value never writes more. static constexpr uint16_t MAX_FEW_REGISTERS = 4; diff --git a/tests/components/modbus/modbus_helpers_test.cpp b/tests/components/modbus/modbus_helpers_test.cpp index 21c264ea69..a42625760d 100644 --- a/tests/components/modbus/modbus_helpers_test.cpp +++ b/tests/components/modbus/modbus_helpers_test.cpp @@ -427,14 +427,37 @@ TEST(ModbusHelpersTest, RegistersToNumberMatchesPayloadToNumber) { } } +TEST(ModbusHelpersTest, RegistersToNumberMatchesPayloadToNumberForQwords) { + // The word shuffle the QWORD_R decode replaces is the least obvious code in the byte path, so pin + // it against that path rather than against registers_to_value(). The top bit is set, which is where + // U_QWORD's unsigned value and this function's int64_t return deliberately diverge. + const uint16_t registers[] = {0xF123, 0x4567, 0x89AB, 0xCDEF}; + const std::vector bytes{0xF1, 0x23, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF}; + for (auto value_type : + {SensorValueType::U_QWORD, SensorValueType::S_QWORD, SensorValueType::U_QWORD_R, SensorValueType::S_QWORD_R}) { + EXPECT_EQ(registers_to_number(registers, 4, value_type), + payload_to_number(std::span(bytes), value_type, 0, 0xFFFFFFFF)) + << "value_type=" << static_cast(value_type); + } +} + +TEST(ModbusHelpersTest, RegistersToNumberTreatsRawAndBitAsNothingToDecode) { + // Both have no fixed-width number, so they decode to 0 whatever the span holds - including none. + const uint16_t registers[] = {0x1234}; + EXPECT_EQ(registers_to_number(registers, 1, SensorValueType::RAW), std::optional(0)); + EXPECT_EQ(registers_to_number(registers, 0, SensorValueType::RAW), std::optional(0)); + EXPECT_EQ(registers_to_number(registers, 0, SensorValueType::BIT), std::optional(0)); +} + TEST(ModbusHelpersTest, RegistersToNumberRejectsTruncatedMultiRegisterValue) { const uint16_t registers[] = {0x1234}; EXPECT_FALSE(registers_to_number(registers, 1, SensorValueType::U_DWORD).has_value()); } // --- registers_to_value ---------------------------------------------------- -// The compile-time decoder must agree with the runtime one for every type it supports, -// so the two implementations cannot drift apart. +// registers_to_number() dispatches to registers_to_value(), so this checks the dispatch table picks +// the right specialisation for each type, not that two implementations agree. The independent check +// against the byte decoder is RegistersToNumberMatchesPayloadToNumber below. template void expect_matches_registers_to_number(const uint16_t *registers) { const auto expected = registers_to_number(registers, register_width_for(VALUE_TYPE), VALUE_TYPE); @@ -472,6 +495,64 @@ TEST(ModbusHelpersTest, RegistersToUint32CombinesWordsHighFirst) { EXPECT_EQ(registers_to_uint32(0x1234, 0x5678), 0x12345678u); } +// --- value_at --------------------------------------------------------------- +// Addresses are absolute; anything not wholly inside the response yields nullopt. + +TEST(ModbusHelpersTest, ValueAtDecodesByAbsoluteAddress) { + const uint16_t registers[] = {0x1111, 0x2222, 0x3333}; + const std::span span(registers, 3); + EXPECT_EQ(value_at(span, 100, 100), std::optional(0x1111)); + EXPECT_EQ(value_at(span, 100, 102), std::optional(0x3333)); + EXPECT_EQ(value_at(span, 100, 101), std::optional(0x22223333u)); + // Types whose RegisterValueType<> is not an unsigned integer, and the widest bounds check. + const uint16_t floats[] = {0x4048, 0xF5C3, 0xF5C3, 0x4048}; + const std::span float_span(floats, 4); + EXPECT_FLOAT_EQ(value_at(float_span, 10, 10).value_or(0.0f), 3.14f); + EXPECT_FLOAT_EQ(value_at(float_span, 10, 12).value_or(0.0f), 3.14f); + EXPECT_EQ(value_at(float_span, 10, 10), std::optional(0x4048F5C3F5C34048ULL)); + EXPECT_FALSE(value_at(float_span, 10, 11).has_value()); +} + +TEST(ModbusHelpersTest, ValueAtIsUsableInAConstantExpression) { + static constexpr uint16_t REGISTERS[] = {0x1234, 0x5678}; + static_assert(value_at(REGISTERS, 7, 7).value_or(0) == 0x12345678u); + static_assert(!value_at(REGISTERS, 7, 6).has_value()); +} + +TEST(ModbusHelpersTest, ValueAtRejectsAddressesOutsideTheResponse) { + const uint16_t registers[] = {0x1111, 0x2222, 0x3333}; + const std::span span(registers, 3); + // Below the response: must not wrap when the subtraction would go negative. + EXPECT_FALSE(value_at(span, 100, 99).has_value()); + EXPECT_FALSE(value_at(span, 100, 0).has_value()); + // Past the end, and a multi-register value truncated by the end of the response. + EXPECT_FALSE(value_at(span, 100, 103).has_value()); + EXPECT_FALSE(value_at(span, 100, 102).has_value()); + EXPECT_TRUE(value_at(span, 100, 101).has_value()); +} + +TEST(ModbusHelpersTest, ValueAtHandlesAnEmptyResponse) { + EXPECT_FALSE(value_at(std::span(), 0, 0).has_value()); +} + +// --- QWORD decoding --------------------------------------------------------- + +TEST(ModbusHelpersTest, RegistersToValueDecodesQwordBothWordOrders) { + const uint16_t registers[] = {0x0123, 0x4567, 0x89AB, 0xCDEF}; + EXPECT_EQ(registers_to_value(registers), 0x0123456789ABCDEFULL); + const uint16_t reversed[] = {0xCDEF, 0x89AB, 0x4567, 0x0123}; + EXPECT_EQ(registers_to_value(reversed), 0x0123456789ABCDEFULL); + // Signed reading of the same bits, and the sign-extreme case. + EXPECT_EQ(registers_to_value(registers), 0x0123456789ABCDEFLL); + const uint16_t negative[] = {0xFFFF, 0xFFFF, 0xFFFF, 0xFFFE}; + EXPECT_EQ(registers_to_value(negative), -2); + EXPECT_EQ(registers_to_value(negative), 0xFFFFFFFFFFFFFFFEULL); +} + +TEST(ModbusHelpersTest, RegistersToUint64CombinesWordsHighFirst) { + EXPECT_EQ(registers_to_uint64(0x0123, 0x4567, 0x89AB, 0xCDEF), 0x0123456789ABCDEFULL); +} + // --- packed bit helpers ------------------------------------------------------ TEST(ModbusHelpersTest, PackBitsAppendsToContainer) { From bcec1d6cb8124a97203c8899d2d85577581819f0 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Fri, 28 Aug 2026 16:44:37 -0700 Subject: [PATCH 022/147] [modbus] Decode by register address in the modbus sensor components (#18874) --- .../growatt_solar/growatt_solar.cpp | 23 +++-- .../components/growatt_solar/growatt_solar.h | 88 +++++++++---------- .../havells_solar/havells_solar.cpp | 19 ++-- esphome/components/pzemac/pzemac.cpp | 19 ++-- esphome/components/pzemdc/pzemdc.cpp | 19 ++-- esphome/components/sdm_meter/sdm_meter.cpp | 11 ++- .../components/selec_meter/selec_meter.cpp | 11 ++- 7 files changed, 88 insertions(+), 102 deletions(-) diff --git a/esphome/components/growatt_solar/growatt_solar.cpp b/esphome/components/growatt_solar/growatt_solar.cpp index bc3c3d52db..08c3966ed9 100644 --- a/esphome/components/growatt_solar/growatt_solar.cpp +++ b/esphome/components/growatt_solar/growatt_solar.cpp @@ -3,6 +3,8 @@ namespace esphome::growatt_solar { +namespace helpers = modbus::helpers; + static const char *const TAG = "growatt_solar"; static const uint8_t MODBUS_REGISTER_COUNT[] = {33, 95}; // indexed with enum GrowattProtocolVersion @@ -16,23 +18,18 @@ void GrowattSolar::on_read_input_registers(uint16_t start_address, std::span void { - if (sensor == nullptr || reg < start_address) + auto publish_1_reg_sensor_state = [&](sensor::Sensor *sensor, uint16_t reg, float unit) -> void { + if (sensor == nullptr) return; - size_t offset = reg - start_address; - if (offset >= registers.size()) - return; - sensor->publish_state(registers[offset] * unit); + if (auto value = helpers::value_at(registers, start_address, reg)) + sensor->publish_state(*value * unit); }; - auto publish_2_reg_sensor_state = [&](sensor::Sensor *sensor, size_t reg, float unit) -> void { - constexpr auto value_type = modbus::helpers::SensorValueType::U_DWORD; - if (sensor == nullptr || reg < start_address) + auto publish_2_reg_sensor_state = [&](sensor::Sensor *sensor, uint16_t reg, float unit) -> void { + if (sensor == nullptr) return; - size_t offset = reg - start_address; - if (offset + modbus::helpers::register_width_for(value_type) > registers.size()) - return; - sensor->publish_state(modbus::helpers::registers_to_value(registers.data() + offset) * unit); + if (auto value = helpers::value_at(registers, start_address, reg)) + sensor->publish_state(*value * unit); }; switch (this->protocol_version_) { diff --git a/esphome/components/growatt_solar/growatt_solar.h b/esphome/components/growatt_solar/growatt_solar.h index 60706930c7..5b96521476 100644 --- a/esphome/components/growatt_solar/growatt_solar.h +++ b/esphome/components/growatt_solar/growatt_solar.h @@ -17,53 +17,53 @@ enum GrowattProtocolVersion { }; // Register addresses for the RTU protocol. -constexpr size_t RTU_INVERTER_STATUS = 0; // length = 1 -constexpr size_t RTU_PV_ACTIVE_POWER = 1; // length = 2 -constexpr size_t RTU_PV1_VOLTAGE = 3; // length = 1 -constexpr size_t RTU_PV1_CURRENT = 4; // length = 1 -constexpr size_t RTU_PV1_ACTIVE_POWER = 5; // length = 2 -constexpr size_t RTU_PV2_VOLTAGE = 7; // length = 1 -constexpr size_t RTU_PV2_CURRENT = 8; // length = 1 -constexpr size_t RTU_PV2_ACTIVE_POWER = 9; // length = 2 -constexpr size_t RTU_GRID_ACTIVE_POWER = 11; // length = 2 -constexpr size_t RTU_GRID_FREQUENCY = 13; // length = 1 -constexpr size_t RTU_PHASE1_VOLTAGE = 14; // length = 1 -constexpr size_t RTU_PHASE1_CURRENT = 15; // length = 1 -constexpr size_t RTU_PHASE1_ACTIVE_POWER = 16; // length = 2 -constexpr size_t RTU_PHASE2_VOLTAGE = 18; // length = 1 -constexpr size_t RTU_PHASE2_CURRENT = 19; // length = 1 -constexpr size_t RTU_PHASE2_ACTIVE_POWER = 20; // length = 2 -constexpr size_t RTU_PHASE3_VOLTAGE = 22; // length = 1 -constexpr size_t RTU_PHASE3_CURRENT = 23; // length = 1 -constexpr size_t RTU_PHASE3_ACTIVE_POWER = 24; // length = 2 -constexpr size_t RTU_TODAY_PRODUCTION = 26; // length = 2 -constexpr size_t RTU_TOTAL_ENERGY_PRODUCTION = 28; // length = 2 -constexpr size_t RTU_INVERTER_MODULE_TEMP = 32; // length = 1 +constexpr uint16_t RTU_INVERTER_STATUS = 0; // length = 1 +constexpr uint16_t RTU_PV_ACTIVE_POWER = 1; // length = 2 +constexpr uint16_t RTU_PV1_VOLTAGE = 3; // length = 1 +constexpr uint16_t RTU_PV1_CURRENT = 4; // length = 1 +constexpr uint16_t RTU_PV1_ACTIVE_POWER = 5; // length = 2 +constexpr uint16_t RTU_PV2_VOLTAGE = 7; // length = 1 +constexpr uint16_t RTU_PV2_CURRENT = 8; // length = 1 +constexpr uint16_t RTU_PV2_ACTIVE_POWER = 9; // length = 2 +constexpr uint16_t RTU_GRID_ACTIVE_POWER = 11; // length = 2 +constexpr uint16_t RTU_GRID_FREQUENCY = 13; // length = 1 +constexpr uint16_t RTU_PHASE1_VOLTAGE = 14; // length = 1 +constexpr uint16_t RTU_PHASE1_CURRENT = 15; // length = 1 +constexpr uint16_t RTU_PHASE1_ACTIVE_POWER = 16; // length = 2 +constexpr uint16_t RTU_PHASE2_VOLTAGE = 18; // length = 1 +constexpr uint16_t RTU_PHASE2_CURRENT = 19; // length = 1 +constexpr uint16_t RTU_PHASE2_ACTIVE_POWER = 20; // length = 2 +constexpr uint16_t RTU_PHASE3_VOLTAGE = 22; // length = 1 +constexpr uint16_t RTU_PHASE3_CURRENT = 23; // length = 1 +constexpr uint16_t RTU_PHASE3_ACTIVE_POWER = 24; // length = 2 +constexpr uint16_t RTU_TODAY_PRODUCTION = 26; // length = 2 +constexpr uint16_t RTU_TOTAL_ENERGY_PRODUCTION = 28; // length = 2 +constexpr uint16_t RTU_INVERTER_MODULE_TEMP = 32; // length = 1 // Input register addresses for the RTU2 protocol as described // in the "GROWATT INVERTER MODBUS PROTOCOL_II V1.39" document. -constexpr size_t RTU2_INVERTER_STATUS = 0; // length = 1 -constexpr size_t RTU2_PV_ACTIVE_POWER = 1; // length = 2 -constexpr size_t RTU2_PV1_VOLTAGE = 3; // length = 1 -constexpr size_t RTU2_PV1_CURRENT = 4; // length = 1 -constexpr size_t RTU2_PV1_ACTIVE_POWER = 5; // length = 2 -constexpr size_t RTU2_PV2_VOLTAGE = 7; // length = 1 -constexpr size_t RTU2_PV2_CURRENT = 8; // length = 1 -constexpr size_t RTU2_PV2_ACTIVE_POWER = 9; // length = 2 -constexpr size_t RTU2_GRID_ACTIVE_POWER = 35; // length = 2 -constexpr size_t RTU2_GRID_FREQUENCY = 37; // length = 1 -constexpr size_t RTU2_PHASE1_VOLTAGE = 38; // length = 1 -constexpr size_t RTU2_PHASE1_CURRENT = 39; // length = 1 -constexpr size_t RTU2_PHASE1_ACTIVE_POWER = 40; // length = 2 -constexpr size_t RTU2_PHASE2_VOLTAGE = 42; // length = 1 -constexpr size_t RTU2_PHASE2_CURRENT = 43; // length = 1 -constexpr size_t RTU2_PHASE2_ACTIVE_POWER = 44; // length = 2 -constexpr size_t RTU2_PHASE3_VOLTAGE = 46; // length = 1 -constexpr size_t RTU2_PHASE3_CURRENT = 47; // length = 1 -constexpr size_t RTU2_PHASE3_ACTIVE_POWER = 48; // length = 2 -constexpr size_t RTU2_TODAY_PRODUCTION = 53; // length = 2 -constexpr size_t RTU2_TOTAL_ENERGY_PRODUCTION = 55; // length = 2 -constexpr size_t RTU2_INVERTER_MODULE_TEMP = 93; // length = 1 +constexpr uint16_t RTU2_INVERTER_STATUS = 0; // length = 1 +constexpr uint16_t RTU2_PV_ACTIVE_POWER = 1; // length = 2 +constexpr uint16_t RTU2_PV1_VOLTAGE = 3; // length = 1 +constexpr uint16_t RTU2_PV1_CURRENT = 4; // length = 1 +constexpr uint16_t RTU2_PV1_ACTIVE_POWER = 5; // length = 2 +constexpr uint16_t RTU2_PV2_VOLTAGE = 7; // length = 1 +constexpr uint16_t RTU2_PV2_CURRENT = 8; // length = 1 +constexpr uint16_t RTU2_PV2_ACTIVE_POWER = 9; // length = 2 +constexpr uint16_t RTU2_GRID_ACTIVE_POWER = 35; // length = 2 +constexpr uint16_t RTU2_GRID_FREQUENCY = 37; // length = 1 +constexpr uint16_t RTU2_PHASE1_VOLTAGE = 38; // length = 1 +constexpr uint16_t RTU2_PHASE1_CURRENT = 39; // length = 1 +constexpr uint16_t RTU2_PHASE1_ACTIVE_POWER = 40; // length = 2 +constexpr uint16_t RTU2_PHASE2_VOLTAGE = 42; // length = 1 +constexpr uint16_t RTU2_PHASE2_CURRENT = 43; // length = 1 +constexpr uint16_t RTU2_PHASE2_ACTIVE_POWER = 44; // length = 2 +constexpr uint16_t RTU2_PHASE3_VOLTAGE = 46; // length = 1 +constexpr uint16_t RTU2_PHASE3_CURRENT = 47; // length = 1 +constexpr uint16_t RTU2_PHASE3_ACTIVE_POWER = 48; // length = 2 +constexpr uint16_t RTU2_TODAY_PRODUCTION = 53; // length = 2 +constexpr uint16_t RTU2_TOTAL_ENERGY_PRODUCTION = 55; // length = 2 +constexpr uint16_t RTU2_INVERTER_MODULE_TEMP = 93; // length = 1 class GrowattSolar final : public PollingComponent, public modbus::ModbusClientDevice { public: diff --git a/esphome/components/havells_solar/havells_solar.cpp b/esphome/components/havells_solar/havells_solar.cpp index c98dc0de2f..d43dfbb89a 100644 --- a/esphome/components/havells_solar/havells_solar.cpp +++ b/esphome/components/havells_solar/havells_solar.cpp @@ -4,6 +4,8 @@ namespace esphome::havells_solar { +namespace helpers = modbus::helpers; + static const char *const TAG = "havells_solar"; static const uint8_t MODBUS_REGISTER_COUNT = 48; // 48 x 16-bit registers @@ -16,22 +18,17 @@ void HavellsSolar::on_read_holding_registers(uint16_t start_address, std::span void { - if (sensor == nullptr || reg < start_address) + if (sensor == nullptr) return; - size_t offset = reg - start_address; - if (offset >= registers.size()) - return; - sensor->publish_state(registers[offset] * unit); + if (auto value = helpers::value_at(registers, start_address, reg)) + sensor->publish_state(*value * unit); }; auto publish_2_registers = [&](sensor::Sensor *sensor, uint16_t reg, float unit) -> void { - constexpr auto value_type = modbus::helpers::SensorValueType::U_DWORD; - if (sensor == nullptr || reg < start_address) + if (sensor == nullptr) return; - size_t offset = reg - start_address; - if (offset + modbus::helpers::register_width_for(value_type) > registers.size()) - return; - sensor->publish_state(modbus::helpers::registers_to_value(registers.data() + offset) * unit); + if (auto value = helpers::value_at(registers, start_address, reg)) + sensor->publish_state(*value * unit); }; for (uint8_t i = 0; i < 3; i++) { diff --git a/esphome/components/pzemac/pzemac.cpp b/esphome/components/pzemac/pzemac.cpp index 50c626ec7f..409de91124 100644 --- a/esphome/components/pzemac/pzemac.cpp +++ b/esphome/components/pzemac/pzemac.cpp @@ -3,6 +3,8 @@ namespace esphome::pzemac { +namespace helpers = modbus::helpers; + static const char *const TAG = "pzemac"; static const uint8_t PZEM_CMD_RESET_ENERGY = 0x42; @@ -25,22 +27,17 @@ void PZEMAC::on_read_input_registers(uint16_t start_address, std::span void { - if (sensor == nullptr || reg < start_address) + if (sensor == nullptr) return; - size_t offset = reg - start_address; - if (offset >= registers.size()) - return; - sensor->publish_state(registers[offset] / divisor); + if (auto value = helpers::value_at(registers, start_address, reg)) + sensor->publish_state(*value / divisor); }; auto publish_2_registers = [&](sensor::Sensor *sensor, uint16_t reg, float divisor) -> void { - constexpr auto value_type = modbus::helpers::SensorValueType::U_DWORD_R; - if (sensor == nullptr || reg < start_address) + if (sensor == nullptr) return; - size_t offset = reg - start_address; - if (offset + modbus::helpers::register_width_for(value_type) > registers.size()) - return; - sensor->publish_state(modbus::helpers::registers_to_value(registers.data() + offset) / divisor); + if (auto value = helpers::value_at(registers, start_address, reg)) + sensor->publish_state(*value / divisor); }; publish_1_register(this->voltage_sensor_, PZEM_REGISTER_VOLTAGE, 10.0f); diff --git a/esphome/components/pzemdc/pzemdc.cpp b/esphome/components/pzemdc/pzemdc.cpp index 546e4225de..eb9a355806 100644 --- a/esphome/components/pzemdc/pzemdc.cpp +++ b/esphome/components/pzemdc/pzemdc.cpp @@ -3,6 +3,8 @@ namespace esphome::pzemdc { +namespace helpers = modbus::helpers; + static const char *const TAG = "pzemdc"; static const uint8_t PZEM_CMD_RESET_ENERGY = 0x42; @@ -23,22 +25,17 @@ void PZEMDC::on_read_input_registers(uint16_t start_address, std::span void { - if (sensor == nullptr || reg < start_address) + if (sensor == nullptr) return; - size_t offset = reg - start_address; - if (offset >= registers.size()) - return; - sensor->publish_state(registers[offset] / divisor); + if (auto value = helpers::value_at(registers, start_address, reg)) + sensor->publish_state(*value / divisor); }; auto publish_2_registers = [&](sensor::Sensor *sensor, uint16_t reg, float divisor) -> void { - constexpr auto value_type = modbus::helpers::SensorValueType::U_DWORD_R; - if (sensor == nullptr || reg < start_address) + if (sensor == nullptr) return; - size_t offset = reg - start_address; - if (offset + modbus::helpers::register_width_for(value_type) > registers.size()) - return; - sensor->publish_state(modbus::helpers::registers_to_value(registers.data() + offset) / divisor); + if (auto value = helpers::value_at(registers, start_address, reg)) + sensor->publish_state(*value / divisor); }; publish_1_register(this->voltage_sensor_, PZEM_REGISTER_VOLTAGE, 100.0f); diff --git a/esphome/components/sdm_meter/sdm_meter.cpp b/esphome/components/sdm_meter/sdm_meter.cpp index f242b6b36d..c1b359cc97 100644 --- a/esphome/components/sdm_meter/sdm_meter.cpp +++ b/esphome/components/sdm_meter/sdm_meter.cpp @@ -4,6 +4,8 @@ namespace esphome::sdm_meter { +namespace helpers = modbus::helpers; + static const char *const TAG = "sdm_meter"; static const uint8_t MODBUS_REGISTER_COUNT = 80; // 80 x 16-bit registers (40 float values) @@ -16,13 +18,10 @@ void SDMMeter::on_read_input_registers(uint16_t start_address, std::span registers.size()) - return; - sensor->publish_state(modbus::helpers::registers_to_value(registers.data() + offset)); + if (auto value = helpers::value_at(registers, start_address, reg)) + sensor->publish_state(*value); }; for (uint8_t i = 0; i < 3; i++) { diff --git a/esphome/components/selec_meter/selec_meter.cpp b/esphome/components/selec_meter/selec_meter.cpp index 97831e8354..3ad1f8b87c 100644 --- a/esphome/components/selec_meter/selec_meter.cpp +++ b/esphome/components/selec_meter/selec_meter.cpp @@ -4,6 +4,8 @@ namespace esphome::selec_meter { +namespace helpers = modbus::helpers; + static const char *const TAG = "selec_meter"; static const uint8_t MODBUS_REGISTER_COUNT = 34; // 34 x 16-bit registers @@ -17,13 +19,10 @@ void SelecMeter::on_read_input_registers(uint16_t start_address, std::span void { - constexpr auto value_type = modbus::helpers::SensorValueType::FP32_R; - if (sensor == nullptr || reg < start_address) + if (sensor == nullptr) return; - size_t offset = reg - start_address; - if (offset + modbus::helpers::register_width_for(value_type) > registers.size()) - return; - sensor->publish_state(modbus::helpers::registers_to_value(registers.data() + offset) * unit); + if (auto value = helpers::value_at(registers, start_address, reg)) + sensor->publish_state(*value * unit); }; publish(this->total_active_energy_sensor_, SELEC_TOTAL_ACTIVE_ENERGY, NO_DEC_UNIT); From ce163b82585ea29d67333f57202665dd9b4b37fe Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 28 Aug 2026 20:41:57 -0500 Subject: [PATCH 023/147] [ci] Cache clang-tidy idedata and key ESP-IDF cache on Python version (#18868) --- .../cache-clang-tidy-idedata/action.yml | 50 ++++++++++++++++ .github/actions/cache-esp-idf/action.yml | 16 ++++-- .github/workflows/ci.yml | 24 ++++++++ script/clang_tidy_hash.py | 57 ++++++++++++++++--- script/determine-jobs.py | 24 ++------ script/helpers.py | 11 ++-- tests/script/test_clang_tidy_hash.py | 37 ++++++++++++ 7 files changed, 180 insertions(+), 39 deletions(-) create mode 100644 .github/actions/cache-clang-tidy-idedata/action.yml diff --git a/.github/actions/cache-clang-tidy-idedata/action.yml b/.github/actions/cache-clang-tidy-idedata/action.yml new file mode 100644 index 0000000000..18f3c2b31a --- /dev/null +++ b/.github/actions/cache-clang-tidy-idedata/action.yml @@ -0,0 +1,50 @@ +name: Cache clang-tidy idedata +description: > + Cache the clang-tidy idedata and the headers it references under .temp + (headers only, about 30MB per env). Run after restore-python and cache-esp-idf. +inputs: + environment: + description: 'clang-tidy environment (e.g. esp32-idf-tidy).' + required: true +runs: + using: composite + steps: + - name: Compute cache key + id: key + shell: bash + run: | + . venv/bin/activate + [ -n "${{ inputs.environment }}" ] || { echo "::error::cache-clang-tidy-idedata: 'environment' input is empty"; exit 1; } + hash=$(python -c 'import sys; sys.path.insert(0, "script"); from clang_tidy_hash import idedata_cache_hash; print(idedata_cache_hash("${{ inputs.environment }}"))') + pyver=$(python -c 'import platform; print(platform.python_version())') + # Generating idedata is what installs ESP-IDF; never skip it over a missing + # install. This also skips the save, so a dev run that installs ESP-IDF + # warms the idedata cache on the next run. + if [ -d ~/.esphome-idf/frameworks ]; then + echo "skip=false" >> "$GITHUB_OUTPUT" + else + echo "ESP-IDF install missing, not using the clang-tidy idedata cache" + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + echo "key=${{ runner.os }}-tidy-idedata-${{ inputs.environment }}-$hash-py$pyver" >> "$GITHUB_OUTPUT" + { + echo "path<> "$GITHUB_OUTPUT" + # Mirror cache-esp-idf: write on dev, restore-only on PRs. The post-step + # save only runs when the job succeeded, so a failed generation is never saved. + # Extend the extension list if a component ships extensionless headers. + - name: Cache clang-tidy idedata (write on dev) + if: (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && steps.key.outputs.skip != 'true' + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.key.outputs.path }} + key: ${{ steps.key.outputs.key }} + - name: Cache clang-tidy idedata (restore-only off dev) + if: github.ref != 'refs/heads/dev' && !contains(github.event.pull_request.labels.*.name, 'ci-cache-write') && steps.key.outputs.skip != 'true' + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ steps.key.outputs.path }} + key: ${{ steps.key.outputs.key }} diff --git a/.github/actions/cache-esp-idf/action.yml b/.github/actions/cache-esp-idf/action.yml index b884e1e4c6..58c9b69cd7 100644 --- a/.github/actions/cache-esp-idf/action.yml +++ b/.github/actions/cache-esp-idf/action.yml @@ -26,6 +26,9 @@ runs: # The native-IDF version is pinned in code, not in any file that feeds the # other cache keys, so resolve it explicitly. Keying on it means the cache # invalidates on a version bump (actions/cache never overwrites a key). + # Also key on the Python version: the cached IDF venv links to the + # runner's toolcache interpreter and is reinstalled every run after a + # runner image bump. id: version shell: bash run: | @@ -36,19 +39,22 @@ runs: version=$(python -c 'from esphome.components.esp32 import ESP_IDF_FRAMEWORK_VERSION_LOOKUP as L; print(L["recommended"])') fi echo "version=$version" >> "$GITHUB_OUTPUT" + echo "python-version=$(python -c 'import platform; print(platform.python_version())')" >> "$GITHUB_OUTPUT" # Mirror the adjacent PlatformIO cache: only dev-branch runs write the # shared cache (so it lives in the default-branch scope readable by all # PRs), and PRs are restore-only -- they never push multi-GB artifacts into - # their own scope / the repo quota (e.g. on a version-bump PR). + # their own scope / the repo quota (e.g. on a version-bump PR). The + # ci-cache-write label lets a PR write into its own scope to test the hit path; + # that costs about 1GB of the repo cache quota per run, so remove it when done. - name: Cache ESP-IDF install (write on dev) - if: github.ref == 'refs/heads/dev' && inputs.restore-only != 'true' + if: (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && inputs.restore-only != 'true' uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.esphome-idf - key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }} + key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }}-py${{ steps.version.outputs.python-version }} - name: Cache ESP-IDF install (restore-only off dev) - if: github.ref != 'refs/heads/dev' || inputs.restore-only == 'true' + if: github.ref != 'refs/heads/dev' && !contains(github.event.pull_request.labels.*.name, 'ci-cache-write') || inputs.restore-only == 'true' uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.esphome-idf - key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }} + key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }}-py${{ steps.version.outputs.python-version }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbf6e070b4..b8840f74f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -649,6 +649,12 @@ jobs: with: framework: arduino + - name: Cache clang-tidy idedata + if: matrix.cache_idf + uses: ./.github/actions/cache-clang-tidy-idedata + with: + environment: esp32-arduino-tidy + - name: Cache nRF Connect SDK install if: matrix.cache_sdk_nrf uses: ./.github/actions/cache-sdk-nrf @@ -730,6 +736,11 @@ jobs: - name: Cache ESP-IDF install uses: ./.github/actions/cache-esp-idf + - name: Cache clang-tidy idedata + uses: ./.github/actions/cache-clang-tidy-idedata + with: + environment: esp32-idf-tidy + - name: Register problem matchers run: | echo "::add-matcher::.github/workflows/matchers/gcc.json" @@ -809,6 +820,11 @@ jobs: - name: Cache ESP-IDF install uses: ./.github/actions/cache-esp-idf + - name: Cache clang-tidy idedata + uses: ./.github/actions/cache-clang-tidy-idedata + with: + environment: esp32-idf-tidy + - name: Register problem matchers run: | echo "::add-matcher::.github/workflows/matchers/gcc.json" @@ -866,16 +882,19 @@ jobs: name: Run script/clang-tidy for ESP32 S3 # yamllint disable-line rule:line-length options: --environment esp32s3-idf-tidy --grep SOC_TEMP_SENSOR_SUPPORTED --grep USE_ESP32_VARIANT_ESP32S3 --grep USE_LOGGER_USB_CDC + tidy_environment: esp32s3-idf-tidy - id: clang-tidy name: Run script/clang-tidy for ESP32 P4 # P4 has no native Wi-Fi/BLE; those run over the hosted co-processor, # so their code paths differ -- lint them under the P4 build too. # yamllint disable-line rule:line-length options: --environment esp32p4-idf-tidy --grep USE_ESP32_VARIANT_ESP32P4 --grep USE_ESP32_HOSTED --grep USE_WIFI --grep USE_BLE + tidy_environment: esp32p4-idf-tidy - id: clang-tidy name: Run script/clang-tidy for ESP32 C6 # yamllint disable-line rule:line-length options: --environment esp32c6-idf-tidy --grep SOC_LP_I2C_SUPPORTED --grep USE_ESP32_VARIANT_ESP32C6 --grep USE_OPENTHREAD --grep USE_ZIGBEE + tidy_environment: esp32c6-idf-tidy steps: - name: Check out code from GitHub @@ -893,6 +912,11 @@ jobs: - name: Cache ESP-IDF install uses: ./.github/actions/cache-esp-idf + - name: Cache clang-tidy idedata + uses: ./.github/actions/cache-clang-tidy-idedata + with: + environment: ${{ matrix.tidy_environment }} + - name: Register problem matchers run: | echo "::add-matcher::.github/workflows/matchers/gcc.json" diff --git a/script/clang_tidy_hash.py b/script/clang_tidy_hash.py index f4fd5a4dff..bdc97bd766 100644 --- a/script/clang_tidy_hash.py +++ b/script/clang_tidy_hash.py @@ -1,13 +1,10 @@ -"""Files that affect clang-tidy results, and a content hash over them. +"""Files that affect clang-tidy results and the idedata built from them. -``CLANG_TIDY_GLOBAL_FILES`` (plus ``SDKCONFIG_DEFAULTS_PREFIX``) is the single -source of truth for which files influence clang-tidy output. A change to any of -them can surface warnings in source files a PR didn't touch, so: - -* ``script/determine-jobs.py`` runs a full clang-tidy scan when one changes, and -* ``calculate_clang_tidy_hash()`` folds them into the idedata cache key used by - ``script/helpers.py`` (a content hash, unlike an mtime check, stays correct - across git checkouts). +``CLANG_TIDY_GLOBAL_FILES`` (plus ``SDKCONFIG_DEFAULTS_PREFIX``) lists the files +that influence clang-tidy output; ``script/determine-jobs.py`` runs a full scan +when one changes. ``ESP_IDF_INFRA_TRIGGER_*`` lists the native ESP-IDF build +code. ``idedata_cache_hash()`` folds the right set into the idedata cache key +used by ``script/helpers.py`` and the CI cache action. """ from __future__ import annotations @@ -31,6 +28,18 @@ CLANG_TIDY_GLOBAL_FILES = ( # this prefix at the repo root. SDKCONFIG_DEFAULTS_PREFIX = "sdkconfig.defaults" +# Native ESP-IDF build infra: determine-jobs forces an esp32 compile when these +# change, and they feed the clang-tidy idedata cache key. +ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/", "esphome/build_helpers/") +ESP_IDF_INFRA_TRIGGER_FILES = frozenset( + { + "esphome/build_gen/espidf.py", + "esphome/framework_helpers.py", + "esphome/platformio/library.py", + "esphome/platformio/extra_script.py", + } +) + def read_file_bytes(path: Path) -> bytes: """Read bytes from a file.""" @@ -66,3 +75,33 @@ def calculate_clang_tidy_hash(repo_root: Path | None = None) -> str: hasher.update(read_file_bytes(path)) return hasher.hexdigest() + + +def calculate_idedata_cache_hash(repo_root: Path | None = None) -> str: + """Clang-tidy hash plus the Python that generates the idedata.""" + repo_root = _ensure_repo_root(repo_root) + + hasher = hashlib.sha256() + hasher.update(calculate_clang_tidy_hash(repo_root).encode()) + + paths = {repo_root / name for name in ESP_IDF_INFRA_TRIGGER_FILES} + for prefix in ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES: + # .pyc files appear between the CI key computation and load_idedata's. + paths.update( + path + for path in (repo_root / prefix).rglob("*") + if "__pycache__" not in path.parts + ) + for path in sorted(paths): + if path.is_file(): + hasher.update(str(path.relative_to(repo_root)).encode()) + hasher.update(read_file_bytes(path)) + + return hasher.hexdigest() + + +def idedata_cache_hash(environment: str, repo_root: Path | None = None) -> str: + """Hash gating the cached idedata of one clang-tidy environment.""" + if "esp32" in environment: + return calculate_idedata_cache_hash(repo_root) + return calculate_clang_tidy_hash(repo_root) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 2bdf7807a9..9eead4b38c 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -58,7 +58,12 @@ from pathlib import Path import sys from typing import Any -from clang_tidy_hash import CLANG_TIDY_GLOBAL_FILES, SDKCONFIG_DEFAULTS_PREFIX +from clang_tidy_hash import ( + CLANG_TIDY_GLOBAL_FILES, + ESP_IDF_INFRA_TRIGGER_FILES, + ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES, + SDKCONFIG_DEFAULTS_PREFIX, +) from helpers import ( CPP_FILE_EXTENSIONS, ESPHOME_TESTS_COMPONENTS_PATH, @@ -524,23 +529,6 @@ def _esp32_platformio_path_or_file_trigger(files: list[str]) -> bool: return False -# Native-build infra: changes under esphome/espidf/, the shared -# esphome/build_helpers/ package, or the modules the native ESP-IDF build -# imports affect every esp32 IDF build (now the default toolchain) but aren't -# components, so the component matrix wouldn't otherwise force any esp32 -# compile. When they change we fold the `esp32` component into the matrix so -# the default native-IDF build path is still compiled on an infra-only PR. -ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES = ("esphome/espidf/", "esphome/build_helpers/") -ESP_IDF_INFRA_TRIGGER_FILES = frozenset( - { - "esphome/build_gen/espidf.py", - "esphome/framework_helpers.py", - "esphome/platformio/library.py", - "esphome/platformio/extra_script.py", - } -) - - def _esp_idf_infra_changed(files: list[str]) -> bool: """Whether any changed file is ESP-IDF build/runner infrastructure.""" for file in files: diff --git a/script/helpers.py b/script/helpers.py index 9e3969e5ce..e648bb91bb 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -809,17 +809,14 @@ def load_idedata(environment: str) -> dict[str, Any]: start_time = time.time() print(f"Loading IDE data for environment '{environment}'...") - # Reuse the clang-tidy input hash as the cache key: it already covers every - # file baked into the generated idedata (platformio.ini, sdkconfig.defaults, - # esphome/idf_component.yml), so this can't drift from that file list. A - # content hash -- unlike an mtime comparison -- stays correct across git - # checkouts, which don't preserve mtimes. - from clang_tidy_hash import calculate_clang_tidy_hash + # Content hash of the idedata inputs (data files and the generator code); a + # content hash, unlike mtimes, stays correct across git checkouts. + from clang_tidy_hash import idedata_cache_hash temp_idedata = Path(temp_folder) / f"idedata-{environment}.json" temp_hash = Path(temp_folder) / f"idedata-{environment}.hash" - cache_key = calculate_clang_tidy_hash() + cache_key = idedata_cache_hash(environment) changed = ( not temp_idedata.is_file() or not temp_hash.is_file() diff --git a/tests/script/test_clang_tidy_hash.py b/tests/script/test_clang_tidy_hash.py index b5a9d8ebe9..decae4fd13 100644 --- a/tests/script/test_clang_tidy_hash.py +++ b/tests/script/test_clang_tidy_hash.py @@ -81,3 +81,40 @@ def test_read_file_bytes(tmp_path: Path) -> None: result = clang_tidy_hash.read_file_bytes(test_file) assert result == test_content + + +def test_calculate_idedata_cache_hash_changes_with_infra_code(tmp_path: Path) -> None: + _populate(tmp_path) + infra = tmp_path / "esphome" / "espidf" / "clang_tidy.py" + infra.parent.mkdir(parents=True) + infra.write_text("a") + before = clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path) + assert before == clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path) + infra.write_text("b") + assert clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path) != before + + +def test_calculate_idedata_cache_hash_includes_listed_files(tmp_path: Path) -> None: + _populate(tmp_path) + before = clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path) + listed = tmp_path / "esphome" / "platformio" / "library.py" + listed.parent.mkdir(parents=True) + listed.write_text("x") + assert clang_tidy_hash.calculate_idedata_cache_hash(repo_root=tmp_path) != before + + +def test_idedata_cache_hash_only_widens_for_esp32(tmp_path: Path) -> None: + _populate(tmp_path) + infra = tmp_path / "esphome" / "espidf" / "clang_tidy.py" + infra.parent.mkdir(parents=True) + infra.write_text("a") + esp32_before = clang_tidy_hash.idedata_cache_hash("esp32-idf-tidy", tmp_path) + other_before = clang_tidy_hash.idedata_cache_hash("esp8266-arduino-tidy", tmp_path) + infra.write_text("b") + assert ( + clang_tidy_hash.idedata_cache_hash("esp32-idf-tidy", tmp_path) != esp32_before + ) + assert ( + clang_tidy_hash.idedata_cache_hash("esp8266-arduino-tidy", tmp_path) + == other_before + ) From 2576a0e3408c85af6c789c28b2a6a57b965ae970 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 28 Aug 2026 20:50:24 -0500 Subject: [PATCH 024/147] [ci] Drop picolibc from the cached ESP-IDF toolchains (#18871) --- .github/actions/cache-esp-idf/action.yml | 14 ++++++++-- .github/actions/prune-esp-idf/action.yml | 34 ++++++++++++++++++++++++ .github/workflows/ci.yml | 16 +++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 .github/actions/prune-esp-idf/action.yml diff --git a/.github/actions/cache-esp-idf/action.yml b/.github/actions/cache-esp-idf/action.yml index 58c9b69cd7..38bcc80eb6 100644 --- a/.github/actions/cache-esp-idf/action.yml +++ b/.github/actions/cache-esp-idf/action.yml @@ -46,15 +46,25 @@ runs: # their own scope / the repo quota (e.g. on a version-bump PR). The # ci-cache-write label lets a PR write into its own scope to test the hit path; # that costs about 1GB of the repo cache quota per run, so remove it when done. + # -slim: bump when prune-esp-idf changes what it removes; a key is never overwritten. - name: Cache ESP-IDF install (write on dev) if: (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && inputs.restore-only != 'true' uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.esphome-idf - key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }}-py${{ steps.version.outputs.python-version }} + key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }}-py${{ steps.version.outputs.python-version }}-slim - name: Cache ESP-IDF install (restore-only off dev) if: github.ref != 'refs/heads/dev' && !contains(github.event.pull_request.labels.*.name, 'ci-cache-write') || inputs.restore-only == 'true' uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.esphome-idf - key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }}-py${{ steps.version.outputs.python-version }} + key: ${{ runner.os }}-esphome-idf-${{ steps.version.outputs.version }}-py${{ steps.version.outputs.python-version }}-slim + # Install explicitly so the prune below sees the toolchains on a cache miss + # too, instead of the install happening inside the first build step. + - name: Install ESP-IDF + shell: bash + run: | + . venv/bin/activate + python -c 'from esphome.espidf.framework import check_esp_idf_install; check_esp_idf_install("${{ steps.version.outputs.version }}")' + - name: Prune ESP-IDF install + uses: ./.github/actions/prune-esp-idf diff --git a/.github/actions/prune-esp-idf/action.yml b/.github/actions/prune-esp-idf/action.yml new file mode 100644 index 0000000000..e0e7c5bd4e --- /dev/null +++ b/.github/actions/prune-esp-idf/action.yml @@ -0,0 +1,34 @@ +name: Prune ESP-IDF install +description: > + Remove the picolibc sysroots (1.1GB of the 3.9GB install) from the native + ESP-IDF toolchains; IDF 5.x links newlib. Skipped when an IDF 6 install is + present, which links picolibc (see esp32/__init__.py). +runs: + using: composite + steps: + - name: Prune picolibc + shell: bash + run: | + shopt -s nullglob + prefix="${ESPHOME_ESP_IDF_PREFIX:-$HOME/.esphome-idf}" + prefix="${prefix/#\~/$HOME}" + for fw in "$prefix"/frameworks/*/; do + case "$(basename "$fw")" in + [6-9].*) echo "IDF $(basename "$fw") installed, keeping picolibc"; exit 0 ;; + esac + done + n=0 + for dir in "$prefix"/tools/*-esp-elf/*/*-esp-elf/picolibc; do + echo "Removing $dir ($(du -sh "$dir" | cut -f1))" + rm -rf "$dir" + n=$((n + 1)) + done + # The marker rides along in the cache entry so a restored slim tree stays quiet. + if [ "$n" -gt 0 ]; then + touch "$prefix/.picolibc-pruned" + elif [ -d "$prefix/tools" ] && [ ! -f "$prefix/.picolibc-pruned" ]; then + echo "::warning::no picolibc sysroots matched under $prefix/tools" + fi + if [ -d "$prefix" ]; then + du -sh "$prefix" + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8840f74f8..5a58d0fe9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -704,6 +704,10 @@ jobs: # Also cache libdeps, store them in a ~/.platformio subfolder PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps + - name: Prune ESP-IDF install before cache save + if: matrix.cache_idf && (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) + uses: ./.github/actions/prune-esp-idf + - name: Suggested changes run: script/ci-suggest-changes ${{ matrix.ignore_errors && '|| true' || '' }} # yamllint disable-line rule:line-length @@ -775,6 +779,10 @@ jobs: # Also cache libdeps, store them in a ~/.platformio subfolder PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps + - name: Prune ESP-IDF install before cache save + if: (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) + uses: ./.github/actions/prune-esp-idf + - name: Suggested changes run: script/ci-suggest-changes if: always() @@ -859,6 +867,10 @@ jobs: # Also cache libdeps, store them in a ~/.platformio subfolder PLATFORMIO_LIBDEPS_DIR: ~/.platformio/libdeps + - name: Prune ESP-IDF install before cache save + if: (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) + uses: ./.github/actions/prune-esp-idf + - name: Suggested changes run: script/ci-suggest-changes if: always() @@ -950,6 +962,10 @@ jobs: script/clang-tidy --fix --changed ${{ matrix.options }} fi + - name: Prune ESP-IDF install before cache save + if: (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) + uses: ./.github/actions/prune-esp-idf + - name: Suggested changes run: script/ci-suggest-changes if: always() From 06bc3d70c24e899b3509d50c6f425127f1081888 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:02:27 +1000 Subject: [PATCH 025/147] [mipi_rgb] Add Elecrow Crowpanel Advance 7 (#18810) --- esphome/components/mipi_rgb/models/elecrow.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 esphome/components/mipi_rgb/models/elecrow.py diff --git a/esphome/components/mipi_rgb/models/elecrow.py b/esphome/components/mipi_rgb/models/elecrow.py new file mode 100644 index 0000000000..acc36beb74 --- /dev/null +++ b/esphome/components/mipi_rgb/models/elecrow.py @@ -0,0 +1,28 @@ +from . import RgbDriverChip + +# fmt: off +RgbDriverChip( + "CROWPANEL-ADVANCE-7", + requires={"psram"}, + initsequence=(), + pclk_frequency="20MHz", + hsync_pulse_width=4, + hsync_front_porch=8, + hsync_back_porch=8, + vsync_pulse_width=4, + vsync_front_porch=8, + vsync_back_porch=8, + pclk_inverted=True, + color_order="RGB", + width=800, + height=480, + de_pin=42, + hsync_pin=40, + vsync_pin=41, + pclk_pin=39, + data_pins={ + "red": [7, 17, 18, 3, 46], + "green": [9, 10, 11, 12, 13, 14], + "blue": [21, 47, 48, 45, 38], + }, +) From 3fea080ed8f7abc110bc86f9c72c09e1b0451770 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 28 Aug 2026 21:08:15 -0500 Subject: [PATCH 026/147] [core] Hash downloaded file paths at the default data dir location (#18824) --- esphome/core/__init__.py | 9 ++++-- esphome/yaml_util.py | 22 +++++++++++-- tests/unit_tests/core/test_config.py | 28 +++++++++++++++++ tests/unit_tests/test_yaml_util.py | 47 ++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 4 deletions(-) diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index 77efc91bef..6e3f91af22 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -783,7 +783,8 @@ class EsphomeCore: can compare a locally computed hash against the one a device advertises. Machine-local data is kept out of the input: build_path (which embeds ESPHOME_BUILD_PATH and OS path separators) is excluded, - and Path values are dumped relative to the config directory. + and Path values are dumped relative to the config directory, with + the data directory always at its default ``.esphome`` location. """ if self._config_hash is None: from esphome import yaml_util @@ -794,11 +795,15 @@ class EsphomeCore: esphome_conf = dict(esphome_conf) esphome_conf.pop(CONF_BUILD_PATH, None) config[CONF_ESPHOME] = esphome_conf + relative_to = data_dir = None + if self.config_path is not None: + relative_to, data_dir = self.config_dir, self.data_dir config_str = yaml_util.dump( config, show_secrets=True, sort_keys=True, - relative_to=self.config_dir if self.config_path is not None else None, + relative_to=relative_to, + data_dir=data_dir, ) self._config_hash = fnv1a_32bit_hash(config_str) return self._config_hash diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index c280e550c9..7c6cf691b9 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -1057,11 +1057,19 @@ def _load_yaml_internal_with_type( loader.dispose() -def dump(dict_, show_secrets=False, sort_keys=False, relative_to: Path | None = None): +def dump( + dict_, + show_secrets=False, + sort_keys=False, + relative_to: Path | None = None, + data_dir: Path | None = None, +): """Dump YAML to a string and remove null. When ``relative_to`` is given, Path values are dumped relative to that - directory (POSIX form) so the output is machine independent. + directory (POSIX form) so the output is machine independent; Path values + under ``data_dir`` are then dumped as ``.esphome/``. ``data_dir`` + has no effect unless ``relative_to`` is also given. """ if show_secrets: _SECRET_VALUES.clear() @@ -1073,6 +1081,7 @@ def dump(dict_, show_secrets=False, sort_keys=False, relative_to: Path | None = class _Dumper(ESPHomeDumper): _redact_sensitive = not show_secrets _relative_to = relative_to + _data_dir = data_dir return yaml.dump( dict_, @@ -1231,6 +1240,9 @@ class ESPHomeDumper(yaml.SafeDumper): # directory (in POSIX form) so the output does not depend on where the # config lives on the machine that produced it. _relative_to: Path | None = None + # Paths under this directory are dumped as ``.esphome/`` so the + # add-on's ``/data`` mount matches the CLI layout. + _data_dir: Path | None = None def represent_mapping(self, tag, mapping, flow_style=None): value = [] @@ -1274,6 +1286,12 @@ class ESPHomeDumper(yaml.SafeDumper): # path that still cannot be relativized (e.g. a different drive) # keeps its POSIX form so separators stay stable across OSes. path = Path(os.path.normpath(value)) + # Checked first: the default data dir sits inside the config dir. + if self._data_dir is not None and path.is_relative_to( + data_dir := os.path.normpath(self._data_dir) + ): + rel = Path(".esphome") / path.relative_to(data_dir) + return self.represent_stringify(rel.as_posix()) with suppress(ValueError): path = path.relative_to( os.path.normpath(self._relative_to), walk_up=True diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index 68b165c0d0..8ab3ad5d15 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -1127,6 +1127,34 @@ def test_config_hash_same_for_different_config_dirs(tmp_path: Path) -> None: assert hash1 == hash2 +def test_config_hash_same_for_different_data_dirs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Test that downloaded file paths hash the same wherever data_dir lives.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + CORE.reset() + CORE.config_path = config_dir / "device.yaml" + CORE.config = { + "esphome": {"name": "test"}, + "file": config_dir / ".esphome" / "image" / "c44630d6", + } + hash1 = CORE.config_hash + + other_data_dir = tmp_path / "data" + CORE.reset() + monkeypatch.setenv("ESPHOME_DATA_DIR", str(other_data_dir)) + CORE.config_path = config_dir / "device.yaml" + CORE.config = { + "esphome": {"name": "test"}, + "file": other_data_dir / "image" / "c44630d6", + } + hash2 = CORE.config_hash + + assert hash1 == hash2 + + def test_make_app_name_cpp_no_mac_simple() -> None: """Test simple name without MAC suffix returns string literal.""" cpp_expr, global_decl, byte_len = make_app_name_cpp( diff --git a/tests/unit_tests/test_yaml_util.py b/tests/unit_tests/test_yaml_util.py index 3bdbd04396..8e1f9c25c0 100644 --- a/tests/unit_tests/test_yaml_util.py +++ b/tests/unit_tests/test_yaml_util.py @@ -1706,6 +1706,53 @@ def test_dump_path_dotdot_reference_outside_anchor() -> None: assert output.strip() == "file: ../shared/font.ttf" +@pytest.mark.parametrize( + "data_dir", + [ + pytest.param(Path("/config/.esphome"), id="cli"), + pytest.param(Path("/data"), id="addon"), + ], +) +def test_dump_path_under_data_dir_uses_default_location(data_dir: Path) -> None: + """Test that Path values under data_dir dump as .esphome/ for any layout.""" + anchor = Path("/config").absolute() + path = data_dir.absolute() / "image" / "c44630d6" + output = yaml_util.dump( + {"file": path}, relative_to=anchor, data_dir=data_dir.absolute() + ) + assert output.strip() == "file: .esphome/image/c44630d6" + + +def test_dump_path_equal_to_data_dir() -> None: + """Test that the data dir itself dumps as .esphome, matching the default layout.""" + anchor = Path("/config").absolute() + data_dir = Path("/data").absolute() + output = yaml_util.dump({"dir": data_dir}, relative_to=anchor, data_dir=data_dir) + assert output.strip() == "dir: .esphome" + default = yaml_util.dump( + {"dir": anchor / ".esphome"}, relative_to=anchor, data_dir=anchor / ".esphome" + ) + assert default == output + + +def test_dump_path_outside_data_dir_still_relative_to_anchor() -> None: + """Test that data_dir does not affect paths that are not under it.""" + anchor = Path("/config").absolute() + path = anchor / "fonts" / "arial.ttf" + output = yaml_util.dump( + {"file": path}, relative_to=anchor, data_dir=Path("/data").absolute() + ) + assert output.strip() == "file: fonts/arial.ttf" + + +def test_dump_path_data_dir_without_relative_to_is_unchanged() -> None: + """Test that data_dir alone does not change the output.""" + data_dir = Path("/data").absolute() + path = data_dir / "image" / "c44630d6" + output = yaml_util.dump({"file": path}, data_dir=data_dir) + assert output.strip() == f"file: {path}" + + def test_dump_relative_to_does_not_leak_between_calls() -> None: """Test that the relative_to flag is scoped to a single dump call.""" anchor = Path("/config/esphome").absolute() From 6957576867bd15a4520ae62baa5974ae1fde6065 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 28 Aug 2026 22:09:12 -0500 Subject: [PATCH 027/147] [esp32] Skip full rebuild on sdkconfig change with the esp-idf toolchain (#18876) --- esphome/components/esp32/__init__.py | 8 ++- .../components/esp32/test_sdkconfig.py | 72 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 tests/unit_tests/components/esp32/test_sdkconfig.py diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 48bb7bf6a1..8ea08b37d2 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -3249,7 +3249,13 @@ def _write_sdkconfig(): if write_file_if_changed(internal_path, contents): # internal changed, update real one write_file_if_changed(sdk_path, contents) - clean_build(clear_pio_cache=False) + if not CORE.using_toolchain_esp_idf: + # PIO's dependency tracking under-declares sdkconfig inputs + # (ldgen, linker scripts); without a clean the image can be + # unbootable (esphome#15336). The esp-idf toolchain tracks + # sdkconfig via IDF's cmake and has_outdated_files(), so a + # reconfigure suffices there; everything else fails safe. + clean_build(clear_pio_cache=False) def _write_idf_component_yml(): diff --git a/tests/unit_tests/components/esp32/test_sdkconfig.py b/tests/unit_tests/components/esp32/test_sdkconfig.py new file mode 100644 index 0000000000..b5a562f4d1 --- /dev/null +++ b/tests/unit_tests/components/esp32/test_sdkconfig.py @@ -0,0 +1,72 @@ +"""Tests for the esp32 sdkconfig write and its toolchain-gated clean.""" + +from __future__ import annotations + +import os +from pathlib import Path +import time +from unittest.mock import patch + +import pytest + +from esphome.components.esp32 import _write_sdkconfig +from esphome.components.esp32.const import KEY_SDKCONFIG_OPTIONS +from esphome.const import KEY_CORE, KEY_ESP32, KEY_FRAMEWORK_VERSION, Toolchain +from esphome.core import CORE +from esphome.espidf.toolchain import has_outdated_files + + +def _setup_core(tmp_path: Path, toolchain: Toolchain | None) -> None: + CORE.config_path = tmp_path / "test.yaml" + CORE.build_path = tmp_path + CORE.toolchain = toolchain + CORE.data[KEY_ESP32] = {KEY_SDKCONFIG_OPTIONS: {"CONFIG_X": "y"}} + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: "5.5.5"} + + +def _seed_configured_build(tmp_path: Path) -> None: + """A settled native build: configure outputs predate what comes next.""" + build = tmp_path / "build" + (build / "config").mkdir(parents=True) + (build / "config" / "sdkconfig.h").write_text("") + (build / "CMakeCache.txt").write_text("") + (build / "build.ninja").write_text("") + # Explicitly older than what the test writes next: has_outdated_files() + # compares st_mtime with a strict >, so same-tick writes would pass + past = time.time() - 60 + for f in build.rglob("*"): + os.utime(f, (past, past)) + + +@pytest.mark.parametrize( + ("toolchain", "clean_expected"), + [(Toolchain.ESP_IDF, False), (Toolchain.PLATFORMIO, True), (None, True)], +) +def test_write_sdkconfig_cleans_only_on_platformio( + tmp_path: Path, toolchain: Toolchain | None, clean_expected: bool +) -> None: + """A changed sdkconfig forces a full clean only under PlatformIO; the + esp-idf toolchain reconfigures via has_outdated_files() instead; an + unresolved toolchain fails safe onto the clean.""" + _setup_core(tmp_path, toolchain) + _seed_configured_build(tmp_path) + with ( + patch.object(CORE, "name", "test"), + patch("esphome.components.esp32.clean_build") as clean, + ): + _write_sdkconfig() + assert "CONFIG_X" in CORE.relative_build_path("sdkconfig.test").read_text() + assert clean.called is clean_expected + if clean_expected: + clean.assert_called_once_with(clear_pio_cache=False) + # The change must still trigger a reconfigure: the internal + # sdkconfig snapshot is now newer than build/CMakeCache.txt + assert has_outdated_files() is True + clean.reset_mock() + # A settled configure restamps the cache; an unchanged rewrite + # must then neither clean nor mark the build stale + future = time.time() + 60 + os.utime(CORE.relative_build_path("build/CMakeCache.txt"), (future, future)) + _write_sdkconfig() + clean.assert_not_called() + assert has_outdated_files() is False From cd28a8a03e1fd00cde1e94e65ad089f3823ef274 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 29 Aug 2026 23:25:25 -0500 Subject: [PATCH 028/147] [internal_temperature] Re-include esp_phy on the original ESP32 so the PHY blob links (#18884) --- esphome/components/esp32/__init__.py | 2 +- esphome/components/internal_temperature/sensor.py | 6 ++++++ ...exclusion_reincludes_internal_temperature.yaml | 11 +++++++++++ .../exclusion_stays_internal_temperature_s3.yaml | 11 +++++++++++ tests/component_tests/esp32/test_esp32.py | 15 +++++++++++++++ 5 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 tests/component_tests/esp32/config/exclusion_reincludes_internal_temperature.yaml create mode 100644 tests/component_tests/esp32/config/exclusion_stays_internal_temperature_s3.yaml diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 8ea08b37d2..b0290d7a84 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -246,7 +246,7 @@ DEFAULT_EXCLUDED_IDF_COMPONENTS = ( "esp_https_server", # HTTPS server - ESPHome has its own web server "esp_lcd", # LCD controller drivers - only needed by display component "esp_local_ctrl", # Local control over HTTPS/BLE - ESPHome has native API - "esp_phy", # RF PHY - esp_wifi/bt/ieee802154 pull it back when they are in the build + "esp_phy", # RF PHY - re-included by internal_temperature on the original ESP32; esp_wifi/bt/ieee802154 pull it back "esp_wifi", # WiFi stack - re-included by request_wifi(), espnow; bt pulls it back for BLE builds "espcoredump", # Core dump support - ESPHome has its own debug component "fatfs", # FAT filesystem - ESPHome doesn't use filesystem storage diff --git a/esphome/components/internal_temperature/sensor.py b/esphome/components/internal_temperature/sensor.py index d3101f4a7c..40ac216f0c 100644 --- a/esphome/components/internal_temperature/sensor.py +++ b/esphome/components/internal_temperature/sensor.py @@ -1,5 +1,7 @@ import esphome.codegen as cg from esphome.components import sensor +from esphome.components.esp32 import get_esp32_variant, include_builtin_idf_component +from esphome.components.esp32.const import VARIANT_ESP32 from esphome.components.zephyr import zephyr_add_prj_conf from esphome.config_helpers import filter_source_files_from_platform import esphome.config_validation as cv @@ -48,6 +50,10 @@ async def to_code(config: ConfigType) -> None: var = await sensor.new_sensor(config) await cg.register_component(var, config) + if CORE.is_esp32 and get_esp32_variant() == VARIANT_ESP32: + # temprature_sens_read() lives in the esp_phy blob, which is excluded by default + include_builtin_idf_component("esp_phy") + if CORE.using_zephyr and CORE.is_nrf52: zephyr_add_prj_conf("SENSOR", True) zephyr_add_prj_conf("TEMP_NRF5", True) diff --git a/tests/component_tests/esp32/config/exclusion_reincludes_internal_temperature.yaml b/tests/component_tests/esp32/config/exclusion_reincludes_internal_temperature.yaml new file mode 100644 index 0000000000..d5a0aaf157 --- /dev/null +++ b/tests/component_tests/esp32/config/exclusion_reincludes_internal_temperature.yaml @@ -0,0 +1,11 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +sensor: + - platform: internal_temperature + name: Internal Temperature diff --git a/tests/component_tests/esp32/config/exclusion_stays_internal_temperature_s3.yaml b/tests/component_tests/esp32/config/exclusion_stays_internal_temperature_s3.yaml new file mode 100644 index 0000000000..6d4dbf90b5 --- /dev/null +++ b/tests/component_tests/esp32/config/exclusion_stays_internal_temperature_s3.yaml @@ -0,0 +1,11 @@ +esphome: + name: test + +esp32: + board: esp32-s3-devkitc-1 + framework: + type: esp-idf + +sensor: + - platform: internal_temperature + name: Internal Temperature diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index c72c4c3a6b..db7ed6b3fc 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -313,6 +313,12 @@ def test_esp32_configuration_errors( ("esp_wifi",), id="espnow", ), + pytest.param( + # temprature_sens_read() on the original ESP32 lives in the esp_phy blob. + "exclusion_reincludes_internal_temperature.yaml", + ("esp_phy",), + id="internal_temperature", + ), ], ) def test_default_exclusions_reincluded_by_owning_components( @@ -337,6 +343,15 @@ def test_default_exclusions_reincluded_by_owning_components( assert ("esp_http_server" in excluded) == ("esp_http_server" not in reincluded) +def test_esp_phy_stays_excluded_for_internal_temperature_on_newer_variants( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Only the original ESP32 reads the PHY blob; other variants use esp_driver_tsens.""" + generate_main(component_config_path("exclusion_stays_internal_temperature_s3.yaml")) + assert "esp_phy" in CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS] + + def test_nvs_sec_provider_stays_excluded_when_encryption_is_off( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], From a811aa840cc52d4d7bb152255a793e50cf109e55 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Sun, 30 Aug 2026 01:14:12 -0700 Subject: [PATCH 029/147] [modbus] Speed up the bus with microsecond-accurate timing (#12421) --- esphome/components/modbus/modbus.cpp | 182 +++++++++++------- esphome/components/modbus/modbus.h | 21 +- tests/components/modbus/common.h | 9 +- .../components/modbus/modbus_framing_test.cpp | 64 ++++++ 4 files changed, 203 insertions(+), 73 deletions(-) create mode 100644 tests/components/modbus/modbus_framing_test.cpp diff --git a/esphome/components/modbus/modbus.cpp b/esphome/components/modbus/modbus.cpp index aa998d283a..f77492f48b 100644 --- a/esphome/components/modbus/modbus.cpp +++ b/esphome/components/modbus/modbus.cpp @@ -3,6 +3,7 @@ #include #include "esphome/core/application.h" +#include "esphome/core/hal.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" @@ -12,29 +13,49 @@ static const char *const TAG = "modbus"; static constexpr size_t MODBUS_MAX_LOG_BYTES = 64; -// Approximate bits per character on the wire (depends on parity/stop bit config) -static constexpr uint32_t MODBUS_BITS_PER_CHAR = 11; -static constexpr uint32_t MS_PER_SEC = 1000; +static constexpr uint32_t US_PER_SEC = 1000000; +static constexpr uint32_t US_PER_MS = 1000; + +// Minimum interframe delay per the Modbus spec (fixed 1750us above 19200 baud) +static constexpr uint32_t MODBUS_MIN_FRAME_DELAY_US = 1750; + +// Diagnostics only: the backdated byte stamp can precede last_send_ (echo, or noise during our own +// send), where an unsigned wrap would print ~4.29e9. +static uint32_t us_since_send(uint32_t last_modbus_byte, uint32_t last_send) { + const uint32_t elapsed = last_modbus_byte - last_send; + return (int32_t) elapsed < 0 ? 0 : elapsed; +} void Modbus::setup() { if (this->flow_control_pin_ != nullptr) { this->flow_control_pin_->setup(); } - this->frame_delay_ms_ = - std::max(2, // 1750us minimum per spec - rounded up to 2ms. - // 3.5 characters * 11 bits per character * 1000ms/sec / (bits/sec) (Standard modbus frame delay) - (uint16_t) (3.5 * MODBUS_BITS_PER_CHAR * MS_PER_SEC / this->parent_->get_baud_rate()) + 1); + // RTU specifies 11 bits per character but 8N1 is 10, so derive it from the framing. The schema + // forbids a zero, so one here means the hub never set it (weikai): fall back to 8N1 and a 1 baud floor. + const uint8_t data_bits = this->parent_->get_data_bits() != 0 ? this->parent_->get_data_bits() : 8; + const uint8_t stop_bits = this->parent_->get_stop_bits() != 0 ? this->parent_->get_stop_bits() : 1; + const uint32_t baud_rate = std::max(1u, this->parent_->get_baud_rate()); + this->bits_per_char_ = static_cast( + 1 + data_bits + (this->parent_->get_parity() == uart::UART_CONFIG_PARITY_NONE ? 0 : 1) + stop_bits); + + // 3.5 characters * bits per character * 1e6 us/sec / (bits/sec) (Standard modbus frame delay) + this->frame_delay_us_ = + std::max(MODBUS_MIN_FRAME_DELAY_US, (uint32_t) (3.5 * this->bits_per_char_ * US_PER_SEC / baud_rate) + 1); // When rx_full_threshold is configured (non-zero), the UART has a hardware FIFO with a // meaningful threshold (e.g., ESP32 native UART), so we can calculate a precise delay. // Otherwise (e.g., USB UART), use 50ms to handle data arriving in chunks. - static constexpr uint16_t DEFAULT_LONG_RX_BUFFER_DELAY_MS = 50; + static constexpr uint32_t DEFAULT_LONG_RX_BUFFER_DELAY_US = 50 * US_PER_MS; size_t rx_threshold = this->parent_->get_rx_full_threshold(); - this->long_rx_buffer_delay_ms_ = - rx_threshold != uart::UARTComponent::RX_FULL_THRESHOLD_UNSET - ? (rx_threshold * MODBUS_BITS_PER_CHAR * MS_PER_SEC / this->parent_->get_baud_rate()) + 1 - : DEFAULT_LONG_RX_BUFFER_DELAY_MS; + this->long_rx_buffer_delay_us_ = rx_threshold != uart::UARTComponent::RX_FULL_THRESHOLD_UNSET + ? (uint32_t) (rx_threshold * this->bits_per_char_ * US_PER_SEC / baud_rate) + 1 + : DEFAULT_LONG_RX_BUFFER_DELAY_US; + + // The idle-timeout interrupt fires rx_timeout characters after the last byte, so that much silence + // has already passed by the time we read it: backdate so the gap measures silence on the wire. + this->rx_detect_latency_us_ = + (uint32_t) (this->parent_->get_rx_timeout() * this->bits_per_char_ * US_PER_SEC / baud_rate); } void Modbus::loop() { @@ -52,7 +73,7 @@ void ModbusClientHub::loop() { // Send-wait watchdog: only the cheap time check runs at loop rate; expire_waiting_() looks the // entry up and holds off if the response has started arriving. if (this->waiting_for_response_ && - this->last_receive_check_ - this->last_send_ > this->last_send_tx_offset_ + this->send_wait_time_) { + this->last_receive_check_ - this->last_send_ > this->last_send_tx_offset_ + this->send_wait_time_us_) { this->expire_waiting_(); } @@ -72,7 +93,7 @@ void ModbusClientHub::expire_waiting_() { } // Only a genuine WAITING entry warrants the log (a cleared or interrupted shell timing out is expected). if (cmd->state == FrameState::WAITING) { - ESP_LOGW(TAG, "Stop waiting for response from %" PRIu8 " %" PRIu32 "ms after last send", cmd->frame.address(), + ESP_LOGW(TAG, "Stop waiting for response from %" PRIu8 " %" PRIu32 "us after last send", cmd->frame.address(), this->last_receive_check_ - this->last_send_); } // Deliver on_no_response directly, the way the parse path delivers response()/error(): the entry @@ -86,37 +107,47 @@ void ModbusClientHub::expire_waiting_() { bool Modbus::timeout_() { // If the response frame is finished (including interframe delay) - we timeout. // The long_rx_buffer_delay accounts for long responses (larger than the UART rx_full_threshold) to avoid timeouts - // when the buffer is filling the back half of the response - const uint16_t timeout = std::max( - (uint16_t) this->frame_delay_ms_, - (uint16_t) (this->rx_buffer_.size() >= this->parent_->get_rx_full_threshold() ? this->long_rx_buffer_delay_ms_ - : 0)); + // when the buffer is filling the back half of the response. The latch decides, not the current size: + // parsing a leading frame can shrink the buffer below the threshold while the rest is still streaming. + // The latency term covers the final batch, which is idle-delivered. + const uint32_t timeout = + this->exceeded_rx_full_threshold_ + ? std::max(this->frame_delay_us_, this->long_rx_buffer_delay_us_ + this->rx_detect_latency_us_) + : this->frame_delay_us_; return this->last_receive_check_ - this->last_modbus_byte_ > timeout; } +// We use micros() here and elsewhere instead of App.get_loop_component_start_time() to avoid stale timestamps +// It's critical in all timestamp comparisons that the left timestamp comes before the right one in time +// If we use a cached value in place of micros() and last_modbus_byte_ is updated inside our loop +// then the comparison is backwards (small negative which wraps to large positive) and will cause a false timeout +// So in this component we don't use any cached timestamp values to avoid these annoying bugs. +// Compare before subtracting: a signed difference would read a bus idle past half the micros() wrap +// (~35 min) as a huge delay still owed. +static inline uint32_t remaining_delay(uint32_t elapsed, uint32_t required) { + return elapsed >= required ? 0 : required - elapsed; +} + int32_t Modbus::tx_delay_remaining() { - // millis() here and everywhere in this component, never a cached loop timestamp: a cached "now" can - // predate last_modbus_byte_, and the unsigned subtraction then wraps huge and forces a false timeout. - const uint32_t now = millis(); - return std::max({(int32_t) 0, - (int32_t) (this->last_send_tx_offset_ + this->frame_delay_ms_ - (now - this->last_send_)), - (int32_t) (this->frame_delay_ms_ - (now - this->last_modbus_byte_))}); + const uint32_t now = micros(); + return (int32_t) std::max(remaining_delay(now - this->last_send_, this->last_send_tx_offset_ + this->frame_delay_us_), + remaining_delay(now - this->last_modbus_byte_, this->frame_delay_us_)); } int32_t ModbusClientHub::tx_delay_remaining() { - const uint32_t now = millis(); - return std::max({(int32_t) 0, - (int32_t) (this->last_send_tx_offset_ + this->frame_delay_ms_ + this->turnaround_delay_ms_ - - (now - this->last_send_)), - (int32_t) (this->frame_delay_ms_ + this->turnaround_delay_ms_ - (now - this->last_modbus_byte_))}); + const uint32_t now = micros(); + return (int32_t) std::max( + remaining_delay(now - this->last_send_, + this->last_send_tx_offset_ + this->frame_delay_us_ + this->turnaround_delay_us_), + remaining_delay(now - this->last_modbus_byte_, this->frame_delay_us_ + this->turnaround_delay_us_)); } bool Modbus::tx_blocked() { // Blocked while any rx bytes are pending, or within tx_delay of the last byte in either direction // (receivers must see our previous tx as done, and more rx may be coming). A remaining delay up to - // MODBUS_TX_MAX_DELAY_MS doesn't block - send_frame_ absorbs it instead of looping on small waits. - return this->available() || !this->rx_buffer_.empty() || this->tx_delay_remaining() > MODBUS_TX_MAX_DELAY_MS; + // MODBUS_TX_MAX_DELAY_US doesn't block - send_frame_ absorbs it instead of looping on small waits. + return this->available() || !this->rx_buffer_.empty() || this->tx_delay_remaining() > MODBUS_TX_MAX_DELAY_US; } bool ModbusClientHub::tx_blocked() { return this->waiting_for_response_ || this->Modbus::tx_blocked(); } @@ -133,20 +164,26 @@ bool ModbusClientHub::tx_buffer_empty() { } void Modbus::receive_bytes_() { - this->last_receive_check_ = millis(); + this->last_receive_check_ = micros(); size_t bytes = this->available(); if (bytes) { size_t buffer_size = this->rx_buffer_.size(); - this->last_modbus_byte_ = this->last_receive_check_; + // Below the threshold the batch can only be idle-delivered, so its last byte finished one detection + // latency ago; at or above it the frame may still be streaming, so stamp now. + this->last_modbus_byte_ = bytes < this->parent_->get_rx_full_threshold() + ? this->last_receive_check_ - this->rx_detect_latency_us_ + : this->last_receive_check_; this->rx_buffer_.resize(buffer_size + bytes); if (!this->read_array(this->rx_buffer_.data() + buffer_size, bytes)) { this->rx_buffer_.resize(buffer_size); return; } + if (this->rx_buffer_.size() >= this->parent_->get_rx_full_threshold()) + this->exceeded_rx_full_threshold_ = true; if (buffer_size == 0) { - ESP_LOGV(TAG, "Received first byte %" PRIu8 " (0X%x) of %zu bytes %" PRIu32 "ms after last send", - this->rx_buffer_[0], this->rx_buffer_[0], this->rx_buffer_.size(), millis() - this->last_send_); + ESP_LOGV(TAG, "Received first byte %" PRIu8 " (0X%x) of %zu bytes %" PRIu32 "us after last send", + this->rx_buffer_[0], this->rx_buffer_[0], this->rx_buffer_.size(), micros() - this->last_send_); } } } @@ -299,8 +336,8 @@ void ModbusClientHub::process_modbus_server_frame(uint8_t address, std::spanwaiting_for_response_ ? this->find_waiting_() : nullptr; if (cmd == nullptr) { ESP_LOGW(TAG, - "Received unexpected frame from address %" PRIu8 ", function code 0x%X, %" PRIu32 "ms after last send", - address, function_code, this->last_modbus_byte_ - this->last_send_); + "Received unexpected frame from address %" PRIu8 ", function code 0x%X, %" PRIu32 "us after last send", + address, function_code, us_since_send(this->last_modbus_byte_, this->last_send_)); return; } @@ -310,9 +347,9 @@ void ModbusClientHub::process_modbus_server_frame(uint8_t address, std::span %" PRIu8 " or function code 0x%X <> 0x%X, %" PRIu32 - "ms after last send", + "us after last send", address, expected_address, (function_code & FUNCTION_CODE_MASK), expected_function_code, - this->last_modbus_byte_ - this->last_send_); + us_since_send(this->last_modbus_byte_, this->last_send_)); // Unexpected frame: flip a WAITING entry to an INTERRUPTED shell that ignores the rest of this // transaction and blocks tx until the send-wait timeout, where it gets its on_no_response. cmd->interrupt(); @@ -325,8 +362,8 @@ void ModbusClientHub::process_modbus_server_frame(uint8_t address, std::spanlast_modbus_byte_ - this->last_send_); + "us after last send", + address, us_since_send(this->last_modbus_byte_, this->last_send_)); return; } @@ -337,12 +374,12 @@ void ModbusClientHub::process_modbus_server_frame(uint8_t address, std::spansweep_needed_ = true; if (helpers::is_function_code_exception(function_code)) { uint8_t exception = pdu[1]; // exception frames are fixed-length, so the code is always present - ESP_LOGW(TAG, "Error function code: 0x%X exception: %" PRIu8 ", address: %" PRIu8 ", %" PRIu32 "ms after last send", - function_code, exception, address, this->last_modbus_byte_ - this->last_send_); + ESP_LOGW(TAG, "Error function code: 0x%X exception: %" PRIu8 ", address: %" PRIu8 ", %" PRIu32 "us after last send", + function_code, exception, address, us_since_send(this->last_modbus_byte_, this->last_send_)); cmd->error(static_cast(exception)); } else if (!cmd->response(pdu)) { - ESP_LOGV(TAG, "Ignoring response from %" PRIu8 " - no callback device set, %" PRIu32 "ms after last send", address, - this->last_modbus_byte_ - this->last_send_); + ESP_LOGV(TAG, "Ignoring response from %" PRIu8 " - no callback device set, %" PRIu32 "us after last send", address, + us_since_send(this->last_modbus_byte_, this->last_send_)); } } @@ -738,9 +775,15 @@ void ModbusServerHub::process_modbus_client_frame_(uint8_t address, uint8_t func // Callers gate on tx_blocked() first, but the pre-send delay below can span several ms, so re-check // after it and refuse (return false) if a byte arrived in that window rather than transmit over it. bool Modbus::send_frame_(const ModbusFrame &frame) { - const int32_t tx_delay_remaining = this->tx_delay_remaining(); + int32_t tx_delay_remaining = this->tx_delay_remaining(); if (tx_delay_remaining > 0) { - delay(tx_delay_remaining); + // delay() only lands on tick boundaries, so yield with it to get close, then busy-wait the rest. + if (tx_delay_remaining > (int32_t) (2 * US_PER_MS)) { + delay((tx_delay_remaining - US_PER_MS) / US_PER_MS); + tx_delay_remaining = this->tx_delay_remaining(); + } + if (tx_delay_remaining > 0) + delayMicroseconds(tx_delay_remaining); } if (this->tx_blocked()) { @@ -755,14 +798,15 @@ bool Modbus::send_frame_(const ModbusFrame &frame) { this->last_send_tx_offset_ = 0; } else { this->write_array(frame.data.data(), frame.size()); - this->last_send_tx_offset_ = frame.size() * MODBUS_BITS_PER_CHAR * MS_PER_SEC / this->parent_->get_baud_rate() + 1; + this->last_send_tx_offset_ = + frame.size() * this->bits_per_char_ * US_PER_SEC / std::max(1u, this->parent_->get_baud_rate()) + 1; } - uint32_t now = millis(); + uint32_t now = micros(); #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE char hex_buf[format_hex_pretty_size(MODBUS_MAX_LOG_BYTES)]; #endif - ESP_LOGV(TAG, "Write: %s %" PRIu32 "ms after last send, %" PRIu32 "ms after last receive", + ESP_LOGV(TAG, "Write: %s %" PRIu32 "us after last send, %" PRIu32 "us after last receive", format_hex_pretty_to(hex_buf, frame.data.data(), frame.size()), now - this->last_send_, now - this->last_modbus_byte_); this->last_send_ = now; @@ -800,20 +844,25 @@ void ModbusClientHub::send_next_frame_() { void ModbusClientHub::dump_config() { ESP_LOGCONFIG(TAG, "Modbus:\n" - " Send Wait Time: %" PRIu16 " ms\n" - " Turnaround Time: %" PRIu16 " ms\n" - " Frame Delay: %" PRIu16 " ms\n" - " Long Rx Buffer Delay: %" PRIu16 " ms", - this->send_wait_time_, this->turnaround_delay_ms_, this->frame_delay_ms_, - this->long_rx_buffer_delay_ms_); + " Send Wait Time: %" PRIu32 " ms\n" + " Turnaround Time: %" PRIu32 " ms\n" + " Frame Delay: %" PRIu32 " us\n" + " Long Rx Buffer Delay: %" PRIu32 " us\n" + " Bits Per Character: %" PRIu8 "\n" + " Rx Detect Latency: %" PRIu32 " us", + this->send_wait_time_us_ / US_PER_MS, this->turnaround_delay_us_ / US_PER_MS, this->frame_delay_us_, + this->long_rx_buffer_delay_us_, this->bits_per_char_, this->rx_detect_latency_us_); LOG_PIN(" Flow Control Pin: ", this->flow_control_pin_); } void ModbusServerHub::dump_config() { ESP_LOGCONFIG(TAG, "Modbus:\n" - " Frame Delay: %" PRIu16 " ms\n" - " Long Rx Buffer Delay: %" PRIu16 " ms", - this->frame_delay_ms_, this->long_rx_buffer_delay_ms_); + " Frame Delay: %" PRIu32 " us\n" + " Long Rx Buffer Delay: %" PRIu32 " us\n" + " Bits Per Character: %" PRIu8 "\n" + " Rx Detect Latency: %" PRIu32 " us", + this->frame_delay_us_, this->long_rx_buffer_delay_us_, this->bits_per_char_, + this->rx_detect_latency_us_); LOG_PIN(" Flow Control Pin: ", this->flow_control_pin_); } @@ -1142,7 +1191,8 @@ void ModbusServerHub::send_raw_(const uint8_t *payload, uint16_t len) { // without a heap allocation. Only one server reply is ever waiting, so a single buffer suffices. std::memcpy(this->deferred_payload_.data(), payload, len); this->deferred_payload_len_ = len; - this->set_timeout("deferred_send", this->tx_delay_remaining(), [this]() { + // set_timeout() takes milliseconds; round the microsecond delay up so we never fire early. + this->set_timeout("deferred_send", (this->tx_delay_remaining() + US_PER_MS - 1) / US_PER_MS, [this]() { ModbusFrame frame(this->deferred_payload_[0], this->deferred_payload_.data() + 1, this->deferred_payload_len_ - 1); if (!this->send_frame_(frame)) @@ -1162,11 +1212,11 @@ void Modbus::clear_rx_buffer_(const LogString *reason, bool warn, size_t bytes_t bytes = bytes_to_clear; if (bytes > 0) { if (warn) { - ESP_LOGW(TAG, "Clearing buffer of %zu bytes - %s %" PRIu32 "ms after last send", bytes, LOG_STR_ARG(reason), - millis() - this->last_send_); + ESP_LOGW(TAG, "Clearing buffer of %zu bytes - %s %" PRIu32 "us after last send", bytes, LOG_STR_ARG(reason), + micros() - this->last_send_); } else { - ESP_LOGV(TAG, "Clearing buffer of %zu bytes - %s %" PRIu32 "ms after last send", bytes, LOG_STR_ARG(reason), - millis() - this->last_send_); + ESP_LOGV(TAG, "Clearing buffer of %zu bytes - %s %" PRIu32 "us after last send", bytes, LOG_STR_ARG(reason), + micros() - this->last_send_); } if (bytes == this->rx_buffer_.size()) { this->rx_buffer_.clear(); @@ -1174,6 +1224,8 @@ void Modbus::clear_rx_buffer_(const LogString *reason, bool warn, size_t bytes_t this->rx_buffer_.erase(this->rx_buffer_.begin(), this->rx_buffer_.begin() + bytes); } } + if (this->rx_buffer_.empty()) + this->exceeded_rx_full_threshold_ = false; } void ModbusClientDevice::dispatch_response_(std::span request_pdu, std::span response_pdu, diff --git a/esphome/components/modbus/modbus.h b/esphome/components/modbus/modbus.h index 69a7eb82e3..7d7818239d 100644 --- a/esphome/components/modbus/modbus.h +++ b/esphome/components/modbus/modbus.h @@ -19,7 +19,7 @@ namespace esphome::modbus { // Tx queue backstop: duplicates dedup into one entry, so only a runaway generator of distinct frames // (e.g. a loop writing a changing value) could grow the heap unboundedly. static constexpr uint16_t MODBUS_TX_BUFFER_SIZE = 128; -static constexpr uint16_t MODBUS_TX_MAX_DELAY_MS = 5; +static constexpr uint16_t MODBUS_TX_MAX_DELAY_US = 5000; // Typical frames -- reads and single-register/coil writes -- are exactly 8 bytes // (address + 5-byte PDU + 2-byte CRC). @@ -70,12 +70,18 @@ class Modbus : public uart::UARTDevice, public Component { bool send_frame_(const ModbusFrame &frame); uint16_t find_frame_end_by_crc_(uint16_t min_length) const; + // All timestamps and durations below are micros()-based uint32_t last_modbus_byte_{0}; uint32_t last_receive_check_{0}; uint32_t last_send_{0}; uint32_t last_send_tx_offset_{0}; - uint16_t frame_delay_ms_{5}; - uint16_t long_rx_buffer_delay_ms_{0}; + uint32_t frame_delay_us_{5000}; + uint32_t long_rx_buffer_delay_us_{0}; + uint32_t rx_detect_latency_us_{0}; + // Bits on the wire per character (start + data + optional parity + stop); 12 at most. + uint8_t bits_per_char_{11}; + // Latched when a read reaches rx_full_threshold, cleared when the buffer drains. + bool exceeded_rx_full_threshold_{false}; GPIOPin *flow_control_pin_{nullptr}; @@ -232,8 +238,9 @@ class ModbusClientHub : public Modbus { ModbusClientHub() = default; void dump_config() override; void loop() override; - void set_send_wait_time(uint16_t time_in_ms) { this->send_wait_time_ = time_in_ms; } - void set_turnaround_time(uint16_t time_in_ms) { this->turnaround_delay_ms_ = time_in_ms; } + // Config arrives in milliseconds; stored internally in microseconds like all other timing. + void set_send_wait_time(uint16_t time_in_ms) { this->send_wait_time_us_ = time_in_ms * 1000UL; } + void set_turnaround_time(uint16_t time_in_ms) { this->turnaround_delay_us_ = time_in_ms * 1000UL; } bool tx_buffer_empty(); bool tx_blocked() override; ESPDEPRECATED("Use queue_pdu() with create_client_pdu() instead. Removed in 2026.10.0", "2026.4.0") @@ -279,8 +286,8 @@ class ModbusClientHub : public Modbus { // End the wait for a response on send-wait timeout (the loop() watchdog body); see FrameState. void expire_waiting_(); - uint16_t send_wait_time_{2000}; - uint16_t turnaround_delay_ms_{0}; + uint32_t send_wait_time_us_{2000000}; + uint32_t turnaround_delay_us_{0}; // Set on transmit, cleared on the transaction-ending transition; send_next_frame_ won't select // while it is set, so at most one frame is awaiting a response. diff --git a/tests/components/modbus/common.h b/tests/components/modbus/common.h index e6c37b0e6d..83d30b3f6d 100644 --- a/tests/components/modbus/common.h +++ b/tests/components/modbus/common.h @@ -11,7 +11,14 @@ namespace esphome::modbus::testing { // A UART that discards all writes, for tests that never inspect the wire. class NullUART : public uart::UARTComponent { public: - NullUART() { this->set_baud_rate(115200); } + // 8N1, matching what the uart schema emits for a real hub; the framing drives the modbus + // interframe timing, so leaving data/stop bits at their zero defaults would not be representative. + NullUART() { + this->set_baud_rate(115200); + this->set_data_bits(8); + this->set_stop_bits(1); + this->set_parity(uart::UART_CONFIG_PARITY_NONE); + } void write_array(const uint8_t *data, size_t len) override {} bool peek_byte(uint8_t *data) override { return false; } bool read_array(uint8_t *data, size_t len) override { return false; } diff --git a/tests/components/modbus/modbus_framing_test.cpp b/tests/components/modbus/modbus_framing_test.cpp new file mode 100644 index 0000000000..a102db5a51 --- /dev/null +++ b/tests/components/modbus/modbus_framing_test.cpp @@ -0,0 +1,64 @@ +#include + +#include + +#include "common.h" +#include "esphome/components/modbus/modbus.h" + +namespace esphome::modbus::testing { + +namespace { + +// Exposes the timing values setup() derives from the UART framing. +class FramingProbeHub : public ModbusClientHub { + public: + uint32_t bits_per_char() const { return this->bits_per_char_; } + uint32_t frame_delay_us() const { return this->frame_delay_us_; } +}; + +class FramedUART : public NullUART { + public: + FramedUART(uint32_t baud_rate, uint8_t data_bits, uint8_t stop_bits, uart::UARTParityOptions parity) { + this->set_baud_rate(baud_rate); + this->set_data_bits(data_bits); + this->set_stop_bits(stop_bits); + this->set_parity(parity); + } +}; + +} // namespace + +// 8N1 is 10 bits on the wire, so t3.5 at 9600 baud is 3.5 * 10 / 9600 = 3645.8us. +TEST(ModbusFraming, EightNoneOneDerivesTenBits) { + FramedUART uart(9600, 8, 1, uart::UART_CONFIG_PARITY_NONE); + FramingProbeHub hub; + hub.set_uart_parent(&uart); + hub.setup(); + + EXPECT_EQ(hub.bits_per_char(), 10u); + EXPECT_EQ(hub.frame_delay_us(), 3646u); +} + +// Spec-conformant RTU framing is 11 bits, which lengthens the interframe gap to +// 3.5 * 11 / 9600 = 4010.4us, rounded up. +TEST(ModbusFraming, EightEvenOneDerivesElevenBits) { + FramedUART uart(9600, 8, 1, uart::UART_CONFIG_PARITY_EVEN); + FramingProbeHub hub; + hub.set_uart_parent(&uart); + hub.setup(); + + EXPECT_EQ(hub.bits_per_char(), 11u); + EXPECT_EQ(hub.frame_delay_us(), 4011u); +} + +// Above 19200 baud the spec's fixed 1750us floor governs instead of 3.5 characters. +TEST(ModbusFraming, FastBaudUsesSpecFloor) { + FramedUART uart(115200, 8, 1, uart::UART_CONFIG_PARITY_NONE); + FramingProbeHub hub; + hub.set_uart_parent(&uart); + hub.setup(); + + EXPECT_EQ(hub.frame_delay_us(), 1750u); +} + +} // namespace esphome::modbus::testing From b9eda644cd0219e560933b5151c0912f4c648278 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:21:07 +1000 Subject: [PATCH 030/147] [lvgl] Add radial and conical gradients (#18818) --- esphome/components/lvgl/defines.py | 3 +- esphome/components/lvgl/gradient.py | 195 +++++++++++++++++++++--- tests/components/lvgl/lvgl-package.yaml | 65 ++++++++ 3 files changed, 239 insertions(+), 24 deletions(-) diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index 81a4d2b4ab..61d15752be 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -483,6 +483,7 @@ LV_ANIM = LvConstant( LV_GRAD_DIR = LvConstant("LV_GRAD_DIR_", "NONE", "HOR", "VER") LV_DITHER = LvConstant("LV_DITHER_", "NONE", "ORDERED", "ERR_DIFF") +LV_GRAD_EXTEND = LvConstant("LV_GRAD_EXTEND_", "PAD", "REPEAT", "REFLECT") LV_LOG_LEVELS = { "VERBOSE": "TRACE", @@ -904,7 +905,7 @@ LV_COLOR_FORMATS = ( LV_DEFINES = ( "LV_USE_FREERTOS_TASK_NOTIFY", "LV_DRAW_BUF_STRIDE_ALIGN", "LV_USE_DRAW_SW", "LV_DRAW_SW_DRAW_UNIT_CNT", - "LV_DRAW_SW_COMPLEX", "LV_USE_DRAW_PXP", "LV_USE_PXP_DRAW_THREAD", "LV_USE_DRAW_G2D", + "LV_DRAW_SW_COMPLEX", "LV_USE_DRAW_SW_COMPLEX_GRADIENTS", "LV_USE_DRAW_PXP", "LV_USE_PXP_DRAW_THREAD", "LV_USE_DRAW_G2D", "LV_USE_G2D_DRAW_THREAD", "LV_VG_LITE_USE_BOX_SHADOW", "LV_VG_LITE_THORVG_16PIXELS_ALIGN", "LV_LOG_USE_TIMESTAMP", "LV_LOG_USE_FILE_LINE", "LV_USE_OBJ_ID_BUILTIN", "LV_USE_OBJ_PROPERTY_NAME", "LV_ATTRIBUTE_MEM_ALIGN_SIZE", "LV_FONT_MONTSERRAT_14", "LV_USE_FONT_PLACEHOLDER", "LV_WIDGETS_HAS_DEFAULT_VALUE", "LV_USE_ARCLABEL", diff --git a/esphome/components/lvgl/gradient.py b/esphome/components/lvgl/gradient.py index 2f1be20772..8db183fabe 100644 --- a/esphome/components/lvgl/gradient.py +++ b/esphome/components/lvgl/gradient.py @@ -13,18 +13,40 @@ from esphome.core import ID from esphome.cpp_generator import MockObj from .defines import ( + CONF_END_ANGLE, CONF_GRADIENTS, CONF_OPA, + CONF_START_ANGLE, LV_DITHER, + LV_GRAD_EXTEND, add_define, add_lv_use, add_warning, ) -from .lv_validation import lv_color, lv_percentage, opacity +from .lv_validation import ( + lv_angle_degrees, + lv_color, + lv_percentage, + opacity, + pixels_or_percent, +) from .lvcode import lv from .types import lv_color_t, lv_gradient_t, lv_opa_t CONF_STOPS = "stops" +CONF_LINEAR = "linear" +CONF_RADIAL = "radial" +CONF_CONICAL = "conical" +CONF_EXTEND = "extend" +CONF_FROM_X = "from_x" +CONF_FROM_Y = "from_y" +CONF_TO_X = "to_x" +CONF_TO_Y = "to_y" +CONF_CENTER_X = "center_x" +CONF_CENTER_Y = "center_y" +CONF_FOCAL_X = "focal_x" +CONF_FOCAL_Y = "focal_y" +CONF_FOCAL_RADIUS = "focal_radius" def min_stops(value): @@ -33,27 +55,109 @@ def min_stops(value): return value +STOPS_SCHEMA = cv.All( + [ + cv.Schema( + { + cv.Required(CONF_COLOR): lv_color, + cv.Optional(CONF_OPA, default=1.0): opacity, + cv.Required(CONF_POSITION): lv_percentage, + } + ) + ], + min_stops, +) + +LINEAR_SCHEMA = cv.Schema( + { + cv.Required(CONF_FROM_X): pixels_or_percent, + cv.Required(CONF_FROM_Y): pixels_or_percent, + cv.Required(CONF_TO_X): pixels_or_percent, + cv.Required(CONF_TO_Y): pixels_or_percent, + cv.Optional(CONF_EXTEND, default="PAD"): LV_GRAD_EXTEND.one_of, + } +) + +RADIAL_SCHEMA = cv.Schema( + { + cv.Required(CONF_CENTER_X): pixels_or_percent, + cv.Required(CONF_CENTER_Y): pixels_or_percent, + cv.Required(CONF_TO_X): pixels_or_percent, + cv.Required(CONF_TO_Y): pixels_or_percent, + cv.Optional(CONF_FOCAL_X): pixels_or_percent, + cv.Optional(CONF_FOCAL_Y): pixels_or_percent, + # No default: gradient_validator() must be able to tell whether this was actually + # given, to require it alongside focal_x/focal_y rather than silently drop it. + # LVGL's lv_grad_radial_set_focal() takes this as a scalar, not lv_pct() - + # unlike every other coordinate here, a percentage is not accepted. + cv.Optional(CONF_FOCAL_RADIUS): cv.positive_int, + cv.Optional(CONF_EXTEND, default="PAD"): LV_GRAD_EXTEND.one_of, + } +) + +CONICAL_SCHEMA = cv.Schema( + { + cv.Required(CONF_CENTER_X): pixels_or_percent, + cv.Required(CONF_CENTER_Y): pixels_or_percent, + cv.Optional(CONF_START_ANGLE, default=0): lv_angle_degrees, + cv.Optional(CONF_END_ANGLE, default=360): lv_angle_degrees, + cv.Optional(CONF_EXTEND, default="PAD"): LV_GRAD_EXTEND.one_of, + } +) + + +def gradient_validator(config): + direction = config[CONF_DIRECTION] + for gradient_direction, key in ( + ("LINEAR", CONF_LINEAR), + ("RADIAL", CONF_RADIAL), + ("CONICAL", CONF_CONICAL), + ): + if direction == gradient_direction: + if key not in config: + raise cv.Invalid( + f"'{key}' is required for {gradient_direction} gradient direction" + ) + elif key in config: + raise cv.Invalid( + f"'{key}' is only valid with 'direction: {gradient_direction}'" + ) + if CONF_RADIAL in config: + radial = config[CONF_RADIAL] + has_focal_x = CONF_FOCAL_X in radial + has_focal_y = CONF_FOCAL_Y in radial + has_focal_radius = CONF_FOCAL_RADIUS in radial + if has_focal_x != has_focal_y or (has_focal_radius and not has_focal_x): + raise cv.Invalid( + "'focal_x', 'focal_y' and 'focal_radius' must be specified together " + "in 'radial'" + ) + return config + + GRADIENT_SCHEMA = cv.ensure_list( - cv.Schema( - { - cv.GenerateID(CONF_ID): cv.declare_id(lv_gradient_t), - cv.Required(CONF_DIRECTION): cv.one_of( - "HOR", "HORIZONTAL", "VER", "VERTICAL", upper=True - ), - cv.Optional(CONF_DITHER): LV_DITHER.one_of, - cv.Required(CONF_STOPS): cv.All( - [ - cv.Schema( - { - cv.Required(CONF_COLOR): lv_color, - cv.Optional(CONF_OPA, default=1.0): opacity, - cv.Required(CONF_POSITION): lv_percentage, - } - ) - ], - min_stops, - ), - } + cv.All( + cv.Schema( + { + cv.GenerateID(CONF_ID): cv.declare_id(lv_gradient_t), + cv.Required(CONF_DIRECTION): cv.one_of( + "HOR", + "HORIZONTAL", + "VER", + "VERTICAL", + "LINEAR", + "RADIAL", + "CONICAL", + upper=True, + ), + cv.Optional(CONF_DITHER): LV_DITHER.one_of, + cv.Optional(CONF_LINEAR): LINEAR_SCHEMA, + cv.Optional(CONF_RADIAL): RADIAL_SCHEMA, + cv.Optional(CONF_CONICAL): CONICAL_SCHEMA, + cv.Required(CONF_STOPS): STOPS_SCHEMA, + } + ), + gradient_validator, ) ) @@ -65,15 +169,60 @@ async def gradients_to_code(config): add_warning( "The 'dither' option for gradients is not supported by LVGL 9.x and will be ignored" ) + if any( + x[CONF_DIRECTION] in ("LINEAR", "RADIAL", "CONICAL") + for x in config.get(CONF_GRADIENTS, ()) + ): + # LVGL's software renderer only draws these gradient types when this is enabled; without + # it they silently fall back to a plain horizontal gradient. + add_define("LV_USE_DRAW_SW_COMPLEX_GRADIENTS") for gradient in config.get(CONF_GRADIENTS, ()): var = MockObj(cg.new_Pvariable(gradient[CONF_ID]), "->") idbase = gradient[CONF_ID].id stops = sorted(gradient[CONF_STOPS], key=itemgetter(CONF_POSITION)) max_stops = max(max_stops, len(stops)) - if gradient[CONF_DIRECTION].startswith("VER"): + direction = gradient[CONF_DIRECTION] + if direction.startswith("VER"): lv.grad_vertical_init(var) - else: + elif direction.startswith("HOR"): lv.grad_horizontal_init(var) + elif direction == "LINEAR": + linear = gradient[CONF_LINEAR] + lv.grad_linear_init( + var, + await pixels_or_percent.process(linear[CONF_FROM_X]), + await pixels_or_percent.process(linear[CONF_FROM_Y]), + await pixels_or_percent.process(linear[CONF_TO_X]), + await pixels_or_percent.process(linear[CONF_TO_Y]), + await LV_GRAD_EXTEND.process(linear[CONF_EXTEND]), + ) + elif direction == "RADIAL": + radial = gradient[CONF_RADIAL] + lv.grad_radial_init( + var, + await pixels_or_percent.process(radial[CONF_CENTER_X]), + await pixels_or_percent.process(radial[CONF_CENTER_Y]), + await pixels_or_percent.process(radial[CONF_TO_X]), + await pixels_or_percent.process(radial[CONF_TO_Y]), + await LV_GRAD_EXTEND.process(radial[CONF_EXTEND]), + ) + if CONF_FOCAL_X in radial: + lv.grad_radial_set_focal( + var, + await pixels_or_percent.process(radial[CONF_FOCAL_X]), + await pixels_or_percent.process(radial[CONF_FOCAL_Y]), + radial.get(CONF_FOCAL_RADIUS, 0), + ) + elif direction == "CONICAL": + conical = gradient[CONF_CONICAL] + lv.grad_conical_init( + var, + await pixels_or_percent.process(conical[CONF_CENTER_X]), + await pixels_or_percent.process(conical[CONF_CENTER_Y]), + await lv_angle_degrees.process(conical[CONF_START_ANGLE]), + await lv_angle_degrees.process(conical[CONF_END_ANGLE]), + await LV_GRAD_EXTEND.process(conical[CONF_EXTEND]), + ) stop_colors = cg.static_const_array( ID(idbase + "_colors_", type=lv_color_t), [await lv_color.process(x[CONF_COLOR]) for x in stops], diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index 57be4e9043..b457ec2c0b 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -209,6 +209,63 @@ lvgl: position: 212 - color: 0xFF0000 position: 255 + - id: linear_grad + direction: LINEAR + linear: + from_x: 0% + from_y: 0% + to_x: 100% + to_y: 0% + extend: REFLECT + stops: + - color: 0xFF0000 + position: 0 + - color: 0x0000FF + position: 255 + - id: radial_grad + direction: RADIAL + radial: + center_x: 50% + center_y: 50% + to_x: 100% + to_y: 50% + extend: PAD + stops: + - color: 0xFFFFFF + position: 0 + - color: 0x000000 + position: 255 + - id: radial_focal_grad + direction: RADIAL + radial: + center_x: 50% + center_y: 50% + to_x: 100% + to_y: 50% + focal_x: 40% + focal_y: 40% + focal_radius: 10 + extend: REPEAT + stops: + - color: 0xFF0000 + position: 0 + - color: 0x0000FF + position: 255 + - id: conical_grad + direction: CONICAL + conical: + center_x: 50% + center_y: 50% + start_angle: 0 + end_angle: 360 + extend: PAD + stops: + - color: 0xFF0000 + position: 0 + - color: 0x00FF00 + position: 127 + - color: 0xFF0000 + position: 255 style_definitions: - id: style_test @@ -1070,6 +1127,14 @@ lvgl: logger.log: format: Slider released at %d/%d with value %.0f args: ['(int) point.x', '(int) point.y', x] + + # Exercises the style-application path for a complex gradient, not just its + # lv_grad_*_init() codegen: the other new gradients are only ever declared. + - obj: + bg_opa: cover + bg_grad: conical_grad + width: 40 + height: 40 - button: styles: spin_button id: spin_up From 0607c228f5d545b2b5582a2db2f2fbddfd7c75c0 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:25:29 +0000 Subject: [PATCH 031/147] Bump aioesphomeapi from 46.2.1 to 46.3.0 (#18892) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 63abc9c645..a065492dfa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyserial==3.5 platformio==6.1.19 esptool==5.3.1 click==8.3.3 -aioesphomeapi==46.2.1 +aioesphomeapi==46.3.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi zeroconf==0.150.0 puremagic==2.2.0 From fb65096ea3dc4cccf53681b501dec01fd2ec9538 Mon Sep 17 00:00:00 2001 From: Brandon Harvey <8107750+bharvey88@users.noreply.github.com> Date: Sun, 30 Aug 2026 13:52:51 -0500 Subject: [PATCH 032/147] [api] Add description and example metadata to user-defined actions (#18881) Co-authored-by: J. Nick Koston --- esphome/codegen.py | 1 + esphome/components/api/__init__.py | 138 ++++++++++++++-- esphome/components/api/api.proto | 3 + esphome/components/api/api_pb2.cpp | 18 ++ esphome/components/api/api_pb2.h | 11 +- esphome/components/api/api_pb2_dump.cpp | 9 + esphome/components/api/list_entities.cpp | 3 +- esphome/components/api/user_services.cpp | 43 +++++ esphome/components/api/user_services.h | 93 ++++++----- esphome/components/const/__init__.py | 1 + esphome/core/defines.h | 2 + esphome/cpp_types.py | 1 + .../api/test_action_metadata.py | 155 ++++++++++++++++++ .../api/test_action_metadata.yaml | 14 ++ .../api/test_action_metadata_common.yaml | 18 ++ .../api/test_action_metadata_esp8266.yaml | 14 ++ .../api/test_action_metadata_shorthand.yaml | 19 +++ .../api/test_homeassistant_action.py | 4 +- tests/components/api/common-base.yaml | 6 +- tests/components/api/common.yaml | 3 +- .../fixtures/api_action_metadata.yaml | 26 +++ tests/integration/test_api_action_metadata.py | 65 ++++++++ 22 files changed, 591 insertions(+), 56 deletions(-) create mode 100644 tests/component_tests/api/test_action_metadata.py create mode 100644 tests/component_tests/api/test_action_metadata.yaml create mode 100644 tests/component_tests/api/test_action_metadata_common.yaml create mode 100644 tests/component_tests/api/test_action_metadata_esp8266.yaml create mode 100644 tests/component_tests/api/test_action_metadata_shorthand.yaml create mode 100644 tests/integration/fixtures/api_action_metadata.yaml create mode 100644 tests/integration/test_api_action_metadata.py diff --git a/esphome/codegen.py b/esphome/codegen.py index 2aa6a70abd..5debb52b4e 100644 --- a/esphome/codegen.py +++ b/esphome/codegen.py @@ -78,6 +78,7 @@ from esphome.cpp_types import ( # noqa: F401 StringRef, arduino_json_ns, bool_, + char, const_char_ptr, double, esphome_ns, diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 2e891a9663..3568318dad 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -5,6 +5,7 @@ from typing import Any from esphome import automation from esphome.automation import Condition import esphome.codegen as cg +from esphome.components.const import CONF_DESCRIPTION from esphome.components.logger import request_log_listener # ENCRYPTION_SCHEMA and validate_encryption_key are re-exported for external @@ -41,10 +42,12 @@ from esphome.const import ( CONF_TAG, CONF_THEN, CONF_TRIGGER_ID, + CONF_TYPE, CONF_VARIABLES, ) from esphome.core import CORE, ID, CoroPriority, EsphomeError, coroutine_with_priority from esphome.cpp_generator import MockObj, TemplateArgsType +from esphome.helpers import fnv1_hash from esphome.types import ConfigFragmentType, ConfigType # Compat alias: downstream consumers (e.g. device-builder) referenced the @@ -125,6 +128,7 @@ SERVICE_ARG_FALLBACK_TYPES: dict[str, MockObj] = { } CONF_BATCH_DELAY = "batch_delay" CONF_CUSTOM_SERVICES = "custom_services" +CONF_EXAMPLE = "example" CONF_HOMEASSISTANT_SERVICES = "homeassistant_services" CONF_HOMEASSISTANT_STATES = "homeassistant_states" CONF_LISTEN_BACKLOG = "listen_backlog" @@ -228,14 +232,30 @@ def _validate_supports_response(value: Any) -> str: return cv.enum(SUPPORTS_RESPONSE_OPTIONS, lower=True)(value) +# ESP8266 copies every string of an action into a stack buffer sized by codegen; keep it small +ESP8266_ACTION_STRINGS_MAX_TOTAL = 384 + +VARIABLE_SCHEMA = cv.Schema( + { + cv.Required(CONF_TYPE): cv.one_of(*SERVICE_ARG_NATIVE_TYPES, lower=True), + cv.Optional(CONF_DESCRIPTION): cv.string_strict, + cv.Optional(CONF_EXAMPLE): cv.string_strict, + } +) + +# Accepts the plain `name: type` shorthand or the full mapping form +validate_variable = cv.maybe_simple_value(VARIABLE_SCHEMA, key=CONF_TYPE) + + ACTIONS_SCHEMA = automation.validate_automation( { cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(UserServiceTrigger), cv.Exclusive(CONF_SERVICE, group_of_exclusion=CONF_ACTION): cv.valid_name, cv.Exclusive(CONF_ACTION, group_of_exclusion=CONF_ACTION): cv.valid_name, + cv.Optional(CONF_DESCRIPTION): cv.string_strict, cv.Optional(CONF_VARIABLES, default={}): cv.Schema( { - cv.validate_id_name: cv.one_of(*SERVICE_ARG_NATIVE_TYPES, lower=True), + cv.validate_id_name: validate_variable, } ), # No default - auto-detected by _auto_detect_supports_response @@ -352,6 +372,85 @@ CONFIG_SCHEMA = cv.All( ) +def _has_action_metadata(actions: list[ConfigType]) -> bool: + # Empty strings count as unset, matching _action_strings + return any( + conf.get(CONF_DESCRIPTION) + or any( + var_.get(CONF_DESCRIPTION) or var_.get(CONF_EXAMPLE) + for var_ in conf[CONF_VARIABLES].values() + ) + for conf in actions + ) + + +def _action_strings(conf: ConfigType, has_metadata: bool) -> list[str | None]: + """Strings of one action in the table order UserServiceStatic (user_services.h) expects.""" + # An empty description or example is treated as unset + strings: list[str | None] = [conf[CONF_ACTION]] + if has_metadata: + strings.append(conf.get(CONF_DESCRIPTION) or None) + for name, var_ in conf[CONF_VARIABLES].items(): + strings.append(name) + if has_metadata: + strings += [ + var_.get(CONF_DESCRIPTION) or None, + var_.get(CONF_EXAMPLE) or None, + ] + return strings + + +def _action_strings_size(strings: list[str | None]) -> int: + """Bytes needed to copy every string out of flash, each with its terminator.""" + return sum( + len(string.encode("utf-8")) + 1 for string in strings if string is not None + ) + + +def _validate_esp8266_action_strings(config: ConfigType) -> ConfigType: + if not CORE.is_esp8266: + return config + actions = config.get(CONF_ACTIONS, []) + has_metadata = _has_action_metadata(actions) + for conf in actions: + size = _action_strings_size(_action_strings(conf, has_metadata)) + if size > ESP8266_ACTION_STRINGS_MAX_TOTAL: + raise cv.Invalid( + f"Action '{conf[CONF_ACTION]}' has {size} bytes of name, variable name, " + f"description and example text; ESP8266 allows at most " + f"{ESP8266_ACTION_STRINGS_MAX_TOTAL} bytes per action" + ) + return config + + +FINAL_VALIDATE_SCHEMA = _validate_esp8266_action_strings + + +def _add_action_strings( + index: int, strings: list[str | None], interned: dict[str, MockObj] +) -> MockObj: + """Emit the PROGMEM string table for one action. + + Each string is its own PROGMEM array because on ESP8266 .rodata is RAM, and identical + strings are shared between actions through `interned`. + """ + entries: list[MockObj] = [] + for string in strings: + if string is None: + entries.append(cg.nullptr) + continue + if (var := interned.get(string)) is None: + var = interned[string] = cg.progmem_array( + ID(f"api_action_str{len(interned)}", is_declaration=True, type=cg.char), + string, + ) + entries.append(var) + return cg.progmem_array( + ID(f"api_action{index}_strings", is_declaration=True, type=cg.const_char_ptr), + entries, + ) + + @coroutine_with_priority(CoroPriority.WEB) async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) @@ -371,8 +470,10 @@ async def to_code(config: ConfigType) -> None: cg.add_define("MAX_API_CONNECTIONS", config[CONF_MAX_CONNECTIONS]) cg.add_define("API_MAX_SEND_QUEUE", config[CONF_MAX_SEND_QUEUE]) + actions = config.get(CONF_ACTIONS, []) + has_user_actions = bool(actions) or config[CONF_CUSTOM_SERVICES] # Set USE_API_USER_DEFINED_ACTIONS if any services are enabled - if config.get(CONF_ACTIONS) or config[CONF_CUSTOM_SERVICES]: + if has_user_actions: cg.add_define("USE_API_USER_DEFINED_ACTIONS") # Set USE_API_CUSTOM_SERVICES if external components need dynamic service registration @@ -385,10 +486,17 @@ async def to_code(config: ConfigType) -> None: if config[CONF_HOMEASSISTANT_STATES]: cg.add_define("USE_API_HOMEASSISTANT_STATES") - if actions := config.get(CONF_ACTIONS, []): + scratch_size = 0 + if actions: + # Metadata is compiled in for every action once any action declares it, because the + # string table layout is fixed by the define rather than per action + has_metadata = _has_action_metadata(actions) + if has_metadata: + cg.add_define("USE_API_USER_DEFINED_ACTION_METADATA") + interned_strings: dict[str, MockObj] = {} # Collect all triggers first, then register all at once with initializer_list triggers: list[cg.MockObj] = [] - for conf in actions: + for index, conf in enumerate(actions): func_args: list[tuple[MockObj, str]] = [] service_template_args: list[MockObj] = [] # User service argument types @@ -421,22 +529,23 @@ async def to_code(config: ConfigType) -> None: conf.get(CONF_THEN, []) ) - service_arg_names: list[str] = [] for name, var_ in conf[CONF_VARIABLES].items(): - if has_non_synchronous and var_ in SERVICE_ARG_FALLBACK_TYPES: - native = SERVICE_ARG_FALLBACK_TYPES[var_] + var_type = var_[CONF_TYPE] + if has_non_synchronous and var_type in SERVICE_ARG_FALLBACK_TYPES: + native = SERVICE_ARG_FALLBACK_TYPES[var_type] else: - native = SERVICE_ARG_NATIVE_TYPES[var_] + native = SERVICE_ARG_NATIVE_TYPES[var_type] service_template_args.append(native) func_args.append((native, name)) - service_arg_names.append(name) + strings = _action_strings(conf, has_metadata) + table = _add_action_strings(index, strings, interned_strings) + if CORE.is_esp8266: + scratch_size = max(scratch_size, _action_strings_size(strings)) # Template args: supports_response mode, then user service arg types templ = cg.TemplateArguments(supports_response, *service_template_args) + # Key is hashed here because the name is not readable at runtime on ESP8266 trigger = cg.new_Pvariable( - conf[CONF_TRIGGER_ID], - templ, - conf[CONF_ACTION], - service_arg_names, + conf[CONF_TRIGGER_ID], templ, table, fnv1_hash(conf[CONF_ACTION]) ) triggers.append(trigger) auto = await automation.build_automation(trigger, func_args, conf) @@ -458,6 +567,9 @@ async def to_code(config: ConfigType) -> None: cg.add(auto.add_actions([unregister_action])) # Register all services at once - single allocation, no reallocations cg.add(var.initialize_user_services(triggers)) + if CORE.is_esp8266 and has_user_actions: + # Stack buffer that list-entities copies PROGMEM strings into, sized for the largest action + cg.add_define("API_USER_ACTION_STRINGS_SCRATCH_SIZE", max(scratch_size, 1)) if CONF_ON_CLIENT_CONNECTED in config: cg.add_define("USE_API_CLIENT_CONNECTED_TRIGGER") diff --git a/esphome/components/api/api.proto b/esphome/components/api/api.proto index c11700782e..3a0e0abea9 100644 --- a/esphome/components/api/api.proto +++ b/esphome/components/api/api.proto @@ -1034,6 +1034,8 @@ message ListEntitiesServicesArgument { option (ifdef) = "USE_API_USER_DEFINED_ACTIONS"; string name = 1; ServiceArgType type = 2; + string description = 3 [(field_ifdef) = "USE_API_USER_DEFINED_ACTION_METADATA"]; + string example = 4 [(field_ifdef) = "USE_API_USER_DEFINED_ACTION_METADATA"]; } message ListEntitiesServicesResponse { option (id) = 41; @@ -1044,6 +1046,7 @@ message ListEntitiesServicesResponse { fixed32 key = 2 [(force) = true]; repeated ListEntitiesServicesArgument args = 3 [(fixed_vector) = true]; SupportsResponseType supports_response = 4; + string description = 5 [(field_ifdef) = "USE_API_USER_DEFINED_ACTION_METADATA"]; } message ExecuteServiceArgument { option (ifdef) = "USE_API_USER_DEFINED_ACTIONS"; diff --git a/esphome/components/api/api_pb2.cpp b/esphome/components/api/api_pb2.cpp index f56d791b67..2de1f0a15c 100644 --- a/esphome/components/api/api_pb2.cpp +++ b/esphome/components/api/api_pb2.cpp @@ -1275,12 +1275,24 @@ uint8_t *ListEntitiesServicesArgument::encode(ProtoWriteBuffer &buffer PROTO_ENC uint8_t *__restrict__ pos = buffer.get_pos(); ProtoEncode::encode_string(pos PROTO_ENCODE_DEBUG_ARG, 1, this->name); ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 2, static_cast(this->type)); +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + ProtoEncode::encode_string(pos PROTO_ENCODE_DEBUG_ARG, 3, this->description); +#endif +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + ProtoEncode::encode_string(pos PROTO_ENCODE_DEBUG_ARG, 4, this->example); +#endif return pos; } uint32_t ListEntitiesServicesArgument::calculate_size() const { uint32_t size = 0; size += ProtoSize::calc_length(1, this->name.size()); size += this->type ? 2 : 0; +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + size += ProtoSize::calc_length(1, this->description.size()); +#endif +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + size += ProtoSize::calc_length(1, this->example.size()); +#endif return size; } uint8_t *ListEntitiesServicesResponse::encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const { @@ -1291,6 +1303,9 @@ uint8_t *ListEntitiesServicesResponse::encode(ProtoWriteBuffer &buffer PROTO_ENC ProtoEncode::encode_sub_message(pos PROTO_ENCODE_DEBUG_ARG, buffer, 3, it); } ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 4, static_cast(this->supports_response)); +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + ProtoEncode::encode_string(pos PROTO_ENCODE_DEBUG_ARG, 5, this->description); +#endif return pos; } uint32_t ListEntitiesServicesResponse::calculate_size() const { @@ -1303,6 +1318,9 @@ uint32_t ListEntitiesServicesResponse::calculate_size() const { } } size += this->supports_response ? 2 : 0; +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + size += ProtoSize::calc_length(1, this->description.size()); +#endif return size; } bool ExecuteServiceArgument::decode_varint(uint32_t field_id, proto_varint_value_t value) { diff --git a/esphome/components/api/api_pb2.h b/esphome/components/api/api_pb2.h index bed28d2956..5c3429a63a 100644 --- a/esphome/components/api/api_pb2.h +++ b/esphome/components/api/api_pb2.h @@ -1317,6 +1317,12 @@ class ListEntitiesServicesArgument final : public ProtoMessage { public: StringRef name{}; enums::ServiceArgType type{}; +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + StringRef description{}; +#endif +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + StringRef example{}; +#endif uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const; uint32_t calculate_size() const; #ifdef HAS_PROTO_MESSAGE_DUMP @@ -1328,7 +1334,7 @@ class ListEntitiesServicesArgument final : public ProtoMessage { class ListEntitiesServicesResponse final : public ProtoMessage { public: static constexpr uint16_t MESSAGE_TYPE = 41; - static constexpr uint8_t ESTIMATED_SIZE = 50; + static constexpr uint8_t ESTIMATED_SIZE = 59; #ifdef HAS_PROTO_MESSAGE_DUMP const LogString *message_name() const override { return LOG_STR("list_entities_services_response"); } #endif @@ -1336,6 +1342,9 @@ class ListEntitiesServicesResponse final : public ProtoMessage { uint32_t key{0}; FixedVector args{}; enums::SupportsResponseType supports_response{}; +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + StringRef description{}; +#endif uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const; uint32_t calculate_size() const; #ifdef HAS_PROTO_MESSAGE_DUMP diff --git a/esphome/components/api/api_pb2_dump.cpp b/esphome/components/api/api_pb2_dump.cpp index 846c0ad652..dced81ee30 100644 --- a/esphome/components/api/api_pb2_dump.cpp +++ b/esphome/components/api/api_pb2_dump.cpp @@ -1500,6 +1500,12 @@ const char *ListEntitiesServicesArgument::dump_to(DumpBuffer &out) const { MessageDumpHelper helper(out, ESPHOME_PSTR("ListEntitiesServicesArgument")); dump_field(out, ESPHOME_PSTR("name"), this->name); dump_field(out, ESPHOME_PSTR("type"), static_cast(this->type)); +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + dump_field(out, ESPHOME_PSTR("description"), this->description); +#endif +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + dump_field(out, ESPHOME_PSTR("example"), this->example); +#endif return out.c_str(); } const char *ListEntitiesServicesResponse::dump_to(DumpBuffer &out) const { @@ -1512,6 +1518,9 @@ const char *ListEntitiesServicesResponse::dump_to(DumpBuffer &out) const { out.append("\n"); } dump_field(out, ESPHOME_PSTR("supports_response"), static_cast(this->supports_response)); +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + dump_field(out, ESPHOME_PSTR("description"), this->description); +#endif return out.c_str(); } const char *ExecuteServiceArgument::dump_to(DumpBuffer &out) const { diff --git a/esphome/components/api/list_entities.cpp b/esphome/components/api/list_entities.cpp index 57ff616ca7..507b098fb4 100644 --- a/esphome/components/api/list_entities.cpp +++ b/esphome/components/api/list_entities.cpp @@ -99,7 +99,8 @@ ListEntitiesIterator::ListEntitiesIterator(APIConnection *client) : client_(clie static constexpr uint8_t SERVICE_YIELD_INTERVAL = 3; bool ListEntitiesIterator::on_service(UserServiceDescriptor *service) { - auto resp = service->encode_list_service_response(); + UserActionScratch scratch; + auto resp = service->encode_list_service_response(scratch); if (!this->client_->send_message(resp)) return false; // at_ is this service's index diff --git a/esphome/components/api/user_services.cpp b/esphome/components/api/user_services.cpp index 28a43c656c..fad3cde29b 100644 --- a/esphome/components/api/user_services.cpp +++ b/esphome/components/api/user_services.cpp @@ -1,9 +1,52 @@ #include "user_services.h" +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include "esphome/core/string_ref.h" namespace esphome::api { +StringRef UserServiceStatic::str_(size_t idx, std::span &scratch) const { + const char *s = progmem_read_ptr(&this->strings_[idx]); + if (s == nullptr) + return {}; +#ifdef USE_ESP8266 + // Codegen sizes the scratch buffer for the largest service; the bound only guards other callers + if (scratch.empty()) + return {}; + size_t len = strnlen_P(s, scratch.size() - 1); + progmem_memcpy(scratch.data(), s, len); + scratch[len] = '\0'; + StringRef ref(scratch.data(), len); + scratch = scratch.subspan(len + 1); + return ref; +#else + return StringRef(s); +#endif +} + +ListEntitiesServicesResponse UserServiceStatic::encode_list_service_response_( + std::span arg_types, std::span scratch) const { + ListEntitiesServicesResponse msg; + msg.name = this->str_(0, scratch); + msg.key = this->key_; + msg.supports_response = this->supports_response_; +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + msg.description = this->str_(1, scratch); +#endif + msg.args.init(arg_types.size()); + for (size_t i = 0; i < arg_types.size(); i++) { + size_t base = USER_ACTION_HEADER_STRINGS + i * USER_ACTION_ARG_STRINGS; + auto &arg = msg.args.emplace_back(); + arg.type = arg_types[i]; + arg.name = this->str_(base, scratch); +#ifdef USE_API_USER_DEFINED_ACTION_METADATA + arg.description = this->str_(base + 1, scratch); + arg.example = this->str_(base + 2, scratch); +#endif + } + return msg; +} + template<> bool get_execute_arg_value(const ExecuteServiceArgument &arg) { return arg.bool_; } template<> int32_t get_execute_arg_value(const ExecuteServiceArgument &arg) { if (arg.legacy_int != 0) diff --git a/esphome/components/api/user_services.h b/esphome/components/api/user_services.h index ea57d0944b..3b17bdb7bc 100644 --- a/esphome/components/api/user_services.h +++ b/esphome/components/api/user_services.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -19,7 +20,9 @@ class APIServer; class UserServiceDescriptor { public: - virtual ListEntitiesServicesResponse encode_list_service_response() = 0; + /// Build the list-entities message. On ESP8266 the strings live in PROGMEM and are copied into + /// `scratch`, so the returned message is only valid while `scratch` is; other platforms ignore it. + virtual ListEntitiesServicesResponse encode_list_service_response(std::span scratch) = 0; virtual bool execute_service(const ExecuteServiceRequest &req) = 0; #ifdef USE_API_USER_DEFINED_ACTION_RESPONSES @@ -34,29 +37,51 @@ template T get_execute_arg_value(const ExecuteServiceArgument &arg); template enums::ServiceArgType to_service_arg_type(); -// Base class for YAML-defined services (most common case) -// Stores only pointers to string literals in flash - no heap allocation -template class UserServiceBase : public UserServiceDescriptor { - public: - UserServiceBase(const char *name, const std::array &arg_names, - enums::SupportsResponseType supports_response = enums::SUPPORTS_RESPONSE_NONE) - : name_(name), arg_names_(arg_names), supports_response_(supports_response) { - this->key_ = fnv1_hash(name); - } +// Scratch buffer list-entities hands to encode_list_service_response(); only ESP8266 copies into it +#ifdef USE_ESP8266 +using UserActionScratch = std::array; +#else +using UserActionScratch = std::array; +#endif - ListEntitiesServicesResponse encode_list_service_response() override { - ListEntitiesServicesResponse msg; - msg.name = StringRef(this->name_); - msg.key = this->key_; - msg.supports_response = this->supports_response_; +// Non-template base for YAML-defined services so the list-entities encoder is compiled once. +// All strings live in one PROGMEM pointer table emitted by codegen (see _action_strings in +// __init__.py), so each service costs a single pointer of RAM. Layout: the action name, then +// each argument name; with USE_API_USER_DEFINED_ACTION_METADATA the action description follows +// the name and every argument is (name, description, example). Unset metadata is nullptr. +#ifdef USE_API_USER_DEFINED_ACTION_METADATA +static constexpr size_t USER_ACTION_HEADER_STRINGS = 2; +static constexpr size_t USER_ACTION_ARG_STRINGS = 3; +#else +static constexpr size_t USER_ACTION_HEADER_STRINGS = 1; +static constexpr size_t USER_ACTION_ARG_STRINGS = 1; +#endif +class UserServiceStatic : public UserServiceDescriptor { + public: + UserServiceStatic(const char *const *strings, uint32_t key, + enums::SupportsResponseType supports_response = enums::SUPPORTS_RESPONSE_NONE) + : strings_(strings), key_(key), supports_response_(supports_response) {} + + protected: + ListEntitiesServicesResponse encode_list_service_response_(std::span arg_types, + std::span scratch) const; + /// Reference table entry `idx`; nullptr gives an empty StringRef. + /// On ESP8266 the bytes are copied out of PROGMEM into `scratch` with a terminator, and the span + /// is advanced past the copy. + StringRef str_(size_t idx, std::span &scratch) const; + + const char *const *strings_; // PROGMEM pointer table, read with progmem_read_ptr() + uint32_t key_; + enums::SupportsResponseType supports_response_; +}; + +template class UserServiceBase : public UserServiceStatic { + public: + using UserServiceStatic::UserServiceStatic; + + ListEntitiesServicesResponse encode_list_service_response(std::span scratch) override { std::array arg_types = {to_service_arg_type()...}; - msg.args.init(sizeof...(Ts)); - for (size_t i = 0; i < sizeof...(Ts); i++) { - auto &arg = msg.args.emplace_back(); - arg.type = arg_types[i]; - arg.name = StringRef(this->arg_names_[i]); - } - return msg; + return this->encode_list_service_response_(arg_types, scratch); } bool execute_service(const ExecuteServiceRequest &req) override { @@ -89,12 +114,6 @@ template class UserServiceBase : public UserServiceDescriptor { void execute_(const ArgsContainer &args, uint32_t call_id, bool return_response, std::index_sequence /*type*/) { this->execute(call_id, return_response, (get_execute_arg_value(args[S]))...); } - - // Pointers to string literals in flash - no heap allocation - const char *name_; - std::array arg_names_; - uint32_t key_{0}; - enums::SupportsResponseType supports_response_{enums::SUPPORTS_RESPONSE_NONE}; }; // Separate class for custom_api_device services (rare case) @@ -106,7 +125,7 @@ template class UserServiceDynamic : public UserServiceDescriptor this->key_ = fnv1_hash(this->name_.c_str()); } - ListEntitiesServicesResponse encode_list_service_response() override { + ListEntitiesServicesResponse encode_list_service_response(std::span /*scratch*/) override { ListEntitiesServicesResponse msg; msg.name = StringRef(this->name_); msg.key = this->key_; @@ -167,8 +186,8 @@ template class UserServiceTrigger final : public UserServiceBase, public Trigger { public: - UserServiceTrigger(const char *name, const std::array &arg_names) - : UserServiceBase(name, arg_names, enums::SUPPORTS_RESPONSE_NONE) {} + UserServiceTrigger(const char *const *strings, uint32_t key) + : UserServiceBase(strings, key, enums::SUPPORTS_RESPONSE_NONE) {} protected: void execute(uint32_t /*call_id*/, bool /*return_response*/, Ts... x) override { this->trigger(x...); } @@ -179,8 +198,8 @@ template class UserServiceTrigger final : public UserServiceBase, public Trigger { public: - UserServiceTrigger(const char *name, const std::array &arg_names) - : UserServiceBase(name, arg_names, enums::SUPPORTS_RESPONSE_OPTIONAL) {} + UserServiceTrigger(const char *const *strings, uint32_t key) + : UserServiceBase(strings, key, enums::SUPPORTS_RESPONSE_OPTIONAL) {} protected: void execute(uint32_t call_id, bool return_response, Ts... x) override { @@ -193,8 +212,8 @@ template class UserServiceTrigger final : public UserServiceBase, public Trigger { public: - UserServiceTrigger(const char *name, const std::array &arg_names) - : UserServiceBase(name, arg_names, enums::SUPPORTS_RESPONSE_ONLY) {} + UserServiceTrigger(const char *const *strings, uint32_t key) + : UserServiceBase(strings, key, enums::SUPPORTS_RESPONSE_ONLY) {} protected: void execute(uint32_t call_id, bool /*return_response*/, Ts... x) override { this->trigger(call_id, x...); } @@ -205,8 +224,8 @@ template class UserServiceTrigger final : public UserServiceBase, public Trigger { public: - UserServiceTrigger(const char *name, const std::array &arg_names) - : UserServiceBase(name, arg_names, enums::SUPPORTS_RESPONSE_STATUS) {} + UserServiceTrigger(const char *const *strings, uint32_t key) + : UserServiceBase(strings, key, enums::SUPPORTS_RESPONSE_STATUS) {} protected: void execute(uint32_t call_id, bool /*return_response*/, Ts... x) override { this->trigger(call_id, x...); } diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index 10710c8d29..e445a4abde 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -16,6 +16,7 @@ CONF_CO2_EQUIVALENT = "co2_equivalent" CONF_COLOR_DEPTH = "color_depth" CONF_CRC_ENABLE = "crc_enable" CONF_DATA_BITS = "data_bits" +CONF_DESCRIPTION = "description" CONF_DRAW_ROUNDING = "draw_rounding" CONF_ENABLE_OTA_DOWNGRADE_PROTECTION = "enable_ota_downgrade_protection" CONF_ENABLED = "enabled" diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 625d4879f5..1f5a10d47d 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -218,9 +218,11 @@ #define USE_API_PLAINTEXT #define USE_API_USER_DEFINED_ACTIONS #define USE_API_CUSTOM_SERVICES +#define USE_API_USER_DEFINED_ACTION_METADATA #define USE_API_USER_DEFINED_ACTION_RESPONSES #define USE_API_USER_DEFINED_ACTION_RESPONSES_JSON #define API_MAX_SEND_QUEUE 8 +#define API_USER_ACTION_STRINGS_SCRATCH_SIZE 64 #define MAX_API_CONNECTIONS 6 // The Improv library is not in the Zephyr tidy environment #define USE_IMPROV_SERIAL diff --git a/esphome/cpp_types.py b/esphome/cpp_types.py index aeaa4480a8..45d6559b3f 100644 --- a/esphome/cpp_types.py +++ b/esphome/cpp_types.py @@ -14,6 +14,7 @@ std_string_ref = std_ns.namespace("string &") std_vector = std_ns.class_("vector") std_span = std_ns.class_("span") int8 = global_ns.namespace("int8_t") +char = global_ns.namespace("char") uint8 = global_ns.namespace("uint8_t") uint16 = global_ns.namespace("uint16_t") uint32 = global_ns.namespace("uint32_t") diff --git a/tests/component_tests/api/test_action_metadata.py b/tests/component_tests/api/test_action_metadata.py new file mode 100644 index 0000000000..adbfdf306e --- /dev/null +++ b/tests/component_tests/api/test_action_metadata.py @@ -0,0 +1,155 @@ +"""Tests for user-defined action field metadata (description / example).""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.components.api import ( + _action_strings, + _action_strings_size, + _has_action_metadata, + _validate_esp8266_action_strings, + validate_variable, +) +from esphome.config_validation import Invalid +from esphome.const import PlatformFramework +from esphome.core import CORE +from esphome.cpp_generator import safe_exp +from esphome.helpers import fnv1_hash +from tests.component_tests.helpers import get_define_value +from tests.component_tests.types import SetCoreConfigCallable + +CONFIG = "tests/component_tests/api/test_action_metadata.yaml" +CONFIG_ESP8266 = "tests/component_tests/api/test_action_metadata_esp8266.yaml" +CONFIG_SHORTHAND = "tests/component_tests/api/test_action_metadata_shorthand.yaml" + + +def test_metadata_is_emitted_as_progmem_table( + generate_main: Callable[[str | Path], str], +) -> None: + """Every action string is a PROGMEM array referenced from one PROGMEM table.""" + main_cpp = generate_main(CONFIG) + + assert ( + 'static constexpr char api_action_str0[] PROGMEM = "play_buzzer";' in main_cpp + ) + assert ( + 'static constexpr char api_action_str1[] PROGMEM = "Play an RTTTL melody on the buzzer";' + in main_cpp + ) + assert ( + 'static constexpr char api_action_str4[] PROGMEM = "two_short:d=4,o=5,b=100:16e6,16e6";' + in main_cpp + ) + assert ( + "static constexpr const char * api_action0_strings[] PROGMEM = {" + "api_action_str0, api_action_str1, api_action_str2, api_action_str3, " + "api_action_str4, api_action_str5, nullptr, nullptr};" in main_cpp + ) + # An action without metadata still carries the metadata slots (as nullptr) + assert ( + "static constexpr const char * api_action1_strings[] PROGMEM = {" + "api_action_str6, nullptr, api_action_str7, nullptr, nullptr};" in main_cpp + ) + assert f"(api_action0_strings, {safe_exp(fnv1_hash('play_buzzer'))});" in main_cpp + assert "USE_API_USER_DEFINED_ACTION_METADATA" in {d.name for d in CORE.defines} + assert get_define_value("API_USER_ACTION_STRINGS_SCRATCH_SIZE") is None + + +def test_esp8266_sizes_scratch_buffer_for_largest_action( + generate_main: Callable[[str | Path], str], +) -> None: + """ESP8266 gets a scratch buffer define equal to the byte total of the largest action.""" + generate_main(CONFIG_ESP8266) + + # play_buzzer: name, description, two variable names, one description, one example, + # each with a terminator + assert get_define_value("API_USER_ACTION_STRINGS_SCRATCH_SIZE") == "117" + + +def test_shorthand_variables_emit_no_metadata( + generate_main: Callable[[str | Path], str], +) -> None: + """The name: type shorthand emits a name-only table and no define.""" + main_cpp = generate_main(CONFIG_SHORTHAND) + + assert ( + "static constexpr const char * api_action0_strings[] PROGMEM = " + "{api_action_str0, api_action_str1};" in main_cpp + ) + assert "USE_API_USER_DEFINED_ACTION_METADATA" not in {d.name for d in CORE.defines} + + +def test_variable_shorthand_normalizes_to_mapping() -> None: + """A bare type string validates to the mapping form.""" + assert validate_variable("string") == {"type": "string"} + + +@pytest.mark.parametrize( + "value", + [ + {"description": "no type given"}, + {"type": "string", "selector": "text"}, + "stringy", + {"type": "stringy"}, + ], +) +def test_variable_rejects_invalid(value: object) -> None: + """Missing or unknown type and unknown keys raise in both forms.""" + with pytest.raises(Invalid): + validate_variable(value) + + +def _oversized_action_config() -> dict: + return { + "actions": [ + { + "action": "big", + "description": "x" * 300, + "variables": {"a": {"type": "string", "example": "y" * 300}}, + } + ] + } + + +def test_esp8266_rejects_actions_over_string_budget( + set_core_config: SetCoreConfigCallable, +) -> None: + set_core_config(PlatformFramework.ESP8266_ARDUINO) + with pytest.raises(Invalid, match="ESP8266 allows at most 384 bytes"): + _validate_esp8266_action_strings(_oversized_action_config()) + + +def test_other_platforms_have_no_string_budget( + set_core_config: SetCoreConfigCallable, +) -> None: + set_core_config(PlatformFramework.ESP32_IDF) + config = _oversized_action_config() + assert _validate_esp8266_action_strings(config) is config + + +def test_empty_metadata_is_unset_and_not_counted() -> None: + """An empty description or example emits nullptr and takes no scratch space.""" + conf = { + "action": "a", + "description": "", + "variables": {"b": {"type": "int", "description": "", "example": "ex"}}, + } + strings = _action_strings(conf, has_metadata=True) + assert strings == ["a", None, "b", None, "ex"] + # Every emitted string counts its terminator: "a" + "b" + "ex" + assert _action_strings_size(strings) == 2 + 2 + 3 + + +def test_empty_metadata_does_not_enable_the_define() -> None: + actions = [ + { + "action": "a", + "description": "", + "variables": {"b": {"type": "int", "example": ""}}, + } + ] + assert not _has_action_metadata(actions) + actions[0]["variables"]["b"]["example"] = "1" + assert _has_action_metadata(actions) diff --git a/tests/component_tests/api/test_action_metadata.yaml b/tests/component_tests/api/test_action_metadata.yaml new file mode 100644 index 0000000000..c998713874 --- /dev/null +++ b/tests/component_tests/api/test_action_metadata.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp32: + board: esp32dev + +wifi: + ssid: MySSID + password: password1 + +logger: + +packages: + api: !include test_action_metadata_common.yaml diff --git a/tests/component_tests/api/test_action_metadata_common.yaml b/tests/component_tests/api/test_action_metadata_common.yaml new file mode 100644 index 0000000000..bd161efe63 --- /dev/null +++ b/tests/component_tests/api/test_action_metadata_common.yaml @@ -0,0 +1,18 @@ +api: + actions: + - action: play_buzzer + description: Play an RTTTL melody on the buzzer + variables: + song_str: + type: string + description: RTTTL melody string + example: "two_short:d=4,o=5,b=100:16e6,16e6" + volume: + type: int + then: + - logger.log: Action Called + - action: plain_action + variables: + value: int + then: + - logger.log: Action Called diff --git a/tests/component_tests/api/test_action_metadata_esp8266.yaml b/tests/component_tests/api/test_action_metadata_esp8266.yaml new file mode 100644 index 0000000000..94a5839b28 --- /dev/null +++ b/tests/component_tests/api/test_action_metadata_esp8266.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp8266: + board: d1_mini + +wifi: + ssid: MySSID + password: password1 + +logger: + +packages: + api: !include test_action_metadata_common.yaml diff --git a/tests/component_tests/api/test_action_metadata_shorthand.yaml b/tests/component_tests/api/test_action_metadata_shorthand.yaml new file mode 100644 index 0000000000..aa2e1ab424 --- /dev/null +++ b/tests/component_tests/api/test_action_metadata_shorthand.yaml @@ -0,0 +1,19 @@ +esphome: + name: test + +esp32: + board: esp32dev + +wifi: + ssid: MySSID + password: password1 + +logger: + +api: + actions: + - action: plain_action + variables: + value: int + then: + - logger.log: Action Called diff --git a/tests/component_tests/api/test_homeassistant_action.py b/tests/component_tests/api/test_homeassistant_action.py index 611353e7c5..6ee5ac3412 100644 --- a/tests/component_tests/api/test_homeassistant_action.py +++ b/tests/component_tests/api/test_homeassistant_action.py @@ -9,7 +9,7 @@ def test_synchronous_chain_keeps_zero_copy_args(generate_main): assert ( "api::UserServiceTrigger" - '("zero_copy_args", {"message"})' in main_cpp + "(api_action0_strings," in main_cpp ) @@ -22,7 +22,7 @@ def test_response_callback_args_are_owning(generate_main): assert ( "api::UserServiceTrigger" - '("response_args", {"message"})' in main_cpp + "(api_action1_strings," in main_cpp ) assert "api::HomeAssistantServiceCallAction" in main_cpp assert "api::HomeAssistantServiceCallAction" not in main_cpp diff --git a/tests/components/api/common-base.yaml b/tests/components/api/common-base.yaml index c9eb200471..5e3139da48 100644 --- a/tests/components/api/common-base.yaml +++ b/tests/components/api/common-base.yaml @@ -61,8 +61,12 @@ api: reboot_timeout: 0min actions: - action: hello_world + description: Log a greeting variables: - name: string + name: + type: string + description: Name to greet + example: World then: - logger.log: format: Hello World %s! diff --git a/tests/components/api/common.yaml b/tests/components/api/common.yaml index 6115838b6d..42eb32a92a 100644 --- a/tests/components/api/common.yaml +++ b/tests/components/api/common.yaml @@ -1,4 +1,5 @@ -<<: !include common-base.yaml +packages: + base: !include common-base.yaml api: encryption: diff --git a/tests/integration/fixtures/api_action_metadata.yaml b/tests/integration/fixtures/api_action_metadata.yaml new file mode 100644 index 0000000000..802b965110 --- /dev/null +++ b/tests/integration/fixtures/api_action_metadata.yaml @@ -0,0 +1,26 @@ +esphome: + name: api-action-metadata-test +host: +api: + batch_delay: 0ms + actions: + - action: play_buzzer + description: Play an RTTTL melody on the buzzer + variables: + song_str: + type: string + description: RTTTL melody string + example: "two_short:d=4,o=5,b=100:16e6,16e6" + volume: + type: int + then: + - logger.log: + format: "Buzzer: %s" + args: [song_str.c_str()] + - action: plain_action + variables: + value: int + then: + - logger.log: "Plain action called" + +logger: diff --git a/tests/integration/test_api_action_metadata.py b/tests/integration/test_api_action_metadata.py new file mode 100644 index 0000000000..74d40f141b --- /dev/null +++ b/tests/integration/test_api_action_metadata.py @@ -0,0 +1,65 @@ +"""Integration test for user-defined action field metadata.""" + +from __future__ import annotations + +import asyncio +import re + +import pytest + +from esphome.helpers import fnv1_hash + +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_api_action_metadata( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Action and argument metadata reach the client and the actions still run.""" + loop = asyncio.get_running_loop() + buzzer_called = loop.create_future() + plain_called = loop.create_future() + buzzer_pattern = re.compile(r"Buzzer: two_short") + plain_pattern = re.compile(r"Plain action called") + + def check_output(line: str) -> None: + if not buzzer_called.done() and buzzer_pattern.search(line): + buzzer_called.set_result(True) + elif not plain_called.done() and plain_pattern.search(line): + plain_called.set_result(True) + + async with ( + run_compiled(yaml_config, line_callback=check_output), + api_client_connected() as client, + ): + _, services = await client.list_entities_services() + + by_name = {service.name: service for service in services} + assert set(by_name) == {"play_buzzer", "plain_action"} + # Keys are hashed at codegen time and must match what the client expects + for name, service in by_name.items(): + assert service.key == fnv1_hash(name), name + + buzzer = by_name["play_buzzer"] + assert buzzer.description == "Play an RTTTL melody on the buzzer" + args = {arg.name: arg for arg in buzzer.args} + assert args["song_str"].description == "RTTTL melody string" + assert args["song_str"].example == "two_short:d=4,o=5,b=100:16e6,16e6" + # An arg without metadata sends empty strings + assert args["volume"].description == "" + assert args["volume"].example == "" + + # An action without metadata sends empty strings + plain = by_name["plain_action"] + assert plain.description == "" + assert plain.args[0].description == "" + + await client.execute_service( + buzzer, {"song_str": "two_short:d=4,o=5,b=100:16e6,16e6", "volume": 3} + ) + await client.execute_service(plain, {"value": 1}) + await asyncio.wait_for(buzzer_called, timeout=5.0) + await asyncio.wait_for(plain_called, timeout=5.0) From cb54e57d847a71c1d8cac2f61e0404cdbc39a59a Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Sun, 30 Aug 2026 14:43:28 -0700 Subject: [PATCH 033/147] [modbus] Yield the whole-millisecond part of the interframe wait (#18898) --- esphome/components/modbus/modbus.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/esphome/components/modbus/modbus.cpp b/esphome/components/modbus/modbus.cpp index f77492f48b..25687ba106 100644 --- a/esphome/components/modbus/modbus.cpp +++ b/esphome/components/modbus/modbus.cpp @@ -777,9 +777,10 @@ void ModbusServerHub::process_modbus_client_frame_(uint8_t address, uint8_t func bool Modbus::send_frame_(const ModbusFrame &frame) { int32_t tx_delay_remaining = this->tx_delay_remaining(); if (tx_delay_remaining > 0) { - // delay() only lands on tick boundaries, so yield with it to get close, then busy-wait the rest. - if (tx_delay_remaining > (int32_t) (2 * US_PER_MS)) { - delay((tx_delay_remaining - US_PER_MS) / US_PER_MS); + // Yield the whole-ms part: delay() never blocks past the request on FreeRTOS, and only slightly + // over elsewhere, which just lengthens the gap. The recompute below makes the remainder exact. + if (tx_delay_remaining >= (int32_t) US_PER_MS) { + delay(tx_delay_remaining / US_PER_MS); tx_delay_remaining = this->tx_delay_remaining(); } if (tx_delay_remaining > 0) @@ -814,6 +815,9 @@ bool Modbus::send_frame_(const ModbusFrame &frame) { } void ModbusClientHub::send_next_frame_() { + if (this->tx_buffer_.empty()) + return; + if (this->tx_blocked()) return; From 2f1d3f8299fe7b8239e8a0af4aa0fad018ded3e6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 30 Aug 2026 17:34:16 -0500 Subject: [PATCH 034/147] [core] Deduplicate the host program path lookup (#18897) --- esphome/__main__.py | 33 +++++++++++----------- tests/unit_tests/test_main.py | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 632d2ba3d0..1ebf194205 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1670,20 +1670,26 @@ def command_compile(args: ArgsProtocol, config: ConfigType) -> int | None: if exit_code != 0: return exit_code if CORE.is_host: - if CORE.using_toolchain_esp_idf: - from esphome.espidf import toolchain - - program_path = str(toolchain.get_elf_path()) - else: - from esphome.platformio.toolchain import get_idedata - - program_path = str(get_idedata(config).firmware_elf_path) - _LOGGER.info("Successfully compiled program to path '%s'", program_path) + _LOGGER.info( + "Successfully compiled program to path '%s'", _host_program_path(config) + ) else: _LOGGER.info("Successfully compiled program.") return 0 +def _host_program_path(config: ConfigType) -> str: + """Return the compiled host ELF path.""" + if CORE.using_toolchain_esp_idf: + from esphome.espidf import toolchain + + return str(toolchain.get_elf_path()) + from esphome.platformio.toolchain import get_idedata + + # Memoized by compile_program's own call; this is a dict lookup + return str(get_idedata(config).firmware_elf_path) + + def command_upload(args: ArgsProtocol, config: ConfigType) -> int | None: # Get devices, resolving special identifiers like OTA devices = choose_upload_log_host( @@ -1728,14 +1734,7 @@ def command_run(args: ArgsProtocol, config: ConfigType) -> int | None: return exit_code _LOGGER.info("Successfully compiled program.") if CORE.is_host: - if CORE.using_toolchain_esp_idf: - from esphome.espidf import toolchain - - program_path = str(toolchain.get_elf_path()) - else: - from esphome.platformio.toolchain import get_idedata - - program_path = str(get_idedata(config).firmware_elf_path) + program_path = _host_program_path(config) _LOGGER.info("Running program from path '%s'", program_path) return run_external_process(program_path) diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 08c99e2119..15b1105ed0 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -112,6 +112,7 @@ from esphome.const import ( PLATFORM_BK72XX, PLATFORM_ESP32, PLATFORM_ESP8266, + PLATFORM_HOST, PLATFORM_NRF52, PLATFORM_RP2, Toolchain, @@ -7254,3 +7255,54 @@ async def test_wrap_to_code_comment_is_insertion_order_independent() -> None: assert first == second assert second.index("alpha") < second.index("beta") assert second.index("a: 2") < second.index("z: 1") + + +def test_host_program_path_platformio_toolchain() -> None: + """Host + PlatformIO toolchain reads the memoized idedata path.""" + setup_core(platform=PLATFORM_HOST) + idedata = SimpleNamespace(firmware_elf_path="/build/x/.pioenvs/x/program") + with patch( + "esphome.platformio.toolchain.get_idedata", return_value=idedata + ) as mock_get: + assert main._host_program_path({}) == "/build/x/.pioenvs/x/program" + mock_get.assert_called_once_with({}) + + +def test_host_program_path_esp_idf_toolchain() -> None: + """Host + native ESP-IDF toolchain asks the espidf toolchain for the ELF.""" + setup_core(platform=PLATFORM_HOST) + CORE.toolchain = Toolchain.ESP_IDF + with patch( + "esphome.espidf.toolchain.get_elf_path", return_value=Path("/b/app.elf") + ): + assert main._host_program_path({}) == str(Path("/b/app.elf")) + + +def test_command_compile_host_logs_program_path( + caplog: pytest.LogCaptureFixture, +) -> None: + """command_compile on host logs the compiled program path.""" + setup_core(platform=PLATFORM_HOST) + with ( + patch.object(main, "write_cpp", return_value=0), + patch.object(main, "compile_program", return_value=0), + patch.object(main, "_host_program_path", return_value="/b/program"), + caplog.at_level(logging.INFO), + ): + assert main.command_compile(SimpleNamespace(only_generate=False), {}) == 0 + assert "Successfully compiled program to path '/b/program'" in caplog.text + + +def test_command_run_host_executes_program(caplog: pytest.LogCaptureFixture) -> None: + """command_run on host logs and executes the compiled program directly.""" + setup_core(platform=PLATFORM_HOST) + with ( + patch.object(main, "write_cpp", return_value=0), + patch.object(main, "compile_program", return_value=0), + patch.object(main, "_host_program_path", return_value="/b/program"), + patch.object(main, "run_external_process", return_value=0) as mock_run, + caplog.at_level(logging.INFO), + ): + assert main.command_run(SimpleNamespace(), {}) == 0 + mock_run.assert_called_with("/b/program") + assert "Running program from path '/b/program'" in caplog.text From fc977b7b5e53770bd8244163b3221861228b04d8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 30 Aug 2026 17:34:32 -0500 Subject: [PATCH 035/147] [ci] Cache the integration test PlatformIO dir (#18896) --- .github/workflows/ci.yml | 24 ++++++++++++++++++++++++ script/determine-jobs.py | 21 +++++++++++++++++++-- tests/integration/conftest.py | 3 ++- tests/script/test_determine_jobs.py | 7 +++++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a58d0fe9b..0df4da6386 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -355,6 +355,15 @@ jobs: fail-fast: false matrix: bucket: ${{ fromJson(needs.determine-jobs.outputs.integration-test-buckets) }} + env: + # What the cache steps persist; libdeps is excluded (keyed per xdist + # worker and env, it never crosses runs). + INTEGRATION_PIO_CACHE_PATH: | + ~/.esphome-integration-tests/platformio/platforms + ~/.esphome-integration-tests/platformio/packages + ~/.esphome-integration-tests/platformio/appstate.json + ~/.esphome-integration-tests/platformio/.cache + ~/.esphome-integration-tests/platformio/.esphome.pio.stamp.json steps: - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -373,6 +382,14 @@ jobs: uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.13" + - name: Restore integration PlatformIO cache + # Native platform + toolchain installed by shared_platformio_cache in + # tests/integration/conftest.py; a miss self-heals, so no restore-keys. + id: pio-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.INTEGRATION_PIO_CACHE_PATH }} + key: integration-pio-v1-${{ runner.os }}-py${{ steps.python.outputs.python-version }}-${{ hashFiles('requirements.txt', 'tests/integration/fixtures/cache_init.yaml', 'esphome/components/host/__init__.py') }} - name: Restore Python virtual environment id: cache-venv uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -416,6 +433,13 @@ jobs: # esphome stores the PlatformIO ccache under the machine-global cache # dir (see _ccache_env() in esphome/platformio/toolchain.py). run: CCACHE_DIR="$HOME/.cache/esphome/platformio-ccache" ccache -s + - name: Save integration PlatformIO cache + # Bucket 0 only; the others would race the same immutable key. + if: success() && (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && strategy.job-index == 0 && steps.pio-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.INTEGRATION_PIO_CACHE_PATH }} + key: ${{ steps.pio-cache.outputs.cache-primary-key }} import-time: name: Check import esphome.__main__ time diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 9eead4b38c..add1af5bba 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -101,6 +101,17 @@ COMPONENT_TEST_BATCH_SIZE = 40 INTEGRATION_TESTS_SPLIT_THRESHOLD = 10 INTEGRATION_TESTS_SPLIT_BUCKETS = 3 +# platformio and aioesphomeapi (requirements.txt), the pytest stack +# (requirements_test.txt) and the fixture every session compiles; a change +# to any runs the full matrix +INTEGRATION_TESTS_TRIGGER_FILES = frozenset( + { + "requirements.txt", + "requirements_test.txt", + "tests/integration/fixtures/cache_init.yaml", + } +) + def _split_list(items: list[str], n: int) -> list[list[str]]: """Split a list into n roughly-equal contiguous parts (matches script/clang-tidy).""" @@ -221,12 +232,15 @@ def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[s 3. Integration test infrastructure files changed - conftest.py, types.py, const.py, entity_utils.py, state_utils.py, etc. + 4. A file in INTEGRATION_TESTS_TRIGGER_FILES changed + - The dependency pins and the session init fixture affect every test + Returns (run_all=False, [test_files...]) when: - 4. Specific integration test files changed + 5. Specific integration test files changed - Only those specific test files are returned - 5. Components used by integration tests (or their dependencies) changed + 6. Components used by integration tests (or their dependencies) changed - Only test files whose fixtures use the changed components are returned Args: @@ -244,6 +258,9 @@ def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[s # If any core files changed, run all integration tests return (True, []) + if any(f in INTEGRATION_TESTS_TRIGGER_FILES for f in files): + return (True, []) + # If infrastructure Python files changed (conftest, utils, etc.), run all tests # Excludes test files (test_*.py), fixtures, and non-Python files (README.md) if any( diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 483d5392af..12b1407fe1 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -78,7 +78,8 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]: @pytest.fixture(scope="session") def shared_platformio_cache() -> Generator[Path]: """Initialize a shared PlatformIO cache for all integration tests.""" - # Use a dedicated directory for integration tests to avoid conflicts + # Use a dedicated directory for integration tests to avoid conflicts. + # CI caches parts of this path; keep in sync with ci.yml integration-tests. test_cache_dir = Path.home() / ".esphome-integration-tests" cache_dir = test_cache_dir / "platformio" diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 565f8c563f..7b641e275e 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -552,6 +552,13 @@ def test_determine_integration_tests( assert run_all is True assert test_files == [] + # Dependency pins and the session init fixture trigger run_all + for trigger in sorted(determine_jobs.INTEGRATION_TESTS_TRIGGER_FILES): + with patch.object(determine_jobs, "changed_files", return_value=[trigger]): + run_all, test_files = determine_jobs.determine_integration_tests() + assert run_all is True + assert test_files == [] + # Python files directly in esphome/ do NOT trigger tests with patch.object( determine_jobs, "changed_files", return_value=["esphome/config.py"] From 67f7532940a0b491a2d175f55b7a7eda57208ccc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 24 Aug 2026 15:56:59 -0500 Subject: [PATCH 036/147] [espidf] Always reconfigure after component discovery (#18730) --- esphome/espidf/toolchain.py | 28 ++++--- tests/unit_tests/test_espidf_toolchain.py | 91 +++++++++++++++++++++++ 2 files changed, 108 insertions(+), 11 deletions(-) diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index bb6452acf2..2afd2ed68a 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -386,17 +386,23 @@ def run_compile(config, verbose: bool) -> int: return rc _LOGGER.info("Regenerating CMakeLists.txt with discovered components...") write_project(minimal=False) - if CORE.testing_mode: - # Reconfigure again so cmake is up to date with the full - # component list before the build's idf.py invocation runs -- - # idf.py build would otherwise re-run cmake and regenerate - # memory.ld, wiping the DRAM/IRAM patches applied below. - # Outside testing mode ninja's own configure-time dep on - # CMakeLists.txt handles the re-run as part of the build step. - rc = run_reconfigure() - if rc != 0: - _LOGGER.error("Reconfigure with discovered components failed") - return rc + # Explicit reconfigure: ninja only re-runs cmake when CMakeLists.txt + # is strictly newer than build.ninja, which fails on coarse-mtime + # filesystems (#18682). Also keeps idf.py from regenerating memory.ld + # in testing mode. + rc = run_reconfigure() + if rc != 0: + _LOGGER.error("Reconfigure with discovered components failed") + return rc + # cmake does not rewrite CMakeCache.txt when only properties change, + # so restamp it or every build repeats discovery. Only after success, + # or a failed reconfigure would be marked fresh. build.ninja is + # restamped too so the cache is not newer and ninja does not + # re-run cmake. + for name in ("build/CMakeCache.txt", "build/build.ninja"): + path = CORE.relative_build_path(name) + if path.is_file(): + os.utime(path) # In testing mode, generate the linker script first, patch DRAM/IRAM sizes, # then build. memory.ld is regenerated by ninja during the build phase, diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 26d812af8b..5d55ed3288 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -373,6 +373,97 @@ def test_run_idf_py_jobs_sets_build_jobs_env(setup_core: Path) -> None: assert "IDF_PY_BUILD_JOBS" not in env +def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> None: + """After a successful discovery reconfigure the reference CMakeCache.txt + is restamped; cmake does not rewrite it when only properties or plain + variables change, so the staleness flag would otherwise never clear.""" + _setup_build(setup_core) + config = {CONF_ESPHOME: {}} + cmakecache = CORE.relative_build_path("build/CMakeCache.txt") + build_ninja = CORE.relative_build_path("build/build.ninja") + cmakecache.parent.mkdir(parents=True, exist_ok=True) + cmakecache.write_text("") + build_ninja.write_text("") + old = cmakecache.stat().st_mtime - 100 + os.utime(cmakecache, (old, old)) + os.utime(build_ninja, (old, old)) + + with ( + patch.object(toolchain, "need_reconfigure", return_value=True), + patch("esphome.build_gen.espidf.write_project"), + patch.object(toolchain, "run_reconfigure", return_value=0), + patch.object(toolchain, "run_idf_py", return_value=0), + patch.object(toolchain, "print_summary"), + ): + assert toolchain.run_compile(config, verbose=False) == 0 + + assert cmakecache.stat().st_mtime > old + # build.ninja must not be older than the cache or ninja re-runs cmake + assert build_ninja.stat().st_mtime >= cmakecache.stat().st_mtime + + +def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None: + """A discovery pass that produced no CMakeCache.txt (nothing to restamp) + still completes normally.""" + _setup_build(setup_core) + config = {CONF_ESPHOME: {}} + + with ( + patch.object(toolchain, "need_reconfigure", return_value=True), + patch("esphome.build_gen.espidf.write_project"), + patch.object(toolchain, "run_reconfigure", return_value=0), + patch.object(toolchain, "run_idf_py", return_value=0), + patch.object(toolchain, "print_summary"), + ): + assert toolchain.run_compile(config, verbose=False) == 0 + + assert not CORE.relative_build_path("build/CMakeCache.txt").exists() + + +def test_run_compile_reconfigures_after_full_write_outside_testing_mode( + setup_core: Path, +) -> None: + """The full CMakeLists write is followed by a reconfigure (#18682); a + failure there stops the build and leaves the cache unstamped.""" + _setup_build(setup_core) + config = {CONF_ESPHOME: {}} + cmakecache = CORE.relative_build_path("build/CMakeCache.txt") + cmakecache.parent.mkdir(parents=True, exist_ok=True) + cmakecache.write_text("") + old = cmakecache.stat().st_mtime - 100 + os.utime(cmakecache, (old, old)) + calls: list[tuple] = [] + reconfigures = 0 + + def record_write(minimal: bool = False) -> None: + calls.append(("write_project", minimal)) + + def record_reconfigure() -> int: + nonlocal reconfigures + reconfigures += 1 + calls.append(("run_reconfigure",)) + return 1 if reconfigures == 2 else 0 + + with ( + patch.object(toolchain, "need_reconfigure", return_value=True), + patch("esphome.build_gen.espidf.write_project", side_effect=record_write), + patch.object(toolchain, "run_reconfigure", side_effect=record_reconfigure), + patch.object(toolchain, "run_idf_py", return_value=0) as mock_build, + patch.object(toolchain, "print_summary"), + ): + assert not CORE.testing_mode + assert toolchain.run_compile(config, verbose=False) == 1 + + assert calls == [ + ("write_project", True), + ("run_reconfigure",), + ("write_project", False), + ("run_reconfigure",), + ] + mock_build.assert_not_called() + assert cmakecache.stat().st_mtime == old + + def test_run_compile_passes_compile_process_limit(setup_core: Path) -> None: """compile_process_limit is forwarded to run_idf_py as the job limit.""" _setup_build(setup_core) From a5d8c45b678bab3dff09b2848ffa40cc2c05e84b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 24 Aug 2026 16:20:25 -0500 Subject: [PATCH 037/147] [gpio] Fix one_wire reset busy-waiting with interrupts off when delay wraps (#18733) --- esphome/components/gpio/one_wire/gpio_one_wire.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/esphome/components/gpio/one_wire/gpio_one_wire.cpp b/esphome/components/gpio/one_wire/gpio_one_wire.cpp index 1fecfbf0dd..f445efeca3 100644 --- a/esphome/components/gpio/one_wire/gpio_one_wire.cpp +++ b/esphome/components/gpio/one_wire/gpio_one_wire.cpp @@ -55,8 +55,11 @@ int HOT IRAM_ATTR GPIOOneWireBus::reset_int() { delayMicroseconds(1); } - // delay J - delayMicroseconds(start + 480 - micros()); + // delay J: finish the 480us slot, but never spin if it already elapsed + // (unsigned wrap here would busy-wait for minutes with interrupts off) + uint32_t elapsed = micros() - start; + if (elapsed < 480) + delayMicroseconds(480 - elapsed); this->pin_.digital_write(true); this->pin_.pin_mode(gpio::FLAG_OUTPUT); return r ? 1 : 0; From 82ca5365c91514ca2e56fea644552706dc75e11f Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:44:28 +0000 Subject: [PATCH 038/147] Bump bundled esphome-device-builder to 1.13.0 (#18743) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9f27d51059..d46f01838e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.12.4 +RUN uv pip install --no-cache-dir esphome-device-builder==1.13.0 RUN \ platformio settings set enable_telemetry No \ From ae460b430c94e9978b1bc18b4bc3ca83d48c4493 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:33:39 +1000 Subject: [PATCH 039/147] [lvgl] Fix on_value/on_update triggers for LVGL select entities (#18778) Co-authored-by: Claude Sonnet 5 --- esphome/components/lvgl/lvgl_esphome.cpp | 8 ++-- esphome/components/lvgl/lvgl_esphome.h | 6 +-- esphome/components/lvgl/select/lvgl_select.h | 15 ++----- esphome/components/lvgl/types.py | 3 ++ .../dropdown_update_fires_event_test.yaml | 36 ++++++++++++++++ .../lvgl/test_dropdown_update_fires_event.py | 41 +++++++++++++++++++ 6 files changed, 90 insertions(+), 19 deletions(-) create mode 100644 tests/component_tests/lvgl/config/dropdown_update_fires_event_test.yaml create mode 100644 tests/component_tests/lvgl/test_dropdown_update_fires_event.py diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index b66a904437..b3ce950db2 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -551,21 +551,21 @@ std::string LvSelectable::get_selected_text() { return this->options_[selected]; } -static std::string join_string(std::vector options) { +static std::string join_string(const FixedVector &options) { return std::accumulate( options.begin(), options.end(), std::string(), - [](const std::string &a, const std::string &b) -> std::string { return a + (!a.empty() ? "\n" : "") + b; }); + [](const std::string &a, const char *b) -> std::string { return a + (!a.empty() ? "\n" : "") + b; }); } void LvSelectable::set_selected_text(const std::string &text, lv_anim_enable_t anim) { - auto index = std::find(this->options_.begin(), this->options_.end(), text); + auto *index = std::find(this->options_.begin(), this->options_.end(), text); if (index != this->options_.end()) { this->set_selected_index(index - this->options_.begin(), anim); lv_obj_send_event(this->obj, lv_update_event, nullptr); } } -void LvSelectable::set_options(std::vector options) { +void LvSelectable::set_options(FixedVector options) { auto index = this->get_selected_index(); if (index >= options.size()) index = options.size() - 1; diff --git a/esphome/components/lvgl/lvgl_esphome.h b/esphome/components/lvgl/lvgl_esphome.h index 9221ab9542..0771de175e 100644 --- a/esphome/components/lvgl/lvgl_esphome.h +++ b/esphome/components/lvgl/lvgl_esphome.h @@ -499,12 +499,12 @@ class LvSelectable : public LvCompound { virtual void set_selected_index(size_t index, lv_anim_enable_t anim) = 0; void set_selected_text(const std::string &text, lv_anim_enable_t anim); std::string get_selected_text(); - const std::vector &get_options() { return this->options_; } - void set_options(std::vector options); + const FixedVector &get_options() { return this->options_; } + void set_options(FixedVector options); protected: virtual void set_option_string(const char *options) = 0; - std::vector options_{}; + FixedVector options_{}; }; #ifdef USE_LVGL_DROPDOWN diff --git a/esphome/components/lvgl/select/lvgl_select.h b/esphome/components/lvgl/select/lvgl_select.h index e36357328c..dafdd91eb5 100644 --- a/esphome/components/lvgl/select/lvgl_select.h +++ b/esphome/components/lvgl/select/lvgl_select.h @@ -50,19 +50,10 @@ class LVGLSelect final : public select::Select, public Component { protected: void control(size_t index) override { this->widget_->set_selected_index(index, this->anim_); - this->publish(); - } - void set_options_() { - // Widget uses std::vector, SelectTraits uses FixedVector - // Convert by extracting c_str() pointers - const auto &opts = this->widget_->get_options(); - FixedVector opt_ptrs; - opt_ptrs.init(opts.size()); - for (const auto &opt : opts) { - opt_ptrs.push_back(opt.c_str()); - } - this->traits.set_options(opt_ptrs); + // The update event fires the widget's on_value/on_update triggers + lv_obj_send_event(this->widget_->obj, lv_update_event, nullptr); } + void set_options_() { this->traits.set_options(this->widget_->get_options()); } LvSelectable *widget_; lv_anim_enable_t anim_; diff --git a/esphome/components/lvgl/types.py b/esphome/components/lvgl/types.py index 61efe385e6..cc8d9438a9 100644 --- a/esphome/components/lvgl/types.py +++ b/esphome/components/lvgl/types.py @@ -3,6 +3,8 @@ from esphome.const import CONF_TEXT, CONF_VALUE from esphome.cpp_generator import MockObj from esphome.cpp_types import Component, esphome_ns +from .defines import CONF_SELECTED_INDEX + class LvType(cg.MockObjClass): def __init__(self, *args, **kwargs): @@ -112,3 +114,4 @@ class LvSelect(LvType): parents=parens, **kwargs, ) + self.value_property = CONF_SELECTED_INDEX diff --git a/tests/component_tests/lvgl/config/dropdown_update_fires_event_test.yaml b/tests/component_tests/lvgl/config/dropdown_update_fires_event_test.yaml new file mode 100644 index 0000000000..2fe59b2f1a --- /dev/null +++ b/tests/component_tests/lvgl/config/dropdown_update_fires_event_test.yaml @@ -0,0 +1,36 @@ +esphome: + name: test-dropdown-update-event + on_boot: + - lvgl.dropdown.update: + id: test_dropdown + selected_index: 2 + +esp32: + board: lolin_c3_mini + +spi: + mosi_pin: + number: GPIO2 + ignore_strapping_warning: true + clk_pin: GPIO1 + +display: + - platform: mipi_spi + data_rate: 20MHz + model: st7735 + cs_pin: + number: GPIO8 + ignore_strapping_warning: true + dc_pin: GPIO3 + +lvgl: + widgets: + - dropdown: + id: test_dropdown + options: + - First + - Second + - Third + on_update: + - lambda: |- + ESP_LOGD("test", "dropdown updated"); diff --git a/tests/component_tests/lvgl/test_dropdown_update_fires_event.py b/tests/component_tests/lvgl/test_dropdown_update_fires_event.py new file mode 100644 index 0000000000..1e034ad6eb --- /dev/null +++ b/tests/component_tests/lvgl/test_dropdown_update_fires_event.py @@ -0,0 +1,41 @@ +"""Regression test: lvgl.dropdown.update with selected_index must fire on_value/on_update. + +LvSelect (backing both dropdown and roller) did not set `value_property`, so the generic +update-action machinery in automation.py never sent the synthetic update event for a +`selected_index:` change made via `lvgl.dropdown.update`/`lvgl.roller.update`, unlike `value:` +on number widgets or `text:` on text widgets. Fixed by setting `LvSelect.value_property` to +`CONF_SELECTED_INDEX`. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from esphome.__main__ import generate_cpp_contents +from esphome.config import read_config +from esphome.core import CORE + + +@pytest.fixture(scope="module") +def main_cpp(request: pytest.FixtureRequest) -> str: + config_path = ( + Path(request.fspath).parent / "config" / "dropdown_update_fires_event_test.yaml" + ) + original_path = CORE.config_path + try: + CORE.config_path = config_path + CORE.config = read_config({}) + generate_cpp_contents(CORE.config) + return CORE.cpp_main_section + finally: + CORE.config_path = original_path + CORE.reset() + + +def test_dropdown_update_sends_update_event(main_cpp: str) -> None: + assert ( + "lv_obj_send_event(test_dropdown->obj, lvgl::lv_update_event, nullptr)" + in main_cpp + ) From c75252d4257d94eb6cd2108c8fd82a7881922326 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 23:11:26 -0500 Subject: [PATCH 040/147] [http_request] Default watchdog_timeout from timeout on ESP32 (#18732) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- esphome/components/http_request/__init__.py | 35 +++++++++++++++- .../http_request/http_request_idf.cpp | 3 +- .../component_tests/http_request/__init__.py | 0 .../config/test_esp32_default.yaml | 12 ++++++ .../config/test_esp32_explicit.yaml | 13 ++++++ .../config/test_esp32_platform_wider.yaml | 13 ++++++ .../http_request/config/test_esp32_stock.yaml | 11 +++++ .../http_request/config/test_esp8266.yaml | 13 ++++++ .../http_request/config/test_rp2040.yaml | 13 ++++++ .../component_tests/http_request/test_init.py | 42 +++++++++++++++++++ 10 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 tests/component_tests/http_request/__init__.py create mode 100644 tests/component_tests/http_request/config/test_esp32_default.yaml create mode 100644 tests/component_tests/http_request/config/test_esp32_explicit.yaml create mode 100644 tests/component_tests/http_request/config/test_esp32_platform_wider.yaml create mode 100644 tests/component_tests/http_request/config/test_esp32_stock.yaml create mode 100644 tests/component_tests/http_request/config/test_esp8266.yaml create mode 100644 tests/component_tests/http_request/config/test_rp2040.yaml create mode 100644 tests/component_tests/http_request/test_init.py diff --git a/esphome/components/http_request/__init__.py b/esphome/components/http_request/__init__.py index 54d7f5c77b..923cd49acf 100644 --- a/esphome/components/http_request/__init__.py +++ b/esphome/components/http_request/__init__.py @@ -16,12 +16,15 @@ from esphome.const import ( CONF_TIMEOUT, CONF_URL, CONF_WATCHDOG_TIMEOUT, + PLATFORM_ESP32, PLATFORM_HOST, PlatformFramework, __version__, ) -from esphome.core import CORE, Lambda +from esphome.core import CORE, Lambda, TimePeriodMilliseconds +import esphome.final_validate as fv from esphome.helpers import IS_MACOS +from esphome.types import ConfigType DEPENDENCIES = ["network"] AUTO_LOAD = ["json", "watchdog"] @@ -91,6 +94,34 @@ def validate_ssl_verification(config): return config +# esp_http_client_open() runs DNS, TCP connect and the TLS handshake with no +# watchdog feed in between; each can take up to `timeout` on ESP-IDF. +WATCHDOG_TIMEOUT_MULTIPLIER = 3 +# Headroom over the exact worst case so a fully stalled open does not land on +# the watchdog deadline. +WATCHDOG_TIMEOUT_MARGIN_MS = 1000 + + +def default_watchdog_timeout(config: ConfigType) -> None: + """Arm the request watchdog on ESP32 when the user did not set it. + + The default never goes below the platform task watchdog, so a user who + widened `esp32.watchdog_timeout` keeps that window during requests. + """ + if not CORE.is_esp32 or CONF_WATCHDOG_TIMEOUT in config: + return + derived_ms = ( + config[CONF_TIMEOUT].total_milliseconds * WATCHDOG_TIMEOUT_MULTIPLIER + + WATCHDOG_TIMEOUT_MARGIN_MS + ) + platform_ms = fv.full_config.get()[PLATFORM_ESP32][ + CONF_WATCHDOG_TIMEOUT + ].total_milliseconds + config[CONF_WATCHDOG_TIMEOUT] = TimePeriodMilliseconds( + milliseconds=max(derived_ms, platform_ms) + ) + + def _declare_request_class(value): if CORE.is_host: return cv.declare_id(HttpRequestHost)(value) @@ -150,6 +181,8 @@ CONFIG_SCHEMA = cv.All( validate_ssl_verification, ) +FINAL_VALIDATE_SCHEMA = default_watchdog_timeout + async def to_code(config): var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/http_request/http_request_idf.cpp b/esphome/components/http_request/http_request_idf.cpp index a437540241..55a1331667 100644 --- a/esphome/components/http_request/http_request_idf.cpp +++ b/esphome/components/http_request/http_request_idf.cpp @@ -142,12 +142,13 @@ std::shared_ptr HttpRequestIDF::perform(const std::string &url, c const char *buf = body.c_str(); while (write_left > 0) { int written = esp_http_client_write(client, buf + write_index, write_left); - if (written < 0) { + if (written <= 0) { err = ESP_FAIL; break; } write_left -= written; write_index += written; + container->feed_wdt(); } } diff --git a/tests/component_tests/http_request/__init__.py b/tests/component_tests/http_request/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/http_request/config/test_esp32_default.yaml b/tests/component_tests/http_request/config/test_esp32_default.yaml new file mode 100644 index 0000000000..86744dcb11 --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp32_default.yaml @@ -0,0 +1,12 @@ +esphome: + name: test + +esp32: + board: nodemcu-32s + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s diff --git a/tests/component_tests/http_request/config/test_esp32_explicit.yaml b/tests/component_tests/http_request/config/test_esp32_explicit.yaml new file mode 100644 index 0000000000..e0d0074caa --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp32_explicit.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + board: nodemcu-32s + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s + watchdog_timeout: 20s diff --git a/tests/component_tests/http_request/config/test_esp32_platform_wider.yaml b/tests/component_tests/http_request/config/test_esp32_platform_wider.yaml new file mode 100644 index 0000000000..77a85da2ff --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp32_platform_wider.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + board: nodemcu-32s + watchdog_timeout: 60s + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s diff --git a/tests/component_tests/http_request/config/test_esp32_stock.yaml b/tests/component_tests/http_request/config/test_esp32_stock.yaml new file mode 100644 index 0000000000..70d2701466 --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp32_stock.yaml @@ -0,0 +1,11 @@ +esphome: + name: test + +esp32: + board: nodemcu-32s + +wifi: + ssid: test + password: testtest + +http_request: diff --git a/tests/component_tests/http_request/config/test_esp8266.yaml b/tests/component_tests/http_request/config/test_esp8266.yaml new file mode 100644 index 0000000000..d0698dc57e --- /dev/null +++ b/tests/component_tests/http_request/config/test_esp8266.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp8266: + board: d1_mini + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s + verify_ssl: false diff --git a/tests/component_tests/http_request/config/test_rp2040.yaml b/tests/component_tests/http_request/config/test_rp2040.yaml new file mode 100644 index 0000000000..030736c30d --- /dev/null +++ b/tests/component_tests/http_request/config/test_rp2040.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +rp2: + board: rpipicow + +wifi: + ssid: test + password: testtest + +http_request: + timeout: 10s + verify_ssl: false diff --git a/tests/component_tests/http_request/test_init.py b/tests/component_tests/http_request/test_init.py new file mode 100644 index 0000000000..446c4acbd0 --- /dev/null +++ b/tests/component_tests/http_request/test_init.py @@ -0,0 +1,42 @@ +"""Tests for the http_request watchdog timeout default.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.config import read_config +from esphome.const import CONF_WATCHDOG_TIMEOUT +from esphome.core import CORE, TimePeriodMilliseconds + + +@pytest.mark.parametrize( + ("yaml_file", "expected_ms"), + [ + # stock 4.5s timeout: 3 x 4.5s plus 1s margin + ("test_esp32_stock.yaml", 14500), + # 3 x 10s plus 1s margin + ("test_esp32_default.yaml", 31000), + # esp32.watchdog_timeout: 60s is wider than the derived value and wins + ("test_esp32_platform_wider.yaml", 60000), + # explicit value is kept as is + ("test_esp32_explicit.yaml", 20000), + ], +) +def test_esp32_watchdog_timeout( + component_config_path: Callable[[str], Path], yaml_file: str, expected_ms: int +) -> None: + CORE.config_path = component_config_path(yaml_file) + config = read_config({}) + assert config["http_request"][CONF_WATCHDOG_TIMEOUT] == TimePeriodMilliseconds( + milliseconds=expected_ms + ) + + +@pytest.mark.parametrize("yaml_file", ["test_esp8266.yaml", "test_rp2040.yaml"]) +def test_other_platforms_leave_watchdog_unset( + component_config_path: Callable[[str], Path], yaml_file: str +) -> None: + CORE.config_path = component_config_path(yaml_file) + config = read_config({}) + assert CONF_WATCHDOG_TIMEOUT not in config["http_request"] From 8929fc43d854de7595babe3b9eb2282a08a78922 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 27 Aug 2026 01:04:57 +1000 Subject: [PATCH 041/147] [mipi_spi] Fix dimensions for jc3636518v2 (#18786) --- esphome/components/mipi_spi/models/jc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/esphome/components/mipi_spi/models/jc.py b/esphome/components/mipi_spi/models/jc.py index ca9adb4a72..8d2591aefe 100644 --- a/esphome/components/mipi_spi/models/jc.py +++ b/esphome/components/mipi_spi/models/jc.py @@ -266,8 +266,6 @@ DriverChip( "JC3636W518V2", height=360, width=360, - offset_height=1, - draw_rounding=1, cs_pin=10, reset_pin=47, invert_colors=True, From 20d4fe1a4178444c77375a562dd0498237db88ba Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Aug 2026 10:16:44 -0500 Subject: [PATCH 042/147] [mdns] Skip MDNS.update() while the ESP8266 radio cannot transmit (#18785) --- esphome/components/mdns/mdns_esp8266.cpp | 14 +++++++++++++- esphome/components/wifi/wifi_component.cpp | 6 +++--- esphome/components/wifi/wifi_component.h | 7 +++++++ esphome/components/wifi/wifi_component_esp8266.cpp | 2 +- esphome/components/wifi/wifi_component_esp_idf.cpp | 2 +- 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/esphome/components/mdns/mdns_esp8266.cpp b/esphome/components/mdns/mdns_esp8266.cpp index f6d5786675..1f0b3c9519 100644 --- a/esphome/components/mdns/mdns_esp8266.cpp +++ b/esphome/components/mdns/mdns_esp8266.cpp @@ -41,7 +41,19 @@ static void register_esp8266(MDNSComponent *, StaticVectorset_interval(MDNS_POLL_ID, MDNS_UPDATE_INTERVAL_MS, []() { MDNS.update(); }); + this->set_interval(MDNS_POLL_ID, MDNS_UPDATE_INTERVAL_MS, []() { +#ifdef USE_MDNS_WIFI_LISTENER + // MDNS.update() can suspend the loop in UdpContext::sendTimeout() while a send is + // failing (radio off-channel during a roam scan, or mid reconnect); an incoming + // packet then re-enters LEAmDNS from lwIP and corrupts shared UdpContext state. + // Skip the tick while the radio cannot transmit (#18760), but keep polling while + // the AP is serving clients (AP-only or fallback AP with the STA down). + auto *wifi = wifi::global_wifi_component; + if (wifi->is_roaming() || (!wifi->is_connected() && !wifi->is_ap_active())) + return; +#endif + MDNS.update(); + }); this->set_timeout(MDNS_POLL_STOP_ID, MDNS_POLL_WINDOW_MS, [this]() { this->cancel_interval(MDNS_POLL_ID); }); } #endif diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 127eb50df1..3a42ace424 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -530,7 +530,7 @@ void WiFiComponent::log_discarded_scan_result_(const char *ssid, const uint8_t * #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE // Skip logging during roaming scans to avoid log buffer overflow // (roaming scans typically find many networks but only care about same-SSID APs) - if (this->roaming_state_ == RoamingState::SCANNING) { + if (this->is_roaming_scan_active()) { return; } char bssid_s[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; @@ -835,7 +835,7 @@ void WiFiComponent::loop() { // Post-connect roaming: check for better AP if (this->post_connect_roaming_) { - if (this->roaming_state_ == RoamingState::SCANNING) { + if (this->is_roaming_scan_active()) { if (this->scan_done_) { this->process_roaming_scan_(); } @@ -2152,7 +2152,7 @@ void WiFiComponent::retry_connect() { // Roam connection failed - transition to reconnecting ESP_LOGD(TAG, "Roam failed, reconnecting (attempt %u/%u)", this->roaming_attempts_, ROAMING_MAX_ATTEMPTS); this->roaming_state_ = RoamingState::RECONNECTING; - } else if (this->roaming_state_ == RoamingState::SCANNING) { + } else if (this->is_roaming_scan_active()) { // Disconnected during roam scan - transition to RECONNECTING so the attempts // counter is preserved when reconnection succeeds (IDLE would reset it) ESP_LOGD(TAG, "Disconnected during roam scan (attempt %u/%u)", this->roaming_attempts_, ROAMING_MAX_ATTEMPTS); diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index ea043fd5c6..bf991beece 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -475,6 +475,13 @@ class WiFiComponent final : public Component { bool is_connected() const { return this->connected_; } + /// True while a post-connect roaming scan holds the radio off-channel. + bool is_roaming_scan_active() const { return this->roaming_state_ == RoamingState::SCANNING; } + + /// True while a post-connect roam is in progress (scanning off-channel, reassociating, + /// or recovering from a failed roam). + bool is_roaming() const { return this->roaming_state_ != RoamingState::IDLE; } + #ifdef USE_ESP32 /// esp_netif handle of the station interface, used by network for default-route /// arbitration. nullptr until wifi_lazy_init_() has run. diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index acaa94b13c..10c973a624 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -717,7 +717,7 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { static constexpr uint32_t SCAN_ACTIVE_MAX_DEFAULT_MS = 500; static constexpr uint32_t SCAN_ACTIVE_MIN_ROAMING_MS = 100; static constexpr uint32_t SCAN_ACTIVE_MAX_ROAMING_MS = 300; - bool roaming = this->roaming_state_ == RoamingState::SCANNING; + bool roaming = this->is_roaming_scan_active(); if (passive) { config.scan_time.passive = roaming ? SCAN_PASSIVE_ROAMING_MS : SCAN_PASSIVE_DEFAULT_MS; } else { diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index 24cb060edb..4b339528a2 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -1064,7 +1064,7 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { // When scanning while connected (roaming), return to home channel between // each scanned channel to maintain the connection (helps with BLE/WiFi coexistence) #ifdef CONFIG_SOC_WIFI_SUPPORTED - if (this->roaming_state_ == RoamingState::SCANNING) { + if (this->is_roaming_scan_active()) { config.coex_background_scan = true; } #endif From 4e0a5e5a5b0a877752c38108da72fe93bfa01a88 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:33:52 -0500 Subject: [PATCH 043/147] Bump bundled esphome-device-builder to 1.13.1 (#18807) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d46f01838e..0da8048c57 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.13.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.13.1 RUN \ platformio settings set enable_telemetry No \ From 0e4f46001346d1ac54b26c6d9eaa7d92d2fab612 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 31 Aug 2026 11:25:12 +1200 Subject: [PATCH 044/147] Bump version to 2026.8.2 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 3d9a6f7221..ce1070cbbd 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.8.1 +PROJECT_NUMBER = 2026.8.2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 53da67a4f4..06f843a2c0 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.8.1" +__version__ = "2026.8.2" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From fbe306f00b83d480a745004788870b2dce5505ad Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 30 Aug 2026 19:30:40 -0500 Subject: [PATCH 045/147] [homeassistant] Add integration test for binary sensor initial state triggers (#18894) --- ...assistant_binary_sensor_initial_state.yaml | 59 +++++++++++ tests/integration/log_utils.py | 5 + ...meassistant_binary_sensor_initial_state.py | 98 +++++++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 tests/integration/fixtures/api_homeassistant_binary_sensor_initial_state.yaml create mode 100644 tests/integration/test_api_homeassistant_binary_sensor_initial_state.py diff --git a/tests/integration/fixtures/api_homeassistant_binary_sensor_initial_state.yaml b/tests/integration/fixtures/api_homeassistant_binary_sensor_initial_state.yaml new file mode 100644 index 0000000000..0e47a7f1fa --- /dev/null +++ b/tests/integration/fixtures/api_homeassistant_binary_sensor_initial_state.yaml @@ -0,0 +1,59 @@ +esphome: + name: ha-bs-initial + +host: + +api: + +logger: + level: DEBUG + +binary_sensor: + # trigger_on_initial_state: true must fire on_press for the first state from HA + - platform: homeassistant + name: Initial On + entity_id: binary_sensor.initial_on + trigger_on_initial_state: true + on_press: + - logger.log: "initial_on on_press" + on_release: + - logger.log: "initial_on on_release" + + # Default (false) must not fire on the first state, only on later changes + - platform: homeassistant + name: Default + entity_id: binary_sensor.default + on_press: + - logger.log: "default on_press" + on_release: + - logger.log: "default on_release" + + # Real HA startup shape: 'unavailable' arrives before the first real state + - platform: homeassistant + name: Unavailable First + entity_id: binary_sensor.unavailable_first + trigger_on_initial_state: true + on_press: + - logger.log: "unavailable_first on_press" + on_release: + - logger.log: "unavailable_first on_release" + + # Initial 'off' must fire on_release when trigger_on_initial_state is set + - platform: homeassistant + name: Initial Off + entity_id: binary_sensor.initial_off + trigger_on_initial_state: true + on_press: + - logger.log: "initial_off on_press" + on_release: + - logger.log: "initial_off on_release" + + # Same 'unavailable' first shape without the flag; must stay quiet on the + # first real state and only fire on the later change + - platform: homeassistant + name: Default Unavailable First + entity_id: binary_sensor.default_unavail + on_press: + - logger.log: "default_unavail on_press" + on_release: + - logger.log: "default_unavail on_release" diff --git a/tests/integration/log_utils.py b/tests/integration/log_utils.py index 0bfbb57b1f..c605351bb8 100644 --- a/tests/integration/log_utils.py +++ b/tests/integration/log_utils.py @@ -28,6 +28,11 @@ class LineWaiter: self._future.set_result(line) self._future = None + async def wait_for_each(self, *texts: str, timeout: float = 10.0) -> None: + """Await each text in turn; a text may match a line already received.""" + for text in texts: + await self.wait_for(text, timeout=timeout) + async def wait_for(self, *needles: str, timeout: float = 10.0) -> str: """Return the first line, past or future, containing every needle.""" for line in self.lines: diff --git a/tests/integration/test_api_homeassistant_binary_sensor_initial_state.py b/tests/integration/test_api_homeassistant_binary_sensor_initial_state.py new file mode 100644 index 0000000000..4f7dda6eee --- /dev/null +++ b/tests/integration/test_api_homeassistant_binary_sensor_initial_state.py @@ -0,0 +1,98 @@ +"""Test on_press/on_release for homeassistant binary sensors on the first HA state.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from .log_utils import LineWaiter +from .types import APIClientConnectedFactory, RunCompiledFunction + +ENTITIES = ( + "binary_sensor.initial_on", + "binary_sensor.default", + "binary_sensor.unavailable_first", + "binary_sensor.initial_off", + "binary_sensor.default_unavail", +) + + +@pytest.mark.asyncio +async def test_api_homeassistant_binary_sensor_initial_state( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """The first state from HA fires on_press only with trigger_on_initial_state.""" + loop = asyncio.get_running_loop() + waiter = LineWaiter() + subscribed: set[str] = set() + all_subscribed = loop.create_future() + + def on_state_sub(entity_id: str, _attribute: str | None) -> None: + subscribed.add(entity_id) + if not all_subscribed.done() and subscribed.issuperset(ENTITIES): + all_subscribed.set_result(None) + + async with ( + run_compiled(yaml_config, line_callback=waiter.callback), + api_client_connected() as client, + ): + client.subscribe_home_assistant_states(on_state_sub) + try: + await asyncio.wait_for(all_subscribed, timeout=5.0) + except TimeoutError: + pytest.fail(f"never subscribed: {set(ENTITIES) - subscribed}") + + # First state from HA + client.send_home_assistant_state("binary_sensor.initial_on", "", "on") + client.send_home_assistant_state("binary_sensor.default", "", "on") + client.send_home_assistant_state( + "binary_sensor.unavailable_first", "", "unavailable" + ) + client.send_home_assistant_state("binary_sensor.unavailable_first", "", "on") + client.send_home_assistant_state( + "binary_sensor.default_unavail", "", "unavailable" + ) + client.send_home_assistant_state("binary_sensor.default_unavail", "", "on") + client.send_home_assistant_state("binary_sensor.initial_off", "", "off") + + await waiter.wait_for("initial_on on_press", timeout=5.0) + await waiter.wait_for("unavailable_first on_press", timeout=5.0) + # Pin that the 'unavailable' message actually arrived and was rejected + await waiter.wait_for("Can't convert 'unavailable'", timeout=5.0) + # initial_off is the last state sent, so this wait also proves the + # earlier 'default' initial state was already processed + await waiter.wait_for("initial_off on_release", timeout=5.0) + # Both 'unavailable' senders must have been seen and rejected + assert sum("Can't convert 'unavailable'" in line for line in waiter.lines) == 2 + # Guard every phase 2 needle against being satisfied by a stale + # phase 1 line, and pin that the initial states fired nothing else + for absent in ( + "initial_on on_release", + "default on_press", + "default on_release", + "default_unavail on_press", + "default_unavail on_release", + "unavailable_first on_release", + "initial_off on_press", + ): + assert not any(absent in line for line in waiter.lines), ( + f"unexpected trigger before the second state change: {absent}" + ) + + # A later change fires for all of them + client.send_home_assistant_state("binary_sensor.initial_on", "", "off") + client.send_home_assistant_state("binary_sensor.default", "", "off") + client.send_home_assistant_state("binary_sensor.unavailable_first", "", "off") + client.send_home_assistant_state("binary_sensor.initial_off", "", "on") + client.send_home_assistant_state("binary_sensor.default_unavail", "", "off") + await waiter.wait_for_each( + "initial_on on_release", + "default on_release", + "default_unavail on_release", + "unavailable_first on_release", + "initial_off on_press", + timeout=5.0, + ) From 7b97c8739073e0ace05e35f472b3aa7eace2ba82 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Sun, 30 Aug 2026 20:43:59 -0500 Subject: [PATCH 046/147] [improv_serial] Support non-Wi-Fi network interfaces (#17598) Co-authored-by: J. Nick Koston --- esphome/components/improv_serial/__init__.py | 2 +- .../improv_serial/improv_serial_component.cpp | 159 +++++++++++++++--- .../improv_serial/improv_serial_component.h | 28 ++- .../improv_serial/common-ethernet.yaml | 17 ++ .../test-ethernet.esp32-idf.yaml | 2 + .../wifi/wifi_component.cpp | 2 + .../external_components/wifi/wifi_component.h | 11 ++ 7 files changed, 193 insertions(+), 28 deletions(-) create mode 100644 tests/components/improv_serial/common-ethernet.yaml create mode 100644 tests/components/improv_serial/test-ethernet.esp32-idf.yaml diff --git a/esphome/components/improv_serial/__init__.py b/esphome/components/improv_serial/__init__.py index 11e9f1ea62..a34e2ab793 100644 --- a/esphome/components/improv_serial/__init__.py +++ b/esphome/components/improv_serial/__init__.py @@ -16,7 +16,7 @@ from esphome.types import ConfigType AUTO_LOAD = ["improv_base"] CODEOWNERS = ["@esphome/core"] -DEPENDENCIES = ["logger", "wifi"] +DEPENDENCIES = ["logger", "network"] improv_serial_ns = cg.esphome_ns.namespace("improv_serial") diff --git a/esphome/components/improv_serial/improv_serial_component.cpp b/esphome/components/improv_serial/improv_serial_component.cpp index 0fb18e9b0d..ffa7b79d9b 100644 --- a/esphome/components/improv_serial/improv_serial_component.cpp +++ b/esphome/components/improv_serial/improv_serial_component.cpp @@ -1,5 +1,5 @@ #include "improv_serial_component.h" -#ifdef USE_WIFI +#ifdef USE_IMPROV_SERIAL #include "esphome/core/application.h" #include "esphome/core/defines.h" #include "esphome/core/hal.h" @@ -7,7 +7,10 @@ #include "esphome/core/version.h" #include "esphome/components/logger/logger.h" +#include "esphome/components/network/util.h" +#ifdef USE_WIFI #include "esphome/components/wifi/scan_list.h" +#endif #include @@ -26,13 +29,17 @@ void ImprovSerialComponent::setup() { this->hw_serial_ = logger::global_logger->get_hw_serial(); #endif - if (wifi::global_wifi_component->has_sta()) { + // The Improv state machine tracks Wi-Fi provisioning only. General device + // connectivity (e.g. Ethernet) is reported separately via GET_NETWORK_STATE. +#ifdef USE_WIFI + if (wifi::global_wifi_component != nullptr && wifi::global_wifi_component->has_sta()) { this->state_ = improv::STATE_PROVISIONED; - } else if (!wifi::global_wifi_component->is_disabled()) { + } else if (wifi::global_wifi_component != nullptr && !wifi::global_wifi_component->is_disabled()) { // Respect Wi-Fi's disabled state; forcing a scan while disabled throws // the wifi component into an invalid state from which it cannot recover. wifi::global_wifi_component->start_scanning(); } +#endif } void ImprovSerialComponent::loop() { @@ -55,8 +62,14 @@ void ImprovSerialComponent::loop() { } } - if (this->state_ == improv::STATE_PROVISIONING) { - if (wifi::global_wifi_component->is_connected()) { +#ifdef USE_WIFI + if (this->state_ == improv::STATE_PROVISIONING && wifi::global_wifi_component != nullptr && + wifi::global_wifi_component->is_connected()) { + // Being connected is not enough: re-provisioning a device that is already online leaves the + // prior network up until it drops, so check that the joined network is the requested one + // before reporting success. Same test as the wifi.connect action. + char ssid_buf[wifi::SSID_BUFFER_SIZE]; + if (strcmp(wifi::global_wifi_component->wifi_ssid_to(ssid_buf), this->connecting_sta_.get_ssid().c_str()) == 0) { wifi::global_wifi_component->save_wifi_sta(this->connecting_sta_.get_ssid(), this->connecting_sta_.get_password()); this->connecting_sta_ = {}; @@ -66,6 +79,7 @@ void ImprovSerialComponent::loop() { this->send_settings_response_(improv::WIFI_SETTINGS); } } +#endif } void ImprovSerialComponent::dump_config() { ESP_LOGCONFIG(TAG, "Improv Serial:"); } @@ -143,15 +157,17 @@ void ImprovSerialComponent::write_data_(const uint8_t *data, const size_t size) #endif } -void ImprovSerialComponent::send_settings_response_(improv::Command command) { - std::array buf; - improv::RpcResponseBuilder builder(buf, command); -#ifdef USE_IMPROV_SERIAL_NEXT_URL - this->add_next_url_(builder, MAX_NEXT_URL_LEN); -#endif #ifdef USE_WEBSERVER - for (auto &ip : wifi::global_wifi_component->wifi_sta_ip_addresses()) { - if (ip.is_ip4()) { +void ImprovSerialComponent::add_webserver_urls_(improv::RpcResponseBuilder &builder, [[maybe_unused]] bool wifi_first) { + // The webserver listens on every interface, so advertise each one that has a usable IPv4. + // network::get_ip_addresses() can't be used here: it returns only the highest-priority + // interface's addresses, which are all-unset (0.0.0.0) when e.g. Ethernet has no link while + // the device is online via Wi-Fi, and 0.0.0.0 must not become the advertised URL. OpenThread + // is omitted: it only ever has IPv6 addresses, which cannot form an IPv4 http:// URL. + const auto append_urls = [&builder](const network::IPAddresses &addresses) { + for (const auto &ip : addresses) { + if (!ip.is_ip4() || !ip.is_set()) + continue; char ip_buf[network::IP_ADDRESS_BUFFER_SIZE]; ip.str_to(ip_buf); // "http://" (7) + IP (40) + ":" (1) + port (5) + null (1) = 54 @@ -162,9 +178,43 @@ void ImprovSerialComponent::send_settings_response_(improv::Command command) { if (!builder.add_string(webserver_url, len)) { ESP_LOGW(TAG, "Response full; URL dropped"); } - break; } - } + }; +#ifdef USE_WIFI + // Clients redirect to the first URL, so the interface the client just configured has to lead: + // another interface's address can be on a subnet that client cannot reach. + const auto append_wifi_urls = [&append_urls]() { + if (wifi::global_wifi_component != nullptr) + append_urls(wifi::global_wifi_component->get_ip_addresses()); + }; + if (wifi_first) + append_wifi_urls(); +#endif +#ifdef USE_ETHERNET + if (ethernet::global_eth_component != nullptr) + append_urls(ethernet::global_eth_component->get_ip_addresses()); +#endif +#ifdef USE_MODEM + if (modem::global_modem_component != nullptr) + append_urls(modem::global_modem_component->get_ip_addresses()); +#endif +#ifdef USE_WIFI + if (!wifi_first) + append_wifi_urls(); +#endif +} +#endif // USE_WEBSERVER + +void ImprovSerialComponent::send_settings_response_(improv::Command command) { + std::array buf; + improv::RpcResponseBuilder builder(buf, command); +#ifdef USE_IMPROV_SERIAL_NEXT_URL + this->add_next_url_(builder, MAX_NEXT_URL_LEN); +#endif +#ifdef USE_WEBSERVER + // This response only ever answers Wi-Fi provisioning, so lead with the Wi-Fi URL as it did + // before other interfaces were reported. + this->add_webserver_urls_(builder, /*wifi_first=*/true); #endif this->send_response_(builder.finish(false)); } @@ -231,7 +281,8 @@ bool ImprovSerialComponent::parse_improv_serial_byte_(uint8_t byte) { bool ImprovSerialComponent::parse_improv_payload_(improv::ImprovCommand &command) { switch (command.command) { case improv::WIFI_SETTINGS: { - if (wifi::global_wifi_component->is_disabled()) { +#ifdef USE_WIFI + if (wifi::global_wifi_component == nullptr || wifi::global_wifi_component->is_disabled()) { // Wi-Fi is disabled, so we can't provision. Respond immediately // instead of letting the client wait out its provisioning timeout. ESP_LOGW(TAG, "Wi-Fi is disabled; cannot provision"); @@ -243,21 +294,32 @@ bool ImprovSerialComponent::parse_improv_payload_(improv::ImprovCommand &command sta.set_password(command.password.c_str()); this->connecting_sta_ = sta; + // Sampled before start_connecting(): the old connection drops asynchronously after it. + const bool switching = wifi::global_wifi_component->is_connected(); wifi::global_wifi_component->set_sta(sta); wifi::global_wifi_component->start_connecting(sta); this->set_state_(improv::STATE_PROVISIONING); ESP_LOGD(TAG, "Received settings: SSID=%s, password=" LOG_SECRET("%s"), command.ssid.c_str(), command.password.c_str()); - this->set_timeout("wifi-connect-timeout", 30000, [this]() { this->on_wifi_connect_timeout_(); }); + this->set_timeout("wifi-connect-timeout", switching ? WIFI_SWITCH_TIMEOUT_MS : WIFI_CONNECT_TIMEOUT_MS, + [this]() { this->on_wifi_connect_timeout_(); }); +#else + // No Wi-Fi support compiled in; there is nothing to provision. + ESP_LOGW(TAG, "Wi-Fi not supported; cannot provision"); + this->set_error_(improv::ERROR_UNABLE_TO_CONNECT); +#endif return true; } - case improv::GET_CURRENT_STATE: - if (wifi::global_wifi_component->is_disabled()) { - // Wi-Fi is disabled; report the Improv "stopped" state so a client can tell - // the user that provisioning is unavailable. Reported transiently without - // disturbing our internal provisioning state machine, so a later `wifi.enable` - // still reports the correct state. + case improv::GET_CURRENT_STATE: { + // This state machine tracks Wi-Fi provisioning only. When Wi-Fi is disabled or not + // compiled in, provisioning is unavailable -> report STOPPED so the client doesn't + // offer a Wi-Fi form. General connectivity (e.g. Ethernet) is reported separately + // via GET_NETWORK_STATE. +#ifdef USE_WIFI + if (wifi::global_wifi_component == nullptr || wifi::global_wifi_component->is_disabled()) { + // Reported transiently without disturbing our internal provisioning state machine, + // so a later `wifi.enable` still reports the correct state. this->send_current_state_(improv::STATE_STOPPED); return true; } @@ -265,14 +327,20 @@ bool ImprovSerialComponent::parse_improv_payload_(improv::ImprovCommand &command if (this->state_ == improv::STATE_PROVISIONED) { this->send_settings_response_(improv::GET_CURRENT_STATE); } +#else + this->send_current_state_(improv::STATE_STOPPED); +#endif return true; + } case improv::GET_DEVICE_INFO: { this->send_version_info_(); return true; } case improv::GET_WIFI_NETWORKS: { - const auto &results = wifi::global_wifi_component->get_scan_result(); + // Declared out here because the terminating empty response is sent with or without Wi-Fi std::array buf; +#ifdef USE_WIFI + const auto &results = wifi::global_wifi_component->get_scan_result(); for (const auto &scan : results) { bool with_auth = false; if (!wifi::should_show_scan_entry(results, scan, with_auth)) @@ -289,11 +357,52 @@ bool ImprovSerialComponent::parse_improv_payload_(improv::ImprovCommand &command builder.add_string(YESNO(with_auth)); this->send_response_(builder.finish(false)); } +#endif // USE_WIFI // Send empty response to signify the end of the list. improv::RpcResponseBuilder builder(buf, improv::GET_WIFI_NETWORKS); this->send_response_(builder.finish(false)); return true; } + case improv::GET_NETWORK_STATE: { + // Reports general device connectivity and which network interfaces are present, decoupled + // from the Wi-Fi-only provisioning state machine. data[0] is a decimal flags byte; + // when online, the reachable device URL(s) follow. + uint8_t flags = 0; + if (network::is_connected()) + flags |= improv::NETWORK_IS_ONLINE; +#ifdef USE_WIFI + flags |= improv::NETWORK_SUPPORTS_WIFI; +#endif +#ifdef USE_ETHERNET + flags |= improv::NETWORK_SUPPORTS_ETHERNET; +#endif +#ifdef USE_OPENTHREAD + flags |= improv::NETWORK_SUPPORTS_THREAD; +#endif +#ifdef USE_MODEM + flags |= improv::NETWORK_SUPPORTS_MODEM; +#endif + std::array buf; + improv::RpcResponseBuilder builder(buf, improv::GET_NETWORK_STATE); + // Every flag bit fits int8_t's positive range, so int8_to_str renders the byte + static_assert(improv::NETWORK_SUPPORTS_MODEM <= 0x7F, "network flags no longer fit int8_to_str"); + char flags_buf[4]; // uint8_t: max "255" + null + char *flags_end = int8_to_str(flags_buf, static_cast(flags)); + builder.add_string(flags_buf, flags_end - flags_buf); +#ifdef USE_WEBSERVER + // Not tied to one interface, so follow the configured priority the way + // network::get_ip_addresses() does: a wifi-first network priority list leads with Wi-Fi. + if (flags & improv::NETWORK_IS_ONLINE) { +#if defined(USE_NETWORK_PRIMARY_INTERFACE_WIFI) && defined(USE_WIFI) + this->add_webserver_urls_(builder, /*wifi_first=*/true); +#else + this->add_webserver_urls_(builder, /*wifi_first=*/false); +#endif + } +#endif + this->send_response_(builder.finish(false)); + return true; + } default: { ESP_LOGW(TAG, "Unknown payload"); this->set_error_(improv::ERROR_UNKNOWN_RPC); @@ -331,12 +440,14 @@ void ImprovSerialComponent::send_response_(std::span response) { this->write_data_(response.data(), response.size()); } +#ifdef USE_WIFI void ImprovSerialComponent::on_wifi_connect_timeout_() { this->set_error_(improv::ERROR_UNABLE_TO_CONNECT); this->set_state_(improv::STATE_AUTHORIZED); ESP_LOGW(TAG, "Timed out while connecting to Wi-Fi network"); wifi::global_wifi_component->clear_sta(); } +#endif ImprovSerialComponent *global_improv_serial_component = // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) diff --git a/esphome/components/improv_serial/improv_serial_component.h b/esphome/components/improv_serial/improv_serial_component.h index 692873bbb6..68cdd75214 100644 --- a/esphome/components/improv_serial/improv_serial_component.h +++ b/esphome/components/improv_serial/improv_serial_component.h @@ -2,15 +2,19 @@ #include "esphome/components/improv_base/improv_base.h" #include "esphome/components/logger/logger.h" -#include "esphome/components/wifi/wifi_component.h" +#include "esphome/components/network/util.h" #include "esphome/core/component.h" #include "esphome/core/defines.h" #include "esphome/core/helpers.h" -#ifdef USE_WIFI +#ifdef USE_IMPROV_SERIAL #include #include #include +#ifdef USE_WIFI +#include "esphome/components/wifi/wifi_component.h" +#endif + #ifdef USE_IMPROV_SERIAL_UART #include "esphome/components/uart/uart_component.h" #elif defined(USE_ESP32) @@ -48,13 +52,22 @@ enum ImprovSerialType : uint8_t { static const uint16_t IMPROV_SERIAL_TIMEOUT = 100; static const uint8_t IMPROV_SERIAL_VERSION = 1; +#ifdef USE_WIFI +// Wi-Fi connect failure timers: a fresh provision reports at 30 s (stock behavior), while +// switching networks on an already-connected device (disconnect + reconnect) can legitimately +// take longer; 90 s matches esp32_improv's default wifi_timeout. +static const uint32_t WIFI_CONNECT_TIMEOUT_MS = 30000; +static const uint32_t WIFI_SWITCH_TIMEOUT_MS = 90000; +#endif + // The serial frame length field is one byte static constexpr size_t MAX_SERIAL_RESPONSE = 255; // command + data length + trailing byte static constexpr size_t RPC_RESPONSE_OVERHEAD = 3; static constexpr size_t MAX_SERIAL_PAYLOAD = MAX_SERIAL_RESPONSE - RPC_RESPONSE_OVERHEAD; #ifdef USE_WEBSERVER -// length byte + "http://" + IPv4 + ":" + port +// length byte + "http://" + IPv4 + ":" + port. Reserves the first URL only; a device with +// several interfaces online adds the rest best-effort and warns if one no longer fits. static constexpr size_t WEBSERVER_URL_RESERVE = 1 + 7 + 15 + 1 + 5; #else static constexpr size_t WEBSERVER_URL_RESERVE = 0; @@ -84,8 +97,15 @@ class ImprovSerialComponent final : public Component, public improv_base::Improv void send_current_state_(improv::State state); void set_error_(improv::Error error); void send_response_(std::span response); +#ifdef USE_WIFI void on_wifi_connect_timeout_(); +#endif +#ifdef USE_WEBSERVER + /// Append one web server URL per interface that has a usable IPv4. With wifi_first the Wi-Fi + /// URL leads, for responses to Wi-Fi provisioning; otherwise interfaces go in priority order. + void add_webserver_urls_(improv::RpcResponseBuilder &builder, [[maybe_unused]] bool wifi_first); +#endif void send_settings_response_(improv::Command command); void send_version_info_(); @@ -167,7 +187,9 @@ class ImprovSerialComponent final : public Component, public improv_base::Improv std::vector rx_buffer_; uint32_t last_read_byte_{0}; +#ifdef USE_WIFI wifi::WiFiAP connecting_sta_; +#endif improv::State state_{improv::STATE_AUTHORIZED}; }; diff --git a/tests/components/improv_serial/common-ethernet.yaml b/tests/components/improv_serial/common-ethernet.yaml new file mode 100644 index 0000000000..c1d8190c13 --- /dev/null +++ b/tests/components/improv_serial/common-ethernet.yaml @@ -0,0 +1,17 @@ +ethernet: + type: W5500 + clk_pin: 19 + mosi_pin: 21 + miso_pin: 17 + cs_pin: 18 + interrupt_pin: 36 + reset_pin: 12 + clock_speed: 10Mhz + +logger: + hardware_uart: UART0 + +# Exercises the per-interface webserver URL collection at compile time +web_server: + +improv_serial: diff --git a/tests/components/improv_serial/test-ethernet.esp32-idf.yaml b/tests/components/improv_serial/test-ethernet.esp32-idf.yaml new file mode 100644 index 0000000000..2dd3a1551e --- /dev/null +++ b/tests/components/improv_serial/test-ethernet.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + improv_serial: !include common-ethernet.yaml diff --git a/tests/integration/fixtures/external_components/wifi/wifi_component.cpp b/tests/integration/fixtures/external_components/wifi/wifi_component.cpp index d1e19a1a0a..b29b1a2bd1 100644 --- a/tests/integration/fixtures/external_components/wifi/wifi_component.cpp +++ b/tests/integration/fixtures/external_components/wifi/wifi_component.cpp @@ -29,6 +29,8 @@ void WiFiComponent::set_sta(const WiFiAP &ap) { ESP_LOGI(TAG, "set_sta ssid=%s", void WiFiComponent::start_connecting(const WiFiAP &ap) { ESP_LOGI(TAG, "start_connecting ssid=%s", ap.get_ssid().c_str()); + // Connecting succeeds immediately, so the requested network is the connected one + this->connected_ssid_ = ap.get_ssid().c_str(); } void WiFiComponent::clear_sta() { ESP_LOGI(TAG, "clear_sta"); } diff --git a/tests/integration/fixtures/external_components/wifi/wifi_component.h b/tests/integration/fixtures/external_components/wifi/wifi_component.h index a68f811ebd..6fe6e84e9d 100644 --- a/tests/integration/fixtures/external_components/wifi/wifi_component.h +++ b/tests/integration/fixtures/external_components/wifi/wifi_component.h @@ -13,11 +13,15 @@ #include "esphome/core/component.h" #include "esphome/core/string_ref.h" +#include +#include #include #include namespace esphome::wifi { +static constexpr size_t SSID_BUFFER_SIZE = 33; + class WiFiAP { public: void set_ssid(const char *ssid) { this->ssid_ = ssid; } @@ -58,6 +62,12 @@ class WiFiComponent : public Component { bool is_disabled() const { return false; } // Always connected so network::is_connected() keeps the API server accepting clients bool is_connected() const { return true; } + // Reports the network start_connecting() was last asked for, so a consumer checking that it + // joined the network it requested (rather than an earlier one) sees the connect succeed + const char *wifi_ssid_to(std::span buffer) { + snprintf(buffer.data(), buffer.size(), "%s", this->connected_ssid_.c_str()); + return buffer.data(); + } void start_scanning(); const std::vector &get_scan_result() const { return this->scan_result_; } void set_sta(const WiFiAP &ap); @@ -70,6 +80,7 @@ class WiFiComponent : public Component { protected: std::vector scan_result_; + std::string connected_ssid_; }; extern WiFiComponent *global_wifi_component; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) From 404da26a373885c23314763ca675029216be933f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 31 Aug 2026 04:08:05 -0500 Subject: [PATCH 047/147] [core] Keep conditional log string literals in flash on ESP8266 (#18907) --- esphome/components/aqi/aqi_sensor.cpp | 6 ++- .../components/binary_sensor/automation.cpp | 6 ++- .../components/bme680_bsec/bme680_bsec.cpp | 5 ++- esphome/components/cs5460a/cs5460a.cpp | 13 +++--- esphome/components/dht/dht.cpp | 4 +- esphome/components/emc2101/emc2101.cpp | 2 +- esphome/components/ens210/ens210.cpp | 2 +- .../components/esphome/ota/ota_esphome.cpp | 3 +- .../components/feedback/feedback_cover.cpp | 17 ++++---- .../fingerprint_grow/fingerprint_grow.cpp | 4 +- .../graphical_display_menu.cpp | 26 ++++++------ .../gt911/touchscreen/gt911_touchscreen.cpp | 6 ++- esphome/components/haier/haier_base.cpp | 3 +- esphome/components/haier/hon_climate.cpp | 10 ++--- esphome/components/hdc302x/hdc302x.cpp | 2 +- esphome/components/he60r/he60r.cpp | 8 ++-- esphome/components/hlk_fm22x/hlk_fm22x.cpp | 3 +- esphome/components/hlw8012/hlw8012.cpp | 3 +- .../components/ili9xxx/ili9xxx_display.cpp | 4 +- .../components/ina2xx_base/ina2xx_base.cpp | 3 +- esphome/components/it8951/it8951.cpp | 42 ++++++++++--------- esphome/components/lc709203f/lc709203f.cpp | 3 +- esphome/components/ld2420/ld2420.cpp | 3 +- esphome/components/ld6002b/ld6002b.cpp | 3 +- esphome/components/max31856/max31856.cpp | 6 ++- esphome/components/max31865/max31865.cpp | 14 ++++--- esphome/components/mcp4461/mcp4461.cpp | 5 ++- .../components/media_player/media_player.cpp | 2 +- esphome/components/mipi_spi/mipi_spi.cpp | 3 +- .../modbus_server/modbus_server.cpp | 8 ++-- esphome/components/nextion/nextion.cpp | 5 ++- esphome/components/pcm5122/pcm5122.cpp | 3 +- esphome/components/pn7150/pn7150.cpp | 4 +- esphome/components/pn7160/pn7160.cpp | 4 +- esphome/components/pylontech/pylontech.cpp | 3 +- esphome/components/rd03d/rd03d.cpp | 5 ++- .../remote_receiver/remote_receiver.cpp | 19 +++++---- .../resistance/resistance_sensor.cpp | 4 +- esphome/components/sen21231/sen21231.cpp | 2 +- esphome/components/senseair/senseair.cpp | 5 ++- .../components/serial_proxy/serial_proxy.cpp | 10 ++--- esphome/components/sgp4x/sgp4x.cpp | 2 +- esphome/components/sprinkler/sprinkler.cpp | 2 +- esphome/components/switch/switch.cpp | 3 +- esphome/components/sx127x/sx127x.cpp | 5 ++- .../thermostat/thermostat_climate.cpp | 8 ++-- esphome/components/tsl2591/tsl2591.cpp | 2 +- .../components/tuya/select/tuya_select.cpp | 2 +- esphome/components/tuya/tuya.cpp | 2 +- esphome/components/veml7700/veml7700.cpp | 8 ++-- esphome/components/vl53l0x/vl53l0x_sensor.cpp | 3 +- .../components/water_heater/water_heater.cpp | 7 ++-- esphome/components/weikai/weikai.cpp | 14 +++---- esphome/components/whirlpool/whirlpool.cpp | 2 +- esphome/components/wifi/wifi_component.cpp | 10 +++-- esphome/components/wireguard/wireguard.cpp | 9 ++-- esphome/components/wl_134/wl_134.cpp | 4 +- .../components/zwave_proxy/zwave_proxy.cpp | 13 +++--- 58 files changed, 214 insertions(+), 165 deletions(-) diff --git a/esphome/components/aqi/aqi_sensor.cpp b/esphome/components/aqi/aqi_sensor.cpp index 4bb964d5ee..e78f301b30 100644 --- a/esphome/components/aqi/aqi_sensor.cpp +++ b/esphome/components/aqi/aqi_sensor.cpp @@ -23,8 +23,10 @@ void AQISensor::setup() { void AQISensor::dump_config() { ESP_LOGCONFIG(TAG, "AQI Sensor:"); - ESP_LOGCONFIG(TAG, " Calculation Type: %s", this->aqi_calc_type_ == AQI_TYPE ? "AQI" : "CAQI"); - ESP_LOGCONFIG(TAG, " Extended Range: %s", this->extended_range_ ? "enabled" : "disabled"); + ESP_LOGCONFIG(TAG, " Calculation Type: %s", + this->aqi_calc_type_ == AQI_TYPE ? LOG_STR_LITERAL("AQI") : LOG_STR_LITERAL("CAQI")); + ESP_LOGCONFIG(TAG, " Extended Range: %s", + this->extended_range_ ? LOG_STR_LITERAL("enabled") : LOG_STR_LITERAL("disabled")); if (this->pm_2_5_sensor_ != nullptr) { ESP_LOGCONFIG(TAG, " PM2.5 Sensor: '%s'", this->pm_2_5_sensor_->get_name().c_str()); } diff --git a/esphome/components/binary_sensor/automation.cpp b/esphome/components/binary_sensor/automation.cpp index 1a3c1f7536..65c7dbbdb6 100644 --- a/esphome/components/binary_sensor/automation.cpp +++ b/esphome/components/binary_sensor/automation.cpp @@ -100,13 +100,15 @@ void MultiClickTriggerBase::schedule_is_valid_(uint32_t min_length) { } this->is_valid_ = false; this->set_timeout(MULTICLICK_IS_VALID_ID, min_length, [this]() { - ESP_LOGV(TAG, "Multi Click: You can now %s the button.", this->parent_->state ? "RELEASE" : "PRESS"); + ESP_LOGV(TAG, "Multi Click: You can now %s the button.", + this->parent_->state ? LOG_STR_LITERAL("RELEASE") : LOG_STR_LITERAL("PRESS")); this->is_valid_ = true; }); } void MultiClickTriggerBase::schedule_is_not_valid_(uint32_t max_length) { this->set_timeout(MULTICLICK_IS_NOT_VALID_ID, max_length, [this]() { - ESP_LOGV(TAG, "Multi Click: You waited too long to %s.", this->parent_->state ? "RELEASE" : "PRESS"); + ESP_LOGV(TAG, "Multi Click: You waited too long to %s.", + this->parent_->state ? LOG_STR_LITERAL("RELEASE") : LOG_STR_LITERAL("PRESS")); this->is_valid_ = false; this->schedule_cooldown_(); }); diff --git a/esphome/components/bme680_bsec/bme680_bsec.cpp b/esphome/components/bme680_bsec/bme680_bsec.cpp index 823f32c446..8e16a28e33 100644 --- a/esphome/components/bme680_bsec/bme680_bsec.cpp +++ b/esphome/components/bme680_bsec/bme680_bsec.cpp @@ -162,8 +162,9 @@ void BME680BSECComponent::dump_config() { " Supply Voltage: %sV\n" " Sample Rate: %s\n" " State Save Interval: %" PRIu32 "ms", - this->temperature_offset_, this->iaq_mode_ == IAQ_MODE_STATIC ? "Static" : "Mobile", - this->supply_voltage_ == SUPPLY_VOLTAGE_3V3 ? "3.3" : "1.8", + this->temperature_offset_, + this->iaq_mode_ == IAQ_MODE_STATIC ? LOG_STR_LITERAL("Static") : LOG_STR_LITERAL("Mobile"), + this->supply_voltage_ == SUPPLY_VOLTAGE_3V3 ? LOG_STR_LITERAL("3.3") : LOG_STR_LITERAL("1.8"), BME680_BSEC_SAMPLE_RATE_LOG(this->sample_rate_), this->state_save_interval_ms_); LOG_SENSOR(" ", "Temperature", this->temperature_sensor_); diff --git a/esphome/components/cs5460a/cs5460a.cpp b/esphome/components/cs5460a/cs5460a.cpp index c9e8f3cf47..1f9233841a 100644 --- a/esphome/components/cs5460a/cs5460a.cpp +++ b/esphome/components/cs5460a/cs5460a.cpp @@ -249,7 +249,7 @@ bool CS5460AComponent::check_status_() { bool dir = status & (1 << 21); if (current_gain_ < 0) dir = !dir; - ESP_LOGI(TAG, "Energy counter %s pulse", dir ? "negative" : "positive"); + ESP_LOGI(TAG, "Energy counter %s pulse", dir ? LOG_STR_LITERAL("negative") : LOG_STR_LITERAL("positive")); clear |= 1 << 22; } @@ -319,7 +319,9 @@ void CS5460AComponent::dump_config() { ESP_LOGCONFIG(TAG, "CS5460A:\n" " Init status: %s", - state == COMPONENT_STATE_LOOP ? "OK" : (state == COMPONENT_STATE_FAILED ? "failed" : "other")); + state == COMPONENT_STATE_LOOP + ? LOG_STR_LITERAL("OK") + : (state == COMPONENT_STATE_FAILED ? LOG_STR_LITERAL("failed") : LOG_STR_LITERAL("other"))); LOG_PIN(" CS Pin: ", cs_); ESP_LOGCONFIG(TAG, " Samples / cycle: %" PRIu32 "\n" @@ -330,9 +332,10 @@ void CS5460AComponent::dump_config() { " Current HPF: %s\n" " Voltage HPF: %s\n" " Pulse energy: %.2f Wh", - samples_, phase_offset_, pga_gain_ == CS5460A_PGA_GAIN_50X ? "50x" : "10x", current_gain_, - voltage_gain_, current_hpf_ ? "enabled" : "disabled", voltage_hpf_ ? "enabled" : "disabled", - pulse_energy_wh_); + samples_, phase_offset_, + pga_gain_ == CS5460A_PGA_GAIN_50X ? LOG_STR_LITERAL("50x") : LOG_STR_LITERAL("10x"), current_gain_, + voltage_gain_, current_hpf_ ? LOG_STR_LITERAL("enabled") : LOG_STR_LITERAL("disabled"), + voltage_hpf_ ? LOG_STR_LITERAL("enabled") : LOG_STR_LITERAL("disabled"), pulse_energy_wh_); LOG_SENSOR(" ", "Voltage", voltage_sensor_); LOG_SENSOR(" ", "Current", current_sensor_); LOG_SENSOR(" ", "Power", power_sensor_); diff --git a/esphome/components/dht/dht.cpp b/esphome/components/dht/dht.cpp index 5b7b6a268f..a9117be4e1 100644 --- a/esphome/components/dht/dht.cpp +++ b/esphome/components/dht/dht.cpp @@ -20,8 +20,8 @@ void DHT::dump_config() { "DHT:\n" " %sModel: %s\n" " Internal pull-up: %s", - this->is_auto_detect_ ? "Auto-detected " : "", - this->model_ == DHT_MODEL_DHT11 ? "DHT11" : "DHT22 or equivalent", + this->is_auto_detect_ ? LOG_STR_LITERAL("Auto-detected ") : "", + this->model_ == DHT_MODEL_DHT11 ? LOG_STR_LITERAL("DHT11") : LOG_STR_LITERAL("DHT22 or equivalent"), ONOFF(this->t_pin_->get_flags() & gpio::FLAG_PULLUP)); LOG_PIN(" Pin: ", this->t_pin_); LOG_UPDATE_INTERVAL(this); diff --git a/esphome/components/emc2101/emc2101.cpp b/esphome/components/emc2101/emc2101.cpp index f46082f5e7..bb041bfd6d 100644 --- a/esphome/components/emc2101/emc2101.cpp +++ b/esphome/components/emc2101/emc2101.cpp @@ -93,7 +93,7 @@ void Emc2101Component::dump_config() { if (this->is_failed()) { ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); } - ESP_LOGCONFIG(TAG, " Mode: %s", this->dac_mode_ ? "DAC" : "PWM"); + ESP_LOGCONFIG(TAG, " Mode: %s", this->dac_mode_ ? LOG_STR_LITERAL("DAC") : LOG_STR_LITERAL("PWM")); if (this->dac_mode_) { ESP_LOGCONFIG(TAG, " DAC Conversion Rate: %X", this->dac_conversion_rate_); } else { diff --git a/esphome/components/ens210/ens210.cpp b/esphome/components/ens210/ens210.cpp index 468c627d4b..11b73afe37 100644 --- a/esphome/components/ens210/ens210.cpp +++ b/esphome/components/ens210/ens210.cpp @@ -216,7 +216,7 @@ void ENS210Component::extract_measurement_(uint32_t val, int *data, int *status) // Sets ENS210 to low (true) or high (false) power. Returns false on I2C problems. bool ENS210Component::set_low_power_(bool enable) { uint8_t low_power_cmd = enable ? 0x01 : 0x00; - ESP_LOGD(TAG, "Enable low power: %s", enable ? "true" : "false"); + ESP_LOGD(TAG, "Enable low power: %s", enable ? LOG_STR_LITERAL("true") : LOG_STR_LITERAL("false")); bool result = this->write_byte(ENS210_REGISTER_SYS_CTRL, low_power_cmd); delay(ENS210_BOOTING_MS); return result; diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 74f84b71fb..9f15eaaede 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -128,7 +128,8 @@ void ESPHomeOTAComponent::dump_config() { esp_partition_iterator_release(it); esp_bootloader_desc_t bootloader_desc; esp_err_t err = esp_ota_get_bootloader_description(nullptr, &bootloader_desc); - ESP_LOGCONFIG(TAG, " Bootloader: ESP-IDF %s", (err == ESP_OK) ? bootloader_desc.idf_ver : "version unknown"); + ESP_LOGCONFIG(TAG, " Bootloader: ESP-IDF %s", + (err == ESP_OK) ? bootloader_desc.idf_ver : LOG_STR_LITERAL("version unknown")); #endif // USE_ESP32 #endif // USE_OTA_PARTITIONS } diff --git a/esphome/components/feedback/feedback_cover.cpp b/esphome/components/feedback/feedback_cover.cpp index 1139e6fa18..4baffc74f8 100644 --- a/esphome/components/feedback/feedback_cover.cpp +++ b/esphome/components/feedback/feedback_cover.cpp @@ -93,7 +93,8 @@ void FeedbackCover::set_open_sensor(binary_sensor::BinarySensor *open_feedback) // setup callbacks to react to sensor changes open_feedback->add_on_state_callback([this](bool state) { - ESP_LOGD(TAG, "'%s' - Open feedback '%s'.", this->name_.c_str(), state ? "STARTED" : "ENDED"); + ESP_LOGD(TAG, "'%s' - Open feedback '%s'.", this->name_.c_str(), + state ? LOG_STR_LITERAL("STARTED") : LOG_STR_LITERAL("ENDED")); this->recompute_position_(); if (!state && this->infer_endstop_ && this->current_trigger_operation_ == COVER_OPERATION_OPENING) { this->endstop_reached_(true); @@ -106,7 +107,8 @@ void FeedbackCover::set_close_sensor(binary_sensor::BinarySensor *close_feedback this->close_feedback_ = close_feedback; close_feedback->add_on_state_callback([this](bool state) { - ESP_LOGD(TAG, "'%s' - Close feedback '%s'.", this->name_.c_str(), state ? "STARTED" : "ENDED"); + ESP_LOGD(TAG, "'%s' - Close feedback '%s'.", this->name_.c_str(), + state ? LOG_STR_LITERAL("STARTED") : LOG_STR_LITERAL("ENDED")); this->recompute_position_(); if (!state && this->infer_endstop_ && this->current_trigger_operation_ == COVER_OPERATION_CLOSING) { this->endstop_reached_(false); @@ -144,7 +146,8 @@ void FeedbackCover::endstop_reached_(bool open_endstop) { // from a position slightly past the endpoint if (this->current_trigger_operation_ == (open_endstop ? COVER_OPERATION_OPENING : COVER_OPERATION_CLOSING)) { float dur = (now - this->start_dir_time_) / 1e3f; - ESP_LOGD(TAG, "'%s' - %s endstop reached. Took %.1fs.", this->name_.c_str(), open_endstop ? "Open" : "Close", dur); + ESP_LOGD(TAG, "'%s' - %s endstop reached. Took %.1fs.", this->name_.c_str(), + open_endstop ? LOG_STR_LITERAL("Open") : LOG_STR_LITERAL("Close"), dur); // if there is no external mechanism, stop the cover if (!this->has_built_in_endstop_) { @@ -366,7 +369,7 @@ void FeedbackCover::start_direction_(CoverOperation dir) { // the case when an obstacle appears while moving is handled in the callback if (obstacle != nullptr && obstacle->state) { ESP_LOGD(TAG, "'%s' - %s obstacle detected. Action not started.", this->name_.c_str(), - dir == COVER_OPERATION_OPENING ? "Open" : "Close"); + dir == COVER_OPERATION_OPENING ? LOG_STR_LITERAL("Open") : LOG_STR_LITERAL("Close")); return; } #endif @@ -383,9 +386,9 @@ void FeedbackCover::start_direction_(CoverOperation dir) { this->set_current_operation_(dir, true); this->prev_command_trigger_ = trig; ESP_LOGD(TAG, "'%s' - Firing '%s' trigger.", this->name_.c_str(), - dir == COVER_OPERATION_OPENING ? "OPEN" - : dir == COVER_OPERATION_CLOSING ? "CLOSE" - : "STOP"); + dir == COVER_OPERATION_OPENING ? LOG_STR_LITERAL("OPEN") + : dir == COVER_OPERATION_CLOSING ? LOG_STR_LITERAL("CLOSE") + : LOG_STR_LITERAL("STOP")); trig->trigger(); } } diff --git a/esphome/components/fingerprint_grow/fingerprint_grow.cpp b/esphome/components/fingerprint_grow/fingerprint_grow.cpp index b38d42191b..07630f121a 100644 --- a/esphome/components/fingerprint_grow/fingerprint_grow.cpp +++ b/esphome/components/fingerprint_grow/fingerprint_grow.cpp @@ -547,8 +547,8 @@ void FingerprintGrowComponent::dump_config() { " System Identifier Code: 0x%.4X\n" " Touch Sensing Pin: %s\n" " Sensor Power Pin: %s", - this->system_identifier_code_, this->has_sensing_pin_ ? sensing_pin_buf : "None", - this->has_power_pin_ ? power_pin_buf : "None"); + this->system_identifier_code_, this->has_sensing_pin_ ? sensing_pin_buf : LOG_STR_LITERAL("None"), + this->has_power_pin_ ? power_pin_buf : LOG_STR_LITERAL("None")); if (this->idle_period_to_sleep_ms_ < UINT32_MAX) { ESP_LOGCONFIG(TAG, " Idle Period to Sleep: %" PRIu32 " ms", this->idle_period_to_sleep_ms_); } else { diff --git a/esphome/components/graphical_display_menu/graphical_display_menu.cpp b/esphome/components/graphical_display_menu/graphical_display_menu.cpp index b3c3b27e06..f0642d2e8c 100644 --- a/esphome/components/graphical_display_menu/graphical_display_menu.cpp +++ b/esphome/components/graphical_display_menu/graphical_display_menu.cpp @@ -35,18 +35,20 @@ void GraphicalDisplayMenu::setup() { } void GraphicalDisplayMenu::dump_config() { - ESP_LOGCONFIG(TAG, - "Graphical Display Menu\n" - " Has Display: %s\n" - " Popup Mode: %s\n" - " Advanced Drawing Mode: %s\n" - " Has Font: %s\n" - " Mode: %s\n" - " Active: %s\n" - " Menu items:", - YESNO(this->display_ != nullptr), YESNO(this->display_ != nullptr), YESNO(this->display_ == nullptr), - YESNO(this->font_ != nullptr), - this->mode_ == display_menu_base::MENU_MODE_ROTARY ? "Rotary" : "Joystick", YESNO(this->active_)); + ESP_LOGCONFIG( + TAG, + "Graphical Display Menu\n" + " Has Display: %s\n" + " Popup Mode: %s\n" + " Advanced Drawing Mode: %s\n" + " Has Font: %s\n" + " Mode: %s\n" + " Active: %s\n" + " Menu items:", + YESNO(this->display_ != nullptr), YESNO(this->display_ != nullptr), YESNO(this->display_ == nullptr), + YESNO(this->font_ != nullptr), + this->mode_ == display_menu_base::MENU_MODE_ROTARY ? LOG_STR_LITERAL("Rotary") : LOG_STR_LITERAL("Joystick"), + YESNO(this->active_)); for (size_t i = 0; i < this->displayed_item_->items_size(); i++) { auto *item = this->displayed_item_->get_item(i); ESP_LOGCONFIG(TAG, " %i: %s (Type: %s, Immediate Edit: %s)", i, item->get_text().c_str(), diff --git a/esphome/components/gt911/touchscreen/gt911_touchscreen.cpp b/esphome/components/gt911/touchscreen/gt911_touchscreen.cpp index 2152ae7b84..8ced267947 100644 --- a/esphome/components/gt911/touchscreen/gt911_touchscreen.cpp +++ b/esphome/components/gt911/touchscreen/gt911_touchscreen.cpp @@ -69,10 +69,12 @@ void GT911Touchscreen::setup_internal_() { // Direct MCU pin: attach a hardware interrupt, no polling needed. this->attach_interrupt_(static_cast(this->interrupt_pin_), active_high ? gpio::INTERRUPT_RISING_EDGE : gpio::INTERRUPT_FALLING_EDGE); - ESP_LOGD(TAG, "Interrupt pin: hardware interrupt, active %s", active_high ? "HIGH" : "LOW"); + ESP_LOGD(TAG, "Interrupt pin: hardware interrupt, active %s", + active_high ? LOG_STR_LITERAL("HIGH") : LOG_STR_LITERAL("LOW")); } else { // IO expander pin: leave as output for configuration only. - ESP_LOGD(TAG, "Interrupt pin: IO expander polling mode, active %s", active_high ? "HIGH" : "LOW"); + ESP_LOGD(TAG, "Interrupt pin: IO expander polling mode, active %s", + active_high ? LOG_STR_LITERAL("HIGH") : LOG_STR_LITERAL("LOW")); } } } diff --git a/esphome/components/haier/haier_base.cpp b/esphome/components/haier/haier_base.cpp index 294aa53b03..48f72dc16b 100644 --- a/esphome/components/haier/haier_base.cpp +++ b/esphome/components/haier/haier_base.cpp @@ -248,7 +248,8 @@ void HaierClimateBase::setup() { void HaierClimateBase::dump_config() { LOG_CLIMATE("", "Haier Climate", this); - ESP_LOGCONFIG(TAG, " Device communication status: %s", this->valid_connection() ? "established" : "none"); + ESP_LOGCONFIG(TAG, " Device communication status: %s", + this->valid_connection() ? LOG_STR_LITERAL("established") : LOG_STR_LITERAL("none")); } void HaierClimateBase::loop() { diff --git a/esphome/components/haier/hon_climate.cpp b/esphome/components/haier/hon_climate.cpp index 881a2328cb..0ce4142fd4 100644 --- a/esphome/components/haier/hon_climate.cpp +++ b/esphome/components/haier/hon_climate.cpp @@ -343,11 +343,11 @@ void HonClimate::dump_config() { this->hvac_hardware_info_.value().software_version_, this->hvac_hardware_info_.value().hardware_version_, this->hvac_hardware_info_.value().device_name_); ESP_LOGCONFIG(TAG, " Device features:%s%s%s%s%s", - (this->hvac_hardware_info_.value().functions_[0] ? " interactive" : ""), - (this->hvac_hardware_info_.value().functions_[1] ? " controller-device" : ""), - (this->hvac_hardware_info_.value().functions_[2] ? " crc" : ""), - (this->hvac_hardware_info_.value().functions_[3] ? " multinode" : ""), - (this->hvac_hardware_info_.value().functions_[4] ? " role" : "")); + (this->hvac_hardware_info_.value().functions_[0] ? LOG_STR_LITERAL(" interactive") : ""), + (this->hvac_hardware_info_.value().functions_[1] ? LOG_STR_LITERAL(" controller-device") : ""), + (this->hvac_hardware_info_.value().functions_[2] ? LOG_STR_LITERAL(" crc") : ""), + (this->hvac_hardware_info_.value().functions_[3] ? LOG_STR_LITERAL(" multinode") : ""), + (this->hvac_hardware_info_.value().functions_[4] ? LOG_STR_LITERAL(" role") : "")); ESP_LOGCONFIG(TAG, " Active alarms: %s", buf_to_hex(this->active_alarms_, sizeof(this->active_alarms_)).c_str()); } } diff --git a/esphome/components/hdc302x/hdc302x.cpp b/esphome/components/hdc302x/hdc302x.cpp index b50d34169a..53d4c7f016 100644 --- a/esphome/components/hdc302x/hdc302x.cpp +++ b/esphome/components/hdc302x/hdc302x.cpp @@ -38,7 +38,7 @@ void HDC302XComponent::dump_config() { ESP_LOGCONFIG(TAG, "HDC302x:\n" " Heater: %s", - this->heater_active_ ? "active" : "inactive"); + this->heater_active_ ? LOG_STR_LITERAL("active") : LOG_STR_LITERAL("inactive")); LOG_I2C_DEVICE(this); LOG_UPDATE_INTERVAL(this); LOG_SENSOR(" ", "Temperature", this->temp_sensor_); diff --git a/esphome/components/he60r/he60r.cpp b/esphome/components/he60r/he60r.cpp index 84edbb2866..ea662e3ba9 100644 --- a/esphome/components/he60r/he60r.cpp +++ b/esphome/components/he60r/he60r.cpp @@ -59,7 +59,7 @@ void HE60rCover::endstop_reached_(CoverOperation operation) { if (this->last_command_ == operation) { float dur = (float) (now - this->start_dir_time_) / 1e3f; ESP_LOGD(TAG, "'%s' - %s endstop reached. Took %.1fs.", this->name_.c_str(), - operation == COVER_OPERATION_OPENING ? "Open" : "Close", dur); + operation == COVER_OPERATION_OPENING ? LOG_STR_LITERAL("Open") : LOG_STR_LITERAL("Close"), dur); } this->publish_state(); } @@ -213,9 +213,9 @@ void HE60rCover::start_direction_(CoverOperation dir) { if (this->current_operation == dir) return; ESP_LOGD(TAG, "'%s' - Direction '%s' requested.", this->name_.c_str(), - dir == COVER_OPERATION_OPENING ? "OPEN" - : dir == COVER_OPERATION_CLOSING ? "CLOSE" - : "STOP"); + dir == COVER_OPERATION_OPENING ? LOG_STR_LITERAL("OPEN") + : dir == COVER_OPERATION_CLOSING ? LOG_STR_LITERAL("CLOSE") + : LOG_STR_LITERAL("STOP")); if (dir == this->next_direction_) { // either moving and needs to stop, or stopped and will move correctly on one trigger diff --git a/esphome/components/hlk_fm22x/hlk_fm22x.cpp b/esphome/components/hlk_fm22x/hlk_fm22x.cpp index 964d26dfbc..a924259802 100644 --- a/esphome/components/hlk_fm22x/hlk_fm22x.cpp +++ b/esphome/components/hlk_fm22x/hlk_fm22x.cpp @@ -336,7 +336,8 @@ void HlkFm22xComponent::dump_config() { } if (this->enrolling_binary_sensor_) { LOG_BINARY_SENSOR(" ", "Enrolling", this->enrolling_binary_sensor_); - ESP_LOGCONFIG(TAG, " Current Value: %s", this->enrolling_binary_sensor_->state ? "ON" : "OFF"); + ESP_LOGCONFIG(TAG, " Current Value: %s", + this->enrolling_binary_sensor_->state ? LOG_STR_LITERAL("ON") : LOG_STR_LITERAL("OFF")); } if (this->face_count_sensor_) { LOG_SENSOR(" ", "Face Count", this->face_count_sensor_); diff --git a/esphome/components/hlw8012/hlw8012.cpp b/esphome/components/hlw8012/hlw8012.cpp index c92c76a20a..9ef81f075d 100644 --- a/esphome/components/hlw8012/hlw8012.cpp +++ b/esphome/components/hlw8012/hlw8012.cpp @@ -97,7 +97,8 @@ void HLW8012Component::update() { if (this->change_mode_every_ != 0 && this->change_mode_at_++ == this->change_mode_every_) { this->current_mode_ = !this->current_mode_; - ESP_LOGV(TAG, "Changing mode to %s mode", this->current_mode_ ? "CURRENT" : "VOLTAGE"); + ESP_LOGV(TAG, "Changing mode to %s mode", + this->current_mode_ ? LOG_STR_LITERAL("CURRENT") : LOG_STR_LITERAL("VOLTAGE")); this->change_mode_at_ = 0; this->sel_pin_->digital_write(this->current_mode_); } diff --git a/esphome/components/ili9xxx/ili9xxx_display.cpp b/esphome/components/ili9xxx/ili9xxx_display.cpp index e8840c0cf1..0ed18c45da 100644 --- a/esphome/components/ili9xxx/ili9xxx_display.cpp +++ b/esphome/components/ili9xxx/ili9xxx_display.cpp @@ -116,8 +116,8 @@ void ILI9XXXDisplay::dump_config() { " Mirror_x: %s\n" " Mirror_y: %s\n" " Invert colors: %s", - this->color_order_ == display::COLOR_ORDER_BGR ? "BGR" : "RGB", YESNO(this->swap_xy_), - YESNO(this->mirror_x_), YESNO(this->mirror_y_), YESNO(this->pre_invertcolors_)); + this->color_order_ == display::COLOR_ORDER_BGR ? LOG_STR_LITERAL("BGR") : LOG_STR_LITERAL("RGB"), + YESNO(this->swap_xy_), YESNO(this->mirror_x_), YESNO(this->mirror_y_), YESNO(this->pre_invertcolors_)); if (this->is_failed()) { ESP_LOGCONFIG(TAG, " => Failed to init Memory: YES!"); diff --git a/esphome/components/ina2xx_base/ina2xx_base.cpp b/esphome/components/ina2xx_base/ina2xx_base.cpp index d3acf00eef..fec5cd2f13 100644 --- a/esphome/components/ina2xx_base/ina2xx_base.cpp +++ b/esphome/components/ina2xx_base/ina2xx_base.cpp @@ -209,7 +209,8 @@ void INA2XX::dump_config() { " CURRENT_LSB = %f\n" " SHUNT_CAL = %d", this->shunt_resistance_ohm_, this->max_current_a_, this->shunt_tempco_ppm_c_, - (uint8_t) this->adc_range_, this->adc_range_ ? "±40.96 mV" : "±163.84 mV", this->current_lsb_, + (uint8_t) this->adc_range_, + this->adc_range_ ? LOG_STR_LITERAL("±40.96 mV") : LOG_STR_LITERAL("±163.84 mV"), this->current_lsb_, this->shunt_cal_); ESP_LOGCONFIG(TAG, " ADC Samples = %d; ADC times: Bus = %d μs, Shunt = %d μs, Temp = %d μs", diff --git a/esphome/components/it8951/it8951.cpp b/esphome/components/it8951/it8951.cpp index cc2bddeda7..179c2e5f63 100644 --- a/esphome/components/it8951/it8951.cpp +++ b/esphome/components/it8951/it8951.cpp @@ -740,7 +740,7 @@ bool IT8951Display::prepare_update_region_(UpdateMode &mode) { this->reset_dirty_region_(); ESP_LOGV(TAG, "Update: %ux%u@%u,%u mode=%u (%s)", width, height, x, y, static_cast(mode), - this->grayscale_ ? "grayscale" : "mono"); + this->grayscale_ ? LOG_STR_LITERAL("grayscale") : LOG_STR_LITERAL("mono")); return true; } @@ -1063,25 +1063,27 @@ void IT8951Display::dump_config() { strncpy(force_temperature, "(controller default)", sizeof(force_temperature)); force_temperature[sizeof(force_temperature) - 1] = '\0'; } - ESP_LOGCONFIG(TAG, - " Model preset: %s" - "\n Dimensions: %dx%d" - "\n Buffer: %u bytes" - "\n Image buffer addr: 0x%04X%04X" - "\n VCOM: %.02fV (set selector 0x%04X)" - "\n Force temperature: %s" - "\n Display command: %s" - "\n Sleep when done: %s" - "\n Full update every: %u" - "\n Inverted colors: %s" - "\n Pixel format: %s" - "\n Reset duration: %" PRIu32 "ms", - this->name_ != nullptr ? this->name_ : "(unknown)", this->get_width_internal(), - this->get_height_internal(), static_cast(this->buffer_length_), this->img_buf_addr_h_, - this->img_buf_addr_l_, static_cast(this->vcom_) / 1000.0f, this->vcom_register_, - force_temperature, this->use_legacy_dpy_area_ ? "DPY_AREA (0x0034, legacy)" : "DPY_BUF_AREA (0x0037)", - YESNO(this->sleep_when_done_), this->full_update_every_, YESNO(this->invert_colors_), - this->grayscale_ ? "4bpp grayscale" : "1bpp monochrome", this->reset_duration_); + ESP_LOGCONFIG( + TAG, + " Model preset: %s" + "\n Dimensions: %dx%d" + "\n Buffer: %u bytes" + "\n Image buffer addr: 0x%04X%04X" + "\n VCOM: %.02fV (set selector 0x%04X)" + "\n Force temperature: %s" + "\n Display command: %s" + "\n Sleep when done: %s" + "\n Full update every: %u" + "\n Inverted colors: %s" + "\n Pixel format: %s" + "\n Reset duration: %" PRIu32 "ms", + this->name_ != nullptr ? this->name_ : LOG_STR_LITERAL("(unknown)"), this->get_width_internal(), + this->get_height_internal(), static_cast(this->buffer_length_), this->img_buf_addr_h_, + this->img_buf_addr_l_, static_cast(this->vcom_) / 1000.0f, this->vcom_register_, force_temperature, + this->use_legacy_dpy_area_ ? LOG_STR_LITERAL("DPY_AREA (0x0034, legacy)") + : LOG_STR_LITERAL("DPY_BUF_AREA (0x0037)"), + YESNO(this->sleep_when_done_), this->full_update_every_, YESNO(this->invert_colors_), + this->grayscale_ ? LOG_STR_LITERAL("4bpp grayscale") : LOG_STR_LITERAL("1bpp monochrome"), this->reset_duration_); LOG_PIN(" Reset Pin: ", this->reset_pin_); LOG_PIN(" Busy Pin: ", this->busy_pin_); LOG_PIN(" CS Pin: ", this->cs_); diff --git a/esphome/components/lc709203f/lc709203f.cpp b/esphome/components/lc709203f/lc709203f.cpp index cbd733b611..a5dda6ca43 100644 --- a/esphome/components/lc709203f/lc709203f.cpp +++ b/esphome/components/lc709203f/lc709203f.cpp @@ -150,7 +150,8 @@ void Lc709203f::dump_config() { " Pack Size: %d mAH\n" " Pack APA: 0x%02X\n" " Pack Rated Voltage: 3.%sV", - this->pack_size_, this->apa_, this->pack_voltage_ == 0x0000 ? "8" : "7"); + this->pack_size_, this->apa_, + this->pack_voltage_ == 0x0000 ? LOG_STR_LITERAL("8") : LOG_STR_LITERAL("7")); LOG_I2C_DEVICE(this); LOG_UPDATE_INTERVAL(this); LOG_SENSOR(" ", "Voltage", this->voltage_sensor_); diff --git a/esphome/components/ld2420/ld2420.cpp b/esphome/components/ld2420/ld2420.cpp index 4aa00f8fd4..e342ead414 100644 --- a/esphome/components/ld2420/ld2420.cpp +++ b/esphome/components/ld2420/ld2420.cpp @@ -701,7 +701,8 @@ uint8_t LD2420Component::set_config_mode(bool enable) { cmd_frame.data_length += sizeof(CMD_PROTOCOL_VER); } cmd_frame.footer = CMD_FRAME_FOOTER; - ESP_LOGV(TAG, "Sending set config %s command: %2X", enable ? "enable" : "disable", cmd_frame.command); + ESP_LOGV(TAG, "Sending set config %s command: %2X", enable ? LOG_STR_LITERAL("enable") : LOG_STR_LITERAL("disable"), + cmd_frame.command); return this->send_cmd_from_array(cmd_frame); } diff --git a/esphome/components/ld6002b/ld6002b.cpp b/esphome/components/ld6002b/ld6002b.cpp index ca6b9b9552..73fc7df331 100644 --- a/esphome/components/ld6002b/ld6002b.cpp +++ b/esphome/components/ld6002b/ld6002b.cpp @@ -437,7 +437,8 @@ void LD6002BComponent::dump_config() { "HLK-LD6002B:\n" " Auto wake: %s\n" " Max data length: %u", - this->auto_wake_ ? "true" : "false", static_cast(this->max_data_len_)); + this->auto_wake_ ? LOG_STR_LITERAL("true") : LOG_STR_LITERAL("false"), + static_cast(this->max_data_len_)); if (this->wakeup_pin_ != nullptr) { LOG_PIN(" Wake-up Pin: ", this->wakeup_pin_); ESP_LOGCONFIG(TAG, " Wake Pulse: %" PRIu32 "ms", this->wakeup_pulse_ms_); diff --git a/esphome/components/max31856/max31856.cpp b/esphome/components/max31856/max31856.cpp index 4062d21bee..b5bad8ef74 100644 --- a/esphome/components/max31856/max31856.cpp +++ b/esphome/components/max31856/max31856.cpp @@ -23,8 +23,10 @@ void MAX31856Sensor::setup() { void MAX31856Sensor::dump_config() { LOG_SENSOR("", "MAX31856", this); LOG_PIN(" CS Pin: ", this->cs_); - ESP_LOGCONFIG(TAG, " Mains Filter: %s", - (filter_ == FILTER_60HZ ? "60 Hz" : (filter_ == FILTER_50HZ ? "50 Hz" : "Unknown!"))); + ESP_LOGCONFIG( + TAG, " Mains Filter: %s", + (filter_ == FILTER_60HZ ? LOG_STR_LITERAL("60 Hz") + : (filter_ == FILTER_50HZ ? LOG_STR_LITERAL("50 Hz") : LOG_STR_LITERAL("Unknown!")))); if (this->thermocouple_type_ < 0 || this->thermocouple_type_ > 7) { ESP_LOGCONFIG(TAG, " Thermocouple Type: Unknown"); } else { diff --git a/esphome/components/max31865/max31865.cpp b/esphome/components/max31865/max31865.cpp index 220fb4e704..e5a6fca8fb 100644 --- a/esphome/components/max31865/max31865.cpp +++ b/esphome/components/max31865/max31865.cpp @@ -80,12 +80,14 @@ void MAX31865Sensor::dump_config() { LOG_SENSOR("", "MAX31865", this); LOG_PIN(" CS Pin: ", this->cs_); LOG_UPDATE_INTERVAL(this); - ESP_LOGCONFIG(TAG, - " Reference Resistance: %.2fΩ\n" - " RTD: %u-wire %.2fΩ\n" - " Mains Filter: %s", - reference_resistance_, rtd_wires_, rtd_nominal_resistance_, - (filter_ == FILTER_60HZ ? "60 Hz" : (filter_ == FILTER_50HZ ? "50 Hz" : "Unknown!"))); + ESP_LOGCONFIG( + TAG, + " Reference Resistance: %.2fΩ\n" + " RTD: %u-wire %.2fΩ\n" + " Mains Filter: %s", + reference_resistance_, rtd_wires_, rtd_nominal_resistance_, + (filter_ == FILTER_60HZ ? LOG_STR_LITERAL("60 Hz") + : (filter_ == FILTER_50HZ ? LOG_STR_LITERAL("50 Hz") : LOG_STR_LITERAL("Unknown!")))); } void MAX31865Sensor::read_data_() { diff --git a/esphome/components/mcp4461/mcp4461.cpp b/esphome/components/mcp4461/mcp4461.cpp index abc74b9e6d..cc53f9de7f 100644 --- a/esphome/components/mcp4461/mcp4461.cpp +++ b/esphome/components/mcp4461/mcp4461.cpp @@ -319,7 +319,7 @@ uint16_t Mcp4461Component::read_wiper_level_(uint8_t wiper_idx, bool *ok) { if (!(this->read_16_(reg, &buf))) { this->error_code_ = MCP4461_STATUS_I2C_ERROR; this->status_set_warning(); - ESP_LOGW(TAG, "Error fetching %swiper %u value", (wiper_idx > 3) ? "nonvolatile " : "", wiper_idx); + ESP_LOGW(TAG, "Error fetching %swiper %u value", (wiper_idx > 3) ? LOG_STR_LITERAL("nonvolatile ") : "", wiper_idx); return 0; } if (ok != nullptr) { @@ -377,7 +377,8 @@ void Mcp4461Component::write_wiper_level_(uint8_t wiper, uint16_t value) { if (!(this->mcp4461_write_(this->get_wiper_address_(wiper), value, nonvolatile))) { this->error_code_ = MCP4461_STATUS_I2C_ERROR; this->status_set_warning(); - ESP_LOGW(TAG, "Error writing %swiper %u level %u", (wiper > 3) ? "nonvolatile " : "", wiper, value); + ESP_LOGW(TAG, "Error writing %swiper %u level %u", (wiper > 3) ? LOG_STR_LITERAL("nonvolatile ") : "", wiper, + value); } } diff --git a/esphome/components/media_player/media_player.cpp b/esphome/components/media_player/media_player.cpp index 7dce74117a..6c3eef912e 100644 --- a/esphome/components/media_player/media_player.cpp +++ b/esphome/components/media_player/media_player.cpp @@ -122,7 +122,7 @@ void MediaPlayerCall::perform() { ESP_LOGV(TAG, " Volume: %.2f", this->volume_.value()); } if (this->announcement_.has_value()) { - ESP_LOGV(TAG, " Announcement: %s", this->announcement_.value() ? "yes" : "no"); + ESP_LOGV(TAG, " Announcement: %s", this->announcement_.value() ? LOG_STR_LITERAL("yes") : LOG_STR_LITERAL("no")); } this->parent_->control(*this); } diff --git a/esphome/components/mipi_spi/mipi_spi.cpp b/esphome/components/mipi_spi/mipi_spi.cpp index 2eec3b12d1..80ae96720b 100644 --- a/esphome/components/mipi_spi/mipi_spi.cpp +++ b/esphome/components/mipi_spi/mipi_spi.cpp @@ -25,7 +25,8 @@ void internal_dump_config(const char *model, int width, int height, int offset_w " SPI Bus width: %d", model, width, height, YESNO(madctl & MADCTL_MV), YESNO(madctl & (MADCTL_MX | MADCTL_XFLIP)), YESNO(madctl & (MADCTL_MY | MADCTL_YFLIP)), YESNO(has_hardware_rotation), YESNO(invert_colors), - (madctl & MADCTL_BGR) ? "BGR" : "RGB", display_bits, is_big_endian ? "Big" : "Little", spi_mode, + (madctl & MADCTL_BGR) ? LOG_STR_LITERAL("BGR") : LOG_STR_LITERAL("RGB"), display_bits, + is_big_endian ? LOG_STR_LITERAL("Big") : LOG_STR_LITERAL("Little"), spi_mode, static_cast(data_rate / 1000000), bus_width); LOG_PIN(" CS Pin: ", cs); LOG_PIN(" Reset Pin: ", reset); diff --git a/esphome/components/modbus_server/modbus_server.cpp b/esphome/components/modbus_server/modbus_server.cpp index feb0e67725..65bf4ef2f4 100644 --- a/esphome/components/modbus_server/modbus_server.cpp +++ b/esphome/components/modbus_server/modbus_server.cpp @@ -269,7 +269,8 @@ void ModbusServer::dump_config() { " Enabled: %s\n" " Register Last Address: 0x%02X\n" " Register Value: %" PRIu16, - this->address_, this->server_courtesy_response_.enabled ? "true" : "false", + this->address_, + this->server_courtesy_response_.enabled ? LOG_STR_LITERAL("true") : LOG_STR_LITERAL("false"), this->server_courtesy_response_.register_last_address, this->server_courtesy_response_.register_value); #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE @@ -280,8 +281,9 @@ void ModbusServer::dump_config() { } ESP_LOGCONFIG(TAG, "server bits"); for (auto &b : this->server_bits_) { - ESP_LOGCONFIG(TAG, " Address=0x%04X readable=%s writable=%s", b->address, b->read_lambda ? "true" : "false", - b->write_lambda ? "true" : "false"); + ESP_LOGCONFIG(TAG, " Address=0x%04X readable=%s writable=%s", b->address, + b->read_lambda ? LOG_STR_LITERAL("true") : LOG_STR_LITERAL("false"), + b->write_lambda ? LOG_STR_LITERAL("true") : LOG_STR_LITERAL("false")); } #endif } diff --git a/esphome/components/nextion/nextion.cpp b/esphome/components/nextion/nextion.cpp index bdc66adb70..97910ba3d5 100644 --- a/esphome/components/nextion/nextion.cpp +++ b/esphome/components/nextion/nextion.cpp @@ -595,7 +595,8 @@ void Nextion::process_nextion_commands_() { uint8_t page_id = to_process[0]; uint8_t component_id = to_process[1]; uint8_t touch_event = to_process[2]; // 0 -> release, 1 -> press - ESP_LOGV(TAG, "Touch %s: page %u comp %u", touch_event ? "PRESS" : "RELEASE", page_id, component_id); + ESP_LOGV(TAG, "Touch %s: page %u comp %u", touch_event ? LOG_STR_LITERAL("PRESS") : LOG_STR_LITERAL("RELEASE"), + page_id, component_id); for (auto *touch : this->touch_) { touch->process_touch(page_id, component_id, touch_event != 0); } @@ -628,7 +629,7 @@ void Nextion::process_nextion_commands_() { const uint16_t x = (uint16_t(to_process[0]) << 8) | to_process[1]; const uint16_t y = (uint16_t(to_process[2]) << 8) | to_process[3]; const uint8_t touch_event = to_process[4]; // 0 -> release, 1 -> press - ESP_LOGV(TAG, "Touch %s at %u,%u", touch_event ? "PRESS" : "RELEASE", x, y); + ESP_LOGV(TAG, "Touch %s at %u,%u", touch_event ? LOG_STR_LITERAL("PRESS") : LOG_STR_LITERAL("RELEASE"), x, y); break; } diff --git a/esphome/components/pcm5122/pcm5122.cpp b/esphome/components/pcm5122/pcm5122.cpp index d178cb83b8..4f6417f6c0 100644 --- a/esphome/components/pcm5122/pcm5122.cpp +++ b/esphome/components/pcm5122/pcm5122.cpp @@ -119,7 +119,8 @@ void PCM5122::dump_config() { " Channel mix: %s\n" " Volume range: %.1f dB to %.1f dB\n" " Muted: %s", - this->bits_per_sample_, this->analog_gain_ == PCM5122_ANALOG_GAIN_0DB ? "0 dB" : "-6 dB", + this->bits_per_sample_, + this->analog_gain_ == PCM5122_ANALOG_GAIN_0DB ? LOG_STR_LITERAL("0 dB") : LOG_STR_LITERAL("-6 dB"), channel_mix_str, this->volume_min_db_, this->volume_max_db_, YESNO(this->is_muted_)); LOG_PIN(" Enable Pin: ", this->enable_pin_); } diff --git a/esphome/components/pn7150/pn7150.cpp b/esphome/components/pn7150/pn7150.cpp index 2a2724f56b..4e679c664a 100644 --- a/esphome/components/pn7150/pn7150.cpp +++ b/esphome/components/pn7150/pn7150.cpp @@ -243,8 +243,8 @@ uint8_t PN7150::reset_core_(const bool reset_config, const bool power) { } ESP_LOGD(TAG, "Configuration %s, NCI version: %s", - rx.get_message()[nfc::NCI_PKT_PAYLOAD_OFFSET + 2] ? "reset" : "retained", - rx.get_message()[nfc::NCI_PKT_PAYLOAD_OFFSET + 1] == 0x20 ? "2.0" : "1.0"); + rx.get_message()[nfc::NCI_PKT_PAYLOAD_OFFSET + 2] ? LOG_STR_LITERAL("reset") : LOG_STR_LITERAL("retained"), + rx.get_message()[nfc::NCI_PKT_PAYLOAD_OFFSET + 1] == 0x20 ? LOG_STR_LITERAL("2.0") : LOG_STR_LITERAL("1.0")); return nfc::STATUS_OK; } diff --git a/esphome/components/pn7160/pn7160.cpp b/esphome/components/pn7160/pn7160.cpp index 7abd89b371..f2cbfa6bcf 100644 --- a/esphome/components/pn7160/pn7160.cpp +++ b/esphome/components/pn7160/pn7160.cpp @@ -265,8 +265,8 @@ uint8_t PN7160::reset_core_(const bool reset_config, const bool power) { } ESP_LOGD(TAG, "Configuration %s, NCI version: %s, Manufacturer ID: 0x%02X", - rx.get_message()[4] ? "reset" : "retained", rx.get_message()[5] == 0x20 ? "2.0" : "1.0", - rx.get_message()[6]); + rx.get_message()[4] ? LOG_STR_LITERAL("reset") : LOG_STR_LITERAL("retained"), + rx.get_message()[5] == 0x20 ? LOG_STR_LITERAL("2.0") : LOG_STR_LITERAL("1.0"), rx.get_message()[6]); rx.get_message().erase(rx.get_message().begin(), rx.get_message().begin() + 8); char mfr_buf[nfc::FORMAT_BYTES_BUFFER_SIZE]; ESP_LOGD(TAG, "Manufacturer info: %s", nfc::format_bytes_to(mfr_buf, rx.get_message())); diff --git a/esphome/components/pylontech/pylontech.cpp b/esphome/components/pylontech/pylontech.cpp index 0973699da8..54d9e5c654 100644 --- a/esphome/components/pylontech/pylontech.cpp +++ b/esphome/components/pylontech/pylontech.cpp @@ -137,7 +137,8 @@ void PylontechComponent::process_line_(std::string &buffer) { } else if (strcmp(token_buf, "Power") == 0) { // header line i.e. "Power Volt Curr" and so on this->has_tlow_id_ = buffer.find("Tlow.Id") != std::string::npos; - ESP_LOGD(TAG, "header line %s Tlow.Id: %s", this->has_tlow_id_ ? "with" : "without", + ESP_LOGD(TAG, "header line %s Tlow.Id: %s", + this->has_tlow_id_ ? LOG_STR_LITERAL("with") : LOG_STR_LITERAL("without"), buffer.substr(0, buffer.size() - 2).c_str()); return; } else { diff --git a/esphome/components/rd03d/rd03d.cpp b/esphome/components/rd03d/rd03d.cpp index 2eb76a1087..18328def9f 100644 --- a/esphome/components/rd03d/rd03d.cpp +++ b/esphome/components/rd03d/rd03d.cpp @@ -55,8 +55,9 @@ void RD03DComponent::setup() { void RD03DComponent::dump_config() { ESP_LOGCONFIG(TAG, "RD-03D:"); if (this->tracking_mode_.has_value()) { - ESP_LOGCONFIG(TAG, " Tracking Mode: %s", - *this->tracking_mode_ == TrackingMode::SINGLE_TARGET ? "single" : "multi"); + ESP_LOGCONFIG( + TAG, " Tracking Mode: %s", + *this->tracking_mode_ == TrackingMode::SINGLE_TARGET ? LOG_STR_LITERAL("single") : LOG_STR_LITERAL("multi")); } if (this->throttle_ > 0) { ESP_LOGCONFIG(TAG, " Throttle: %" PRIu32 "ms", this->throttle_); diff --git a/esphome/components/remote_receiver/remote_receiver.cpp b/esphome/components/remote_receiver/remote_receiver.cpp index 36152d8854..bbcb7ae765 100644 --- a/esphome/components/remote_receiver/remote_receiver.cpp +++ b/esphome/components/remote_receiver/remote_receiver.cpp @@ -76,15 +76,16 @@ void RemoteReceiverComponent::setup() { } void RemoteReceiverComponent::dump_config() { - ESP_LOGCONFIG(TAG, - "Remote Receiver:\n" - " Buffer Size: %" PRIu32 "\n" - " Tolerance: %" PRIu32 "%s\n" - " Filter out pulses shorter than: %" PRIu32 " us\n" - " Signal is done after %" PRIu32 " us of no changes", - this->buffer_size_, this->tolerance_, - (this->tolerance_mode_ == remote_base::TOLERANCE_MODE_TIME) ? " us" : "%", this->filter_us_, - this->idle_us_); + ESP_LOGCONFIG( + TAG, + "Remote Receiver:\n" + " Buffer Size: %" PRIu32 "\n" + " Tolerance: %" PRIu32 "%s\n" + " Filter out pulses shorter than: %" PRIu32 " us\n" + " Signal is done after %" PRIu32 " us of no changes", + this->buffer_size_, this->tolerance_, + (this->tolerance_mode_ == remote_base::TOLERANCE_MODE_TIME) ? LOG_STR_LITERAL(" us") : LOG_STR_LITERAL("%"), + this->filter_us_, this->idle_us_); LOG_PIN(" Pin: ", this->pin_); } diff --git a/esphome/components/resistance/resistance_sensor.cpp b/esphome/components/resistance/resistance_sensor.cpp index 6056509093..7522d026a4 100644 --- a/esphome/components/resistance/resistance_sensor.cpp +++ b/esphome/components/resistance/resistance_sensor.cpp @@ -11,8 +11,8 @@ void ResistanceSensor::dump_config() { " Configuration: %s\n" " Resistor: %.2fΩ\n" " Reference Voltage: %.1fV", - this->configuration_ == UPSTREAM ? "UPSTREAM" : "DOWNSTREAM", this->resistor_, - this->reference_voltage_); + this->configuration_ == UPSTREAM ? LOG_STR_LITERAL("UPSTREAM") : LOG_STR_LITERAL("DOWNSTREAM"), + this->resistor_, this->reference_voltage_); } void ResistanceSensor::process_(float value) { if (std::isnan(value)) { diff --git a/esphome/components/sen21231/sen21231.cpp b/esphome/components/sen21231/sen21231.cpp index b42ba2fa1d..3f6212e7f2 100644 --- a/esphome/components/sen21231/sen21231.cpp +++ b/esphome/components/sen21231/sen21231.cpp @@ -13,7 +13,7 @@ void Sen21231Sensor::dump_config() { if (this->is_failed()) { ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); } - ESP_LOGI(TAG, "SEN21231: %s", this->is_failed() ? "FAILED" : "OK"); + ESP_LOGI(TAG, "SEN21231: %s", this->is_failed() ? LOG_STR_LITERAL("FAILED") : LOG_STR_LITERAL("OK")); LOG_UPDATE_INTERVAL(this); } diff --git a/esphome/components/senseair/senseair.cpp b/esphome/components/senseair/senseair.cpp index 0e8e4cef97..f5017cff75 100644 --- a/esphome/components/senseair/senseair.cpp +++ b/esphome/components/senseair/senseair.cpp @@ -89,8 +89,9 @@ void SenseAirComponent::background_calibration_result() { } // Check if 5th bit (register CI6) is set - ESP_LOGI(TAG, "SenseAir Result=%s (%02x%02x%02x %02x%02x %02x%02x)", (response[4] & 0b100000) != 0 ? "OK" : "NOT_OK", - response[0], response[1], response[2], response[3], response[4], response[5], response[6]); + ESP_LOGI(TAG, "SenseAir Result=%s (%02x%02x%02x %02x%02x %02x%02x)", + (response[4] & 0b100000) != 0 ? LOG_STR_LITERAL("OK") : LOG_STR_LITERAL("NOT_OK"), response[0], response[1], + response[2], response[3], response[4], response[5], response[6]); } void SenseAirComponent::abc_enable() { diff --git a/esphome/components/serial_proxy/serial_proxy.cpp b/esphome/components/serial_proxy/serial_proxy.cpp index 94cefc8700..2ab0d4ebb4 100644 --- a/esphome/components/serial_proxy/serial_proxy.cpp +++ b/esphome/components/serial_proxy/serial_proxy.cpp @@ -82,11 +82,11 @@ void SerialProxy::dump_config() { " RTS Pin: %s\n" " DTR Pin: %s", this->instance_index_, this->name_ != nullptr ? this->name_ : "", - this->port_type_ == api::enums::SERIAL_PROXY_PORT_TYPE_RS485 ? "RS485" - : this->port_type_ == api::enums::SERIAL_PROXY_PORT_TYPE_RS232 ? "RS232" - : "TTL", - this->rts_pin_ != nullptr ? "configured" : "not configured", - this->dtr_pin_ != nullptr ? "configured" : "not configured"); + this->port_type_ == api::enums::SERIAL_PROXY_PORT_TYPE_RS485 ? LOG_STR_LITERAL("RS485") + : this->port_type_ == api::enums::SERIAL_PROXY_PORT_TYPE_RS232 ? LOG_STR_LITERAL("RS232") + : LOG_STR_LITERAL("TTL"), + this->rts_pin_ != nullptr ? LOG_STR_LITERAL("configured") : LOG_STR_LITERAL("not configured"), + this->dtr_pin_ != nullptr ? LOG_STR_LITERAL("configured") : LOG_STR_LITERAL("not configured")); } SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uint32_t baudrate, bool flow_control, diff --git a/esphome/components/sgp4x/sgp4x.cpp b/esphome/components/sgp4x/sgp4x.cpp index bc6fe794a0..0cf5c31483 100644 --- a/esphome/components/sgp4x/sgp4x.cpp +++ b/esphome/components/sgp4x/sgp4x.cpp @@ -284,7 +284,7 @@ void SGP4xComponent::dump_config() { " Type: %s\n" " Serial number: %" PRIu64 "\n" " Minimum Samples: %f", - this->sgp_type_ == SGP41 ? "SGP41" : "SGP40", this->serial_number_, + this->sgp_type_ == SGP41 ? LOG_STR_LITERAL("SGP41") : LOG_STR_LITERAL("SGP40"), this->serial_number_, GasIndexAlgorithm_INITIAL_BLACKOUT); } LOG_UPDATE_INTERVAL(this); diff --git a/esphome/components/sprinkler/sprinkler.cpp b/esphome/components/sprinkler/sprinkler.cpp index 2edceb76a5..9fd0d9208b 100644 --- a/esphome/components/sprinkler/sprinkler.cpp +++ b/esphome/components/sprinkler/sprinkler.cpp @@ -1335,7 +1335,7 @@ void Sprinkler::all_valves_off_(const bool include_pump) { this->set_pump_state(this->valve_pump_switch(valve_index), false); } } - ESP_LOGD(TAG, "All valves stopped%s", include_pump ? ", including pumps" : ""); + ESP_LOGD(TAG, "All valves stopped%s", include_pump ? LOG_STR_LITERAL(", including pumps") : ""); } void Sprinkler::prep_full_cycle_() { diff --git a/esphome/components/switch/switch.cpp b/esphome/components/switch/switch.cpp index 101a0b9ffa..8413c7b493 100644 --- a/esphome/components/switch/switch.cpp +++ b/esphome/components/switch/switch.cpp @@ -26,7 +26,8 @@ void Switch::turn_off() { this->write_state(this->inverted_); } void Switch::toggle() { - ESP_LOGV(TAG, "'%s' Toggling %s.", this->get_name().c_str(), this->state ? "OFF" : "ON"); + ESP_LOGV(TAG, "'%s' Toggling %s.", this->get_name().c_str(), + this->state ? LOG_STR_LITERAL("OFF") : LOG_STR_LITERAL("ON")); this->write_state(this->inverted_ == this->state); } optional Switch::get_initial_state() { diff --git a/esphome/components/sx127x/sx127x.cpp b/esphome/components/sx127x/sx127x.cpp index 040a3064bc..cd81f08914 100644 --- a/esphome/components/sx127x/sx127x.cpp +++ b/esphome/components/sx127x/sx127x.cpp @@ -479,8 +479,9 @@ void SX127x::dump_config() { " Rx Start: %s\n" " Rx Floor: %.1f dBm\n" " Packet Mode: %s", - shaping, this->modulation_ == MOD_FSK ? "FSK" : "OOK", this->bitrate_, TRUEFALSE(this->bitsync_), - TRUEFALSE(this->rx_start_), this->rx_floor_, TRUEFALSE(this->packet_mode_)); + shaping, this->modulation_ == MOD_FSK ? LOG_STR_LITERAL("FSK") : LOG_STR_LITERAL("OOK"), + this->bitrate_, TRUEFALSE(this->bitsync_), TRUEFALSE(this->rx_start_), this->rx_floor_, + TRUEFALSE(this->packet_mode_)); if (this->packet_mode_) { ESP_LOGCONFIG(TAG, " CRC Enable: %s", TRUEFALSE(this->crc_enable_)); } diff --git a/esphome/components/thermostat/thermostat_climate.cpp b/esphome/components/thermostat/thermostat_climate.cpp index c10eb5b9f5..e830d359c6 100644 --- a/esphome/components/thermostat/thermostat_climate.cpp +++ b/esphome/components/thermostat/thermostat_climate.cpp @@ -1432,7 +1432,8 @@ void ThermostatClimate::dump_config() { ESP_LOGCONFIG(TAG, " On boot, restore from: %s\n" " Use Start-up Delay: %s", - this->on_boot_restore_from_ == thermostat::DEFAULT_PRESET ? "DEFAULT_PRESET" : "MEMORY", + this->on_boot_restore_from_ == thermostat::DEFAULT_PRESET ? LOG_STR_LITERAL("DEFAULT_PRESET") + : LOG_STR_LITERAL("MEMORY"), YESNO(this->use_startup_delay_)); if (this->supports_two_points_) { ESP_LOGCONFIG(TAG, " Minimum Set Point Differential: %.1f°C", this->set_point_minimum_differential_); @@ -1550,7 +1551,8 @@ void ThermostatClimate::dump_config() { ESP_LOGCONFIG(TAG, " Supported PRESETS:"); for (const auto &entry : this->preset_config_) { const auto *preset_name = LOG_STR_ARG(climate::climate_preset_to_string(entry.preset)); - ESP_LOGCONFIG(TAG, " %s:%s", preset_name, entry.preset == this->default_preset_ ? " (default)" : ""); + ESP_LOGCONFIG(TAG, " %s:%s", preset_name, + entry.preset == this->default_preset_ ? LOG_STR_LITERAL(" (default)") : ""); this->dump_preset_config_(preset_name, entry.config); } } @@ -1561,7 +1563,7 @@ void ThermostatClimate::dump_config() { const auto *preset_name = entry.name; ESP_LOGCONFIG(TAG, " %s:%s", preset_name, (this->default_custom_preset_ != nullptr && strcmp(entry.name, this->default_custom_preset_) == 0) - ? " (default)" + ? LOG_STR_LITERAL(" (default)") : ""); this->dump_preset_config_(preset_name, entry.config); } diff --git a/esphome/components/tsl2591/tsl2591.cpp b/esphome/components/tsl2591/tsl2591.cpp index fb34dd833d..2a5d6a4ee4 100644 --- a/esphome/components/tsl2591/tsl2591.cpp +++ b/esphome/components/tsl2591/tsl2591.cpp @@ -269,7 +269,7 @@ uint32_t TSL2591Component::get_combined_illuminance() { break; } // we only log this if we need any delay, since normally we don't - ESP_LOGD(TAG, " after %3d ms: ADC valid? %s", d, avalid ? "true" : "false"); + ESP_LOGD(TAG, " after %3d ms: ADC valid? %s", d, avalid ? LOG_STR_LITERAL("true") : LOG_STR_LITERAL("false")); delay(mini_delay); } if (!avalid) { diff --git a/esphome/components/tuya/select/tuya_select.cpp b/esphome/components/tuya/select/tuya_select.cpp index f0fc47f504..057f7f8ea6 100644 --- a/esphome/components/tuya/select/tuya_select.cpp +++ b/esphome/components/tuya/select/tuya_select.cpp @@ -39,7 +39,7 @@ void TuyaSelect::dump_config() { " Select has datapoint ID %u\n" " Data type: %s\n" " Options are:", - this->select_id_, this->is_int_ ? "int" : "enum"); + this->select_id_, this->is_int_ ? LOG_STR_LITERAL("int") : LOG_STR_LITERAL("enum")); const auto &options = this->traits.get_options(); for (size_t i = 0; i < this->mappings_.size(); i++) { ESP_LOGCONFIG(TAG, " %i: %s", this->mappings_.at(i), options.at(i)); diff --git a/esphome/components/tuya/tuya.cpp b/esphome/components/tuya/tuya.cpp index 15ab4b6dc3..82fb96d787 100644 --- a/esphome/components/tuya/tuya.cpp +++ b/esphome/components/tuya/tuya.cpp @@ -259,7 +259,7 @@ void Tuya::handle_command_(uint8_t command, uint8_t version, const uint8_t *buff st.payload[0] = 0x04; this->send_command_(st); ESP_LOGI(TAG, "%s received (%s), replied with WIFI_STATE confirming connection established", - is_select ? "WIFI_SELECT" : "WIFI_RESET", mode_str); + is_select ? LOG_STR_LITERAL("WIFI_SELECT") : LOG_STR_LITERAL("WIFI_RESET"), mode_str); break; } case TuyaCommandType::DATAPOINT_DELIVER: diff --git a/esphome/components/veml7700/veml7700.cpp b/esphome/components/veml7700/veml7700.cpp index 594c9da170..6c609f4fe7 100644 --- a/esphome/components/veml7700/veml7700.cpp +++ b/esphome/components/veml7700/veml7700.cpp @@ -259,7 +259,7 @@ ErrorCode VEML7700Component::configure_() { ErrorCode VEML7700Component::reconfigure_time_and_gain_(IntegrationTime time, Gain gain, bool shutdown) { ESP_LOGV(TAG, "Reconfigure time and gain (%d ms, %s) %s", get_itime_ms(time), get_gain_str(gain), - shutdown ? "Shutting down" : "Turning back on"); + shutdown ? LOG_STR_LITERAL("Shutting down") : LOG_STR_LITERAL("Turning back on")); ConfigurationRegister als_conf{0}; als_conf.raw = 0; @@ -272,7 +272,7 @@ ErrorCode VEML7700Component::reconfigure_time_and_gain_(IntegrationTime time, Ga als_conf.ALS_GAIN = gain; auto err = this->write_register((uint8_t) CommandRegisters::ALS_CONF_0, als_conf.raw_bytes, VEML_REG_SIZE); if (err != i2c::ERROR_OK) { - ESP_LOGW(TAG, "%s failed", shutdown ? "Shutdown" : "Turn on"); + ESP_LOGW(TAG, "%s failed", shutdown ? LOG_STR_LITERAL("Shutdown") : LOG_STR_LITERAL("Turn on")); } return err; @@ -363,8 +363,8 @@ void VEML7700Component::apply_lux_calculation_(Readings &data) { data.fake_infrared_lux = reduce_to_zero(data.white_lux, data.als_lux); ESP_LOGV(TAG, "%s mode - ALS = %.1f lx, WHITE = %.1f lx, FAKE_IR = %.1f lx", - this->automatic_mode_enabled_ ? "Automatic" : "Manual", data.als_lux, data.white_lux, - data.fake_infrared_lux); + this->automatic_mode_enabled_ ? LOG_STR_LITERAL("Automatic") : LOG_STR_LITERAL("Manual"), data.als_lux, + data.white_lux, data.fake_infrared_lux); } void VEML7700Component::apply_lux_compensation_(Readings &data) { diff --git a/esphome/components/vl53l0x/vl53l0x_sensor.cpp b/esphome/components/vl53l0x/vl53l0x_sensor.cpp index df7929f676..49eb3d00a1 100644 --- a/esphome/components/vl53l0x/vl53l0x_sensor.cpp +++ b/esphome/components/vl53l0x/vl53l0x_sensor.cpp @@ -31,7 +31,8 @@ void VL53L0XSensor::dump_config() { ESP_LOGCONFIG(TAG, " Timeout: %" PRIu32 "%s\n" " Timing Budget %" PRIu32 "us ", - this->timeout_us_, this->timeout_us_ > 0 ? "us" : " (no timeout)", this->measurement_timing_budget_us_); + this->timeout_us_, this->timeout_us_ > 0 ? LOG_STR_LITERAL("us") : LOG_STR_LITERAL(" (no timeout)"), + this->measurement_timing_budget_us_); } void VL53L0XSensor::setup() { diff --git a/esphome/components/water_heater/water_heater.cpp b/esphome/components/water_heater/water_heater.cpp index 9862253ad9..1dc2d008a1 100644 --- a/esphome/components/water_heater/water_heater.cpp +++ b/esphome/components/water_heater/water_heater.cpp @@ -100,10 +100,11 @@ void WaterHeaterCall::perform() { ESP_LOGV(TAG, " Target Temperature High: %.2f", this->target_temperature_high_); } if (this->state_mask_ & WATER_HEATER_STATE_AWAY) { - ESP_LOGV(TAG, " Away: %s", (this->state_ & WATER_HEATER_STATE_AWAY) ? "YES" : "NO"); + ESP_LOGV(TAG, " Away: %s", + (this->state_ & WATER_HEATER_STATE_AWAY) ? LOG_STR_LITERAL("YES") : LOG_STR_LITERAL("NO")); } if (this->state_mask_ & WATER_HEATER_STATE_ON) { - ESP_LOGV(TAG, " On: %s", (this->state_ & WATER_HEATER_STATE_ON) ? "YES" : "NO"); + ESP_LOGV(TAG, " On: %s", (this->state_ & WATER_HEATER_STATE_ON) ? LOG_STR_LITERAL("YES") : LOG_STR_LITERAL("NO")); } this->parent_->control(*this); } @@ -178,7 +179,7 @@ void WaterHeater::publish_state() { ESP_LOGV(TAG, " Away: YES"); } if (traits.has_feature_flags(WATER_HEATER_SUPPORTS_ON_OFF)) { - ESP_LOGV(TAG, " On: %s", (this->state_ & WATER_HEATER_STATE_ON) ? "YES" : "NO"); + ESP_LOGV(TAG, " On: %s", (this->state_ & WATER_HEATER_STATE_ON) ? LOG_STR_LITERAL("YES") : LOG_STR_LITERAL("NO")); } #if defined(USE_WATER_HEATER) && defined(USE_CONTROLLER_REGISTRY) diff --git a/esphome/components/weikai/weikai.cpp b/esphome/components/weikai/weikai.cpp index a19dce4db3..043df86be9 100644 --- a/esphome/components/weikai/weikai.cpp +++ b/esphome/components/weikai/weikai.cpp @@ -134,7 +134,7 @@ void WeikaiComponent::loop() { } bool status = children_[i]->uart_receive_test_(message); ESP_LOGI(TAG, "Test %s => send/received %u bytes %s - execution time %" PRIu32 " ms", message, RING_BUFFER_SIZE, - status ? "correctly" : "with error", elapsed_ms(time)); + status ? LOG_STR_LITERAL("correctly") : LOG_STR_LITERAL("with error"), elapsed_ms(time)); } } @@ -238,9 +238,9 @@ void WeikaiComponent::set_pin_direction_(uint8_t pin, gpio::Flags flags) { void WeikaiGPIOPin::setup() { ESP_LOGCONFIG(TAG, "Setting GPIO pin %d mode to %s", this->pin_, - flags_ == gpio::FLAG_INPUT ? "Input" - : this->flags_ == gpio::FLAG_OUTPUT ? "Output" - : "NOT SPECIFIED"); + this->flags_ == gpio::FLAG_INPUT ? LOG_STR_LITERAL("Input") + : this->flags_ == gpio::FLAG_OUTPUT ? LOG_STR_LITERAL("Output") + : LOG_STR_LITERAL("NOT SPECIFIED")); this->pin_mode(this->flags_); } @@ -420,7 +420,7 @@ bool WeikaiChannel::read_array(uint8_t *buffer, size_t length) { this->receive_buffer_.pop(buffer[i]); } ESP_LOGVV(TAG, "read_array(ch=%d buffer[0]=%02X, length=%d): status %s", this->channel_, *buffer, length, - status ? "OK" : "ERROR"); + status ? LOG_STR_LITERAL("OK") : LOG_STR_LITERAL("ERROR")); return status; } @@ -558,8 +558,8 @@ bool WeikaiChannel::uart_receive_test_(char *message) { } } - ESP_LOGV(TAG, "%s => received %d bytes status %s - exec time %d µs", message, received, status ? "OK" : "ERROR", - micros() - start_exec); + ESP_LOGV(TAG, "%s => received %d bytes status %s - exec time %d µs", message, received, + status ? LOG_STR_LITERAL("OK") : LOG_STR_LITERAL("ERROR"), micros() - start_exec); return status; } diff --git a/esphome/components/whirlpool/whirlpool.cpp b/esphome/components/whirlpool/whirlpool.cpp index ace96d78fc..f560917f41 100644 --- a/esphome/components/whirlpool/whirlpool.cpp +++ b/esphome/components/whirlpool/whirlpool.cpp @@ -103,7 +103,7 @@ void WhirlpoolClimate::transmit_state() { } // Swing - ESP_LOGV(TAG, "send swing %s", this->send_swing_cmd_ ? "true" : "false"); + ESP_LOGV(TAG, "send swing %s", this->send_swing_cmd_ ? LOG_STR_LITERAL("true") : LOG_STR_LITERAL("false")); if (this->send_swing_cmd_) { if (this->swing_mode == climate::CLIMATE_SWING_VERTICAL || this->swing_mode == climate::CLIMATE_SWING_OFF) { remote_state[2] |= 128; diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 82755f39f7..694e616476 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -933,7 +933,7 @@ void WiFiComponent::loop() { if (semaphore_count > 0 && !this->is_high_performance_mode_) { // Transition to high-performance mode (no power save) ESP_LOGV(TAG, "Switching to high-performance mode (%" PRIu32 " active %s)", (uint32_t) semaphore_count, - semaphore_count == 1 ? "request" : "requests"); + semaphore_count == 1 ? LOG_STR_LITERAL("request") : LOG_STR_LITERAL("requests")); this->power_save_ = WIFI_POWER_SAVE_NONE; if (this->wifi_apply_power_save_()) { this->is_high_performance_mode_ = true; @@ -1181,8 +1181,9 @@ void WiFiComponent::start_connecting(const WiFiAP &ap) { " CA Cert: %s\n" " Client Cert: %s\n" " Client Key: %s", - ca_cert_present ? "present" : "not present", client_cert_present ? "present" : "not present", - client_key_present ? "present" : "not present"); + ca_cert_present ? LOG_STR_LITERAL("present") : LOG_STR_LITERAL("not present"), + client_cert_present ? LOG_STR_LITERAL("present") : LOG_STR_LITERAL("not present"), + client_key_present ? LOG_STR_LITERAL("present") : LOG_STR_LITERAL("not present")); } else { #endif ESP_LOGV(TAG, " Password: " LOG_SECRET("'%s'"), ap.password_.c_str()); @@ -1316,7 +1317,8 @@ void WiFiComponent::print_connect_params_() { ESP_LOGCONFIG(TAG, " BTM: %s\n" " RRM: %s", - this->btm_ ? "enabled" : "disabled", this->rrm_ ? "enabled" : "disabled"); + this->btm_ ? LOG_STR_LITERAL("enabled") : LOG_STR_LITERAL("disabled"), + this->rrm_ ? LOG_STR_LITERAL("enabled") : LOG_STR_LITERAL("disabled")); #endif } diff --git a/esphome/components/wireguard/wireguard.cpp b/esphome/components/wireguard/wireguard.cpp index 2f07344d3b..fc06569fba 100644 --- a/esphome/components/wireguard/wireguard.cpp +++ b/esphome/components/wireguard/wireguard.cpp @@ -146,18 +146,19 @@ void Wireguard::dump_config() { " Peer Pre-shared Key: " LOG_SECRET("%s"), this->address_, this->netmask_, private_key_masked, this->peer_endpoint_, this->peer_port_, this->peer_public_key_, - (this->preshared_key_ != nullptr ? preshared_key_masked : "NOT IN USE")); + (this->preshared_key_ != nullptr ? preshared_key_masked : LOG_STR_LITERAL("NOT IN USE"))); // clang-format on ESP_LOGCONFIG(TAG, " Peer Allowed IPs:"); for (const AllowedIP &allowed_ip : this->allowed_ips_) { ESP_LOGCONFIG(TAG, " - %s/%s", allowed_ip.ip, allowed_ip.netmask); } ESP_LOGCONFIG(TAG, " Peer Persistent Keepalive: %d%s", this->keepalive_, - (this->keepalive_ > 0 ? "s" : " (DISABLED)")); + (this->keepalive_ > 0 ? LOG_STR_LITERAL("s") : LOG_STR_LITERAL(" (DISABLED)"))); ESP_LOGCONFIG(TAG, " Reboot Timeout: %" PRIu32 "%s", (this->reboot_timeout_ / 1000), - (this->reboot_timeout_ != 0 ? "s" : " (DISABLED)")); + (this->reboot_timeout_ != 0 ? LOG_STR_LITERAL("s") : LOG_STR_LITERAL(" (DISABLED)"))); // be careful: if proceed_allowed_ is true, require connection is false - ESP_LOGCONFIG(TAG, " Require Connection to Proceed: %s", (this->proceed_allowed_ ? "NO" : "YES")); + ESP_LOGCONFIG(TAG, " Require Connection to Proceed: %s", + (this->proceed_allowed_ ? LOG_STR_LITERAL("NO") : LOG_STR_LITERAL("YES"))); LOG_UPDATE_INTERVAL(this); } diff --git a/esphome/components/wl_134/wl_134.cpp b/esphome/components/wl_134/wl_134.cpp index f3eb17965d..5e86d5a441 100644 --- a/esphome/components/wl_134/wl_134.cpp +++ b/esphome/components/wl_134/wl_134.cpp @@ -76,8 +76,8 @@ Wl134Component::Rfid134Error Wl134Component::read_packet_() { " isAnimal: %s\n" " Reserved0: %d\n" " Reserved1: %" PRId32, - reading.id, reading.country, reading.isData ? "true" : "false", reading.isAnimal ? "true" : "false", - reading.reserved0, reading.reserved1); + reading.id, reading.country, reading.isData ? LOG_STR_LITERAL("true") : LOG_STR_LITERAL("false"), + reading.isAnimal ? LOG_STR_LITERAL("true") : LOG_STR_LITERAL("false"), reading.reserved0, reading.reserved1); char buf[20]; // "%03d" (3) + "%012" PRId64 (12) + null = 16 max buf_append_printf(buf, sizeof(buf), 0, "%03d%012" PRId64, reading.country, reading.id); diff --git a/esphome/components/zwave_proxy/zwave_proxy.cpp b/esphome/components/zwave_proxy/zwave_proxy.cpp index 68750295e1..b0d18a3e6a 100644 --- a/esphome/components/zwave_proxy/zwave_proxy.cpp +++ b/esphome/components/zwave_proxy/zwave_proxy.cpp @@ -180,11 +180,11 @@ void ZWaveProxy::process_uart_slow_() { void ZWaveProxy::dump_config() { char hex_buf[format_hex_pretty_size(ZWAVE_HOME_ID_SIZE)]; - ESP_LOGCONFIG( - TAG, - "Z-Wave Proxy:\n" - " Home ID: %s", - this->home_id_ready_ ? format_hex_pretty_to(hex_buf, this->home_id_.data(), this->home_id_.size()) : "unknown"); + ESP_LOGCONFIG(TAG, + "Z-Wave Proxy:\n" + " Home ID: %s", + this->home_id_ready_ ? format_hex_pretty_to(hex_buf, this->home_id_.data(), this->home_id_.size()) + : LOG_STR_LITERAL("unknown")); } void ZWaveProxy::api_connection_authenticated(api::APIConnection *conn) { @@ -510,7 +510,8 @@ bool ZWaveProxy::response_handler_slow_() { return false; // No response handled } - ESP_LOGVV(TAG, "Sending %s (0x%02X)", this->last_response_ == ZWAVE_FRAME_TYPE_ACK ? "ACK" : "NAK/CAN", + ESP_LOGVV(TAG, "Sending %s (0x%02X)", + this->last_response_ == ZWAVE_FRAME_TYPE_ACK ? LOG_STR_LITERAL("ACK") : LOG_STR_LITERAL("NAK/CAN"), this->last_response_); this->write_byte(this->last_response_); this->parsing_state_ = ZWAVE_PARSING_STATE_WAIT_START; From 575a540c9f1e7902a3b1f6d8a676bcb611cdac3d Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 31 Aug 2026 07:19:20 -0400 Subject: [PATCH 048/147] [esp32] Fix ESP32-S31 GPIO validation (#18904) --- esphome/components/esp32/gpio_esp32_s31.py | 24 ++++++++------ tests/component_tests/esp32/test_esp32.py | 37 ++++++++++++++++++++++ 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/esphome/components/esp32/gpio_esp32_s31.py b/esphome/components/esp32/gpio_esp32_s31.py index d49240723b..c53c32c99c 100644 --- a/esphome/components/esp32/gpio_esp32_s31.py +++ b/esphome/components/esp32/gpio_esp32_s31.py @@ -5,11 +5,15 @@ import esphome.config_validation as cv from esphome.const import CONF_INPUT, CONF_MODE, CONF_NUMBER, CONF_SCL, CONF_SDA from esphome.pins import check_strapping_pin -# Per the ESP32-S31 datasheet (page 96): -# https://documentation.espressif.com/esp32-s31_datasheet_en.pdf -_ESP32S31_SPI_FLASH_PINS: set[int] = {27, 28, 29, 31, 32, 33} -# GPIO60/GPIO61 set the boot mode; GPIO37 selects the JTAG signal source. -_ESP32S31_STRAPPING_PINS: set[int] = {37, 60, 61} +# Per the ESP32-S31 datasheet, the SPI flash and PSRAM interfaces use +# dedicated package pins (SPICS/SPIQ/SPIWP/SPIHD/SPICLK/SPID) outside the +# GPIO matrix, so no GPIOs are reserved for them. GPIO29 and GPIO41 do not +# exist on this chip (SOC_GPIO_VALID_GPIO_MASK excludes them). +# https://documentation.espressif.com/esp32-s31_datasheet_en.html +_ESP32S31_INVALID_PINS: set[int] = {29, 41} +# GPIO60/GPIO61 set the boot mode; GPIO37 selects the JTAG signal source; +# GPIO36 sets the VDD_SPI voltage. +_ESP32S31_STRAPPING_PINS: set[int] = {36, 37, 60, 61} # LP I2C is fixed to GPIO6 (SCL) / GPIO7 (SDA) per the datasheet IO MUX table. _ESP32S31_I2C_LP_PINS = {"SDA": 7, "SCL": 6} @@ -19,10 +23,8 @@ _LOGGER = logging.getLogger(__name__) def esp32_s31_validate_gpio_pin(value: int) -> int: if value < 0 or value > 61: raise cv.Invalid(f"Invalid pin number: {value} (must be 0-61)") - if value in _ESP32S31_SPI_FLASH_PINS: - raise cv.Invalid( - f"GPIO{value} is reserved for the SPI flash interface on ESP32-S31 and cannot be used." - ) + if value in _ESP32S31_INVALID_PINS: + raise cv.Invalid(f"GPIO{value} does not exist on ESP32-S31.") return value @@ -33,6 +35,10 @@ def esp32_s31_validate_supports(value: dict[str, Any]) -> dict[str, Any]: if num < 0 or num > 61: raise cv.Invalid(f"Invalid pin number: {num} (must be 0-61)") + # Checked here as well so ignore_pin_validation_error cannot bypass it; + # these pins are not bonded and can never work + if num in _ESP32S31_INVALID_PINS: + raise cv.Invalid(f"GPIO{num} does not exist on ESP32-S31.") if is_input: # All ESP32 pins support input mode pass diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index db7ed6b3fc..190f2d2896 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -1268,3 +1268,40 @@ def test_parse_pio_platform_version(value: str, expected: str) -> None: from esphome.components.esp32 import _parse_pio_platform_version assert _parse_pio_platform_version(value) == expected + + +def test_esp32_s31_gpio_validation( + set_core_config: SetCoreConfigCallable, + caplog: pytest.LogCaptureFixture, +) -> None: + """S31: flash uses dedicated pins so GPIO27-33 are normal pins, GPIO29 and + GPIO41 do not exist, and GPIO36 is a strapping pin.""" + from esphome.components.esp32.const import VARIANT_ESP32S31 + from esphome.components.esp32.gpio import validate_supports + from esphome.const import CONF_INPUT, CONF_MODE, CONF_OPEN_DRAIN, CONF_OUTPUT + + set_core_config( + PlatformFramework.ESP32_IDF, platform_data={KEY_VARIANT: VARIANT_ESP32S31} + ) + + input_mode = {CONF_INPUT: True, CONF_OUTPUT: False, CONF_OPEN_DRAIN: False} + + # Previously reserved for the flash interface, which uses dedicated pins + for num in (27, 28, 31, 32, 33): + pin = {CONF_NUMBER: num, CONF_IGNORE_PIN_VALIDATION_ERROR: False} + assert validate_gpio_pin(pin)[CONF_NUMBER] == num + + for num in (29, 41): + with pytest.raises(cv.Invalid, match=f"GPIO{num} does not exist"): + validate_gpio_pin( + {CONF_NUMBER: num, CONF_IGNORE_PIN_VALIDATION_ERROR: False} + ) + # Also rejected in validate_supports so ignore_pin_validation_error + # cannot bypass it + with pytest.raises(cv.Invalid, match=f"GPIO{num} does not exist"): + validate_supports({CONF_NUMBER: num, CONF_MODE: input_mode}) + + pin = {CONF_NUMBER: 36, CONF_MODE: input_mode} + with caplog.at_level("WARNING"): + validate_supports(pin) + assert "GPIO36 is a strapping PIN" in caplog.text From 5e58b312e26cf09efe2dfba9af3d13f28d678866 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:20:20 +1000 Subject: [PATCH 049/147] [mipi_rgb] Add ESP32S31 support (#18914) --- esphome/components/mipi_rgb/display.py | 9 ++- esphome/components/mipi_rgb/mipi_rgb.cpp | 5 +- esphome/components/mipi_rgb/mipi_rgb.h | 2 +- .../mipi_rgb/test_mipi_rgb_config.py | 59 ++++++++++++++++++- 4 files changed, 69 insertions(+), 6 deletions(-) diff --git a/esphome/components/mipi_rgb/display.py b/esphome/components/mipi_rgb/display.py index e23e19a000..b91528160e 100644 --- a/esphome/components/mipi_rgb/display.py +++ b/esphome/components/mipi_rgb/display.py @@ -12,7 +12,12 @@ from esphome.components.const import ( CONF_DRAW_ROUNDING, ) from esphome.components.display import CONF_SHOW_TEST_CARD -from esphome.components.esp32 import VARIANT_ESP32P4, VARIANT_ESP32S3, only_on_variant +from esphome.components.esp32 import ( + VARIANT_ESP32P4, + VARIANT_ESP32S3, + VARIANT_ESP32S31, + only_on_variant, +) from esphome.components.mipi import ( COLOR_ORDERS, CONF_DE_PIN, @@ -226,7 +231,7 @@ def _config_schema(config: ConfigType) -> ConfigType: config = cv.All( schema, cv.only_on_esp32, - only_on_variant(supported=[VARIANT_ESP32S3, VARIANT_ESP32P4]), + only_on_variant(supported=[VARIANT_ESP32S3, VARIANT_ESP32P4, VARIANT_ESP32S31]), )(config) model = MODELS[config[CONF_MODEL].upper()] model.check_requirements() diff --git a/esphome/components/mipi_rgb/mipi_rgb.cpp b/esphome/components/mipi_rgb/mipi_rgb.cpp index 7421d8ad83..aeb04c155c 100644 --- a/esphome/components/mipi_rgb/mipi_rgb.cpp +++ b/esphome/components/mipi_rgb/mipi_rgb.cpp @@ -1,4 +1,4 @@ -#if defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32P4) +#if defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32P4) || defined(USE_ESP32_VARIANT_ESP32S31) #include "mipi_rgb.h" #include "esphome/core/gpio.h" #include "esphome/core/hal.h" @@ -400,4 +400,5 @@ void MipiRgb::dump_config() { } } // namespace esphome::mipi_rgb -#endif // defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32P4) +#endif // defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32P4) || + // defined(USE_ESP32_VARIANT_ESP32S31) diff --git a/esphome/components/mipi_rgb/mipi_rgb.h b/esphome/components/mipi_rgb/mipi_rgb.h index 1480004833..87b35781e2 100644 --- a/esphome/components/mipi_rgb/mipi_rgb.h +++ b/esphome/components/mipi_rgb/mipi_rgb.h @@ -1,6 +1,6 @@ #pragma once -#if defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32P4) +#if defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32P4) || defined(USE_ESP32_VARIANT_ESP32S31) #include "esphome/core/gpio.h" #include "esphome/components/display/display.h" #include "esp_lcd_panel_ops.h" diff --git a/tests/component_tests/mipi_rgb/test_mipi_rgb_config.py b/tests/component_tests/mipi_rgb/test_mipi_rgb_config.py index e85327c0ab..497aba4df1 100644 --- a/tests/component_tests/mipi_rgb/test_mipi_rgb_config.py +++ b/tests/component_tests/mipi_rgb/test_mipi_rgb_config.py @@ -10,7 +10,13 @@ from esphome import config_validation as cv # via ch422g) can be validated by the mipi_rgb CONFIG_SCHEMA in this test. import esphome.components.ch422g # noqa: F401 from esphome.components.display import get_display_metadata -from esphome.components.esp32 import KEY_BOARD, VARIANT_ESP32S3 +from esphome.components.esp32 import ( + KEY_BOARD, + VARIANT_ESP32C3, + VARIANT_ESP32P4, + VARIANT_ESP32S3, + VARIANT_ESP32S31, +) import esphome.components.pca9554 # noqa: F401 import esphome.components.xl9535 # noqa: F401 from esphome.const import ( @@ -135,3 +141,54 @@ def test_metadata_records_rotation( config = CONFIG_SCHEMA({**base, "id": "unrotated"}) assert get_display_metadata(config["id"]).rotation == 0 + + +@pytest.mark.parametrize( + ("variant", "board"), + [ + (VARIANT_ESP32S3, "esp32-s3-devkitc-1"), + (VARIANT_ESP32P4, "esp32-p4-evboard"), + # No dedicated board is registered for ESP32-S31 yet; an unknown board + # name simply skips per-board pin validation. + (VARIANT_ESP32S31, "esp32-s31-devkitc"), + ], +) +def test_configuration_succeeds_on_supported_variants( + variant: str, board: str, set_core_config: SetCoreConfigCallable +) -> None: + """mipi_rgb requires a chip with an RGB LCD peripheral: S3, P4 or S31.""" + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_BOARD: board, KEY_VARIANT: variant}, + ) + + from esphome.components.mipi_rgb.display import CONFIG_SCHEMA + + CONFIG_SCHEMA({"model": "ESP32-8048S070", "data_pins": DATA_PINS, "pclk_pin": 21}) + + +def test_only_on_variant_rejects_unsupported_variant( + set_core_config: SetCoreConfigCallable, +) -> None: + """A variant without the RGB LCD peripheral (e.g. ESP32-C3) is rejected. + + Exercises the exact ``only_on_variant`` call used by ``mipi_rgb.display`` + directly, since building a full model config with GPIO numbers that are + also valid on an unsupported variant like ESP32-C3 is unrelated to what + this checks. + """ + from esphome.components.esp32 import only_on_variant + + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_VARIANT: VARIANT_ESP32C3}, + ) + + validator = only_on_variant( + supported=[VARIANT_ESP32S3, VARIANT_ESP32P4, VARIANT_ESP32S31] + ) + with pytest.raises( + cv.Invalid, + match=r"This feature is only available on ESP32S3, ESP32P4, ESP32S31", + ): + validator({}) From 2890afe0e5a4299c81a5f7e0873e70553c31a5cb Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Mon, 31 Aug 2026 22:38:20 +1000 Subject: [PATCH 050/147] [esp32] Fix S31 reserved pins (#18915) Co-authored-by: Claude Sonnet 5 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- esphome/components/esp32/gpio_esp32_s31.py | 13 +++++++----- tests/component_tests/esp32/test_esp32.py | 20 +++++++++++++----- .../mipi_rgb/test_mipi_rgb_config.py | 21 +++++++++++++------ 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/esphome/components/esp32/gpio_esp32_s31.py b/esphome/components/esp32/gpio_esp32_s31.py index c53c32c99c..7ccb7cdb90 100644 --- a/esphome/components/esp32/gpio_esp32_s31.py +++ b/esphome/components/esp32/gpio_esp32_s31.py @@ -5,11 +5,10 @@ import esphome.config_validation as cv from esphome.const import CONF_INPUT, CONF_MODE, CONF_NUMBER, CONF_SCL, CONF_SDA from esphome.pins import check_strapping_pin -# Per the ESP32-S31 datasheet, the SPI flash and PSRAM interfaces use -# dedicated package pins (SPICS/SPIQ/SPIWP/SPIHD/SPICLK/SPID) outside the -# GPIO matrix, so no GPIOs are reserved for them. GPIO29 and GPIO41 do not -# exist on this chip (SOC_GPIO_VALID_GPIO_MASK excludes them). -# https://documentation.espressif.com/esp32-s31_datasheet_en.html +# Per the ESP32-S31 IDF DOCS and datasheet: +# https://docs.espressif.com/projects/esp-idf/en/v6.1/esp32s31/api-reference/peripherals/gpio.html +# https://documentation.espressif.com/esp32-s31_datasheet_en.pdf +_ESP32S31_SPI_FLASH_PINS: set[int] = {26, 27, 28, 30, 31, 32} _ESP32S31_INVALID_PINS: set[int] = {29, 41} # GPIO60/GPIO61 set the boot mode; GPIO37 selects the JTAG signal source; # GPIO36 sets the VDD_SPI voltage. @@ -25,6 +24,10 @@ def esp32_s31_validate_gpio_pin(value: int) -> int: raise cv.Invalid(f"Invalid pin number: {value} (must be 0-61)") if value in _ESP32S31_INVALID_PINS: raise cv.Invalid(f"GPIO{value} does not exist on ESP32-S31.") + if value in _ESP32S31_SPI_FLASH_PINS: + raise cv.Invalid( + f"GPIO{value} is reserved for the SPI flash interface on ESP32-S31 and cannot be used." + ) return value diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index 190f2d2896..bef273badd 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -1274,8 +1274,9 @@ def test_esp32_s31_gpio_validation( set_core_config: SetCoreConfigCallable, caplog: pytest.LogCaptureFixture, ) -> None: - """S31: flash uses dedicated pins so GPIO27-33 are normal pins, GPIO29 and - GPIO41 do not exist, and GPIO36 is a strapping pin.""" + """S31: GPIO26-28/30-32 are reserved for the SPI flash interface, GPIO29 + and GPIO41 do not exist, GPIO33 is a normal pin, and GPIO36 is a + strapping pin.""" from esphome.components.esp32.const import VARIANT_ESP32S31 from esphome.components.esp32.gpio import validate_supports from esphome.const import CONF_INPUT, CONF_MODE, CONF_OPEN_DRAIN, CONF_OUTPUT @@ -1286,9 +1287,18 @@ def test_esp32_s31_gpio_validation( input_mode = {CONF_INPUT: True, CONF_OUTPUT: False, CONF_OPEN_DRAIN: False} - # Previously reserved for the flash interface, which uses dedicated pins - for num in (27, 28, 31, 32, 33): - pin = {CONF_NUMBER: num, CONF_IGNORE_PIN_VALIDATION_ERROR: False} + # Not reserved; a normal GPIO + pin = {CONF_NUMBER: 33, CONF_IGNORE_PIN_VALIDATION_ERROR: False} + assert validate_gpio_pin(pin)[CONF_NUMBER] == 33 + + # Reserved for the SPI flash interface, but can be bypassed with + # ignore_pin_validation_error + for num in (26, 27, 28, 30, 31, 32): + with pytest.raises(cv.Invalid, match=f"GPIO{num} is reserved"): + validate_gpio_pin( + {CONF_NUMBER: num, CONF_IGNORE_PIN_VALIDATION_ERROR: False} + ) + pin = {CONF_NUMBER: num, CONF_IGNORE_PIN_VALIDATION_ERROR: True} assert validate_gpio_pin(pin)[CONF_NUMBER] == num for num in (29, 41): diff --git a/tests/component_tests/mipi_rgb/test_mipi_rgb_config.py b/tests/component_tests/mipi_rgb/test_mipi_rgb_config.py index 497aba4df1..ac8e111ddb 100644 --- a/tests/component_tests/mipi_rgb/test_mipi_rgb_config.py +++ b/tests/component_tests/mipi_rgb/test_mipi_rgb_config.py @@ -144,17 +144,22 @@ def test_metadata_records_rotation( @pytest.mark.parametrize( - ("variant", "board"), + ("variant", "board", "model"), [ - (VARIANT_ESP32S3, "esp32-s3-devkitc-1"), - (VARIANT_ESP32P4, "esp32-p4-evboard"), + # ESP32-8048S070 is a real Sunton board wired for ESP32-S3 (e.g. its + # default de_pin is GPIO41, which doesn't exist on S31), so it is + # only meaningful as a config on that variant. + (VARIANT_ESP32S3, "esp32-s3-devkitc-1", "ESP32-8048S070"), + # P4 and S31 use the pin-agnostic CUSTOM model so this only checks + # that the chip itself is accepted, independent of board wiring. + (VARIANT_ESP32P4, "esp32-p4-evboard", "CUSTOM"), # No dedicated board is registered for ESP32-S31 yet; an unknown board # name simply skips per-board pin validation. - (VARIANT_ESP32S31, "esp32-s31-devkitc"), + (VARIANT_ESP32S31, "esp32-s31-devkitc", "CUSTOM"), ], ) def test_configuration_succeeds_on_supported_variants( - variant: str, board: str, set_core_config: SetCoreConfigCallable + variant: str, board: str, model: str, set_core_config: SetCoreConfigCallable ) -> None: """mipi_rgb requires a chip with an RGB LCD peripheral: S3, P4 or S31.""" set_core_config( @@ -164,7 +169,11 @@ def test_configuration_succeeds_on_supported_variants( from esphome.components.mipi_rgb.display import CONFIG_SCHEMA - CONFIG_SCHEMA({"model": "ESP32-8048S070", "data_pins": DATA_PINS, "pclk_pin": 21}) + config = {"model": model, "data_pins": DATA_PINS, "pclk_pin": 21} + if model == "CUSTOM": + config[CONF_INIT_SEQUENCE] = [[0xA0, 0x01]] + config[CONF_DIMENSIONS] = {CONF_WIDTH: 480, CONF_HEIGHT: 480} + CONFIG_SCHEMA(config) def test_only_on_variant_rejects_unsupported_variant( From 813c0006842681e1408d27e017897abd74a87b69 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 31 Aug 2026 08:51:33 -0400 Subject: [PATCH 051/147] [adc] Add ESP32-S31 support (#18887) --- esphome/components/adc/__init__.py | 23 ++++++ esphome/components/adc/adc_sensor_esp32.cpp | 78 +++++++++++---------- esphome/components/adc/sensor.py | 9 +++ 3 files changed, 72 insertions(+), 38 deletions(-) diff --git a/esphome/components/adc/__init__.py b/esphome/components/adc/__init__.py index 5c763a4f4c..c397e746b0 100644 --- a/esphome/components/adc/__init__.py +++ b/esphome/components/adc/__init__.py @@ -13,6 +13,7 @@ from esphome.components.esp32 import ( VARIANT_ESP32P4, VARIANT_ESP32S2, VARIANT_ESP32S3, + VARIANT_ESP32S31, get_esp32_variant, ) import esphome.config_validation as cv @@ -156,6 +157,17 @@ ESP32_VARIANT_ADC1_PIN_TO_CHANNEL = { 9: adc_channel_t.ADC_CHANNEL_8, 10: adc_channel_t.ADC_CHANNEL_9, }, + # https://github.com/espressif/esp-idf/blob/master/components/soc/esp32s31/include/soc/adc_channel.h + VARIANT_ESP32S31: { + 42: adc_channel_t.ADC_CHANNEL_0, + 43: adc_channel_t.ADC_CHANNEL_1, + 44: adc_channel_t.ADC_CHANNEL_2, + 45: adc_channel_t.ADC_CHANNEL_3, + 46: adc_channel_t.ADC_CHANNEL_4, + 47: adc_channel_t.ADC_CHANNEL_5, + 48: adc_channel_t.ADC_CHANNEL_6, + 49: adc_channel_t.ADC_CHANNEL_7, + }, } # pin to adc2 channel mapping @@ -225,6 +237,17 @@ ESP32_VARIANT_ADC2_PIN_TO_CHANNEL = { 19: adc_channel_t.ADC_CHANNEL_8, 20: adc_channel_t.ADC_CHANNEL_9, }, + # https://github.com/espressif/esp-idf/blob/master/components/soc/esp32s31/include/soc/adc_channel.h + VARIANT_ESP32S31: { + 50: adc_channel_t.ADC_CHANNEL_0, + 51: adc_channel_t.ADC_CHANNEL_1, + 52: adc_channel_t.ADC_CHANNEL_2, + 53: adc_channel_t.ADC_CHANNEL_3, + 54: adc_channel_t.ADC_CHANNEL_4, + 55: adc_channel_t.ADC_CHANNEL_5, + 56: adc_channel_t.ADC_CHANNEL_6, + 57: adc_channel_t.ADC_CHANNEL_7, + }, } diff --git a/esphome/components/adc/adc_sensor_esp32.cpp b/esphome/components/adc/adc_sensor_esp32.cpp index a0f7a1ed08..c9887cea7c 100644 --- a/esphome/components/adc/adc_sensor_esp32.cpp +++ b/esphome/components/adc/adc_sensor_esp32.cpp @@ -74,9 +74,7 @@ void ADCSensor::setup() { if (this->calibration_handle_ == nullptr) { adc_cali_handle_t handle = nullptr; -#if USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32C5 || USE_ESP32_VARIANT_ESP32C6 || \ - USE_ESP32_VARIANT_ESP32C61 || USE_ESP32_VARIANT_ESP32H2 || USE_ESP32_VARIANT_ESP32P4 || USE_ESP32_VARIANT_ESP32S3 - // RISC-V variants (except C2) and S3 use curve fitting calibration +#if defined(ADC_CALI_SCHEME_CURVE_FITTING_SUPPORTED) adc_cali_curve_fitting_config_t cali_config = {}; // Zero initialize first #if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 3, 0) cali_config.chan = this->channel_; @@ -94,7 +92,7 @@ void ADCSensor::setup() { ESP_LOGW(TAG, "Curve fitting calibration failed with error %d, will use uncalibrated readings", err); this->setup_flags_.calibration_complete = false; } -#else // ESP32, ESP32-S2, and ESP32-C2 use line fitting calibration +#elif defined(ADC_CALI_SCHEME_LINE_FITTING_SUPPORTED) adc_cali_line_fitting_config_t cali_config = { .unit_id = this->adc_unit_, .atten = this->attenuation_, @@ -112,7 +110,11 @@ void ADCSensor::setup() { ESP_LOGW(TAG, "Line fitting calibration failed with error %d, will use uncalibrated readings", err); this->setup_flags_.calibration_complete = false; } -#endif // ESP32C3 || ESP32C5 || ESP32C6 || ESP32C61 || ESP32H2 || ESP32P4 || ESP32S3 +#else // No calibration scheme available + (void) handle; + ESP_LOGD(TAG, "No calibration scheme for this variant, readings are uncalibrated"); + this->setup_flags_.calibration_complete = false; +#endif } this->setup_flags_.init_complete = true; @@ -121,23 +123,28 @@ void ADCSensor::setup() { void ADCSensor::dump_config() { LOG_SENSOR("", "ADC Sensor", this); LOG_PIN(" Pin: ", this->pin_); - ESP_LOGCONFIG( - TAG, - " Channel: %d\n" - " Unit: %s\n" - " Attenuation: %s\n" - " Samples: %i\n" - " Sampling mode: %s\n" - " Setup Status:\n" - " Handle Init: %s\n" - " Config: %s\n" - " Calibration: %s\n" - " Overall Init: %s", - this->channel_, LOG_STR_ARG(adc_unit_to_str(this->adc_unit_)), - this->autorange_ ? "Auto" : LOG_STR_ARG(attenuation_to_str(this->attenuation_)), this->sample_count_, - LOG_STR_ARG(sampling_mode_to_str(this->sampling_mode_)), - this->setup_flags_.handle_init_complete ? "OK" : "FAILED", this->setup_flags_.config_complete ? "OK" : "FAILED", - this->setup_flags_.calibration_complete ? "OK" : "FAILED", this->setup_flags_.init_complete ? "OK" : "FAILED"); +#if defined(ADC_CALI_SCHEME_CURVE_FITTING_SUPPORTED) || defined(ADC_CALI_SCHEME_LINE_FITTING_SUPPORTED) + const char *calibration_status = this->setup_flags_.calibration_complete ? "OK" : "FAILED"; +#else + const char *calibration_status = "N/A"; // This variant has no calibration scheme +#endif + ESP_LOGCONFIG(TAG, + " Channel: %d\n" + " Unit: %s\n" + " Attenuation: %s\n" + " Samples: %i\n" + " Sampling mode: %s\n" + " Setup Status:\n" + " Handle Init: %s\n" + " Config: %s\n" + " Calibration: %s\n" + " Overall Init: %s", + this->channel_, LOG_STR_ARG(adc_unit_to_str(this->adc_unit_)), + this->autorange_ ? "Auto" : LOG_STR_ARG(attenuation_to_str(this->attenuation_)), this->sample_count_, + LOG_STR_ARG(sampling_mode_to_str(this->sampling_mode_)), + this->setup_flags_.handle_init_complete ? "OK" : "FAILED", + this->setup_flags_.config_complete ? "OK" : "FAILED", calibration_status, + this->setup_flags_.init_complete ? "OK" : "FAILED"); LOG_UPDATE_INTERVAL(this); } @@ -184,12 +191,11 @@ float ADCSensor::sample_fixed_attenuation_() { } else { ESP_LOGW(TAG, "ADC calibration conversion failed with error %d, disabling calibration", err); if (this->calibration_handle_ != nullptr) { -#if USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32C5 || USE_ESP32_VARIANT_ESP32C6 || \ - USE_ESP32_VARIANT_ESP32C61 || USE_ESP32_VARIANT_ESP32H2 || USE_ESP32_VARIANT_ESP32P4 || USE_ESP32_VARIANT_ESP32S3 +#if defined(ADC_CALI_SCHEME_CURVE_FITTING_SUPPORTED) adc_cali_delete_scheme_curve_fitting(this->calibration_handle_); -#else // Other ESP32 variants use line fitting calibration +#elif defined(ADC_CALI_SCHEME_LINE_FITTING_SUPPORTED) adc_cali_delete_scheme_line_fitting(this->calibration_handle_); -#endif // ESP32C3 || ESP32C5 || ESP32C6 || ESP32C61 || ESP32H2 || ESP32P4 || ESP32S3 +#endif this->calibration_handle_ = nullptr; } } @@ -217,10 +223,9 @@ float ADCSensor::sample_autorange_() { // Need to recalibrate for the new attenuation if (this->calibration_handle_ != nullptr) { // Delete old calibration handle -#if USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32C5 || USE_ESP32_VARIANT_ESP32C6 || \ - USE_ESP32_VARIANT_ESP32C61 || USE_ESP32_VARIANT_ESP32H2 || USE_ESP32_VARIANT_ESP32P4 || USE_ESP32_VARIANT_ESP32S3 +#if defined(ADC_CALI_SCHEME_CURVE_FITTING_SUPPORTED) adc_cali_delete_scheme_curve_fitting(this->calibration_handle_); -#else +#elif defined(ADC_CALI_SCHEME_LINE_FITTING_SUPPORTED) adc_cali_delete_scheme_line_fitting(this->calibration_handle_); #endif this->calibration_handle_ = nullptr; @@ -229,8 +234,7 @@ float ADCSensor::sample_autorange_() { // Create new calibration handle for this attenuation adc_cali_handle_t handle = nullptr; -#if USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32C5 || USE_ESP32_VARIANT_ESP32C6 || \ - USE_ESP32_VARIANT_ESP32C61 || USE_ESP32_VARIANT_ESP32H2 || USE_ESP32_VARIANT_ESP32P4 || USE_ESP32_VARIANT_ESP32S3 +#if defined(ADC_CALI_SCHEME_CURVE_FITTING_SUPPORTED) adc_cali_curve_fitting_config_t cali_config = {}; #if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 3, 0) cali_config.chan = this->channel_; @@ -242,7 +246,7 @@ float ADCSensor::sample_autorange_() { err = adc_cali_create_scheme_curve_fitting(&cali_config, &handle); ESP_LOGVV(TAG, "Autorange atten=%d: Calibration handle creation %s (err=%d)", atten, (err == ESP_OK) ? "SUCCESS" : "FAILED", err); -#else +#elif defined(ADC_CALI_SCHEME_LINE_FITTING_SUPPORTED) adc_cali_line_fitting_config_t cali_config = { .unit_id = this->adc_unit_, .atten = atten, @@ -264,10 +268,9 @@ float ADCSensor::sample_autorange_() { if (err != ESP_OK) { ESP_LOGW(TAG, "ADC read failed in autorange with error %d", err); if (handle != nullptr) { -#if USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32C5 || USE_ESP32_VARIANT_ESP32C6 || \ - USE_ESP32_VARIANT_ESP32C61 || USE_ESP32_VARIANT_ESP32H2 || USE_ESP32_VARIANT_ESP32P4 || USE_ESP32_VARIANT_ESP32S3 +#if defined(ADC_CALI_SCHEME_CURVE_FITTING_SUPPORTED) adc_cali_delete_scheme_curve_fitting(handle); -#else +#elif defined(ADC_CALI_SCHEME_LINE_FITTING_SUPPORTED) adc_cali_delete_scheme_line_fitting(handle); #endif } @@ -286,10 +289,9 @@ float ADCSensor::sample_autorange_() { ESP_LOGVV(TAG, "Autorange atten=%d: UNCALIBRATED FALLBACK - raw=%d -> %.6fV (3.3V ref)", atten, raw, voltage); } // Clean up calibration handle -#if USE_ESP32_VARIANT_ESP32C3 || USE_ESP32_VARIANT_ESP32C5 || USE_ESP32_VARIANT_ESP32C6 || \ - USE_ESP32_VARIANT_ESP32C61 || USE_ESP32_VARIANT_ESP32H2 || USE_ESP32_VARIANT_ESP32P4 || USE_ESP32_VARIANT_ESP32S3 +#if defined(ADC_CALI_SCHEME_CURVE_FITTING_SUPPORTED) adc_cali_delete_scheme_curve_fitting(handle); -#else +#elif defined(ADC_CALI_SCHEME_LINE_FITTING_SUPPORTED) adc_cali_delete_scheme_line_fitting(handle); #endif } else { diff --git a/esphome/components/adc/sensor.py b/esphome/components/adc/sensor.py index 5d1031825e..8cdea4f01a 100644 --- a/esphome/components/adc/sensor.py +++ b/esphome/components/adc/sensor.py @@ -3,6 +3,7 @@ import logging import esphome.codegen as cg from esphome.components import sensor, voltage_sampler from esphome.components.esp32 import ( + VARIANT_ESP32S31, get_esp32_variant, include_builtin_idf_component, require_adc_oneshot_iram, @@ -56,6 +57,14 @@ def validate_config(config: ConfigType) -> ConfigType: if config[CONF_RAW] and config.get(CONF_ATTENUATION, None) == "auto": raise cv.Invalid("Automatic attenuation cannot be used when raw output is set") + # The S31 ADC supports a single attenuation level (SOC_ADC_ATTEN_NUM is 1) + if ( + CORE.is_esp32 + and get_esp32_variant() == VARIANT_ESP32S31 + and config.get(CONF_ATTENUATION, "0db") != "0db" + ): + raise cv.Invalid("ESP32-S31 only supports 'attenuation: 0db'") + if config.get(CONF_ATTENUATION, None) == "auto" and config.get(CONF_SAMPLES, 1) > 1: raise cv.Invalid( "Automatic attenuation cannot be used when multisampling is set" From bbe806f1e6108b0d0597e2d1a055e6321ba99fd8 Mon Sep 17 00:00:00 2001 From: Brandon Harvey <8107750+bharvey88@users.noreply.github.com> Date: Mon, 31 Aug 2026 12:45:38 -0500 Subject: [PATCH 052/147] [sen6x] Add VOC/NOx algorithm tuning (#18779) --- esphome/components/sen6x/sen6x.cpp | 79 ++++++++++++++++--- esphome/components/sen6x/sen6x.h | 42 +++++++++- esphome/components/sen6x/sensor.py | 76 ++++++++++++++++-- tests/components/sen6x/common.yaml | 13 +++ .../components/sen6x/validate.esp32-idf.yaml | 18 +++++ 5 files changed, 208 insertions(+), 20 deletions(-) create mode 100644 tests/components/sen6x/validate.esp32-idf.yaml diff --git a/esphome/components/sen6x/sen6x.cpp b/esphome/components/sen6x/sen6x.cpp index 2a6ea64735..ed6cb24c52 100644 --- a/esphome/components/sen6x/sen6x.cpp +++ b/esphome/components/sen6x/sen6x.cpp @@ -9,9 +9,11 @@ static const char *const TAG = "sen6x"; static constexpr uint8_t POLL_RETRIES = 24; // 24 attempts static constexpr uint32_t I2C_READ_DELAY = 20; // 20 ms to wait for I2C read to complete +static constexpr uint32_t CMD_EXEC_DELAY = 20; // execution time of set commands (datasheet section 4.8) static constexpr uint32_t POLL_INTERVAL = 50; // 50 ms between poll attempts -// Single numeric timeout ID — the chain is sequential so only one is active at a time. +// Numeric timeout IDs. Each chain is sequential, so only one timeout per ID is active at a time. static constexpr uint32_t TIMEOUT_POLL = 1; +static constexpr uint32_t TIMEOUT_SETUP_STEP = 2; static constexpr uint16_t SEN6X_CMD_GET_DATA_READY_STATUS = 0x0202; static constexpr uint16_t SEN6X_CMD_GET_FIRMWARE_VERSION = 0xD100; static constexpr uint16_t SEN6X_CMD_GET_PRODUCT_NAME = 0xD014; @@ -26,6 +28,8 @@ static constexpr uint16_t SEN6X_CMD_READ_MEASUREMENT_SEN69C = 0x04B5; static constexpr uint16_t SEN6X_CMD_START_MEASUREMENTS = 0x0021; static constexpr uint16_t SEN6X_CMD_RESET = 0xD304; +static constexpr uint16_t SEN6X_CMD_VOC_ALGORITHM_TUNING = 0x60D0; +static constexpr uint16_t SEN6X_CMD_NOX_ALGORITHM_TUNING = 0x60E1; static inline void set_read_command_and_words(SEN6XComponent::Sen6xType type, uint16_t &read_cmd, uint8_t &read_words) { read_cmd = SEN6X_CMD_READ_MEASUREMENT; @@ -143,21 +147,76 @@ void SEN6XComponent::setup() { this->firmware_version_minor_ = raw_firmware_version & 0xFF; ESP_LOGI(TAG, "Firmware: %u.%u", this->firmware_version_major_, this->firmware_version_minor_); - if (!this->write_command(SEN6X_CMD_START_MEASUREMENTS)) { - ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); - this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL)); - return; - } - - this->set_timeout(60000, [this]() { this->startup_complete_ = true; }); - this->initialized_ = true; - ESP_LOGD(TAG, "Initialized"); + // Step 4: write configuration commands one at a time, then start measurements. + // Delay the first step so it doesn't run in the same loop tick as the read above. + this->set_timeout(TIMEOUT_SETUP_STEP, CMD_EXEC_DELAY, [this]() { this->run_next_setup_step_(); }); }); }); }); }); } +// One configuration write per invocation, spaced by CMD_EXEC_DELAY. Cases without a +// configured value fall through; each taken case must advance setup_step_index_ so the +// next invocation resumes at the following step. These writes are optional, so a failure +// only warns and the chain continues to the mandatory start-measurements write. +void SEN6XComponent::run_next_setup_step_() { + switch (this->setup_step_index_) { + // Tuning writes are skipped when setup() disabled the sensor for this variant + case 0: + this->setup_step_index_++; + if (this->voc_sensor_ != nullptr && this->voc_tuning_params_.has_value()) { + this->write_tuning_parameters_(SEN6X_CMD_VOC_ALGORITHM_TUNING, this->voc_tuning_params_.value()); + break; + } + [[fallthrough]]; + case 1: + this->setup_step_index_++; + if (this->nox_sensor_ != nullptr && this->nox_tuning_params_.has_value()) { + this->write_tuning_parameters_(SEN6X_CMD_NOX_ALGORITHM_TUNING, this->nox_tuning_params_.value()); + break; + } + [[fallthrough]]; + default: + this->finish_setup_(); + return; + } + this->set_timeout(TIMEOUT_SETUP_STEP, CMD_EXEC_DELAY, [this]() { this->run_next_setup_step_(); }); +} + +void SEN6XComponent::finish_setup_() { + if (!this->write_command(SEN6X_CMD_START_MEASUREMENTS)) { + ESP_LOGE(TAG, "Write 0x%04X failed, error %d", SEN6X_CMD_START_MEASUREMENTS, this->last_error_); + this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL)); + return; + } + + this->set_timeout(60000, [this]() { this->startup_complete_ = true; }); + this->initialized_ = true; + ESP_LOGD(TAG, "Initialized"); +} + +// Writes one optional configuration command. A failure warns and returns false, but does +// not stop setup: the sensor still measures with that setting left at its default. +bool SEN6XComponent::write_config_words_(uint16_t i2c_command, const uint16_t *data, uint8_t len) { + if (!this->write_command(i2c_command, data, len)) { + ESP_LOGE(TAG, "Write 0x%04X failed, error %d", i2c_command, this->last_error_); + this->status_set_warning(); + return false; + } + return true; +} + +bool SEN6XComponent::write_tuning_parameters_(uint16_t i2c_command, const GasTuning &tuning) { + uint16_t params[6] = {tuning.index_offset, + tuning.learning_time_offset_hours, + tuning.learning_time_gain_hours, + tuning.gating_max_duration_minutes, + tuning.std_initial, + tuning.gain_factor}; + return this->write_config_words_(i2c_command, params, 6); +} + void SEN6XComponent::dump_config() { ESP_LOGCONFIG(TAG, "sen6x:\n" diff --git a/esphome/components/sen6x/sen6x.h b/esphome/components/sen6x/sen6x.h index 041bf3b1aa..64ce3371fc 100644 --- a/esphome/components/sen6x/sen6x.h +++ b/esphome/components/sen6x/sen6x.h @@ -1,11 +1,25 @@ #pragma once #include "esphome/core/component.h" +#include "esphome/core/optional.h" #include "esphome/components/sensor/sensor.h" #include "esphome/components/sensirion_common/i2c_sensirion.h" namespace esphome::sen6x { +// The NOx algorithm requires std_initial to stay at 50 (Sensirion datasheet) +static constexpr uint16_t NOX_STD_INITIAL = 50; + +// Raw parameter block for the VOC/NOx algorithm tuning commands +struct GasTuning { + uint16_t index_offset; + uint16_t learning_time_offset_hours; + uint16_t learning_time_gain_hours; + uint16_t gating_max_duration_minutes; + uint16_t std_initial; + uint16_t gain_factor; +}; + class SEN6XComponent final : public PollingComponent, public sensirion_common::SensirionI2CDevice { SUB_SENSOR(pm_1_0) SUB_SENSOR(pm_2_5) @@ -27,22 +41,46 @@ class SEN6XComponent final : public PollingComponent, public sensirion_common::S enum Sen6xType { SEN62, SEN63C, SEN65, SEN66, SEN68, SEN69C, UNKNOWN }; void set_type(const std::string &type) { sen6x_type_ = infer_type_from_product_name_(type); } + void set_voc_algorithm_tuning(uint16_t index_offset, uint16_t learning_time_offset_hours, + uint16_t learning_time_gain_hours, uint16_t gating_max_duration_minutes, + uint16_t std_initial, uint16_t gain_factor) { + this->voc_tuning_params_ = GasTuning{ + index_offset, learning_time_offset_hours, learning_time_gain_hours, gating_max_duration_minutes, std_initial, + gain_factor}; + } + void set_nox_algorithm_tuning(uint16_t index_offset, uint16_t learning_time_offset_hours, + uint16_t learning_time_gain_hours, uint16_t gating_max_duration_minutes, + uint16_t gain_factor) { + this->nox_tuning_params_ = GasTuning{index_offset, + learning_time_offset_hours, + learning_time_gain_hours, + gating_max_duration_minutes, + NOX_STD_INITIAL, + gain_factor}; + } protected: Sen6xType infer_type_from_product_name_(const std::string &product_name); + void run_next_setup_step_(); + void finish_setup_(); + bool write_config_words_(uint16_t i2c_command, const uint16_t *data, uint8_t len); + bool write_tuning_parameters_(uint16_t i2c_command, const GasTuning &tuning); void poll_data_ready_(); void read_measurements_(); void parse_and_publish_measurements_(); - bool initialized_{false}; std::string product_name_; - Sen6xType sen6x_type_{UNKNOWN}; std::string serial_number_; + optional voc_tuning_params_; + optional nox_tuning_params_; + Sen6xType sen6x_type_{UNKNOWN}; uint16_t read_cmd_{0}; + uint8_t setup_step_index_{0}; uint8_t firmware_version_major_{0}; uint8_t firmware_version_minor_{0}; uint8_t poll_retries_remaining_{0}; uint8_t read_words_{0}; + bool initialized_{false}; bool startup_complete_{false}; }; diff --git a/esphome/components/sen6x/sensor.py b/esphome/components/sen6x/sensor.py index b0ffdc53a4..4c0242f2e3 100644 --- a/esphome/components/sen6x/sensor.py +++ b/esphome/components/sen6x/sensor.py @@ -3,15 +3,22 @@ from esphome.components import i2c, sensirion_common, sensor from esphome.components.const import CONF_NOX_INDEX, CONF_VOC_INDEX import esphome.config_validation as cv from esphome.const import ( + CONF_ALGORITHM_TUNING, CONF_CO2, CONF_FORMALDEHYDE, + CONF_GAIN_FACTOR, + CONF_GATING_MAX_DURATION_MINUTES, CONF_HUMIDITY, CONF_ID, + CONF_INDEX_OFFSET, + CONF_LEARNING_TIME_GAIN_HOURS, + CONF_LEARNING_TIME_OFFSET_HOURS, CONF_NOX, CONF_PM_1_0, CONF_PM_2_5, CONF_PM_4_0, CONF_PM_10_0, + CONF_STD_INITIAL, CONF_TEMPERATURE, CONF_TYPE, CONF_VOC, @@ -44,6 +51,42 @@ SEN6XComponent = sen6x_ns.class_( ) +def _gas_index_schema( + *, + index_offset: int, + gating_max_duration: int, + std_initial: int | None, +) -> cv.Schema: + """Sensor schema for a gas index sensor with optional algorithm tuning. + + std_initial is only configurable for VOC; the NOx algorithm requires 50. + """ + tuning_schema = { + cv.Optional(CONF_INDEX_OFFSET, default=index_offset): cv.int_range( + min=1, max=250 + ), + cv.Optional(CONF_LEARNING_TIME_OFFSET_HOURS, default=12): cv.int_range( + min=1, max=1000 + ), + cv.Optional(CONF_LEARNING_TIME_GAIN_HOURS, default=12): cv.int_range( + min=1, max=1000 + ), + cv.Optional( + CONF_GATING_MAX_DURATION_MINUTES, default=gating_max_duration + ): cv.int_range(min=0, max=3000), + cv.Optional(CONF_GAIN_FACTOR, default=230): cv.int_range(min=1, max=1000), + } + if std_initial is not None: + tuning_schema[cv.Optional(CONF_STD_INITIAL, default=std_initial)] = ( + cv.int_range(min=10, max=5000) + ) + return sensor.sensor_schema( + icon=ICON_RADIATOR, + accuracy_decimals=0, + state_class=STATE_CLASS_MEASUREMENT, + ).extend({cv.Optional(CONF_ALGORITHM_TUNING): cv.Schema(tuning_schema)}) + + CONFIG_SCHEMA = cv.All( cv.rename_key(CONF_VOC, CONF_VOC_INDEX, removed_in="2027.2.0", component="sen6x"), cv.rename_key(CONF_NOX, CONF_NOX_INDEX, removed_in="2027.2.0", component="sen6x"), @@ -94,15 +137,15 @@ CONFIG_SCHEMA = cv.All( device_class=DEVICE_CLASS_HUMIDITY, state_class=STATE_CLASS_MEASUREMENT, ), - cv.Optional(CONF_VOC_INDEX): sensor.sensor_schema( - icon=ICON_RADIATOR, - accuracy_decimals=0, - state_class=STATE_CLASS_MEASUREMENT, + cv.Optional(CONF_VOC_INDEX): _gas_index_schema( + index_offset=100, + gating_max_duration=180, + std_initial=50, ), - cv.Optional(CONF_NOX_INDEX): sensor.sensor_schema( - icon=ICON_RADIATOR, - accuracy_decimals=0, - state_class=STATE_CLASS_MEASUREMENT, + cv.Optional(CONF_NOX_INDEX): _gas_index_schema( + index_offset=1, + gating_max_duration=720, + std_initial=None, ), cv.Optional(CONF_CO2): sensor.sensor_schema( unit_of_measurement=UNIT_PARTS_PER_MILLION, @@ -149,3 +192,20 @@ async def to_code(config: ConfigType) -> None: if cfg := config.get(key): sens = await sensor.new_sensor(cfg) cg.add(getattr(var, func_name)(sens)) + + for key, setter in ( + (CONF_VOC_INDEX, "set_voc_algorithm_tuning"), + (CONF_NOX_INDEX, "set_nox_algorithm_tuning"), + ): + if (tuning := config.get(key, {}).get(CONF_ALGORITHM_TUNING)) is not None: + args = [ + tuning[CONF_INDEX_OFFSET], + tuning[CONF_LEARNING_TIME_OFFSET_HOURS], + tuning[CONF_LEARNING_TIME_GAIN_HOURS], + tuning[CONF_GATING_MAX_DURATION_MINUTES], + ] + # std_initial is in the schema for VOC only + if (std_initial := tuning.get(CONF_STD_INITIAL)) is not None: + args.append(std_initial) + args.append(tuning[CONF_GAIN_FACTOR]) + cg.add(getattr(var, setter)(*args)) diff --git a/tests/components/sen6x/common.yaml b/tests/components/sen6x/common.yaml index 859e012c4a..c9b6f22c0f 100644 --- a/tests/components/sen6x/common.yaml +++ b/tests/components/sen6x/common.yaml @@ -28,8 +28,21 @@ sensor: accuracy_decimals: 1 nox_index: name: NOx Index + algorithm_tuning: + index_offset: 8 + learning_time_offset_hours: 6 + learning_time_gain_hours: 24 + gating_max_duration_minutes: 900 + gain_factor: 180 voc_index: name: VOC Index + algorithm_tuning: + index_offset: 120 + learning_time_offset_hours: 6 + learning_time_gain_hours: 24 + gating_max_duration_minutes: 240 + std_initial: 75 + gain_factor: 180 co2: name: Carbon Dioxide formaldehyde: diff --git a/tests/components/sen6x/validate.esp32-idf.yaml b/tests/components/sen6x/validate.esp32-idf.yaml new file mode 100644 index 0000000000..3ae23af4ac --- /dev/null +++ b/tests/components/sen6x/validate.esp32-idf.yaml @@ -0,0 +1,18 @@ +# Config-only: partial algorithm_tuning blocks, so the schema defaults fill in the +# keys that are left out. +packages: + i2c: !include ../../test_build_components/common/i2c/esp32-idf.yaml + +sensor: + - platform: sen6x + id: sen6x_partial_tuning + type: SEN65 + i2c_id: i2c_bus + voc_index: + name: VOC Index + algorithm_tuning: + index_offset: 60 + nox_index: + name: NOx Index + algorithm_tuning: + gain_factor: 45 From 06d477bea73cd415cf7e118cce07285e1b5a6057 Mon Sep 17 00:00:00 2001 From: mfishma Date: Mon, 31 Aug 2026 10:47:55 -0700 Subject: [PATCH 053/147] [whynter] Fix truncating Fahrenheit temps that should be rounded (#18813) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/whynter/whynter.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/whynter/whynter.cpp b/esphome/components/whynter/whynter.cpp index b8a8db4d7c..f5f304aa17 100644 --- a/esphome/components/whynter/whynter.cpp +++ b/esphome/components/whynter/whynter.cpp @@ -84,8 +84,8 @@ void Whynter::transmit_state() { if (fahrenheit_) { remote_state |= UNIT_MASK; - uint8_t temp = - (uint8_t) clamp(esphome::celsius_to_fahrenheit(this->target_temperature), TEMP_MIN_F, TEMP_MAX_F); + uint8_t temp = (uint8_t) roundf( + clamp(esphome::celsius_to_fahrenheit(this->target_temperature), TEMP_MIN_F, TEMP_MAX_F)); temp = esphome::reverse_bits(temp); remote_state |= temp; } else { From ad69718eccce320e1134a2f04a92711ad4ab8955 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 31 Aug 2026 13:01:13 -0500 Subject: [PATCH 054/147] [core] Configure the platform again after prefetching its packages (#18830) --- esphome/platformio/prefetch.py | 68 +++++++++++++------ tests/unit_tests/test_platformio_prefetch.py | 70 ++++++++++++++++++++ 2 files changed, 118 insertions(+), 20 deletions(-) diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index ef8c27c9aa..1df0a4b328 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -16,8 +16,9 @@ name and promote with an atomic rename. from __future__ import annotations +from collections.abc import Iterator from concurrent.futures import ThreadPoolExecutor -from contextlib import suppress +from contextlib import contextmanager, suppress import hashlib import json import logging @@ -43,6 +44,17 @@ from esphome.helpers import get_bool_env, get_usable_cpu_count, rmtree _LOGGER = logging.getLogger(__name__) + +@contextmanager +def _preserved_sys_path() -> Iterator[None]: + """Platform setup may rewrite sys.path (pioarduino's penv does); undo it.""" + saved = list(sys.path) + try: + yield + finally: + sys.path[:] = saved + + # Concurrent registry resolutions / HEAD probes (each is network-bound) _RESOLVE_WORKERS = 8 @@ -96,6 +108,14 @@ class _Resolved(NamedTuple): cached: bool +class _Group(NamedTuple): + """The installable ``(name, spec)`` entries of one package manager.""" + + manager: Any + entries: list[tuple[str, Any]] + is_platform: bool + + # Child records a no-work run; the parent skips the next spawn while valid _SENTINEL_NAME = ".esphome_prefetch.json" _SENTINEL_SCHEMA = 1 @@ -772,13 +792,9 @@ def _preinstall( # poison the next wave; pio run installs the rest cleanly _LOGGER.warning("Skipping the dependency wave") return - # The builtin probe may construct platforms whose setup rewrites - # sys.path (see _prefetch); restore it for later imports - saved_sys_path = list(sys.path) - try: + # The builtin probe may construct platforms + with _preserved_sys_path(): next_entries = _dependency_entries(manager, installed, seen) - finally: - sys.path[:] = saved_sys_path if next_entries: # Terminates without a cap: every wave admits only never-seen # names, so a cycle yields an empty next wave @@ -803,15 +819,13 @@ def _prefetch(build_dir: Path, env: str) -> None: return # The platform (manifest plus build scripts) installs first and - # resolves the rest. Its setup may rewrite sys.path (pioarduino's penv - # setup does); restore it so later imports here still resolve. - saved_sys_path = list(sys.path) - pm = PlatformPackageManager() - _sweep_stale_sidecars(Path(pm.get_download_dir()), pm.DOWNLOAD_CACHE_EXPIRE) - pkg = pm.install(platform_spec, skip_dependencies=True) - p = PlatformFactory.new(pkg) - p.configure_project_packages(env, ["run"]) - sys.path[:] = saved_sys_path + # resolves the rest + with _preserved_sys_path(): + pm = PlatformPackageManager() + _sweep_stale_sidecars(Path(pm.get_download_dir()), pm.DOWNLOAD_CACHE_EXPIRE) + pkg = pm.install(platform_spec, skip_dependencies=True) + p = PlatformFactory.new(pkg) + p.configure_project_packages(env, ["run"]) specs = [ p.get_package_spec(name) @@ -851,9 +865,9 @@ def _prefetch(build_dir: Path, env: str) -> None: seen: set[str] = set() jobs: list[tuple[str, int, Any]] = [] - groups: list[tuple[Any, list[tuple[str, Any]]]] = [] + groups: list[_Group] = [] unresolved = 0 - for mgr, batch in ((p.pm, specs), (lm, lib_specs)): + for mgr, batch, is_platform in ((p.pm, specs, True), (lm, lib_specs, False)): entries: list[tuple[str, Any]] = [] for build_jobs in (_registry_jobs, _uri_jobs): batch_jobs, failed, installable = build_jobs(mgr, batch, seen) @@ -861,7 +875,7 @@ def _prefetch(build_dir: Path, env: str) -> None: unresolved += failed entries += installable if entries: - groups.append((mgr, entries)) + groups.append(_Group(mgr, entries, is_platform)) sentinel = build_dir / _SENTINEL_NAME if jobs or groups: @@ -890,7 +904,8 @@ def _prefetch(build_dir: Path, env: str) -> None: encoding="utf-8", ) - for mgr, entries in groups: + platform_packages_installed = False + for mgr, entries, is_platform in groups: # One install per destination: pio derives the directory from # the package name, so key on the name part to_install = { @@ -901,6 +916,8 @@ def _prefetch(build_dir: Path, env: str) -> None: if to_install: try: _preinstall(mgr, list(to_install.values())) + if is_platform: + platform_packages_installed = True except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught # Each group degrades independently; pio run installs # whatever this one did not @@ -910,6 +927,17 @@ def _prefetch(build_dir: Path, env: str) -> None: failure_reason(err), ) _LOGGER.debug("Pre-install group failure detail", exc_info=True) + if platform_packages_installed: + # pioarduino installs its real toolchains from configure (the registry + # package is a stub); settle that here so pio run does not redo it + with _preserved_sys_path(), ThreadPoolExecutor(max_workers=1) as ex: + # A worker so SIGTERM joins it; exception() so a postinstall exit only warns + err = ex.submit(p.configure_project_packages, env, ["run"]).exception() + if err is not None: + _LOGGER.warning( + "Could not settle platform packages: %s", failure_reason(err) + ) + _LOGGER.debug("Platform settle failure detail", exc_info=err) def _sigterm(_signum, _frame) -> None: diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 91fb78c6af..d0785d2724 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1417,6 +1417,76 @@ def test_prefetch_installs_cached_archives_without_downloads( assert not (tmp_path / pf._SENTINEL_NAME).exists() +@pytest.mark.parametrize( + ("platform_group", "lib_group", "expected"), + [ + ( + [("toolchain-x@1", _FakeSpec(name="toolchain-x"))], + [], + ["configure", "install", "configure"], + ), + ([], [("noise-c@1.0", _FakeSpec(name="noise-c"))], ["configure", "install"]), + ], +) +def test_prefetch_reconfigures_only_after_platform_installs( + tmp_path: Path, platform_group: list, lib_group: list, expected: list[str] +) -> None: + """Installed platform packages get a second configure pass; libraries do not.""" + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") + order: list[str] = [] + fake_platform = MagicMock() + fake_platform.packages = {} + fake_platform.configure_project_packages.side_effect = lambda env, targets: ( + order.append("configure") + ) + config = _fake_config( + tmp_path, {"platform": "fake/p@1", "lib_deps": ["esphome/noise-c@1.0"]} + ) + modules = _pio_modules(tmp_path, fake_platform, MagicMock(), config) + with ( + patch.dict("sys.modules", modules), + patch.object( + pf, + "_registry_jobs", + side_effect=[([], 0, platform_group), ([], 0, lib_group)], + ), + patch.object(pf, "_uri_jobs", return_value=([], 0, [])), + patch.object(pf, "_preinstall", side_effect=lambda *_: order.append("install")), + ): + pf._prefetch(tmp_path, "testenv") + assert order == expected + + +@pytest.mark.parametrize( + "err", [RuntimeError("idf_tools.py failed"), SystemExit("postinstall exited")] +) +def test_prefetch_settle_failure_warns_and_continues( + tmp_path: Path, caplog: pytest.LogCaptureFixture, err: BaseException +) -> None: + """A failing second configure pass only costs the speedup.""" + _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") + fake_platform = MagicMock() + fake_platform.packages = {} + fake_platform.configure_project_packages.side_effect = [None, err] + config = _fake_config(tmp_path, {"platform": "fake/p@1"}) + modules = _pio_modules(tmp_path, fake_platform, MagicMock(), config) + with ( + patch.dict("sys.modules", modules), + patch.object( + pf, + "_registry_jobs", + side_effect=[ + ([], 0, [("toolchain-x@1", _FakeSpec(name="toolchain-x"))]), + ([], 0, []), + ], + ), + patch.object(pf, "_uri_jobs", return_value=([], 0, [])), + patch.object(pf, "_preinstall"), + ): + pf._prefetch(tmp_path, "testenv") + assert f"Could not settle platform packages: {err}" in caplog.text + + def test_preinstall_extracts_in_parallel_under_one_lock(tmp_path: Path) -> None: """The manager lock wraps the whole batch; per-thread managers share its package dir; one failing install leaves the rest alone.""" From 5d56517e147d700114fa16085ac682c957f101e9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 31 Aug 2026 13:19:49 -0500 Subject: [PATCH 055/147] [api] Keep action dropped warning strings in flash on ESP8266 (#18905) --- esphome/components/api/api_server.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 751f2e4c3b..43d35363d3 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -433,8 +433,10 @@ void APIServer::send_homeassistant_action(const HomeassistantActionRequest &call // Home Assistant subscribes to actions shortly *after* authenticating, so actions // fired right at connection time (on_client_connected, on_time_sync, ...) can // arrive before the subscription and are lost - warn instead of failing silently. - ESP_LOGW(TAG, "Home Assistant %s '%s' dropped; %s", call.is_event ? "event" : "action", call.service.c_str(), - this->is_connected() ? "client has not subscribed to actions (yet)" : "no client connected"); + ESP_LOGW(TAG, "Home Assistant %s '%s' dropped; %s", + call.is_event ? LOG_STR_LITERAL("event") : LOG_STR_LITERAL("action"), call.service.c_str(), + this->is_connected() ? LOG_STR_LITERAL("client has not subscribed to actions (yet)") + : LOG_STR_LITERAL("no client connected")); } } #ifdef USE_API_HOMEASSISTANT_ACTION_RESPONSES From 1ba1aebfa1943d3a3b57232ee9a7c01c2d742cf2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 31 Aug 2026 13:20:47 -0500 Subject: [PATCH 056/147] [ci] Balance integration test buckets by recorded durations (#18895) --- .github/workflows/ci.yml | 23 ++- .../workflows/sync-integration-durations.yml | 98 ++++++++++++ script/determine-jobs.py | 63 ++++---- script/helpers.py | 65 ++++++++ script/update_integration_test_durations.py | 119 +++++++++++++++ tests/integration/conftest.py | 9 ++ .../integration_test_durations.json | 142 ++++++++++++++++++ tests/script/test_determine_jobs.py | 130 +++++++++++++--- tests/script/test_helpers.py | 27 ++++ .../test_update_integration_test_durations.py | 130 ++++++++++++++++ 10 files changed, 755 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/sync-integration-durations.yml create mode 100755 script/update_integration_test_durations.py create mode 100644 tests/integration/integration_test_durations.json create mode 100644 tests/script/test_update_integration_test_durations.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0df4da6386..a874a023b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,6 +92,7 @@ jobs: outputs: core-ci: ${{ steps.determine.outputs.core-ci }} integration-tests: ${{ steps.determine.outputs.integration-tests }} + integration-run-all: ${{ steps.determine.outputs.integration-run-all }} integration-test-buckets: ${{ steps.determine.outputs.integration-test-buckets }} clang-tidy: ${{ steps.determine.outputs.clang-tidy }} clang-tidy-mode: ${{ steps.determine.outputs.clang-tidy-mode }} @@ -152,6 +153,9 @@ jobs: # Extract individual fields echo "core-ci=$(echo "$output" | jq -r '.core_ci')" >> $GITHUB_OUTPUT echo "integration-tests=$(echo "$output" | jq -r '.integration_tests')" >> $GITHUB_OUTPUT + # A missing key must fail here, not silently disable the junit upload + run_all=$(echo "$output" | jq -r 'if has("integration_run_all") then .integration_run_all else error("integration_run_all missing") end') + echo "integration-run-all=${run_all}" >> $GITHUB_OUTPUT echo "integration-test-buckets=$(echo "$output" | jq -c '.integration_test_buckets')" >> $GITHUB_OUTPUT echo "clang-tidy=$(echo "$output" | jq -r '.clang_tidy')" >> $GITHUB_OUTPUT echo "clang-tidy-mode=$(echo "$output" | jq -r '.clang_tidy_mode')" >> $GITHUB_OUTPUT @@ -427,8 +431,25 @@ jobs: run: | . venv/bin/activate mapfile -t test_files < <(echo "$BUCKET_TESTS" | jq -r '.[]') + if [ "${#test_files[@]}" -eq 0 ]; then + echo "::error::Empty integration test bucket; pytest would collect the whole tree" + exit 1 + fi echo "Bucket ${{ matrix.bucket.name }}: running ${#test_files[@]} integration tests" - pytest -vv --no-cov --tb=native --durations=30 -n auto "${test_files[@]}" + pytest -vv --no-cov --tb=native --durations=30 -n auto --dist worksteal \ + --junitxml=junit-integration.xml "${test_files[@]}" + - name: Upload junit timings + # Consumed by sync-integration-durations.yml through + # script/update_integration_test_durations.py; only full matrix dev + # runs produce usable data. + if: github.ref == 'refs/heads/dev' && needs.determine-jobs.outputs.integration-run-all == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: junit-integration-${{ strategy.job-index }} + path: junit-integration.xml + if-no-files-found: error + # A full cron period of margin for the weekly refresh + retention-days: 14 - name: Print ccache statistics # esphome stores the PlatformIO ccache under the machine-global cache # dir (see _ccache_env() in esphome/platformio/toolchain.py). diff --git a/.github/workflows/sync-integration-durations.yml b/.github/workflows/sync-integration-durations.yml new file mode 100644 index 0000000000..d09a1cf242 --- /dev/null +++ b/.github/workflows/sync-integration-durations.yml @@ -0,0 +1,98 @@ +--- +name: Refresh integration test durations + +on: + workflow_dispatch: + schedule: + - cron: "45 5 * * 1" + +# Repo writes (branch push, PR open) happen via the App token minted below, +# so the workflow's GITHUB_TOKEN does not need any write scopes. +permissions: + contents: read + actions: read # gh api / gh run download for the CI junit artifacts + +jobs: + sync: + name: Refresh integration test durations + runs-on: ubuntu-latest + if: github.repository == 'esphome/esphome' + steps: + - name: Generate a token + id: generate-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.ESPHOME_GITHUB_APP_CLIENT_ID }} + private-key: ${{ secrets.ESPHOME_GITHUB_APP_PRIVATE_KEY }} + permission-contents: write # push the sync branch + permission-pull-requests: write # open or refresh the sync PR + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + + - name: Refresh from the newest usable dev run + env: + GH_TOKEN: ${{ github.token }} + run: | + # Only full matrix dev runs upload junit-integration-* artifacts + # (see the integration-tests job); the merge script re-checks + # coverage regardless. + # Newest-first candidates via their bucket-0 artifact. Fork PRs run + # their own ci.yml, so name and branch are spoofable; require + # same-repo. Assignment failures trip set -e and fail loudly. + candidates=$( + gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=junit-integration-0&per_page=100" \ + --jq '.artifacts[] | select(.expired | not) | .workflow_run + | select(.head_branch == "dev" and .head_repository_id != null + and .head_repository_id == .repository_id) + | .id' + ) + # Green runs first, then the rest newest first; a run missing a + # bucket fails the coverage check and the next one is tried + green="" + rest="" + for id in ${candidates}; do + conclusion=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${id}" --jq '.conclusion // ""') + if [ "${conclusion}" = "success" ]; then + green="${green} ${id}" + elif [ -n "${conclusion}" ]; then + rest="${rest} ${id}" + fi + done + # helpers.py imports colorama; the script needs nothing else + pip install colorama + for id in ${green} ${rest}; do + rm -rf /tmp/junit + if ! gh run download "${id}" --repo "${GITHUB_REPOSITORY}" -p "junit-integration-*" -D /tmp/junit; then + echo "::warning::Could not download artifacts for run ${id}; trying the next" + continue + fi + status=0 + python script/update_integration_test_durations.py /tmp/junit || status=$? + if [ "${status}" -eq 0 ]; then + echo "Refreshed from run ${id}" + exit 0 + fi + # Only EXIT_LOW_COVERAGE (3) from the script advances to the next run + [ "${status}" -eq 3 ] || exit 1 + echo "::warning::Run ${id} covers too few test files; trying the next" + done + echo "::error::No dev CI run with usable junit artifacts in range; the feed is starved" + exit 1 + + - name: Commit changes + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + commit-message: "[ci] Refresh integration test durations" + committer: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> + author: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> + branch: sync/integration-durations + delete-branch: true + title: "[ci] Refresh integration test durations" + body-path: .github/PULL_REQUEST_TEMPLATE.md + token: ${{ steps.generate-token.outputs.token }} diff --git a/script/determine-jobs.py b/script/determine-jobs.py index add1af5bba..f5412af21d 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -53,8 +53,10 @@ from collections import Counter from enum import StrEnum from functools import cache import json +import math import os from pathlib import Path +import statistics import sys from typing import Any @@ -67,7 +69,9 @@ from clang_tidy_hash import ( from helpers import ( CPP_FILE_EXTENSIONS, ESPHOME_TESTS_COMPONENTS_PATH, + INTEGRATION_TESTS_PATH, PYTHON_FILE_EXTENSIONS, + all_integration_test_files, base_python_changed, changed_files, core_changed, @@ -83,6 +87,8 @@ from helpers import ( get_target_branch, git_ls_files, is_validate_only_file, + load_integration_durations, + lpt_partition, root_path, ) from split_components_for_ci import create_intelligent_batches @@ -96,10 +102,13 @@ CLANG_TIDY_SPLIT_THRESHOLD = 65 # Isolated components count as 10x, groupable components count as 1x COMPONENT_TEST_BATCH_SIZE = 40 -# Integration test bucketing: when more than the threshold tests are scheduled, -# fan out across this many parallel jobs. Below the threshold, a single job runs. +# Above the threshold, fan out across up to this many jobs, balanced by the +# recorded per-file durations. The target is serial junit-time weight per +# bucket, not wall time (calibrated with the conftest compile cap); it +# sizes the bucket count for small subsets. INTEGRATION_TESTS_SPLIT_THRESHOLD = 10 -INTEGRATION_TESTS_SPLIT_BUCKETS = 3 +INTEGRATION_TESTS_SPLIT_BUCKETS = 5 +INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT = 360.0 # platformio and aioesphomeapi (requirements.txt), the pytest stack # (requirements_test.txt) and the fixture every session compiles; a change @@ -113,27 +122,13 @@ INTEGRATION_TESTS_TRIGGER_FILES = frozenset( ) -def _split_list(items: list[str], n: int) -> list[list[str]]: - """Split a list into n roughly-equal contiguous parts (matches script/clang-tidy).""" - k, m = divmod(len(items), n) - return [items[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n)] - - -def _all_integration_test_files() -> list[str]: - """Return all integration test file paths, sorted, relative to repo root.""" - return sorted( - str(p.relative_to(root_path)) - for p in (Path(root_path) / "tests" / "integration").glob("test_*.py") - ) - - def _compute_integration_test_buckets( integration_run_all: bool, integration_test_files: list[str], ) -> tuple[bool, list[dict[str, Any]]]: """Compute (run_integration, buckets) from the determine_integration_tests result. - Pure function for unit testing — no I/O beyond `_all_integration_test_files` + Pure function for unit testing — no I/O beyond `all_integration_test_files` when `integration_run_all` is set. `buckets` is a list of `{name, tests}` dicts where `tests` is a JSON-friendly @@ -141,7 +136,7 @@ def _compute_integration_test_buckets( shell word-splitting / glob hazards. """ if integration_run_all: - files = _all_integration_test_files() + files = all_integration_test_files() else: files = sorted(integration_test_files) @@ -152,12 +147,23 @@ def _compute_integration_test_buckets( return False, [] if len(files) > INTEGRATION_TESTS_SPLIT_THRESHOLD: - parts = [ - part for part in _split_list(files, INTEGRATION_TESTS_SPLIT_BUCKETS) if part - ] + durations = load_integration_durations() + # Unrecorded files weigh the recording's median; with no recording a + # file weighs a whole bucket, which keeps the full fan-out + default = ( + statistics.median(durations.values()) + if durations + else INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT + ) + weights = {f: durations.get(f, default) for f in files} + count = min( + INTEGRATION_TESTS_SPLIT_BUCKETS, + math.ceil(sum(weights.values()) / INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT), + ) + # count <= SPLIT_BUCKETS < threshold < len(files): no group is empty + parts = [sorted(part) for part in lpt_partition(files, weights, count)] buckets = [ - {"name": f"{i + 1}/{len(parts)}", "tests": part} - for i, part in enumerate(parts) + {"name": f"{i + 1}/{count}", "tests": part} for i, part in enumerate(parts) ] else: buckets = [{"name": "1/1", "tests": files}] @@ -264,9 +270,9 @@ def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[s # If infrastructure Python files changed (conftest, utils, etc.), run all tests # Excludes test files (test_*.py), fixtures, and non-Python files (README.md) if any( - f.startswith("tests/integration/") + f.startswith(INTEGRATION_TESTS_PATH) and f.endswith(".py") - and not f.startswith("tests/integration/test_") + and not f.startswith(f"{INTEGRATION_TESTS_PATH}test_") and "/fixtures/" not in f for f in files ): @@ -277,9 +283,9 @@ def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[s fixture_to_test_files = get_fixture_to_test_files() for f in files: - if f.startswith("tests/integration/test_") and f.endswith(".py"): + if f.startswith(f"{INTEGRATION_TESTS_PATH}test_") and f.endswith(".py"): test_files.add(f) - elif f.startswith("tests/integration/fixtures/"): + elif f.startswith(f"{INTEGRATION_TESTS_PATH}fixtures/"): if f.endswith(".yaml"): # Fixture YAML changed - add corresponding test file(s) test_files.update(fixture_to_test_files.get(Path(f).stem, ())) @@ -1415,6 +1421,7 @@ def main() -> None: output: dict[str, Any] = { "core_ci": run_core_ci, "integration_tests": run_integration, + "integration_run_all": integration_run_all, "integration_test_buckets": integration_test_buckets, "clang_tidy": run_clang_tidy, "clang_tidy_mode": clang_tidy_mode, diff --git a/script/helpers.py b/script/helpers.py index e648bb91bb..bf22e15808 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -43,6 +43,53 @@ ESPHOME_TESTS_COMPONENTS_PATH = "tests/components/" # Tuple of component and test paths for efficient startswith checks COMPONENT_AND_TESTS_PATHS = (ESPHOME_COMPONENTS_PATH, ESPHOME_TESTS_COMPONENTS_PATH) +# Integration tests path prefix +INTEGRATION_TESTS_PATH = "tests/integration/" + +# Per-file integration test durations from CI junit output; shared by the +# reader (determine-jobs) and writer (update_integration_test_durations) +INTEGRATION_TEST_DURATIONS_FILE = "tests/integration/integration_test_durations.json" + + +def all_integration_test_files() -> list[str]: + """Return all integration test file paths, sorted, relative to repo root.""" + return sorted( + p.relative_to(root_path).as_posix() + for p in (Path(root_path) / "tests" / "integration").glob("test_*.py") + ) + + +def load_integration_durations() -> dict[str, float]: + """Return recorded per-file pytest durations in seconds; empty when unavailable.""" + try: + raw = json.loads( + (Path(root_path) / INTEGRATION_TEST_DURATIONS_FILE).read_text() + ) + if not isinstance(raw, dict): + print( + f"integration durations unavailable: expected an object, " + f"got {type(raw).__name__}", + file=sys.stderr, + ) + return {} + except (OSError, ValueError) as err: + # The file ships in the repo; degrade to unweighted bucketing, loudly + print(f"integration durations unavailable: {err}", file=sys.stderr) + return {} + durations = { + key: seconds + for key, value in raw.items() + if isinstance(value, (int, float)) and (seconds := float(value)) > 0 + } + if len(durations) != len(raw): + # One bad entry must not discard the whole recording + print( + f"dropped {len(raw) - len(durations)} invalid duration entries", + file=sys.stderr, + ) + return durations + + # Base bus components - these ARE the bus implementations and should not # be flagged as needing migration since they are the platform/base components BASE_BUS_COMPONENTS = { @@ -1545,3 +1592,21 @@ def get_cpp_changed_components(files: list[str]) -> list[str]: if file.startswith(ESPHOME_COMPONENTS_PATH): affected.update(find_children_of_component(components_graph, component)) return sorted(c for c in affected if has_cpp_unit_tests(c, tests_dir)) + + +def lpt_partition( + items: list[str], weights: dict[str, float], count: int +) -> list[list[str]]: + """Partition items into `count` weight-balanced groups (LPT greedy). + + Heaviest item first into the lightest group. Ties keep input order, so + pass pre-sorted items for deterministic output. script/clang-tidy's + split_list is the unweighted contiguous sibling. + """ + groups: list[list[str]] = [[] for _ in range(count)] + group_weights = [0.0] * count + for item in sorted(items, key=lambda i: -weights[i]): + lightest = min(range(count), key=group_weights.__getitem__) + groups[lightest].append(item) + group_weights[lightest] += weights[item] + return groups diff --git a/script/update_integration_test_durations.py b/script/update_integration_test_durations.py new file mode 100755 index 0000000000..bbb959c0b2 --- /dev/null +++ b/script/update_integration_test_durations.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Merge CI junit output into tests/integration/integration_test_durations.json. + +The integration-tests CI job uploads one junit XML artifact per bucket on +full matrix dev runs. Download a run's artifacts and merge the per file +durations into the recording used by script/determine-jobs.py: + + gh run download --repo esphome/esphome -p "junit-integration-*" -D /tmp/junit + script/update_integration_test_durations.py /tmp/junit + +Missing files keep their previous recording and deleted files drop out; a +run covering under 90% of the test files aborts unless --allow-partial. +""" + +from __future__ import annotations + +import argparse +from collections import defaultdict +import json +from pathlib import Path +import sys +import xml.etree.ElementTree as ET + +from helpers import ( + INTEGRATION_TEST_DURATIONS_FILE, + INTEGRATION_TESTS_PATH, + all_integration_test_files, + load_integration_durations, + root_path, +) + +DURATIONS_FILE = Path(root_path) / INTEGRATION_TEST_DURATIONS_FILE +MIN_COVERAGE = 0.9 +# Exit code for the expected "run covers too few files" refusal, so the +# refresh workflow can move on to the next candidate run +EXIT_LOW_COVERAGE = 3 + + +def collect_durations(junit_dir: Path, known_files: set[str]) -> dict[str, float]: + """Sum junit testcase times per integration test file, in seconds.""" + durations: defaultdict[str, float] = defaultdict(float) + unmatched = 0 + xml_files = sorted(junit_dir.rglob("*.xml")) + if not xml_files: + raise SystemExit(f"no junit XML files found under {junit_dir}") + for xml_file in xml_files: + for testcase in ET.parse(xml_file).getroot().iter("testcase"): + # Skipped/errored testcases carry time="0"; recording them would + # overwrite a good previous duration + if any( + testcase.find(tag) is not None + for tag in ("skipped", "error", "failure") + ): + continue + # classname is the dotted module plus any test class, e.g. + # tests.integration.test_x or tests.integration.test_x.TestFoo + parts = testcase.get("classname", "").split(".") + if parts[:2] != ["tests", "integration"] or len(parts) < 3: + unmatched += 1 + continue + path = f"{INTEGRATION_TESTS_PATH}{parts[2]}.py" + if path not in known_files: + print(f"skipping unknown test module {path}", file=sys.stderr) + continue + durations[path] += float(testcase.get("time", "0")) + if unmatched: + # A junit naming change would otherwise shrink the recording silently + raise SystemExit( + f"{unmatched} testcases with unexpected classnames; the junit layout changed" + ) + # An all-skipped file totals 0.0; let the merge keep its previous entry + return {k: v for k, v in durations.items() if v > 0} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "junit_dir", type=Path, help="directory containing downloaded junit XML files" + ) + parser.add_argument( + "--allow-partial", + action="store_true", + help="merge a run covering under 90%% of the test files", + ) + args = parser.parse_args() + + on_disk = set(all_integration_test_files()) + if not on_disk: + raise SystemExit("no integration test files found; wrong checkout root?") + collected = collect_durations(args.junit_dir, on_disk) + coverage = len(collected.keys() & on_disk) / len(on_disk) + if coverage < MIN_COVERAGE and not args.allow_partial: + print( + f"artifacts cover only {coverage:.0%} of {len(on_disk)} test files; " + "use a full matrix run or pass --allow-partial to merge anyway", + file=sys.stderr, + ) + return EXIT_LOW_COVERAGE + + # Validated load: a bad previous entry cannot survive the round trip, and + # an unreadable file aborts rather than being overwritten + previous = load_integration_durations() + if DURATIONS_FILE.is_file() and not previous: + raise SystemExit(f"{DURATIONS_FILE} is unreadable; refusing to overwrite it") + # New recordings win, absent files keep theirs, deleted files drop out + merged = { + path: collected.get(path, previous.get(path)) + for path in sorted(on_disk) + if path in collected or path in previous + } + DURATIONS_FILE.write_text( + json.dumps({k: round(v, 2) for k, v in merged.items()}, indent=2) + "\n" + ) + print(f"wrote {len(merged)} entries to {DURATIONS_FILE} ({coverage:.0%} fresh)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 12b1407fe1..6777e6cabc 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -23,6 +23,7 @@ import pytest_asyncio import esphome.config from esphome.core import CORE +from esphome.helpers import get_usable_cpu_count from esphome.platformio.toolchain import get_idedata from .const import ( @@ -67,6 +68,14 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]: env["PLATFORMIO_LIBDEPS_DIR"] = str(cache_dir / "libdeps" / worker) # Prevent cache cleaning during integration tests env["ESPHOME_SKIP_CLEAN_BUILD"] = "1" + # Cap each compile's -j so several xdist workers do not each spawn a + # full-width compiler fan-out on the same machine. An explicit env wins. + if "ESPHOME_DEFAULT_COMPILE_PROCESS_LIMIT" not in os.environ: + workers = int(os.environ.get("PYTEST_XDIST_WORKER_COUNT", "1")) + # Floor of 2 keeps a lone tail compile from running fully serial + env["ESPHOME_DEFAULT_COMPILE_PROCESS_LIMIT"] = str( + max(2, get_usable_cpu_count() // workers) + ) # Compile with THIS tree's esphome sources, not wherever the venv's editable # install points (which may be a different git worktree or checkout). repo_root = str(Path(__file__).resolve().parent.parent.parent) diff --git a/tests/integration/integration_test_durations.json b/tests/integration/integration_test_durations.json new file mode 100644 index 0000000000..9bada5cd36 --- /dev/null +++ b/tests/integration/integration_test_durations.json @@ -0,0 +1,142 @@ +{ + "tests/integration/test_action_concurrent_reentry.py": 45.23, + "tests/integration/test_addressable_light_transition.py": 74.47, + "tests/integration/test_alarm_control_panel_state_transitions.py": 74.1, + "tests/integration/test_api_action_metadata.py": 62.1, + "tests/integration/test_api_action_responses.py": 71.08, + "tests/integration/test_api_action_timeout.py": 21.64, + "tests/integration/test_api_conditional_memory.py": 13.72, + "tests/integration/test_api_custom_services.py": 24.16, + "tests/integration/test_api_get_time_response_timezone.py": 23.48, + "tests/integration/test_api_homeassistant.py": 37.87, + "tests/integration/test_api_homeassistant_action_no_subscriber.py": 14.38, + "tests/integration/test_api_list_entities_backpressure.py": 26.85, + "tests/integration/test_api_message_size_batching.py": 33.36, + "tests/integration/test_api_reboot_timeout.py": 13.63, + "tests/integration/test_api_string_lambda.py": 25.04, + "tests/integration/test_api_vv_logging.py": 16.6, + "tests/integration/test_api_zero_psk_provisioning.py": 43.14, + "tests/integration/test_areas_and_devices.py": 25.98, + "tests/integration/test_automation_wait_actions.py": 21.91, + "tests/integration/test_automations.py": 42.43, + "tests/integration/test_batch_delay_zero_rapid_transitions.py": 16.65, + "tests/integration/test_binary_sensor_autorepeat_filter.py": 28.67, + "tests/integration/test_binary_sensor_invalidate_state.py": 23.69, + "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 22.99, + "tests/integration/test_build_info.py": 24.96, + "tests/integration/test_camera_mock.py": 14.47, + "tests/integration/test_climate_control_action.py": 31.07, + "tests/integration/test_climate_custom_modes.py": 28.59, + "tests/integration/test_continuation_actions.py": 14.96, + "tests/integration/test_cover_control_action.py": 26.14, + "tests/integration/test_crc8_helper.py": 10.92, + "tests/integration/test_device_id_in_state.py": 64.97, + "tests/integration/test_duplicate_entities.py": 30.81, + "tests/integration/test_entity_icon.py": 32.85, + "tests/integration/test_fan_turn_on_action.py": 24.91, + "tests/integration/test_fnv1_hash_object_id.py": 12.54, + "tests/integration/test_fnv1a_hash.py": 21.8, + "tests/integration/test_gpio_expander_cache.py": 5.2, + "tests/integration/test_host_logger_thread_safety.py": 21.7, + "tests/integration/test_host_mode_basic.py": 13.62, + "tests/integration/test_host_mode_batch_delay.py": 14.56, + "tests/integration/test_host_mode_climate_basic_state.py": 30.95, + "tests/integration/test_host_mode_climate_control.py": 29.06, + "tests/integration/test_host_mode_empty_string_options.py": 27.22, + "tests/integration/test_host_mode_entity_fields.py": 30.95, + "tests/integration/test_host_mode_fan_preset.py": 14.44, + "tests/integration/test_host_mode_many_entities.py": 54.13, + "tests/integration/test_host_mode_many_entities_multiple_connections.py": 32.17, + "tests/integration/test_host_mode_noise_encryption.py": 42.77, + "tests/integration/test_host_mode_reconnect.py": 4.06, + "tests/integration/test_host_mode_sensor.py": 13.47, + "tests/integration/test_host_ota.py": 21.4, + "tests/integration/test_host_preferences.py": 25.43, + "tests/integration/test_host_preferences_suspend_resume.py": 19.2, + "tests/integration/test_improv_serial_uart.py": 31.52, + "tests/integration/test_large_message_batching.py": 15.64, + "tests/integration/test_legacy_area.py": 22.63, + "tests/integration/test_legacy_climate_compat.py": 26.13, + "tests/integration/test_legacy_fan_compat.py": 24.05, + "tests/integration/test_light_automations.py": 30.86, + "tests/integration/test_light_binary_effect_off_phase.py": 23.19, + "tests/integration/test_light_calls.py": 32.35, + "tests/integration/test_light_constant_brightness.py": 29.89, + "tests/integration/test_light_control_action.py": 29.06, + "tests/integration/test_light_dim_relative_action.py": 29.61, + "tests/integration/test_light_effect_zero_brightness.py": 18.68, + "tests/integration/test_light_initial_state.py": 24.49, + "tests/integration/test_light_toggle_action.py": 26.46, + "tests/integration/test_lock_automations.py": 23.28, + "tests/integration/test_logger_buffered_recursion_guard.py": 24.29, + "tests/integration/test_loop_disable_enable.py": 45.28, + "tests/integration/test_loop_interval_decoupling.py": 28.35, + "tests/integration/test_loop_interval_default_not_pulled_forward.py": 21.97, + "tests/integration/test_micros_to_millis.py": 20.79, + "tests/integration/test_multi_click_trigger.py": 26.2, + "tests/integration/test_multi_device_preferences.py": 16.87, + "tests/integration/test_noise_encryption_key_protection.py": 77.05, + "tests/integration/test_object_id_api_verification.py": 73.51, + "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 62.33, + "tests/integration/test_object_id_no_friendly_name.py": 43.47, + "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 32.21, + "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 56.86, + "tests/integration/test_online_image_bmp.py": 50.9, + "tests/integration/test_oversized_payloads.py": 53.2, + "tests/integration/test_preference_key_stability.py": 26.09, + "tests/integration/test_runtime_stats.py": 18.34, + "tests/integration/test_safe_mode_loop_runs.py": 10.07, + "tests/integration/test_scheduler_blocking_warning.py": 40.91, + "tests/integration/test_scheduler_bulk_cleanup.py": 23.14, + "tests/integration/test_scheduler_defer_cancel.py": 24.54, + "tests/integration/test_scheduler_defer_cancel_regular.py": 13.48, + "tests/integration/test_scheduler_defer_fifo_simple.py": 26.86, + "tests/integration/test_scheduler_defer_stress.py": 27.23, + "tests/integration/test_scheduler_heap_stress.py": 24.02, + "tests/integration/test_scheduler_internal_id_no_collision.py": 24.57, + "tests/integration/test_scheduler_interval_reschedule.py": 13.12, + "tests/integration/test_scheduler_interval_zero_coerced.py": 22.91, + "tests/integration/test_scheduler_null_name.py": 23.46, + "tests/integration/test_scheduler_numeric_id_test.py": 24.54, + "tests/integration/test_scheduler_pool.py": 25.0, + "tests/integration/test_scheduler_rapid_cancellation.py": 14.68, + "tests/integration/test_scheduler_recursive_timeout.py": 25.35, + "tests/integration/test_scheduler_removed_item_race.py": 26.19, + "tests/integration/test_scheduler_self_keyed.py": 23.43, + "tests/integration/test_scheduler_simultaneous_callbacks.py": 22.16, + "tests/integration/test_scheduler_string_test.py": 15.22, + "tests/integration/test_script_array_params.py": 14.67, + "tests/integration/test_script_delay_params.py": 15.65, + "tests/integration/test_script_queued.py": 24.93, + "tests/integration/test_script_queued_idle_loop.py": 5.04, + "tests/integration/test_script_wait_on_boot.py": 13.08, + "tests/integration/test_select_stringref_trigger.py": 29.6, + "tests/integration/test_sensor_filters_delta.py": 28.01, + "tests/integration/test_sensor_filters_ring_buffer.py": 25.04, + "tests/integration/test_sensor_filters_sliding_window.py": 71.5, + "tests/integration/test_sensor_filters_value_list.py": 16.94, + "tests/integration/test_sensor_timeout_filter.py": 29.48, + "tests/integration/test_socket_wake_gate_tcp.py": 20.36, + "tests/integration/test_status_flags.py": 37.42, + "tests/integration/test_strftime_to.py": 22.61, + "tests/integration/test_syslog.py": 16.34, + "tests/integration/test_template_alarm_control_panel_many_sensors.py": 29.81, + "tests/integration/test_template_text_save.py": 25.43, + "tests/integration/test_text_command.py": 23.34, + "tests/integration/test_text_sensor_raw_state.py": 69.57, + "tests/integration/test_uart_mock_ld2410.py": 37.95, + "tests/integration/test_uart_mock_ld2412.py": 93.22, + "tests/integration/test_uart_mock_ld2420.py": 43.24, + "tests/integration/test_uart_mock_ld2450.py": 31.75, + "tests/integration/test_uart_mock_modbus.py": 667.4, + "tests/integration/test_udp.py": 9.38, + "tests/integration/test_use_address_runtime.py": 37.05, + "tests/integration/test_valve_control_action.py": 24.47, + "tests/integration/test_varint_five_byte_device_id.py": 25.03, + "tests/integration/test_wait_until_mid_loop_timing.py": 23.73, + "tests/integration/test_wait_until_on_boot.py": 9.16, + "tests/integration/test_wait_until_ordering.py": 13.3, + "tests/integration/test_wait_until_reentrant_restart.py": 25.23, + "tests/integration/test_wake_loop_forces_phase_b.py": 23.34, + "tests/integration/test_water_heater_template.py": 17.67 +} diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 7b641e275e..4971821969 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -151,9 +151,14 @@ def test_main_all_tests_should_run( patch.object(determine_jobs, "_is_clang_tidy_full_scan", return_value=False), patch.object( determine_jobs, - "_all_integration_test_files", + "all_integration_test_files", return_value=fake_test_files, ), + patch.object( + determine_jobs, + "load_integration_durations", + return_value=dict.fromkeys(fake_test_files, 200.0), + ), patch.object( determine_jobs, "get_changed_components", @@ -189,24 +194,12 @@ def test_main_all_tests_should_run( output = json.loads(captured.out) assert output["integration_tests"] is True - # run_all=True expands to the full glob and pre-buckets into 3 parts. - # Each bucket's `tests` is a JSON list of file paths. + assert output["integration_run_all"] is True + # run_all=True expands to the full glob; balance and naming are pinned + # by the unit tests, main() only needs to round-trip the structure assert isinstance(output["integration_test_buckets"], list) - assert len(output["integration_test_buckets"]) == 3 - assert [b["name"] for b in output["integration_test_buckets"]] == [ - "1/3", - "2/3", - "3/3", - ] - for bucket in output["integration_test_buckets"]: - assert isinstance(bucket["tests"], list) - for path in bucket["tests"]: - assert isinstance(path, str) bucket_files = [f for b in output["integration_test_buckets"] for f in b["tests"]] - assert bucket_files == fake_test_files - # Bucket sizes are balanced (max-min difference at most 1). - sizes = [len(b["tests"]) for b in output["integration_test_buckets"]] - assert max(sizes) - min(sizes) <= 1 + assert sorted(bucket_files) == fake_test_files assert output["clang_tidy"] is True assert output["clang_tidy_mode"] in ["nosplit", "split"] assert output["clang_format"] is True @@ -509,14 +502,24 @@ def test_compute_integration_test_buckets_at_threshold_stays_single() -> None: def test_compute_integration_test_buckets_just_over_threshold_splits() -> None: - """One file over the threshold triggers the 3-bucket fan-out, balanced.""" + """One file over the threshold fans out fully when the weights demand it.""" n = determine_jobs.INTEGRATION_TESTS_SPLIT_THRESHOLD + 1 files = [f"tests/integration/test_{i:02d}.py" for i in range(n)] - run, buckets = determine_jobs._compute_integration_test_buckets(False, files) + with patch.object( + determine_jobs, + "load_integration_durations", + return_value=dict.fromkeys(files, 200.0), + ): + run, buckets = determine_jobs._compute_integration_test_buckets(False, files) assert run is True - assert [b["name"] for b in buckets] == ["1/3", "2/3", "3/3"] - union = [path for b in buckets for path in b["tests"]] + # threshold+1 files x 200s caps at the maximum bucket count. + n_buckets = determine_jobs.INTEGRATION_TESTS_SPLIT_BUCKETS + assert [b["name"] for b in buckets] == [ + f"{i + 1}/{n_buckets}" for i in range(n_buckets) + ] + union = sorted(path for b in buckets for path in b["tests"]) assert union == sorted(files) + # Equal weights => bucket sizes are balanced (difference at most 1). sizes = [len(b["tests"]) for b in buckets] assert max(sizes) - min(sizes) <= 1 @@ -526,7 +529,7 @@ def test_compute_integration_test_buckets_run_all_with_empty_glob_disables_run() ): """run_all=True but glob returns no files => run suppressed (otherwise pytest would collect tests outside tests/integration/).""" - with patch.object(determine_jobs, "_all_integration_test_files", return_value=[]): + with patch.object(determine_jobs, "all_integration_test_files", return_value=[]): run, buckets = determine_jobs._compute_integration_test_buckets(True, []) assert run is False assert buckets == [] @@ -3146,3 +3149,86 @@ def test_memory_impact_elf_layouts_are_found(tmp_path: Path) -> None: elf.write_text("") assert find_elf_path(build_path) == elf, f"{platform} ELF not found" + + +def test_compute_integration_test_buckets_no_durations_full_fanout() -> None: + """Without recorded durations the fan-out stays at the maximum.""" + files = [f"tests/integration/test_{i:03d}.py" for i in range(15)] + with patch.object(determine_jobs, "load_integration_durations", return_value={}): + run, buckets = determine_jobs._compute_integration_test_buckets(False, files) + assert run is True + assert len(buckets) == determine_jobs.INTEGRATION_TESTS_SPLIT_BUCKETS + assert sorted(f for b in buckets for f in b["tests"]) == files + + +def test_compute_integration_test_buckets_adaptive_count() -> None: + """A small recorded total weight collapses to one bucket above the threshold.""" + files = [f"tests/integration/test_{i:03d}.py" for i in range(15)] + with patch.object( + determine_jobs, + "load_integration_durations", + return_value=dict.fromkeys(files, 10.0), + ): + run, buckets = determine_jobs._compute_integration_test_buckets(False, files) + assert run is True + # 15 files x 10s recorded = 150s, under the per-bucket weight target. + assert [b["name"] for b in buckets] == ["1/1"] + assert buckets[0]["tests"] == files + + +def test_compute_integration_test_buckets_duration_weighted() -> None: + """Heavy files spread across buckets instead of clustering by sorted name.""" + files = [f"tests/integration/test_{i:03d}.py" for i in range(12)] + durations = dict.fromkeys(files, 10.0) + durations[files[0]] = 600.0 + durations[files[1]] = 600.0 + with patch.object( + determine_jobs, "load_integration_durations", return_value=durations + ): + run, buckets = determine_jobs._compute_integration_test_buckets(False, files) + assert run is True + assert len(buckets) >= 2 + heavy_buckets = [b for b in buckets if set(files[:2]) & set(b["tests"])] + assert len(heavy_buckets) == 2, "heavy files should land in different buckets" + assert sorted(f for b in buckets for f in b["tests"]) == files + + +def test_load_integration_durations_missing_or_corrupt(tmp_path: Path) -> None: + """Missing or unparsable durations data degrades to an empty mapping.""" + with patch.object(helpers, "root_path", str(tmp_path)): + assert determine_jobs.load_integration_durations() == {} + durations_file = tmp_path / helpers.INTEGRATION_TEST_DURATIONS_FILE + durations_file.parent.mkdir(parents=True) + durations_file.write_text("not json") + assert determine_jobs.load_integration_durations() == {} + durations_file.write_text('{"tests/integration/test_a.py": 12.5}') + assert determine_jobs.load_integration_durations() == { + "tests/integration/test_a.py": 12.5 + } + # Non-positive entries are dropped, valid ones survive + durations_file.write_text( + '{"tests/integration/test_a.py": 12.5, "tests/integration/test_b.py": -1}' + ) + assert determine_jobs.load_integration_durations() == { + "tests/integration/test_a.py": 12.5 + } + # One non-numeric entry cannot discard the whole recording + durations_file.write_text( + '{"tests/integration/test_a.py": 12.5, "tests/integration/test_b.py": null}' + ) + assert determine_jobs.load_integration_durations() == { + "tests/integration/test_a.py": 12.5 + } + # A non-dict top level degrades to empty + durations_file.write_text("[12.5]") + assert determine_jobs.load_integration_durations() == {} + + +def test_committed_integration_durations_are_sane() -> None: + """The committed recording itself holds positive bounded floats.""" + raw = json.loads( + (Path(helpers.root_path) / helpers.INTEGRATION_TEST_DURATIONS_FILE).read_text() + ) + assert raw, "committed durations file missing or empty" + assert all(isinstance(v, (int, float)) and 0 < v < 86400 for v in raw.values()) + assert all(k.startswith("tests/integration/test_") for k in raw) diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 38b8c57368..7d4059da2f 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -2120,3 +2120,30 @@ def test_get_cpp_changed_components_independent_of_cwd( assert helpers.get_cpp_changed_components( ["tests/components/time/__init__.py"] ) == ["time"] + + +def test_lpt_partition_balances_skewed_weights() -> None: + """Heavy items spread across groups instead of clustering.""" + items = [f"i{n}" for n in range(6)] + weights = {"i0": 100.0, "i1": 90.0, "i2": 10.0, "i3": 10.0, "i4": 5.0, "i5": 5.0} + groups = helpers.lpt_partition(items, weights, 2) + group_weights = sorted(sum(weights[i] for i in g) for g in groups) + # Contiguous split would give 200 vs 20; LPT lands at 110 vs 110 + assert group_weights == [110.0, 110.0] + assert sorted(i for g in groups for i in g) == items + + +def test_lpt_partition_more_groups_than_items() -> None: + """Surplus groups come back empty; every item still lands somewhere.""" + items = ["a", "b"] + groups = helpers.lpt_partition(items, {"a": 1.0, "b": 1.0}, 4) + assert len(groups) == 4 + assert sorted(i for g in groups for i in g) == items + assert sum(not g for g in groups) == 2 + + +def test_lpt_partition_tie_determinism() -> None: + """Equal weights assign in input order, so output is reproducible.""" + items = [f"i{n}" for n in range(4)] + weights = dict.fromkeys(items, 1.0) + assert helpers.lpt_partition(items, weights, 2) == [["i0", "i2"], ["i1", "i3"]] diff --git a/tests/script/test_update_integration_test_durations.py b/tests/script/test_update_integration_test_durations.py new file mode 100644 index 0000000000..f2f373d4bb --- /dev/null +++ b/tests/script/test_update_integration_test_durations.py @@ -0,0 +1,130 @@ +"""Unit tests for script/update_integration_test_durations.py.""" + +import json +from pathlib import Path +import sys +from unittest.mock import patch + +import pytest + +# Add the script directory to Python path so we can import the module +script_dir = str((Path(__file__).parent / ".." / ".." / "script").resolve()) +sys.path.insert(0, script_dir) + +import helpers # noqa: E402 +import update_integration_test_durations as uitd # noqa: E402 + +JUNIT_TEMPLATE = """ +{testcases} +""" + +KNOWN = { + "tests/integration/test_a.py", + "tests/integration/test_b.py", +} + + +def _write_junit(path: Path, testcases: str) -> None: + path.write_text(JUNIT_TEMPLATE.format(testcases=testcases), encoding="utf-8") + + +def test_collect_durations_sums_per_file(tmp_path: Path) -> None: + """Testcases from the same module sum.""" + _write_junit( + tmp_path / "a.xml", + '' + '' + '', + ) + assert uitd.collect_durations(tmp_path, KNOWN) == { + "tests/integration/test_a.py": 3.5, + "tests/integration/test_b.py": 4.0, + } + + +def test_collect_durations_class_based_testcase(tmp_path: Path) -> None: + """A class-based classname still maps to its module file.""" + _write_junit( + tmp_path / "a.xml", + '', + ) + assert uitd.collect_durations(tmp_path, KNOWN) == { + "tests/integration/test_a.py": 2.5 + } + + +def test_collect_durations_unknown_module_skipped( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A classname that maps to no known file is skipped with a warning.""" + _write_junit( + tmp_path / "a.xml", + '', + ) + assert uitd.collect_durations(tmp_path, KNOWN) == {} + assert "test_gone" in capsys.readouterr().err + + +def test_collect_durations_skips_skipped_testcases(tmp_path: Path) -> None: + """Skipped testcases do not record a bogus zero duration.""" + _write_junit( + tmp_path / "a.xml", + '' + "", + ) + assert uitd.collect_durations(tmp_path, KNOWN) == {} + + +def test_collect_durations_unexpected_classname_aborts(tmp_path: Path) -> None: + """A classname outside tests.integration means the junit layout changed.""" + _write_junit( + tmp_path / "a.xml", + '', + ) + with pytest.raises(SystemExit): + uitd.collect_durations(tmp_path, KNOWN) + + +def test_collect_durations_empty_dir_aborts(tmp_path: Path) -> None: + """No junit XML at all is a hard error, not an empty recording.""" + with pytest.raises(SystemExit): + uitd.collect_durations(tmp_path, KNOWN) + + +def test_main_merges_partial_run(tmp_path: Path) -> None: + """A partial run merges over the previous data instead of truncating it.""" + tests_dir = tmp_path / "tests" / "integration" + tests_dir.mkdir(parents=True) + for name in ("test_a", "test_b", "test_c"): + (tests_dir / f"{name}.py").write_text("", encoding="utf-8") + durations_file = tmp_path / helpers.INTEGRATION_TEST_DURATIONS_FILE + durations_file.write_text( + json.dumps( + { + "tests/integration/test_a.py": 5.0, + "tests/integration/test_b.py": 7.0, + "tests/integration/test_gone.py": 9.0, + } + ), + encoding="utf-8", + ) + junit_dir = tmp_path / "junit" + junit_dir.mkdir() + _write_junit( + junit_dir / "a.xml", + '', + ) + with ( + patch.object(helpers, "root_path", str(tmp_path)), + patch.object(uitd, "DURATIONS_FILE", durations_file), + ): + # 1 of 3 files covered: refused without --allow-partial + with patch.object(sys, "argv", ["uitd", str(junit_dir)]): + assert uitd.main() == uitd.EXIT_LOW_COVERAGE + with patch.object(sys, "argv", ["uitd", str(junit_dir), "--allow-partial"]): + assert uitd.main() == 0 + # test_a updated, test_b kept, deleted test_gone dropped + assert json.loads(durations_file.read_text()) == { + "tests/integration/test_a.py": 6.0, + "tests/integration/test_b.py": 7.0, + } From ea1e0c6f6897ebb901664ccc1155047dda4de0b9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 31 Aug 2026 13:27:53 -0500 Subject: [PATCH 057/147] [core] Keep scheduler dump cancelled marker string in flash on ESP8266 (#18906) --- esphome/core/scheduler.cpp | 15 ++++++++------- esphome/core/scheduler.h | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index e9c5bf2c04..afb323f78e 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -193,7 +193,7 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type } #ifdef ESPHOME_DEBUG_SCHEDULER - this->debug_log_timer_(item, name_type, static_name, hash_or_id, type, delay, now_64); + this->debug_log_timer_(item, name_type, static_name, hash_or_id, delay, now_64); #endif /* ESPHOME_DEBUG_SCHEDULER */ } @@ -438,9 +438,10 @@ uint32_t HOT Scheduler::call(uint32_t now) { SchedulerNameLog name_log; bool is_cancelled = is_item_removed_(item); ESP_LOGD(TAG, " %s '%s/%s' interval=%" PRIu32 " next_execution in %" PRIu64 "ms at %" PRIu64 "%s", - item->get_type_str(), LOG_STR_ARG(item->get_source()), + LOG_STR_ARG(item->get_type_str()), LOG_STR_ARG(item->get_source()), name_log.format(item->get_name_type(), item->get_name(), item->get_name_hash_or_id()), item->interval, - item->get_next_execution() - now_64, item->get_next_execution(), is_cancelled ? " [CANCELLED]" : ""); + item->get_next_execution() - now_64, item->get_next_execution(), + is_cancelled ? LOG_STR_LITERAL(" [CANCELLED]") : LOG_STR_LITERAL("")); old_items.push_back(item); } @@ -512,7 +513,7 @@ uint32_t HOT Scheduler::call(uint32_t now) { { SchedulerNameLog name_log; ESP_LOGV(TAG, "Running %s '%s/%s' with interval=%" PRIu32 " next_execution=%" PRIu64 " (now=%" PRIu64 ")", - item->get_type_str(), LOG_STR_ARG(item->get_source()), + LOG_STR_ARG(item->get_type_str()), LOG_STR_ARG(item->get_source()), name_log.format(item->get_name_type(), item->get_name(), item->get_name_hash_or_id()), item->interval, item->get_next_execution(), now_64); } @@ -794,7 +795,7 @@ void Scheduler::trim_freelist() { #ifdef ESPHOME_DEBUG_SCHEDULER void Scheduler::debug_log_timer_(const SchedulerItem *item, NameType name_type, const char *static_name, - uint32_t hash_or_id, SchedulerItem::Type type, uint32_t delay, uint64_t now) { + uint32_t hash_or_id, uint32_t delay, uint64_t now) { // Validate static strings in debug mode if (name_type == NameType::STATIC_STRING && static_name != nullptr) { validate_static_string(static_name); @@ -802,8 +803,8 @@ void Scheduler::debug_log_timer_(const SchedulerItem *item, NameType name_type, // Debug logging SchedulerNameLog name_log; - const char *type_str = (type == SchedulerItem::TIMEOUT) ? "timeout" : "interval"; - if (type == SchedulerItem::TIMEOUT) { + const char *type_str = LOG_STR_ARG(item->get_type_str()); + if (item->type == SchedulerItem::TIMEOUT) { ESP_LOGD(TAG, "set_%s(name='%s/%s', %s=%" PRIu32 ")", type_str, LOG_STR_ARG(item->get_source()), name_log.format(name_type, static_name, hash_or_id), type_str, delay); } else { diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 8ef3499a11..56fc83f12f 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -254,7 +254,7 @@ class Scheduler { // This is correct because millis_major_ that creates these values is also 16 bits. next_execution_high_ = static_cast(value >> 32); } - constexpr const char *get_type_str() const { return (type == TIMEOUT) ? "timeout" : "interval"; } + const LogString *get_type_str() const { return (type == TIMEOUT) ? LOG_STR("timeout") : LOG_STR("interval"); } // The owning component, or nullptr for SELF_POINTER items (whose slot holds source_name instead). // All component access goes through this so SELF_POINTER items read as component-less. Component *get_component() const { return name_type_ == NameType::SELF_POINTER ? nullptr : component; } @@ -404,7 +404,7 @@ class Scheduler { #ifdef ESPHOME_DEBUG_SCHEDULER // Helper for debug logging in set_timer_common_ - extracted to reduce code size void debug_log_timer_(const SchedulerItem *item, NameType name_type, const char *static_name, uint32_t hash_or_id, - SchedulerItem::Type type, uint32_t delay, uint64_t now); + uint32_t delay, uint64_t now); #endif /* ESPHOME_DEBUG_SCHEDULER */ #ifndef ESPHOME_THREAD_SINGLE From 61e37cfc8614af14191864988556046d9730a944 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Tue, 1 Sep 2026 07:01:53 +1000 Subject: [PATCH 058/147] [docker] Use GITHUB_REPOSITORY instead of hardcoded esphome/esphome (#18916) --- docker/generate_tags.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/generate_tags.py b/docker/generate_tags.py index 31f98c4614..a54205f1bf 100755 --- a/docker/generate_tags.py +++ b/docker/generate_tags.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import argparse +import os import re CHANNEL_DEV = "dev" @@ -64,7 +65,8 @@ def main(): suffix = f"-{args.suffix}" if args.suffix else "" - image_name = f"esphome/esphome{suffix}" + repository = (os.environ.get("GITHUB_REPOSITORY") or "esphome/esphome").strip().lower() + image_name = f"{repository}{suffix}" print(f"channel={channel}") From 24af3fd8343ad7fb8c7db6f5c5f126af240a1e6a Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Tue, 1 Sep 2026 07:13:36 +1000 Subject: [PATCH 059/147] [lvgl] Fix user_ flags (#18902) --- esphome/components/lvgl/defines.py | 4 ---- tests/components/lvgl/lvgl-package.yaml | 26 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index 61d15752be..1eee8041f9 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -628,10 +628,6 @@ OBJ_FLAGS = ( "send_draw_task_events", "widget_1", "widget_2", - "user_1", - "user_2", - "user_3", - "user_4", ) LV_OBJ_FLAG = LvConstant("LV_OBJ_FLAG_", *OBJ_FLAGS) diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index b457ec2c0b..07c492db35 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -188,6 +188,8 @@ lvgl: dark_mode: true obj: border_width: 1 + user_1: + bg_color: black gradients: - id: color_bar @@ -717,6 +719,30 @@ lvgl: id: button_with_text text: Clicked + # Exercises the LV_STATE_USER_1..USER_4 states: setting them at creation + # (both literal and lambda), styling each of them individually, and + # setting/clearing them at runtime with lvgl.widget.update. + - button: + id: user_flags_button + text: User flags + state: + user_1: true + user_2: !lambda return true; + user_1: + bg_color: 0xFF00FF + user_2: + bg_color: 0x00FFFF + user_3: + bg_color: 0xFFFF00 + user_4: + bg_color: 0x808080 + on_click: + - lvgl.widget.update: + id: user_flags_button + state: + user_3: true + user_4: !lambda return !lv_obj_has_state(id(user_flags_button), LV_STATE_USER_4); + - button: layout: 2x1 id: button_button From 7a784d11358cf82a2566ff5585504080cb8cc2f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:16:22 -0400 Subject: [PATCH 060/147] Bump zeroconf from 0.150.0 to 0.150.4 (#18924) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a065492dfa..3b3f3029ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ esptool==5.3.1 click==8.3.3 aioesphomeapi==46.3.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi -zeroconf==0.150.0 +zeroconf==0.150.4 puremagic==2.2.0 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import From fc9afcb201acba3bd8961b1e6011528bc5b8ea0c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:17:58 -0400 Subject: [PATCH 061/147] Bump ruff from 0.16.4 to 0.16.5 (#18920) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index cf4b028b0e..df37a10cb4 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.0.7 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating -ruff==0.16.4 # also change in .pre-commit-config.yaml when updating +ruff==0.16.5 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating prek==0.4.14 # also change in .github/workflows/ci.yml when updating From 78beb75b3bfbde8b6006301611bf6114b6fbbaf8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:20:00 -0400 Subject: [PATCH 062/147] Bump github/codeql-action/analyze from 4.37.8 to 4.37.9 (#18926) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b46f9adab6..12c6c6c60c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -84,6 +84,6 @@ jobs: exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: category: "/language:${{matrix.language}}" From ebe6c2d0495af2d72e2405326fad7efde20d41bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:20:12 -0400 Subject: [PATCH 063/147] Bump github/codeql-action/init from 4.37.8 to 4.37.9 (#18927) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 12c6c6c60c..aab3dea592 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,7 +56,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} From c20b619684a84e44800c56e160df8ecab0315250 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:20:43 -0400 Subject: [PATCH 064/147] Bump platformdirs from 4.11.4 to 4.11.5 (#18923) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3b3f3029ce..0d0fe9591b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ bleak==3.0.2 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 -platformdirs==4.11.4 # native esp-idf toolchain global cache dir +platformdirs==4.11.5 # native esp-idf toolchain global cache dir ninja==1.13.0 # native esp8266 arduino toolchain build driver filelock==3.32.4 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg From d43786937384df398eac11a084ef2f6e37f3d8e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:23:29 -0400 Subject: [PATCH 065/147] Bump cryptography from 48.0.1 to 50.0.1 (#18922) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0d0fe9591b..f19559dca8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # cryptography 49+ ships no Intel macOS wheels (arm64 only); esptool caps <49 there. # Keep 48.0.1, the last universal2 release, so esphome stays installable on Intel Macs. -cryptography==50.0.0; platform_system != "Darwin" or platform_machine != "x86_64" +cryptography==50.0.1; platform_system != "Darwin" or platform_machine != "x86_64" cryptography==48.0.1; platform_system == "Darwin" and platform_machine == "x86_64" voluptuous==0.16.0 PyYAML==6.0.3 From ca44db107cc63cb7104b554b6f56d94d27b2c83a Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:33:52 -0400 Subject: [PATCH 066/147] [docker] Fix generate_tags.py formatting (#18928) --- docker/generate_tags.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/generate_tags.py b/docker/generate_tags.py index a54205f1bf..b35aed91f0 100755 --- a/docker/generate_tags.py +++ b/docker/generate_tags.py @@ -65,7 +65,9 @@ def main(): suffix = f"-{args.suffix}" if args.suffix else "" - repository = (os.environ.get("GITHUB_REPOSITORY") or "esphome/esphome").strip().lower() + repository = ( + (os.environ.get("GITHUB_REPOSITORY") or "esphome/esphome").strip().lower() + ) image_name = f"{repository}{suffix}" print(f"channel={channel}") From 081ef3d30dc63811c547056c2d4d593c7d44b101 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:46:47 -0400 Subject: [PATCH 067/147] Bump prek from 0.4.14 to 0.5.0 (#18921) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index df37a10cb4..e837953878 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,7 @@ pylint==4.0.7 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating ruff==0.16.5 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating -prek==0.4.14 # also change in .github/workflows/ci.yml when updating +prek==0.5.0 # also change in .github/workflows/ci.yml when updating # Unit tests pytest==9.1.1 From afb0022dd0eb882a06e77190a4e4055c7dd05c16 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Mon, 31 Aug 2026 16:53:39 -0700 Subject: [PATCH 068/147] [core] Lint: require braces around single ESP_LOG control-statement bodies (#18727) --- esphome/components/alpha3/alpha3.cpp | 3 +- esphome/components/api/api_connection.cpp | 3 +- esphome/components/bk72xx_ble/bdk_scan.cpp | 3 +- esphome/components/bk72xx_ble/bk72xx_ble.cpp | 15 +- .../bk72xx_ble_tracker/bk72xx_ble_tracker.cpp | 3 +- .../ble_client/output/ble_binary_output.cpp | 3 +- esphome/components/bme680/bme680.cpp | 6 +- esphome/components/climate/climate.cpp | 18 ++- esphome/components/dht/dht.cpp | 3 +- .../camera_web_server.cpp | 3 +- esphome/components/fan/fan.cpp | 3 +- .../hbridge/switch/hbridge_switch.cpp | 3 +- esphome/components/he60r/he60r.cpp | 6 +- .../components/hoermann_hcp/hoermann_hcp.cpp | 3 +- .../key_collector/key_collector.cpp | 21 ++- esphome/components/ln882h_ble/ln882h_ble.cpp | 3 +- esphome/components/lvgl/lvgl_esphome.cpp | 3 +- esphome/components/mipi_dsi/mipi_dsi.cpp | 3 +- esphome/components/mipi_rgb/mipi_rgb.cpp | 3 +- esphome/components/mipi_spi/mipi_spi.cpp | 9 +- esphome/components/modbus/modbus.cpp | 6 +- esphome/components/mqtt/mqtt_component.cpp | 6 +- esphome/components/one_wire/one_wire_bus.cpp | 3 +- .../packet_transport/packet_transport.cpp | 12 +- esphome/components/qwiic_pir/qwiic_pir.cpp | 3 +- .../components/rpi_dpi_rgb/rpi_dpi_rgb.cpp | 3 +- .../components/shelly_dimmer/stm32flash.cpp | 3 +- esphome/components/spi/spi.h | 3 +- esphome/components/spi/spi_esp_idf.cpp | 9 +- esphome/components/st7701s/st7701s.cpp | 3 +- .../tuya/water_heater/tuya_water_heater.cpp | 12 +- esphome/components/udp/udp_component.cpp | 9 +- .../uponor_smatrix/uponor_smatrix.cpp | 3 +- esphome/components/usb_uart/pl2303.cpp | 3 +- .../components/wake_on_lan/wake_on_lan.cpp | 3 +- esphome/components/weikai/weikai.cpp | 15 +- script/ci-custom.py | 148 ++++++++++++++++++ tests/script/test_ci_custom.py | 147 +++++++++++++++++ 38 files changed, 437 insertions(+), 71 deletions(-) create mode 100644 tests/script/test_ci_custom.py diff --git a/esphome/components/alpha3/alpha3.cpp b/esphome/components/alpha3/alpha3.cpp index 048c365616..92b00d87cb 100644 --- a/esphome/components/alpha3/alpha3.cpp +++ b/esphome/components/alpha3/alpha3.cpp @@ -162,8 +162,9 @@ void Alpha3::send_request_(uint8_t *request, size_t len) { auto status = esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->geni_handle_, len, request, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) + if (status) { ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); + } } void Alpha3::update() { diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index bc088ca473..9c609aa047 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -2391,8 +2391,9 @@ void APIConnection::process_batch_() { } else if (payload_size == 0) { // payload_size == 0 with remove set means encoding hit OOM and the // connection is being dropped; warn only for a genuinely oversized message - if (!this->flags_.remove) + if (!this->flags_.remove) { ESP_LOGW(TAG, "Message too large to send: type=%u", item.message_type); + } this->clear_batch_(); } return; diff --git a/esphome/components/bk72xx_ble/bdk_scan.cpp b/esphome/components/bk72xx_ble/bdk_scan.cpp index f17f21c06b..3192fc79d7 100644 --- a/esphome/components/bk72xx_ble/bdk_scan.cpp +++ b/esphome/components/bk72xx_ble/bdk_scan.cpp @@ -62,8 +62,9 @@ BdkActivityState bdk_scan_state(uint8_t activity_idx) { uint8_t bdk_scan_acquire_activity() { uint8_t idx = app_ble_get_idle_actv_idx_handle(SCAN_ACTV); - if (idx == INVALID_ACTIVITY_IDX) + if (idx == INVALID_ACTIVITY_IDX) { ESP_LOGE(TAG, "Scan start failed: no idle activity handle"); + } return idx; } diff --git a/esphome/components/bk72xx_ble/bk72xx_ble.cpp b/esphome/components/bk72xx_ble/bk72xx_ble.cpp index 52401114e6..7a4efff455 100644 --- a/esphome/components/bk72xx_ble/bk72xx_ble.cpp +++ b/esphome/components/bk72xx_ble/bk72xx_ble.cpp @@ -181,8 +181,9 @@ void BK72xxBLE::enable() { break; } } - if (!bdaddr_live) + if (!bdaddr_live) { ESP_LOGW(TAG, "Controller address still unset after init; BLE stack may not have started"); + } #endif this->state_ = BLEComponentState::ACTIVE; @@ -210,8 +211,9 @@ void BK72xxBLE::loop() { // Re-check a settled scan; scan_start() refills the bring-up budget. // WARN: the only report of a drop that recovers inside its budget. if (this->scan_start(this->requested_.interval, this->requested_.window, this->requested_.active) != - ScanOpResult::SETTLED) + ScanOpResult::SETTLED) { ESP_LOGW(TAG, "Controller dropped the scan; restarting"); + } } // Drain the lock-free ring filled by the BLE task; all per-report work runs @@ -230,8 +232,9 @@ void BK72xxBLE::loop() { // Log dropped reports — only reachable when reports were processed; drops can // only occur while the queue is full, and only this loop drains it. uint16_t dropped = this->report_queue_.get_and_reset_dropped_count(); - if (dropped > 0) + if (dropped > 0) { ESP_LOGW(TAG, "Dropped %u scan reports due to queue overflow", dropped); + } } void BK72xxBLE::get_mac_lsb_first(uint8_t out[MAC_ADDRESS_SIZE]) const { @@ -449,8 +452,9 @@ ScanOpResult BK72xxBLE::advance_stop_(BdkActivityState state, bool ready) { if (!ready) { // Acting mid-operation could delete an activity whose start lands // afterwards, leaking the slot with the radio on; wait. - if (this->last_result_ == ScanOpResult::SETTLED) + if (this->last_result_ == ScanOpResult::SETTLED) { ESP_LOGD(TAG, "Scan stop deferred (controller busy)"); + } return ScanOpResult::PENDING; } // Settled, so CREATED unambiguously means "never started". @@ -474,8 +478,9 @@ ScanOpResult BK72xxBLE::advance_start_(BdkActivityState state, bool ready) { return ScanOpResult::PENDING; } if (!ready) { - if (this->last_result_ == ScanOpResult::SETTLED) + if (this->last_result_ == ScanOpResult::SETTLED) { ESP_LOGD(TAG, "Scan start deferred (controller busy)"); + } return ScanOpResult::PENDING; } if (state == BdkActivityState::CREATED) { diff --git a/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.cpp b/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.cpp index 1b4e6245ae..0939e1259f 100644 --- a/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.cpp +++ b/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.cpp @@ -69,8 +69,9 @@ void BK72xxBLETracker::on_ota_global_state(ota::OTAState state, float progress, this->stop_scan(); // The transfer starves the loop; a deferred stop would leave the radio // scanning for the whole update, so drain it here, bounded. - if (!this->parent_->flush_pending_stop(OTA_STOP_FLUSH_MS)) + if (!this->parent_->flush_pending_stop(OTA_STOP_FLUSH_MS)) { ESP_LOGE(TAG, "Scan still stopping at OTA start; the radio may contend with the update"); + } } else if (state == ota::OTA_ERROR || state == ota::OTA_ABORT) { // On success the device reboots, so restore only on a failed/aborted update; // loop() restarts the scan on its next iteration (continuous idle branch). diff --git a/esphome/components/ble_client/output/ble_binary_output.cpp b/esphome/components/ble_client/output/ble_binary_output.cpp index 1cb83b9d8b..5d53c59708 100644 --- a/esphome/components/ble_client/output/ble_binary_output.cpp +++ b/esphome/components/ble_client/output/ble_binary_output.cpp @@ -80,8 +80,9 @@ void BLEBinaryOutput::write_state(bool state) { esp_err_t err = esp_ble_gattc_write_char(this->parent()->get_gattc_if(), this->parent()->get_conn_id(), this->char_handle_, sizeof(state_as_uint), &state_as_uint, this->write_type_, ESP_GATT_AUTH_REQ_NONE); - if (err != ESP_GATT_OK) + if (err != ESP_GATT_OK) { ESP_LOGW(TAG, "[%s] Write error, err=%d", this->char_uuid_.to_str(char_buf), err); + } } } // namespace esphome::ble_client diff --git a/esphome/components/bme680/bme680.cpp b/esphome/components/bme680/bme680.cpp index ef98174e06..164424de09 100644 --- a/esphome/components/bme680/bme680.cpp +++ b/esphome/components/bme680/bme680.cpp @@ -327,10 +327,12 @@ void BME680Component::read_data_() { ESP_LOGD(TAG, "Got temperature=%.1f°C pressure=%.1fhPa humidity=%.1f%% gas_resistance=%.1fΩ", temperature, pressure, humidity, gas_resistance); - if (!gas_valid) + if (!gas_valid) { ESP_LOGW(TAG, "Gas measurement unsuccessful, reading invalid!"); - if (!heat_stable) + } + if (!heat_stable) { ESP_LOGW(TAG, "Heater unstable, reading invalid! (Normal for a few readings after a power cycle)"); + } if (this->temperature_sensor_ != nullptr) this->temperature_sensor_->publish_state(temperature); diff --git a/esphome/components/climate/climate.cpp b/esphome/components/climate/climate.cpp index 6ca9e394f7..34684a87e1 100644 --- a/esphome/components/climate/climate.cpp +++ b/esphome/components/climate/climate.cpp @@ -749,33 +749,39 @@ void Climate::dump_traits_(const char *tag) { } if (!traits.get_supported_modes().empty()) { ESP_LOGCONFIG(tag, " Supported modes:"); - for (ClimateMode m : traits.get_supported_modes()) + for (ClimateMode m : traits.get_supported_modes()) { ESP_LOGCONFIG(tag, " - %s", LOG_STR_ARG(climate_mode_to_string(m))); + } } if (!traits.get_supported_fan_modes().empty()) { ESP_LOGCONFIG(tag, " Supported fan modes:"); - for (ClimateFanMode m : traits.get_supported_fan_modes()) + for (ClimateFanMode m : traits.get_supported_fan_modes()) { ESP_LOGCONFIG(tag, " - %s", LOG_STR_ARG(climate_fan_mode_to_string(m))); + } } if (!traits.get_supported_custom_fan_modes().empty()) { ESP_LOGCONFIG(tag, " Supported custom fan modes:"); - for (const char *s : traits.get_supported_custom_fan_modes()) + for (const char *s : traits.get_supported_custom_fan_modes()) { ESP_LOGCONFIG(tag, " - %s", s); + } } if (!traits.get_supported_presets().empty()) { ESP_LOGCONFIG(tag, " Supported presets:"); - for (ClimatePreset p : traits.get_supported_presets()) + for (ClimatePreset p : traits.get_supported_presets()) { ESP_LOGCONFIG(tag, " - %s", LOG_STR_ARG(climate_preset_to_string(p))); + } } if (!traits.get_supported_custom_presets().empty()) { ESP_LOGCONFIG(tag, " Supported custom presets:"); - for (const char *s : traits.get_supported_custom_presets()) + for (const char *s : traits.get_supported_custom_presets()) { ESP_LOGCONFIG(tag, " - %s", s); + } } if (!traits.get_supported_swing_modes().empty()) { ESP_LOGCONFIG(tag, " Supported swing modes:"); - for (ClimateSwingMode m : traits.get_supported_swing_modes()) + for (ClimateSwingMode m : traits.get_supported_swing_modes()) { ESP_LOGCONFIG(tag, " - %s", LOG_STR_ARG(climate_swing_mode_to_string(m))); + } } } diff --git a/esphome/components/dht/dht.cpp b/esphome/components/dht/dht.cpp index a9117be4e1..2196f3a982 100644 --- a/esphome/components/dht/dht.cpp +++ b/esphome/components/dht/dht.cpp @@ -154,8 +154,9 @@ bool HOT IRAM_ATTR DHT::read_sensor_(float *temperature, float *humidity, bool r } } if (error_code != 0) { - if (report_errors) + if (report_errors) { ESP_LOGW(TAG, ESP_LOG_MSG_COMM_FAIL); + } return false; } diff --git a/esphome/components/esp32_camera_web_server/camera_web_server.cpp b/esphome/components/esp32_camera_web_server/camera_web_server.cpp index 88579e9632..bee231d132 100644 --- a/esphome/components/esp32_camera_web_server/camera_web_server.cpp +++ b/esphome/components/esp32_camera_web_server/camera_web_server.cpp @@ -210,8 +210,9 @@ esp_err_t CameraWebServer::streaming_handler_(struct httpd_req *req) { if (!image) { // A shutdown is not a lost frame: wait_for_image_() returns empty as soon // as running_ clears, and the loop condition below ends the stream anyway. - if (this->running_) + if (this->running_) { ESP_LOGW(TAG, "STREAM: failed to acquire frame"); + } res = ESP_FAIL; } if (res == ESP_OK) { diff --git a/esphome/components/fan/fan.cpp b/esphome/components/fan/fan.cpp index 7dc0b5c6fe..65521e63d5 100644 --- a/esphome/components/fan/fan.cpp +++ b/esphome/components/fan/fan.cpp @@ -334,8 +334,9 @@ void Fan::dump_traits_(const char *tag, const char *prefix) { } if (traits.supports_preset_modes()) { ESP_LOGCONFIG(tag, "%s Supported presets:", prefix); - for (const char *s : traits.supported_preset_modes()) + for (const char *s : traits.supported_preset_modes()) { ESP_LOGCONFIG(tag, "%s - %s", prefix, s); + } } } diff --git a/esphome/components/hbridge/switch/hbridge_switch.cpp b/esphome/components/hbridge/switch/hbridge_switch.cpp index 1012a264f2..c8e472d7aa 100644 --- a/esphome/components/hbridge/switch/hbridge_switch.cpp +++ b/esphome/components/hbridge/switch/hbridge_switch.cpp @@ -29,8 +29,9 @@ void HBridgeSwitch::dump_config() { LOG_PIN(" On Pin: ", this->on_pin_); LOG_PIN(" Off Pin: ", this->off_pin_); ESP_LOGCONFIG(TAG, " Pulse length: %" PRId32 " ms", this->pulse_length_); - if (this->wait_time_) + if (this->wait_time_) { ESP_LOGCONFIG(TAG, " Wait time %" PRId32 " ms", this->wait_time_); + } } void HBridgeSwitch::write_state(bool state) { diff --git a/esphome/components/he60r/he60r.cpp b/esphome/components/he60r/he60r.cpp index ea662e3ba9..f49224f17c 100644 --- a/esphome/components/he60r/he60r.cpp +++ b/esphome/components/he60r/he60r.cpp @@ -44,8 +44,9 @@ void HE60rCover::dump_config() { " Close Duration: %.1fs", this->open_duration_ / 1e3f, this->close_duration_ / 1e3f); auto restore = this->restore_state_(); - if (restore.has_value()) + if (restore.has_value()) { ESP_LOGCONFIG(TAG, " Saved position %d%%", (int) (restore->position * 100.f)); + } } void HE60rCover::endstop_reached_(CoverOperation operation) { @@ -77,8 +78,9 @@ void HE60rCover::process_rx_(uint8_t data) { ESP_LOGV(TAG, "Process RX data %X", data); if (!this->query_seen_) { this->query_seen_ = data == QUERY_BYTE; - if (!this->query_seen_) + if (!this->query_seen_) { ESP_LOGD(TAG, "RX Byte %02X", data); + } return; } switch (data) { diff --git a/esphome/components/hoermann_hcp/hoermann_hcp.cpp b/esphome/components/hoermann_hcp/hoermann_hcp.cpp index 17df927eb7..4aa2c79bb1 100644 --- a/esphome/components/hoermann_hcp/hoermann_hcp.cpp +++ b/esphome/components/hoermann_hcp/hoermann_hcp.cpp @@ -257,8 +257,9 @@ void HoermannHcp::on_state_reg_(uint16_t value) { } } // The low byte can change on its own, so only report a state we cannot decode once. - if (state != (previous >> 8)) + if (state != (previous >> 8)) { ESP_LOGW(TAG, "Unknown door state 0x%02X", state); + } } // Low byte of register 6: bit 0x10 is the lamp, bit 0x04 the relay. The reference implementation records diff --git a/esphome/components/key_collector/key_collector.cpp b/esphome/components/key_collector/key_collector.cpp index 69b7a6a7c6..42f02d39d4 100644 --- a/esphome/components/key_collector/key_collector.cpp +++ b/esphome/components/key_collector/key_collector.cpp @@ -16,26 +16,33 @@ void KeyCollector::loop() { void KeyCollector::dump_config() { #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_CONFIG ESP_LOGCONFIG(TAG, "Key Collector:"); - if (this->min_length_ > 0) + if (this->min_length_ > 0) { ESP_LOGCONFIG(TAG, " min length: %d", this->min_length_); - if (this->max_length_ > 0) + } + if (this->max_length_ > 0) { ESP_LOGCONFIG(TAG, " max length: %d", this->max_length_); - if (!this->back_keys_.empty()) + } + if (!this->back_keys_.empty()) { ESP_LOGCONFIG(TAG, " erase keys '%s'", this->back_keys_.c_str()); - if (!this->clear_keys_.empty()) + } + if (!this->clear_keys_.empty()) { ESP_LOGCONFIG(TAG, " clear keys '%s'", this->clear_keys_.c_str()); - if (!this->start_keys_.empty()) + } + if (!this->start_keys_.empty()) { ESP_LOGCONFIG(TAG, " start keys '%s'", this->start_keys_.c_str()); + } if (!this->end_keys_.empty()) { ESP_LOGCONFIG(TAG, " end keys '%s'\n" " end key is required: %s", this->end_keys_.c_str(), ONOFF(this->end_key_required_)); } - if (!this->allowed_keys_.empty()) + if (!this->allowed_keys_.empty()) { ESP_LOGCONFIG(TAG, " allowed keys '%s'", this->allowed_keys_.c_str()); - if (this->timeout_ > 0) + } + if (this->timeout_ > 0) { ESP_LOGCONFIG(TAG, " entry timeout: %0.1f", this->timeout_ / 1000.0); + } #endif } diff --git a/esphome/components/ln882h_ble/ln882h_ble.cpp b/esphome/components/ln882h_ble/ln882h_ble.cpp index 021e138f08..0b15bf434c 100644 --- a/esphome/components/ln882h_ble/ln882h_ble.cpp +++ b/esphome/components/ln882h_ble/ln882h_ble.cpp @@ -333,8 +333,9 @@ void LN882HBLE::loop() { // the queue empty — from the very first report on. Checking here keeps that // failure visible instead of producing a scanner that is silently dead. uint16_t dropped = this->report_queue_.get_and_reset_dropped_count(); - if (dropped > 0) + if (dropped > 0) { ESP_LOGW(TAG, "Dropped %u scan reports (queue full or out of memory for a report slot)", dropped); + } // Drain the lock-free ring filled by the rw task; all per-report work runs // here on the main task, then the report returns to the pool. BLEScanReport *report = this->report_queue_.pop(); diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index a10fdb0582..2c988473a9 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -1059,8 +1059,9 @@ static void *lv_alloc_draw_buf(size_t size, bool internal) { void *buffer; size = LV_ROUND_UP(size, LV_DRAW_BUF_ALIGN); buffer = heap_caps_aligned_alloc(LV_DRAW_BUF_ALIGN, size, internal ? MALLOC_CAP_8BIT : cap_bits); // NOLINT - if (buffer == nullptr) + if (buffer == nullptr) { ESP_LOGW(esphome::lvgl::TAG, "Failed to allocate %zu bytes for %sdraw buffer", size, internal ? "internal " : ""); + } return buffer; } diff --git a/esphome/components/mipi_dsi/mipi_dsi.cpp b/esphome/components/mipi_dsi/mipi_dsi.cpp index 0ff934ae94..0850b50c85 100644 --- a/esphome/components/mipi_dsi/mipi_dsi.cpp +++ b/esphome/components/mipi_dsi/mipi_dsi.cpp @@ -237,8 +237,9 @@ void MipiDsi::write_to_display_(int x_start, int y_start, int w, int h, const ui xSemaphoreTake(this->io_lock_, portMAX_DELAY); } } - if (err != ESP_OK) + if (err != ESP_OK) { ESP_LOGE(TAG, "lcd_lcd_panel_draw_bitmap failed: %s", esp_err_to_name(err)); + } } bool MipiDsi::check_buffer_() { diff --git a/esphome/components/mipi_rgb/mipi_rgb.cpp b/esphome/components/mipi_rgb/mipi_rgb.cpp index aeb04c155c..f43bbab21c 100644 --- a/esphome/components/mipi_rgb/mipi_rgb.cpp +++ b/esphome/components/mipi_rgb/mipi_rgb.cpp @@ -243,8 +243,9 @@ void MipiRgb::write_to_display_(int x_start, int y_start, int w, int h, const ui ptr += stride; // next line } } - if (err != ESP_OK) + if (err != ESP_OK) { ESP_LOGE(TAG, "lcd_lcd_panel_draw_bitmap failed: %s", esp_err_to_name(err)); + } } bool MipiRgb::check_buffer_() { diff --git a/esphome/components/mipi_spi/mipi_spi.cpp b/esphome/components/mipi_spi/mipi_spi.cpp index 80ae96720b..b2658de6e8 100644 --- a/esphome/components/mipi_spi/mipi_spi.cpp +++ b/esphome/components/mipi_spi/mipi_spi.cpp @@ -31,12 +31,15 @@ void internal_dump_config(const char *model, int width, int height, int offset_w LOG_PIN(" CS Pin: ", cs); LOG_PIN(" Reset Pin: ", reset); LOG_PIN(" DC Pin: ", dc); - if (offset_width != 0) + if (offset_width != 0) { ESP_LOGCONFIG(TAG, " Offset width: %d", offset_width); - if (offset_height != 0) + } + if (offset_height != 0) { ESP_LOGCONFIG(TAG, " Offset height: %d", offset_height); - if (brightness.has_value()) + } + if (brightness.has_value()) { ESP_LOGCONFIG(TAG, " Brightness: %u", brightness.value()); + } } } // namespace esphome::mipi_spi diff --git a/esphome/components/modbus/modbus.cpp b/esphome/components/modbus/modbus.cpp index 25687ba106..f428236a82 100644 --- a/esphome/components/modbus/modbus.cpp +++ b/esphome/components/modbus/modbus.cpp @@ -1199,15 +1199,17 @@ void ModbusServerHub::send_raw_(const uint8_t *payload, uint16_t len) { this->set_timeout("deferred_send", (this->tx_delay_remaining() + US_PER_MS - 1) / US_PER_MS, [this]() { ModbusFrame frame(this->deferred_payload_[0], this->deferred_payload_.data() + 1, this->deferred_payload_len_ - 1); - if (!this->send_frame_(frame)) + if (!this->send_frame_(frame)) { ESP_LOGE(TAG, "Deferred server reply dropped: transmission still blocked"); + } }); return; } ModbusFrame frame(payload[0], payload + 1, len - 1); - if (!this->send_frame_(frame)) + if (!this->send_frame_(frame)) { ESP_LOGE(TAG, "Server reply dropped: a frame arrived during the send delay"); + } } void Modbus::clear_rx_buffer_(const LogString *reason, bool warn, size_t bytes_to_clear) { diff --git a/esphome/components/mqtt/mqtt_component.cpp b/esphome/components/mqtt/mqtt_component.cpp index 18a759725f..a80cea6bd6 100644 --- a/esphome/components/mqtt/mqtt_component.cpp +++ b/esphome/components/mqtt/mqtt_component.cpp @@ -39,10 +39,12 @@ inline char *append_char(char *p, char c) { // Function implementation of LOG_MQTT_COMPONENT macro to reduce code size void log_mqtt_component(const char *tag, MQTTComponent *obj, bool state_topic, bool command_topic) { char buf[MQTT_DEFAULT_TOPIC_MAX_LEN]; - if (state_topic) + if (state_topic) { ESP_LOGCONFIG(tag, " State Topic: '%s'", obj->get_state_topic_to_(buf).c_str()); - if (command_topic) + } + if (command_topic) { ESP_LOGCONFIG(tag, " Command Topic: '%s'", obj->get_command_topic_to_(buf).c_str()); + } } void MQTTComponent::set_qos(uint8_t qos) { this->qos_ = qos; } diff --git a/esphome/components/one_wire/one_wire_bus.cpp b/esphome/components/one_wire/one_wire_bus.cpp index c7ea59050c..b62e4f47d4 100644 --- a/esphome/components/one_wire/one_wire_bus.cpp +++ b/esphome/components/one_wire/one_wire_bus.cpp @@ -18,8 +18,9 @@ const std::vector &OneWireBus::get_devices() { return this->devices_; bool OneWireBus::reset_() { int res = this->reset_int(); - if (res == -1) + if (res == -1) { ESP_LOGE(TAG, "1-wire bus is held low"); + } return res == 1; } diff --git a/esphome/components/packet_transport/packet_transport.cpp b/esphome/components/packet_transport/packet_transport.cpp index a21f0e2f63..998e1be5fc 100644 --- a/esphome/components/packet_transport/packet_transport.cpp +++ b/esphome/components/packet_transport/packet_transport.cpp @@ -551,12 +551,14 @@ void PacketTransport::dump_config() { " Ping-pong: %s", this->platform_name_, YESNO(this->is_encrypted_()), YESNO(this->ping_pong_enable_)); #ifdef USE_SENSOR - for (const auto &sensor : this->sensors_) + for (const auto &sensor : this->sensors_) { ESP_LOGCONFIG(TAG, " Sensor: %s", sensor.id); + } #endif #ifdef USE_BINARY_SENSOR - for (const auto &sensor : this->binary_sensors_) + for (const auto &sensor : this->binary_sensors_) { ESP_LOGCONFIG(TAG, " Binary Sensor: %s", sensor.id); + } #endif for (const auto &host : this->providers_) { ESP_LOGCONFIG(TAG, " Remote host: %s", host.first.c_str()); @@ -564,15 +566,17 @@ void PacketTransport::dump_config() { #ifdef USE_SENSOR auto rs = this->remote_sensors_.find(host.first.c_str()); if (rs != this->remote_sensors_.end()) { - for (const auto &key : rs->second | std::views::keys) + for (const auto &key : rs->second | std::views::keys) { ESP_LOGCONFIG(TAG, " Sensor: %s", key.c_str()); + } } #endif #ifdef USE_BINARY_SENSOR auto rbs = this->remote_binary_sensors_.find(host.first.c_str()); if (rbs != this->remote_binary_sensors_.end()) { - for (const auto &key : rbs->second | std::views::keys) + for (const auto &key : rbs->second | std::views::keys) { ESP_LOGCONFIG(TAG, " Binary Sensor: %s", key.c_str()); + } } #endif } diff --git a/esphome/components/qwiic_pir/qwiic_pir.cpp b/esphome/components/qwiic_pir/qwiic_pir.cpp index baf8dc122d..eb338db772 100644 --- a/esphome/components/qwiic_pir/qwiic_pir.cpp +++ b/esphome/components/qwiic_pir/qwiic_pir.cpp @@ -124,8 +124,9 @@ void QwiicPIRComponent::dump_config() { void QwiicPIRComponent::clear_events_() { // Clear event status register - if (!this->write_byte(QWIIC_PIR_EVENT_STATUS, 0x00)) + if (!this->write_byte(QWIIC_PIR_EVENT_STATUS, 0x00)) { ESP_LOGW(TAG, "Failed to clear events"); + } } } // namespace esphome::qwiic_pir diff --git a/esphome/components/rpi_dpi_rgb/rpi_dpi_rgb.cpp b/esphome/components/rpi_dpi_rgb/rpi_dpi_rgb.cpp index aacb217965..c0afc0607e 100644 --- a/esphome/components/rpi_dpi_rgb/rpi_dpi_rgb.cpp +++ b/esphome/components/rpi_dpi_rgb/rpi_dpi_rgb.cpp @@ -75,8 +75,9 @@ void RpiDpiRgb::draw_pixels_at(int x_start, int y_start, int w, int h, const uin break; } } - if (err != ESP_OK) + if (err != ESP_OK) { ESP_LOGE(TAG, "lcd_lcd_panel_draw_bitmap failed: %s", esp_err_to_name(err)); + } } int RpiDpiRgb::get_width() { diff --git a/esphome/components/shelly_dimmer/stm32flash.cpp b/esphome/components/shelly_dimmer/stm32flash.cpp index c758b0a312..beb015851e 100644 --- a/esphome/components/shelly_dimmer/stm32flash.cpp +++ b/esphome/components/shelly_dimmer/stm32flash.cpp @@ -629,8 +629,9 @@ stm32_unique_ptr stm32_init(uart::UARTDevice *stream, const uint8_t flags, const stm->pid = (buf[1] << 8) | buf[2]; if (returned > 2) { ESP_LOGD(TAG, "This bootloader returns %d extra bytes in PID:", returned); - for (auto i = 2; i <= returned; i++) + for (auto i = 2; i <= returned; i++) { ESP_LOGD(TAG, " %02x", buf[i]); + } } if (stm32_get_ack(stm) != STM32_ERR_OK) { return make_stm32_with_deletor(nullptr); diff --git a/esphome/components/spi/spi.h b/esphome/components/spi/spi.h index f8233c48d1..2dfb3c75a8 100644 --- a/esphome/components/spi/spi.h +++ b/esphome/components/spi/spi.h @@ -406,8 +406,9 @@ class SPIClient { this->release_device_, this->write_only_); #ifdef USE_SPI_PSRAM_DMA this->delegate_->set_psram_dma(this->psram_dma_); - if (this->psram_dma_) + if (this->psram_dma_) { esph_log_config("spi_device", "PSRAM DMA: enabled"); + } #endif } diff --git a/esphome/components/spi/spi_esp_idf.cpp b/esphome/components/spi/spi_esp_idf.cpp index 45d38c1719..95b5e4f14b 100644 --- a/esphome/components/spi/spi_esp_idf.cpp +++ b/esphome/components/spi/spi_esp_idf.cpp @@ -42,8 +42,9 @@ class SPIDelegateHw : public SPIDelegate { if (this->release_device_) this->add_device_(); if (this->is_ready()) { - if (spi_device_acquire_bus(this->handle_, portMAX_DELAY) != ESP_OK) + if (spi_device_acquire_bus(this->handle_, portMAX_DELAY) != ESP_OK) { ESP_LOGE(TAG, "Failed to acquire SPI bus"); + } SPIDelegate::begin_transaction(); } else { ESP_LOGW(TAG, "SPI device not ready, cannot begin transaction"); @@ -63,8 +64,9 @@ class SPIDelegateHw : public SPIDelegate { ~SPIDelegateHw() override { esp_err_t const err = spi_bus_remove_device(this->handle_); - if (err != ESP_OK) + if (err != ESP_OK) { ESP_LOGE(TAG, "Remove device failed - err %X", err); + } } // do a transfer. either txbuf or rxbuf (but not both) may be null. @@ -284,8 +286,9 @@ class SPIBusHw : public SPIBus { } buscfg.max_transfer_sz = MAX_TRANSFER_SIZE; auto err = spi_bus_initialize(channel, &buscfg, SPI_DMA_CH_AUTO); - if (err != ESP_OK) + if (err != ESP_OK) { ESP_LOGE(TAG, "Bus init failed - err %X", err); + } } SPIDelegate *get_delegate(uint32_t data_rate, SPIBitOrder bit_order, SPIMode mode, GPIOPin *cs_pin, diff --git a/esphome/components/st7701s/st7701s.cpp b/esphome/components/st7701s/st7701s.cpp index 3ffef86f3e..83f7bc9ce5 100644 --- a/esphome/components/st7701s/st7701s.cpp +++ b/esphome/components/st7701s/st7701s.cpp @@ -78,8 +78,9 @@ void ST7701S::draw_pixels_at(int x_start, int y_start, int w, int h, const uint8 break; } } - if (err != ESP_OK) + if (err != ESP_OK) { esph_log_e(TAG, "lcd_lcd_panel_draw_bitmap failed: %s", esp_err_to_name(err)); + } } void ST7701S::draw_pixel_at(int x, int y, Color color) { diff --git a/esphome/components/tuya/water_heater/tuya_water_heater.cpp b/esphome/components/tuya/water_heater/tuya_water_heater.cpp index 2fca3bf581..e1c78530e3 100644 --- a/esphome/components/tuya/water_heater/tuya_water_heater.cpp +++ b/esphome/components/tuya/water_heater/tuya_water_heater.cpp @@ -177,14 +177,18 @@ water_heater::WaterHeaterMode TuyaWaterHeater::default_on_mode_() const { void TuyaWaterHeater::dump_config() { LOG_WATER_HEATER("", "Tuya Water Heater", this); - if (this->switch_id_.has_value()) + if (this->switch_id_.has_value()) { ESP_LOGCONFIG(TAG, " Switch has datapoint ID %u", *this->switch_id_); - if (this->mode_id_.has_value()) + } + if (this->mode_id_.has_value()) { ESP_LOGCONFIG(TAG, " Mode has datapoint ID %u", *this->mode_id_); - if (this->target_temperature_id_.has_value()) + } + if (this->target_temperature_id_.has_value()) { ESP_LOGCONFIG(TAG, " Target Temperature has datapoint ID %u", *this->target_temperature_id_); - if (this->current_temperature_id_.has_value()) + } + if (this->current_temperature_id_.has_value()) { ESP_LOGCONFIG(TAG, " Current Temperature has datapoint ID %u", *this->current_temperature_id_); + } } } // namespace esphome::tuya diff --git a/esphome/components/udp/udp_component.cpp b/esphome/components/udp/udp_component.cpp index c144212ecf..858516c746 100644 --- a/esphome/components/udp/udp_component.cpp +++ b/esphome/components/udp/udp_component.cpp @@ -129,8 +129,9 @@ void UDPComponent::dump_config() { " Listen Port: %u\n" " Broadcast Port: %u", this->listen_port_, this->broadcast_port_); - for (const char *address : this->addresses_) + for (const char *address : this->addresses_) { ESP_LOGCONFIG(TAG, " Address: %s", address); + } if (this->listen_address_.has_value()) { char addr_buf[network::IP_ADDRESS_BUFFER_SIZE]; ESP_LOGCONFIG(TAG, " Listen address: %s", this->listen_address_.value().str_to(addr_buf)); @@ -145,8 +146,9 @@ void UDPComponent::send_packet(const uint8_t *data, size_t size) { #if defined(USE_SOCKET_IMPL_BSD_SOCKETS) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) for (const auto &saddr : this->sockaddrs_) { auto result = this->broadcast_socket_->sendto(data, size, 0, &saddr, sizeof(saddr)); - if (result < 0) + if (result < 0) { ESP_LOGW(TAG, "sendto() error %d", errno); + } } #endif #ifdef USE_SOCKET_IMPL_LWIP_TCP @@ -155,8 +157,9 @@ void UDPComponent::send_packet(const uint8_t *data, size_t size) { if (this->udp_client_.beginPacketMulticast(saddr, this->broadcast_port_, iface, 128) != 0) { this->udp_client_.write(data, size); auto result = this->udp_client_.endPacket(); - if (result == 0) + if (result == 0) { ESP_LOGW(TAG, "udp.write() error"); + } } } #endif diff --git a/esphome/components/uponor_smatrix/uponor_smatrix.cpp b/esphome/components/uponor_smatrix/uponor_smatrix.cpp index 0ba19f5cd7..c77f3468c7 100644 --- a/esphome/components/uponor_smatrix/uponor_smatrix.cpp +++ b/esphome/components/uponor_smatrix/uponor_smatrix.cpp @@ -110,8 +110,9 @@ bool UponorSmatrixComponent::parse_byte_(uint8_t byte) { // Handle packet size_t data_len = (packet_len - 6) / 3; if (data_len == 0) { - if (packet[4] == UPONOR_ID_REQUEST) + if (packet[4] == UPONOR_ID_REQUEST) { ESP_LOGVV(TAG, "Ignoring request packet for device 0x%08" PRIX32 "", device_address); + } return true; } diff --git a/esphome/components/usb_uart/pl2303.cpp b/esphome/components/usb_uart/pl2303.cpp index a9f7348331..db177fd308 100644 --- a/esphome/components/usb_uart/pl2303.cpp +++ b/esphome/components/usb_uart/pl2303.cpp @@ -194,8 +194,9 @@ std::vector USBUartTypePL2303::parse_descriptors(usb_device_handle_t dev } } - if (cdc_devs.empty()) + if (cdc_devs.empty()) { ESP_LOGE(TAG, "PL2303: failed to find bulk IN+OUT endpoints"); + } return cdc_devs; } diff --git a/esphome/components/wake_on_lan/wake_on_lan.cpp b/esphome/components/wake_on_lan/wake_on_lan.cpp index a514a55d80..e46b96c86a 100644 --- a/esphome/components/wake_on_lan/wake_on_lan.cpp +++ b/esphome/components/wake_on_lan/wake_on_lan.cpp @@ -40,8 +40,9 @@ void WakeOnLanButton::press_action() { memcpy(buffer + i * sizeof(this->macaddr_) + sizeof(PREFIX), this->macaddr_, sizeof(this->macaddr_)); } if (this->broadcast_socket_->sendto(buffer, sizeof(buffer), 0, reinterpret_cast(&saddr), - addr_len) <= 0) + addr_len) <= 0) { ESP_LOGW(TAG, "sendto() error %d", errno); + } #else IPAddress broadcast = IPAddress(255, 255, 255, 255); for (auto ip : esphome::network::get_ip_addresses()) { diff --git a/esphome/components/weikai/weikai.cpp b/esphome/components/weikai/weikai.cpp index 043df86be9..b95d474fd3 100644 --- a/esphome/components/weikai/weikai.cpp +++ b/esphome/components/weikai/weikai.cpp @@ -348,14 +348,18 @@ size_t WeikaiChannel::rx_in_fifo_() { uint8_t const fsr = this->reg(WKREG_FSR); if (fsr & (FSR_RFOE | FSR_RFLB | FSR_RFFE | FSR_RFPE)) { char bin_buf[9]; - if (fsr & FSR_RFOE) + if (fsr & FSR_RFOE) { ESP_LOGE(TAG, "Receive data overflow FSR=%s", format_bin_to(bin_buf, fsr)); - if (fsr & FSR_RFLB) + } + if (fsr & FSR_RFLB) { ESP_LOGE(TAG, "Receive line break FSR=%s", format_bin_to(bin_buf, fsr)); - if (fsr & FSR_RFFE) + } + if (fsr & FSR_RFFE) { ESP_LOGE(TAG, "Receive frame error FSR=%s", format_bin_to(bin_buf, fsr)); - if (fsr & FSR_RFPE) + } + if (fsr & FSR_RFPE) { ESP_LOGE(TAG, "Receive parity error FSR=%s", format_bin_to(bin_buf, fsr)); + } } if ((available == 0) && (fsr & FSR_RFDAT)) { // here we should be very careful because we can have something like this: @@ -495,8 +499,9 @@ void print_buffer(std::vector buffer) { hex_buffer[(3 * 32) + 1] = 0; for (size_t i = 0; i < buffer.size(); i++) { snprintf(&hex_buffer[3 * (i % 32)], sizeof(hex_buffer), "%02X ", buffer[i]); - if (i % 32 == 31) + if (i % 32 == 31) { ESP_LOGI(TAG, " %s", hex_buffer); + } } if (buffer.size() % 32) { // null terminate if incomplete line diff --git a/script/ci-custom.py b/script/ci-custom.py index 724a350884..f481fda860 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -319,6 +319,154 @@ def lint_no_long_delays(fname, match): ) +# An if/else/for/while whose only body is an unbraced ESP_LOG*() call. When the build's compile-time +# log level drops that macro, the body expands to nothing and the compiler warns (-Wempty-body). +# clang-tidy's brace check does not catch these (ShortStatementLines allows short unbraced bodies), so +# this fills that gap. Matched against comment/string-masked content, so commented-out or quoted code +# is ignored. Both spellings are covered: core/log.h defines the uppercase ESP_LOG*() macros and +# the lowercase esph_log_*() ones, and both expand to nothing below their log level. +# 'for' allows ';' inside its parentheses (the classic C-style header); 'if'/'while' do not, so their +# condition cannot run past the statement it guards. The 'for' header permits one level of nested +# parens so it stays bounded to its own statement: without that, it can run past the loop body and +# latch onto a later ')', mis-reporting the line and skipping the '#' preprocessor check below. +ESP_LOG_NEEDS_BRACES_RE = re.compile( + r"(?:\bif\s*\([^{};]*\)|\bwhile\s*\([^{};]*\)|\bfor\s*\((?:[^{}()]|\([^{}()]*\))*\)|\belse\b)" + r"[ \t]*\n?[ \t]*(?:ESP_LOG[A-Z]*|esph_log_[a-z]+)\s*\(", + re.MULTILINE, +) + + +def _mask_cpp_comments_strings(s): + """Return s with // and /* */ comments and string/char/raw-string literals blanked to spaces + (length and newlines preserved) so a regex only matches real code. Parentheses in real code are + kept, so callers can still balance them on the masked text.""" + out = list(s) + i = 0 + n = len(s) + while i < n: + c = s[i] + # Raw string literal: an optional encoding prefix, then R"delim( ... )delim". The body may + # contain quotes, //, /* and unbalanced parens, so it must be consumed as one unit. + if c == "R" and i + 1 < n and s[i + 1] == '"': + j = i + 2 + delim = "" + while j < n and s[j] not in "( \t\r\n\\" and len(delim) < 16: + delim += s[j] + j += 1 + if j < n and s[j] == "(": + closing = ")" + delim + '"' + end = s.find(closing, j + 1) + end = n if end == -1 else end + len(closing) + for k in range(i, end): + if s[k] != "\n": + out[k] = " " + i = end + continue + i += 1 + elif c == "/" and i + 1 < n and s[i + 1] == "/": + while i < n and s[i] != "\n": + out[i] = " " + i += 1 + elif c == "/" and i + 1 < n and s[i + 1] == "*": + out[i] = out[i + 1] = " " + i += 2 + while i < n and not (s[i] == "*" and i + 1 < n and s[i + 1] == "/"): + if s[i] != "\n": + out[i] = " " + i += 1 + if i < n: + out[i] = " " + if i + 1 < n: + out[i + 1] = " " + i += 2 + # A "'" after an alphanumeric or '_' is a C++ digit separator (1'000), not a literal opener. + elif c == '"' or ( + c == "'" and not (i and (s[i - 1].isalnum() or s[i - 1] == "_")) + ): + quote = c + out[i] = " " + i += 1 + while i < n: + if s[i] == "\\": + out[i] = " " + if i + 1 < n: + out[i + 1] = " " + i += 2 + continue + if s[i] == quote: + out[i] = " " + i += 1 + break + if s[i] != "\n": + out[i] = " " + i += 1 + else: + i += 1 + return "".join(out) + + +def _log_statement_end(masked, open_paren): + """Index of the ';' ending the ESP_LOG call whose '(' is at open_paren, or None. Balanced on the + masked text so quotes/comments inside the arguments do not confuse the paren count.""" + depth = 0 + i = open_paren + n = len(masked) + while i < n: + ch = masked[i] + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + j = i + 1 + while j < n and masked[j] != ";": + if not masked[j].isspace(): + return None + j += 1 + return j if j < n else None + i += 1 + return None + + +@lint_content_check(include=cpp_include) +def lint_esp_log_needs_braces(fname, content): + # Cheap bailout: no log call means nothing to flag, and skips masking the file entirely. + if "ESP_LOG" not in content and "esph_log_" not in content: + return [] + masked = _mask_cpp_comments_strings(content) + errors = [] + for match in ESP_LOG_NEEDS_BRACES_RE.finditer(masked): + pos = match.start() + line_start = content.rfind("\n", 0, pos) + 1 + # Skip preprocessor conditionals (#if/#else/#elif): not C++ control statements. + if content[line_start:pos].lstrip().startswith("#"): + continue + # A '// NOLINT' may sit at the end of the log line (where the message says to put it) or on the + # control-statement line, so scan the whole statement rather than only up to the ESP_LOG token. + stmt_end = _log_statement_end(masked, match.end() - 1) + nolint_end = ( + content.find("\n", stmt_end) if stmt_end is not None else match.end() + ) + if nolint_end == -1: + nolint_end = len(content) + if "NOLINT" in content[pos:nolint_end]: + continue + snippet = content[pos : match.end()].replace("\n", " ").strip() + errors.append( + ( + content.count("\n", 0, pos) + 1, + pos - line_start + 1, + ( + f"{highlight(snippet)} - an if/else/for/while body that is a single log " + "call must be wrapped in braces. When the log level compiles the macro out, the " + "body becomes empty and the compiler warns (-Wempty-body). Add { } around the " + "log call (or a '// NOLINT' comment if this is genuinely intended)." + ), + ) + ) + return errors + + @lint_content_check( include=[ "esphome/const.py", diff --git a/tests/script/test_ci_custom.py b/tests/script/test_ci_custom.py new file mode 100644 index 0000000000..d340a816c6 --- /dev/null +++ b/tests/script/test_ci_custom.py @@ -0,0 +1,147 @@ +"""Unit tests for the ESP_LOG-needs-braces lint rule in script/ci-custom.py. + +The rule flags an if/else/for/while whose only body is an unbraced ESP_LOG*() call (which becomes an +empty statement -- and a -Wempty-body warning -- once the log level compiles the macro out). These +tests pin the comment/string/raw-string masker, the accepted control-statement shapes, and the +NOLINT escape hatch at both placements a contributor would try. +""" + +import importlib.util +from pathlib import Path +import sys + +SCRIPT_DIR = (Path(__file__).parent / ".." / ".." / "script").resolve() +sys.path.insert(0, str(SCRIPT_DIR)) +_spec = importlib.util.spec_from_file_location("ci_custom", SCRIPT_DIR / "ci-custom.py") +ci_custom = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(ci_custom) + +mask = ci_custom._mask_cpp_comments_strings + + +def _lint(content: str) -> list: + return ci_custom.lint_esp_log_needs_braces("test.cpp", content) + + +# --- masker --- + + +def test_mask_preserves_length_newlines_and_real_parens() -> None: + src = 'foo("bar") + baz();\nqux();\n' + masked = mask(src) + assert len(masked) == len(src) + assert masked.count("\n") == src.count("\n") + assert masked.count("(") == src.count("(") # real parens survive for balancing + + +def test_mask_blanks_line_and_block_comments() -> None: + assert "ESP_LOGD" not in mask("a; // if (x) ESP_LOGD(t);\n") + assert "ESP_LOGD" not in mask("a; /* if (x) ESP_LOGD(t); */ b;\n") + + +def test_mask_blanks_string_literals() -> None: + assert "if" not in mask('x = "if (y) ESP_LOGD";\n') + + +def test_mask_handles_raw_string_without_desync() -> None: + # A raw string full of quotes/parens must be consumed as one unit; code after it stays intact. + src = 's.print(R"()");\nreturn;\n' + masked = mask(src) + assert "href" not in masked + assert "return;" in masked # not swallowed by a desynced string scan + + +# --- rule: flags real violations --- + + +def test_flags_unbraced_if_next_line() -> None: + assert _lint("if (x)\n ESP_LOGD(t);\n") + + +def test_flags_unbraced_same_line() -> None: + assert _lint("if (x) ESP_LOGW(t);\n") + + +def test_flags_c_style_for() -> None: + assert _lint("for (int i = 0; i < n; i++)\n ESP_LOGD(t, i);\n") + + +def test_flags_range_for_and_else() -> None: + assert _lint("for (auto &x : v)\n ESP_LOGCONFIG(t);\n") + assert _lint("else\n ESP_LOGE(t);\n") + + +def test_flags_for_header_with_nested_call() -> None: + assert _lint("for (auto it = v.begin(); it != v.end(); ++it)\n ESP_LOGD(t);\n") + + +def test_for_header_does_not_reach_into_a_later_statement() -> None: + # The 'for' header is bounded to its own statement, so it cannot swallow the loop body and latch + # onto a later ')'. Without that, the '#if' line below is reported as an unbraced body even though + # the '#' preprocessor check should skip it. + assert not _lint( + "for (int i = 0; i < n; i++)\n arr[i] = 0;\n#if defined(USE_X)\n ESP_LOGD(t);\n#endif\n" + ) + + +def test_violation_after_a_for_loop_is_reported_at_its_own_line() -> None: + errors = _lint( + "for (int i = 0; i < n; i++)\n sum += a[i];\nif (verbose)\n ESP_LOGD(t, sum);\n" + ) + lines = [line for line, _col, _msg in errors] + assert lines == [3] # the 'if', not the 'for' on line 1 + + +def test_flags_lowercase_esph_log_family() -> None: + # core/log.h defines esph_log_*() alongside ESP_LOG*(); both expand to nothing below their level. + assert _lint('if (x)\n esph_log_config(t, "m");\n') + assert _lint('if (err != ESP_OK)\n esph_log_e(t, "m");\n') + + +def test_digit_separator_does_not_disable_the_rest_of_the_file() -> None: + # A "'" digit separator must not be read as a char-literal opener, which blanked everything after. + assert _lint("uint32_t x = 1'000;\nif (y)\n ESP_LOGD(t);\n") + + +def test_mask_still_blanks_real_char_literals() -> None: + assert "ESP_LOGD" not in mask("char c = '\"'; // if (x) ESP_LOGD(t);\n") + assert not _lint("char sep = ';';\nif (x) {\n ESP_LOGD(t);\n}\n") + + +def test_flags_multiline_log_body() -> None: + assert _lint('if (x)\n ESP_LOGD(t, "%d %d",\n a, b);\n') + + +def test_raw_string_before_violation_still_caught() -> None: + # Regression for the masker desyncing on a raw string and disabling the check for the rest. + assert _lint('s.print(R"()");\nif (y)\n ESP_LOGD(t);\n') + + +# --- rule: ignores non-violations --- + + +def test_ignores_braced_body() -> None: + assert not _lint("if (x) {\n ESP_LOGD(t);\n}\n") + + +def test_ignores_commented_out_code() -> None: + assert not _lint("// if (x) ESP_LOGD(t);\n") + + +def test_ignores_preprocessor_else() -> None: + assert not _lint("#else\n ESP_LOGCONFIG(t);\n#endif\n") + + +def test_ignores_non_log_body() -> None: + assert not _lint("if (x)\n return false;\n") + + +# --- NOLINT escape hatch, both placements --- + + +def test_nolint_at_end_of_log_line_suppresses() -> None: + assert not _lint("if (x)\n ESP_LOGD(t); // NOLINT\n") + + +def test_nolint_on_control_line_suppresses() -> None: + assert not _lint("if (x) // NOLINT\n ESP_LOGD(t);\n") From 0f982f03b2e2085fab26f27e7310b62a7c924578 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 31 Aug 2026 19:55:53 -0400 Subject: [PATCH 069/147] [core] Prefetch tool-scons by PlatformIO's core spec (#18831) --- esphome/platformio/prefetch.py | 18 +++++++++--------- tests/unit_tests/test_platformio_prefetch.py | 16 ++++++++++------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 1df0a4b328..5097239065 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -832,16 +832,16 @@ def _prefetch(build_dir: Path, env: str) -> None: for name, opts in p.packages.items() if not opts.get("optional") ] - # PIO's build engine installs outside the platform package list; - # skipped when the platform lists it itself - if not any(s.name == "tool-scons" for s in specs): - specs.append( - PackageSpec( - owner="platformio", - name="tool-scons", - requirements=get_core_dependencies()["tool-scons"], - ) + # PIO's build engine installs tool-scons by its own registry spec at build + # start; a platform URL copy has no owner to match it, so prefetch that spec + specs = [s for s in specs if s.name != "tool-scons"] + specs.append( + PackageSpec( + owner="platformio", + name="tool-scons", + requirements=get_core_dependencies()["tool-scons"], ) + ) lib_deps = config.get(f"env:{env}", "lib_deps", []) # pio run's storage dir for this env, with its compatibility # qualifiers: an unqualified library install could land a different diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index d0785d2724..379ef52ebd 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -13,6 +13,7 @@ from types import SimpleNamespace from unittest.mock import MagicMock, patch from filelock import Timeout +from platformio.dependencies import get_core_dependencies from platformio.package.manager._install import PackageManagerInstallMixin from platformio.package.manager.base import BasePackageManager from platformio.package.manager.library import LibraryPackageManager @@ -1728,30 +1729,31 @@ def test_preinstall_unlocks_even_when_pool_fails(tmp_path: Path) -> None: m.unlock.assert_called_once_with() -def test_prefetch_skips_duplicate_tool_scons(tmp_path: Path) -> None: - """A platform that lists tool-scons itself does not get it appended.""" +def test_prefetch_replaces_platform_tool_scons_with_core_spec(tmp_path: Path) -> None: + """A platform's own tool-scons spec gives way to the core's registry spec.""" _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") fake_platform = MagicMock() fake_platform.packages = {"tool-scons": {"optional": False}} fake_platform.get_package_spec.side_effect = lambda name: _FakeSpec( - uri=None, name=name + uri="https://x/scons.zip", name=name, owner=None ) config = _fake_config(tmp_path, {"platform": "fake/p@1"}) modules = _pio_modules(tmp_path, fake_platform, MagicMock(), config) - batches: list[list[str]] = [] + batches: list[list] = [] with ( patch.dict("sys.modules", modules), patch.object( pf, "_registry_jobs", side_effect=lambda mgr, specs, seen: ( - batches.append([s.name for s in specs]) or ([], 0, []) + batches.append(list(specs)) or ([], 0, []) ), ), patch.object(pf, "_uri_jobs", return_value=([], 0, [])), ): pf._prefetch(tmp_path, "testenv") - assert batches[0] == ["tool-scons"] + (spec,) = batches[0] + assert (spec.name, spec.owner, spec.uri) == ("tool-scons", "platformio", None) def test_platformio_private_api_contract() -> None: @@ -1784,6 +1786,8 @@ def test_platformio_private_api_contract() -> None: assert callable(getattr(BasePackageManager, name)) # The dependency wave mirrors install_dependency's builtin skip assert callable(LibraryPackageManager.is_builtin_lib) + # The prefetch keys tool-scons on this core dependency + assert "tool-scons" in get_core_dependencies() # The pre-install passes these positionally / by keyword assert "compatibility" in inspect.signature(BasePackageManager.__init__).parameters lib_params = inspect.signature(LibraryPackageManager.__init__).parameters From 5dbc8ffe4c249fa9353b4cbe70c1e1ab01ad9377 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:17:52 +1200 Subject: [PATCH 070/147] [epaper_spi] Add UC8179 mono driver and Seeed reTerminal E1001 model (#17568) --- .../epaper_spi/epaper_spi_uc8179.cpp | 139 ++++++++++++++++++ .../components/epaper_spi/epaper_spi_uc8179.h | 52 +++++++ .../components/epaper_spi/models/uc8179.py | 93 ++++++++++++ .../epaper_spi/config/uc8179_e1001_test.yaml | 15 ++ tests/component_tests/epaper_spi/test_init.py | 17 +++ .../epaper_spi/test.esp32-s3-idf.yaml | 42 ++++++ 6 files changed, 358 insertions(+) create mode 100644 esphome/components/epaper_spi/epaper_spi_uc8179.cpp create mode 100644 esphome/components/epaper_spi/epaper_spi_uc8179.h create mode 100644 esphome/components/epaper_spi/models/uc8179.py create mode 100644 tests/component_tests/epaper_spi/config/uc8179_e1001_test.yaml diff --git a/esphome/components/epaper_spi/epaper_spi_uc8179.cpp b/esphome/components/epaper_spi/epaper_spi_uc8179.cpp new file mode 100644 index 0000000000..2a4ff2969a --- /dev/null +++ b/esphome/components/epaper_spi/epaper_spi_uc8179.cpp @@ -0,0 +1,139 @@ +#include "epaper_spi_uc8179.h" + +#include + +#include "esphome/core/log.h" + +namespace esphome::epaper_spi { + +static constexpr const char *const TAG = "epaper_spi.uc8179"; + +bool EPaperUC8179::initialise(bool partial) { + EPaperBase::initialise(partial); // send the model init sequence + this->partial_ = partial; + ESP_LOGV(TAG, "Power on"); + // POWER ON must precede the waveform/mode registers and the data transfer + // (the original driver powers on and busy-waits before writing them). + // The state machine busy-waits before entering TRANSFER_DATA. + this->command(0x04); + // Give the busy line time to assert before the state machine polls it + this->next_delay_ = 100; + return true; +} + +// Set up the refresh mode. Must be called after power-on has completed. +void EPaperUC8179::set_refresh_mode_() { + if (!this->is_using_partial_update_()) { + return; // plain full refresh uses the mode set by the init sequence + } + // Fast and partial refresh use flipped data polarity and a floating border + this->cmd_data(0x50, {0xA9, 0x07}); + // Force the waveform via the temperature registers: 0x5A selects the fast + // full-refresh waveform, 0x6E the partial-refresh waveform + this->cmd_data(0xE0, {0x02}); + if (this->partial_) { + this->cmd_data(0xE5, {0x6E}); + this->command(0x91); // enter partial mode + // Set the partial window to the full screen + const uint16_t x_end = this->width_ - 1; + const uint16_t y_end = this->height_ - 1; + this->cmd_data(0x90, {0x00, 0x00, static_cast(x_end >> 8), static_cast(x_end & 0xFF), 0x00, 0x00, + static_cast(y_end >> 8), static_cast(y_end & 0xFF), 0x01}); + } else { + this->cmd_data(0xE5, {0x5A}); + this->command(0x92); // exit partial mode + } +} + +bool HOT EPaperUC8179::transfer_data() { + const uint32_t start_time = millis(); + const size_t buffer_length = this->buffer_length_; + if (this->current_data_index_ == 0) { + this->set_refresh_mode_(); + } + // Fast full refresh sends the previous-image plane as well, so that every pixel transitions + const bool two_pass = this->is_using_partial_update_() && !this->partial_; + // Plain full refresh sends inverted data (buffer is 1=white, the wire wants 0=white); + // in fast/partial mode the data polarity is flipped via the VCOM/data-interval + // register instead, so the new-image plane is sent unmodified + const bool invert_new_data = !this->is_using_partial_update_(); + + uint8_t bytes_to_send[MAX_TRANSFER_SIZE]; + + // Phase 1 (fast full refresh only): previous image via 0x10 (DTM1), inverse of the new image + if (two_pass && this->current_data_index_ < buffer_length) { + if (this->current_data_index_ == 0) { + this->command(0x10); // DATA START TRANSMISSION 1 (previous image) + } + this->start_data_(); + while (this->current_data_index_ < buffer_length) { + const size_t bytes_to_copy = std::min(MAX_TRANSFER_SIZE, buffer_length - this->current_data_index_); + for (size_t i = 0; i < bytes_to_copy; i++) { + bytes_to_send[i] = ~this->buffer_[this->current_data_index_ + i]; + } + this->write_array(bytes_to_send, bytes_to_copy); + this->current_data_index_ += bytes_to_copy; + if (millis() - start_time > MAX_TRANSFER_TIME) { + this->disable(); + return false; + } + } + this->disable(); + } + + // Phase 2: new image via 0x13 (DTM2) + const size_t offset = two_pass ? buffer_length : 0; + const size_t total = offset + buffer_length; + if (this->current_data_index_ < total) { + if (this->current_data_index_ == offset) { + this->command(0x13); // DATA START TRANSMISSION 2 (new image) + } + this->start_data_(); + while (this->current_data_index_ < total) { + const size_t bytes_to_copy = std::min(MAX_TRANSFER_SIZE, total - this->current_data_index_); + const size_t data_idx = this->current_data_index_ - offset; + for (size_t i = 0; i < bytes_to_copy; i++) { + const uint8_t byte = this->buffer_[data_idx + i]; + bytes_to_send[i] = invert_new_data ? ~byte : byte; + } + this->write_array(bytes_to_send, bytes_to_copy); + this->current_data_index_ += bytes_to_copy; + if (millis() - start_time > MAX_TRANSFER_TIME) { + this->disable(); + return false; + } + } + this->disable(); + } + + this->current_data_index_ = 0; + return true; +} + +void EPaperUC8179::power_on() { + // Power-on is sent at the end of initialise() instead, because the + // waveform/mode registers and the data transfer must follow it +} + +void EPaperUC8179::refresh_screen(bool /*partial*/) { + ESP_LOGV(TAG, "Refresh"); + this->command(0x12); // DISPLAY REFRESH + // Delay the next busy poll: the busy line takes a short time to assert after + // the refresh command, and polling too early would read it as already idle + this->next_delay_ = 100; +} + +void EPaperUC8179::power_off() { + ESP_LOGV(TAG, "Power off"); + this->command(0x02); // POWER OFF +} + +void EPaperUC8179::deep_sleep() { + // Deep sleep loses the previous-image RAM that partial refresh compares against + if (!this->is_using_partial_update_()) { + ESP_LOGV(TAG, "Deep sleep"); + this->cmd_data(0x07, {0xA5}); // DEEP SLEEP with check code + } +} + +} // namespace esphome::epaper_spi diff --git a/esphome/components/epaper_spi/epaper_spi_uc8179.h b/esphome/components/epaper_spi/epaper_spi_uc8179.h new file mode 100644 index 0000000000..85c0eb623e --- /dev/null +++ b/esphome/components/epaper_spi/epaper_spi_uc8179.h @@ -0,0 +1,52 @@ +#pragma once + +#include "epaper_spi.h" + +namespace esphome::epaper_spi { + +/** + * Monochrome e-paper displays using the UC8179 controller. + * Supports: 7.5" V2 (EPD_7in5_V2), 800x480 pixels, as used by the + * Waveshare 7.5" V2 HAT and the Seeed reTerminal E1001. + * + * Buffer layout: 1 bit per pixel, 1=white, 0=black (the base class default). + * + * The INITIALISE state sends the panel configuration followed by power-on + * (0x04); the state machine busy-waits for power-on to complete before + * TRANSFER_DATA, which first writes the waveform/mode registers (these are + * only accepted while powered) and then the image data. The state machine + * busy-waits again before triggering REFRESH_SCREEN (0x12). + * + * Three refresh modes are used, following the Waveshare EPD_7in5_V2 examples: + * - full_update_every == 1: plain full refresh. The new image is sent + * inverted to DTM2 (0x13) and the controller uses its normal waveform. + * - full_update_every > 1, full update: fast full refresh. The data polarity + * is flipped via the VCOM/data-interval register, a fast waveform is forced + * via the temperature registers, and the image is sent to both DTM1 (0x10, + * inverted) and DTM2 (0x13) so that every pixel transitions. + * - full_update_every > 1, partial update: partial refresh. A partial-update + * waveform is forced, partial mode is entered with a full-screen window and + * only DTM2 is sent; the controller compares against its previous-image RAM. + */ +class EPaperUC8179 final : public EPaperBase { + public: + EPaperUC8179(const char *name, uint16_t width, uint16_t height, const uint8_t *init_sequence, + size_t init_sequence_length) + : EPaperBase(name, width, height, init_sequence, init_sequence_length, DISPLAY_TYPE_BINARY) { + this->buffer_length_ = this->row_width_ * height; + } + + protected: + bool initialise(bool partial) override; + bool transfer_data() override; + void refresh_screen(bool partial) override; + void power_on() override; + void power_off() override; + void deep_sleep() override; + void set_refresh_mode_(); + + // Set by initialise() so transfer_data() knows which planes to send + bool partial_{}; +}; + +} // namespace esphome::epaper_spi diff --git a/esphome/components/epaper_spi/models/uc8179.py b/esphome/components/epaper_spi/models/uc8179.py new file mode 100644 index 0000000000..bea133c328 --- /dev/null +++ b/esphome/components/epaper_spi/models/uc8179.py @@ -0,0 +1,93 @@ +"""Monochrome e-paper displays using the UC8179 controller. + +Supported models: +- waveshare-7.5in-v2: 7.5" mono display, 800x480 pixels (EPD_7in5_V2) +- seeed-reterminal-e1001: Seeed reTerminal E1001, which uses the same + 7.5" 800x480 panel on an integrated ESP32-S3 board + +Panel configuration and power-on (0x04) are both sent during the INITIALISE +state; the state machine's built-in busy wait then covers the power-on delay +before the waveform/mode registers and image data are transferred. + +These displays support fast full and partial refresh: set ``full_update_every`` +greater than 1 to enable it. Every ``full_update_every``-th update is a fast +full refresh, with partial refreshes in between. +""" + +from typing import Any + +from esphome.const import CONF_DATA_RATE + +from . import EpaperModel + + +class UC8179(EpaperModel): + """EpaperModel class for monochrome displays using the UC8179 controller.""" + + def __init__( + self, + name: str, + class_name: str = "EPaperUC8179", + data_rate: str = "10MHz", + **defaults: Any, + ) -> None: + defaults.setdefault(CONF_DATA_RATE, data_rate) + super().__init__(name, class_name, **defaults) + + def get_init_sequence(self, config: dict) -> tuple: + """Generate the initialization sequence for UC8179 mono displays. + + Panel configuration only — the driver appends power-on (0x04) at the + end of the INITIALISE state, and the state machine busy-waits for it + to complete before the data transfer starts. + """ + width, height = self.get_dimensions(config) + return ( + # POWER SETTING + (0x01, 0x07, 0x07, 0x3F, 0x3F), + # BOOSTER SOFT START + (0x06, 0x17, 0x17, 0x28, 0x17), + # PANEL SETTING (black/white mode, LUT from OTP) + (0x00, 0x1F), + # RESOLUTION SETTING (width x height) + ( + 0x61, + (width >> 8) & 0xFF, + width & 0xFF, + (height >> 8) & 0xFF, + height & 0xFF, + ), + # DUAL SPI MODE (disabled) + (0x15, 0x00), + # VCOM AND DATA INTERVAL SETTING + (0x50, 0x10, 0x07), + # TCON SETTING + (0x60, 0x22), + ) + + +uc8179 = UC8179("uc8179") + +# Waveshare 7.5" V2 mono (EPD_7in5_V2) — 800x480, UC8179 controller +waveshare_7_5_v2 = uc8179.extend( + "waveshare-7.5in-v2", + width=800, + height=480, +) + +# Seeed reTerminal E1001 — 7.5" mono e-paper (800x480), same panel as the +# Waveshare 7.5" V2, driven by an integrated ESP32-S3 board +waveshare_7_5_v2.extend( + "seeed-reterminal-e1001", + cs_pin=10, + dc_pin=11, + reset_pin=12, + busy_pin={ + "number": 13, + "inverted": True, + "mode": { + "input": True, + "pullup": True, + }, + }, +) diff --git a/tests/component_tests/epaper_spi/config/uc8179_e1001_test.yaml b/tests/component_tests/epaper_spi/config/uc8179_e1001_test.yaml new file mode 100644 index 0000000000..73f956c8ee --- /dev/null +++ b/tests/component_tests/epaper_spi/config/uc8179_e1001_test.yaml @@ -0,0 +1,15 @@ +esphome: + name: test + +esp32: + board: esp32-s3-devkitc-1 + variant: esp32s3 + +spi: + clk_pin: GPIO7 + mosi_pin: GPIO9 + +display: + - platform: epaper_spi + id: epaper_display + model: seeed-reterminal-e1001 diff --git a/tests/component_tests/epaper_spi/test_init.py b/tests/component_tests/epaper_spi/test_init.py index 7a0507542e..5e2e7d6013 100644 --- a/tests/component_tests/epaper_spi/test_init.py +++ b/tests/component_tests/epaper_spi/test_init.py @@ -439,6 +439,23 @@ def test_enable_pin_multiple( assert all(pin["mode"]["output"] is True for pin in enable_pins) +def test_uc8179_e1001_code_generation( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Test that the reTerminal E1001 model generates the UC8179 driver and init sequence.""" + main_cpp = generate_main(component_config_path("uc8179_e1001_test.yaml")) + + # The model must instantiate the UC8179 driver class with the panel dimensions + assert "epaper_spi::EPaperUC8179" in main_cpp + assert re.search(r'"SEEED-RETERMINAL-E1001",\s*800,\s*480', main_cpp) + + # The generated init sequence must contain the UC8179 resolution setting + # for 800x480: command 0x61, 4 data bytes 0x03 0x20 0x01 0xE0 + # (rendered as decimal in the generated array) + assert "97, 4, 3, 32, 1, 224" in main_cpp + + def test_enable_pin_code_generation( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], diff --git a/tests/components/epaper_spi/test.esp32-s3-idf.yaml b/tests/components/epaper_spi/test.esp32-s3-idf.yaml index 9cca528744..602aeb8d0e 100644 --- a/tests/components/epaper_spi/test.esp32-s3-idf.yaml +++ b/tests/components/epaper_spi/test.esp32-s3-idf.yaml @@ -255,3 +255,45 @@ display: it.filled_rectangle(0, 0, it.get_width(), it.get_height(), Color::WHITE); it.circle(it.get_width() / 2, it.get_height() / 2, 100, Color::BLACK); it.circle(it.get_width() / 2, it.get_height() / 2, 60, Color(255, 0, 0)); + + # Waveshare 7.5" V2 mono (800x480, UC8179 controller, EPD_7in5_V2) + # full_update_every > 1 exercises the fast/partial refresh paths + - platform: epaper_spi + spi_id: spi_bus + model: waveshare-7.5in-v2 + full_update_every: 4 + cs_pin: + allow_other_uses: true + number: GPIO5 + dc_pin: + allow_other_uses: true + number: GPIO17 + reset_pin: + allow_other_uses: true + number: GPIO16 + busy_pin: + allow_other_uses: true + number: GPIO4 + inverted: true + lambda: |- + it.filled_rectangle(0, 0, it.get_width(), it.get_height(), Color::WHITE); + it.circle(it.get_width() / 2, it.get_height() / 2, 100, Color::BLACK); + + # Seeed reTerminal E1001 - 7.5" mono e-paper (800x480, UC8179) + # Pins overridden to avoid conflicts with the E1002 defaults above + - platform: epaper_spi + spi_id: spi_bus + model: seeed-reterminal-e1001 + cs_pin: + allow_other_uses: true + number: GPIO5 + dc_pin: + allow_other_uses: true + number: GPIO17 + reset_pin: + allow_other_uses: true + number: GPIO16 + busy_pin: + allow_other_uses: true + number: GPIO4 + inverted: true From d6758377d14a8ab63781a4c035162166d4889a12 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 31 Aug 2026 22:25:58 -0400 Subject: [PATCH 071/147] [core] Clone git libraries in parallel in the library prefetch (#18836) --- esphome/git.py | 9 ++ esphome/platformio/library.py | 103 +++++++++++++++----- tests/unit_tests/test_git.py | 19 ++++ tests/unit_tests/test_platformio_library.py | 81 ++++++++++++++- 4 files changed, 184 insertions(+), 28 deletions(-) diff --git a/esphome/git.py b/esphome/git.py index 9815377f51..14145a639b 100644 --- a/esphome/git.py +++ b/esphome/git.py @@ -457,6 +457,15 @@ def _clone_complete_marker_path(repo_dir: Path) -> Path: return repo_dir / ".git" / _CLONE_COMPLETE_MARKER +def has_complete_clone( + url: str, ref: str | None, domain: str, subpath: Path | None = None +) -> bool: + """Lock-free probe for a complete clone; can go stale immediately, so + best-effort decisions only, never a substitute for ``clone_or_update``.""" + repo_dir = _repo_entry_dir(_cache_key(url, ref), domain, subpath) + return _clone_complete_marker_path(repo_dir).is_file() + + def _clear_clone_complete_marker(repo_dir: Path) -> None: """Best-effort removal of the completion marker. diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 0402311a9a..3ff60f8aaa 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -13,7 +13,7 @@ regardless of which toolchain consumes the result. """ from collections import deque -from collections.abc import Callable, Iterable +from collections.abc import Callable, Hashable, Iterable from dataclasses import dataclass, field from functools import partial import glob @@ -99,6 +99,17 @@ class Source: ) -> Path: raise NotImplementedError + def prefetch_key(self, dir_suffix: str) -> Hashable | None: + """Prefetch dedup identity; None = not prefetchable. Sources that + could write one cache dir must return equal keys (workers must never + share a dir); a coarser key only skips a prefetch.""" + return None + + def is_cached(self, dir_suffix: str, salt: str = "", namespace: str = "") -> bool: + """Whether a completed fetch exists; only consulted when + ``prefetch_key()`` is not None, True is the safe default.""" + return True + def source_root(self, build_path: Path) -> Path: """Directory holding the library's own files (manifest + sources). @@ -127,6 +138,9 @@ class URLSource(Source): h.update(salt.encode()) return base_dir / h.hexdigest()[:8] / dir_suffix + def prefetch_key(self, dir_suffix: str) -> Hashable | None: + return self.url if self.size else None + def is_cached(self, dir_suffix: str, salt: str = "", namespace: str = "") -> bool: """Whether a completed extraction already exists for this source.""" return ( @@ -177,14 +191,29 @@ class GitSource(Source): self.url = url self.ref = ref - def download( - self, dir_suffix: str, force: bool = False, salt: str = "", namespace: str = "" - ) -> Path: + @staticmethod + def _domain(salt: str, namespace: str) -> str: domain = DOMAIN if namespace: domain = f"{domain}/{namespace}" if salt: domain = f"{domain}/{salt}" + return domain + + def prefetch_key(self, dir_suffix: str) -> Hashable | None: + # The clone target dir is hash(url@ref)/ + return (self.url, self.ref, dir_suffix) + + def is_cached(self, dir_suffix: str, salt: str = "", namespace: str = "") -> bool: + """Whether a completed clone already exists for this source.""" + return git.has_complete_clone( + self.url, self.ref, self._domain(salt, namespace), Path(dir_suffix) + ) + + def download( + self, dir_suffix: str, force: bool = False, salt: str = "", namespace: str = "" + ) -> Path: + domain = self._domain(salt, namespace) path, _ = git.clone_or_update( url=self.url, ref=self.ref, @@ -988,56 +1017,78 @@ def _fetch_source( ) +def _clone_source( + component: ConvertedLibrary, + salt: str, + namespace: str, + tracker: Callable[[int], None], +) -> None: + # No byte progress from git; one tick so a cancelled batch stops here + tracker(0) + component.source.download( + component.get_sanitized_name(), salt=salt, namespace=namespace + ) + + def _prefetch_wave( wave: list[tuple[str, ConvertedLibrary]], salt: str, namespace: str ) -> None: - """Best-effort parallel download of a wave's registry archives. + """Best-effort parallel fetch of a wave's registry archives and git clones. - The walk's own ``download()`` stays authoritative; duplicate URLs + The walk's own ``download()`` stays authoritative; duplicate sources prefetch once so two threads never share a cache directory. Archives whose size the registry did not report are left to the sequential loop, whose per-file bars don't interleave. A node a sibling in the - same wave supersedes has its archive fetched in vain (knowing better + same wave supersedes has its source fetched in vain (knowing better would need the manifests being downloaded). """ try: - components: list[ConvertedLibrary] = [] - seen: set[str] = set() + archives: list[ConvertedLibrary] = [] + clones: list[ConvertedLibrary] = [] + seen: set[Hashable] = set() for _key, component in wave: source = component.source - if not isinstance(source, URLSource) or not source.size: + name = component.get_sanitized_name() + dedup_key = source.prefetch_key(name) + if dedup_key is None or dedup_key in seen: continue - if source.url in seen: - continue - seen.add(source.url) + seen.add(dedup_key) try: - cached = source.is_cached( - component.get_sanitized_name(), salt=salt, namespace=namespace - ) + cached = source.is_cached(name, salt=salt, namespace=namespace) except OSError as err: # Best-effort, but visibly: a systematic probe failure makes - # every warm build re-download every archive + # every warm build re-fetch every source _LOGGER.warning("Cache probe for %s failed: %s", component.name, err) cached = False if cached: # A warm build must stay silent continue - components.append(component) - if not components: + (archives if isinstance(source, URLSource) else clones).append(component) + if not archives and not clones: return # Single-item waves (a dependency chain discovers one archive per # wave) go through the same runner: one download method, one bar - _LOGGER.info( - "Downloading %d library archive(s): %s", - len(components), - ", ".join(c.name for c in components), - ) + if archives: + _LOGGER.info( + "Downloading %d library archive(s): %s", + len(archives), + ", ".join(c.name for c in archives), + ) + if clones: + _LOGGER.info( + "Cloning %d library repo(s): %s", + len(clones), + ", ".join(c.name for c in clones), + ) failures = run_batch_downloads( "Downloading libraries", [ (c.name, c.source.size, partial(_fetch_source, c, salt, namespace)) - for c in components - ], + for c in archives + ] + # Size 0: clones share the worker pool without skewing the + # byte bar, whose total stays the archive sum + + [(c.name, 0, partial(_clone_source, c, salt, namespace)) for c in clones], ) # The sequential call below retries and raises the real error warn_prefetch_failures( diff --git a/tests/unit_tests/test_git.py b/tests/unit_tests/test_git.py index e296d48a46..0f7e0339c9 100644 --- a/tests/unit_tests/test_git.py +++ b/tests/unit_tests/test_git.py @@ -714,6 +714,25 @@ def test_run_git_command_without_git_dir_raises_error( git.run_git_command(["git", "clone", "https://invalid.url/repo.git"]) +def test_has_complete_clone(tmp_path: Path) -> None: + """The lock-free probe tracks the completion marker, subpath included.""" + CORE.config_path = tmp_path / "test.yaml" + + url = "https://github.com/test/repo" + subpath = Path("lib") + assert not git.has_complete_clone(url, "v1", "test_domain", subpath) + + repo_dir = _compute_repo_dir(url, "v1", "test_domain") / subpath + (repo_dir / ".git").mkdir(parents=True) + # A directory without the marker is an incomplete clone + assert not git.has_complete_clone(url, "v1", "test_domain", subpath) + + _mark_clone_complete(repo_dir) + assert git.has_complete_clone(url, "v1", "test_domain", subpath) + # The ref is part of the cache key + assert not git.has_complete_clone(url, "v2", "test_domain", subpath) + + def test_clone_or_update_with_never_refresh( tmp_path: Path, mock_run_git_command: Mock ) -> None: diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 0a16b118fc..3bae39b3c1 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -638,7 +638,7 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel( setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: """Registry archives in one wave download concurrently, deduped by URL; - git/local sources and failures are left to the sequential call.""" + local sources and failures are left to the sequential call.""" calls: list[str] = [] def fake_download( @@ -658,7 +658,7 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel( # into the same cache directory) ("b2", ConvertedLibrary("b2", "1.0", URLSource("https://x/b.tar.gz", 1))), ("c", ConvertedLibrary("c", "1.0", URLSource("https://x/boom.tar.gz", 1))), - ("g", ConvertedLibrary("g", "*", lib.GitSource("https://x/g.git", None))), + ("l", ConvertedLibrary("l", "*", LocalSource("/some/lib"))), ] lib._prefetch_wave(wave, "", "idf") assert sorted(calls) == [ @@ -670,6 +670,83 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel( assert "Prefetch of c failed (retrying sequentially)" in caplog.text +def test_prefetch_wave_clones_git_sources_in_parallel( + setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Git sources join the same prefetch batch as the archives, deduped by + clone target; a clone failure warns and is left to the sequential call.""" + caplog.set_level("INFO") + calls: list[str] = [] + + def fake_clone(self, dir_suffix, force=False, salt="", namespace=""): + calls.append(f"{self}/{dir_suffix}") + if "boom" in self.url: + raise RuntimeError("boom") + + monkeypatch.setattr(GitSource, "download", fake_clone) + wave = [ + ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))), + ("g", ConvertedLibrary("g", "*", GitSource("https://x/g.git", "v1"))), + # Same url@ref and target dir must clone once + ("g2", ConvertedLibrary("g", "*", GitSource("https://x/g.git", "v1"))), + ("h", ConvertedLibrary("h", "*", GitSource("https://x/boom.git", None))), + ] + monkeypatch.setattr( + URLSource, "download", lambda self, dir_suffix, progress=None, **kw: None + ) + lib._prefetch_wave(wave, "", "idf") + assert sorted(calls) == ["https://x/boom.git/h", "https://x/g.git#v1/g"] + assert "Cloning 2 library repo(s): g, h" in caplog.text + assert "Prefetch of h failed (retrying sequentially)" in caplog.text + + +def test_source_base_prefetch_defaults() -> None: + """The base Source is not prefetchable and reports cached (nothing to do).""" + source = Source() + assert source.prefetch_key("x") is None + assert source.is_cached("x") is True + + +def test_prefetch_wave_single_clone_uses_the_batch( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A wave with only git sources still clones through the batch runner.""" + caplog.set_level("INFO") + calls: list[str] = [] + monkeypatch.setattr(GitSource, "is_cached", lambda self, *a, **kw: False) + monkeypatch.setattr( + GitSource, + "download", + lambda self, dir_suffix, force=False, salt="", namespace="": calls.append( + self.url + ), + ) + lib._prefetch_wave( + [("g", ConvertedLibrary("g", "*", GitSource("https://x/g.git", None)))], + "", + "idf", + ) + assert calls == ["https://x/g.git"] + assert "Cloning 1 library repo(s): g" in caplog.text + assert "Downloading" not in caplog.text + + +def test_prefetch_wave_warm_git_cache_is_silent( + setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """An already-complete clone is neither re-fetched nor announced.""" + caplog.set_level("INFO") + monkeypatch.setattr( + GitSource, + "download", + lambda self, dir_suffix, **kw: (_ for _ in ()).throw(AssertionError("cloned")), + ) + monkeypatch.setattr(GitSource, "is_cached", lambda self, *a, **kw: True) + wave = [("g", ConvertedLibrary("g", "*", GitSource("https://x/g.git", None)))] + lib._prefetch_wave(wave, "", "idf") + assert "Cloning" not in caplog.text + + def test_prefetch_wave_unknown_size_left_to_sequential( setup_core, monkeypatch: pytest.MonkeyPatch ) -> None: From fe3788ff4783ab93192a3c979585bc2a4eac660f Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:56:03 +1000 Subject: [PATCH 072/147] [esp32][mipi_rgb] Add ESP32-S31 support for execute_from_psram (#18929) Co-authored-by: Claude Sonnet 5 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- esphome/components/esp32/__init__.py | 17 +++++----- esphome/components/mipi_rgb/mipi_rgb.cpp | 7 +--- esphome/components/mipi_rgb/mipi_rgb.h | 9 ++++-- .../esp32/config/execute_from_psram_s31.yaml | 13 ++++++++ tests/component_tests/esp32/test_esp32.py | 32 ++++++++++++++++--- 5 files changed, 58 insertions(+), 20 deletions(-) create mode 100644 tests/component_tests/esp32/config/execute_from_psram_s31.yaml diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index b0290d7a84..3f5a34bc73 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -182,6 +182,13 @@ SIGNED_OTA_V1_ECDSA_VARIANTS = { VARIANT_ESP32, } +# Variants that support execution from PSRAM +PSRAM_XIP_VARIANTS = { + VARIANT_ESP32S3, + VARIANT_ESP32P4, + VARIANT_ESP32S31, +} + # NVS encryption (HMAC peripheral scheme) is only available on variants that # expose the HMAC peripheral (SOC_HMAC_SUPPORTED in soc_caps.h). The original # ESP32 and ESP32-C2 do not have it. New variants with an HMAC peripheral @@ -1523,7 +1530,7 @@ def final_validate(config) -> None: ) ) if advanced[CONF_EXECUTE_FROM_PSRAM]: - if config[CONF_VARIANT] not in {VARIANT_ESP32S3, VARIANT_ESP32P4}: + if config[CONF_VARIANT] not in PSRAM_XIP_VARIANTS: errs.append( cv.Invalid( f"'{CONF_EXECUTE_FROM_PSRAM}' is not available on this esp32 variant", @@ -2727,13 +2734,7 @@ async def to_code(config): _configure_lwip_max_sockets(conf) if advanced[CONF_EXECUTE_FROM_PSRAM]: - if variant == VARIANT_ESP32S3: - add_idf_sdkconfig_option("CONFIG_SPIRAM_FETCH_INSTRUCTIONS", True) - add_idf_sdkconfig_option("CONFIG_SPIRAM_RODATA", True) - elif variant == VARIANT_ESP32P4: - add_idf_sdkconfig_option("CONFIG_SPIRAM_XIP_FROM_PSRAM", True) - else: - raise ValueError("Unhandled ESP32 variant") + add_idf_sdkconfig_option("CONFIG_SPIRAM_XIP_FROM_PSRAM", True) # Apply LWIP core locking for better socket performance # This is already enabled by default in Arduino framework, where it provides diff --git a/esphome/components/mipi_rgb/mipi_rgb.cpp b/esphome/components/mipi_rgb/mipi_rgb.cpp index f43bbab21c..c11044c288 100644 --- a/esphome/components/mipi_rgb/mipi_rgb.cpp +++ b/esphome/components/mipi_rgb/mipi_rgb.cpp @@ -5,7 +5,7 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include -#include +#include #include namespace esphome::mipi_rgb { @@ -177,11 +177,6 @@ void MipiRgb::common_setup_() { ESP_LOGCONFIG(TAG, "MipiRgb setup complete"); } -void MipiRgb::loop() { - if (this->handle_ != nullptr) - esp_lcd_rgb_panel_restart(this->handle_); -} - void MipiRgb::update() { if (this->is_failed()) return; diff --git a/esphome/components/mipi_rgb/mipi_rgb.h b/esphome/components/mipi_rgb/mipi_rgb.h index 87b35781e2..f528943c1b 100644 --- a/esphome/components/mipi_rgb/mipi_rgb.h +++ b/esphome/components/mipi_rgb/mipi_rgb.h @@ -3,7 +3,7 @@ #if defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32P4) || defined(USE_ESP32_VARIANT_ESP32S31) #include "esphome/core/gpio.h" #include "esphome/components/display/display.h" -#include "esp_lcd_panel_ops.h" +#include #ifdef USE_SPI #include "esphome/components/spi/spi.h" #endif @@ -25,7 +25,12 @@ class MipiRgb : public display::Display { public: MipiRgb(int width, int height) : width_(width), height_(height) {} void setup() override; - void loop() override; +#ifdef USE_ESP32_VARIANT_ESP32S3 + void loop() override { + if (this->handle_ != nullptr) + esp_lcd_rgb_panel_restart(this->handle_); + } +#endif void update() override; void fill(Color color) override; void draw_pixels_at(int x_start, int y_start, int w, int h, const uint8_t *ptr, display::ColorOrder order, diff --git a/tests/component_tests/esp32/config/execute_from_psram_s31.yaml b/tests/component_tests/esp32/config/execute_from_psram_s31.yaml new file mode 100644 index 0000000000..493c9f989e --- /dev/null +++ b/tests/component_tests/esp32/config/execute_from_psram_s31.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + variant: esp32s31 + board: esp32-s31-devkitc + framework: + type: esp-idf + advanced: + execute_from_psram: true + +psram: + mode: octal diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index bef273badd..759020c732 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -203,6 +203,18 @@ def test_esp32_rejects_unsupported_cli_toolchain( r"'execute_from_psram' requires PSRAM to be configured @ data\['framework'\]\['advanced'\]\['execute_from_psram'\]", id="execute_from_psram_requires_psram_p4_config", ), + pytest.param( + { + "variant": "esp32s31", + "board": "esp32-s31-devkitc", + "framework": { + "type": "esp-idf", + "advanced": {"execute_from_psram": True}, + }, + }, + r"'execute_from_psram' requires PSRAM to be configured @ data\['framework'\]\['advanced'\]\['execute_from_psram'\]", + id="execute_from_psram_requires_psram_s31_config", + ), pytest.param( { "variant": "esp32s3", @@ -422,12 +434,12 @@ def test_execute_from_psram_s3_sdkconfig( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], ) -> None: - """Test that execute_from_psram on ESP32-S3 sets the correct sdkconfig options.""" + """Test that execute_from_psram on ESP32-S3 sets the correct sdkconfig option.""" generate_main(component_config_path("execute_from_psram_s3.yaml")) sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] - assert sdkconfig.get("CONFIG_SPIRAM_FETCH_INSTRUCTIONS") is True - assert sdkconfig.get("CONFIG_SPIRAM_RODATA") is True - assert "CONFIG_SPIRAM_XIP_FROM_PSRAM" not in sdkconfig + assert sdkconfig.get("CONFIG_SPIRAM_XIP_FROM_PSRAM") is True + assert "CONFIG_SPIRAM_FETCH_INSTRUCTIONS" not in sdkconfig + assert "CONFIG_SPIRAM_RODATA" not in sdkconfig def test_execute_from_psram_p4_sdkconfig( @@ -442,6 +454,18 @@ def test_execute_from_psram_p4_sdkconfig( assert "CONFIG_SPIRAM_RODATA" not in sdkconfig +def test_execute_from_psram_s31_sdkconfig( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Test that execute_from_psram on ESP32-S31 sets the correct sdkconfig option.""" + generate_main(component_config_path("execute_from_psram_s31.yaml")) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert sdkconfig.get("CONFIG_SPIRAM_XIP_FROM_PSRAM") is True + assert "CONFIG_SPIRAM_FETCH_INSTRUCTIONS" not in sdkconfig + assert "CONFIG_SPIRAM_RODATA" not in sdkconfig + + def test_nvs_encryption_sdkconfig( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], From 68ffd5a77324f43345d5bbcab6db1edc9c42e390 Mon Sep 17 00:00:00 2001 From: Oliver Kleinecke Date: Tue, 1 Sep 2026 15:12:49 +0200 Subject: [PATCH 073/147] [safe_mode] Uncover silent error in safe-mode (#18749) Co-authored-by: Oliver Kleinecke Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- esphome/components/safe_mode/safe_mode.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/esphome/components/safe_mode/safe_mode.cpp b/esphome/components/safe_mode/safe_mode.cpp index ce029b4f55..8fd5911ab5 100644 --- a/esphome/components/safe_mode/safe_mode.cpp +++ b/esphome/components/safe_mode/safe_mode.cpp @@ -255,12 +255,17 @@ bool SafeModeComponent::should_enter_safe_mode(uint8_t num_attempts, uint32_t en } void SafeModeComponent::write_rtc_(uint32_t val) { - this->rtc_.save(&val); - global_preferences->sync(); + if (!this->rtc_.save(&val)) { + ESP_LOGE(TAG, "Failed to set rtc value (%" PRIu32 ")", val); + return; + } + if (!global_preferences->sync()) { + ESP_LOGE(TAG, "Failed to persist rtc value (%" PRIu32 ")", val); + } } uint32_t SafeModeComponent::read_rtc_() { - uint32_t val; + uint32_t val = 0; if (!this->rtc_.load(&val)) return 0; return val; @@ -272,7 +277,9 @@ void SafeModeComponent::clean_rtc() { // before sync, the boot wasn't really successful anyway and the counter should // remain incremented. uint32_t val = 0; - this->rtc_.save(&val); + if (!this->rtc_.save(&val)) { + ESP_LOGE(TAG, "Failed to clear boot loop counter"); + } } void SafeModeComponent::on_safe_shutdown() { From f9824ee83f2c0e866bf6ca76e3fb5a8a10ef5b64 Mon Sep 17 00:00:00 2001 From: Zebble Date: Tue, 1 Sep 2026 14:23:21 -0400 Subject: [PATCH 074/147] [core] Move CONF_KEYS to the shared component constants (#18933) Co-authored-by: Claude --- esphome/components/const/__init__.py | 2 ++ esphome/components/lvgl/widgets/table.py | 3 +-- esphome/components/matrix_keypad/__init__.py | 4 +--- esphome/components/sx1509/__init__.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index e445a4abde..49a625e3f1 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -14,6 +14,7 @@ CONF_CHANNEL_COLORS = "channel_colors" CONF_CLIMATE_ID = "climate_id" CONF_CO2_EQUIVALENT = "co2_equivalent" CONF_COLOR_DEPTH = "color_depth" +CONF_COLUMNS = "columns" CONF_CRC_ENABLE = "crc_enable" CONF_DATA_BITS = "data_bits" CONF_DESCRIPTION = "description" @@ -25,6 +26,7 @@ CONF_GYROSCOPE_RANGE = "gyroscope_range" CONF_IAQ = "iaq" CONF_IGNORE_NOT_FOUND = "ignore_not_found" CONF_IS_WRGB = "is_wrgb" +CONF_KEYS = "keys" CONF_LABEL = "label" CONF_LIBRETINY = "libretiny" CONF_LOOP = "loop" diff --git a/esphome/components/lvgl/widgets/table.py b/esphome/components/lvgl/widgets/table.py index efae2be2be..f000ea1846 100644 --- a/esphome/components/lvgl/widgets/table.py +++ b/esphome/components/lvgl/widgets/table.py @@ -2,7 +2,7 @@ from contextlib import ExitStack from esphome import automation import esphome.codegen as cg -from esphome.components.const import CONF_ROWS +from esphome.components.const import CONF_COLUMNS, CONF_ROWS import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_ITEMS, CONF_ROW, CONF_TEXT, CONF_WIDTH from esphome.core import ID @@ -20,7 +20,6 @@ from .label import CONF_LABEL CONF_TABLE = "table" CONF_CELLS = "cells" -CONF_COLUMNS = "columns" CONF_ROW_COUNT = "row_count" CONF_COLUMN_COUNT = "column_count" CONF_MERGE_RIGHT = "merge_right" diff --git a/esphome/components/matrix_keypad/__init__.py b/esphome/components/matrix_keypad/__init__.py index 47cf4793b1..2e43eaf7e2 100644 --- a/esphome/components/matrix_keypad/__init__.py +++ b/esphome/components/matrix_keypad/__init__.py @@ -1,7 +1,7 @@ from esphome import automation, pins import esphome.codegen as cg from esphome.components import key_provider -from esphome.components.const import CONF_ROWS +from esphome.components.const import CONF_COLUMNS, CONF_KEYS, CONF_ROWS import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_ON_KEY, CONF_PIN, CONF_TRIGGER_ID from esphome.types import ConfigType @@ -21,8 +21,6 @@ MatrixKeyTrigger = matrix_keypad_ns.class_( ) CONF_KEYPAD_ID = "keypad_id" -CONF_COLUMNS = "columns" -CONF_KEYS = "keys" CONF_DEBOUNCE_TIME = "debounce_time" CONF_HAS_DIODES = "has_diodes" CONF_HAS_PULLDOWNS = "has_pulldowns" diff --git a/esphome/components/sx1509/__init__.py b/esphome/components/sx1509/__init__.py index c1e4e11d54..7694b8f732 100644 --- a/esphome/components/sx1509/__init__.py +++ b/esphome/components/sx1509/__init__.py @@ -1,6 +1,7 @@ from esphome import automation, pins import esphome.codegen as cg from esphome.components import i2c, key_provider +from esphome.components.const import CONF_KEYS import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -19,7 +20,6 @@ from esphome.cpp_generator import MockObj from esphome.types import ConfigType CONF_KEYPAD = "keypad" -CONF_KEYS = "keys" CONF_KEY_ROWS = "key_rows" CONF_KEY_COLUMNS = "key_columns" CONF_SLEEP_TIME = "sleep_time" From 0aff9e1c54f45dedaf111643c0e157dd87a669b9 Mon Sep 17 00:00:00 2001 From: Andrej Walilko <3455017+ch604@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:33:35 -0400 Subject: [PATCH 075/147] [d01] add D01 pm2.5 sensor support (#17788) Co-authored-by: Andrej Walilko Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- CODEOWNERS | 1 + esphome/components/d01/__init__.py | 0 esphome/components/d01/d01.cpp | 45 ++++++++++++++++++++++ esphome/components/d01/d01.h | 14 +++++++ esphome/components/d01/sensor.py | 45 ++++++++++++++++++++++ tests/components/d01/common.yaml | 3 ++ tests/components/d01/test.esp32-idf.yaml | 7 ++++ tests/components/d01/test.esp8266-ard.yaml | 7 ++++ tests/components/d01/test.rp2040-ard.yaml | 7 ++++ 9 files changed, 129 insertions(+) create mode 100644 esphome/components/d01/__init__.py create mode 100644 esphome/components/d01/d01.cpp create mode 100644 esphome/components/d01/d01.h create mode 100644 esphome/components/d01/sensor.py create mode 100644 tests/components/d01/common.yaml create mode 100644 tests/components/d01/test.esp32-idf.yaml create mode 100644 tests/components/d01/test.esp8266-ard.yaml create mode 100644 tests/components/d01/test.rp2040-ard.yaml diff --git a/CODEOWNERS b/CODEOWNERS index 13fae0664b..7bb7f310a3 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -131,6 +131,7 @@ esphome/components/cst816/* @clydebarrow esphome/components/cst9220/* @clydebarrow esphome/components/ct_clamp/* @jesserockz esphome/components/current_based/* @djwmarcx +esphome/components/d01/* @ch604 esphome/components/dac7678/* @NickB1 esphome/components/daikin_arc/* @MagicBear esphome/components/daikin_brc/* @hagak diff --git a/esphome/components/d01/__init__.py b/esphome/components/d01/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esphome/components/d01/d01.cpp b/esphome/components/d01/d01.cpp new file mode 100644 index 0000000000..f7a0c08ec4 --- /dev/null +++ b/esphome/components/d01/d01.cpp @@ -0,0 +1,45 @@ +#include "d01.h" +#include "esphome/core/log.h" + +// uart specification for d01 sensor from https://manuals.plus/ae/1005006417362019: +// +// A frame of serial output data includes 4 bytes, formatted as follows: +// __Characteristic byte: Fixed value 0xA5. +// __Data byte: DATAH is the high 7 bits of the concentration value, and DATAL is the low 7 bits of the concentration +// value. +// __Check byte: The low 7 bits of the sum of all bytes before the check byte. +// +// If the serial output is 4 bytes of data: 0*A5 0*01 0*2C 0*52, then DATAH = 0*01 = 1, DATAL = 0*2C = 44. +// Concentration value = 1*128 + 44 = 172 µg/m³. +// +// The PM2.5 dust concentration value obtained from the dust sensor needs to be calibrated with a K value coefficient +// based on the TSI instrument's photometric method. It is generally recommended to use 0.4. + +namespace esphome::d01 { + +static const char *const TAG = "d01"; + +static const uint8_t D01_FRAME_HEADER = 0xA5; + +void D01SensorComponent::dump_config() { LOG_SENSOR(" ", "D01 PM2.5", this); } + +void D01SensorComponent::loop() { + uint8_t buf[4]; + while (this->available() >= 4) { + if (this->peek() != D01_FRAME_HEADER) { + this->read(); + continue; + } + this->read_array(buf, 4); + uint8_t sum = (buf[0] + buf[1] + buf[2]) & 0x7F; + if (sum != buf[3]) { + ESP_LOGW(TAG, "checksum mismatch"); + continue; + } + uint16_t latest_concentration = (buf[1] & 0x7F) * 128 + (buf[2] & 0x7F); + ESP_LOGV(TAG, "Unadjusted PM2.5 Concentration: %d µg/m³", latest_concentration); + this->publish_state(latest_concentration); + } +} + +} // namespace esphome::d01 diff --git a/esphome/components/d01/d01.h b/esphome/components/d01/d01.h new file mode 100644 index 0000000000..73c7a8711d --- /dev/null +++ b/esphome/components/d01/d01.h @@ -0,0 +1,14 @@ +#pragma once +#include "esphome/core/component.h" +#include "esphome/components/sensor/sensor.h" +#include "esphome/components/uart/uart.h" + +namespace esphome::d01 { + +class D01SensorComponent final : public sensor::Sensor, public Component, public uart::UARTDevice { + public: + void dump_config() override; + void loop() override; +}; + +} // namespace esphome::d01 diff --git a/esphome/components/d01/sensor.py b/esphome/components/d01/sensor.py new file mode 100644 index 0000000000..5bc5a4e424 --- /dev/null +++ b/esphome/components/d01/sensor.py @@ -0,0 +1,45 @@ +import esphome.codegen as cg +from esphome.components import sensor, uart +import esphome.config_validation as cv +from esphome.const import ( + DEVICE_CLASS_PM25, + ICON_BLUR, + STATE_CLASS_MEASUREMENT, + UNIT_MICROGRAMS_PER_CUBIC_METER, +) +from esphome.types import ConfigType + +CODEOWNERS = ["@ch604"] +DEPENDENCIES = ["uart"] + +d01_ns = cg.esphome_ns.namespace("d01") +D01SensorComponent = d01_ns.class_( + "D01SensorComponent", sensor.Sensor, uart.UARTDevice, cg.Component +) + + +CONFIG_SCHEMA = ( + sensor.sensor_schema( + D01SensorComponent, + unit_of_measurement=UNIT_MICROGRAMS_PER_CUBIC_METER, + icon=ICON_BLUR, + accuracy_decimals=0, + device_class=DEVICE_CLASS_PM25, + state_class=STATE_CLASS_MEASUREMENT, + ) + .extend(cv.COMPONENT_SCHEMA) + .extend(uart.UART_DEVICE_SCHEMA) +) + +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "d01", + baud_rate=9600, + require_rx=True, + require_tx=False, +) + + +async def to_code(config: ConfigType) -> None: + var = await sensor.new_sensor(config) + await cg.register_component(var, config) + await uart.register_uart_device(var, config) diff --git a/tests/components/d01/common.yaml b/tests/components/d01/common.yaml new file mode 100644 index 0000000000..b59ec06ff0 --- /dev/null +++ b/tests/components/d01/common.yaml @@ -0,0 +1,3 @@ +sensor: + - platform: d01 + name: D01 PM2.5 Concentration diff --git a/tests/components/d01/test.esp32-idf.yaml b/tests/components/d01/test.esp32-idf.yaml new file mode 100644 index 0000000000..b658bfbede --- /dev/null +++ b/tests/components/d01/test.esp32-idf.yaml @@ -0,0 +1,7 @@ +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +packages: + uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + d01: !include common.yaml diff --git a/tests/components/d01/test.esp8266-ard.yaml b/tests/components/d01/test.esp8266-ard.yaml new file mode 100644 index 0000000000..876615ae9f --- /dev/null +++ b/tests/components/d01/test.esp8266-ard.yaml @@ -0,0 +1,7 @@ +substitutions: + tx_pin: GPIO0 + rx_pin: GPIO2 + +packages: + uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml + d01: !include common.yaml diff --git a/tests/components/d01/test.rp2040-ard.yaml b/tests/components/d01/test.rp2040-ard.yaml new file mode 100644 index 0000000000..00ed175b42 --- /dev/null +++ b/tests/components/d01/test.rp2040-ard.yaml @@ -0,0 +1,7 @@ +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +packages: + uart: !include ../../test_build_components/common/uart/rp2040-ard.yaml + d01: !include common.yaml From 3f68930001385b4f66f5fa799eaa79a370bf3da5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 1 Sep 2026 16:11:57 -0400 Subject: [PATCH 076/147] [ota] Add Noise encryption to the OTA platform (#18489) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- THREAT_MODEL.md | 38 +- esphome/__main__.py | 29 +- esphome/components/esphome/ota/__init__.py | 135 +++++- .../components/esphome/ota/ota_esphome.cpp | 121 ++++-- esphome/components/esphome/ota/ota_esphome.h | 70 ++- .../esphome/ota/ota_esphome_noise.cpp | 279 ++++++++++++ esphome/components/noise/__init__.py | 9 + esphome/components/ota/ota_backend.h | 1 + esphome/core/defines.h | 1 + esphome/espota2.py | 198 ++++++++- .../noise/test_encryption_key.py | 11 +- tests/component_tests/ota/test_esphome_ota.py | 314 +++++++++++++- tests/components/ota/encryption.yaml | 9 + tests/components/ota/encryption_inherit.yaml | 12 + .../ota/test-encryption.esp32-idf.yaml | 2 + .../ota/test-encryption.esp8266-ard.yaml | 2 + .../ota/test-encryption.rp2040-ard.yaml | 2 + .../test-encryption_inherit.esp8266-ard.yaml | 2 + .../fixtures/host_ota_encrypted.yaml | 11 + tests/integration/test_host_ota.py | 57 +++ tests/unit_tests/test_espota2_noise.py | 407 ++++++++++++++++++ tests/unit_tests/test_main.py | 108 ++++- 22 files changed, 1772 insertions(+), 46 deletions(-) create mode 100644 esphome/components/esphome/ota/ota_esphome_noise.cpp create mode 100644 tests/components/ota/encryption.yaml create mode 100644 tests/components/ota/encryption_inherit.yaml create mode 100644 tests/components/ota/test-encryption.esp32-idf.yaml create mode 100644 tests/components/ota/test-encryption.esp8266-ard.yaml create mode 100644 tests/components/ota/test-encryption.rp2040-ard.yaml create mode 100644 tests/components/ota/test-encryption_inherit.esp8266-ard.yaml create mode 100644 tests/integration/fixtures/host_ota_encrypted.yaml create mode 100644 tests/unit_tests/test_espota2_noise.py diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index 5816f38176..b4f557e55b 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -23,7 +23,8 @@ For this repository there are two trusted inputs by design: 1. **The configuration.** Anyone who can supply or edit a YAML config is trusted (see below). 2. **Authenticated peers of a running device** — clients holding the device's - API encryption key / password, OTA password, or web server credentials. + API/OTA encryption key, API password, OTA password, or web server + credentials. The security boundary is therefore **unauthenticated network traffic vs. those trusted inputs.** A bug that lets an unauthenticated attacker cross it is a @@ -76,8 +77,8 @@ These *are* security bugs in this repo, and we want to hear about them privately captive portal, etc.) **without** valid credentials. - Authentication or encryption bypass on the device — reaching API calls, OTA updates, or the web server without the configured key/password. -- Flaws that weaken the device's API encryption (Noise), OTA, or web server auth - below their documented guarantees. +- Flaws that weaken the device's API or OTA encryption (Noise), OTA auth, or + web server auth below their documented guarantees. ## The web server is an open HTTP API by design @@ -121,6 +122,37 @@ and any memory-safety or protocol bug in the server reachable without credential This section documents the current design and scope; it is not a judgment that the design is optimal or that it will not change. +## OTA update encryption + +The `esphome` OTA platform optionally encrypts updates with the same Noise +`NNpsk0` pattern the native API uses; one key protects the device. With an +`encryption:` block configured the guarantees are: the firmware image is +confidential in transit, the uploader is authenticated by the pre-shared key, +and the plaintext negotiation preceding the handshake is bound into the +handshake prologue, so stripping or tampering with it fails the first MAC. +Both ends fail closed with no override: a device built with a key refuses +plaintext uploads, and the CLI refuses to send plaintext when a key is +configured. + +Defeating any of that without the key is in scope: a keyed device accepting a +plaintext or downgraded upload, getting past the MAC, or recovering image +contents from captured traffic. + +The following are **not** vulnerabilities, by design: + +- Plaintext OTA on a device with no `encryption:` block. That is the + documented default, authenticated (if at all) by the OTA password. +- The enablement window: turning encryption on takes one last upload of the + encryption-enabled firmware over the existing plaintext channel, with the + pre-existing plaintext exposure. +- The web OTA `/update` endpoint alongside encryption. The `web_server` + component keeps it always reachable, and `captive_portal:` auto-loads it + for the fallback AP window; validation warns about both combinations, and + the operator keeps the recovery path. +- CLI retry behavior on transport or MAC failures; every attempt renegotiates + a fresh handshake with fresh ephemerals, so retrying does not weaken + authentication. + ## Explicitly out of scope - Local attackers who already have shell access on the host that runs `esphome`. diff --git a/esphome/__main__.py b/esphome/__main__.py index 1ebf194205..b3d58ad13b 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -26,7 +26,9 @@ from esphome.const import ( CONF_DEASSERT_RTS_DTR, CONF_DISABLED, CONF_DISCOVER_IP, + CONF_ENCRYPTION, CONF_ESPHOME, + CONF_KEY, CONF_LEVEL, CONF_LOG, CONF_LOG_TOPIC, @@ -1336,6 +1338,19 @@ def _upload_via_native_api( remote_port = int(ota_conf[CONF_PORT]) password = ota_conf.get(CONF_PASSWORD) + # Fail closed: an encryption block whose key did not resolve must never + # fall back to a plaintext upload + noise_psk = None + if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: + noise_psk = encryption_conf.get(CONF_KEY) + if not noise_psk: + raise EsphomeError( + "OTA encryption is configured but no key was resolved; " + "set the key under 'ota: encryption:' or 'api: encryption:'" + ) + # Ensure the key is a string, as required by the underlying OTA implementation. + # It arrives here as a SensitiveStr which aioesphomeapi rejects. + noise_psk = str(noise_psk) def check_partition_access(option_string: str) -> None: if not ota_conf.get("allow_partition_access"): @@ -1366,7 +1381,9 @@ def _upload_via_native_api( if ota_type == espota2.OTA_TYPE_UPDATE_BOOTLOADER: _validate_bootloader_binary(binary) - return espota2.run_ota(network_devices, remote_port, password, binary, ota_type) + return espota2.run_ota( + network_devices, remote_port, password, binary, ota_type, noise_psk + ) def _upload_via_web_server( @@ -1375,6 +1392,16 @@ def _upload_via_web_server( from esphome import web_server_ota from esphome.web_server_helpers import get_web_server_connection + if any( + ota_item.get(CONF_PLATFORM) == CONF_ESPHOME + and ota_item.get(CONF_ENCRYPTION) is not None + for ota_item in config.get(CONF_OTA, []) + ): + _LOGGER.warning( + "This config has OTA encryption, but the web_server OTA path sends " + "the image over plaintext HTTP; use the esphome OTA platform to " + "keep it confidential" + ) remote_port, username, password = get_web_server_connection(config) return web_server_ota.run_ota( network_devices, remote_port, username, password, binary diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index 3ef4c7ba13..1fec9e5c9b 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -1,12 +1,20 @@ import logging import esphome.codegen as cg +from esphome.components.noise import ( + decode_encryption_key, + encryption_schema, + is_reserved_key, +) from esphome.components.ota import BASE_OTA_SCHEMA, OTAComponent, ota_to_code from esphome.config_helpers import merge_config import esphome.config_validation as cv from esphome.const import ( + CONF_API, + CONF_ENCRYPTION, CONF_ESPHOME, CONF_ID, + CONF_KEY, CONF_NUM_ATTEMPTS, CONF_OTA, CONF_PASSWORD, @@ -15,6 +23,7 @@ from esphome.const import ( CONF_REBOOT_TIMEOUT, CONF_SAFE_MODE, CONF_VERSION, + CONF_WEB_SERVER, ) from esphome.core import CORE, coroutine_with_priority from esphome.coroutine import CoroPriority @@ -22,6 +31,7 @@ import esphome.final_validate as fv from esphome.types import ConfigType CONF_ALLOW_PARTITION_ACCESS = "allow_partition_access" +CONF_CAPTIVE_PORTAL = "captive_portal" _LOGGER = logging.getLogger(__name__) @@ -30,7 +40,15 @@ CODEOWNERS = ["@esphome/core"] DEPENDENCIES = ["network"] -AUTO_LOAD = ["sha256", "socket"] +def AUTO_LOAD(config: ConfigType) -> list[str]: + """Auto-load noise only when encryption is configured.""" + base = ["sha256", "socket"] + # A falsy config is a tooling probe for the maximal set (None from + # dependency resolution, {} from the components-graph platform probe); + # a validated config always carries defaults, never empty + if not config or CONF_ENCRYPTION in config: + return base + ["noise"] + return base esphome = cg.esphome_ns.namespace("esphome") @@ -67,11 +85,24 @@ def ota_esphome_final_validate(config: ConfigType) -> None: CONF_PASSWORD in merged_ota_esphome_configs_by_port[conf_port] and CONF_PASSWORD in ota_conf and merged_ota_esphome_configs_by_port[conf_port][CONF_PASSWORD] - != ota_conf.get(CONF_PASSWORD) + != ota_conf[CONF_PASSWORD] ): raise cv.Invalid( f"Found multiple configurations but {CONF_PASSWORD} is inconsistent" ) + # Encryption blocks conflict only when both pin a key; a bare + # `encryption:` (a package/device split) is compatible with a + # keyed one, and merge_config yields the keyed result + merged_key = ( + merged_ota_esphome_configs_by_port[conf_port] + .get(CONF_ENCRYPTION, {}) + .get(CONF_KEY) + ) + other_key = ota_conf.get(CONF_ENCRYPTION, {}).get(CONF_KEY) + if merged_key and other_key and merged_key != other_key: + raise cv.Invalid( + f"Found multiple configurations but {CONF_ENCRYPTION} is inconsistent" + ) ports_with_merged_configs.append(conf_port) merged_ota_esphome_configs_by_port[conf_port] = merge_config( @@ -94,6 +125,20 @@ def ota_esphome_final_validate(config: ConfigType) -> None: new_ota_conf.extend(merged_ota_esphome_configs_by_port.values()) + api_conf = full_conf.get(CONF_API) or {} + for ota_conf in merged_ota_esphome_configs_by_port.values(): + # Merging same-port blocks can combine a password from one block with + # encryption from another; re-check the exclusion on the merged result. + _validate_no_password_with_encryption(ota_conf) + if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: + _resolve_encryption_key(encryption_conf, api_conf) + if any( + conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf + ) and any( + CONF_ENCRYPTION in conf for conf in merged_ota_esphome_configs_by_port.values() + ): + _warn_web_server_ota(full_conf) + full_conf[CONF_OTA] = new_ota_conf fv.full_config.set(full_conf) @@ -107,6 +152,73 @@ def ota_esphome_final_validate(config: ConfigType) -> None: ) +def _warn_web_server_ota(full_conf: ConfigType) -> None: + """The web_server ota platform accepts the same image over plaintext HTTP + with basic auth, bypassing the encryption; warn rather than fail so the + operator keeps the recovery path.""" + if CONF_CAPTIVE_PORTAL in full_conf and CONF_WEB_SERVER not in full_conf: + # The captive_portal auto-load: the endpoint only exists while the + # fallback AP is active + _LOGGER.warning( + "OTA encryption does not cover the %s OTA platform (auto-loaded " + "by captive_portal); the plaintext /update endpoint stays " + "reachable while the fallback AP is active", + CONF_WEB_SERVER, + ) + else: + _LOGGER.warning( + "OTA encryption does not cover the %s OTA platform; its " + "plaintext /update endpoint accepts the same image", + CONF_WEB_SERVER, + ) + + +def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) -> None: + """Resolve the one encryption key per device into the ota block. + + An explicit ota key must match the api key, a bare block inherits it, + a runtime provisioned api key cannot be inherited, and the all-zeros + provisioning sentinel is rejected (the device treats it as no key). + """ + api_key = api_conf.get(CONF_ENCRYPTION, {}).get(CONF_KEY) + if ota_key := encryption_conf.get(CONF_KEY): + if api_key and ota_key != api_key: + raise cv.Invalid( + f"'{CONF_OTA}' {CONF_ENCRYPTION} {CONF_KEY} must match the " + f"'{CONF_API}' {CONF_ENCRYPTION} {CONF_KEY}; omit the " + f"'{CONF_OTA}' {CONF_KEY} to use the '{CONF_API}' one" + ) + elif not api_key: + if CONF_ENCRYPTION in api_conf: + raise cv.Invalid( + f"the '{CONF_API}' {CONF_ENCRYPTION} {CONF_KEY} is provisioned at " + f"runtime and cannot be inherited at build time; set an explicit " + f"'{CONF_OTA}' {CONF_ENCRYPTION} {CONF_KEY}" + ) + raise cv.Invalid( + f"'{CONF_OTA}' {CONF_ENCRYPTION} has no {CONF_KEY} and there is no " + f"'{CONF_API}' {CONF_ENCRYPTION} {CONF_KEY} to inherit; set one of them" + ) + else: + encryption_conf[CONF_KEY] = api_key + if is_reserved_key(encryption_conf[CONF_KEY]): + raise cv.Invalid( + f"The all-zeros {CONF_KEY} is reserved and provides no protection; " + f"generate a real key with: openssl rand -base64 32" + ) + + +# Also called on merged same-port configs in final validate, where schemas +# do not run +def _validate_no_password_with_encryption(config: ConfigType) -> ConfigType: + if CONF_PASSWORD in config and CONF_ENCRYPTION in config: + raise cv.Invalid( + f"'{CONF_PASSWORD}' cannot be combined with '{CONF_ENCRYPTION}'; the " + f"encryption key already authenticates the uploader, remove '{CONF_PASSWORD}'" + ) + return config + + def _consume_ota_sockets(config: ConfigType) -> ConfigType: """Register socket needs for OTA component.""" from esphome.components import socket @@ -134,6 +246,7 @@ CONFIG_SCHEMA = cv.All( ): cv.port, cv.Optional(CONF_ALLOW_PARTITION_ACCESS, default=False): cv.boolean, cv.Optional(CONF_PASSWORD): cv.sensitive(), + cv.Optional(CONF_ENCRYPTION): encryption_schema, cv.Optional(CONF_NUM_ATTEMPTS): cv.invalid( f"'{CONF_SAFE_MODE}' (and its related configuration variables) has moved from 'ota' to its own component. See https://esphome.io/components/safe_mode" ), @@ -147,12 +260,24 @@ CONFIG_SCHEMA = cv.All( ) .extend(BASE_OTA_SCHEMA) .extend(cv.COMPONENT_SCHEMA), + _validate_no_password_with_encryption, _consume_ota_sockets, ) FINAL_VALIDATE_SCHEMA = ota_esphome_final_validate +def FILTER_SOURCE_FILES() -> list[str]: + """Filter out the noise transport when no ota entry configures encryption.""" + for ota_conf in CORE.config.get(CONF_OTA, []): + if ( + ota_conf.get(CONF_PLATFORM) == CONF_ESPHOME + and ota_conf.get(CONF_ENCRYPTION) is not None + ): + return [] + return ["ota_esphome_noise.cpp"] + + @coroutine_with_priority(CoroPriority.OTA_UPDATES) async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) @@ -171,6 +296,12 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_ALLOW_PARTITION_ACCESS): cg.add_define("USE_OTA_PARTITIONS") + if (encryption_conf := config.get(CONF_ENCRYPTION)) is not None: + # A missing key was resolved from the api component in final validate. + key = encryption_conf[CONF_KEY] + cg.add_define("USE_OTA_ENCRYPTION") + cg.add(var.set_noise_psk(list(decode_encryption_key(key)))) + # Build flag so lwip_fast_select.c (a .c file that can't include defines.h) sees it. cg.add_build_flag("-DUSE_OTA_PLATFORM_ESPHOME") diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 9f15eaaede..396a47bc52 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -27,7 +27,6 @@ namespace esphome { static const char *const TAG = "esphome.ota"; static constexpr uint16_t OTA_BLOCK_SIZE = 8192; -static constexpr size_t OTA_BUFFER_SIZE = 1024; // buffer size for OTA data transfer static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer @@ -105,6 +104,11 @@ void ESPHomeOTAComponent::dump_config() { ESP_LOGCONFIG(TAG, " Password configured"); } #endif +#ifdef USE_OTA_ENCRYPTION + if (this->noise_ctx_.has_psk()) { + ESP_LOGCONFIG(TAG, " Encryption configured"); + } +#endif #ifdef USE_OTA_PARTITIONS ESP_LOGCONFIG(TAG, " Partition access allowed\n" @@ -149,8 +153,10 @@ void ESPHomeOTAComponent::loop() { static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04; +static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_NOISE = 0x08; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02; +static constexpr uint8_t SERVER_FEATURE_SUPPORTS_NOISE = 0x04; void ESPHomeOTAComponent::handle_handshake_() { /// Handle the OTA handshake and authentication. @@ -202,8 +208,7 @@ void ESPHomeOTAComponent::handle_handshake_() { } // Validate magic bytes - static const uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45}; - if (memcmp(this->handshake_buf_, MAGIC_BYTES, 5) != 0) { + if (memcmp(this->handshake_buf_, MAGIC_BYTES, sizeof(MAGIC_BYTES)) != 0) { ESP_LOGW(TAG, "Magic bytes mismatch! 0x%02X-0x%02X-0x%02X-0x%02X-0x%02X", this->handshake_buf_[0], this->handshake_buf_[1], this->handshake_buf_[2], this->handshake_buf_[3], this->handshake_buf_[4]); this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_MAGIC); @@ -235,6 +240,19 @@ void ESPHomeOTAComponent::handle_handshake_() { } this->ota_features_ = this->handshake_buf_[0]; ESP_LOGV(TAG, "Features: 0x%02X", this->ota_features_); + +#ifdef USE_OTA_ENCRYPTION + // Fail closed: with a PSK configured the client must negotiate encryption + // (which requires the extended protocol); refuse plaintext uploads. + static constexpr uint8_t NOISE_REQUIRED_FEATURES = + CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; + if (this->noise_ctx_.has_psk() && (this->ota_features_ & NOISE_REQUIRED_FEATURES) != NOISE_REQUIRED_FEATURES) { + ESP_LOGW(TAG, "Client does not support encryption"); + this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_ENCRYPTION_REQUIRED); + return; + } +#endif + this->transition_ota_state_(OTAState::FEATURE_ACK); const bool supports_compression = @@ -250,6 +268,11 @@ void ESPHomeOTAComponent::handle_handshake_() { this->handshake_buf_[1] = (supports_compression ? SERVER_FEATURE_SUPPORTS_COMPRESSION : 0); #ifdef USE_OTA_PARTITIONS this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS; +#endif +#ifdef USE_OTA_ENCRYPTION + if (this->noise_ctx_.has_psk()) { + this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; + } #endif } else { this->handshake_buf_[0] = @@ -265,6 +288,20 @@ void ESPHomeOTAComponent::handle_handshake_() { if (!this->try_write_(ack_size, LOG_STR("ack feature"))) { return; } +#ifdef USE_OTA_ENCRYPTION + // With a PSK configured the rest of the session runs inside the noise + // transport; the client sends the first handshake frame next, so there + // is nothing to do until data arrives. + if (this->noise_ctx_.has_psk()) { + // handshake_buf_ still holds the feature ack composed above; a + // would-block re-entry lands here without rebuilding it + if (!this->noise_start_session_(this->handshake_buf_[1])) { + return; + } + this->transition_ota_state_(OTAState::NOISE_HANDSHAKE); + return; + } +#endif #ifdef USE_OTA_PASSWORD // If password is set, move to auth phase if (!this->password_.empty()) { @@ -302,6 +339,16 @@ void ESPHomeOTAComponent::handle_handshake_() { this->handle_data_(); return; +#ifdef USE_OTA_ENCRYPTION + case OTAState::NOISE_HANDSHAKE: + if (!this->handle_noise_handshake_()) { + return; + } + this->transition_ota_state_(OTAState::DATA); + this->handle_data_(); + return; +#endif + default: break; } @@ -340,6 +387,8 @@ void ESPHomeOTAComponent::handle_data_() { /// Raw TCP (8266, RP2040): setblocking is no-op; SO_RCVTIMEO uses /// wakeable_delay() in read(); /// write() always returns immediately + // Backend calls overwrite this with OK; reset to UNKNOWN before any + // goto error that follows a successful begin()/write() ota::OTAResponseTypes error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; size_t total = 0; uint32_t last_progress = 0; @@ -361,11 +410,11 @@ void ESPHomeOTAComponent::handle_data_() { this->client_->setblocking(true); // Acknowledge auth OK - 1 byte - this->write_byte_(ota::OTA_RESPONSE_AUTH_OK); + this->data_write_byte_(ota::OTA_RESPONSE_AUTH_OK); if (this->extended_proto_) { // Read ota type, 1 byte - if (!this->readall_(buf, 1)) { + if (!this->data_readall_(buf, 1)) { this->log_read_error_(LOG_STR("OTA type")); goto error; // NOLINT(cppcoreguidelines-avoid-goto) } @@ -374,7 +423,7 @@ void ESPHomeOTAComponent::handle_data_() { ESP_LOGV(TAG, "OTA type is 0x%02x", ota_type); // Read size, 4 bytes MSB first - if (!this->readall_(buf, 4)) { + if (!this->data_readall_(buf, 4)) { this->log_read_error_(LOG_STR("size")); goto error; // NOLINT(cppcoreguidelines-avoid-goto) } @@ -405,11 +454,12 @@ void ESPHomeOTAComponent::handle_data_() { goto error; // NOLINT(cppcoreguidelines-avoid-goto) // Acknowledge prepare OK - 1 byte - this->write_byte_(ota::OTA_RESPONSE_UPDATE_PREPARE_OK); + this->data_write_byte_(ota::OTA_RESPONSE_UPDATE_PREPARE_OK); // Read binary MD5, 32 bytes - if (!this->readall_(buf, 32)) { + if (!this->data_readall_(buf, 32)) { this->log_read_error_(LOG_STR("MD5 checksum")); + error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; goto error; // NOLINT(cppcoreguidelines-avoid-goto) } sbuf[32] = '\0'; @@ -417,7 +467,7 @@ void ESPHomeOTAComponent::handle_data_() { this->backend_->set_update_md5(sbuf); // Acknowledge MD5 OK - 1 byte - this->write_byte_(ota::OTA_RESPONSE_BIN_MD5_OK); + this->data_write_byte_(ota::OTA_RESPONSE_BIN_MD5_OK); // Track when we last received data so a silently-vanished peer (no FIN/RST // delivered, e.g. uploader killed mid-transfer or NAT/router dropped state) @@ -433,19 +483,35 @@ void ESPHomeOTAComponent::handle_data_() { } size_t remaining = ota_size - total; size_t requested = remaining < OTA_BUFFER_SIZE ? remaining : OTA_BUFFER_SIZE; - ssize_t read = this->client_->read(buf, requested); - if (read == -1) { - const int err = errno; - if (this->would_block_(err)) { - // read() already waited up to SO_RCVTIMEO for data, just feed WDT - App.feed_wdt(); - continue; + ssize_t read; +#ifdef USE_OTA_ENCRYPTION + if (this->noise_ != nullptr) { + // One frame per call; noise_read_data_ waits internally (readall_), so + // there is no would-block retry here and failures are already logged. + read = this->noise_read_data_(buf, requested); + if (read <= 0) { + error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; + goto error; // NOLINT(cppcoreguidelines-avoid-goto) + } + } else +#endif + { + read = this->client_->read(buf, requested); + if (read == -1) { + const int err = errno; + if (this->would_block_(err)) { + // read() already waited up to SO_RCVTIMEO for data, just feed WDT + App.feed_wdt(); + continue; + } + ESP_LOGW(TAG, "Read err %d", err); + error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; + goto error; // NOLINT(cppcoreguidelines-avoid-goto) + } else if (read == 0) { + ESP_LOGW(TAG, "Remote closed"); + error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; + goto error; // NOLINT(cppcoreguidelines-avoid-goto) } - ESP_LOGW(TAG, "Read err %d", err); - goto error; // NOLINT(cppcoreguidelines-avoid-goto) - } else if (read == 0) { - ESP_LOGW(TAG, "Remote closed"); - goto error; // NOLINT(cppcoreguidelines-avoid-goto) } last_data_ms = millis(); @@ -457,7 +523,7 @@ void ESPHomeOTAComponent::handle_data_() { total += read; #if USE_OTA_VERSION == 2 while (size_acknowledged + OTA_BLOCK_SIZE <= total || (total == ota_size && size_acknowledged < ota_size)) { - this->write_byte_(ota::OTA_RESPONSE_CHUNK_OK); + this->data_write_byte_(ota::OTA_RESPONSE_CHUNK_OK); size_acknowledged += OTA_BLOCK_SIZE; } #endif @@ -476,7 +542,7 @@ void ESPHomeOTAComponent::handle_data_() { } // Acknowledge receive OK - 1 byte - this->write_byte_(ota::OTA_RESPONSE_RECEIVE_OK); + this->data_write_byte_(ota::OTA_RESPONSE_RECEIVE_OK); error_code = this->backend_->end(); if (error_code != ota::OTA_RESPONSE_OK) { @@ -485,10 +551,10 @@ void ESPHomeOTAComponent::handle_data_() { } // Acknowledge Update end OK - 1 byte - this->write_byte_(ota::OTA_RESPONSE_UPDATE_END_OK); + this->data_write_byte_(ota::OTA_RESPONSE_UPDATE_END_OK); // Read ACK - if (!this->readall_(buf, 1) || buf[0] != ota::OTA_RESPONSE_OK) { + if (!this->data_readall_(buf, 1) || buf[0] != ota::OTA_RESPONSE_OK) { this->log_read_error_(LOG_STR("ack")); // do not go to error, this is not fatal } @@ -511,7 +577,7 @@ void ESPHomeOTAComponent::handle_data_() { App.safe_reboot(); error: - this->write_byte_(static_cast(error_code)); + this->data_write_byte_(static_cast(error_code)); // Abort backend before cleanup - cleanup_connection_() destroys the backend. // Always call abort() unconditionally: backends register external partitions before @@ -678,6 +744,9 @@ void ESPHomeOTAComponent::cleanup_connection_() { this->backend_ = nullptr; #ifdef USE_OTA_PASSWORD this->cleanup_auth_(); +#endif +#ifdef USE_OTA_ENCRYPTION + this->noise_ = nullptr; #endif // Intentionally no disable_loop() — letting loop() run one more iteration catches // any connection that queued on the listener mid-session (otherwise the wake flag, diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index 979e3f2d7d..fd164b8138 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -4,6 +4,9 @@ #ifdef USE_OTA #include "esphome/components/ota/ota_backend_factory.h" #include "esphome/components/socket/socket.h" +#ifdef USE_OTA_ENCRYPTION +#include "esphome/components/noise/noise_handshake.h" +#endif #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/preferences.h" @@ -24,7 +27,10 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { AUTH_SEND, // Sending authentication request AUTH_READ, // Reading authentication data #endif // USE_OTA_PASSWORD - DATA, // BLOCKING! Processing OTA data (update, etc.) +#ifdef USE_OTA_ENCRYPTION + NOISE_HANDSHAKE, // Exchanging Noise handshake frames +#endif + DATA, // BLOCKING! Processing OTA data (update, etc.) }; #ifdef USE_OTA_PASSWORD void set_auth_password(const std::string &password) { password_ = password; } @@ -38,6 +44,10 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { } #endif // USE_OTA_PASSWORD +#ifdef USE_OTA_ENCRYPTION + void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#endif + /// Manually set the port OTA should listen on void set_port(uint16_t port) { this->port_ = port; } @@ -63,6 +73,48 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { bool writeall_(const uint8_t *buf, size_t len); inline bool write_byte_(uint8_t byte) { return this->writeall_(&byte, 1); } +#ifdef USE_OTA_ENCRYPTION + // Heap-allocated only while an encrypted OTA session is active. + struct NoiseSession { + ~NoiseSession(); + noise::NoiseResponderHandshake handshake; + NoiseCipherState *send_cipher{nullptr}; + NoiseCipherState *recv_cipher{nullptr}; + uint16_t frame_len{0}; // total frame size once the header is parsed, 0 until then + uint16_t frame_pos{0}; // bytes read or written so far + bool writing{false}; // a produced handshake frame is still being flushed + uint8_t frame_buf[noise::FRAME_HEADER_SIZE + 1 + noise::MAX_HANDSHAKE_SIZE]; + }; + bool noise_start_session_(uint8_t server_feature_flags); + bool handle_noise_handshake_(); + bool noise_try_read_frame_(); + bool noise_try_write_frame_(); + void noise_send_reject_(const LogString *reason); + ssize_t noise_decrypt_(uint8_t *buf, size_t len); + ssize_t noise_read_frame_blocking_(uint8_t *buf, size_t min_ciphertext, size_t max_ciphertext); + bool noise_readall_(uint8_t *buf, size_t len); + ssize_t noise_read_data_(uint8_t *buf, size_t capacity); + bool noise_write_byte_(uint8_t byte); +#endif // USE_OTA_ENCRYPTION + + // Data-phase I/O dispatch: through the noise transport when a session is + // active, straight to the socket otherwise. + inline bool data_write_byte_(uint8_t byte) { +#ifdef USE_OTA_ENCRYPTION + if (this->noise_ != nullptr) + return this->noise_write_byte_(byte); +#endif + return this->write_byte_(byte); + } + // When encrypted, buf must have room for len + noise::MAC_SIZE bytes. + inline bool data_readall_(uint8_t *buf, size_t len) { +#ifdef USE_OTA_ENCRYPTION + if (this->noise_ != nullptr) + return this->noise_readall_(buf, len); +#endif + return this->readall_(buf, len); + } + bool try_read_(size_t to_read, const LogString *desc); bool try_write_(size_t to_write, const LogString *desc); @@ -91,6 +143,10 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { std::string password_; std::unique_ptr auth_buf_; #endif // USE_OTA_PASSWORD +#ifdef USE_OTA_ENCRYPTION + noise::NoiseContext noise_ctx_; + std::unique_ptr noise_; +#endif // USE_OTA_ENCRYPTION socket::ListenSocket *server_{nullptr}; std::unique_ptr client_; @@ -98,6 +154,18 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { uint32_t client_connect_time_{0}; static constexpr size_t HANDSHAKE_BUF_SIZE = 5; + // Buffer size for OTA data transfer. The upload client derives its maximum + // encrypted frame plaintext from this (espota2.NOISE_MAX_PLAINTEXT is this + // minus the 16-byte MAC); both must change together. + static constexpr size_t OTA_BUFFER_SIZE = 1040; +#ifdef USE_OTA_ENCRYPTION + // espota2.NOISE_MAX_PLAINTEXT; shrinking the buffer would reject every + // frame a current CLI sends + static constexpr size_t NOISE_CLIENT_MAX_PLAINTEXT = 1024; + static_assert(OTA_BUFFER_SIZE >= NOISE_CLIENT_MAX_PLAINTEXT + noise::MAC_SIZE, + "OTA_BUFFER_SIZE must fit a full encrypted data frame"); +#endif + static constexpr uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45}; #ifdef USE_OTA_PARTITIONS uint32_t running_app_offset_{0}; size_t running_app_size_{0}; diff --git a/esphome/components/esphome/ota/ota_esphome_noise.cpp b/esphome/components/esphome/ota/ota_esphome_noise.cpp new file mode 100644 index 0000000000..7f8331cf96 --- /dev/null +++ b/esphome/components/esphome/ota/ota_esphome_noise.cpp @@ -0,0 +1,279 @@ +#include "ota_esphome.h" +#ifdef USE_OTA +#ifdef USE_OTA_ENCRYPTION +#include "esphome/components/noise/noise.h" +#include "esphome/components/ota/ota_backend.h" +#include "esphome/core/log.h" + +#include +#include + +#ifdef USE_ESP8266 +#include +#endif + +namespace esphome { + +static const char *const TAG = "esphome.ota"; + +#ifdef USE_ESP8266 +static constexpr char OTA_NOISE_PROLOGUE_INIT[] PROGMEM = "NoiseOTAInit"; +#else +static constexpr char OTA_NOISE_PROLOGUE_INIT[] = "NoiseOTAInit"; +#endif +static constexpr size_t OTA_NOISE_PROLOGUE_INIT_LEN = sizeof(OTA_NOISE_PROLOGUE_INIT) - 1; + +ESPHomeOTAComponent::NoiseSession::~NoiseSession() { + if (this->send_cipher != nullptr) { + noise_cipherstate_free(this->send_cipher); + } + if (this->recv_cipher != nullptr) { + noise_cipherstate_free(this->recv_cipher); + } +} + +/** Allocate the session and start the responder handshake. + * + * The prologue binds the whole plaintext preamble, so any tampering with the + * negotiation (a stripped feature flag, a changed version) breaks the first + * handshake MAC on either side: + * "NoiseOTAInit" | magic(5) | OK,version | client_features | FEATURE_FLAGS,server_flags + */ +bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) + this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession()); + if (this->noise_ == nullptr) { + ESP_LOGW(TAG, "Session allocation failed"); + this->cleanup_connection_(); + return false; + } + + static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version + static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1; + static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags + uint8_t prologue[OTA_NOISE_PROLOGUE_INIT_LEN + sizeof(MAGIC_BYTES) + PROLOGUE_ACK_LEN + PROLOGUE_CLIENT_FEATURES_LEN + + PROLOGUE_FEATURE_ACK_LEN]; +#ifdef USE_ESP8266 + memcpy_P(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); +#else + std::memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); +#endif + uint8_t *p = prologue + OTA_NOISE_PROLOGUE_INIT_LEN; + // Magic bytes, already validated in MAGIC_READ + std::memcpy(p, MAGIC_BYTES, sizeof(MAGIC_BYTES)); + p += sizeof(MAGIC_BYTES); + // Our magic ack + *p++ = ota::OTA_RESPONSE_OK; + *p++ = USE_OTA_VERSION; + // The feature byte the client sent + *p++ = this->ota_features_; + // The feature ack we sent (noise requires the extended protocol) + *p++ = ota::OTA_RESPONSE_FEATURE_FLAGS; + *p++ = server_feature_flags; + + int err = this->noise_->handshake.init(this->noise_ctx_.get_psk(), prologue, sizeof(prologue)); + if (err != 0) { + ESP_LOGW(TAG, "Handshake init: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + this->cleanup_connection_(); + return false; + } + return true; +} + +/** Drive the non-blocking handshake from loop(); returns true once the + * transport ciphers are ready. A would-block returns false and the next + * loop() resumes from the NoiseSession cursors; on failure the connection + * is cleaned up. + */ +bool ESPHomeOTAComponent::handle_noise_handshake_() { + NoiseSession &s = *this->noise_; + while (true) { + if (s.writing) { + if (!this->noise_try_write_frame_()) { + return false; // would block, or errored and cleaned up + } + s.writing = false; + s.frame_pos = 0; + s.frame_len = 0; + } + switch (s.handshake.action()) { + case noise::NoiseResponderHandshake::Action::ACTION_READ: { + if (!this->noise_try_read_frame_()) { + return false; + } + const uint16_t payload_len = s.frame_len - noise::FRAME_HEADER_SIZE; + s.frame_pos = 0; + s.frame_len = 0; + if (s.frame_buf[noise::FRAME_HEADER_SIZE] != noise::HANDSHAKE_STATUS_OK) { + ESP_LOGW(TAG, "Bad handshake error byte: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); + this->cleanup_connection_(); + return false; + } + int err = s.handshake.read_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, payload_len - 1); + if (err != 0) { + ESP_LOGW(TAG, "Handshake read: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + this->noise_send_reject_(noise::reject_reason_for(err)); + this->cleanup_connection_(); + return false; + } + break; + } + case noise::NoiseResponderHandshake::Action::ACTION_WRITE: { + size_t msg_len = 0; + int err = + s.handshake.write_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, noise::MAX_HANDSHAKE_SIZE, msg_len); + if (err != 0) { + ESP_LOGW(TAG, "Handshake write: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + this->cleanup_connection_(); + return false; + } + const uint16_t payload_len = msg_len + 1; + noise::write_frame_header(s.frame_buf, payload_len); + s.frame_buf[noise::FRAME_HEADER_SIZE] = noise::HANDSHAKE_STATUS_OK; + s.frame_len = noise::FRAME_HEADER_SIZE + payload_len; + s.frame_pos = 0; + s.writing = true; + break; + } + case noise::NoiseResponderHandshake::Action::ACTION_SPLIT: { + int err = s.handshake.split(s.send_cipher, s.recv_cipher); + if (err != 0) { + ESP_LOGW(TAG, "Handshake split: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + this->cleanup_connection_(); + return false; + } + ESP_LOGD(TAG, "Noise handshake complete"); + return true; + } + default: { + ESP_LOGW(TAG, "Bad handshake state"); + this->cleanup_connection_(); + return false; + } + } + } +} + +/// Non-blocking read of one handshake frame into the session buffer. +bool ESPHomeOTAComponent::noise_try_read_frame_() { + NoiseSession &s = *this->noise_; + while (s.frame_pos < noise::FRAME_HEADER_SIZE) { + ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, noise::FRAME_HEADER_SIZE - s.frame_pos); + if (!this->handle_read_error_(read, LOG_STR("read noise header"))) { + return false; + } + s.frame_pos += read; + } + if (s.frame_len == 0) { + const uint16_t payload_len = encode_uint16(s.frame_buf[1], s.frame_buf[2]); + if (s.frame_buf[0] != noise::FRAME_INDICATOR || payload_len < 1 || payload_len > 1 + noise::MAX_HANDSHAKE_SIZE) { + ESP_LOGW(TAG, "Bad handshake frame: 0x%02X, %u bytes", s.frame_buf[0], payload_len); + this->cleanup_connection_(); + return false; + } + s.frame_len = noise::FRAME_HEADER_SIZE + payload_len; + } + while (s.frame_pos < s.frame_len) { + ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, s.frame_len - s.frame_pos); + if (!this->handle_read_error_(read, LOG_STR("read noise frame"))) { + return false; + } + s.frame_pos += read; + } + return true; +} + +/// Non-blocking write of the pending session-buffer frame. +bool ESPHomeOTAComponent::noise_try_write_frame_() { + NoiseSession &s = *this->noise_; + while (s.frame_pos < s.frame_len) { + ssize_t written = this->client_->write(s.frame_buf + s.frame_pos, s.frame_len - s.frame_pos); + if (!this->handle_write_error_(written, LOG_STR("write noise frame"))) { + return false; + } + s.frame_pos += written; + } + return true; +} + +/// Best-effort explicit reject frame so the client can log a readable reason. +void ESPHomeOTAComponent::noise_send_reject_(const LogString *reason) { + // Every reason here comes from noise::reject_reason_for(), so the exported + // floor is the exact capacity needed + uint8_t data[noise::FRAME_HEADER_SIZE + noise::MAC_FAILURE_PAYLOAD_SIZE]; + const size_t payload_len = + noise::format_reject_payload(data + noise::FRAME_HEADER_SIZE, sizeof(data) - noise::FRAME_HEADER_SIZE, reason); + noise::write_frame_header(data, payload_len); + this->client_->write(data, noise::FRAME_HEADER_SIZE + payload_len); // Best effort, non-blocking +} + +/// Decrypt a ciphertext in place; returns the plaintext size or -1. +ssize_t ESPHomeOTAComponent::noise_decrypt_(uint8_t *buf, size_t len) { + NoiseBuffer mbuf; + noise_buffer_init(mbuf); + noise_buffer_set_inout(mbuf, buf, len, len); + int err = noise_cipherstate_decrypt(this->noise_->recv_cipher, &mbuf); + if (err != 0) { + ESP_LOGW(TAG, "Decrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + return -1; + } + return mbuf.size; +} + +/** Blocking read of one frame whose ciphertext size must be within the given + * bounds, decrypted in place; returns the plaintext size, or -1 on error. + * buf needs max_ciphertext capacity. + */ +ssize_t ESPHomeOTAComponent::noise_read_frame_blocking_(uint8_t *buf, size_t min_ciphertext, size_t max_ciphertext) { + uint8_t header[noise::FRAME_HEADER_SIZE]; + if (!this->readall_(header, sizeof(header))) { + return -1; + } + const size_t ciphertext_len = encode_uint16(header[1], header[2]); + if (header[0] != noise::FRAME_INDICATOR || ciphertext_len < min_ciphertext || ciphertext_len > max_ciphertext) { + ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], ciphertext_len); + return -1; + } + if (!this->readall_(buf, ciphertext_len)) { + return -1; + } + return this->noise_decrypt_(buf, ciphertext_len); +} + +/** Blocking read of one frame whose plaintext must be exactly len bytes + * (control units are one unit per frame). buf needs len + noise::MAC_SIZE + * capacity; the plaintext lands at buf[0..len). + */ +bool ESPHomeOTAComponent::noise_readall_(uint8_t *buf, size_t len) { + return this->noise_read_frame_blocking_(buf, len + noise::MAC_SIZE, len + noise::MAC_SIZE) == (ssize_t) len; +} + +/** Blocking read of one data-phase frame, decrypted in place; returns the + * plaintext size, or -1 on error. buf is the OTA_BUFFER_SIZE data buffer. + * The ciphertext must fit that buffer and its plaintext must fit what the + * caller accepts (the remaining image bytes). + */ +ssize_t ESPHomeOTAComponent::noise_read_data_(uint8_t *buf, size_t capacity) { + const size_t max_ciphertext = std::min(capacity + noise::MAC_SIZE, OTA_BUFFER_SIZE); + return this->noise_read_frame_blocking_(buf, noise::MAC_SIZE + 1, max_ciphertext); +} + +/// Blocking write of one response byte as an encrypted frame. +bool ESPHomeOTAComponent::noise_write_byte_(uint8_t byte) { + uint8_t frame[noise::FRAME_HEADER_SIZE + 1 + noise::MAC_SIZE]; + frame[noise::FRAME_HEADER_SIZE] = byte; + NoiseBuffer mbuf; + noise_buffer_init(mbuf); + noise_buffer_set_inout(mbuf, frame + noise::FRAME_HEADER_SIZE, 1, 1 + noise::MAC_SIZE); + int err = noise_cipherstate_encrypt(this->noise_->send_cipher, &mbuf); + if (err != 0) { + ESP_LOGW(TAG, "Encrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + return false; + } + noise::write_frame_header(frame, mbuf.size); + return this->writeall_(frame, noise::FRAME_HEADER_SIZE + mbuf.size); +} + +} // namespace esphome +#endif // USE_OTA_ENCRYPTION +#endif // USE_OTA diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 3a8e2609ef..0f9328a482 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -45,6 +45,15 @@ def decode_encryption_key(value: str) -> bytes: return decoded +def is_reserved_key(value: str) -> bool: + """Whether the key is the reserved all-zeros provisioning sentinel. + + The device treats it as no key configured, so consumers that require a + real key must reject it. + """ + return not any(decode_encryption_key(value)) + + ENCRYPTION_SCHEMA = cv.Schema( { cv.Optional(CONF_KEY): cv.sensitive(validate_encryption_key), diff --git a/esphome/components/ota/ota_backend.h b/esphome/components/ota/ota_backend.h index 1c24fc320a..7348a0ce90 100644 --- a/esphome/components/ota/ota_backend.h +++ b/esphome/components/ota/ota_backend.h @@ -49,6 +49,7 @@ enum OTAResponseTypes { OTA_RESPONSE_ERROR_BOOTLOADER_VERIFY = 0x91, OTA_RESPONSE_ERROR_BOOTLOADER_UPDATE = 0x92, OTA_RESPONSE_ERROR_VERSION_DOWNGRADE = 0x93, + OTA_RESPONSE_ERROR_ENCRYPTION_REQUIRED = 0x94, OTA_RESPONSE_ERROR_UNKNOWN = 0xFF, }; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 1f5a10d47d..7af41409fd 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -242,6 +242,7 @@ #define USE_RUNTIME_IMAGE_QOI #define USE_RUNTIME_STATS #define USE_OTA +#define USE_OTA_ENCRYPTION #define USE_OTA_PASSWORD #define USE_OTA_VERSION 2 #define USE_TIME_TIMEZONE diff --git a/esphome/espota2.py b/esphome/espota2.py index ca833f1816..ac4cbeeb7c 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -53,6 +53,7 @@ RESPONSE_ERROR_PARTITION_TABLE_UPDATE = 0x90 RESPONSE_ERROR_BOOTLOADER_VERIFY = 0x91 RESPONSE_ERROR_BOOTLOADER_UPDATE = 0x92 RESPONSE_ERROR_VERSION_DOWNGRADE = 0x93 +RESPONSE_ERROR_ENCRYPTION_REQUIRED = 0x94 RESPONSE_ERROR_UNKNOWN = 0xFF OTA_VERSION_1_0 = 1 @@ -63,8 +64,20 @@ MAGIC_BYTES = [0x6C, 0x26, 0xF7, 0x5C, 0x45] CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01 CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02 CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04 +CLIENT_FEATURE_SUPPORTS_NOISE = 0x08 SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01 SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02 +SERVER_FEATURE_SUPPORTS_NOISE = 0x04 + +NOISE_FRAME_INDICATOR = 0x01 +NOISE_HANDSHAKE_OK = 0x00 +# The device decrypts frames in its transfer buffer (OTA_BUFFER_SIZE, sized +# as this plus the 16-byte ChaCha20-Poly1305 MAC). 1024 divides the 8192-byte +# upload block exactly, so blocks tile into full frames with no runt. +NOISE_MAX_PLAINTEXT = 1024 +# Wire contract: the device sends exactly this reject reason for a bad MAC +NOISE_MAC_FAILURE_REASON = "Handshake MAC failure" +NOISE_PROLOGUE_INIT = b"NoiseOTAInit" # OTA types this client knows how to send. Future PRs that add bootloader/partition # updates extend this set. Anything outside the set is rejected up front so callers @@ -171,6 +184,12 @@ _ERROR_MESSAGES: dict[int, str] = { "enabled: the new firmware's version must be newer than the version the " "device is currently running." ), + RESPONSE_ERROR_ENCRYPTION_REQUIRED: ( + "The device requires an encrypted OTA connection but this upload has no " + "encryption key. Add 'encryption:' to the 'ota: platform: esphome' section " + "of the YAML this upload uses, or update your esphome installation if it " + "predates OTA encryption." + ), RESPONSE_ERROR_UNKNOWN: "Unknown error from ESP", } @@ -305,16 +324,149 @@ def send_check( raise OTANetworkError(f"sending {msg}: {err}") from err +class NoiseSocketWrapper: + """Runs the OTA session inside a Noise (ChaCha20-Poly1305) transport. + + Exposes the socket subset perform_ota uses. Frames are indicator 0x01, + 16-bit big-endian length, ciphertext; recv() drains one decrypted frame + at a time, sendall() keeps control units in one frame and splits data + at NOISE_MAX_PLAINTEXT. + """ + + def __init__(self, sock: socket.socket, psk: str, prologue: bytes) -> None: + # Deliberately lazy: the noise stack (noiseprotocol, cryptography) is + # only imported when an encrypted upload actually runs. + try: + from aioesphomeapi.noise import NoiseHandshake + except ImportError as err: + raise OTAError( + "OTA encryption requires a newer aioesphomeapi; update your " + "esphome installation (pip install -U esphome) and retry" + ) from err + # The aioesphomeapi import above already loaded cryptography; bind + # the exception once so recv() pays no per-frame import lookup + from cryptography.exceptions import InvalidTag + + self._invalid_tag = InvalidTag + self._sock = sock + try: + self._handshake = NoiseHandshake(psk, prologue) + except ValueError as err: + raise OTAError(f"Invalid OTA encryption key: {err}") from err + self._encrypt = None + self._decrypt = None + self._buffer = b"" + + # Only harmless socket controls pass through; byte-moving methods are + # deliberately absent so plaintext cannot leak past the transport. + def settimeout(self, timeout: float | None) -> None: + self._sock.settimeout(timeout) + + def setsockopt(self, level: int, optname: int, value: int) -> None: + self._sock.setsockopt(level, optname, value) + + def close(self) -> None: + self._sock.close() + + def do_handshake(self) -> None: + """Run the two-message NNpsk0 handshake and set up the transport ciphers.""" + try: + self._send_frame( + bytes([NOISE_HANDSHAKE_OK]) + self._handshake.write_message() + ) + payload = self._recv_frame() + except OSError as err: + raise OTANetworkError(f"noise handshake: {err}") from err + if not payload: + raise OTANetworkError("Device closed connection during the noise handshake") + if payload[0] != NOISE_HANDSHAKE_OK: + reason = payload[1:].decode("utf-8", "replace") + if reason == NOISE_MAC_FAILURE_REASON: + raise OTAError( + "Device rejected the handshake; is the OTA encryption key correct?" + ) + raise OTAError(f"Device rejected the noise handshake: {reason}") + try: + self._handshake.read_message(payload[1:]) + except (ValueError, self._invalid_tag) as err: + # InvalidTag is a wrong key; ValueError covers a device sending an + # invalid curve point, which cryptography rejects during the DH + raise OTAError( + "Noise handshake failed; is the OTA encryption key correct?" + ) from err + self._encrypt, self._decrypt = self._handshake.get_ciphers() + + def sendall(self, data: bytes) -> None: + frames: list[bytes] = [] + for offset in range(0, len(data), NOISE_MAX_PLAINTEXT): + ciphertext = self._encrypt.encrypt( + data[offset : offset + NOISE_MAX_PLAINTEXT] + ) + frames.append(self._frame_header(len(ciphertext))) + frames.append(ciphertext) + self._sock.sendall(b"".join(frames)) + + def recv(self, amount: int) -> bytes: + if not self._buffer: + ciphertext = self._recv_frame() + if not ciphertext: + return b"" # connection closed at a frame boundary + try: + self._buffer = self._decrypt.decrypt(ciphertext) + except self._invalid_tag as err: + # Retryable: a fresh connection renegotiates the session + raise OTANetworkError( + "Noise decryption failed (MAC mismatch); frame corrupted or tampered" + ) from err + if not self._buffer: + # Reject MAC-only frames so b"" always means the peer closed + raise OTANetworkError("Device sent an empty noise frame") + data = self._buffer[:amount] + self._buffer = self._buffer[amount:] + return data + + @staticmethod + def _frame_header(length: int) -> bytes: + return bytes([NOISE_FRAME_INDICATOR, (length >> 8) & 0xFF, length & 0xFF]) + + def _send_frame(self, payload: bytes) -> None: + self._sock.sendall(self._frame_header(len(payload)) + payload) + + def _recv_frame(self) -> bytes: + header = self._recv_exact(3, closed_ok=True) + if not header: + return b"" # connection closed at a frame boundary + # A malformed frame is a broken transport, not a device error; + # retryable so a fresh session is tried + if header[0] != NOISE_FRAME_INDICATOR: + raise OTANetworkError(f"Bad noise frame indicator 0x{header[0]:02X}") + length = (header[1] << 8) | header[2] + if length == 0: + raise OTANetworkError("Device sent an empty noise frame") + return self._recv_exact(length) + + def _recv_exact(self, amount: int, closed_ok: bool = False) -> bytes: + data = b"" + while len(data) < amount: + chunk = self._sock.recv(amount - len(data)) + if not chunk: + if closed_ok and not data: + return b"" + raise OSError("connection closed inside a noise frame") + data += chunk + return data + + def perform_ota( sock: socket.socket, password: str | None, file_handle: io.IOBase, filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, + noise_psk: str | None = None, ) -> None: - # Validate ota_type up front. It travels as a single byte on the wire, and - # passing an out-of-range value would only surface as a ValueError from - # bytes([ota_type]) deep inside send_check, bypassing OTAError handling. + # Validate up front; an out-of-range value would only surface as a + # ValueError deep inside send_check, bypassing OTAError handling if not isinstance(ota_type, int) or not 0 <= ota_type <= 0xFF: raise OTAError( f"Invalid ota_type {ota_type!r}; expected an integer in range 0-255" @@ -325,6 +477,11 @@ def perform_ota( f"Unsupported OTA type 0x{ota_type:02X}; this ESPHome supports: {supported}" ) + if noise_psk is not None and not noise_psk: + raise OTAError( + "An empty OTA encryption key was provided; refusing to upload in plaintext" + ) + file_contents = file_handle.read() file_size = len(file_contents) _LOGGER.info("Uploading %s (%s bytes)", filename, file_size) @@ -347,6 +504,8 @@ def perform_ota( | CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL ) + if noise_psk: + features_to_send |= CLIENT_FEATURE_SUPPORTS_NOISE send_check(sock, features_to_send, "features") features = receive_exactly( sock, @@ -369,6 +528,31 @@ def perform_ota( else: features = 0 + if noise_psk: + # Fail closed: never fall back to a plaintext upload when an + # encryption key is configured, an active attacker could otherwise + # strip the feature flag and capture the image (it contains the wifi + # credentials and the api encryption key). + if not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + raise OTAError( + "An OTA encryption key is configured but the device did not " + "offer encryption; refusing to send the image in plaintext. " + "If the running firmware predates OTA encryption, first update " + "it without the 'ota: encryption:' block (over a trusted " + "network or via USB), then restore the block and upload again." + ) + # The prologue binds every negotiation byte both sides saw, so any + # tampering with the plaintext preamble breaks the handshake. + prologue = ( + NOISE_PROLOGUE_INIT + + bytes(MAGIC_BYTES) + + bytes([RESPONSE_OK, version, features_to_send]) + + bytes([RESPONSE_FEATURE_FLAGS, features]) + ) + sock = NoiseSocketWrapper(sock, noise_psk, prologue) + sock.do_handshake() + _LOGGER.info("Encrypted connection established") + if ota_type != OTA_TYPE_UPDATE_APP: # Any non-app OTA type requires the extended protocol and the # partition-access server feature. Reject up front so the user gets @@ -572,6 +756,7 @@ def run_ota_impl_( password: str | None, filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, + noise_psk: str | None = None, ) -> tuple[int, str | None]: from esphome.core import CORE @@ -636,7 +821,7 @@ def run_ota_impl_( reached_device = True with contextlib.closing(sock), Path(filename).open("rb") as file_handle: try: - perform_ota(sock, password, file_handle, filename, ota_type) + perform_ota(sock, password, file_handle, filename, ota_type, noise_psk) except OTANetworkError as err: # Transient network failure; retry last_error = str(err) @@ -661,9 +846,12 @@ def run_ota( password: str | None, filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, + noise_psk: str | None = None, ) -> tuple[int, str | None]: try: - return run_ota_impl_(remote_host, remote_port, password, filename, ota_type) + return run_ota_impl_( + remote_host, remote_port, password, filename, ota_type, noise_psk + ) except OTAError as err: _LOGGER.error(err) return 1, None diff --git a/tests/component_tests/noise/test_encryption_key.py b/tests/component_tests/noise/test_encryption_key.py index 62abae6487..10f1eb3d4c 100644 --- a/tests/component_tests/noise/test_encryption_key.py +++ b/tests/component_tests/noise/test_encryption_key.py @@ -5,7 +5,11 @@ from __future__ import annotations import pytest from esphome import config_validation as cv -from esphome.components.noise import decode_encryption_key, validate_encryption_key +from esphome.components.noise import ( + decode_encryption_key, + is_reserved_key, + validate_encryption_key, +) KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @@ -35,3 +39,8 @@ def test_decode_encryption_key_rejects_short_decode() -> None: a zero padded PSK on the device.""" with pytest.raises(cv.Invalid, match="32 bytes"): decode_encryption_key("AAECAw==") + + +def test_is_reserved_key() -> None: + assert is_reserved_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") + assert not is_reserved_key(KEY) diff --git a/tests/component_tests/ota/test_esphome_ota.py b/tests/component_tests/ota/test_esphome_ota.py index cdac430ff7..873f162555 100644 --- a/tests/component_tests/ota/test_esphome_ota.py +++ b/tests/component_tests/ota/test_esphome_ota.py @@ -8,17 +8,25 @@ from typing import Any import pytest from esphome import config_validation as cv -from esphome.components.esphome.ota import ota_esphome_final_validate +from esphome.components.esphome.ota import ( + AUTO_LOAD, + FILTER_SOURCE_FILES, + _validate_no_password_with_encryption, + ota_esphome_final_validate, +) from esphome.const import ( + CONF_API, + CONF_ENCRYPTION, CONF_ESPHOME, CONF_ID, + CONF_KEY, CONF_OTA, CONF_PASSWORD, CONF_PLATFORM, CONF_PORT, CONF_VERSION, ) -from esphome.core import ID +from esphome.core import CORE, ID import esphome.final_validate as fv @@ -103,3 +111,305 @@ def test_non_esphome_ota_unaffected() -> None: assert len(updated[CONF_OTA]) == 3 finally: fv.full_config.reset(token) + + +API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +OTHER_KEY = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" +ZEROS_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + + +def test_encryption_key_inherited_from_api() -> None: + """A bare encryption block resolves to the api encryption key.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: API_KEY}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], + } + token = fv.full_config.set(full_conf) + try: + ota_esphome_final_validate({}) + updated = fv.full_config.get() + assert updated[CONF_OTA][0][CONF_ENCRYPTION][CONF_KEY] == API_KEY + finally: + fv.full_config.reset(token) + + +def test_encryption_explicit_key_matching_api_accepted() -> None: + """An explicit ota key equal to the api key validates.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: API_KEY}}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + ota_esphome_final_validate({}) + updated = fv.full_config.get() + assert updated[CONF_OTA][0][CONF_ENCRYPTION][CONF_KEY] == API_KEY + finally: + fv.full_config.reset(token) + + +def test_encryption_key_differing_from_api_rejected() -> None: + """There is one key per device; an ota key differing from the api key raises.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: API_KEY}}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="must match the 'api' encryption key"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_encryption_explicit_key_without_api_encryption_accepted() -> None: + """An explicit ota key with a plaintext api has nothing to match; it stands.""" + full_conf = { + CONF_API: {}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + ota_esphome_final_validate({}) + updated = fv.full_config.get() + assert updated[CONF_OTA][0][CONF_ENCRYPTION][CONF_KEY] == OTHER_KEY + finally: + fv.full_config.reset(token) + + +def test_encryption_without_any_key_rejected() -> None: + """A bare encryption block with no api key to inherit raises.""" + full_conf = { + CONF_API: {}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="no 'api' encryption key to inherit"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_encryption_explicit_all_zeros_key_rejected() -> None: + """The all-zeros key is the provisioning sentinel; the device would treat + it as no PSK and accept plaintext, so it must fail validation.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_encryption_inherited_all_zeros_key_rejected() -> None: + """An all-zeros api key must not silently disable ota encryption either.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_encryption_key_mismatch_between_merged_configs_rejected() -> None: + """Same-port configs with different encryption keys raise.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}), + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + ] + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="encryption is inconsistent"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) + + +@pytest.mark.parametrize("keyed_first", [True, False]) +def test_encryption_bare_and_keyed_blocks_merge(keyed_first: bool) -> None: + """A bare encryption block (package/device split) is compatible with a + keyed one on the same port; the merge resolves to the keyed result.""" + keyed = _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + bare = _make_ota_config(port=3232, **{CONF_ENCRYPTION: {}}) + full_conf = { + CONF_OTA: [keyed, bare] if keyed_first else [bare, keyed], + } + token = fv.full_config.set(full_conf) + try: + ota_esphome_final_validate({}) + updated = fv.full_config.get() + assert len(updated[CONF_OTA]) == 1 + assert updated[CONF_OTA][0][CONF_ENCRYPTION][CONF_KEY] == OTHER_KEY + finally: + fv.full_config.reset(token) + + +def test_encryption_runtime_provisioned_api_key_not_inheritable() -> None: + """A keyless api encryption block provisions its key at runtime; a bare + ota encryption block cannot inherit it and the message says so.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="provisioned at runtime"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_encryption_explicit_key_with_runtime_provisioned_api_accepted() -> None: + """The documented remedy for a runtime-provisioned api key: set an + explicit ota key.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {}}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + ota_esphome_final_validate({}) + updated = fv.full_config.get() + assert updated[CONF_OTA][0][CONF_ENCRYPTION][CONF_KEY] == OTHER_KEY + finally: + fv.full_config.reset(token) + + +def test_encryption_with_web_server_ota_warns( + caplog: pytest.LogCaptureFixture, +) -> None: + """With the web_server component the plaintext /update endpoint is always + on; the combination validates with a warning.""" + full_conf = { + "web_server": {}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert any("plaintext /update" in record.message for record in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_encryption_with_captive_portal_web_server_ota_warns( + caplog: pytest.LogCaptureFixture, +) -> None: + """captive_portal auto-loads the web_server ota platform without the + web_server component; encryption stays usable and only warns, so the + fallback AP recovery path is not lost.""" + full_conf = { + "captive_portal": {}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert any("captive_portal" in record.message for record in caplog.records) + esphome_conf = next( + conf + for conf in fv.full_config.get()[CONF_OTA] + if conf.get(CONF_PLATFORM) == CONF_ESPHOME + ) + assert esphome_conf[CONF_ENCRYPTION][CONF_KEY] == OTHER_KEY + finally: + fv.full_config.reset(token) + + +def test_web_server_ota_without_encryption_unaffected() -> None: + """web_server ota stays valid alongside an unencrypted esphome entry.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + ota_esphome_final_validate({}) + assert len(fv.full_config.get()[CONF_OTA]) == 2 + finally: + fv.full_config.reset(token) + + +def test_auto_load_pulls_noise_only_for_encryption() -> None: + """A plain ota entry must never pull noise-c into the build.""" + assert AUTO_LOAD({CONF_PORT: 3232}) == ["sha256", "socket"] + assert "noise" in AUTO_LOAD({CONF_ENCRYPTION: {}}) + # Tooling probes must get the maximal set: None from dependency + # resolution, {} from the components-graph platform probe + assert "noise" in AUTO_LOAD(None) + assert "noise" in AUTO_LOAD({}) + + +def test_filter_source_files_excludes_noise_without_encryption() -> None: + """The noise transport source compiles only for encrypted builds.""" + old_config = CORE.config + try: + CORE.config = {CONF_OTA: [_make_ota_config(port=3232)]} + assert FILTER_SOURCE_FILES() == ["ota_esphome_noise.cpp"] + CORE.config = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) + ] + } + assert FILTER_SOURCE_FILES() == [] + finally: + CORE.config = old_config + + +def test_password_with_encryption_rejected() -> None: + """The password and encryption options are mutually exclusive.""" + config = {CONF_PASSWORD: "pw", CONF_ENCRYPTION: {CONF_KEY: API_KEY}} + with pytest.raises(cv.Invalid, match="cannot be combined"): + _validate_no_password_with_encryption(config) + + +def test_password_alone_accepted() -> None: + """A password without encryption still validates.""" + config = {CONF_PASSWORD: "pw"} + assert _validate_no_password_with_encryption(config) is config + + +def test_merged_password_and_encryption_rejected() -> None: + """A password block and an encryption block merged on one port raise.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_PASSWORD: "pw"}), + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}), + ] + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="cannot be combined"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) diff --git a/tests/components/ota/encryption.yaml b/tests/components/ota/encryption.yaml new file mode 100644 index 0000000000..550d35caec --- /dev/null +++ b/tests/components/ota/encryption.yaml @@ -0,0 +1,9 @@ +wifi: + ssid: MySSID + password: password1 + +ota: + - platform: esphome + port: 3288 + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" diff --git a/tests/components/ota/encryption_inherit.yaml b/tests/components/ota/encryption_inherit.yaml new file mode 100644 index 0000000000..15ada6f810 --- /dev/null +++ b/tests/components/ota/encryption_inherit.yaml @@ -0,0 +1,12 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + port: 3289 + encryption: diff --git a/tests/components/ota/test-encryption.esp32-idf.yaml b/tests/components/ota/test-encryption.esp32-idf.yaml new file mode 100644 index 0000000000..000e38168e --- /dev/null +++ b/tests/components/ota/test-encryption.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include encryption.yaml diff --git a/tests/components/ota/test-encryption.esp8266-ard.yaml b/tests/components/ota/test-encryption.esp8266-ard.yaml new file mode 100644 index 0000000000..000e38168e --- /dev/null +++ b/tests/components/ota/test-encryption.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include encryption.yaml diff --git a/tests/components/ota/test-encryption.rp2040-ard.yaml b/tests/components/ota/test-encryption.rp2040-ard.yaml new file mode 100644 index 0000000000..000e38168e --- /dev/null +++ b/tests/components/ota/test-encryption.rp2040-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include encryption.yaml diff --git a/tests/components/ota/test-encryption_inherit.esp8266-ard.yaml b/tests/components/ota/test-encryption_inherit.esp8266-ard.yaml new file mode 100644 index 0000000000..71aa083e7e --- /dev/null +++ b/tests/components/ota/test-encryption_inherit.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include encryption_inherit.yaml diff --git a/tests/integration/fixtures/host_ota_encrypted.yaml b/tests/integration/fixtures/host_ota_encrypted.yaml new file mode 100644 index 0000000000..0d11c99d3d --- /dev/null +++ b/tests/integration/fixtures/host_ota_encrypted.yaml @@ -0,0 +1,11 @@ +esphome: + name: host-ota-test +host: +api: +ota: + - platform: esphome + port: __OTA_PORT__ + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +logger: + level: DEBUG diff --git a/tests/integration/test_host_ota.py b/tests/integration/test_host_ota.py index e1036fdf1c..4e74814534 100644 --- a/tests/integration/test_host_ota.py +++ b/tests/integration/test_host_ota.py @@ -10,6 +10,7 @@ from __future__ import annotations import asyncio from collections.abc import Generator from contextlib import contextmanager +import functools import socket import pytest @@ -111,6 +112,62 @@ async def test_host_ota_self_update( assert proc.pid == pid_before +@pytest.mark.asyncio +async def test_host_ota_encrypted( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> None: + """Encrypted self-OTA succeeds; a plaintext upload to the same device fails.""" + pytest.importorskip("aioesphomeapi.noise") + noise_psk = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + api_port, api_socket = reserved_tcp_port + with _reserve_port() as (ota_port, ota_socket): + yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + api_socket.close() + ota_socket.close() + + loop = asyncio.get_running_loop() + rebooted = loop.create_future() + + def on_log(line: str) -> None: + if not rebooted.done() and "Rebooting safely" in line: + rebooted.set_result(True) + + async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): + await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) + pid_before = proc.pid + + # A plaintext upload must be refused with the device unharmed + rc, _ = await loop.run_in_executor( + None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path + ) + assert rc == 1, "plaintext upload to an encrypted device must fail" + await asyncio.sleep(0.5) + assert proc.returncode is None, "process died on rejected plaintext OTA" + + # The encrypted upload goes through and the device re-execs + rc, _ = await loop.run_in_executor( + None, + functools.partial( + espota2.run_ota, + LOCALHOST, + ota_port, + None, + binary_path, + noise_psk=noise_psk, + ), + ) + assert rc == 0, "encrypted OTA reported failure" + await asyncio.wait_for(rebooted, timeout=10.0) + await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) + assert proc.returncode is None, "process exited instead of execing" + assert proc.pid == pid_before + + @pytest.mark.asyncio async def test_host_ota_rejects_garbage( yaml_config: str, diff --git a/tests/unit_tests/test_espota2_noise.py b/tests/unit_tests/test_espota2_noise.py new file mode 100644 index 0000000000..5b43d05530 --- /dev/null +++ b/tests/unit_tests/test_espota2_noise.py @@ -0,0 +1,407 @@ +"""Unit tests for encrypted OTA uploads in esphome.espota2. + +A fake device implementing the responder side of the wire protocol (via +noiseprotocol, which esphome already has through aioesphomeapi) serves a real +TCP loopback connection, so these exercise the actual handshake, framing, and +cipher interop of the client code. Tests that need the client-side crypto skip +when the installed aioesphomeapi predates the noise module. +""" + +from __future__ import annotations + +import base64 +import hashlib +import io +from pathlib import Path +import socket +import sys +import threading +from unittest.mock import Mock, patch + +import pytest + +from esphome import espota2 + +PSK = base64.b64encode(bytes(range(32))).decode() +OTHER_PSK = base64.b64encode(bytes(range(1, 33))).decode() + +MAGIC = bytes(espota2.MAGIC_BYTES) + + +def _recv_exact(sock: socket.socket, amount: int) -> bytes: + data = b"" + while len(data) < amount: + chunk = sock.recv(amount - len(data)) + if not chunk: + raise ConnectionError("client closed") + data += chunk + return data + + +def _frame(payload: bytes) -> bytes: + return ( + bytes([espota2.NOISE_FRAME_INDICATOR, len(payload) >> 8, len(payload) & 0xFF]) + + payload + ) + + +def _send_frame(sock: socket.socket, payload: bytes) -> None: + sock.sendall(_frame(payload)) + + +def _recv_frame(sock: socket.socket) -> bytes: + header = _recv_exact(sock, 3) + assert header[0] == 0x01 + return _recv_exact(sock, (header[1] << 8) | header[2]) + + +class FakeEncryptedDevice(threading.Thread): + """Responder side of the encrypted OTA wire protocol.""" + + def __init__( + self, + psk: str = PSK, + version: int = 2, + offer_noise: bool = True, + require_noise: bool = True, + prologue_features_override: int | None = None, + ) -> None: + super().__init__(daemon=True) + self.psk = psk + self.version = version + self.offer_noise = offer_noise + self.require_noise = require_noise + self.prologue_features_override = prologue_features_override + self.received: bytes | None = None + self.error: Exception | None = None + self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen(1) + self.port = self.listener.getsockname()[1] + + def run(self) -> None: + try: + sock, _ = self.listener.accept() + sock.settimeout(10) + with sock: + self._serve(sock) + except Exception as err: # noqa: BLE001 - surfaced via join_and_check + self.error = err + finally: + self.listener.close() + + def join_and_check(self) -> None: + self.join(timeout=10) + assert not self.is_alive(), "fake device did not finish" + if self.error is not None: + raise self.error + + def _serve(self, sock: socket.socket) -> None: + assert _recv_exact(sock, 5) == MAGIC + sock.sendall(bytes([espota2.RESPONSE_OK, self.version])) + features = _recv_exact(sock, 1)[0] + noise_negotiated = bool( + features & espota2.CLIENT_FEATURE_SUPPORTS_NOISE + and features & espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + ) + if self.require_noise and not noise_negotiated: + sock.sendall(bytes([espota2.RESPONSE_ERROR_ENCRYPTION_REQUIRED])) + return + server_flags = espota2.SERVER_FEATURE_SUPPORTS_NOISE if self.offer_noise else 0 + sock.sendall(bytes([espota2.RESPONSE_FEATURE_FLAGS, server_flags])) + if not (self.offer_noise and noise_negotiated): + return # the client fails closed; nothing further arrives + + from cryptography.exceptions import InvalidTag + from noise.connection import NoiseConnection + + prologue_features = ( + features + if self.prologue_features_override is None + else self.prologue_features_override + ) + prologue = ( + espota2.NOISE_PROLOGUE_INIT + + MAGIC + + bytes([espota2.RESPONSE_OK, self.version, prologue_features]) + + bytes([espota2.RESPONSE_FEATURE_FLAGS, server_flags]) + ) + proto = NoiseConnection.from_name(b"Noise_NNpsk0_25519_ChaChaPoly_SHA256") + proto.set_as_responder() + proto.set_psks(base64.b64decode(self.psk)) + proto.set_prologue(prologue) + proto.start_handshake() + + msg1 = _recv_frame(sock) + assert msg1[0] == 0x00 + try: + proto.read_message(msg1[1:]) + except InvalidTag: + _send_frame(sock, b"\x01" + espota2.NOISE_MAC_FAILURE_REASON.encode()) + return + _send_frame(sock, b"\x00" + bytes(proto.write_message())) + + def send_byte(byte: int) -> None: + _send_frame(sock, proto.encrypt(bytes([byte]))) + + def recv_unit(length: int) -> bytes: + plaintext = proto.decrypt(_recv_frame(sock)) + assert len(plaintext) == length, "control units must be one per frame" + return plaintext + + send_byte(espota2.RESPONSE_AUTH_OK) + recv_unit(1) # ota type + size = int.from_bytes(recv_unit(4), "big") + send_byte(espota2.RESPONSE_UPDATE_PREPARE_OK) + md5_hex = recv_unit(32) + send_byte(espota2.RESPONSE_BIN_MD5_OK) + + received = b"" + acked = 0 + while len(received) < size: + plaintext = proto.decrypt(_recv_frame(sock)) + assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT + received += plaintext + if self.version >= espota2.OTA_VERSION_2_0: + while acked + espota2.UPLOAD_BLOCK_SIZE <= len(received) or ( + len(received) == size and acked < size + ): + send_byte(espota2.RESPONSE_CHUNK_OK) + acked += espota2.UPLOAD_BLOCK_SIZE + assert hashlib.md5(received).hexdigest().encode() == md5_hex + send_byte(espota2.RESPONSE_RECEIVE_OK) + send_byte(espota2.RESPONSE_UPDATE_END_OK) + assert recv_unit(1) == bytes([espota2.RESPONSE_OK]) + self.received = received + + +def _upload( + device: FakeEncryptedDevice, firmware: bytes, noise_psk: str | None +) -> None: + device.start() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(10) + sock.connect(("127.0.0.1", device.port)) + try: + espota2.perform_ota( + sock, None, io.BytesIO(firmware), Path("firmware.bin"), noise_psk=noise_psk + ) + finally: + sock.close() + + +def test_encrypted_upload_success() -> None: + """A full encrypted v2 upload spanning several 8192-byte blocks.""" + pytest.importorskip("aioesphomeapi.noise") + firmware = bytes(range(256)) * 80 # 20480 bytes, crosses chunk-ack boundaries + device = FakeEncryptedDevice() + with patch("time.sleep"): + _upload(device, firmware, PSK) + device.join_and_check() + assert device.received == firmware + + +def test_encrypted_upload_version_1() -> None: + """Version 1 protocol (no chunk acks) works through the noise transport.""" + pytest.importorskip("aioesphomeapi.noise") + firmware = b"v1 firmware image" * 100 + device = FakeEncryptedDevice(version=1) + with patch("time.sleep"): + _upload(device, firmware, PSK) + device.join_and_check() + assert device.received == firmware + + +def test_wrong_key_fails_with_clear_error() -> None: + """A key mismatch surfaces the device's handshake reject readably.""" + pytest.importorskip("aioesphomeapi.noise") + device = FakeEncryptedDevice(psk=OTHER_PSK) + with pytest.raises(espota2.OTAError, match="encryption key correct"): + _upload(device, b"firmware", PSK) + device.join_and_check() + + +def test_tampered_negotiation_breaks_handshake() -> None: + """A negotiation byte differing between the sides breaks the prologue MAC.""" + pytest.importorskip("aioesphomeapi.noise") + device = FakeEncryptedDevice( + prologue_features_override=espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + ) + with pytest.raises(espota2.OTAError, match="encryption key correct"): + _upload(device, b"firmware", PSK) + device.join_and_check() + + +def test_client_fails_closed_when_device_lacks_encryption() -> None: + """With a key configured, a device not offering noise aborts the upload.""" + device = FakeEncryptedDevice(offer_noise=False, require_noise=False) + with pytest.raises(espota2.OTAError, match="refusing to send the image"): + _upload(device, b"firmware", PSK) + device.join_and_check() + + +def test_plaintext_client_gets_encryption_required_error() -> None: + """A client without a key gets the device's 0x94 error message.""" + device = FakeEncryptedDevice() + with pytest.raises(espota2.OTAError, match="requires an encrypted OTA"): + _upload(device, b"firmware", None) + device.join_and_check() + + +def test_missing_aioesphomeapi_noise_module_message() -> None: + """An aioesphomeapi without the noise module produces a clear error.""" + with ( + patch.dict(sys.modules, {"aioesphomeapi.noise": None}), + pytest.raises(espota2.OTAError, match="requires a newer aioesphomeapi"), + ): + espota2.NoiseSocketWrapper(Mock(), PSK, b"prologue") + + +class ScriptedSocket: + """Serves scripted recv chunks; b"" means the peer closed.""" + + def __init__(self, *chunks: bytes | Exception) -> None: + self.chunks = list(chunks) + self.sent: list[bytes] = [] + + def sendall(self, data: bytes) -> None: + self.sent.append(data) + + def settimeout(self, timeout: float) -> None: + pass + + def recv(self, amount: int) -> bytes: + if not self.chunks: + return b"" + chunk = self.chunks[0] + if isinstance(chunk, Exception): + self.chunks.pop(0) + raise chunk + take, rest = chunk[:amount], chunk[amount:] + if rest: + self.chunks[0] = rest + else: + self.chunks.pop(0) + return take + + +def _wrapper(*chunks: bytes | Exception) -> espota2.NoiseSocketWrapper: + pytest.importorskip("aioesphomeapi.noise") + return espota2.NoiseSocketWrapper(ScriptedSocket(*chunks), PSK, b"prologue") + + +def test_wrapper_rejects_malformed_psk() -> None: + pytest.importorskip("aioesphomeapi.noise") + with pytest.raises(espota2.OTAError, match="Invalid OTA encryption key"): + espota2.NoiseSocketWrapper(ScriptedSocket(), "not-base64!!!", b"prologue") + + +def test_handshake_socket_error_is_network_error() -> None: + wrapper = _wrapper(OSError("boom")) + with pytest.raises(espota2.OTANetworkError, match="noise handshake"): + wrapper.do_handshake() + + +def test_handshake_closed_at_frame_boundary() -> None: + wrapper = _wrapper() + with pytest.raises(espota2.OTANetworkError, match="closed connection during"): + wrapper.do_handshake() + + +def test_handshake_reject_with_other_reason() -> None: + wrapper = _wrapper(_frame(b"\x01Handshake error")) + with pytest.raises( + espota2.OTAError, match="rejected the noise handshake: Handshake error" + ): + wrapper.do_handshake() + + +def test_handshake_garbage_second_message() -> None: + """A valid-looking point with a garbage MAC fails cleanly.""" + wrapper = _wrapper(_frame(b"\x00" + bytes(range(48)))) + with pytest.raises( + espota2.OTAError, match="handshake failed; is the OTA encryption key" + ): + wrapper.do_handshake() + + +def test_handshake_invalid_curve_point() -> None: + """An all-zero x25519 point is rejected as a clean error, not a crash.""" + wrapper = _wrapper(_frame(b"\x00" + bytes(48))) + with pytest.raises( + espota2.OTAError, match="handshake failed; is the OTA encryption key" + ): + wrapper.do_handshake() + + +def test_recv_closed_at_frame_boundary_returns_empty() -> None: + wrapper = _wrapper() + assert wrapper.recv(1) == b"" + + +def test_recv_corrupt_frame_is_retryable_network_error() -> None: + from cryptography.exceptions import InvalidTag + + wrapper = _wrapper(_frame(b"ciphertext")) + wrapper._decrypt = Mock(decrypt=Mock(side_effect=InvalidTag())) + with pytest.raises(espota2.OTANetworkError, match="decryption failed"): + wrapper.recv(1) + + +def test_wrapper_blocks_unencrypted_socket_methods() -> None: + """Byte-moving socket methods must not bypass the encrypted transport.""" + wrapper = _wrapper() + # The harmless socket controls pass through to the wrapped socket + wrapper._sock = Mock() + wrapper.settimeout(1) + wrapper._sock.settimeout.assert_called_once_with(1) + wrapper.setsockopt(6, 1, 1) + wrapper._sock.setsockopt.assert_called_once_with(6, 1, 1) + wrapper.close() + wrapper._sock.close.assert_called_once_with() + with pytest.raises(AttributeError): + _ = wrapper.send + with pytest.raises(AttributeError): + _ = wrapper.recv_into + + +def test_recv_empty_plaintext_frame_is_protocol_error() -> None: + """A MAC-only frame decrypts to nothing; b'' from recv must mean close.""" + wrapper = _wrapper(_frame(bytes(16))) + wrapper._decrypt = Mock(decrypt=Mock(return_value=b"")) + with pytest.raises(espota2.OTANetworkError, match="empty noise frame"): + wrapper.recv(1) + + +def test_recv_frame_bad_indicator_is_retryable() -> None: + wrapper = _wrapper(b"\x02\x00\x01x") + with pytest.raises(espota2.OTANetworkError, match="Bad noise frame indicator"): + wrapper._recv_frame() + + +def test_recv_frame_zero_length_is_retryable() -> None: + wrapper = _wrapper(bytes([espota2.NOISE_FRAME_INDICATOR, 0, 0])) + with pytest.raises(espota2.OTANetworkError, match="empty noise frame"): + wrapper._recv_frame() + + +def test_perform_ota_blank_key_refuses_plaintext() -> None: + with pytest.raises(espota2.OTAError, match="empty OTA encryption key"): + espota2.perform_ota( + ScriptedSocket(), None, io.BytesIO(b"x"), Path("f.bin"), noise_psk="" + ) + + +def test_recv_exact_closed_mid_frame() -> None: + wrapper = _wrapper(_frame(b"partial")[:5]) + with pytest.raises(OSError, match="closed inside a noise frame"): + wrapper._recv_frame() + + +def test_recv_serves_buffered_plaintext_without_new_frame() -> None: + """A second recv drains the decrypted buffer without reading another frame.""" + wrapper = _wrapper(_frame(b"ciphertext")) + wrapper._decrypt = Mock(decrypt=Mock(return_value=b"AB")) + assert wrapper.recv(1) == b"A" # reads and decrypts one frame + assert wrapper.recv(1) == b"B" # served from the buffer, no new frame + wrapper._decrypt.decrypt.assert_called_once() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 15b1105ed0..5372a7203d 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -86,7 +86,9 @@ from esphome.const import ( CONF_BROKER, CONF_DISABLED, CONF_DISCOVER_IP, + CONF_ENCRYPTION, CONF_ESPHOME, + CONF_KEY, CONF_LEVEL, CONF_LOG, CONF_LOG_TOPIC, @@ -2106,10 +2108,65 @@ def test_upload_program_ota_success( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, "secret", expected_firmware, OTA_TYPE_UPDATE_APP + ["192.168.1.100"], 3232, "secret", expected_firmware, OTA_TYPE_UPDATE_APP, None ) +def test_upload_program_ota_encryption_key( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """The resolved encryption key is passed through to run_ota.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + key = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + config = { + CONF_OTA: [ + { + CONF_PLATFORM: CONF_ESPHOME, + CONF_PORT: 3232, + CONF_ENCRYPTION: {CONF_KEY: key}, + } + ] + } + exit_code, host = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + assert host == "192.168.1.100" + expected_firmware = ( + tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" + ) + mock_run_ota.assert_called_once_with( + ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, key + ) + + +def test_upload_program_ota_encryption_without_key_fails_closed( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """An encryption block with no resolved key must never upload plaintext.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + + config = { + CONF_OTA: [ + { + CONF_PLATFORM: CONF_ESPHOME, + CONF_PORT: 3232, + CONF_ENCRYPTION: {}, + } + ] + } + with pytest.raises(EsphomeError, match="no key was resolved"): + upload_program(config, MockArgs(), ["192.168.1.100"]) + mock_run_ota.assert_not_called() + + def test_upload_program_ota_with_file_arg( mock_run_ota: Mock, mock_get_port_type: Mock, @@ -2137,7 +2194,7 @@ def test_upload_program_ota_with_file_arg( assert exit_code == 0 assert host == "192.168.1.100" mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, Path("custom.bin"), OTA_TYPE_UPDATE_APP + ["192.168.1.100"], 3232, None, Path("custom.bin"), OTA_TYPE_UPDATE_APP, None ) @@ -2192,6 +2249,7 @@ def test_upload_program_ota_partition_table_with_file_arg( None, partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, + None, ) @@ -2253,6 +2311,7 @@ def test_upload_program_ota_partition_table_mqttip( None, partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, + None, ) @@ -2440,6 +2499,7 @@ def test_upload_program_ota_bootloader_with_file_arg( None, bootloader_file, OTA_TYPE_UPDATE_BOOTLOADER, + None, ) @@ -2602,6 +2662,42 @@ def test_has_web_server_logging_respects_log_disabled() -> None: assert has_web_server_logging() is False +def test_upload_program_web_server_warns_when_encryption_configured( + mock_run_web_server_ota: Mock, + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Explicitly picking web_server OTA on an encrypted config warns about + the plaintext upload path.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_web_server_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_OTA: [ + { + CONF_PLATFORM: CONF_ESPHOME, + CONF_PORT: 3232, + CONF_ENCRYPTION: {CONF_KEY: "test_key"}, + }, + {CONF_PLATFORM: CONF_WEB_SERVER}, + ], + CONF_WEB_SERVER: { + CONF_PORT: 80, + CONF_AUTH: {CONF_USERNAME: "admin", CONF_PASSWORD: "pw"}, + }, + } + args = MockArgs(ota_platform=CONF_WEB_SERVER) + with caplog.at_level(logging.WARNING): + exit_code, _ = upload_program(config, args, ["192.168.1.100"]) + + assert exit_code == 0 + assert any("plaintext HTTP" in record.message for record in caplog.records) + mock_run_ota.assert_not_called() + + def test_upload_program_web_server_only_auto_dispatches( mock_run_web_server_ota: Mock, mock_run_ota: Mock, @@ -2892,7 +2988,7 @@ def test_upload_program_ota_with_mqtt_resolution( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP + ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None ) @@ -2942,7 +3038,7 @@ def test_upload_program_ota_with_mqtt_empty_broker( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.50"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP + ["192.168.1.50"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None ) # Verify warning was logged assert "MQTT IP discovery failed" in caplog.text @@ -5114,6 +5210,7 @@ def test_upload_program_ota_static_ip_with_mqttip( None, expected_firmware, OTA_TYPE_UPDATE_APP, + None, ) @@ -5163,6 +5260,7 @@ def test_upload_program_ota_multiple_mqttip_resolves_once( None, expected_firmware, OTA_TYPE_UPDATE_APP, + None, ) @@ -5340,7 +5438,7 @@ def test_upload_program_ota_mqtt_timeout_fallback( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP + ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None ) From 379e077b5f4b83a844536746f1c6d8d66b6e094b Mon Sep 17 00:00:00 2001 From: Jake <106696989+JakeLC15@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:05:56 -0400 Subject: [PATCH 077/147] [ds1603l] New sensor DS1603L V1.0 (#13133) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- CODEOWNERS | 1 + esphome/components/ds1603l/__init__.py | 0 esphome/components/ds1603l/ds1603l.cpp | 68 +++++++++++++++++++ esphome/components/ds1603l/ds1603l.h | 30 ++++++++ esphome/components/ds1603l/sensor.py | 43 ++++++++++++ tests/components/ds1603l/common.yaml | 3 + tests/components/ds1603l/test.esp32-idf.yaml | 7 ++ .../components/ds1603l/test.esp8266-ard.yaml | 7 ++ 8 files changed, 159 insertions(+) create mode 100644 esphome/components/ds1603l/__init__.py create mode 100644 esphome/components/ds1603l/ds1603l.cpp create mode 100644 esphome/components/ds1603l/ds1603l.h create mode 100644 esphome/components/ds1603l/sensor.py create mode 100644 tests/components/ds1603l/common.yaml create mode 100644 tests/components/ds1603l/test.esp32-idf.yaml create mode 100644 tests/components/ds1603l/test.esp8266-ard.yaml diff --git a/CODEOWNERS b/CODEOWNERS index 7bb7f310a3..3429a93aa7 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -149,6 +149,7 @@ esphome/components/display_menu_base/* @numo68 esphome/components/dlms_meter/* @latonita @PolarGoose @SimonFischer04 @Tomer27cz esphome/components/dps310/* @kbx81 esphome/components/ds1307/* @badbadc0ffee +esphome/components/ds1603l/* @JakeLC15 esphome/components/ds2484/* @mrk-its esphome/components/ds248x/* @tomwellnitz esphome/components/dsmr/* @glmnet @PolarGoose diff --git a/esphome/components/ds1603l/__init__.py b/esphome/components/ds1603l/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esphome/components/ds1603l/ds1603l.cpp b/esphome/components/ds1603l/ds1603l.cpp new file mode 100644 index 0000000000..b0b0ef8175 --- /dev/null +++ b/esphome/components/ds1603l/ds1603l.cpp @@ -0,0 +1,68 @@ +#include "ds1603l.h" + +#include + +#include "esphome/core/helpers.h" +#include "esphome/core/log.h" + +namespace esphome::ds1603l { + +static const char *const TAG = "ds1603l.sensor"; + +void DS1603L::loop() { + // Assemble frames one byte at a time so a stream that starts mid-frame can realign + uint8_t byte; + while (this->available() > 0 && this->read_byte(&byte)) { + if (this->rx_count_ == 0 && byte != HEADER_BYTE) { + ESP_LOGV(TAG, "Skipping byte 0x%02X while looking for header", byte); + continue; + } + + this->rx_buffer_[this->rx_count_++] = byte; + if (this->rx_count_ < FRAME_SIZE) { + continue; + } + + if (this->parse_data_()) { + this->rx_count_ = 0; + } else { + // The header byte was part of the payload of a misaligned frame, so realign instead of dropping everything + this->resync_(); + } + } +} + +void DS1603L::dump_config() { LOG_SENSOR("", "DS1603L", this); } + +bool DS1603L::parse_data_() { + uint8_t header = this->rx_buffer_[0]; + uint8_t data_h = this->rx_buffer_[1]; + uint8_t data_l = this->rx_buffer_[2]; + uint8_t checksum = this->rx_buffer_[3]; + + uint8_t computed_checksum = (header + data_h + data_l) & 0xFF; + + ESP_LOGV(TAG, "Data: Header=0x%02X, Data_H=0x%02X, Data_L=0x%02X, Checksum=0x%02X", header, data_h, data_l, checksum); + + if (checksum != computed_checksum) { + ESP_LOGW(TAG, "Checksum mismatch: received 0x%02X, expected 0x%02X", checksum, computed_checksum); + return false; + } + + this->publish_state(encode_uint16(data_h, data_l)); + return true; +} + +void DS1603L::resync_() { + // Drop the byte that was treated as the header, then look for the next candidate header in what is left + size_t start = 1; + while (start < this->rx_count_ && this->rx_buffer_[start] != HEADER_BYTE) { + start++; + } + this->rx_count_ -= start; + if (this->rx_count_ > 0) { + memmove(this->rx_buffer_, this->rx_buffer_ + start, this->rx_count_); + } +} + +} // namespace esphome::ds1603l diff --git a/esphome/components/ds1603l/ds1603l.h b/esphome/components/ds1603l/ds1603l.h new file mode 100644 index 0000000000..9041681f5e --- /dev/null +++ b/esphome/components/ds1603l/ds1603l.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +#include "esphome/components/sensor/sensor.h" +#include "esphome/components/uart/uart.h" +#include "esphome/core/component.h" + +namespace esphome::ds1603l { + +class DS1603L final : public sensor::Sensor, public Component, public uart::UARTDevice { + public: + void loop() override; + void dump_config() override; + + protected: + static constexpr uint8_t HEADER_BYTE = 0xFF; + static constexpr size_t FRAME_SIZE = 4; + + // Validates the checksum of the frame in rx_buffer_ and publishes it. Returns false if the frame is invalid. + bool parse_data_(); + // Drops the first buffered byte and realigns the buffer on the next possible header byte. + void resync_(); + + uint8_t rx_buffer_[FRAME_SIZE]; // Buffer for the frame being assembled + size_t rx_count_{0}; // Number of bytes currently in rx_buffer_ +}; + +} // namespace esphome::ds1603l diff --git a/esphome/components/ds1603l/sensor.py b/esphome/components/ds1603l/sensor.py new file mode 100644 index 0000000000..c4f117c603 --- /dev/null +++ b/esphome/components/ds1603l/sensor.py @@ -0,0 +1,43 @@ +import esphome.codegen as cg +from esphome.components import sensor, uart +import esphome.config_validation as cv +from esphome.const import ( + DEVICE_CLASS_DISTANCE, + STATE_CLASS_MEASUREMENT, + UNIT_MILLIMETER, +) +from esphome.types import ConfigType + +CODEOWNERS = ["@JakeLC15"] +DEPENDENCIES = ["uart"] + +ds1603l_ns = cg.esphome_ns.namespace("ds1603l") +DS1603L = ds1603l_ns.class_("DS1603L", sensor.Sensor, cg.Component, uart.UARTDevice) + + +CONFIG_SCHEMA = ( + sensor.sensor_schema( + DS1603L, + unit_of_measurement=UNIT_MILLIMETER, + accuracy_decimals=0, + device_class=DEVICE_CLASS_DISTANCE, + state_class=STATE_CLASS_MEASUREMENT, + ) + .extend(uart.UART_DEVICE_SCHEMA) + .extend(cv.COMPONENT_SCHEMA) +) + +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "ds1603l", + baud_rate=9600, + require_tx=False, + require_rx=True, + data_bits=8, + stop_bits=1, +) + + +async def to_code(config: ConfigType) -> None: + var = await sensor.new_sensor(config) + await cg.register_component(var, config) + await uart.register_uart_device(var, config) diff --git a/tests/components/ds1603l/common.yaml b/tests/components/ds1603l/common.yaml new file mode 100644 index 0000000000..d47ef1b610 --- /dev/null +++ b/tests/components/ds1603l/common.yaml @@ -0,0 +1,3 @@ +sensor: + - platform: ds1603l + name: ds1603l Distance diff --git a/tests/components/ds1603l/test.esp32-idf.yaml b/tests/components/ds1603l/test.esp32-idf.yaml new file mode 100644 index 0000000000..544827f577 --- /dev/null +++ b/tests/components/ds1603l/test.esp32-idf.yaml @@ -0,0 +1,7 @@ +substitutions: + tx_pin: GPIO1 + rx_pin: GPIO3 + +packages: + uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + ds1603l: !include common.yaml diff --git a/tests/components/ds1603l/test.esp8266-ard.yaml b/tests/components/ds1603l/test.esp8266-ard.yaml new file mode 100644 index 0000000000..878e45899b --- /dev/null +++ b/tests/components/ds1603l/test.esp8266-ard.yaml @@ -0,0 +1,7 @@ +substitutions: + tx_pin: GPIO0 + rx_pin: GPIO2 + +packages: + uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml + ds1603l: !include common.yaml From 8bef5b22e112503be305221b6167ea6a2b4e0780 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:36:04 -0400 Subject: [PATCH 078/147] Bump zeroconf from 0.150.4 to 0.151.2 (#18934) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f19559dca8..594b44432d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ esptool==5.3.1 click==8.3.3 aioesphomeapi==46.3.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi -zeroconf==0.150.4 +zeroconf==0.151.2 puremagic==2.2.0 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import From b8480b8424a3b6248c234ed0f78291813129e144 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:36:49 -0400 Subject: [PATCH 079/147] Bump pylint from 4.0.7 to 4.0.8 (#18935) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index e837953878..b1309ec63b 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,4 +1,4 @@ -pylint==4.0.7 +pylint==4.0.8 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating ruff==0.16.5 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating From 6099ac7b533be3c9ecfd44c9463df86572e7f239 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:17:22 +1200 Subject: [PATCH 080/147] [core] Document C++ conventions in AGENTS.md that reviews keep catching (#18941) --- AGENTS.md | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f006ee6087..e932c50f32 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,6 +44,16 @@ This document provides essential context for AI models interacting with this pro ## 4. Coding Conventions & Style Guide +**Read the developer documentation before writing a component.** https://developers.esphome.io covers the +component lifecycle, the main loop, and the reasoning behind the rules below in far more depth than this +file does, and it is the authority when they disagree. The most useful starting points: + +* https://developers.esphome.io/architecture/components/ - component lifecycle, `setup()`, `loop()`, + setup priorities, and how a component is registered. +* https://developers.esphome.io/architecture/components/advanced/ - choosing between `loop()`, + `set_interval`, `set_timeout` and `defer`; waking the loop from another thread; the RAM cost of each. +* https://developers.esphome.io/contributing/code/ - contribution rules, public API and breaking changes. + * **Formatting:** * **Python:** Uses `ruff` and `flake8` for linting and formatting. Configuration is in `pyproject.toml`. * **C++:** Uses `clang-format` for formatting. Configuration is in `.clang-format`. @@ -142,6 +152,47 @@ This document provides essential context for AI models interacting with this pro * **Indentation:** Use spaces (two per indentation level), not tabs * **Type aliases:** Prefer `using type_t = int;` over `typedef int type_t;` * **Line length:** Wrap lines at no more than 120 characters + * **Timing in `loop()`:** Never call `millis()` in a `loop()` body. The current tick's timestamp is + already cached - use `App.get_loop_component_start_time()` (from `esphome/core/application.h`). + Only reach for `millis()` when you genuinely need sub-tick resolution inside a long operation. + * **The main loop runs every 16 ms.** A rate-limit gate shorter than that does nothing: the check + passes on essentially every pass of the loop, so it costs a comparison and buys nothing. Pick an + interval comfortably coarser than 16 ms, or drop the gate entirely and accept running every loop. + ```cpp + // Bad - a 10ms gate against a 16ms loop never holds anything back + static constexpr uint32_t POLL_INTERVAL_MS = 10; + const uint32_t now = millis(); + if (now - this->last_poll_ < POLL_INTERVAL_MS) + return; + this->last_poll_ = now; + ``` + ```cpp + // Good - an interval that actually rate limits, off the cached timestamp + static constexpr uint32_t POLL_INTERVAL_MS = 100; + const uint32_t now = App.get_loop_component_start_time(); + if (now - this->last_poll_ < POLL_INTERVAL_MS) + return; + this->last_poll_ = now; + ``` + Pick the primitive by cadence: under 250 ms use a gated `loop()`; 500 ms and above use + `set_interval`. Full reasoning, including why `set_interval` costs more below 500 ms: + https://developers.esphome.io/architecture/components/advanced/#quick-rule-of-thumb + * **Don't override a default with the same value:** if a base class method already returns what you + want, do not override it. `Component::get_setup_priority()` returns `setup_priority::DATA`, so a + component that wants `DATA` should simply leave it alone. + ```cpp + // Bad - this is exactly what the base class already does + float get_setup_priority() const override { return setup_priority::DATA; } + ``` + * **Logging string literals:** wrap literals passed as `%s` arguments in `LOG_STR_LITERAL()` so they + can be stored in flash rather than RAM. + ```cpp + // Bad + ESP_LOGV(TAG, "Key %u %s", key, pressed ? "pressed" : "released"); + + // Good + ESP_LOGV(TAG, "Key %u %s", key, pressed ? LOG_STR_LITERAL("pressed") : LOG_STR_LITERAL("released")); + ``` * **Constructor parameters vs setters:** Component properties that are both **required** and **invariant** (never change after construction) should be constructor parameters rather than set via setter methods. This makes the dependency explicit and prevents use of the object in an incompletely-initialized state. @@ -562,6 +613,33 @@ This document provides essential context for AI models interacting with this pro Use `cg.add_define("MAX_SERVICES", count)` to set the size from Python configuration. Like `std::array` but with vector-like API (`push_back()`, `size()`) and no STL reallocation code. + **Listener and child-entity registration lists are the most common case, and the most commonly + missed.** A `register_*()` method called once per child at code generation time has a count that + is known at compile time, so it should never be a `std::vector`. Use `cg.slot_counter()`: it + returns a function that each consumer calls once per slot it will occupy, and after every + `to_code` has run it emits the define with the final count. When nothing registers, no define is + emitted and the storage plus its registration method compile out entirely. + ```python + # hub component's __init__.py + _request_listener_slot = cg.slot_counter("MY_COMPONENT_LISTENER_COUNT") + + + async def register_listener(hub: MockObj, var: MockObj) -> None: + _request_listener_slot() + cg.add(hub.register_listener(var)) + ``` + ```cpp + #ifdef MY_COMPONENT_LISTENER_COUNT + void register_listener(MyComponentListener *listener); + #endif + protected: + #ifdef MY_COMPONENT_LISTENER_COUNT + StaticVector listeners_; + #endif + ``` + Request slots from `to_code`, not from a job that runs after `CoroPriority.FINAL` - a late + request raises rather than silently undercounting. + 3. **Runtime-known sizes:** Use `FixedVector` from `esphome/core/helpers.h` when the size is only known at runtime initialization. ```cpp // Bad - generates STL realloc code (_M_realloc_insert) @@ -599,9 +677,25 @@ This document provides essential context for AI models interacting with this pro ``` Linear search on small datasets (1-16 elements) is often faster than hashing/tree overhead, but this depends on lookup frequency and access patterns. For frequent lookups in hot code paths, the O(1) vs O(n) complexity difference may still matter even for small datasets. `std::vector` with simple structs is usually fine—it's the heavy containers (`map`, `set`, `unordered_map`) that should be avoided for small datasets unless profiling shows otherwise. - 5. **Avoid `std::deque`:** It allocates in 512-byte blocks regardless of element size, guaranteeing at least 512 bytes of RAM usage immediately. This is a major source of crashes on memory-constrained devices. + 5. **Strings set once from configuration:** Use `StringRef` (`esphome/core/string_ref.h`) rather than + `std::string`. Code generation passes a string literal that lives in flash for the life of the + program, so storing a `std::string` copies it onto the heap for nothing. `StringRef` is a + non-owning pointer plus length; it does not copy, and it must only ever refer to storage that + outlives it (a string literal, or a buffer owned elsewhere). + ```cpp + // Bad - heap copy of a literal that is already in flash + void set_keys(std::string keys) { this->keys_ = std::move(keys); } + std::string keys_; + ``` + ```cpp + // Good - no allocation + void set_keys(const char *keys) { this->keys_ = StringRef(keys); } + StringRef keys_; + ``` - 6. **Detection:** Look for these patterns in compiler output: + 6. **Avoid `std::deque`:** It allocates in 512-byte blocks regardless of element size, guaranteeing at least 512 bytes of RAM usage immediately. This is a major source of crashes on memory-constrained devices. + + 7. **Detection:** Look for these patterns in compiler output: - Large code sections with STL symbols (vector, map, set) - `alloc`, `realloc`, `dealloc` in symbol names - `_M_realloc_insert`, `_M_default_append` (vector reallocation) From da16c01351e60d33c4ecc6527b8e78e0907a22ed Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:32:47 +0000 Subject: [PATCH 081/147] [ci] Refresh integration test durations (#18944) --- .../integration_test_durations.json | 281 +++++++++--------- 1 file changed, 141 insertions(+), 140 deletions(-) diff --git a/tests/integration/integration_test_durations.json b/tests/integration/integration_test_durations.json index 9bada5cd36..5a5aac3b22 100644 --- a/tests/integration/integration_test_durations.json +++ b/tests/integration/integration_test_durations.json @@ -1,142 +1,143 @@ { - "tests/integration/test_action_concurrent_reentry.py": 45.23, - "tests/integration/test_addressable_light_transition.py": 74.47, - "tests/integration/test_alarm_control_panel_state_transitions.py": 74.1, - "tests/integration/test_api_action_metadata.py": 62.1, - "tests/integration/test_api_action_responses.py": 71.08, - "tests/integration/test_api_action_timeout.py": 21.64, - "tests/integration/test_api_conditional_memory.py": 13.72, - "tests/integration/test_api_custom_services.py": 24.16, - "tests/integration/test_api_get_time_response_timezone.py": 23.48, - "tests/integration/test_api_homeassistant.py": 37.87, - "tests/integration/test_api_homeassistant_action_no_subscriber.py": 14.38, - "tests/integration/test_api_list_entities_backpressure.py": 26.85, - "tests/integration/test_api_message_size_batching.py": 33.36, - "tests/integration/test_api_reboot_timeout.py": 13.63, - "tests/integration/test_api_string_lambda.py": 25.04, - "tests/integration/test_api_vv_logging.py": 16.6, - "tests/integration/test_api_zero_psk_provisioning.py": 43.14, - "tests/integration/test_areas_and_devices.py": 25.98, - "tests/integration/test_automation_wait_actions.py": 21.91, - "tests/integration/test_automations.py": 42.43, - "tests/integration/test_batch_delay_zero_rapid_transitions.py": 16.65, - "tests/integration/test_binary_sensor_autorepeat_filter.py": 28.67, - "tests/integration/test_binary_sensor_invalidate_state.py": 23.69, - "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 22.99, - "tests/integration/test_build_info.py": 24.96, - "tests/integration/test_camera_mock.py": 14.47, - "tests/integration/test_climate_control_action.py": 31.07, - "tests/integration/test_climate_custom_modes.py": 28.59, - "tests/integration/test_continuation_actions.py": 14.96, - "tests/integration/test_cover_control_action.py": 26.14, - "tests/integration/test_crc8_helper.py": 10.92, - "tests/integration/test_device_id_in_state.py": 64.97, - "tests/integration/test_duplicate_entities.py": 30.81, - "tests/integration/test_entity_icon.py": 32.85, - "tests/integration/test_fan_turn_on_action.py": 24.91, - "tests/integration/test_fnv1_hash_object_id.py": 12.54, - "tests/integration/test_fnv1a_hash.py": 21.8, - "tests/integration/test_gpio_expander_cache.py": 5.2, - "tests/integration/test_host_logger_thread_safety.py": 21.7, - "tests/integration/test_host_mode_basic.py": 13.62, - "tests/integration/test_host_mode_batch_delay.py": 14.56, - "tests/integration/test_host_mode_climate_basic_state.py": 30.95, - "tests/integration/test_host_mode_climate_control.py": 29.06, - "tests/integration/test_host_mode_empty_string_options.py": 27.22, - "tests/integration/test_host_mode_entity_fields.py": 30.95, - "tests/integration/test_host_mode_fan_preset.py": 14.44, - "tests/integration/test_host_mode_many_entities.py": 54.13, - "tests/integration/test_host_mode_many_entities_multiple_connections.py": 32.17, - "tests/integration/test_host_mode_noise_encryption.py": 42.77, - "tests/integration/test_host_mode_reconnect.py": 4.06, - "tests/integration/test_host_mode_sensor.py": 13.47, - "tests/integration/test_host_ota.py": 21.4, - "tests/integration/test_host_preferences.py": 25.43, - "tests/integration/test_host_preferences_suspend_resume.py": 19.2, - "tests/integration/test_improv_serial_uart.py": 31.52, - "tests/integration/test_large_message_batching.py": 15.64, - "tests/integration/test_legacy_area.py": 22.63, - "tests/integration/test_legacy_climate_compat.py": 26.13, - "tests/integration/test_legacy_fan_compat.py": 24.05, - "tests/integration/test_light_automations.py": 30.86, - "tests/integration/test_light_binary_effect_off_phase.py": 23.19, - "tests/integration/test_light_calls.py": 32.35, - "tests/integration/test_light_constant_brightness.py": 29.89, - "tests/integration/test_light_control_action.py": 29.06, - "tests/integration/test_light_dim_relative_action.py": 29.61, - "tests/integration/test_light_effect_zero_brightness.py": 18.68, - "tests/integration/test_light_initial_state.py": 24.49, - "tests/integration/test_light_toggle_action.py": 26.46, - "tests/integration/test_lock_automations.py": 23.28, - "tests/integration/test_logger_buffered_recursion_guard.py": 24.29, - "tests/integration/test_loop_disable_enable.py": 45.28, - "tests/integration/test_loop_interval_decoupling.py": 28.35, - "tests/integration/test_loop_interval_default_not_pulled_forward.py": 21.97, - "tests/integration/test_micros_to_millis.py": 20.79, - "tests/integration/test_multi_click_trigger.py": 26.2, - "tests/integration/test_multi_device_preferences.py": 16.87, - "tests/integration/test_noise_encryption_key_protection.py": 77.05, - "tests/integration/test_object_id_api_verification.py": 73.51, - "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 62.33, - "tests/integration/test_object_id_no_friendly_name.py": 43.47, - "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 32.21, - "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 56.86, - "tests/integration/test_online_image_bmp.py": 50.9, - "tests/integration/test_oversized_payloads.py": 53.2, - "tests/integration/test_preference_key_stability.py": 26.09, - "tests/integration/test_runtime_stats.py": 18.34, - "tests/integration/test_safe_mode_loop_runs.py": 10.07, - "tests/integration/test_scheduler_blocking_warning.py": 40.91, - "tests/integration/test_scheduler_bulk_cleanup.py": 23.14, - "tests/integration/test_scheduler_defer_cancel.py": 24.54, - "tests/integration/test_scheduler_defer_cancel_regular.py": 13.48, - "tests/integration/test_scheduler_defer_fifo_simple.py": 26.86, - "tests/integration/test_scheduler_defer_stress.py": 27.23, - "tests/integration/test_scheduler_heap_stress.py": 24.02, - "tests/integration/test_scheduler_internal_id_no_collision.py": 24.57, - "tests/integration/test_scheduler_interval_reschedule.py": 13.12, - "tests/integration/test_scheduler_interval_zero_coerced.py": 22.91, - "tests/integration/test_scheduler_null_name.py": 23.46, - "tests/integration/test_scheduler_numeric_id_test.py": 24.54, - "tests/integration/test_scheduler_pool.py": 25.0, - "tests/integration/test_scheduler_rapid_cancellation.py": 14.68, - "tests/integration/test_scheduler_recursive_timeout.py": 25.35, - "tests/integration/test_scheduler_removed_item_race.py": 26.19, - "tests/integration/test_scheduler_self_keyed.py": 23.43, - "tests/integration/test_scheduler_simultaneous_callbacks.py": 22.16, - "tests/integration/test_scheduler_string_test.py": 15.22, - "tests/integration/test_script_array_params.py": 14.67, - "tests/integration/test_script_delay_params.py": 15.65, - "tests/integration/test_script_queued.py": 24.93, - "tests/integration/test_script_queued_idle_loop.py": 5.04, - "tests/integration/test_script_wait_on_boot.py": 13.08, - "tests/integration/test_select_stringref_trigger.py": 29.6, - "tests/integration/test_sensor_filters_delta.py": 28.01, - "tests/integration/test_sensor_filters_ring_buffer.py": 25.04, - "tests/integration/test_sensor_filters_sliding_window.py": 71.5, - "tests/integration/test_sensor_filters_value_list.py": 16.94, - "tests/integration/test_sensor_timeout_filter.py": 29.48, - "tests/integration/test_socket_wake_gate_tcp.py": 20.36, - "tests/integration/test_status_flags.py": 37.42, - "tests/integration/test_strftime_to.py": 22.61, - "tests/integration/test_syslog.py": 16.34, - "tests/integration/test_template_alarm_control_panel_many_sensors.py": 29.81, - "tests/integration/test_template_text_save.py": 25.43, - "tests/integration/test_text_command.py": 23.34, - "tests/integration/test_text_sensor_raw_state.py": 69.57, - "tests/integration/test_uart_mock_ld2410.py": 37.95, - "tests/integration/test_uart_mock_ld2412.py": 93.22, - "tests/integration/test_uart_mock_ld2420.py": 43.24, - "tests/integration/test_uart_mock_ld2450.py": 31.75, - "tests/integration/test_uart_mock_modbus.py": 667.4, - "tests/integration/test_udp.py": 9.38, - "tests/integration/test_use_address_runtime.py": 37.05, - "tests/integration/test_valve_control_action.py": 24.47, - "tests/integration/test_varint_five_byte_device_id.py": 25.03, - "tests/integration/test_wait_until_mid_loop_timing.py": 23.73, - "tests/integration/test_wait_until_on_boot.py": 9.16, - "tests/integration/test_wait_until_ordering.py": 13.3, - "tests/integration/test_wait_until_reentrant_restart.py": 25.23, - "tests/integration/test_wake_loop_forces_phase_b.py": 23.34, - "tests/integration/test_water_heater_template.py": 17.67 + "tests/integration/test_action_concurrent_reentry.py": 57.91, + "tests/integration/test_addressable_light_transition.py": 21.25, + "tests/integration/test_alarm_control_panel_state_transitions.py": 70.71, + "tests/integration/test_api_action_metadata.py": 66.6, + "tests/integration/test_api_action_responses.py": 36.1, + "tests/integration/test_api_action_timeout.py": 68.86, + "tests/integration/test_api_conditional_memory.py": 15.48, + "tests/integration/test_api_custom_services.py": 18.77, + "tests/integration/test_api_get_time_response_timezone.py": 21.08, + "tests/integration/test_api_homeassistant.py": 65.59, + "tests/integration/test_api_homeassistant_action_no_subscriber.py": 18.44, + "tests/integration/test_api_homeassistant_binary_sensor_initial_state.py": 15.05, + "tests/integration/test_api_list_entities_backpressure.py": 13.88, + "tests/integration/test_api_message_size_batching.py": 29.98, + "tests/integration/test_api_reboot_timeout.py": 16.05, + "tests/integration/test_api_string_lambda.py": 15.31, + "tests/integration/test_api_vv_logging.py": 19.28, + "tests/integration/test_api_zero_psk_provisioning.py": 31.5, + "tests/integration/test_areas_and_devices.py": 24.95, + "tests/integration/test_automation_wait_actions.py": 20.92, + "tests/integration/test_automations.py": 35.19, + "tests/integration/test_batch_delay_zero_rapid_transitions.py": 17.99, + "tests/integration/test_binary_sensor_autorepeat_filter.py": 20.39, + "tests/integration/test_binary_sensor_invalidate_state.py": 18.41, + "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 24.69, + "tests/integration/test_build_info.py": 18.7, + "tests/integration/test_camera_mock.py": 16.23, + "tests/integration/test_climate_control_action.py": 21.14, + "tests/integration/test_climate_custom_modes.py": 20.74, + "tests/integration/test_continuation_actions.py": 16.81, + "tests/integration/test_cover_control_action.py": 20.34, + "tests/integration/test_crc8_helper.py": 9.36, + "tests/integration/test_device_id_in_state.py": 44.67, + "tests/integration/test_duplicate_entities.py": 23.58, + "tests/integration/test_entity_icon.py": 34.35, + "tests/integration/test_fan_turn_on_action.py": 24.23, + "tests/integration/test_fnv1_hash_object_id.py": 16.21, + "tests/integration/test_fnv1a_hash.py": 13.38, + "tests/integration/test_gpio_expander_cache.py": 13.06, + "tests/integration/test_host_logger_thread_safety.py": 23.66, + "tests/integration/test_host_mode_basic.py": 8.01, + "tests/integration/test_host_mode_batch_delay.py": 21.0, + "tests/integration/test_host_mode_climate_basic_state.py": 22.14, + "tests/integration/test_host_mode_climate_control.py": 19.39, + "tests/integration/test_host_mode_empty_string_options.py": 21.76, + "tests/integration/test_host_mode_entity_fields.py": 29.61, + "tests/integration/test_host_mode_fan_preset.py": 20.01, + "tests/integration/test_host_mode_many_entities.py": 39.08, + "tests/integration/test_host_mode_many_entities_multiple_connections.py": 23.92, + "tests/integration/test_host_mode_noise_encryption.py": 42.42, + "tests/integration/test_host_mode_reconnect.py": 3.41, + "tests/integration/test_host_mode_sensor.py": 22.96, + "tests/integration/test_host_ota.py": 29.5, + "tests/integration/test_host_preferences.py": 16.06, + "tests/integration/test_host_preferences_suspend_resume.py": 18.71, + "tests/integration/test_improv_serial_uart.py": 20.22, + "tests/integration/test_large_message_batching.py": 26.56, + "tests/integration/test_legacy_area.py": 22.72, + "tests/integration/test_legacy_climate_compat.py": 14.13, + "tests/integration/test_legacy_fan_compat.py": 14.33, + "tests/integration/test_light_automations.py": 18.81, + "tests/integration/test_light_binary_effect_off_phase.py": 8.38, + "tests/integration/test_light_calls.py": 21.88, + "tests/integration/test_light_constant_brightness.py": 59.45, + "tests/integration/test_light_control_action.py": 31.91, + "tests/integration/test_light_dim_relative_action.py": 14.43, + "tests/integration/test_light_effect_zero_brightness.py": 25.05, + "tests/integration/test_light_initial_state.py": 18.97, + "tests/integration/test_light_toggle_action.py": 17.44, + "tests/integration/test_lock_automations.py": 18.9, + "tests/integration/test_logger_buffered_recursion_guard.py": 18.2, + "tests/integration/test_loop_disable_enable.py": 63.35, + "tests/integration/test_loop_interval_decoupling.py": 17.7, + "tests/integration/test_loop_interval_default_not_pulled_forward.py": 21.56, + "tests/integration/test_micros_to_millis.py": 15.89, + "tests/integration/test_multi_click_trigger.py": 17.23, + "tests/integration/test_multi_device_preferences.py": 19.4, + "tests/integration/test_noise_encryption_key_protection.py": 72.59, + "tests/integration/test_object_id_api_verification.py": 19.22, + "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 16.77, + "tests/integration/test_object_id_no_friendly_name.py": 45.8, + "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 86.73, + "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 40.4, + "tests/integration/test_online_image_bmp.py": 37.24, + "tests/integration/test_oversized_payloads.py": 55.75, + "tests/integration/test_preference_key_stability.py": 25.49, + "tests/integration/test_runtime_stats.py": 29.81, + "tests/integration/test_safe_mode_loop_runs.py": 6.26, + "tests/integration/test_scheduler_blocking_warning.py": 37.98, + "tests/integration/test_scheduler_bulk_cleanup.py": 18.67, + "tests/integration/test_scheduler_defer_cancel.py": 18.46, + "tests/integration/test_scheduler_defer_cancel_regular.py": 16.34, + "tests/integration/test_scheduler_defer_fifo_simple.py": 18.26, + "tests/integration/test_scheduler_defer_stress.py": 17.74, + "tests/integration/test_scheduler_heap_stress.py": 3.89, + "tests/integration/test_scheduler_internal_id_no_collision.py": 20.01, + "tests/integration/test_scheduler_interval_reschedule.py": 16.29, + "tests/integration/test_scheduler_interval_zero_coerced.py": 16.09, + "tests/integration/test_scheduler_null_name.py": 14.69, + "tests/integration/test_scheduler_numeric_id_test.py": 17.08, + "tests/integration/test_scheduler_pool.py": 19.88, + "tests/integration/test_scheduler_rapid_cancellation.py": 4.42, + "tests/integration/test_scheduler_recursive_timeout.py": 4.3, + "tests/integration/test_scheduler_removed_item_race.py": 15.49, + "tests/integration/test_scheduler_self_keyed.py": 25.77, + "tests/integration/test_scheduler_simultaneous_callbacks.py": 14.84, + "tests/integration/test_scheduler_string_test.py": 15.42, + "tests/integration/test_script_array_params.py": 12.73, + "tests/integration/test_script_delay_params.py": 12.69, + "tests/integration/test_script_queued.py": 20.38, + "tests/integration/test_script_queued_idle_loop.py": 25.06, + "tests/integration/test_script_wait_on_boot.py": 15.67, + "tests/integration/test_select_stringref_trigger.py": 19.48, + "tests/integration/test_sensor_filters_delta.py": 27.62, + "tests/integration/test_sensor_filters_ring_buffer.py": 20.27, + "tests/integration/test_sensor_filters_sliding_window.py": 56.28, + "tests/integration/test_sensor_filters_value_list.py": 20.6, + "tests/integration/test_sensor_timeout_filter.py": 22.21, + "tests/integration/test_socket_wake_gate_tcp.py": 16.37, + "tests/integration/test_status_flags.py": 29.68, + "tests/integration/test_strftime_to.py": 17.42, + "tests/integration/test_syslog.py": 18.39, + "tests/integration/test_template_alarm_control_panel_many_sensors.py": 25.61, + "tests/integration/test_template_text_save.py": 19.16, + "tests/integration/test_text_command.py": 16.43, + "tests/integration/test_text_sensor_raw_state.py": 17.19, + "tests/integration/test_uart_mock_ld2410.py": 37.0, + "tests/integration/test_uart_mock_ld2412.py": 40.82, + "tests/integration/test_uart_mock_ld2420.py": 32.7, + "tests/integration/test_uart_mock_ld2450.py": 32.84, + "tests/integration/test_uart_mock_modbus.py": 548.87, + "tests/integration/test_udp.py": 16.67, + "tests/integration/test_use_address_runtime.py": 27.26, + "tests/integration/test_valve_control_action.py": 24.58, + "tests/integration/test_varint_five_byte_device_id.py": 22.5, + "tests/integration/test_wait_until_mid_loop_timing.py": 22.05, + "tests/integration/test_wait_until_on_boot.py": 10.37, + "tests/integration/test_wait_until_ordering.py": 18.23, + "tests/integration/test_wait_until_reentrant_restart.py": 19.35, + "tests/integration/test_wake_loop_forces_phase_b.py": 17.83, + "tests/integration/test_water_heater_template.py": 25.7 } From ecbde8ddf4f12e0530ad262ef9a285b030eb8824 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 3 Sep 2026 07:25:08 +1200 Subject: [PATCH 082/147] [uart] Migrate check_uart_settings to final validation (#18940) --- esphome/components/cm1106/cm1106.cpp | 1 - esphome/components/cm1106/sensor.py | 8 ++++++++ esphome/components/cse7761/cse7761.cpp | 1 - esphome/components/cse7761/sensor.py | 8 +++++++- esphome/components/cse7766/cse7766.cpp | 1 - esphome/components/cse7766/sensor.py | 7 ++++++- esphome/components/daly_bms/__init__.py | 8 ++++++++ esphome/components/daly_bms/daly_bms.cpp | 5 +---- esphome/components/dfplayer/__init__.py | 7 ++++++- esphome/components/dfplayer/dfplayer.cpp | 5 +---- esphome/components/hc8/hc8.cpp | 1 - esphome/components/hc8/sensor.py | 3 +++ esphome/components/he60r/he60r.cpp | 1 - .../hrxl_maxsonar_wr/hrxl_maxsonar_wr.cpp | 2 -- esphome/components/hrxl_maxsonar_wr/sensor.py | 8 ++++++++ esphome/components/hydreon_rgxx/hydreon_rgxx.cpp | 1 - esphome/components/hydreon_rgxx/sensor.py | 8 ++++++++ esphome/components/kamstrup_kmp/kamstrup_kmp.cpp | 2 -- esphome/components/kamstrup_kmp/sensor.py | 8 +++++++- esphome/components/mhz19/mhz19.cpp | 2 -- esphome/components/mhz19/sensor.py | 8 ++++++++ esphome/components/mk2pvrouter/mk2pvrouter.cpp | 5 +---- esphome/components/mk2pvrouter/mk2pvrouter.h | 1 - esphome/components/pm1006/pm1006.cpp | 1 - esphome/components/pm1006/sensor.py | 3 +++ esphome/components/pmsx003/pmsx003.cpp | 2 -- esphome/components/pmsx003/sensor.py | 8 +++++++- esphome/components/pylontech/__init__.py | 8 ++++++++ esphome/components/pylontech/pylontech.cpp | 1 - esphome/components/seeed_mr60fda2/__init__.py | 1 + .../components/seeed_mr60fda2/seeed_mr60fda2.cpp | 2 -- esphome/components/smt100/sensor.py | 8 +++++++- esphome/components/smt100/smt100.cpp | 1 - esphome/components/t6615/sensor.py | 8 +++++++- esphome/components/t6615/t6615.cpp | 1 - esphome/components/teleinfo/__init__.py | 16 ++++++++++++++++ esphome/components/teleinfo/teleinfo.cpp | 7 +------ esphome/components/teleinfo/teleinfo.h | 1 - esphome/components/tormatic/tormatic_cover.cpp | 2 -- esphome/components/uart/uart.h | 2 ++ esphome/components/ufm01/__init__.py | 1 + esphome/components/ufm01/ufm01.cpp | 1 - esphome/components/uponor_smatrix/__init__.py | 2 +- .../components/uponor_smatrix/uponor_smatrix.cpp | 2 -- esphome/components/vbus/__init__.py | 8 ++++++++ esphome/components/vbus/vbus.cpp | 5 +---- esphome/components/wl_134/text_sensor.py | 8 ++++++++ esphome/components/wl_134/wl_134.cpp | 2 -- tests/components/cse7761/test.esp32-idf.yaml | 2 +- tests/components/cse7761/test.esp8266-ard.yaml | 2 +- tests/components/cse7761/test.rp2040-ard.yaml | 2 +- .../components/kamstrup_kmp/test.esp32-idf.yaml | 2 +- .../kamstrup_kmp/test.esp8266-ard.yaml | 2 +- tests/components/pylontech/test.esp32-idf.yaml | 2 +- tests/components/pylontech/test.esp8266-ard.yaml | 2 +- tests/components/pylontech/test.rp2040-ard.yaml | 2 +- tests/components/teleinfo/test.esp32-idf.yaml | 2 +- tests/components/teleinfo/test.esp8266-ard.yaml | 2 +- tests/components/teleinfo/test.rp2040-ard.yaml | 2 +- .../teleinfo/validate-standard.esp32-idf.yaml | 14 ++++++++++++++ .../common/uart_1200_even_7bits/esp32-ard.yaml | 14 ++++++++++++++ .../uart_1200_even_7bits/esp32-c3-ard.yaml | 14 ++++++++++++++ .../uart_1200_even_7bits/esp32-c3-idf.yaml | 14 ++++++++++++++ .../common/uart_1200_even_7bits/esp32-idf.yaml | 14 ++++++++++++++ .../common/uart_1200_even_7bits/esp8266-ard.yaml | 14 ++++++++++++++ .../common/uart_1200_even_7bits/rp2040-ard.yaml | 14 ++++++++++++++ .../common/uart_38400_even/esp32-ard.yaml | 12 ++++++++++++ .../common/uart_38400_even/esp32-c3-ard.yaml | 12 ++++++++++++ .../common/uart_38400_even/esp32-c3-idf.yaml | 12 ++++++++++++ .../common/uart_38400_even/esp32-idf.yaml | 12 ++++++++++++ .../common/uart_38400_even/esp8266-ard.yaml | 12 ++++++++++++ .../common/uart_38400_even/rp2040-ard.yaml | 12 ++++++++++++ 72 files changed, 324 insertions(+), 70 deletions(-) create mode 100644 tests/components/teleinfo/validate-standard.esp32-idf.yaml create mode 100644 tests/test_build_components/common/uart_1200_even_7bits/esp32-ard.yaml create mode 100644 tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-ard.yaml create mode 100644 tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-idf.yaml create mode 100644 tests/test_build_components/common/uart_1200_even_7bits/esp32-idf.yaml create mode 100644 tests/test_build_components/common/uart_1200_even_7bits/esp8266-ard.yaml create mode 100644 tests/test_build_components/common/uart_1200_even_7bits/rp2040-ard.yaml create mode 100644 tests/test_build_components/common/uart_38400_even/esp32-ard.yaml create mode 100644 tests/test_build_components/common/uart_38400_even/esp32-c3-ard.yaml create mode 100644 tests/test_build_components/common/uart_38400_even/esp32-c3-idf.yaml create mode 100644 tests/test_build_components/common/uart_38400_even/esp32-idf.yaml create mode 100644 tests/test_build_components/common/uart_38400_even/esp8266-ard.yaml create mode 100644 tests/test_build_components/common/uart_38400_even/rp2040-ard.yaml diff --git a/esphome/components/cm1106/cm1106.cpp b/esphome/components/cm1106/cm1106.cpp index 7e5d25b7ae..2e3352b895 100644 --- a/esphome/components/cm1106/cm1106.cpp +++ b/esphome/components/cm1106/cm1106.cpp @@ -100,7 +100,6 @@ bool CM1106Component::cm1106_write_command_(const uint8_t *command, size_t comma void CM1106Component::dump_config() { ESP_LOGCONFIG(TAG, "CM1106:"); LOG_SENSOR(" ", "CO2", this->co2_sensor_); - this->check_uart_settings(9600); if (this->is_failed()) { ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); } diff --git a/esphome/components/cm1106/sensor.py b/esphome/components/cm1106/sensor.py index 936c5fc673..a36f0b0059 100644 --- a/esphome/components/cm1106/sensor.py +++ b/esphome/components/cm1106/sensor.py @@ -46,6 +46,14 @@ CONFIG_SCHEMA = ( .extend(uart.UART_DEVICE_SCHEMA) ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "cm1106", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: """Code generation entry point.""" diff --git a/esphome/components/cse7761/cse7761.cpp b/esphome/components/cse7761/cse7761.cpp index 4251751531..103bc84452 100644 --- a/esphome/components/cse7761/cse7761.cpp +++ b/esphome/components/cse7761/cse7761.cpp @@ -58,7 +58,6 @@ void CSE7761Component::dump_config() { ESP_LOGE(TAG, ESP_LOG_MSG_COMM_FAIL); } LOG_UPDATE_INTERVAL(this); - this->check_uart_settings(38400, 1, uart::UART_CONFIG_PARITY_EVEN, 8); } void CSE7761Component::update() { diff --git a/esphome/components/cse7761/sensor.py b/esphome/components/cse7761/sensor.py index b53ed26ca3..5f79be0255 100644 --- a/esphome/components/cse7761/sensor.py +++ b/esphome/components/cse7761/sensor.py @@ -68,7 +68,13 @@ CONFIG_SCHEMA = ( ) FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( - "cse7761", baud_rate=38400, require_rx=True, require_tx=True + "cse7761", + baud_rate=38400, + require_rx=True, + require_tx=True, + data_bits=8, + parity="EVEN", + stop_bits=1, ) diff --git a/esphome/components/cse7766/cse7766.cpp b/esphome/components/cse7766/cse7766.cpp index ce77b62b7b..30f1b7a867 100644 --- a/esphome/components/cse7766/cse7766.cpp +++ b/esphome/components/cse7766/cse7766.cpp @@ -255,7 +255,6 @@ void CSE7766Component::dump_config() { LOG_SENSOR(" ", "Apparent Power", this->apparent_power_sensor_); LOG_SENSOR(" ", "Reactive Power", this->reactive_power_sensor_); LOG_SENSOR(" ", "Power Factor", this->power_factor_sensor_); - this->check_uart_settings(4800, 1, uart::UART_CONFIG_PARITY_EVEN); } } // namespace esphome::cse7766 diff --git a/esphome/components/cse7766/sensor.py b/esphome/components/cse7766/sensor.py index a1a68e18e8..9bed0f3f59 100644 --- a/esphome/components/cse7766/sensor.py +++ b/esphome/components/cse7766/sensor.py @@ -84,7 +84,12 @@ CONFIG_SCHEMA = ( .extend(cv.COMPONENT_SCHEMA) ) FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( - "cse7766", baud_rate=4800, parity="EVEN", require_rx=True + "cse7766", + baud_rate=4800, + require_rx=True, + data_bits=8, + parity="EVEN", + stop_bits=1, ) diff --git a/esphome/components/daly_bms/__init__.py b/esphome/components/daly_bms/__init__.py index ba0be4d3a5..c0d7d0aa62 100644 --- a/esphome/components/daly_bms/__init__.py +++ b/esphome/components/daly_bms/__init__.py @@ -26,6 +26,14 @@ CONFIG_SCHEMA = ( .extend(cv.polling_component_schema("30s")) ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "daly_bms", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/daly_bms/daly_bms.cpp b/esphome/components/daly_bms/daly_bms.cpp index 530d8ad541..45d4db4972 100644 --- a/esphome/components/daly_bms/daly_bms.cpp +++ b/esphome/components/daly_bms/daly_bms.cpp @@ -22,10 +22,7 @@ static const uint8_t DALY_REQUEST_TEMPERATURE = 0x96; void DalyBmsComponent::setup() { this->next_request_ = 1; } -void DalyBmsComponent::dump_config() { - ESP_LOGCONFIG(TAG, "Daly BMS:"); - this->check_uart_settings(9600); -} +void DalyBmsComponent::dump_config() { ESP_LOGCONFIG(TAG, "Daly BMS:"); } void DalyBmsComponent::update() { this->trigger_next_ = true; diff --git a/esphome/components/dfplayer/__init__.py b/esphome/components/dfplayer/__init__.py index d589381461..bb18e6ba8c 100644 --- a/esphome/components/dfplayer/__init__.py +++ b/esphome/components/dfplayer/__init__.py @@ -60,7 +60,12 @@ CONFIG_SCHEMA = cv.All( ).extend(uart.UART_DEVICE_SCHEMA) ) FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( - "dfplayer", baud_rate=9600, require_tx=True + "dfplayer", + baud_rate=9600, + require_tx=True, + data_bits=8, + parity="NONE", + stop_bits=1, ) diff --git a/esphome/components/dfplayer/dfplayer.cpp b/esphome/components/dfplayer/dfplayer.cpp index 5c9d497c87..f81d1cd1b6 100644 --- a/esphome/components/dfplayer/dfplayer.cpp +++ b/esphome/components/dfplayer/dfplayer.cpp @@ -277,9 +277,6 @@ void DFPlayer::loop() { } } } -void DFPlayer::dump_config() { - ESP_LOGCONFIG(TAG, "DFPlayer:"); - this->check_uart_settings(9600); -} +void DFPlayer::dump_config() { ESP_LOGCONFIG(TAG, "DFPlayer:"); } } // namespace esphome::dfplayer diff --git a/esphome/components/hc8/hc8.cpp b/esphome/components/hc8/hc8.cpp index 900acca691..6a19f977a6 100644 --- a/esphome/components/hc8/hc8.cpp +++ b/esphome/components/hc8/hc8.cpp @@ -96,7 +96,6 @@ void HC8Component::dump_config() { " Warmup time: %" PRIu32 " s", this->warmup_seconds_); LOG_SENSOR(" ", "CO2", this->co2_sensor_); - this->check_uart_settings(9600); } } // namespace esphome::hc8 diff --git a/esphome/components/hc8/sensor.py b/esphome/components/hc8/sensor.py index 616162eb40..8a19cce8d1 100644 --- a/esphome/components/hc8/sensor.py +++ b/esphome/components/hc8/sensor.py @@ -47,6 +47,9 @@ FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( baud_rate=9600, require_rx=True, require_tx=True, + data_bits=8, + parity="NONE", + stop_bits=1, ) diff --git a/esphome/components/he60r/he60r.cpp b/esphome/components/he60r/he60r.cpp index f49224f17c..008505e2bb 100644 --- a/esphome/components/he60r/he60r.cpp +++ b/esphome/components/he60r/he60r.cpp @@ -38,7 +38,6 @@ CoverTraits HE60rCover::get_traits() { void HE60rCover::dump_config() { LOG_COVER("", "HE60R Cover", this); - this->check_uart_settings(1200, 1, uart::UART_CONFIG_PARITY_EVEN, 8); ESP_LOGCONFIG(TAG, " Open Duration: %.1fs\n" " Close Duration: %.1fs", diff --git a/esphome/components/hrxl_maxsonar_wr/hrxl_maxsonar_wr.cpp b/esphome/components/hrxl_maxsonar_wr/hrxl_maxsonar_wr.cpp index 270bb2709d..b323dd0436 100644 --- a/esphome/components/hrxl_maxsonar_wr/hrxl_maxsonar_wr.cpp +++ b/esphome/components/hrxl_maxsonar_wr/hrxl_maxsonar_wr.cpp @@ -68,8 +68,6 @@ void HrxlMaxsonarWrComponent::check_buffer_() { void HrxlMaxsonarWrComponent::dump_config() { ESP_LOGCONFIG(TAG, "HRXL MaxSonar WR Sensor:"); LOG_SENSOR(" ", "Distance", this); - // As specified in the sensor's data sheet - this->check_uart_settings(9600, 1, esphome::uart::UART_CONFIG_PARITY_NONE, 8); } } // namespace esphome::hrxl_maxsonar_wr diff --git a/esphome/components/hrxl_maxsonar_wr/sensor.py b/esphome/components/hrxl_maxsonar_wr/sensor.py index e4daacd869..b81a8b273d 100644 --- a/esphome/components/hrxl_maxsonar_wr/sensor.py +++ b/esphome/components/hrxl_maxsonar_wr/sensor.py @@ -23,6 +23,14 @@ CONFIG_SCHEMA = sensor.sensor_schema( state_class=STATE_CLASS_MEASUREMENT, ).extend(uart.UART_DEVICE_SCHEMA) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "hrxl_maxsonar_wr", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = await sensor.new_sensor(config) diff --git a/esphome/components/hydreon_rgxx/hydreon_rgxx.cpp b/esphome/components/hydreon_rgxx/hydreon_rgxx.cpp index 695a823cb7..05557111fc 100644 --- a/esphome/components/hydreon_rgxx/hydreon_rgxx.cpp +++ b/esphome/components/hydreon_rgxx/hydreon_rgxx.cpp @@ -11,7 +11,6 @@ static const char *const PROTOCOL_NAMES[] = {HYDREON_RGXX_PROTOCOL_LIST(, HYDREO static const char *const IGNORE_STRINGS[] = {HYDREON_RGXX_IGNORE_LIST(, HYDREON_RGXX_COMMA)}; void HydreonRGxxComponent::dump_config() { - this->check_uart_settings(9600, 1, esphome::uart::UART_CONFIG_PARITY_NONE, 8); ESP_LOGCONFIG(TAG, "hydreon_rgxx:"); if (this->is_failed()) { ESP_LOGE(TAG, "Connection with hydreon_rgxx failed!"); diff --git a/esphome/components/hydreon_rgxx/sensor.py b/esphome/components/hydreon_rgxx/sensor.py index 58e72571ff..8e269fef9a 100644 --- a/esphome/components/hydreon_rgxx/sensor.py +++ b/esphome/components/hydreon_rgxx/sensor.py @@ -130,6 +130,14 @@ CONFIG_SCHEMA = cv.All( _validate, ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "hydreon_rgxx", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/kamstrup_kmp/kamstrup_kmp.cpp b/esphome/components/kamstrup_kmp/kamstrup_kmp.cpp index 70f6d4eaa7..24e5d25921 100644 --- a/esphome/components/kamstrup_kmp/kamstrup_kmp.cpp +++ b/esphome/components/kamstrup_kmp/kamstrup_kmp.cpp @@ -26,8 +26,6 @@ void KamstrupKMPComponent::dump_config() { LOG_SENSOR(" ", "Custom Sensor", this->custom_sensors_[i]); ESP_LOGCONFIG(TAG, " Command: 0x%04X", this->custom_commands_[i]); } - - this->check_uart_settings(1200, 2, uart::UART_CONFIG_PARITY_NONE, 8); } void KamstrupKMPComponent::update() { diff --git a/esphome/components/kamstrup_kmp/sensor.py b/esphome/components/kamstrup_kmp/sensor.py index 6465012897..f6c236b72d 100644 --- a/esphome/components/kamstrup_kmp/sensor.py +++ b/esphome/components/kamstrup_kmp/sensor.py @@ -102,7 +102,13 @@ CONFIG_SCHEMA = ( ) FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( - "kamstrup_kmp", baud_rate=1200, require_rx=True, require_tx=True + "kamstrup_kmp", + baud_rate=1200, + require_rx=True, + require_tx=True, + data_bits=8, + parity="NONE", + stop_bits=2, ) diff --git a/esphome/components/mhz19/mhz19.cpp b/esphome/components/mhz19/mhz19.cpp index ff518808d9..707d952f83 100644 --- a/esphome/components/mhz19/mhz19.cpp +++ b/esphome/components/mhz19/mhz19.cpp @@ -143,8 +143,6 @@ void MHZ19Component::dump_config() { ESP_LOGCONFIG(TAG, "MH-Z19:"); LOG_SENSOR(" ", "CO2", this->co2_sensor_); LOG_SENSOR(" ", "Temperature", this->temperature_sensor_); - this->check_uart_settings(9600); - if (this->abc_boot_logic_ == MHZ19_ABC_ENABLED) { ESP_LOGCONFIG(TAG, " Automatic baseline calibration enabled on boot"); } else if (this->abc_boot_logic_ == MHZ19_ABC_DISABLED) { diff --git a/esphome/components/mhz19/sensor.py b/esphome/components/mhz19/sensor.py index 33cb27080c..5852686608 100644 --- a/esphome/components/mhz19/sensor.py +++ b/esphome/components/mhz19/sensor.py @@ -80,6 +80,14 @@ CONFIG_SCHEMA = ( .extend(uart.UART_DEVICE_SCHEMA) ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "mhz19", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/mk2pvrouter/mk2pvrouter.cpp b/esphome/components/mk2pvrouter/mk2pvrouter.cpp index a9c922602b..0c0476fb11 100644 --- a/esphome/components/mk2pvrouter/mk2pvrouter.cpp +++ b/esphome/components/mk2pvrouter/mk2pvrouter.cpp @@ -163,10 +163,7 @@ void Mk2PVRouter::publish_value_(const char *tag, const char *val) { #endif } -void Mk2PVRouter::dump_config() { - ESP_LOGCONFIG(TAG, "Mk2PVRouter:"); - this->check_uart_settings(BAUD_RATE, 1, uart::UART_CONFIG_PARITY_EVEN, 7); -} +void Mk2PVRouter::dump_config() { ESP_LOGCONFIG(TAG, "Mk2PVRouter:"); } #ifdef MK2PVROUTER_LISTENER_COUNT void Mk2PVRouter::register_mk2pvrouter_listener(Mk2PVRouterListener *listener) { diff --git a/esphome/components/mk2pvrouter/mk2pvrouter.h b/esphome/components/mk2pvrouter/mk2pvrouter.h index f542436f1d..fc23cf49e8 100644 --- a/esphome/components/mk2pvrouter/mk2pvrouter.h +++ b/esphome/components/mk2pvrouter/mk2pvrouter.h @@ -43,7 +43,6 @@ class Mk2PVRouter final : public Component, public uart::UARTDevice { protected: static constexpr size_t CRC_SUFFIX_LEN = 1; - static constexpr uint32_t BAUD_RATE = 9600; enum class State : uint8_t { WAITING_FOR_START, diff --git a/esphome/components/pm1006/pm1006.cpp b/esphome/components/pm1006/pm1006.cpp index 6a325c57dc..d4c6824713 100644 --- a/esphome/components/pm1006/pm1006.cpp +++ b/esphome/components/pm1006/pm1006.cpp @@ -16,7 +16,6 @@ void PM1006Component::dump_config() { ESP_LOGCONFIG(TAG, "PM1006:"); LOG_SENSOR(" ", "PM2.5", this->pm_2_5_sensor_); LOG_UPDATE_INTERVAL(this); - this->check_uart_settings(9600); } void PM1006Component::update() { diff --git a/esphome/components/pm1006/sensor.py b/esphome/components/pm1006/sensor.py index 8274726ac4..447671ebb3 100644 --- a/esphome/components/pm1006/sensor.py +++ b/esphome/components/pm1006/sensor.py @@ -48,6 +48,9 @@ def validate_interval_uart(config: ConfigType) -> None: baud_rate=9600, require_rx=True, require_tx=interval.total_milliseconds != SCHEDULER_DONT_RUN, + data_bits=8, + parity="NONE", + stop_bits=1, )(config) diff --git a/esphome/components/pmsx003/pmsx003.cpp b/esphome/components/pmsx003/pmsx003.cpp index 6275ff60c2..f8d890ac9e 100644 --- a/esphome/components/pmsx003/pmsx003.cpp +++ b/esphome/components/pmsx003/pmsx003.cpp @@ -46,8 +46,6 @@ void PMSX003Component::dump_config() { } else { ESP_LOGCONFIG(TAG, " Mode: passive with sleep/wake cycles"); } - - this->check_uart_settings(9600); } void PMSX003Component::loop() { diff --git a/esphome/components/pmsx003/sensor.py b/esphome/components/pmsx003/sensor.py index fe784c5ffe..dc85380203 100644 --- a/esphome/components/pmsx003/sensor.py +++ b/esphome/components/pmsx003/sensor.py @@ -302,7 +302,13 @@ CONFIG_SCHEMA = cv.All( def final_validate(config: ConfigType) -> None: require_tx = config[CONF_UPDATE_INTERVAL] > cv.time_period("0s") schema = uart.final_validate_device_schema( - "pmsx003", baud_rate=9600, require_rx=True, require_tx=require_tx + "pmsx003", + baud_rate=9600, + require_rx=True, + require_tx=require_tx, + data_bits=8, + parity="NONE", + stop_bits=1, ) schema(config) diff --git a/esphome/components/pylontech/__init__.py b/esphome/components/pylontech/__init__.py index 4ab606d9f9..242a613a6c 100644 --- a/esphome/components/pylontech/__init__.py +++ b/esphome/components/pylontech/__init__.py @@ -41,6 +41,14 @@ CONFIG_SCHEMA = cv.All( .extend(uart.UART_DEVICE_SCHEMA) ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "pylontech", + baud_rate=115200, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/pylontech/pylontech.cpp b/esphome/components/pylontech/pylontech.cpp index 54d9e5c654..932b71ba55 100644 --- a/esphome/components/pylontech/pylontech.cpp +++ b/esphome/components/pylontech/pylontech.cpp @@ -33,7 +33,6 @@ static const uint8_t ASCII_LF = 0x0A; PylontechComponent::PylontechComponent() {} void PylontechComponent::dump_config() { - this->check_uart_settings(115200, 1, esphome::uart::UART_CONFIG_PARITY_NONE, 8); ESP_LOGCONFIG(TAG, "pylontech:"); if (this->is_failed()) { ESP_LOGE(TAG, "Connection with pylontech failed!"); diff --git a/esphome/components/seeed_mr60fda2/__init__.py b/esphome/components/seeed_mr60fda2/__init__.py index de6e8ad57b..159a1ece9c 100644 --- a/esphome/components/seeed_mr60fda2/__init__.py +++ b/esphome/components/seeed_mr60fda2/__init__.py @@ -31,6 +31,7 @@ FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( require_tx=True, require_rx=True, baud_rate=115200, + data_bits=8, parity="NONE", stop_bits=1, ) diff --git a/esphome/components/seeed_mr60fda2/seeed_mr60fda2.cpp b/esphome/components/seeed_mr60fda2/seeed_mr60fda2.cpp index 4875aa5cff..2d1cd0fbb4 100644 --- a/esphome/components/seeed_mr60fda2/seeed_mr60fda2.cpp +++ b/esphome/components/seeed_mr60fda2/seeed_mr60fda2.cpp @@ -33,8 +33,6 @@ void MR60FDA2Component::dump_config() { // Initialisation functions void MR60FDA2Component::setup() { - this->check_uart_settings(115200); - this->current_frame_locate_ = LOCATE_FRAME_HEADER; this->current_frame_id_ = 0; this->current_frame_len_ = 0; diff --git a/esphome/components/smt100/sensor.py b/esphome/components/smt100/sensor.py index 632a1e7547..7ba7da801c 100644 --- a/esphome/components/smt100/sensor.py +++ b/esphome/components/smt100/sensor.py @@ -68,7 +68,13 @@ CONFIG_SCHEMA = ( ) FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( - "smt100", baud_rate=9600, require_rx=True, require_tx=True + "smt100", + baud_rate=9600, + require_rx=True, + require_tx=True, + data_bits=8, + parity="NONE", + stop_bits=1, ) diff --git a/esphome/components/smt100/smt100.cpp b/esphome/components/smt100/smt100.cpp index ed33fc54c5..2889a9fb4d 100644 --- a/esphome/components/smt100/smt100.cpp +++ b/esphome/components/smt100/smt100.cpp @@ -65,7 +65,6 @@ void SMT100Component::dump_config() { LOG_SENSOR(TAG, "Temperature", this->temperature_sensor_); LOG_SENSOR(TAG, "Moisture", this->moisture_sensor_); LOG_UPDATE_INTERVAL(this); - this->check_uart_settings(9600); } int SMT100Component::readline_(int readch, char *buffer, int len) { diff --git a/esphome/components/t6615/sensor.py b/esphome/components/t6615/sensor.py index 6f3ef372bc..44dba52ae8 100644 --- a/esphome/components/t6615/sensor.py +++ b/esphome/components/t6615/sensor.py @@ -33,7 +33,13 @@ CONFIG_SCHEMA = ( ) FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( - "t6615", baud_rate=19200, require_rx=True, require_tx=True + "t6615", + baud_rate=19200, + require_rx=True, + require_tx=True, + data_bits=8, + parity="NONE", + stop_bits=1, ) diff --git a/esphome/components/t6615/t6615.cpp b/esphome/components/t6615/t6615.cpp index 1a98e48c14..982cc181b7 100644 --- a/esphome/components/t6615/t6615.cpp +++ b/esphome/components/t6615/t6615.cpp @@ -88,7 +88,6 @@ void T6615Component::query_ppm_() { void T6615Component::dump_config() { ESP_LOGCONFIG(TAG, "T6615:"); LOG_SENSOR(" ", "CO2", this->co2_sensor_); - this->check_uart_settings(19200); } } // namespace esphome::t6615 diff --git a/esphome/components/teleinfo/__init__.py b/esphome/components/teleinfo/__init__.py index f9233511e1..67aad11d0f 100644 --- a/esphome/components/teleinfo/__init__.py +++ b/esphome/components/teleinfo/__init__.py @@ -35,6 +35,22 @@ CONFIG_SCHEMA = ( ) +def _final_validate(config: ConfigType) -> ConfigType: + # Historical mode runs at 1200 baud, standard mode at 9600 baud. + baud_rate = 1200 if config[CONF_HISTORICAL_MODE] else 9600 + uart.final_validate_device_schema( + "teleinfo", + baud_rate=baud_rate, + data_bits=7, + parity="EVEN", + stop_bits=1, + )(config) + return config + + +FINAL_VALIDATE_SCHEMA = _final_validate + + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID], config[CONF_HISTORICAL_MODE]) await cg.register_component(var, config) diff --git a/esphome/components/teleinfo/teleinfo.cpp b/esphome/components/teleinfo/teleinfo.cpp index e00895d162..17d3d6c099 100644 --- a/esphome/components/teleinfo/teleinfo.cpp +++ b/esphome/components/teleinfo/teleinfo.cpp @@ -184,10 +184,7 @@ void TeleInfo::publish_value_(const std::string &tag, const std::string &val) { element->publish_val(val); } } -void TeleInfo::dump_config() { - ESP_LOGCONFIG(TAG, "TeleInfo:"); - this->check_uart_settings(baud_rate_, 1, uart::UART_CONFIG_PARITY_EVEN, 7); -} +void TeleInfo::dump_config() { ESP_LOGCONFIG(TAG, "TeleInfo:"); } TeleInfo::TeleInfo(bool historical_mode) { if (historical_mode) { /* @@ -195,11 +192,9 @@ TeleInfo::TeleInfo(bool historical_mode) { */ checksum_area_end_ = 2; separator_ = 0x20; - baud_rate_ = 1200; } else { checksum_area_end_ = 1; separator_ = 0x9; - baud_rate_ = 9600; } } void TeleInfo::register_teleinfo_listener(TeleInfoListener *listener) { teleinfo_listeners_.push_back(listener); } diff --git a/esphome/components/teleinfo/teleinfo.h b/esphome/components/teleinfo/teleinfo.h index 4aab3bf2cd..b1bf586e9c 100644 --- a/esphome/components/teleinfo/teleinfo.h +++ b/esphome/components/teleinfo/teleinfo.h @@ -31,7 +31,6 @@ class TeleInfo final : public PollingComponent, public uart::UARTDevice { std::vector teleinfo_listeners_{}; protected: - uint32_t baud_rate_; int checksum_area_end_; int separator_; char buf_[MAX_BUF_SIZE]; diff --git a/esphome/components/tormatic/tormatic_cover.cpp b/esphome/components/tormatic/tormatic_cover.cpp index 7004c4f836..5c8d6623b6 100644 --- a/esphome/components/tormatic/tormatic_cover.cpp +++ b/esphome/components/tormatic/tormatic_cover.cpp @@ -36,8 +36,6 @@ cover::CoverTraits Tormatic::get_traits() { void Tormatic::dump_config() { LOG_COVER("", "Tormatic Cover", this); - this->check_uart_settings(9600, 1, uart::UART_CONFIG_PARITY_NONE, 8); - ESP_LOGCONFIG(TAG, " Open Duration: %.1fs\n" " Close Duration: %.1fs", diff --git a/esphome/components/uart/uart.h b/esphome/components/uart/uart.h index 899d349e21..eda5b72ea8 100644 --- a/esphome/components/uart/uart.h +++ b/esphome/components/uart/uart.h @@ -3,6 +3,7 @@ #include #include "esphome/core/component.h" #include "esphome/core/hal.h" +#include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "uart_component.h" @@ -66,6 +67,7 @@ class UARTDevice { } /// Check that the configuration of the UART bus matches the provided values and otherwise print a warning + ESPDEPRECATED("Use uart.final_validate_device_schema() in Python instead. Removed in 2027.3.0", "2026.9.0") void check_uart_settings(uint32_t baud_rate, uint8_t stop_bits = 1, UARTParityOptions parity = UART_CONFIG_PARITY_NONE, uint8_t data_bits = 8); diff --git a/esphome/components/ufm01/__init__.py b/esphome/components/ufm01/__init__.py index ca0ea57796..85ca0eecae 100644 --- a/esphome/components/ufm01/__init__.py +++ b/esphome/components/ufm01/__init__.py @@ -30,6 +30,7 @@ FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( require_tx=True, require_rx=True, baud_rate=2400, + data_bits=8, parity="EVEN", stop_bits=1, ) diff --git a/esphome/components/ufm01/ufm01.cpp b/esphome/components/ufm01/ufm01.cpp index bafdb5d853..880132bad3 100644 --- a/esphome/components/ufm01/ufm01.cpp +++ b/esphome/components/ufm01/ufm01.cpp @@ -213,7 +213,6 @@ void UFM01Component::dump_config() { LOG_BINARY_SENSOR(" ", "Empty Tube", this->empty_tube_binary_sensor_); LOG_BINARY_SENSOR(" ", "Flow Rate Out Of Range", this->flow_rate_out_of_range_binary_sensor_); #endif - this->check_uart_settings(2400, 1, uart::UART_CONFIG_PARITY_EVEN, 8); } void UFM01Component::on_active_frame_(uint8_t data[FRAME_SIZE]) { diff --git a/esphome/components/uponor_smatrix/__init__.py b/esphome/components/uponor_smatrix/__init__.py index 093408e868..ba686dc22a 100644 --- a/esphome/components/uponor_smatrix/__init__.py +++ b/esphome/components/uponor_smatrix/__init__.py @@ -50,7 +50,7 @@ FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( require_tx=True, require_rx=True, data_bits=8, - parity=None, + parity="NONE", stop_bits=1, ) diff --git a/esphome/components/uponor_smatrix/uponor_smatrix.cpp b/esphome/components/uponor_smatrix/uponor_smatrix.cpp index c77f3468c7..74974548af 100644 --- a/esphome/components/uponor_smatrix/uponor_smatrix.cpp +++ b/esphome/components/uponor_smatrix/uponor_smatrix.cpp @@ -29,8 +29,6 @@ void UponorSmatrixComponent::dump_config() { } #endif - this->check_uart_settings(19200); - if (!this->unknown_devices_.empty()) { ESP_LOGCONFIG(TAG, " Detected unknown device addresses:"); for (auto device_address : this->unknown_devices_) { diff --git a/esphome/components/vbus/__init__.py b/esphome/components/vbus/__init__.py index 94857050f2..fd54658912 100644 --- a/esphome/components/vbus/__init__.py +++ b/esphome/components/vbus/__init__.py @@ -29,6 +29,14 @@ CONFIG_SCHEMA = uart.UART_DEVICE_SCHEMA.extend( } ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "vbus", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) diff --git a/esphome/components/vbus/vbus.cpp b/esphome/components/vbus/vbus.cpp index 81714a2049..080567e7f9 100644 --- a/esphome/components/vbus/vbus.cpp +++ b/esphome/components/vbus/vbus.cpp @@ -11,10 +11,7 @@ static const char *const TAG = "vbus"; // Maximum bytes to log in verbose hex output (16 frames * 4 bytes = 64 bytes typical) static constexpr size_t VBUS_MAX_LOG_BYTES = 64; -void VBus::dump_config() { - ESP_LOGCONFIG(TAG, "VBus:"); - check_uart_settings(9600); -} +void VBus::dump_config() { ESP_LOGCONFIG(TAG, "VBus:"); } static void septet_spread(uint8_t *data, int start, int count, uint8_t septet) { for (int i = 0; i < count; i++, septet >>= 1) { diff --git a/esphome/components/wl_134/text_sensor.py b/esphome/components/wl_134/text_sensor.py index af5e705786..2e3021504f 100644 --- a/esphome/components/wl_134/text_sensor.py +++ b/esphome/components/wl_134/text_sensor.py @@ -21,6 +21,14 @@ CONFIG_SCHEMA = ( .extend(uart.UART_DEVICE_SCHEMA) ) +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "wl_134", + baud_rate=9600, + data_bits=8, + parity="NONE", + stop_bits=1, +) + async def to_code(config: ConfigType) -> None: var = await text_sensor.new_text_sensor(config) diff --git a/esphome/components/wl_134/wl_134.cpp b/esphome/components/wl_134/wl_134.cpp index 5e86d5a441..858f974f2b 100644 --- a/esphome/components/wl_134/wl_134.cpp +++ b/esphome/components/wl_134/wl_134.cpp @@ -110,7 +110,5 @@ uint64_t Wl134Component::hex_lsb_ascii_to_uint64_(const uint8_t *text, uint8_t t void Wl134Component::dump_config() { ESP_LOGCONFIG(TAG, "WL-134 Sensor:"); LOG_TEXT_SENSOR("", "Tag", this); - // As specified in the sensor's data sheet - this->check_uart_settings(9600, 1, esphome::uart::UART_CONFIG_PARITY_NONE, 8); } } // namespace esphome::wl_134 diff --git a/tests/components/cse7761/test.esp32-idf.yaml b/tests/components/cse7761/test.esp32-idf.yaml index a6a8fee7e9..b9ae061c25 100644 --- a/tests/components/cse7761/test.esp32-idf.yaml +++ b/tests/components/cse7761/test.esp32-idf.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO14 packages: - uart_38400: !include ../../test_build_components/common/uart_38400/esp32-idf.yaml + uart_38400_even: !include ../../test_build_components/common/uart_38400_even/esp32-idf.yaml <<: !include common.yaml diff --git a/tests/components/cse7761/test.esp8266-ard.yaml b/tests/components/cse7761/test.esp8266-ard.yaml index 134274ffb8..0d57039e1c 100644 --- a/tests/components/cse7761/test.esp8266-ard.yaml +++ b/tests/components/cse7761/test.esp8266-ard.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO3 packages: - uart_38400: !include ../../test_build_components/common/uart_38400/esp8266-ard.yaml + uart_38400_even: !include ../../test_build_components/common/uart_38400_even/esp8266-ard.yaml <<: !include common.yaml diff --git a/tests/components/cse7761/test.rp2040-ard.yaml b/tests/components/cse7761/test.rp2040-ard.yaml index b813e0f7f1..65e6252c51 100644 --- a/tests/components/cse7761/test.rp2040-ard.yaml +++ b/tests/components/cse7761/test.rp2040-ard.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO5 packages: - uart_38400: !include ../../test_build_components/common/uart_38400/rp2040-ard.yaml + uart_38400_even: !include ../../test_build_components/common/uart_38400_even/rp2040-ard.yaml <<: !include common.yaml diff --git a/tests/components/kamstrup_kmp/test.esp32-idf.yaml b/tests/components/kamstrup_kmp/test.esp32-idf.yaml index 1016905720..4e1ff86fb7 100644 --- a/tests/components/kamstrup_kmp/test.esp32-idf.yaml +++ b/tests/components/kamstrup_kmp/test.esp32-idf.yaml @@ -1,4 +1,4 @@ packages: - uart_1200: !include ../../test_build_components/common/uart_1200/esp32-idf.yaml + uart_1200_none_2stopbits: !include ../../test_build_components/common/uart_1200_none_2stopbits/esp32-idf.yaml <<: !include common.yaml diff --git a/tests/components/kamstrup_kmp/test.esp8266-ard.yaml b/tests/components/kamstrup_kmp/test.esp8266-ard.yaml index f55c18eb76..631516eba9 100644 --- a/tests/components/kamstrup_kmp/test.esp8266-ard.yaml +++ b/tests/components/kamstrup_kmp/test.esp8266-ard.yaml @@ -3,6 +3,6 @@ substitutions: uart_rx_pin: GPIO3 packages: - uart_1200: !include ../../test_build_components/common/uart_1200/esp8266-ard.yaml + uart_1200_none_2stopbits: !include ../../test_build_components/common/uart_1200_none_2stopbits/esp8266-ard.yaml <<: !include common.yaml diff --git a/tests/components/pylontech/test.esp32-idf.yaml b/tests/components/pylontech/test.esp32-idf.yaml index b415125e84..7d5c371187 100644 --- a/tests/components/pylontech/test.esp32-idf.yaml +++ b/tests/components/pylontech/test.esp32-idf.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO5 packages: - uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + uart_115200: !include ../../test_build_components/common/uart_115200/esp32-idf.yaml <<: !include common.yaml diff --git a/tests/components/pylontech/test.esp8266-ard.yaml b/tests/components/pylontech/test.esp8266-ard.yaml index 96ab4ef6ac..c49b2bfee1 100644 --- a/tests/components/pylontech/test.esp8266-ard.yaml +++ b/tests/components/pylontech/test.esp8266-ard.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO2 packages: - uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml + uart_115200: !include ../../test_build_components/common/uart_115200/esp8266-ard.yaml <<: !include common.yaml diff --git a/tests/components/pylontech/test.rp2040-ard.yaml b/tests/components/pylontech/test.rp2040-ard.yaml index b28f2b5e05..5b2785b792 100644 --- a/tests/components/pylontech/test.rp2040-ard.yaml +++ b/tests/components/pylontech/test.rp2040-ard.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO5 packages: - uart: !include ../../test_build_components/common/uart/rp2040-ard.yaml + uart_115200: !include ../../test_build_components/common/uart_115200/rp2040-ard.yaml <<: !include common.yaml diff --git a/tests/components/teleinfo/test.esp32-idf.yaml b/tests/components/teleinfo/test.esp32-idf.yaml index b415125e84..3071f9a67b 100644 --- a/tests/components/teleinfo/test.esp32-idf.yaml +++ b/tests/components/teleinfo/test.esp32-idf.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO5 packages: - uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + uart_1200_even_7bits: !include ../../test_build_components/common/uart_1200_even_7bits/esp32-idf.yaml <<: !include common.yaml diff --git a/tests/components/teleinfo/test.esp8266-ard.yaml b/tests/components/teleinfo/test.esp8266-ard.yaml index 96ab4ef6ac..29490b3be3 100644 --- a/tests/components/teleinfo/test.esp8266-ard.yaml +++ b/tests/components/teleinfo/test.esp8266-ard.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO2 packages: - uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml + uart_1200_even_7bits: !include ../../test_build_components/common/uart_1200_even_7bits/esp8266-ard.yaml <<: !include common.yaml diff --git a/tests/components/teleinfo/test.rp2040-ard.yaml b/tests/components/teleinfo/test.rp2040-ard.yaml index b28f2b5e05..f13d5a9f8f 100644 --- a/tests/components/teleinfo/test.rp2040-ard.yaml +++ b/tests/components/teleinfo/test.rp2040-ard.yaml @@ -3,6 +3,6 @@ substitutions: rx_pin: GPIO5 packages: - uart: !include ../../test_build_components/common/uart/rp2040-ard.yaml + uart_1200_even_7bits: !include ../../test_build_components/common/uart_1200_even_7bits/rp2040-ard.yaml <<: !include common.yaml diff --git a/tests/components/teleinfo/validate-standard.esp32-idf.yaml b/tests/components/teleinfo/validate-standard.esp32-idf.yaml new file mode 100644 index 0000000000..2ca014c8af --- /dev/null +++ b/tests/components/teleinfo/validate-standard.esp32-idf.yaml @@ -0,0 +1,14 @@ +packages: + uart_9600_even_7bits: !include ../../test_build_components/common/uart_9600_even_7bits/esp32-idf.yaml + +teleinfo: + id: test_teleinfo_standard + historical_mode: false + update_interval: 60s + +sensor: + - platform: teleinfo + name: sinsts + tag_name: SINSTS + teleinfo_id: test_teleinfo_standard + unit_of_measurement: VA diff --git a/tests/test_build_components/common/uart_1200_even_7bits/esp32-ard.yaml b/tests/test_build_components/common/uart_1200_even_7bits/esp32-ard.yaml new file mode 100644 index 0000000000..931905032d --- /dev/null +++ b/tests/test_build_components/common/uart_1200_even_7bits/esp32-ard.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP32 Arduino tests - 1200 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO17 + rx_pin: GPIO16 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 1200 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-ard.yaml b/tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-ard.yaml new file mode 100644 index 0000000000..a67b0b6ace --- /dev/null +++ b/tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-ard.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP32-C3 Arduino tests - 1200 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO20 + rx_pin: GPIO21 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 1200 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-idf.yaml b/tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-idf.yaml new file mode 100644 index 0000000000..135aaa68c9 --- /dev/null +++ b/tests/test_build_components/common/uart_1200_even_7bits/esp32-c3-idf.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP32-C3 IDF tests - 1200 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO20 + rx_pin: GPIO21 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 1200 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_1200_even_7bits/esp32-idf.yaml b/tests/test_build_components/common/uart_1200_even_7bits/esp32-idf.yaml new file mode 100644 index 0000000000..4cbe16dfd5 --- /dev/null +++ b/tests/test_build_components/common/uart_1200_even_7bits/esp32-idf.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP32 IDF tests - 1200 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO17 + rx_pin: GPIO16 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 1200 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_1200_even_7bits/esp8266-ard.yaml b/tests/test_build_components/common/uart_1200_even_7bits/esp8266-ard.yaml new file mode 100644 index 0000000000..2eedcad6d3 --- /dev/null +++ b/tests/test_build_components/common/uart_1200_even_7bits/esp8266-ard.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for ESP8266 Arduino tests - 1200 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 1200 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_1200_even_7bits/rp2040-ard.yaml b/tests/test_build_components/common/uart_1200_even_7bits/rp2040-ard.yaml new file mode 100644 index 0000000000..d3edc1c1c9 --- /dev/null +++ b/tests/test_build_components/common/uart_1200_even_7bits/rp2040-ard.yaml @@ -0,0 +1,14 @@ +# Common UART configuration for RP2040 Arduino tests - 1200 baud, EVEN parity, 7 data bits + +substitutions: + tx_pin: GPIO0 + rx_pin: GPIO1 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 1200 + parity: EVEN + data_bits: 7 + stop_bits: 1 diff --git a/tests/test_build_components/common/uart_38400_even/esp32-ard.yaml b/tests/test_build_components/common/uart_38400_even/esp32-ard.yaml new file mode 100644 index 0000000000..4235c9c027 --- /dev/null +++ b/tests/test_build_components/common/uart_38400_even/esp32-ard.yaml @@ -0,0 +1,12 @@ +# Common UART configuration for ESP32 Arduino tests - 38400 baud, EVEN parity + +substitutions: + tx_pin: GPIO17 + rx_pin: GPIO16 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 38400 + parity: EVEN diff --git a/tests/test_build_components/common/uart_38400_even/esp32-c3-ard.yaml b/tests/test_build_components/common/uart_38400_even/esp32-c3-ard.yaml new file mode 100644 index 0000000000..c20b7939e9 --- /dev/null +++ b/tests/test_build_components/common/uart_38400_even/esp32-c3-ard.yaml @@ -0,0 +1,12 @@ +# Common UART configuration for ESP32-C3 Arduino tests - 38400 baud, EVEN parity + +substitutions: + tx_pin: GPIO20 + rx_pin: GPIO21 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 38400 + parity: EVEN diff --git a/tests/test_build_components/common/uart_38400_even/esp32-c3-idf.yaml b/tests/test_build_components/common/uart_38400_even/esp32-c3-idf.yaml new file mode 100644 index 0000000000..0aeb13a7c3 --- /dev/null +++ b/tests/test_build_components/common/uart_38400_even/esp32-c3-idf.yaml @@ -0,0 +1,12 @@ +# Common UART configuration for ESP32-C3 IDF tests - 38400 baud, EVEN parity + +substitutions: + tx_pin: GPIO20 + rx_pin: GPIO21 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 38400 + parity: EVEN diff --git a/tests/test_build_components/common/uart_38400_even/esp32-idf.yaml b/tests/test_build_components/common/uart_38400_even/esp32-idf.yaml new file mode 100644 index 0000000000..b79b91448e --- /dev/null +++ b/tests/test_build_components/common/uart_38400_even/esp32-idf.yaml @@ -0,0 +1,12 @@ +# Common UART configuration for ESP32 IDF tests - 38400 baud, EVEN parity + +substitutions: + tx_pin: GPIO17 + rx_pin: GPIO16 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 38400 + parity: EVEN diff --git a/tests/test_build_components/common/uart_38400_even/esp8266-ard.yaml b/tests/test_build_components/common/uart_38400_even/esp8266-ard.yaml new file mode 100644 index 0000000000..373680e8e6 --- /dev/null +++ b/tests/test_build_components/common/uart_38400_even/esp8266-ard.yaml @@ -0,0 +1,12 @@ +# Common UART configuration for ESP8266 Arduino tests - 38400 baud, EVEN parity + +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 38400 + parity: EVEN diff --git a/tests/test_build_components/common/uart_38400_even/rp2040-ard.yaml b/tests/test_build_components/common/uart_38400_even/rp2040-ard.yaml new file mode 100644 index 0000000000..950f7b4957 --- /dev/null +++ b/tests/test_build_components/common/uart_38400_even/rp2040-ard.yaml @@ -0,0 +1,12 @@ +# Common UART configuration for RP2040 Arduino tests - 38400 baud, EVEN parity + +substitutions: + tx_pin: GPIO0 + rx_pin: GPIO1 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 38400 + parity: EVEN From 567f7f9196425e0b8637b16b2a43373e376e2df4 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Wed, 2 Sep 2026 15:01:23 -0500 Subject: [PATCH 083/147] [serial_proxy] Skip no-op reconfigure requests (#18953) Co-authored-by: puddly <32534428+puddly@users.noreply.github.com> --- .../components/serial_proxy/serial_proxy.cpp | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/esphome/components/serial_proxy/serial_proxy.cpp b/esphome/components/serial_proxy/serial_proxy.cpp index 2ab0d4ebb4..c1c1510643 100644 --- a/esphome/components/serial_proxy/serial_proxy.cpp +++ b/esphome/components/serial_proxy/serial_proxy.cpp @@ -130,17 +130,26 @@ SerialProxyResult SerialProxy::configure(api::APIConnection *api_connection, uin return SerialProxyResult::SERIAL_PROXY_RESULT_NOT_SUPPORTED; } - // Apply validated parameters - uart_comp->set_baud_rate(baudrate); - uart_comp->set_stop_bits(stop_bits); - uart_comp->set_data_bits(data_size); - - // Map parity value to UARTParityOptions + // Skip a no-op reconfigure. Clients routinely re-send identical settings on every + // port open, and on a USB UART each apply is a CDC SET_LINE_CODING control transfer. + // Some bridges watch line-coding changes as a signalling channel (a magic baud + // sequence to enter a bootloader, say), so redundant applies are not harmless. static const uart::UARTParityOptions PARITY_MAP[] = { uart::UART_CONFIG_PARITY_NONE, uart::UART_CONFIG_PARITY_EVEN, uart::UART_CONFIG_PARITY_ODD, }; + if (uart_comp->get_baud_rate() == baudrate && uart_comp->get_stop_bits() == stop_bits && + uart_comp->get_data_bits() == data_size && uart_comp->get_parity() == PARITY_MAP[parity]) { + ESP_LOGV(TAG, "Settings unchanged, skipping reconfigure [%" PRIu32 "]", this->instance_index_); + return SerialProxyResult::SERIAL_PROXY_RESULT_OK; + } + + // Apply validated parameters + uart_comp->set_baud_rate(baudrate); + uart_comp->set_stop_bits(stop_bits); + uart_comp->set_data_bits(data_size); + uart_comp->set_parity(PARITY_MAP[parity]); // load_settings() is available on ESP8266 and ESP32 platforms From f0e2eb96bdcc3411d42432b606d56902088f644c Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 3 Sep 2026 07:39:32 +1000 Subject: [PATCH 084/147] [snapshot][SDL] Display headless mode and snapshots (#17917) Co-authored-by: Claude Opus 5 Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- .github/workflows/ci.yml | 15 +- .gitignore | 2 + CODEOWNERS | 1 + esphome/components/sdl/__init__.py | 253 ++++++++++++++++ esphome/components/sdl/binary_sensor.py | 253 +--------------- esphome/components/sdl/display.py | 53 +++- esphome/components/sdl/sdl_esphome.cpp | 274 +++++++++++++++--- esphome/components/sdl/sdl_esphome.h | 35 ++- .../components/sdl/touchscreen/__init__.py | 4 +- esphome/components/snapshot/__init__.py | 76 +++++ .../components/snapshot/display/__init__.py | 61 ++++ .../snapshot/display/snapshot_display.cpp | 80 +++++ .../snapshot/display/snapshot_display.h | 48 +++ esphome/components/snapshot/snapshot.cpp | 248 ++++++++++++++++ esphome/components/snapshot/snapshot.h | 72 +++++ esphome/core/defines.h | 1 + tests/component_tests/sdl/test_sdl.py | 101 +++++++ tests/components/sdl/common.yaml | 27 ++ tests/components/sdl/validate.host.yaml | 29 ++ tests/components/snapshot/common.yaml | 34 +++ tests/components/snapshot/test.host.yaml | 5 + tests/integration/artifact_utils.py | 26 ++ tests/integration/bmp_utils.py | 161 ++++++++++ .../fixtures/lvgl_headless_render.yaml | 53 ++++ .../fixtures/sdl_headless_screenshot.yaml | 29 ++ .../fixtures/snapshot_display.yaml | 28 ++ .../integration/test_lvgl_headless_render.py | 83 ++++++ .../test_sdl_headless_screenshot.py | 49 ++++ tests/integration/test_snapshot_display.py | 78 +++++ 29 files changed, 1874 insertions(+), 305 deletions(-) create mode 100644 esphome/components/snapshot/__init__.py create mode 100644 esphome/components/snapshot/display/__init__.py create mode 100644 esphome/components/snapshot/display/snapshot_display.cpp create mode 100644 esphome/components/snapshot/display/snapshot_display.h create mode 100644 esphome/components/snapshot/snapshot.cpp create mode 100644 esphome/components/snapshot/snapshot.h create mode 100644 tests/component_tests/sdl/test_sdl.py create mode 100644 tests/components/sdl/validate.host.yaml create mode 100644 tests/components/snapshot/common.yaml create mode 100644 tests/components/snapshot/test.host.yaml create mode 100644 tests/integration/artifact_utils.py create mode 100644 tests/integration/bmp_utils.py create mode 100644 tests/integration/fixtures/lvgl_headless_render.yaml create mode 100644 tests/integration/fixtures/sdl_headless_screenshot.yaml create mode 100644 tests/integration/fixtures/snapshot_display.yaml create mode 100644 tests/integration/test_lvgl_headless_render.py create mode 100644 tests/integration/test_sdl_headless_screenshot.py create mode 100644 tests/integration/test_snapshot_display.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a874a023b9..d7c93b3b86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -374,8 +374,9 @@ jobs: - name: Install apt packages (cached) # ccache speeds up the host compiles. A cache hit never touches apt # (mirror outages cannot hang the job); the timeout bounds the cold - # path. Packages and version must match seed-apt-cache exactly; - # libsdl2-dev is unused here and carried only for cache-key parity. + # path. Packages and version must match seed-apt-cache exactly. + # libsdl2-dev is needed by the headless display tests, which capture + # screenshots. timeout-minutes: 10 uses: awalsh128/cache-apt-pkgs-action@553a35bb8ebd9fcabcb1c9451aa4c98e1b4ca8a9 # v1.6.3 with: @@ -438,6 +439,16 @@ jobs: echo "Bucket ${{ matrix.bucket.name }}: running ${#test_files[@]} integration tests" pytest -vv --no-cov --tb=native --durations=30 -n auto --dist worksteal \ --junitxml=junit-integration.xml "${test_files[@]}" + - name: Upload test artifacts + # Tests that compare rendered output write the image they actually got here, so a + # failure can be looked at without reproducing the whole build locally. + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: integration-test-artifacts-${{ matrix.bucket.name }} + path: test_artifacts/ + if-no-files-found: ignore + retention-days: 7 - name: Upload junit timings # Consumed by sync-integration-durations.yml through # script/update_integration_test_durations.py; only full matrix dev diff --git a/.gitignore b/.gitignore index fdb75824fb..82b00286c7 100644 --- a/.gitignore +++ b/.gitignore @@ -137,6 +137,8 @@ config/ !tests/component_tests/**/config/ tests/build/ tests/.esphome/ +# Output kept by failing tests for inspection; uploaded by CI +test_artifacts/ /.temp-clang-tidy.cpp /.temp/ .pio/ diff --git a/CODEOWNERS b/CODEOWNERS index 3429a93aa7..f91bc00ae5 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -496,6 +496,7 @@ esphome/components/sm2335/* @Cossid esphome/components/sml/* @alengwenus esphome/components/smt100/* @piechade esphome/components/sn74hc165/* @jesserockz +esphome/components/snapshot/* @clydebarrow esphome/components/socket/* @esphome/core esphome/components/sonoff_d1/* @anatoly-savchenkov esphome/components/sound_level/* @kahrendt diff --git a/esphome/components/sdl/__init__.py b/esphome/components/sdl/__init__.py index c58ce8a01e..872d831850 100644 --- a/esphome/components/sdl/__init__.py +++ b/esphome/components/sdl/__init__.py @@ -1 +1,254 @@ +import esphome.codegen as cg + CODEOWNERS = ["@clydebarrow"] + +SDL_KeyCode = cg.global_ns.enum("SDL_KeyCode") + +SDL_KEYS = ( + "SDLK_UNKNOWN", + "SDLK_RETURN", + "SDLK_ESCAPE", + "SDLK_BACKSPACE", + "SDLK_TAB", + "SDLK_SPACE", + "SDLK_EXCLAIM", + "SDLK_QUOTEDBL", + "SDLK_HASH", + "SDLK_PERCENT", + "SDLK_DOLLAR", + "SDLK_AMPERSAND", + "SDLK_QUOTE", + "SDLK_LEFTPAREN", + "SDLK_RIGHTPAREN", + "SDLK_ASTERISK", + "SDLK_PLUS", + "SDLK_COMMA", + "SDLK_MINUS", + "SDLK_PERIOD", + "SDLK_SLASH", + "SDLK_0", + "SDLK_1", + "SDLK_2", + "SDLK_3", + "SDLK_4", + "SDLK_5", + "SDLK_6", + "SDLK_7", + "SDLK_8", + "SDLK_9", + "SDLK_COLON", + "SDLK_SEMICOLON", + "SDLK_LESS", + "SDLK_EQUALS", + "SDLK_GREATER", + "SDLK_QUESTION", + "SDLK_AT", + "SDLK_LEFTBRACKET", + "SDLK_BACKSLASH", + "SDLK_RIGHTBRACKET", + "SDLK_CARET", + "SDLK_UNDERSCORE", + "SDLK_BACKQUOTE", + "SDLK_a", + "SDLK_b", + "SDLK_c", + "SDLK_d", + "SDLK_e", + "SDLK_f", + "SDLK_g", + "SDLK_h", + "SDLK_i", + "SDLK_j", + "SDLK_k", + "SDLK_l", + "SDLK_m", + "SDLK_n", + "SDLK_o", + "SDLK_p", + "SDLK_q", + "SDLK_r", + "SDLK_s", + "SDLK_t", + "SDLK_u", + "SDLK_v", + "SDLK_w", + "SDLK_x", + "SDLK_y", + "SDLK_z", + "SDLK_CAPSLOCK", + "SDLK_F1", + "SDLK_F2", + "SDLK_F3", + "SDLK_F4", + "SDLK_F5", + "SDLK_F6", + "SDLK_F7", + "SDLK_F8", + "SDLK_F9", + "SDLK_F10", + "SDLK_F11", + "SDLK_F12", + "SDLK_PRINTSCREEN", + "SDLK_SCROLLLOCK", + "SDLK_PAUSE", + "SDLK_INSERT", + "SDLK_HOME", + "SDLK_PAGEUP", + "SDLK_DELETE", + "SDLK_END", + "SDLK_PAGEDOWN", + "SDLK_RIGHT", + "SDLK_LEFT", + "SDLK_DOWN", + "SDLK_UP", + "SDLK_NUMLOCKCLEAR", + "SDLK_KP_DIVIDE", + "SDLK_KP_MULTIPLY", + "SDLK_KP_MINUS", + "SDLK_KP_PLUS", + "SDLK_KP_ENTER", + "SDLK_KP_1", + "SDLK_KP_2", + "SDLK_KP_3", + "SDLK_KP_4", + "SDLK_KP_5", + "SDLK_KP_6", + "SDLK_KP_7", + "SDLK_KP_8", + "SDLK_KP_9", + "SDLK_KP_0", + "SDLK_KP_PERIOD", + "SDLK_APPLICATION", + "SDLK_POWER", + "SDLK_KP_EQUALS", + "SDLK_F13", + "SDLK_F14", + "SDLK_F15", + "SDLK_F16", + "SDLK_F17", + "SDLK_F18", + "SDLK_F19", + "SDLK_F20", + "SDLK_F21", + "SDLK_F22", + "SDLK_F23", + "SDLK_F24", + "SDLK_EXECUTE", + "SDLK_HELP", + "SDLK_MENU", + "SDLK_SELECT", + "SDLK_STOP", + "SDLK_AGAIN", + "SDLK_UNDO", + "SDLK_CUT", + "SDLK_COPY", + "SDLK_PASTE", + "SDLK_FIND", + "SDLK_MUTE", + "SDLK_VOLUMEUP", + "SDLK_VOLUMEDOWN", + "SDLK_KP_COMMA", + "SDLK_KP_EQUALSAS400", + "SDLK_ALTERASE", + "SDLK_SYSREQ", + "SDLK_CANCEL", + "SDLK_CLEAR", + "SDLK_PRIOR", + "SDLK_RETURN2", + "SDLK_SEPARATOR", + "SDLK_OUT", + "SDLK_OPER", + "SDLK_CLEARAGAIN", + "SDLK_CRSEL", + "SDLK_EXSEL", + "SDLK_KP_00", + "SDLK_KP_000", + "SDLK_THOUSANDSSEPARATOR", + "SDLK_DECIMALSEPARATOR", + "SDLK_CURRENCYUNIT", + "SDLK_CURRENCYSUBUNIT", + "SDLK_KP_LEFTPAREN", + "SDLK_KP_RIGHTPAREN", + "SDLK_KP_LEFTBRACE", + "SDLK_KP_RIGHTBRACE", + "SDLK_KP_TAB", + "SDLK_KP_BACKSPACE", + "SDLK_KP_A", + "SDLK_KP_B", + "SDLK_KP_C", + "SDLK_KP_D", + "SDLK_KP_E", + "SDLK_KP_F", + "SDLK_KP_XOR", + "SDLK_KP_POWER", + "SDLK_KP_PERCENT", + "SDLK_KP_LESS", + "SDLK_KP_GREATER", + "SDLK_KP_AMPERSAND", + "SDLK_KP_DBLAMPERSAND", + "SDLK_KP_VERTICALBAR", + "SDLK_KP_DBLVERTICALBAR", + "SDLK_KP_COLON", + "SDLK_KP_HASH", + "SDLK_KP_SPACE", + "SDLK_KP_AT", + "SDLK_KP_EXCLAM", + "SDLK_KP_MEMSTORE", + "SDLK_KP_MEMRECALL", + "SDLK_KP_MEMCLEAR", + "SDLK_KP_MEMADD", + "SDLK_KP_MEMSUBTRACT", + "SDLK_KP_MEMMULTIPLY", + "SDLK_KP_MEMDIVIDE", + "SDLK_KP_PLUSMINUS", + "SDLK_KP_CLEAR", + "SDLK_KP_CLEARENTRY", + "SDLK_KP_BINARY", + "SDLK_KP_OCTAL", + "SDLK_KP_DECIMAL", + "SDLK_KP_HEXADECIMAL", + "SDLK_LCTRL", + "SDLK_LSHIFT", + "SDLK_LALT", + "SDLK_LGUI", + "SDLK_RCTRL", + "SDLK_RSHIFT", + "SDLK_RALT", + "SDLK_RGUI", + "SDLK_MODE", + "SDLK_AUDIONEXT", + "SDLK_AUDIOPREV", + "SDLK_AUDIOSTOP", + "SDLK_AUDIOPLAY", + "SDLK_AUDIOMUTE", + "SDLK_MEDIASELECT", + "SDLK_WWW", + "SDLK_MAIL", + "SDLK_CALCULATOR", + "SDLK_COMPUTER", + "SDLK_AC_SEARCH", + "SDLK_AC_HOME", + "SDLK_AC_BACK", + "SDLK_AC_FORWARD", + "SDLK_AC_STOP", + "SDLK_AC_REFRESH", + "SDLK_AC_BOOKMARKS", + "SDLK_BRIGHTNESSDOWN", + "SDLK_BRIGHTNESSUP", + "SDLK_DISPLAYSWITCH", + "SDLK_KBDILLUMTOGGLE", + "SDLK_KBDILLUMDOWN", + "SDLK_KBDILLUMUP", + "SDLK_EJECT", + "SDLK_SLEEP", + "SDLK_APP1", + "SDLK_APP2", + "SDLK_AUDIOREWIND", + "SDLK_AUDIOFASTFORWARD", + "SDLK_SOFTLEFT", + "SDLK_SOFTRIGHT", + "SDLK_CALL", + "SDLK_ENDCALL", +) + +SDL_KEYMAP = {key: getattr(SDL_KeyCode, key) for key in SDL_KEYS} diff --git a/esphome/components/sdl/binary_sensor.py b/esphome/components/sdl/binary_sensor.py index 0fdda25ed3..c978071391 100644 --- a/esphome/components/sdl/binary_sensor.py +++ b/esphome/components/sdl/binary_sensor.py @@ -7,262 +7,15 @@ from esphome.core import Lambda from esphome.cpp_generator import ExpressionStatement, RawExpression from esphome.types import ConfigType -from .display import CONF_SDL_ID, Sdl +from . import SDL_KEYMAP +from .display import CONF_SDL_ID, Sdl, headless_final_validate CODEOWNERS = ["@bdm310"] STATE_ARG = "state" -SDL_KeyCode = cg.global_ns.enum("SDL_KeyCode") +FINAL_VALIDATE_SCHEMA = headless_final_validate("binary_sensor") -SDL_KEYS = ( - "SDLK_UNKNOWN", - "SDLK_RETURN", - "SDLK_ESCAPE", - "SDLK_BACKSPACE", - "SDLK_TAB", - "SDLK_SPACE", - "SDLK_EXCLAIM", - "SDLK_QUOTEDBL", - "SDLK_HASH", - "SDLK_PERCENT", - "SDLK_DOLLAR", - "SDLK_AMPERSAND", - "SDLK_QUOTE", - "SDLK_LEFTPAREN", - "SDLK_RIGHTPAREN", - "SDLK_ASTERISK", - "SDLK_PLUS", - "SDLK_COMMA", - "SDLK_MINUS", - "SDLK_PERIOD", - "SDLK_SLASH", - "SDLK_0", - "SDLK_1", - "SDLK_2", - "SDLK_3", - "SDLK_4", - "SDLK_5", - "SDLK_6", - "SDLK_7", - "SDLK_8", - "SDLK_9", - "SDLK_COLON", - "SDLK_SEMICOLON", - "SDLK_LESS", - "SDLK_EQUALS", - "SDLK_GREATER", - "SDLK_QUESTION", - "SDLK_AT", - "SDLK_LEFTBRACKET", - "SDLK_BACKSLASH", - "SDLK_RIGHTBRACKET", - "SDLK_CARET", - "SDLK_UNDERSCORE", - "SDLK_BACKQUOTE", - "SDLK_a", - "SDLK_b", - "SDLK_c", - "SDLK_d", - "SDLK_e", - "SDLK_f", - "SDLK_g", - "SDLK_h", - "SDLK_i", - "SDLK_j", - "SDLK_k", - "SDLK_l", - "SDLK_m", - "SDLK_n", - "SDLK_o", - "SDLK_p", - "SDLK_q", - "SDLK_r", - "SDLK_s", - "SDLK_t", - "SDLK_u", - "SDLK_v", - "SDLK_w", - "SDLK_x", - "SDLK_y", - "SDLK_z", - "SDLK_CAPSLOCK", - "SDLK_F1", - "SDLK_F2", - "SDLK_F3", - "SDLK_F4", - "SDLK_F5", - "SDLK_F6", - "SDLK_F7", - "SDLK_F8", - "SDLK_F9", - "SDLK_F10", - "SDLK_F11", - "SDLK_F12", - "SDLK_PRINTSCREEN", - "SDLK_SCROLLLOCK", - "SDLK_PAUSE", - "SDLK_INSERT", - "SDLK_HOME", - "SDLK_PAGEUP", - "SDLK_DELETE", - "SDLK_END", - "SDLK_PAGEDOWN", - "SDLK_RIGHT", - "SDLK_LEFT", - "SDLK_DOWN", - "SDLK_UP", - "SDLK_NUMLOCKCLEAR", - "SDLK_KP_DIVIDE", - "SDLK_KP_MULTIPLY", - "SDLK_KP_MINUS", - "SDLK_KP_PLUS", - "SDLK_KP_ENTER", - "SDLK_KP_1", - "SDLK_KP_2", - "SDLK_KP_3", - "SDLK_KP_4", - "SDLK_KP_5", - "SDLK_KP_6", - "SDLK_KP_7", - "SDLK_KP_8", - "SDLK_KP_9", - "SDLK_KP_0", - "SDLK_KP_PERIOD", - "SDLK_APPLICATION", - "SDLK_POWER", - "SDLK_KP_EQUALS", - "SDLK_F13", - "SDLK_F14", - "SDLK_F15", - "SDLK_F16", - "SDLK_F17", - "SDLK_F18", - "SDLK_F19", - "SDLK_F20", - "SDLK_F21", - "SDLK_F22", - "SDLK_F23", - "SDLK_F24", - "SDLK_EXECUTE", - "SDLK_HELP", - "SDLK_MENU", - "SDLK_SELECT", - "SDLK_STOP", - "SDLK_AGAIN", - "SDLK_UNDO", - "SDLK_CUT", - "SDLK_COPY", - "SDLK_PASTE", - "SDLK_FIND", - "SDLK_MUTE", - "SDLK_VOLUMEUP", - "SDLK_VOLUMEDOWN", - "SDLK_KP_COMMA", - "SDLK_KP_EQUALSAS400", - "SDLK_ALTERASE", - "SDLK_SYSREQ", - "SDLK_CANCEL", - "SDLK_CLEAR", - "SDLK_PRIOR", - "SDLK_RETURN2", - "SDLK_SEPARATOR", - "SDLK_OUT", - "SDLK_OPER", - "SDLK_CLEARAGAIN", - "SDLK_CRSEL", - "SDLK_EXSEL", - "SDLK_KP_00", - "SDLK_KP_000", - "SDLK_THOUSANDSSEPARATOR", - "SDLK_DECIMALSEPARATOR", - "SDLK_CURRENCYUNIT", - "SDLK_CURRENCYSUBUNIT", - "SDLK_KP_LEFTPAREN", - "SDLK_KP_RIGHTPAREN", - "SDLK_KP_LEFTBRACE", - "SDLK_KP_RIGHTBRACE", - "SDLK_KP_TAB", - "SDLK_KP_BACKSPACE", - "SDLK_KP_A", - "SDLK_KP_B", - "SDLK_KP_C", - "SDLK_KP_D", - "SDLK_KP_E", - "SDLK_KP_F", - "SDLK_KP_XOR", - "SDLK_KP_POWER", - "SDLK_KP_PERCENT", - "SDLK_KP_LESS", - "SDLK_KP_GREATER", - "SDLK_KP_AMPERSAND", - "SDLK_KP_DBLAMPERSAND", - "SDLK_KP_VERTICALBAR", - "SDLK_KP_DBLVERTICALBAR", - "SDLK_KP_COLON", - "SDLK_KP_HASH", - "SDLK_KP_SPACE", - "SDLK_KP_AT", - "SDLK_KP_EXCLAM", - "SDLK_KP_MEMSTORE", - "SDLK_KP_MEMRECALL", - "SDLK_KP_MEMCLEAR", - "SDLK_KP_MEMADD", - "SDLK_KP_MEMSUBTRACT", - "SDLK_KP_MEMMULTIPLY", - "SDLK_KP_MEMDIVIDE", - "SDLK_KP_PLUSMINUS", - "SDLK_KP_CLEAR", - "SDLK_KP_CLEARENTRY", - "SDLK_KP_BINARY", - "SDLK_KP_OCTAL", - "SDLK_KP_DECIMAL", - "SDLK_KP_HEXADECIMAL", - "SDLK_LCTRL", - "SDLK_LSHIFT", - "SDLK_LALT", - "SDLK_LGUI", - "SDLK_RCTRL", - "SDLK_RSHIFT", - "SDLK_RALT", - "SDLK_RGUI", - "SDLK_MODE", - "SDLK_AUDIONEXT", - "SDLK_AUDIOPREV", - "SDLK_AUDIOSTOP", - "SDLK_AUDIOPLAY", - "SDLK_AUDIOMUTE", - "SDLK_MEDIASELECT", - "SDLK_WWW", - "SDLK_MAIL", - "SDLK_CALCULATOR", - "SDLK_COMPUTER", - "SDLK_AC_SEARCH", - "SDLK_AC_HOME", - "SDLK_AC_BACK", - "SDLK_AC_FORWARD", - "SDLK_AC_STOP", - "SDLK_AC_REFRESH", - "SDLK_AC_BOOKMARKS", - "SDLK_BRIGHTNESSDOWN", - "SDLK_BRIGHTNESSUP", - "SDLK_DISPLAYSWITCH", - "SDLK_KBDILLUMTOGGLE", - "SDLK_KBDILLUMDOWN", - "SDLK_KBDILLUMUP", - "SDLK_EJECT", - "SDLK_SLEEP", - "SDLK_APP1", - "SDLK_APP2", - "SDLK_AUDIOREWIND", - "SDLK_AUDIOFASTFORWARD", - "SDLK_SOFTLEFT", - "SDLK_SOFTRIGHT", - "SDLK_CALL", - "SDLK_ENDCALL", -) - -SDL_KEYMAP = {key: getattr(SDL_KeyCode, key) for key in SDL_KEYS} CONFIG_SCHEMA = ( binary_sensor.binary_sensor_schema(BinarySensor) diff --git a/esphome/components/sdl/display.py b/esphome/components/sdl/display.py index 5ced2edf5a..77b0001c55 100644 --- a/esphome/components/sdl/display.py +++ b/esphome/components/sdl/display.py @@ -4,6 +4,7 @@ from typing import Any import esphome.codegen as cg from esphome.components import display +from esphome.components.snapshot import Snapshot, register_snapshot import esphome.config_validation as cv from esphome.const import ( CONF_DIMENSIONS, @@ -16,14 +17,21 @@ from esphome.const import ( CONF_Y, PLATFORM_HOST, ) +import esphome.final_validate as fv from esphome.types import ConfigType +from . import SDL_KEYMAP + +AUTO_LOAD = ["snapshot"] + sdl_ns = cg.esphome_ns.namespace("sdl") -Sdl = sdl_ns.class_("Sdl", display.Display, cg.Component) +Sdl = sdl_ns.class_("Sdl", display.Display, cg.Component, Snapshot) sdl_window_flags = cg.global_ns.enum("SDL_WindowFlags") CONF_CENTERED_ON_DISPLAY = "centered_on_display" +CONF_HEADLESS = "headless" +CONF_SNAPSHOT_KEY = "snapshot_key" CONF_SDL_OPTIONS = "sdl_options" CONF_SDL_ID = "sdl_id" CONF_WINDOW_OPTIONS = "window_options" @@ -67,12 +75,29 @@ def _validate_position(config: dict) -> dict: raise cv.Invalid("Must specify either 'x' and 'y' or 'centered_on_display'") +def _validate_headless(config: ConfigType) -> ConfigType: + if not config[CONF_HEADLESS]: + return config + if CONF_WINDOW_OPTIONS in config: + raise cv.Invalid( + f"'{CONF_WINDOW_OPTIONS}' has no effect when '{CONF_HEADLESS}' is set - there is no window" + ) + if CONF_SNAPSHOT_KEY in config: + raise cv.Invalid( + f"'{CONF_SNAPSHOT_KEY}' cannot be used when '{CONF_HEADLESS}' is set - " + f"there is no keyboard. Use the 'snapshot.take' action instead" + ) + return config + + CONFIG_SCHEMA = cv.All( display.FULL_DISPLAY_SCHEMA.extend( cv.Schema( { cv.GenerateID(): cv.declare_id(Sdl), cv.Optional(CONF_SDL_OPTIONS, default=""): get_sdl_options, + cv.Optional(CONF_HEADLESS, default=False): cv.boolean, + cv.Optional(CONF_SNAPSHOT_KEY): cv.enum(SDL_KEYMAP), cv.Required(CONF_DIMENSIONS): cv.Any( cv.dimensions, cv.Schema( @@ -99,16 +124,42 @@ CONFIG_SCHEMA = cv.All( } ) ), + _validate_headless, cv.only_on(PLATFORM_HOST), ) +def headless_final_validate(platform: str) -> cv.Schema: + """Build a FINAL_VALIDATE_SCHEMA rejecting a platform whose sdl display is headless. + + Mouse and keyboard platforms are driven by window events, so under a headless display they + would never report anything. + """ + + def validate_display(display_config: ConfigType) -> ConfigType: + if display_config.get(CONF_HEADLESS): + raise cv.Invalid( + f"The sdl {platform} platform needs a window, but its display has " + f"'{CONF_HEADLESS}' set" + ) + return display_config + + return cv.Schema( + {cv.Required(CONF_SDL_ID): fv.id_declaration_match_schema(validate_display)}, + extra=cv.ALLOW_EXTRA, + ) + + async def to_code(config: ConfigType) -> None: for option in config[CONF_SDL_OPTIONS].split(): cg.add_build_flag(option) cg.add_build_flag("-DSDL_BYTEORDER=4321") var = cg.new_Pvariable(config[CONF_ID]) await display.register_display(var, config) + await register_snapshot(var, config) + cg.add(var.set_headless(config[CONF_HEADLESS])) + if (key := config.get(CONF_SNAPSHOT_KEY)) is not None: + cg.add(var.set_snapshot_key(key)) dimensions = config[CONF_DIMENSIONS] if isinstance(dimensions, dict): diff --git a/esphome/components/sdl/sdl_esphome.cpp b/esphome/components/sdl/sdl_esphome.cpp index c99b5081b3..03fc086021 100644 --- a/esphome/components/sdl/sdl_esphome.cpp +++ b/esphome/components/sdl/sdl_esphome.cpp @@ -2,8 +2,17 @@ #include "sdl_esphome.h" #include "esphome/components/display/display_color_utils.h" +#include + namespace esphome::sdl { +namespace { + +// Key under which each window keeps a pointer back to its Sdl instance. +constexpr const char *const WINDOW_DATA_KEY = "esphome_sdl"; + +} // namespace + int Sdl::get_width() { switch (this->rotation_) { case display::DISPLAY_ROTATION_90_DEGREES: @@ -28,17 +37,96 @@ int Sdl::get_height() { } } -void Sdl::setup() { - SDL_Init(SDL_INIT_VIDEO); - this->window_ = SDL_CreateWindow(App.get_name().c_str(), this->pos_x_, this->pos_y_, this->width_, this->height_, - this->window_options_); - this->renderer_ = SDL_CreateRenderer(this->window_, -1, SDL_RENDERER_SOFTWARE); - SDL_RenderSetLogicalSize(this->renderer_, this->width_, this->height_); +void Sdl::destroy_renderer_() { + // Reverse order of creation: the renderer refers to the window or surface it was made from. + if (this->shot_target_ != nullptr) { + SDL_DestroyTexture(this->shot_target_); + this->shot_target_ = nullptr; + } + if (this->texture_ != nullptr) { + SDL_DestroyTexture(this->texture_); + this->texture_ = nullptr; + } + if (this->renderer_ != nullptr) { + SDL_DestroyRenderer(this->renderer_); + this->renderer_ = nullptr; + } + if (this->window_ != nullptr) { + SDL_DestroyWindow(this->window_); + this->window_ = nullptr; + } + if (this->surface_ != nullptr) { + SDL_FreeSurface(this->surface_); + this->surface_ = nullptr; + } +} + +bool Sdl::setup_failed_(const char *what) { + ESP_LOGE(TAG, "%s: %s", what, SDL_GetError()); + // Give back whatever was created before the failure. Without this a half set up display leaves an + // empty window on screen for the life of the process, still registered as an event target. + this->destroy_renderer_(); + return false; +} + +bool Sdl::setup_renderer_() { + SDL_SetMainReady(); + if (this->headless_) { + // SDL_INIT_VIDEO is deliberately not requested: a software renderer bound to a surface needs no + // video device, so this works on a machine with no display server at all. + if (SDL_Init(0) != 0) + return this->setup_failed_("SDL_Init failed"); + this->surface_ = SDL_CreateRGBSurfaceWithFormat(0, this->width_, this->height_, 16, SDL_PIXELFORMAT_RGB565); + if (this->surface_ == nullptr) + return this->setup_failed_("Could not create offscreen surface"); + this->renderer_ = SDL_CreateSoftwareRenderer(this->surface_); + } else { + if (SDL_Init(SDL_INIT_VIDEO) != 0) + return this->setup_failed_("SDL_Init failed"); + this->window_ = SDL_CreateWindow(App.get_name().c_str(), this->pos_x_, this->pos_y_, this->width_, this->height_, + this->window_options_); + if (this->window_ == nullptr) + return this->setup_failed_("Could not create window"); + // Lets loop() find the display an event belongs to, so one display does not act on another's + // input when several windows are open. + SDL_SetWindowData(this->window_, WINDOW_DATA_KEY, this); + this->renderer_ = SDL_CreateRenderer(this->window_, -1, SDL_RENDERER_SOFTWARE); + } + if (this->renderer_ == nullptr) + return this->setup_failed_("Could not create renderer"); + if (SDL_RenderSetLogicalSize(this->renderer_, this->width_, this->height_) != 0) + return this->setup_failed_("Could not set renderer logical size"); this->texture_ = SDL_CreateTexture(this->renderer_, SDL_PIXELFORMAT_RGB565, SDL_TEXTUREACCESS_STATIC, this->width_, this->height_); - SDL_SetTextureBlendMode(this->texture_, SDL_BLENDMODE_BLEND); + if (this->texture_ == nullptr) + return this->setup_failed_("Could not create texture"); + // The texture has no alpha channel, so blending is pointless. Headless it would also force a + // different software blit path onto the 16 bit target surface. + if (SDL_SetTextureBlendMode(this->texture_, this->headless_ ? SDL_BLENDMODE_NONE : SDL_BLENDMODE_BLEND) != 0) + return this->setup_failed_("Could not set texture blend mode"); + return true; } + +void Sdl::setup() { + if (!this->setup_renderer_()) { + this->mark_failed(); + return; + } + if (this->headless_) { + // Nothing generates events, so there is nothing for loop() to do. + this->disable_loop(); + } else if (this->snapshot_key_ != 0) { + this->add_key_listener(this->snapshot_key_, [this](bool down) { + if (down && !this->take_snapshot(nullptr)) { + ESP_LOGW(TAG, "snapshot key did not write a file"); + } + }); + } +} + void Sdl::update() { + if (this->texture_ == nullptr) + return; this->do_update_(); if ((this->x_high_ < this->x_low_) || (this->y_high_ < this->y_low_)) return; @@ -51,12 +139,19 @@ void Sdl::update() { } void Sdl::redraw_(SDL_Rect &rect) { + // Nothing to present when headless - a snapshot blits the whole texture when it needs it, so + // doing it here as well would just burn CPU. draw_pixels_at() calls this on every partial + // update, so it is worth skipping. + if (this->headless_) + return; SDL_RenderCopy(this->renderer_, this->texture_, &rect, &rect); SDL_RenderPresent(this->renderer_); } void Sdl::draw_pixels_at(int x_start, int y_start, int w, int h, const uint8_t *ptr, display::ColorOrder order, display::ColorBitness bitness, bool big_endian, int x_offset, int y_offset, int x_pad) { + if (this->texture_ == nullptr) + return; SDL_Rect rect{x_start, y_start, w, h}; if (this->rotation_ != display::DISPLAY_ROTATION_0_DEGREES || bitness != display::COLOR_BITNESS_565 || big_endian) { Display::draw_pixels_at(x_start, y_start, w, h, ptr, order, bitness, big_endian, x_offset, y_offset, x_pad); @@ -69,7 +164,7 @@ void Sdl::draw_pixels_at(int x_start, int y_start, int w, int h, const uint8_t * } void Sdl::draw_pixel_at(int x, int y, Color color) { - if (!this->get_clipping().inside(x, y)) + if (this->texture_ == nullptr || !this->get_clipping().inside(x, y)) return; if (this->rotation_ == display::DISPLAY_ROTATION_180_DEGREES) { @@ -104,61 +199,148 @@ void Sdl::process_key(uint32_t keycode, bool down) { callback->second(down); } +Sdl *Sdl::instance_for_window_(uint32_t window_id) { + SDL_Window *window = SDL_GetWindowFromID(window_id); + if (window == nullptr) + return nullptr; + return static_cast(SDL_GetWindowData(window, WINDOW_DATA_KEY)); +} + +void Sdl::handle_event_(const SDL_Event &event) { + switch (event.type) { + case SDL_MOUSEBUTTONDOWN: + case SDL_MOUSEBUTTONUP: + if (event.button.button == 1) { + this->mouse_x = event.button.x; + this->mouse_y = event.button.y; + this->mouse_down = event.button.state != 0; + } + break; + + case SDL_MOUSEMOTION: + if (event.motion.state & 1) { + this->mouse_x = event.motion.x; + this->mouse_y = event.motion.y; + this->mouse_down = true; + } else { + this->mouse_down = false; + } + break; + + case SDL_KEYDOWN: + // Ignore auto-repeat, otherwise holding a key floods the listeners. + if (event.key.repeat != 0) + break; + ESP_LOGD(TAG, "keydown %d", event.key.keysym.sym); + this->process_key(event.key.keysym.sym, true); + break; + + case SDL_KEYUP: + ESP_LOGD(TAG, "keyup %d", event.key.keysym.sym); + this->process_key(event.key.keysym.sym, false); + break; + + case SDL_WINDOWEVENT: + switch (event.window.event) { + case SDL_WINDOWEVENT_SIZE_CHANGED: + case SDL_WINDOWEVENT_EXPOSED: + case SDL_WINDOWEVENT_RESIZED: { + SDL_Rect rect{0, 0, this->width_, this->height_}; + this->redraw_(rect); + break; + } + default: + break; + } + break; + + default: + break; + } +} + void Sdl::loop() { SDL_Event e; - if (SDL_PollEvent(&e)) { - switch (e.type) { - case SDL_QUIT: - exit(0); + // Take everything that is waiting, not one event per loop. A touch drag produces a burst of + // motion events, and consuming them one at a time lets the queue grow without bound, so the + // pointer ends up acting on input from further and further in the past. Draining collapses a + // burst to the position it ended at, which is the one the user is asking for anyway. + while (SDL_PollEvent(&e)) { + if (e.type == SDL_QUIT) + exit(0); + // Events carry the window they happened in, so send each one to the display that owns it. + uint32_t window_id; + switch (e.type) { case SDL_MOUSEBUTTONDOWN: case SDL_MOUSEBUTTONUP: - if (e.button.button == 1) { - this->mouse_x = e.button.x; - this->mouse_y = e.button.y; - this->mouse_down = e.button.state != 0; - } + window_id = e.button.windowID; break; - case SDL_MOUSEMOTION: - if (e.motion.state & 1) { - this->mouse_x = e.button.x; - this->mouse_y = e.button.y; - this->mouse_down = true; - } else { - this->mouse_down = false; - } + window_id = e.motion.windowID; break; - case SDL_KEYDOWN: - ESP_LOGD(TAG, "keydown %d", e.key.keysym.sym); - this->process_key(e.key.keysym.sym, true); - break; - case SDL_KEYUP: - ESP_LOGD(TAG, "keyup %d", e.key.keysym.sym); - this->process_key(e.key.keysym.sym, false); + window_id = e.key.windowID; break; - case SDL_WINDOWEVENT: - switch (e.window.event) { - case SDL_WINDOWEVENT_SIZE_CHANGED: - case SDL_WINDOWEVENT_EXPOSED: - case SDL_WINDOWEVENT_RESIZED: { - SDL_Rect rect{0, 0, this->width_, this->height_}; - this->redraw_(rect); - break; - } - default: - break; - } + window_id = e.window.windowID; break; - default: + // Anything else, including the touch events SDL reports alongside the mouse events it + // synthesises from them, is not used here. ESP_LOGV(TAG, "Event %d", e.type); - break; + continue; + } + + Sdl *target = instance_for_window_(window_id); + if (target == nullptr) { + // Nothing to route this to: the window has gone, or it is not one of ours. Say so, otherwise + // input that stops working leaves no trace at all. + ESP_LOGV(TAG, "Event %d for unknown window %u", e.type, window_id); + continue; + } + target->handle_event_(e); + } +} + +bool Sdl::capture_bgr(uint8_t *dest, size_t row_stride) { + if (this->texture_ == nullptr || this->renderer_ == nullptr) { + ESP_LOGE(TAG, "Snapshot requested but SDL is not set up"); + return false; + } + if (this->shot_target_ == nullptr) { + this->shot_target_ = SDL_CreateTexture(this->renderer_, SDL_PIXELFORMAT_RGB565, SDL_TEXTUREACCESS_TARGET, + this->width_, this->height_); + if (this->shot_target_ == nullptr) { + ESP_LOGE(TAG, "Could not create capture texture: %s", SDL_GetError()); + return false; + } + SDL_SetTextureBlendMode(this->shot_target_, SDL_BLENDMODE_NONE); + } + + // Render into an offscreen target first. SDL_RenderReadPixels works in physical output pixels and + // ignores the logical size, so reading straight off a resizable window would read more pixels than + // there is room for. + // Every step is checked: a failed clear or copy would otherwise be read back as a blank or stale + // picture, written out, and reported as a snapshot that worked. + bool ok = false; + if (SDL_SetRenderTarget(this->renderer_, this->shot_target_) == 0) { + ok = SDL_SetRenderDrawColor(this->renderer_, 0, 0, 0, SDL_ALPHA_OPAQUE) == 0 && + SDL_RenderClear(this->renderer_) == 0 && + SDL_RenderCopy(this->renderer_, this->texture_, nullptr, nullptr) == 0 && + SDL_RenderReadPixels(this->renderer_, nullptr, SDL_PIXELFORMAT_BGR24, dest, static_cast(row_stride)) == 0; + if (SDL_SetRenderTarget(this->renderer_, nullptr) != 0) { + // Stuck rendering into shot_target_ from here on, so there's no point continuing. + ESP_LOGE(TAG, "Could not restore the render target: %s", SDL_GetError()); + this->mark_failed(); + return false; } } + if (!ok) { + ESP_LOGE(TAG, "Could not capture the screen: %s", SDL_GetError()); + } + return ok; } } // namespace esphome::sdl diff --git a/esphome/components/sdl/sdl_esphome.h b/esphome/components/sdl/sdl_esphome.h index 635eb1e3f8..54f0d2573f 100644 --- a/esphome/components/sdl/sdl_esphome.h +++ b/esphome/components/sdl/sdl_esphome.h @@ -1,10 +1,12 @@ #pragma once #ifdef USE_HOST +#include "esphome/core/automation.h" #include "esphome/core/component.h" #include "esphome/core/log.h" #include "esphome/core/application.h" #include "esphome/components/display/display.h" +#include "esphome/components/snapshot/snapshot.h" #define SDL_MAIN_HANDLED #include "SDL.h" #include @@ -13,7 +15,7 @@ namespace esphome::sdl { constexpr static const char *const TAG = "sdl"; -class Sdl final : public display::Display { +class Sdl final : public display::Display, public snapshot::Snapshot { public: display::DisplayType get_display_type() override { return display::DISPLAY_TYPE_COLOR; } void update() override; @@ -32,6 +34,9 @@ class Sdl final : public display::Display { this->pos_x_ = pos_x; this->pos_y_ = pos_y; } + void set_headless(bool headless) { this->headless_ = headless; } + void set_snapshot_key(int32_t keycode) { this->snapshot_key_ = keycode; } + int get_width() override; int get_height() override; float get_setup_priority() const override { return setup_priority::HARDWARE; } @@ -51,20 +56,40 @@ class Sdl final : public display::Display { int get_width_internal() override { return this->width_; } int get_height_internal() override { return this->height_; } void redraw_(SDL_Rect &rect); + bool setup_renderer_(); + /// Release the window, surface, renderer and textures, and forget them. + void destroy_renderer_(); + /// Log an SDL failure during setup, release anything already created, and return false. + bool setup_failed_(const char *what); + int snapshot_width() override { return this->width_; } + int snapshot_height() override { return this->height_; } + bool capture_bgr(uint8_t *dest, size_t row_stride) override; + void handle_event_(const SDL_Event &event); + /// The display owning the given window, or nullptr if it is not one of ours. + static Sdl *instance_for_window_(uint32_t window_id); + SDL_Renderer *renderer_{}; + SDL_Window *window_{}; + SDL_Texture *texture_{}; + // Offscreen render target used when headless. SDL_CreateSoftwareRenderer only borrows the + // surface, and the renderer goes back to using it as its output whenever the capture target is + // released, so it has to stay alive as long as the renderer does. + SDL_Surface *surface_{}; + // Capture target, created on first snapshot. + SDL_Texture *shot_target_{}; + std::map> key_callbacks_{}; int width_{}; int height_{}; uint32_t window_options_{0}; int32_t pos_x_{SDL_WINDOWPOS_UNDEFINED}; int32_t pos_y_{SDL_WINDOWPOS_UNDEFINED}; - SDL_Renderer *renderer_{}; - SDL_Window *window_{}; - SDL_Texture *texture_{}; + int32_t snapshot_key_{0}; uint16_t x_low_{0}; uint16_t y_low_{0}; uint16_t x_high_{0}; uint16_t y_high_{0}; - std::map> key_callbacks_{}; + bool headless_{false}; }; + } // namespace esphome::sdl #endif diff --git a/esphome/components/sdl/touchscreen/__init__.py b/esphome/components/sdl/touchscreen/__init__.py index d7af8da403..9b807b4585 100644 --- a/esphome/components/sdl/touchscreen/__init__.py +++ b/esphome/components/sdl/touchscreen/__init__.py @@ -4,10 +4,12 @@ import esphome.config_validation as cv from esphome.const import CONF_ID from esphome.types import ConfigType -from ..display import CONF_SDL_ID, Sdl, sdl_ns +from ..display import CONF_SDL_ID, Sdl, headless_final_validate, sdl_ns SdlTouchscreen = sdl_ns.class_("SdlTouchscreen", touchscreen.Touchscreen) +FINAL_VALIDATE_SCHEMA = headless_final_validate("touchscreen") + CONFIG_SCHEMA = touchscreen.TOUCHSCREEN_SCHEMA.extend( { diff --git a/esphome/components/snapshot/__init__.py b/esphome/components/snapshot/__init__.py new file mode 100644 index 0000000000..bf561a0e0d --- /dev/null +++ b/esphome/components/snapshot/__init__.py @@ -0,0 +1,76 @@ +"""Shared support for writing what a display is showing out to an image file. + +The component itself has no configuration. It provides the ``snapshot.take`` action and the C++ +base class behind it, so any display that can hand over its pixels - the in memory display in this +component, or an SDL window - saves files the same way, under the same directory, with the same +rules about names. +""" + +from dataclasses import dataclass + +from esphome import automation +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome.const import CONF_ID +from esphome.core import CORE, ID +from esphome.cpp_generator import MockObj +from esphome.types import ConfigType, TemplateArgsType + +CODEOWNERS = ["@clydebarrow"] + +DOMAIN = "snapshot" + +CONF_FILENAME = "filename" + +snapshot_ns = cg.esphome_ns.namespace("snapshot") +Snapshot = snapshot_ns.class_("Snapshot") +SnapshotAction = snapshot_ns.class_("SnapshotAction", automation.Action) + + +@automation.register_action( + "snapshot.take", + SnapshotAction, + automation.maybe_simple_id( + { + cv.GenerateID(): cv.use_id(Snapshot), + cv.Optional(CONF_FILENAME): cv.templatable(cv.string), + } + ), + synchronous=True, +) +async def snapshot_take_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + if (filename := config.get(CONF_FILENAME)) is not None: + cg.add(var.set_filename(await cg.templatable(filename, args, cg.std_string))) + return var + + +@dataclass +class SnapshotData: + directory_defined: bool = False + + +def _get_data() -> SnapshotData: + if DOMAIN not in CORE.data: + CORE.data[DOMAIN] = SnapshotData() + return CORE.data[DOMAIN] + + +async def register_snapshot(var: MockObj, config: ConfigType) -> None: + """Set up a component so that the snapshot action can write its picture to a file.""" + data = _get_data() + # Only once, however many displays there are: two defines that say the same thing do not + # compare equal, so asking for this per display repeats the line in defines.h. + if not data.directory_defined: + data.directory_defined = True + cg.add_define( + "ESPHOME_SNAPSHOT_DIR", + (CORE.data_dir / "snapshots" / CORE.name).as_posix(), + ) + cg.add(var.set_snapshot_prefix(str(config[CONF_ID]))) diff --git a/esphome/components/snapshot/display/__init__.py b/esphome/components/snapshot/display/__init__.py new file mode 100644 index 0000000000..68429f164b --- /dev/null +++ b/esphome/components/snapshot/display/__init__.py @@ -0,0 +1,61 @@ +import esphome.codegen as cg +from esphome.components import display +import esphome.config_validation as cv +from esphome.const import ( + CONF_DIMENSIONS, + CONF_HEIGHT, + CONF_ID, + CONF_LAMBDA, + CONF_WIDTH, + PLATFORM_HOST, +) +from esphome.types import ConfigType + +from .. import Snapshot, register_snapshot, snapshot_ns + +# The base class and the file writing live in the parent component, which nothing else in a +# configuration using only this platform would pull in. +AUTO_LOAD = ["snapshot"] + +SnapshotDisplay = snapshot_ns.class_( + "SnapshotDisplay", display.DisplayBuffer, cg.Component, Snapshot +) + +CONFIG_SCHEMA = cv.All( + display.FULL_DISPLAY_SCHEMA.extend( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(SnapshotDisplay), + cv.Required(CONF_DIMENSIONS): cv.Any( + cv.dimensions, + cv.Schema( + { + cv.Required(CONF_WIDTH): cv.positive_not_null_int, + cv.Required(CONF_HEIGHT): cv.positive_not_null_int, + } + ), + ), + } + ) + ), + cv.only_on(PLATFORM_HOST), +) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await display.register_display(var, config) + await register_snapshot(var, config) + + dimensions = config[CONF_DIMENSIONS] + if isinstance(dimensions, dict): + cg.add(var.set_dimensions(dimensions[CONF_WIDTH], dimensions[CONF_HEIGHT])) + else: + (width, height) = dimensions + cg.add(var.set_dimensions(width, height)) + + if lamb := config.get(CONF_LAMBDA): + lambda_ = await cg.process_lambda( + lamb, [(display.DisplayRef, "it")], return_type=cg.void + ) + cg.add(var.set_writer(lambda_)) diff --git a/esphome/components/snapshot/display/snapshot_display.cpp b/esphome/components/snapshot/display/snapshot_display.cpp new file mode 100644 index 0000000000..6297e3e18f --- /dev/null +++ b/esphome/components/snapshot/display/snapshot_display.cpp @@ -0,0 +1,80 @@ +#ifdef USE_HOST +#include "snapshot_display.h" +#include "esphome/components/display/display_color_utils.h" +#include "esphome/core/log.h" + +#include + +namespace esphome::snapshot { + +static const char *const TAG = "snapshot.display"; + +namespace { + +/// Spread a channel that only goes up to `max` over the whole 0 to 255 range, so that the +/// brightest value stays the brightest. This is the same arithmetic SDL uses, which is what makes +/// a picture taken here come out identical to the same picture taken from an SDL window. +constexpr uint8_t expand_channel(uint16_t value, uint16_t max) { return static_cast(value * 255 / max); } + +constexpr uint16_t RED_MAX = 0x1F; +constexpr uint16_t GREEN_MAX = 0x3F; +constexpr uint16_t BLUE_MAX = 0x1F; + +} // namespace + +void SnapshotDisplay::setup() { + this->init_internal_(static_cast(this->width_) * this->height_ * 2); + if (this->buffer_ == nullptr) { + this->mark_failed(LOG_STR("Could not allocate display buffer")); + } +} + +void SnapshotDisplay::dump_config() { LOG_DISPLAY("", "Snapshot", this); } + +void SnapshotDisplay::draw_absolute_pixel_internal(int x, int y, Color color) { + if (this->buffer_ == nullptr || x < 0 || x >= this->width_ || y < 0 || y >= this->height_) + return; + this->pixels_()[y * this->width_ + x] = display::ColorUtil::color_to_565(color, display::COLOR_ORDER_RGB); +} + +void SnapshotDisplay::draw_pixels_at(int x_start, int y_start, int w, int h, const uint8_t *ptr, + display::ColorOrder order, display::ColorBitness bitness, bool big_endian, + int x_offset, int y_offset, int x_pad) { + if (this->buffer_ == nullptr) + return; + // Anything that is not already laid out the way the buffer is, or that would reach outside it, + // goes through the base class, which turns it into one call per pixel with the bounds checked. + const bool copyable = this->rotation_ == display::DISPLAY_ROTATION_0_DEGREES && + bitness == display::COLOR_BITNESS_565 && !big_endian && x_start >= 0 && y_start >= 0 && + x_start + w <= this->width_ && y_start + h <= this->height_; + if (!copyable) { + DisplayBuffer::draw_pixels_at(x_start, y_start, w, h, ptr, order, bitness, big_endian, x_offset, y_offset, x_pad); + return; + } + const size_t stride = static_cast(x_offset) + w + x_pad; + const uint8_t *src = ptr + (stride * y_offset + x_offset) * 2; + for (int y = 0; y != h; y++) { + memcpy(&this->pixels_()[(y_start + y) * this->width_ + x_start], src + y * stride * 2, w * 2); + } +} + +bool SnapshotDisplay::capture_bgr(uint8_t *dest, size_t row_stride) { + if (this->buffer_ == nullptr) { + ESP_LOGE(TAG, "Snapshot requested but there is no buffer to read"); + return false; + } + const uint16_t *src = this->pixels_(); + for (int y = 0; y != this->height_; y++) { + uint8_t *out = dest + y * row_stride; + for (int x = 0; x != this->width_; x++) { + const uint16_t pixel = *src++; + *out++ = expand_channel(pixel & BLUE_MAX, BLUE_MAX); + *out++ = expand_channel((pixel >> 5) & GREEN_MAX, GREEN_MAX); + *out++ = expand_channel(pixel >> 11, RED_MAX); + } + } + return true; +} + +} // namespace esphome::snapshot +#endif diff --git a/esphome/components/snapshot/display/snapshot_display.h b/esphome/components/snapshot/display/snapshot_display.h new file mode 100644 index 0000000000..5317bc6058 --- /dev/null +++ b/esphome/components/snapshot/display/snapshot_display.h @@ -0,0 +1,48 @@ +#pragma once + +#ifdef USE_HOST +#include "esphome/components/display/display_buffer.h" +#include "esphome/components/snapshot/snapshot.h" +#include "esphome/core/component.h" + +namespace esphome::snapshot { + +/// A display with nowhere to show anything: it keeps the picture in memory, where the snapshot +/// action can pick it up. That makes it a way to see what a configuration draws on a machine with +/// no screen, and to check the result in a test. +class SnapshotDisplay final : public display::DisplayBuffer, public Snapshot { + public: + void setup() override; + void update() override { this->do_update_(); } + void dump_config() override; + float get_setup_priority() const override { return setup_priority::HARDWARE; } + display::DisplayType get_display_type() override { return display::DISPLAY_TYPE_COLOR; } + + void set_dimensions(uint16_t width, uint16_t height) { + this->width_ = width; + this->height_ = height; + } + + void draw_pixels_at(int x_start, int y_start, int w, int h, const uint8_t *ptr, display::ColorOrder order, + display::ColorBitness bitness, bool big_endian, int x_offset, int y_offset, int x_pad) override; + + protected: + void draw_absolute_pixel_internal(int x, int y, Color color) override; + int get_width_internal() override { return this->width_; } + int get_height_internal() override { return this->height_; } + + int snapshot_width() override { return this->width_; } + int snapshot_height() override { return this->height_; } + bool capture_bgr(uint8_t *dest, size_t row_stride) override; + + /// The picture, one 16 bit RGB565 value per pixel, topmost row first. Owned by DisplayBuffer as + /// a byte pointer; this is the same memory seen as what is actually stored in it. + uint16_t *pixels_() { return reinterpret_cast(this->buffer_); } + + int width_{}; + int height_{}; +}; + +} // namespace esphome::snapshot + +#endif diff --git a/esphome/components/snapshot/snapshot.cpp b/esphome/components/snapshot/snapshot.cpp new file mode 100644 index 0000000000..995f87710e --- /dev/null +++ b/esphome/components/snapshot/snapshot.cpp @@ -0,0 +1,248 @@ +#ifdef USE_HOST +#include "snapshot.h" +#include "esphome/core/log.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace esphome::snapshot { + +namespace { + +constexpr const char *const TAG = "snapshot"; + +// Longest name we will build a path from. NAME_MAX is 255 and we may append a collision suffix. +constexpr size_t MAX_NAME_LENGTH = 200; +// Give up rather than spin forever if every candidate name is taken. +constexpr unsigned MAX_NAME_ATTEMPTS = 1000; +// A BMP file header followed by a BITMAPINFOHEADER, which is where the pixels start. +constexpr size_t BMP_HEADER_SIZE = 54; +constexpr size_t BMP_INFO_HEADER_SIZE = 40; +constexpr int BMP_BITS_PER_PIXEL = 24; + +/// True if the name already ends in ".bmp". The comparison ignores case, so "shot.BMP" is left +/// alone rather than turned into "shot.BMP.bmp". +bool has_bmp_suffix(const std::string &name) { + return name.size() >= 4 && strcasecmp(name.c_str() + name.size() - 4, ".bmp") == 0; +} + +/// Reduce a user supplied name to a single safe path component. Everything outside the allowed set +/// is replaced, so "..", "/" and absolute paths cannot escape the snapshot directory. +/// Returns an empty string if nothing usable is left. +std::string sanitise_filename(const char *const name, bool *name_changed) { + std::string result; + bool all_dots = true; + bool changed = false; + for (const char *p = name; *p != '\0'; p++) { + if (result.size() >= MAX_NAME_LENGTH) { + changed = true; + break; + } + char c = *p; + if (!(std::isalnum(static_cast(c)) || c == '.' || c == '_' || c == '-')) { + c = '_'; + changed = true; + } + if (c != '.') + all_dots = false; + result.push_back(c); + } + if (all_dots) { + *name_changed = true; + return ""; + } + if (!has_bmp_suffix(result)) + result += ".bmp"; + *name_changed = changed; + return result; +} + +/// Insert "-" before the file extension, e.g. "shot.bmp" -> "shot-1.bmp". +std::string add_suffix(const std::string &name, unsigned attempt) { + char suffix[12]; + snprintf(suffix, sizeof(suffix), "-%u", attempt); + auto dot = name.rfind('.'); + if (dot == std::string::npos) + return name + suffix; + return name.substr(0, dot) + suffix + name.substr(dot); +} + +/// Directory snapshots are written to. The environment variable lets a test redirect output +/// without rebuilding, matching how the host platform handles ESPHOME_PREFDIR. +const char *snapshot_dir() { + const char *dir = getenv("ESPHOME_SNAPSHOT_DIR"); // NOLINT(concurrency-mt-unsafe) + return dir != nullptr && dir[0] != '\0' ? dir : ESPHOME_SNAPSHOT_DIR; +} + +/// Store a value in as many bytes, least significant first, and step the pointer past it. +/// BMP is a little endian format whatever the machine writing it uses. +void put_le(uint8_t *&dest, uint32_t value, size_t bytes) { + for (size_t i = 0; i != bytes; i++) + *dest++ = static_cast(value >> (8 * i)); +} + +/// The number of bytes one row of `width` pixels takes up in the file. Rows are padded out to a +/// multiple of four bytes. +size_t bmp_row_size(int width) { return (static_cast(width) * 3 + 3) & ~size_t{3}; } + +/// Write pixels out as a 24 bit BMP. The rows given start with the topmost and are `row_stride` +/// bytes apart, which must leave room for a whole padded row; a BMP holds its rows the other way +/// up, so they go out last first. +bool write_bmp(FILE *file, const uint8_t *pixels, int width, int height, size_t row_stride) { + const size_t row_size = bmp_row_size(width); + const size_t pixel_bytes = row_size * height; + + uint8_t header[BMP_HEADER_SIZE]; + uint8_t *pos = header; + *pos++ = 'B'; + *pos++ = 'M'; + put_le(pos, static_cast(BMP_HEADER_SIZE + pixel_bytes), 4); + put_le(pos, 0, 4); // reserved + put_le(pos, BMP_HEADER_SIZE, 4); + put_le(pos, BMP_INFO_HEADER_SIZE, 4); + put_le(pos, static_cast(width), 4); + put_le(pos, static_cast(height), 4); + put_le(pos, 1, 2); // one plane + put_le(pos, BMP_BITS_PER_PIXEL, 2); + put_le(pos, 0, 4); // not compressed + put_le(pos, static_cast(pixel_bytes), 4); + put_le(pos, 0, 4); // pixels per metre across, unspecified + put_le(pos, 0, 4); // pixels per metre down, unspecified + put_le(pos, 0, 4); // no palette + put_le(pos, 0, 4); // so no palette entry matters more than another + + if (fwrite(header, 1, sizeof(header), file) != sizeof(header)) + return false; + for (int y = height - 1; y >= 0; y--) { + if (fwrite(pixels + static_cast(y) * row_stride, 1, row_size, file) != row_size) + return false; + } + return true; +} + +/// Reserve a name in the snapshot directory and write the picture to it. +/// With `exact` set the given name is the only one tried; otherwise a number is added on +/// collision. Returns true if a file was written. +bool write_snapshot_file(const uint8_t *pixels, int width, int height, size_t row_stride, const std::string &name, + bool exact) { + const std::string dir = snapshot_dir(); + std::error_code ec; + std::filesystem::create_directories(dir, ec); + if (ec) { + ESP_LOGE(TAG, "Could not create snapshot directory %s: %s", dir.c_str(), ec.message().c_str()); + return false; + } + + // O_EXCL guarantees we never write over a file that is already there. + std::string path; + int fd = -1; + for (unsigned attempt = 0; attempt < MAX_NAME_ATTEMPTS; attempt++) { + path = dir + "/" + (attempt == 0 ? name : add_suffix(name, attempt)); + fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, 0644); + if (fd >= 0) + break; + if (errno != EEXIST) { + ESP_LOGE(TAG, "Could not create %s: %s", path.c_str(), strerror(errno)); + return false; + } + if (exact) { + // The caller asked for this exact name, so silently writing somewhere else would be worse + // than failing - a test asserting on the path would pick up a stale file. + ESP_LOGE(TAG, "Snapshot %s already exists, not overwriting", path.c_str()); + return false; + } + } + if (fd < 0) { + ESP_LOGE(TAG, "Could not find an unused name for %s in %s", name.c_str(), dir.c_str()); + return false; + } + + FILE *file = fdopen(fd, "wb"); + if (file == nullptr) { + ESP_LOGE(TAG, "Could not open %s: %s", path.c_str(), strerror(errno)); + ::close(fd); + ::unlink(path.c_str()); + return false; + } + bool ok = write_bmp(file, pixels, width, height, row_stride); + int saved_errno = ok ? 0 : errno; + // Closing can fail in its own right - the last of the data is still on its way out. + if (fclose(file) != 0) { + if (ok) + saved_errno = errno; + ok = false; + } + if (!ok) { + ESP_LOGE(TAG, "Could not write %s: %s", path.c_str(), strerror(saved_errno)); + // Leave no truncated file behind - it would block a retry under the same name. + ::unlink(path.c_str()); + return false; + } + ESP_LOGI(TAG, "Snapshot written to %s", path.c_str()); + return true; +} + +} // namespace + +// helper function since ESP_LOGW is disallowed in a header file +void Snapshot::log_action_failed() { ESP_LOGW(TAG, "snapshot.take did not write a file"); } + +bool Snapshot::take_snapshot(const char *filename) { + const int width = this->snapshot_width(); + const int height = this->snapshot_height(); + if (width <= 0 || height <= 0) { + ESP_LOGE(TAG, "Snapshot requested but the display is %dx%d", width, height); + return false; + } + + std::string name; + bool exact = false; + if (filename != nullptr) { + bool name_changed = false; + name = sanitise_filename(filename, &name_changed); + exact = !name.empty(); + if (name_changed) { + ESP_LOGW(TAG, "Requested snapshot name '%s' is not an acceptable file name, using '%s' instead", filename, + name.empty() ? "a name made from the time" : name.c_str()); + } + } + if (name.empty()) { + struct timespec now {}; + if (clock_gettime(CLOCK_REALTIME, &now) != 0) + now = {}; + struct tm tm_buf {}; + if (localtime_r(&now.tv_sec, &tm_buf) == nullptr) + tm_buf = {}; + char stamp[32]{}; + // ::strftime to be sure of the one from ; display has an unrelated member of that name + if (::strftime(stamp, sizeof(stamp), "%Y%m%d-%H%M%S", &tm_buf) == 0) + snprintf(stamp, sizeof(stamp), "unknown-time"); + char buffer[MAX_NAME_LENGTH]; + int written = + snprintf(buffer, sizeof(buffer), "%s-%s-%03ld.bmp", this->snapshot_prefix_, stamp, now.tv_nsec / 1000000); + if (written < 0 || static_cast(written) >= sizeof(buffer)) { + ESP_LOGW(TAG, "Could not build a timestamped snapshot name, using a fallback"); + snprintf(buffer, sizeof(buffer), "snapshot.bmp"); + } + name = buffer; + } + + // Rows are padded out to a multiple of four bytes, as the file wants them, so each one can be + // written straight from the buffer. Zeroed on allocation, which is what the padding must be. + const size_t row_stride = bmp_row_size(width); + auto pixels = std::make_unique(row_stride * height); + if (!this->capture_bgr(pixels.get(), row_stride)) + return false; + return write_snapshot_file(pixels.get(), width, height, row_stride, name, exact); +} + +} // namespace esphome::snapshot +#endif diff --git a/esphome/components/snapshot/snapshot.h b/esphome/components/snapshot/snapshot.h new file mode 100644 index 0000000000..bb670e639f --- /dev/null +++ b/esphome/components/snapshot/snapshot.h @@ -0,0 +1,72 @@ +#pragma once + +#ifdef USE_HOST +#include "esphome/core/automation.h" + +#include +#include +#include + +// Directory snapshots are written to. Normally set by codegen to a folder under .esphome; the +// fallback keeps the component compiling for static analysis, where no defines.h is generated. +#ifndef ESPHOME_SNAPSHOT_DIR +#define ESPHOME_SNAPSHOT_DIR "." +#endif + +namespace esphome::snapshot { + +/// Base for anything that can hand over the picture it is showing so it can be written to a file. +/// +/// A subclass says how big the picture is and fills in the pixels. Everything else - picking a +/// name, staying inside the snapshot directory, not writing over anything, and encoding the file - +/// is done here, so every component that can take a snapshot behaves the same way. +class Snapshot { + public: + virtual ~Snapshot() = default; + + /// Set the word generated names start with. Codegen passes the component id, so with more than + /// one display in a device it is clear which one a file came from. + void set_snapshot_prefix(const char *prefix) { this->snapshot_prefix_ = prefix; } + + /// Write the current picture to a BMP file in the snapshot directory. + /// + /// Pass nullptr to have a name made up from the prefix and the current time. A file that is + /// already there is never written over. Returns true if a file was written. + bool take_snapshot(const char *filename); + + /// Log that an action-triggered snapshot did not write a file. + static void log_action_failed(); + + protected: + /// Width of the picture in pixels. + virtual int snapshot_width() = 0; + /// Height of the picture in pixels. + virtual int snapshot_height() = 0; + /// Fill in the picture: three bytes per pixel in blue, green, red order, topmost row first, with + /// `row_stride` bytes from the start of one row to the start of the next. Returns false, having + /// logged why, if the picture could not be read. + virtual bool capture_bgr(uint8_t *dest, size_t row_stride) = 0; + + const char *snapshot_prefix_{"snapshot"}; +}; + +template class SnapshotAction final : public Action, public Parented { + public: + TEMPLATABLE_VALUE(std::string, filename) + + protected: + void play(const Ts &...x) override { + bool ok; + if (this->filename_.has_value()) { + ok = this->parent_->take_snapshot(this->filename_.value(x...).c_str()); + } else { + ok = this->parent_->take_snapshot(nullptr); + } + if (!ok) + this->parent_->log_action_failed(); + } +}; + +} // namespace esphome::snapshot + +#endif diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 7af41409fd..526adf74f0 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -13,6 +13,7 @@ #define ESPHOME_PROJECT_VERSION "v2" #define ESPHOME_PROJECT_VERSION_30 "v2" #define ESPHOME_VARIANT "ESP32" +#define ESPHOME_SNAPSHOT_DIR "." #define ESPHOME_NAME_ADD_MAC_SUFFIX #define ESPHOME_DEBUG_SCHEDULER #define ESPHOME_DEBUG_API diff --git a/tests/component_tests/sdl/test_sdl.py b/tests/component_tests/sdl/test_sdl.py new file mode 100644 index 0000000000..5ab5e17ee6 --- /dev/null +++ b/tests/component_tests/sdl/test_sdl.py @@ -0,0 +1,101 @@ +"""Tests for the sdl display schema, in particular the headless option.""" + +from __future__ import annotations + +import pytest + +from esphome import config_validation as cv +from esphome.components.sdl.display import ( + CONF_SDL_ID, + CONFIG_SCHEMA, + headless_final_validate, +) +from esphome.config import Config +from esphome.const import PlatformFramework +from esphome.core import ID +from esphome.final_validate import full_config +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + + +@pytest.fixture(autouse=True) +def _host_platform(set_core_config: SetCoreConfigCallable) -> None: + set_core_config(PlatformFramework.HOST_NATIVE) + + +def _config(**extra: object) -> ConfigType: + config: ConfigType = { + "dimensions": {"width": 320, "height": 240}, + # sdl2-config is not necessarily installed in the test environment + "sdl_options": "-lSDL2", + } + config.update(extra) + return config + + +def test_defaults_to_windowed() -> None: + """A display without the option is not headless.""" + assert CONFIG_SCHEMA(_config())["headless"] is False + + +def test_headless_accepted() -> None: + """A headless display needs nothing beyond the dimensions.""" + assert CONFIG_SCHEMA(_config(headless=True))["headless"] is True + + +def test_headless_rejects_window_options() -> None: + """Window options are meaningless without a window.""" + with pytest.raises(cv.Invalid, match="has no effect"): + CONFIG_SCHEMA( + _config(headless=True, window_options={"position": {"x": 0, "y": 0}}) + ) + + +def test_headless_rejects_snapshot_key() -> None: + """A headless display has no keyboard, so the action is the only way in.""" + with pytest.raises(cv.Invalid, match="snapshot.take"): + CONFIG_SCHEMA(_config(headless=True, snapshot_key="SDLK_F12")) + + +def test_snapshot_key_accepted_when_windowed() -> None: + """The key is only valid alongside a window.""" + config = CONFIG_SCHEMA(_config(snapshot_key="SDLK_F12")) + assert str(config["snapshot_key"]) == "SDLK_F12" + + +def _declare_sdl_display(headless: bool) -> ID: + """Register a full_config with a single sdl display declaration and return a reference to it. + + Mirrors what the real config pipeline leaves behind: a "display" domain entry plus a + declare_ids record id_declaration_match_schema uses to find it again. + """ + declared_id = ID("my_sdl", is_declaration=True) + fc = Config() + fc["display"] = [ + { + "platform": "sdl", + "id": declared_id, + "headless": headless, + "dimensions": {"width": 320, "height": 240}, + } + ] + fc.declare_ids.append((declared_id, ["display", 0, "id"])) + full_config.set(fc) + return ID("my_sdl") + + +@pytest.mark.parametrize("platform", ["binary_sensor", "touchscreen"]) +def test_headless_final_validate_rejects_headless_display(platform: str) -> None: + """binary_sensor and touchscreen both need a window, so a headless display is rejected.""" + sdl_ref = _declare_sdl_display(headless=True) + schema = headless_final_validate(platform) + with pytest.raises(cv.Invalid, match="needs a window"): + schema({CONF_SDL_ID: sdl_ref}) + + +@pytest.mark.parametrize("platform", ["binary_sensor", "touchscreen"]) +def test_headless_final_validate_accepts_windowed_display(platform: str) -> None: + """The same platforms are accepted once the display has a window.""" + sdl_ref = _declare_sdl_display(headless=False) + schema = headless_final_validate(platform) + schema({CONF_SDL_ID: sdl_ref}) # Should not raise. diff --git a/tests/components/sdl/common.yaml b/tests/components/sdl/common.yaml index 3be86cf8be..1bb0434057 100644 --- a/tests/components/sdl/common.yaml +++ b/tests/components/sdl/common.yaml @@ -14,6 +14,15 @@ display: position: x: 100 y: 100 + snapshot_key: SDLK_F12 + + - platform: sdl + id: headless_display + headless: true + show_test_card: true + dimensions: + width: 320 + height: 240 - platform: sdl id: second_display @@ -46,3 +55,21 @@ binary_sensor: sdl_id: sdl_sdl_display id: key_enter key: SDLK_RETURN + +esphome: + # A name of your own is only good for one snapshot - a second one under the same name fails + # rather than writing over the first - so these run once rather than on a repeating interval. + on_boot: + - delay: 2s + - snapshot.take: + id: headless_display + filename: test_card.bmp + - snapshot.take: + id: headless_display + filename: !lambda 'return "shot.bmp";' + +interval: + # A generated name has the time in it, so this one can repeat. + - interval: 10s + then: + - snapshot.take: sdl_sdl_display diff --git a/tests/components/sdl/validate.host.yaml b/tests/components/sdl/validate.host.yaml new file mode 100644 index 0000000000..883f34675d --- /dev/null +++ b/tests/components/sdl/validate.host.yaml @@ -0,0 +1,29 @@ +# Config-only test for the headless and screenshot options. The combinations that must be +# rejected are covered by tests/component_tests/sdl/test_sdl.py; this file checks that the +# accepted forms validate together. +host: + mac_address: "62:23:45:AF:B3:DD" + +display: + - platform: sdl + id: headless_display + headless: true + dimensions: 320x240 + + - platform: sdl + id: windowed_display + dimensions: 320x240 + snapshot_key: SDLK_F12 + +binary_sensor: + - platform: sdl + sdl_id: windowed_display + id: key_up + key: SDLK_UP + +interval: + - interval: 10s + then: + - snapshot.take: + id: headless_display + filename: periodic.bmp diff --git a/tests/components/snapshot/common.yaml b/tests/components/snapshot/common.yaml new file mode 100644 index 0000000000..9ce2d33a87 --- /dev/null +++ b/tests/components/snapshot/common.yaml @@ -0,0 +1,34 @@ +display: + - platform: snapshot + id: snapshot_display + update_interval: 1s + show_test_card: true + # An odd width exercises the row padding in the BMP writer + dimensions: + width: 101 + height: 64 + + - platform: snapshot + id: snapshot_rotated + rotation: 90 + dimensions: 320x240 + lambda: |- + it.filled_rectangle(0, 0, 40, 20, Color(0xFF, 0x80, 0x00)); + +esphome: + # A name of your own is only good for one snapshot - a second one under the same name fails + # rather than writing over the first - so these run once rather than on a repeating interval. + on_boot: + - delay: 2s + - snapshot.take: + id: snapshot_display + filename: test_card.bmp + - snapshot.take: + id: snapshot_rotated + filename: !lambda 'return "rotated.bmp";' + +interval: + # A generated name has the time in it, so this one can repeat. + - interval: 10s + then: + - snapshot.take: snapshot_display diff --git a/tests/components/snapshot/test.host.yaml b/tests/components/snapshot/test.host.yaml new file mode 100644 index 0000000000..951be2ed04 --- /dev/null +++ b/tests/components/snapshot/test.host.yaml @@ -0,0 +1,5 @@ +host: + mac_address: "62:23:45:AF:B3:DE" + +packages: + snapshot: !include common.yaml diff --git a/tests/integration/artifact_utils.py b/tests/integration/artifact_utils.py new file mode 100644 index 0000000000..cf18946512 --- /dev/null +++ b/tests/integration/artifact_utils.py @@ -0,0 +1,26 @@ +"""Shared utilities for ESPHome integration tests - keeping output from failing tests.""" + +from __future__ import annotations + +from pathlib import Path + +#: Where a failing test leaves output for someone to look at afterwards. pytest's own +#: temporary folder is no use on a CI runner, which throws the whole workspace away when +#: the job ends; the workflow uploads this folder instead when a job fails. +ARTIFACT_DIR = Path(__file__).resolve().parents[2] / "test_artifacts" + + +def keep_artifact(name: str, data: bytes) -> Path: + """Write ``data`` where it can still be read after the run, and return the path. + + Args: + name: File name to write under the artifact folder. + data: Contents to write. + + Returns: + The full path written. + """ + ARTIFACT_DIR.mkdir(parents=True, exist_ok=True) + path = ARTIFACT_DIR / name + path.write_bytes(data) + return path diff --git a/tests/integration/bmp_utils.py b/tests/integration/bmp_utils.py new file mode 100644 index 0000000000..c10aea5ade --- /dev/null +++ b/tests/integration/bmp_utils.py @@ -0,0 +1,161 @@ +"""Shared utilities for ESPHome integration tests - reading BMP snapshots.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from pathlib import Path +import struct + +# Size of the smallest BMP header pair (file header plus BITMAPINFOHEADER). +_MIN_HEADER_SIZE = 54 + +# How long capture_when_drawn() keeps asking for a picture with something on it. +DRAW_TIMEOUT = 15.0 + + +@dataclass(frozen=True) +class Bmp: + """A decoded BMP image.""" + + width: int + height: int + bits: int + #: Pixel data with the per row padding stripped, so it depends only on the image itself. + pixels: bytes + + +class NotABmpError(Exception): + """The data is not a BMP at all, as opposed to a BMP that is still being written.""" + + +def parse_bmp(data: bytes) -> Bmp | None: + """Decode a BMP, or return None if the data is not a complete image yet. + + Raises: + NotABmpError: If the data cannot become a valid BMP however much more is appended. + """ + # Writes go to the file in order, so a short read is always a prefix of what will be there. + # Anything wrong in a prefix we have already read is wrong for good, and worth saying now + # rather than reporting as a timeout later. + if len(data) >= 2 and data[:2] != b"BM": + raise NotABmpError(f"expected a BMP, got {data[:2]!r}") + if len(data) < _MIN_HEADER_SIZE: + return None + file_size = struct.unpack_from(" Bmp: + """Wait for a complete BMP file to appear at ``path`` and return it. + + The file is created before any of its contents are written, so waiting for it to exist is + not enough - a read that wins the race sees a truncated image. Keep reading until the + headers say the whole image is there. + + Args: + path: The file to wait for. + timeout: Maximum time to wait in seconds. + + Returns: + The decoded image. + + Raises: + AssertionError: If no complete image is readable within ``timeout``. + NotABmpError: If what was written is not a BMP. This is reported as soon as it is + seen, so a device that writes the wrong thing is named for what it did rather + than waiting out the timeout. + """ + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while True: + try: + data = path.read_bytes() + except FileNotFoundError: + data = b"" + if (image := parse_bmp(data)) is not None: + return image + if loop.time() >= deadline: + break + await asyncio.sleep(0.05) + if not data: + raise AssertionError(f"no snapshot appeared at {path} within {timeout}s") + raise AssertionError( + f"{path} was still incomplete after {timeout}s ({len(data)} bytes)" + ) + + +def is_blank(image: Bmp) -> bool: + """True if every pixel of the image is the same colour. + + Whole pixels are counted rather than byte values: a plain background is usually made of more + than one distinct byte, so counting bytes would find several of them in a blank screen. + """ + return len({image.pixels[i : i + 3] for i in range(0, len(image.pixels), 3)}) <= 1 + + +async def capture_when_drawn( + take: Callable[[str], Awaitable[None]], + directory: Path, + prefix: str = "drawn", + timeout: float = DRAW_TIMEOUT, +) -> tuple[Bmp, Path]: + """Ask for snapshots until one has something drawn on it, and return it and where it went. + + A display holds one flat colour until it first draws, which is one update interval after it + starts - long enough that a test connecting over the API can easily get in first. Capturing + once and hoping would compare a blank screen against whatever the test expects, reporting a + drawing fault where the real trouble was timing. + + Args: + take: Asks the device for a snapshot under the name it is given. + directory: Where the device writes them. + prefix: Start of the names asked for. Each attempt needs its own, because a snapshot never + writes over a file that is already there. + timeout: How long to keep asking. + + Returns: + The first image that is not one flat colour, and the path it was read from. + + Raises: + AssertionError: If nothing had been drawn within ``timeout``. + """ + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + attempt = 0 + while True: + attempt += 1 + path = directory / f"{prefix}-{attempt}.bmp" + await take(path.name) + image = await wait_for_bmp(path) + if not is_blank(image): + return image, path + if loop.time() >= deadline: + raise AssertionError( + f"the screen was still a single flat colour after {timeout}s and " + f"{attempt} captures - nothing was drawn" + ) + await asyncio.sleep(0.5) diff --git a/tests/integration/fixtures/lvgl_headless_render.yaml b/tests/integration/fixtures/lvgl_headless_render.yaml new file mode 100644 index 0000000000..670b51ab53 --- /dev/null +++ b/tests/integration/fixtures/lvgl_headless_render.yaml @@ -0,0 +1,53 @@ +esphome: + name: lvgl-headless-render-test +host: + +api: + actions: + # The name comes from the test so it can capture more than once: a snapshot never writes over + # a file that is already there, so a fixed name could only ever be captured once. + - action: take_screenshot + variables: + name: string + then: + - snapshot.take: + id: lvgl_display + filename: !lambda return name; + +logger: + level: DEBUG + +display: + # A display with no screen, so what LVGL draws depends on LVGL alone - nothing about the machine + # running the test, and no graphics library outside this repository, can move the result. + - platform: snapshot + id: lvgl_display + auto_clear_enabled: false + dimensions: + width: 300 + height: 300 + +# The widgets are spelled out here rather than left to the built in "Hello World" screen, which +# LVGL builds when nothing is configured: that screen contains a spinner, and an animation cannot +# produce the same picture twice. +# +# Everything that affects the rendered pixels is set explicitly, so the expected hash in the test +# depends only on the drawing code and the built in font. In particular the background comes from a +# full screen object rather than from the theme, so adjusting a theme default does not break this. +lvgl: + displays: lvgl_display + default_font: montserrat_14 + widgets: + - obj: + width: 100% + height: 100% + bg_color: 0x000080 + bg_opa: cover + border_width: 0 + radius: 0 + pad_all: 0 + widgets: + - label: + align: center + text: "Hello World!" + text_color: 0xFFFFFF diff --git a/tests/integration/fixtures/sdl_headless_screenshot.yaml b/tests/integration/fixtures/sdl_headless_screenshot.yaml new file mode 100644 index 0000000000..7ce2df130c --- /dev/null +++ b/tests/integration/fixtures/sdl_headless_screenshot.yaml @@ -0,0 +1,29 @@ +esphome: + name: sdl-headless-screenshot-test +host: + +api: + actions: + # The name comes from the test so it can capture more than once while it waits for the first + # frame: a snapshot never writes over a file that is already there. + - action: take_screenshot + variables: + name: string + then: + - snapshot.take: + id: sdl_display + filename: !lambda return name; + +logger: + level: DEBUG + +display: + - platform: sdl + id: sdl_display + headless: true + show_test_card: true + update_interval: 100ms + # An odd width exercises the row padding in the BMP writer + dimensions: + width: 101 + height: 64 diff --git a/tests/integration/fixtures/snapshot_display.yaml b/tests/integration/fixtures/snapshot_display.yaml new file mode 100644 index 0000000000..d10af09806 --- /dev/null +++ b/tests/integration/fixtures/snapshot_display.yaml @@ -0,0 +1,28 @@ +esphome: + name: snapshot-display-test +host: + +api: + actions: + # The name comes from the test so it can ask for several in a row and check what each one + # does with it. + - action: take_snapshot + variables: + name: string + then: + - snapshot.take: + id: snapshot_display + filename: !lambda return name; + +logger: + level: DEBUG + +display: + - platform: snapshot + id: snapshot_display + show_test_card: true + update_interval: 100ms + # An odd width exercises the row padding in the BMP writer + dimensions: + width: 101 + height: 64 diff --git a/tests/integration/test_lvgl_headless_render.py b/tests/integration/test_lvgl_headless_render.py new file mode 100644 index 0000000000..1c60e49604 --- /dev/null +++ b/tests/integration/test_lvgl_headless_render.py @@ -0,0 +1,83 @@ +"""Integration test that checks what LVGL actually draws, using a display with no screen. + +The rendered screen is compared against a hash rather than a checked in reference image, so the +repository does not have to carry a binary file. If a change to the drawing code or to the bundled +LVGL alters the output, this test fails and prints the hash it saw; update EXPECTED_SHA256 once the +new image has been looked at and found to be correct. + +The picture is drawn and encoded entirely by code in this repository, so nothing installed on the +machine running the test takes part in the result. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest + +from .artifact_utils import keep_artifact +from .bmp_utils import capture_when_drawn +from .types import APIClientConnectedFactory, RunCompiledFunction + +WIDTH = 300 +HEIGHT = 300 + +# sha256 of the pixel data of a 300x300 screen showing "Hello World!" centred in white on a dark +# blue background, drawn with the built in montserrat_14 font. To regenerate, run this test and +# take the hash it reports. +EXPECTED_SHA256 = "a995b002dd1d183c47514da15ab9a60a3e7d788c2e24386a02fddd48655092ed" +# Bundled LVGL version (esphome/components/lvgl/__init__.py, LVGL_VERSION) the hash above was +# generated against. A version bump can shift anti-aliasing enough to change the hash even though +# nothing is actually wrong -- if this test fails, check that first before regenerating the hash. +EXPECTED_LVGL_VERSION = "9.5.0" + + +@pytest.mark.asyncio +async def test_lvgl_headless_render( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """LVGL draws the expected screen on a 300x300 display with no screen behind it.""" + snapshot_dir = tmp_path / "snapshots" + monkeypatch.setenv("ESPHOME_SNAPSHOT_DIR", str(snapshot_dir)) + + async with run_compiled(yaml_config), api_client_connected() as client: + _, services = await client.list_entities_services() + service = next(s for s in services if s.name == "take_screenshot") + + async def take(name: str) -> None: + await client.execute_service(service, {"name": name}) + + # The background is not the whole picture: LVGL must have drawn on it. Waiting for that + # rather than for a fixed time keeps a slow first frame from being reported as a hash + # mismatch, which would look like a drawing regression. + image, capture = await capture_when_drawn(take, snapshot_dir, prefix="render") + assert (image.width, image.height, image.bits) == (WIDTH, HEIGHT, 24) + + digest = hashlib.sha256(image.pixels).hexdigest() + if digest != EXPECTED_SHA256: + # Kept outside the temporary folder so CI can upload it; see artifact_utils. + kept = keep_artifact( + "lvgl_headless_render_actual.bmp", capture.read_bytes() + ) + + from esphome.components.lvgl import LVGL_VERSION + + version_hint = "" + if LVGL_VERSION != EXPECTED_LVGL_VERSION: + version_hint = ( + f"the bundled LVGL version changed ({EXPECTED_LVGL_VERSION} -> " + f"{LVGL_VERSION}), which is the likely cause\n" + ) + pytest.fail( + f"rendered screen does not match the expected hash\n" + f"{version_hint}" + f" expected: {EXPECTED_SHA256}\n" + f" actual: {digest}\n" + f"the image that was rendered has been kept at {kept}\n" + f"on CI it is in the integration-test-artifacts upload for this job" + ) diff --git a/tests/integration/test_sdl_headless_screenshot.py b/tests/integration/test_sdl_headless_screenshot.py new file mode 100644 index 0000000000..f24b21c157 --- /dev/null +++ b/tests/integration/test_sdl_headless_screenshot.py @@ -0,0 +1,49 @@ +"""Integration test for headless SDL rendering and snapshot capture. + +How a file is named and written is the same for every display that can take a snapshot and is +covered by test_snapshot_display; what is tested here is that SDL renders and can be read back +with no display server present. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from .bmp_utils import capture_when_drawn +from .types import APIClientConnectedFactory, RunCompiledFunction + +WIDTH = 101 +HEIGHT = 64 + + +@pytest.mark.asyncio +async def test_sdl_headless_screenshot( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A headless SDL display renders with no display server and can be captured.""" + snapshot_dir = tmp_path / "snapshots" + # The device reads this when it writes a file; the subprocess inherits our environment, so it + # must be set before the binary is launched. + monkeypatch.setenv("ESPHOME_SNAPSHOT_DIR", str(snapshot_dir)) + # Make sure the run really is headless even when the test machine has a display. + monkeypatch.delenv("DISPLAY", raising=False) + monkeypatch.delenv("WAYLAND_DISPLAY", raising=False) + + async with run_compiled(yaml_config), api_client_connected() as client: + _, services = await client.list_entities_services() + service = next(s for s in services if s.name == "take_screenshot") + + async def take(name: str) -> None: + await client.execute_service(service, {"name": name}) + + # The test card is drawn in several colours, so once it is on the screen the picture is + # not one flat shade. Capturing until that is true waits out the first update rather than + # racing it. + image, _ = await capture_when_drawn(take, snapshot_dir) + assert (image.width, image.height, image.bits) == (WIDTH, HEIGHT, 24) diff --git a/tests/integration/test_snapshot_display.py b/tests/integration/test_snapshot_display.py new file mode 100644 index 0000000000..771cf0cf7d --- /dev/null +++ b/tests/integration/test_snapshot_display.py @@ -0,0 +1,78 @@ +"""Integration test for the snapshot display and the file writing shared with other displays.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +from aioesphomeapi import LogLevel +import pytest + +from .bmp_utils import capture_when_drawn, wait_for_bmp +from .types import APIClientConnectedFactory, RunCompiledFunction + +WIDTH = 101 +HEIGHT = 64 + +# Part of the message the writer logs when it will not write over a file that is already there. +REFUSAL_MESSAGE = b"not overwriting" + + +@pytest.mark.asyncio +async def test_snapshot_display( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A display with no screen draws into memory and writes what it drew to a file.""" + snapshot_dir = tmp_path / "snapshots" + # The device reads this when it writes a file; the subprocess inherits our environment, so it + # must be set before the binary is launched. + monkeypatch.setenv("ESPHOME_SNAPSHOT_DIR", str(snapshot_dir)) + + async with run_compiled(yaml_config), api_client_connected() as client: + _, services = await client.list_entities_services() + service = next(s for s in services if s.name == "take_snapshot") + + async def take(name: str) -> None: + await client.execute_service(service, {"name": name}) + + # The test card is drawn in several colours, so once it is on the screen the picture is + # not one flat shade. Capturing until that is true waits out the first update rather than + # racing it. + image, capture = await capture_when_drawn(take, snapshot_dir) + assert (image.width, image.height, image.bits) == (WIDTH, HEIGHT, 24) + + # An extension is only added when there is not one already, whatever its case. + await take("UPPER.BMP") + await wait_for_bmp(snapshot_dir / "UPPER.BMP") + + # A name that tries to lead somewhere else is cut back to one harmless name in the + # snapshot directory. + await take("../escape") + await wait_for_bmp(snapshot_dir / ".._escape.bmp") + + # A second capture under a name already used must fail rather than write over the first. + # Wait for the device to report the refusal: on its own, an unchanged file cannot tell a + # refusal apart from a request the device has not got to yet, so a regression that wrote + # over the file could still pass on a busy machine. + refused = asyncio.Event() + + def on_log(msg) -> None: + if REFUSAL_MESSAGE in msg.message: + refused.set() + + client.subscribe_logs(on_log, log_level=LogLevel.LOG_LEVEL_DEBUG) + + before = capture.read_bytes() + await take(capture.name) + await asyncio.wait_for(refused.wait(), timeout=10.0) + assert capture.read_bytes() == before + # Nothing beyond what was asked for, leaving out however many captures it took to wait + # for the first frame. + written = sorted( + p.name for p in snapshot_dir.iterdir() if not p.name.startswith("drawn-") + ) + assert written == [".._escape.bmp", "UPPER.BMP"] From 81ecb872534532390d9ac5a2c2376d68c8fec955 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:11:56 +1200 Subject: [PATCH 085/147] Bump version to 2026.10.0-dev --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 8f6048b4d8..1619371323 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.9.0-dev +PROJECT_NUMBER = 2026.10.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 6f83f0c937..e1d875f94b 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.9.0-dev" +__version__ = "2026.10.0-dev" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 567a98107884152abda87bb42e3519e898e12b67 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:16:25 +1200 Subject: [PATCH 086/147] Bump version to 2026.9.0b1 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 1619371323..7b2d21027a 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.10.0-dev +PROJECT_NUMBER = 2026.9.0b1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index e1d875f94b..378da14197 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.10.0-dev" +__version__ = "2026.9.0b1" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From d1068d582fedc070cd8611b020f9e6f5188dc68c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:56:42 -0400 Subject: [PATCH 087/147] Bump ninja from 1.13.0 to 1.13.2 (#18952) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 594b44432d..4820579e61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 platformdirs==4.11.5 # native esp-idf toolchain global cache dir -ninja==1.13.0 # native esp8266 arduino toolchain build driver +ninja==1.13.2 # native esp8266 arduino toolchain build driver filelock==3.32.4 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this From 22504309998347b1ee214107a80d627697f4cd6e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:56:51 -0400 Subject: [PATCH 088/147] Bump zeroconf from 0.151.2 to 0.151.3 (#18951) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4820579e61..8731d38b7f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ esptool==5.3.1 click==8.3.3 aioesphomeapi==46.3.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi -zeroconf==0.151.2 +zeroconf==0.151.3 puremagic==2.2.0 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import From f3c786c7848201fb4477233609b0e5ec11ec2010 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:11:56 +1200 Subject: [PATCH 089/147] Bump version to 2026.10.0-dev --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 8f6048b4d8..1619371323 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.9.0-dev +PROJECT_NUMBER = 2026.10.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 6f83f0c937..e1d875f94b 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.9.0-dev" +__version__ = "2026.10.0-dev" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 6b1163649166385b8a1dcc398349c4dbaa9f459c Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Thu, 3 Sep 2026 07:12:36 -0500 Subject: [PATCH 090/147] [remote_transmitter] Fix BK7231N build by limiting the PWM path to BK7238 (#18958) --- esphome/components/remote_transmitter/__init__.py | 14 +++++--------- .../remote_transmitter/remote_transmitter.h | 9 +++++---- .../remote_transmitter_bk72xx.cpp | 11 +++++++---- .../remote_transmitter_libretiny_isr.cpp | 10 +++++----- .../remote_transmitter/test_non_blocking_gate.py | 2 +- .../remote_transmitter/test.bk72xx-ard.yaml | 2 +- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/remote_transmitter/__init__.py b/esphome/components/remote_transmitter/__init__.py index cb2aebec91..58392c48ab 100644 --- a/esphome/components/remote_transmitter/__init__.py +++ b/esphome/components/remote_transmitter/__init__.py @@ -4,11 +4,7 @@ from esphome import automation, pins import esphome.codegen as cg from esphome.components import esp32, esp32_rmt, remote_base from esphome.components.libretiny import get_libretiny_family -from esphome.components.libretiny.const import ( - FAMILY_BK7231N, - FAMILY_BK7238, - FAMILY_RTL8720C, -) +from esphome.components.libretiny.const import FAMILY_BK7238, FAMILY_RTL8720C from esphome.config_helpers import filter_source_files_from_platform import esphome.config_validation as cv from esphome.const import ( @@ -49,7 +45,9 @@ DigitalWriteAction = remote_transmitter_ns.class_( ) -_NON_BLOCKING_LIBRETINY_FAMILIES = (FAMILY_RTL8720C, FAMILY_BK7231N, FAMILY_BK7238) +# Keep in sync with the USE_LIBRETINY_VARIANT_RTL8720C / REMOTE_TRANSMITTER_BK_PWM gates in +# remote_transmitter.h, which decide where set_non_blocking() is declared +_NON_BLOCKING_LIBRETINY_FAMILIES = (FAMILY_RTL8720C, FAMILY_BK7238) def _validate_non_blocking_platform(value: bool) -> bool: @@ -59,9 +57,7 @@ def _validate_non_blocking_platform(value: bool) -> bool: return cv.boolean(value) if CORE.is_libretiny and get_libretiny_family() in _NON_BLOCKING_LIBRETINY_FAMILIES: return cv.boolean(value) - raise cv.Invalid( - "non_blocking is only supported on ESP32, RTL8720C, BK7231N and BK7238" - ) + raise cv.Invalid("non_blocking is only supported on ESP32, RTL8720C and BK7238") MULTI_CONF = True diff --git a/esphome/components/remote_transmitter/remote_transmitter.h b/esphome/components/remote_transmitter/remote_transmitter.h index 313b26364d..4db4e80a60 100644 --- a/esphome/components/remote_transmitter/remote_transmitter.h +++ b/esphome/components/remote_transmitter/remote_transmitter.h @@ -12,10 +12,11 @@ #endif // SOC_RMT_SUPPORTED #endif // USE_ESP32 -// The BK7231N-style PWM block (hardware shadow-load duty updates) enables the ISR-driven -// transmitter on these families; family-level proxy for the SDK's CFG_SOC_NAME gate. -// See remote_transmitter_bk72xx.cpp. -#if defined(USE_LIBRETINY_VARIANT_BK7231N) || defined(USE_LIBRETINY_VARIANT_BK7238) +// Enables the ISR-driven transmitter on Beken. Gated on BK7238 alone: the shadow-load PWM +// block is shared with BK7231N, but LibreTiny builds that family against an older BDK whose +// PWM driver has no pwm_init_param()/pwm_start(). See remote_transmitter_bk72xx.cpp. +// Keep in sync with _NON_BLOCKING_LIBRETINY_FAMILIES in __init__.py. +#ifdef USE_LIBRETINY_VARIANT_BK7238 #define REMOTE_TRANSMITTER_BK_PWM #endif diff --git a/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp index 0081ae47b3..822389ccf9 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp @@ -9,10 +9,13 @@ // with the core's fixes for type-name collisions between the two #include -// Only the BK7231N-style PWM block (shadow registers with a hardware CFG_UPDATA load bit) -// supports glitch-free per-edge duty updates; older SoCs compile the generic bit-bang -// implementation (remote_transmitter.cpp) instead, and this file compiles to nothing. -// REMOTE_TRANSMITTER_BK_PWM is set per-family in remote_transmitter.h. +// Needs the BK7231N-style PWM block (shadow registers with a hardware CFG_UPDATA load bit) +// for glitch-free per-edge duty updates, and an SDK exposing pwm_init_param()/pwm_start(). +// BK7231N has the block but LibreTiny builds it against an older BDK offering only the +// sddev_control API (CMD_PWM_INIT_PARAM), so it stays on the generic bit-bang path until +// someone can add and validate that path on real hardware. Every other Beken SoC lacks the +// block. REMOTE_TRANSMITTER_BK_PWM is set per-family in remote_transmitter.h; when it is +// unset this file compiles to nothing and remote_transmitter.cpp is used instead. namespace esphome::remote_transmitter { diff --git a/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp index 003cdfa986..fad91f593f 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp @@ -3,11 +3,11 @@ #include "esphome/core/hal.h" #include "esphome/core/log.h" -// Envelope chain shared by the LibreTiny families that pace transmission from a hardware -// timer interrupt: RTL8720C (gtimer) and the BK7231N-style PWM block (BKTIMER1). Everything -// platform-specific sits behind five hooks implemented in the per-family files -- carrier -// setup, duty writes, one-shot arming and timer stop. Families without a usable timer keep -// the generic bit-bang implementation and compile none of this. +// Envelope chain shared by the LibreTiny families that pace transmission from a hardware timer +// interrupt: RTL8720C (gtimer) and BK7238 (BKTIMER1). Everything platform-specific sits behind +// five hooks implemented in the per-family files -- carrier setup, duty writes, one-shot arming +// and timer stop. Families without a usable timer keep the generic bit-bang implementation and +// compile none of this. #if defined(USE_LIBRETINY_VARIANT_RTL8720C) || defined(REMOTE_TRANSMITTER_BK_PWM) namespace esphome::remote_transmitter { diff --git a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py index ee2769e177..525ab3329e 100644 --- a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py +++ b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py @@ -26,7 +26,7 @@ from ..types import SetCoreConfigCallable (PlatformFramework.ESP32_IDF, None, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8720C, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8710B, False), - (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231N, True), + (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231N, False), (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7238, True), (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231T, False), (PlatformFramework.ESP8266_ARDUINO, None, False), diff --git a/tests/components/remote_transmitter/test.bk72xx-ard.yaml b/tests/components/remote_transmitter/test.bk72xx-ard.yaml index ea2feafda9..f3e2da9daf 100644 --- a/tests/components/remote_transmitter/test.bk72xx-ard.yaml +++ b/tests/components/remote_transmitter/test.bk72xx-ard.yaml @@ -2,7 +2,7 @@ remote_transmitter: id: xmitr pin: GPIO26 carrier_duty_percent: 50% - # non_blocking is bk7231n/bk7238-only; the CI board is a BK7252 + # non_blocking is bk7238-only; the CI board is a BK7252, so this builds the bit-bang path packages: buttons: !include common-buttons.yaml From b84532d2548ffe0bb6f326ee26160db423e5d936 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:15:06 +0000 Subject: [PATCH 091/147] Bump bundled esphome-device-builder to 1.14.0 (#18960) Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0da8048c57..7952616496 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.13.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.0 RUN \ platformio settings set enable_telemetry No \ From f65ab5629e0401d34d0b0e9be1bc865c76941b9a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 3 Sep 2026 21:16:36 +0200 Subject: [PATCH 092/147] [esp8266] Drop Arduino framework versions before 3.0.0 (#18917) to Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/arduino8266/framework.py | 13 ++--- esphome/components/climate/climate.cpp | 4 +- esphome/components/debug/debug_component.cpp | 4 +- esphome/components/debug/debug_component.h | 4 +- esphome/components/debug/debug_esp8266.cpp | 2 - esphome/components/debug/sensor.py | 7 +-- esphome/components/esp8266/__init__.py | 57 ++++++------------- .../nextion/nextion_upload_arduino.cpp | 6 -- esphome/components/wifi/wifi_component.h | 5 -- .../wifi/wifi_component_esp8266.cpp | 10 +--- esphome/core/log.h | 14 ----- .../components/esp8266/test_boards.py | 17 +----- .../esp8266/test_framework_version.py | 23 ++++++++ .../unit_tests/test_arduino8266_framework.py | 17 ++---- 14 files changed, 62 insertions(+), 121 deletions(-) create mode 100644 tests/unit_tests/components/esp8266/test_framework_version.py diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 1edbe4b36f..663002b3b1 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -44,8 +44,7 @@ def get_arduino8266_tools_path() -> Path: return tools_cache_path(*ARDUINO8266_TOOLS_CACHE) -# 3.1.1 rather than 3.1.0: the registry has no package for 3.1.0, and the -# encoder below cannot name 3.0.0/3.0.1 either (see its docstring) +# 3.1.1 rather than 3.1.0: the registry has no packages for 3.0.0, 3.0.1 or 3.1.0 MIN_FRAMEWORK_VERSION = Version(3, 1, 1) @@ -53,20 +52,16 @@ def framework_package_version(ver: Version) -> str: """Map an Arduino core version to its registry package version (3.1.2 -> 3.30102.0; the leading 3 is the package major). - Exact registry names only for cores > 2.6.2 and >= 3.0.2; callers floor - at MIN_FRAMEWORK_VERSION. + Exact registry names for 3.x cores; callers floor at MIN_FRAMEWORK_VERSION. """ if ver.major > 3: raise EsphomeError( f"Arduino core {ver} is not supported yet; " "the newest known core series is 3.x" ) - if ver <= Version(2, 6, 2): - # Cores <= 2.6.2 use the older 1.x/2.x package-major encodings (same - # boundary as _format_framework_arduino_version's era guard) + if ver.major < 3: raise EsphomeError( - f"Arduino core {ver} uses an older package encoding than this " - "helper implements (newer than 2.6.2)" + f"Arduino core {ver} is not supported; ESPHome requires core 3.x" ) return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" diff --git a/esphome/components/climate/climate.cpp b/esphome/components/climate/climate.cpp index 34684a87e1..f80de151b1 100644 --- a/esphome/components/climate/climate.cpp +++ b/esphome/components/climate/climate.cpp @@ -368,8 +368,8 @@ optional Climate::restore_state_() { } void Climate::save_state_(const ClimateTraits &traits) { -#if (defined(USE_ESP32) || (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0))) && \ - !defined(CLANG_TIDY) +#if (defined(USE_ESP32) || defined(USE_ESP8266)) && !defined(CLANG_TIDY) +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wclass-memaccess" #define TEMP_IGNORE_MEMACCESS #endif diff --git a/esphome/components/debug/debug_component.cpp b/esphome/components/debug/debug_component.cpp index 9020c261c2..97f4522c62 100644 --- a/esphome/components/debug/debug_component.cpp +++ b/esphome/components/debug/debug_component.cpp @@ -22,9 +22,9 @@ void DebugComponent::dump_config() { LOG_SENSOR(" ", "Free space on heap", this->free_sensor_); LOG_SENSOR(" ", "Largest free heap block", this->block_sensor_); LOG_SENSOR(" ", "CPU frequency", this->cpu_frequency_sensor_); -#if defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) +#ifdef USE_ESP8266 LOG_SENSOR(" ", "Heap fragmentation", this->fragmentation_sensor_); -#endif // defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) +#endif // USE_ESP8266 #endif // USE_SENSOR char device_info_buffer[DEVICE_INFO_BUFFER_SIZE]; diff --git a/esphome/components/debug/debug_component.h b/esphome/components/debug/debug_component.h index 20798cf600..b05029f878 100644 --- a/esphome/components/debug/debug_component.h +++ b/esphome/components/debug/debug_component.h @@ -35,7 +35,7 @@ class DebugComponent final : public PollingComponent { #ifdef USE_SENSOR void set_free_sensor(sensor::Sensor *free_sensor) { free_sensor_ = free_sensor; } void set_block_sensor(sensor::Sensor *block_sensor) { block_sensor_ = block_sensor; } -#if (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2)) || defined(USE_ESP32) +#if defined(USE_ESP8266) || defined(USE_ESP32) void set_fragmentation_sensor(sensor::Sensor *fragmentation_sensor) { fragmentation_sensor_ = fragmentation_sensor; } #endif #if defined(USE_ESP32) || defined(USE_LIBRETINY) @@ -61,7 +61,7 @@ class DebugComponent final : public PollingComponent { sensor::Sensor *free_sensor_{nullptr}; sensor::Sensor *block_sensor_{nullptr}; -#if (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2)) || defined(USE_ESP32) +#if defined(USE_ESP8266) || defined(USE_ESP32) sensor::Sensor *fragmentation_sensor_{nullptr}; #endif #if defined(USE_ESP32) || defined(USE_LIBRETINY) diff --git a/esphome/components/debug/debug_esp8266.cpp b/esphome/components/debug/debug_esp8266.cpp index 272123dfc0..acce28818c 100644 --- a/esphome/components/debug/debug_esp8266.cpp +++ b/esphome/components/debug/debug_esp8266.cpp @@ -159,12 +159,10 @@ void DebugComponent::update_platform_() { // NOLINTNEXTLINE(readability-static-accessed-through-instance) this->block_sensor_->publish_state(ESP.getMaxFreeBlockSize()); } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) if (this->fragmentation_sensor_ != nullptr) { // NOLINTNEXTLINE(readability-static-accessed-through-instance) this->fragmentation_sensor_->publish_state(ESP.getHeapFragmentation()); } -#endif #endif } diff --git a/esphome/components/debug/sensor.py b/esphome/components/debug/sensor.py index 72e2efebc2..e53cb0d1e4 100644 --- a/esphome/components/debug/sensor.py +++ b/esphome/components/debug/sensor.py @@ -52,12 +52,9 @@ CONFIG_SCHEMA = { ), cv.Optional(CONF_FRAGMENTATION): cv.All( cv.Any( - cv.All( - cv.only_on_esp8266, - cv.require_framework_version(esp8266_arduino=cv.Version(2, 5, 2)), - ), + cv.only_on_esp8266, cv.only_on_esp32, - msg="This feature is only available on ESP8266 (Arduino 2.5.2+) and ESP32", + msg="This feature is only available on ESP8266 and ESP32", ), sensor.sensor_schema( unit_of_measurement=UNIT_PERCENT, diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 63665e7681..19dbb68f29 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -35,7 +35,7 @@ from esphome.platformio.toolchain import copy_ccache_script from esphome.storage_json import StorageJSON from esphome.types import ConfigType -from .boards import BOARDS, ESP8266_LD_SCRIPTS, board_ld_script +from .boards import BOARDS, board_ld_script from .const import ( CONF_EARLY_PIN_INIT, CONF_ENABLE_SERIAL, @@ -43,8 +43,6 @@ from .const import ( CONF_RESTORE_FROM_FLASH, KEY_BOARD, KEY_ESP8266, - KEY_FLASH_SIZE, - KEY_LDSCRIPT, KEY_PIN_INITIAL_STATES, KEY_SERIAL1_REQUIRED, KEY_SERIAL_REQUIRED, @@ -133,10 +131,6 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # format the given arduino (https://github.com/esp8266/Arduino/releases) version to # a PIO platformio/framework-arduinoespressif8266 value # List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 - if ver <= cv.Version(2, 4, 1): - return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" - if ver <= cv.Version(2, 6, 2): - return f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" # Same encoding the native toolchain uses for its package download, so a # version bump cannot drift between the two paths. from esphome.arduino8266.framework import framework_package_version @@ -159,11 +153,9 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # - https://github.com/esp8266/Arduino/releases # - https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 RECOMMENDED_ARDUINO_FRAMEWORK_VERSION = cv.Version(3, 1, 2) -# The platformio/espressif8266 version to use for arduino 2 framework versions +# The platformio/espressif8266 version to use for arduino 3 framework versions # - https://github.com/platformio/platform-espressif8266/releases # - https://api.registry.platformio.org/v3/packages/platformio/platform/espressif8266 -ARDUINO_2_PLATFORM_VERSION = cv.Version(2, 6, 3) -# for arduino 3 framework versions ARDUINO_3_PLATFORM_VERSION = cv.Version(3, 2, 0) # for arduino 4 framework versions ARDUINO_4_PLATFORM_VERSION = cv.Version(4, 2, 1) @@ -188,6 +180,14 @@ def _arduino_check_versions(value: ConfigType) -> ConfigType: version = cv.Version.parse(cv.version_number(value[CONF_VERSION])) source = value.get(CONF_SOURCE, None) + if version < cv.Version(3, 0, 0): + raise cv.Invalid( + f"Arduino framework {version} is no longer supported; ESPHome requires " + f"C++20, which needs Arduino core 3.x. Use the recommended version " + f"({RECOMMENDED_ARDUINO_FRAMEWORK_VERSION}).", + path=[CONF_VERSION], + ) + value[CONF_VERSION] = str(version) value[CONF_SOURCE] = source or _format_framework_arduino_version(version) @@ -195,12 +195,8 @@ def _arduino_check_versions(value: ConfigType) -> ConfigType: if platform_version is None: if version >= cv.Version(3, 1, 0): platform_version = _parse_platform_version(str(ARDUINO_4_PLATFORM_VERSION)) - elif version >= cv.Version(3, 0, 0): - platform_version = _parse_platform_version(str(ARDUINO_3_PLATFORM_VERSION)) - elif version >= cv.Version(2, 5, 0): - platform_version = _parse_platform_version(str(ARDUINO_2_PLATFORM_VERSION)) else: - platform_version = _parse_platform_version(str(cv.Version(1, 8, 0))) + platform_version = _parse_platform_version(str(ARDUINO_3_PLATFORM_VERSION)) value[CONF_PLATFORM_VERSION] = platform_version if version != RECOMMENDED_ARDUINO_FRAMEWORK_VERSION: @@ -289,29 +285,11 @@ def check_rosetta() -> None: ) -def _choose_ld_script(board: str, ver: cv.Version) -> str | None: - """The flash ld to pin for this board and core, or None for cores - without ld-script support.""" - board_data = BOARDS[board] - ld_scripts = ESP8266_LD_SCRIPTS[board_data[KEY_FLASH_SIZE]] - if ver <= cv.Version(2, 3, 0): - # No ld script support - return None - if ver <= cv.Version(2, 4, 2): - # Old ld script path; the modern per-board override names do not - # exist in this core's SDK, so the override cannot be honored. - # Substituting the size default would move _FS_end and the - # preferences sector, wiping flash-backed state on flash. - if KEY_LDSCRIPT in board_data: - raise EsphomeError( - f"Board {board} requires its {board_data[KEY_LDSCRIPT]} " - f"flash layout, which Arduino core {ver} cannot honor; " - "use a core newer than 2.4.2" - ) - return ld_scripts[0] +def _choose_ld_script(board: str) -> str: + """The flash ld to pin for this board.""" # A per-board override preserves a layout the board shipped with # (see d1_wroom_02 in boards.py) - return board_ld_script(board_data) + return board_ld_script(BOARDS[board]) @coroutine_with_priority(CoroPriority.PLATFORM) @@ -435,10 +413,9 @@ async def to_code(config: ConfigType) -> None: ) if config[CONF_BOARD] in BOARDS: - ld_script = _choose_ld_script(config[CONF_BOARD], ver) - - if ld_script is not None: - cg.add_platformio_option("board_build.ldscript", ld_script) + cg.add_platformio_option( + "board_build.ldscript", _choose_ld_script(config[CONF_BOARD]) + ) CORE.add_job(add_pin_initial_states_array) CORE.add_job(finalize_waveform_config) diff --git a/esphome/components/nextion/nextion_upload_arduino.cpp b/esphome/components/nextion/nextion_upload_arduino.cpp index f02f32d5ca..944fa1db47 100644 --- a/esphome/components/nextion/nextion_upload_arduino.cpp +++ b/esphome/components/nextion/nextion_upload_arduino.cpp @@ -209,14 +209,8 @@ bool Nextion::upload_tft(uint32_t baud_rate, bool exit_reparse) { http_client.setTimeout(this->tft_upload_http_timeout_); bool begin_status = false; -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 7, 0) http_client.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); -#elif USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 6, 0) - http_client.setFollowRedirects(true); -#endif -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 6, 0) http_client.setRedirectLimit(3); -#endif begin_status = http_client.begin(*this->get_wifi_client_(), this->tft_url_.c_str()); if (!begin_status) { this->connection_state_.is_updating_ = false; diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index cfdbc1a968..63df9fbfa5 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -40,11 +40,6 @@ #include #include -#if defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE < VERSION_CODE(2, 4, 0) -extern "C" { -#include -}; -#endif #endif #ifdef USE_RP2 diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index b4a91fb3cd..031da1b355 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -21,7 +21,6 @@ extern "C" { #include "lwip/apps/sntp.h" #include "lwip/netif.h" // struct netif #include -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0) #include "LwipDhcpServer.h" #if USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) #include @@ -30,7 +29,6 @@ extern "C" { #define wifi_softap_set_dhcps_lease_time(time) dhcpSoftAP.set_dhcps_lease_time(time) #define wifi_softap_set_dhcps_offer_option(offer, mode) dhcpSoftAP.set_dhcps_offer_option(offer, mode) #endif -#endif } #include "esphome/core/application.h" @@ -293,7 +291,6 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { conf.bssid_set = 0; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) if (ap.password_.empty()) { conf.threshold.authmode = AUTH_OPEN; } else { @@ -310,7 +307,6 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { } } conf.threshold.rssi = -127; -#endif ETS_UART_INTR_DISABLE(); bool ret = wifi_station_set_config_current(&conf); @@ -602,7 +598,6 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { #endif break; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) case EVENT_OPMODE_CHANGED: { auto it = event->event_info.opmode_changed; ESP_LOGV(TAG, "Changed Mode old=%s new=%s", LOG_STR_ARG(get_op_mode_str(it.old_opmode)), @@ -620,7 +615,6 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { #endif break; } -#endif default: break; } @@ -705,7 +699,6 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { config.bssid = nullptr; config.channel = 0; config.show_hidden = 1; -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) config.scan_type = passive ? WIFI_SCAN_TYPE_PASSIVE : WIFI_SCAN_TYPE_ACTIVE; // Use shorter dwell times for roaming scans - we only need to detect strong // nearby APs, not do a thorough survey. This also reduces off-channel time @@ -724,7 +717,6 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { config.scan_time.active.min = roaming ? SCAN_ACTIVE_MIN_ROAMING_MS : SCAN_ACTIVE_MIN_DEFAULT_MS; config.scan_time.active.max = roaming ? SCAN_ACTIVE_MAX_ROAMING_MS : SCAN_ACTIVE_MAX_DEFAULT_MS; } -#endif bool ret = wifi_station_scan(&config, &WiFiComponent::s_wifi_scan_done_callback); if (!ret) { ESP_LOGV(TAG, "wifi_station_scan failed"); @@ -830,7 +822,7 @@ bool WiFiComponent::wifi_ap_ip_config_(const optional &manual_ip) { return false; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0) && USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) +#if USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) dhcpSoftAP.begin(&info); #endif diff --git a/esphome/core/log.h b/esphome/core/log.h index 272e516808..14d24412ef 100644 --- a/esphome/core/log.h +++ b/esphome/core/log.h @@ -18,7 +18,6 @@ #ifdef USE_STORE_LOG_STR_IN_FLASH #include "WString.h" -#include "esphome/core/defines.h" // for USE_ARDUINO_VERSION_CODE #endif // Include ESP-IDF/Arduino based logging methods here so they don't undefine ours later @@ -177,20 +176,7 @@ struct LogString; #include -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 0) #define LOG_STR_ARG(s) ((PGM_P) (s)) -#else -// Pre-Arduino 2.5, we can't pass a PSTR() to printf(). Emulate support by copying the message to a -// local buffer first. String length is limited to 63 characters. -// https://github.com/esp8266/Arduino/commit/6280e98b0360f85fdac2b8f10707fffb4f6e6e31 -#define LOG_STR_ARG(s) \ - ({ \ - char __buf[64]; \ - __buf[63] = '\0'; \ - strncpy_P(__buf, (PGM_P) (s), 63); \ - __buf; \ - }) -#endif #define LOG_STR(s) (reinterpret_cast(PSTR(s))) #define LOG_STR_LITERAL(s) LOG_STR_ARG(LOG_STR(s)) diff --git a/tests/unit_tests/components/esp8266/test_boards.py b/tests/unit_tests/components/esp8266/test_boards.py index df0e536d42..78213a762a 100644 --- a/tests/unit_tests/components/esp8266/test_boards.py +++ b/tests/unit_tests/components/esp8266/test_boards.py @@ -1,11 +1,7 @@ """Tests for the per-board linker-script rule.""" -import pytest - from esphome.components.esp8266 import _choose_ld_script from esphome.components.esp8266.boards import BOARDS, board_ld_script -import esphome.config_validation as cv -from esphome.core import EsphomeError def test_d1_wroom_02_keeps_its_shipped_layout() -> None: @@ -21,13 +17,6 @@ def test_default_boards_use_the_flash_size_layout() -> None: def test_choose_ld_script_paths() -> None: - """Old cores get the size default, overriding boards hard-error there - (a substituted layout would wipe flash-backed state), modern cores - honor the override.""" - assert _choose_ld_script("nodemcuv2", cv.Version(2, 3, 0)) is None - assert _choose_ld_script("nodemcuv2", cv.Version(2, 4, 2)) == "eagle.flash.4m.ld" - assert _choose_ld_script("d1_wroom_02", cv.Version(2, 7, 4)) == ( - "eagle.flash.2m64.ld" - ) - with pytest.raises(EsphomeError, match="cannot honor"): - _choose_ld_script("d1_wroom_02", cv.Version(2, 4, 2)) + """Default boards get the size layout, overriding boards keep theirs.""" + assert _choose_ld_script("nodemcuv2") == "eagle.flash.4m.ld" + assert _choose_ld_script("d1_wroom_02") == "eagle.flash.2m64.ld" diff --git a/tests/unit_tests/components/esp8266/test_framework_version.py b/tests/unit_tests/components/esp8266/test_framework_version.py new file mode 100644 index 0000000000..0107aff8dd --- /dev/null +++ b/tests/unit_tests/components/esp8266/test_framework_version.py @@ -0,0 +1,23 @@ +"""Tests for the Arduino framework version floor.""" + +import pytest + +from esphome.components.esp8266 import _arduino_check_versions +import esphome.config_validation as cv +from esphome.const import CONF_PLATFORM_VERSION, CONF_VERSION + + +def test_versions_before_3_are_rejected() -> None: + with pytest.raises(cv.Invalid, match="no longer supported") as excinfo: + _arduino_check_versions({CONF_VERSION: "2.7.4"}) + assert excinfo.value.path == [CONF_VERSION] + + +def test_supported_versions_pass() -> None: + value = _arduino_check_versions({CONF_VERSION: "3.0.2"}) + assert value[CONF_VERSION] == "3.0.2" + assert "espressif8266@3.2.0" in value[CONF_PLATFORM_VERSION] + + value = _arduino_check_versions({CONF_VERSION: "recommended"}) + assert value[CONF_VERSION] == "3.1.2" + assert "espressif8266@4.2.1" in value[CONF_PLATFORM_VERSION] diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index bd0a620e10..9f415344ae 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -21,17 +21,12 @@ def _build_path(tmp_path: Path) -> None: def test_framework_package_version() -> None: assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0" assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0" - # 2.6.3+ cores use the same package-major-3 encoding (PlatformIO path) - assert framework.framework_package_version(cv.Version(2, 7, 4)) == "3.20704.0" # A future major bump needs its own encoding, not a doomed registry lookup with pytest.raises(EsphomeError, match="not supported yet"): framework.framework_package_version(cv.Version(4, 0, 0)) - # The boundary matches the PlatformIO era guard; a 2.6.2 pre-release - # keeps this encoding - with pytest.raises(EsphomeError, match="older package encoding"): - framework.framework_package_version(cv.Version(2, 6, 2)) - assert framework.framework_package_version(cv.Version(2, 6, 2, "b1")) == "3.20602.0" - assert framework.framework_package_version(cv.Version(2, 6, 3)) == "3.20603.0" + # Cores before 3.x cannot build ESPHome (C++20) and are rejected + with pytest.raises(EsphomeError, match="requires core 3"): + framework.framework_package_version(cv.Version(2, 7, 4)) def test_format_framework_arduino_version_pins_all_series() -> None: @@ -39,10 +34,10 @@ def test_format_framework_arduino_version_pins_all_series() -> None: era, including the 4.x rejection it now shares with the installer.""" from esphome.components.esp8266 import _format_framework_arduino_version as fmt - assert fmt(cv.Version(2, 4, 1)) == "~1.20401.0" - assert fmt(cv.Version(2, 6, 2)) == "~2.20602.0" - assert fmt(cv.Version(2, 7, 4)) == "~3.20704.0" assert fmt(cv.Version(3, 1, 2)) == "~3.30102.0" + # Pre-3 cores are rejected with the version line anchored + with pytest.raises(cv.Invalid, match="requires core 3"): + fmt(cv.Version(2, 7, 4)) # Anchored to the framework version line, not a bare EsphomeError with pytest.raises(cv.Invalid, match="not supported yet") as excinfo: fmt(cv.Version(4, 0, 0)) From ab800dc09dbe2d1eb1ed1ee8f091282561e0ccc4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:19:29 -0400 Subject: [PATCH 093/147] Bump filelock from 3.32.4 to 3.32.5 (#18963) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8731d38b7f..8a510a2c60 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ requests==2.34.2 py7zr==1.1.3 platformdirs==4.11.5 # native esp-idf toolchain global cache dir ninja==1.13.2 # native esp8266 arduino toolchain build driver -filelock==3.32.4 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg +filelock==3.32.5 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this pyparsing >= 3.3.2 From 51ea97deffbac5c2d1379ce20424c74d6b2259b5 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Sat, 5 Sep 2026 08:38:55 +1200 Subject: [PATCH 094/147] [esp32_ble] Reference count BLE advertising (#18943) --- esphome/components/esp32_ble/ble.cpp | 35 +++++++++++++++---- esphome/components/esp32_ble/ble.h | 13 +++++++ .../esp32_ble_beacon/esp32_ble_beacon.cpp | 2 ++ .../components/esp32_ble_server/__init__.py | 12 +++++++ .../esp32_ble_server/ble_server.cpp | 21 +++++++++-- .../components/esp32_ble_server/ble_server.h | 11 ++++++ .../esp32_improv/esp32_improv_component.cpp | 20 ++++++++++- .../esp32_improv/esp32_improv_component.h | 3 ++ .../esp32_ble_server/config/improv_only.yaml | 13 +++++++ .../config/manufacturer_data_only.yaml | 9 +++++ .../esp32_ble_server/config/own_service.yaml | 14 ++++++++ .../esp32_ble_server/test_esp32_ble_server.py | 28 +++++++++++++++ 12 files changed, 171 insertions(+), 10 deletions(-) create mode 100644 tests/component_tests/esp32_ble_server/config/improv_only.yaml create mode 100644 tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml create mode 100644 tests/component_tests/esp32_ble_server/config/own_service.yaml diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index 6e6fb0e30d..fc95760cf8 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -100,21 +100,38 @@ void ESP32BLE::disable() { #ifdef USE_ESP32_BLE_ADVERTISING void ESP32BLE::advertising_start() { this->advertising_init_(); - if (!this->is_active()) + this->advertising_ref_count_++; + this->advertising_refresh(); +} + +void ESP32BLE::advertising_stop() { + if (this->advertising_ref_count_ == 0) return; - this->advertising_->start(); + this->advertising_ref_count_--; + this->advertising_refresh(); +} + +void ESP32BLE::advertising_refresh() { + if (this->advertising_ == nullptr || !this->is_active()) + return; + // Advertise while any component still needs it, otherwise stop + if (this->advertising_ref_count_ == 0) { + this->advertising_->stop(); + } else { + this->advertising_->start(); + } } void ESP32BLE::advertising_set_service_data(const std::vector &data) { this->advertising_init_(); this->advertising_->set_service_data(data); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_set_manufacturer_data(const std::vector &data) { this->advertising_init_(); this->advertising_->set_manufacturer_data(data); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_set_service_data_and_name(std::span data, bool include_name) { @@ -136,7 +153,7 @@ void ESP32BLE::advertising_set_service_data_and_name(std::span da this->advertising_->set_service_data(data); } - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_register_raw_advertisement_callback(std::function &&callback) { @@ -147,13 +164,13 @@ void ESP32BLE::advertising_register_raw_advertisement_callback(std::functionadvertising_init_(); this->advertising_->add_service_uuid(uuid); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_remove_service_uuid(ESPBTUUID uuid) { this->advertising_init_(); this->advertising_->remove_service_uuid(uuid); - this->advertising_start(); + this->advertising_refresh(); } #endif @@ -575,6 +592,10 @@ void ESP32BLE::loop_handle_state_transition_not_active_() { } this->state_ = BLE_COMPONENT_STATE_ACTIVE; +#ifdef USE_ESP32_BLE_ADVERTISING + // Requests made before the stack was up (or before it was re-enabled) take effect now + this->advertising_refresh(); +#endif } } diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h index 2a355a6c8b..7d2d0438a4 100644 --- a/esphome/components/esp32_ble/ble.h +++ b/esphome/components/esp32_ble/ble.h @@ -114,7 +114,17 @@ class ESP32BLE final : public Component { void set_name(const char *name) { this->name_ = name; } #ifdef USE_ESP32_BLE_ADVERTISING + /** Request advertising on behalf of a component. + * + * Requests are reference counted: advertising runs until every component that called + * advertising_start() has released it again with advertising_stop(). Each component must + * pair its calls, so nothing advertises until something actually asks for it. + */ void advertising_start(); + /// Release a request made with advertising_start(); advertising stops at the last release. + void advertising_stop(); + /// Apply the current payload and request count: advertise while requested, otherwise stop. + void advertising_refresh(); void advertising_set_service_data(const std::vector &data); void advertising_set_manufacturer_data(const std::vector &data); void advertising_set_appearance(uint16_t appearance) { this->appearance_ = appearance; } @@ -226,6 +236,9 @@ class ESP32BLE final : public Component { // 1-byte aligned members (grouped together to minimize padding) BLEComponentState state_{BLE_COMPONENT_STATE_OFF}; // 1 byte (uint8_t enum) bool enable_on_boot_{}; // 1 byte +#ifdef USE_ESP32_BLE_ADVERTISING + uint8_t advertising_ref_count_{0}; // 1 byte, number of components requesting advertising +#endif #ifdef ESPHOME_ESP32_BLE_EXTENDED_AUTH_PARAMS optional auth_req_mode_; diff --git a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp index 9f1723430b..ab728f9f6f 100644 --- a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp +++ b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp @@ -67,6 +67,8 @@ void ESP32BLEBeacon::setup() { this->on_advertise_(); } }); + // A beacon always needs the device to advertise, and never releases the request + global_ble->advertising_start(); } void ESP32BLEBeacon::on_advertise_() { diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py index 855a3be29b..d8095cd702 100644 --- a/esphome/components/esp32_ble_server/__init__.py +++ b/esphome/components/esp32_ble_server/__init__.py @@ -596,6 +596,18 @@ async def to_code(config): cg.add(var.set_parent(parent)) cg.add(parent.advertising_set_appearance(config[CONF_APPEARANCE])) cg.add(var.set_max_clients(config[CONF_MAX_CLIENTS])) + # Only advertise for the server itself when the configuration gives clients something to + # find. A server that is auto-loaded purely to host a runtime service (esp32_improv) stays + # silent until that service asks for advertising. + cg.add( + var.set_advertising_required( + CONF_MANUFACTURER_DATA in config + or any( + not uuid_is(service_config[CONF_UUID], DEVICE_INFORMATION_SERVICE_UUID) + for service_config in config[CONF_SERVICES] + ) + ) + ) if CONF_MANUFACTURER_DATA in config: cg.add(var.set_manufacturer_data(config[CONF_MANUFACTURER_DATA])) for service_config in config[CONF_SERVICES]: diff --git a/esphome/components/esp32_ble_server/ble_server.cpp b/esphome/components/esp32_ble_server/ble_server.cpp index 2dea1666bb..45679b9b98 100644 --- a/esphome/components/esp32_ble_server/ble_server.cpp +++ b/esphome/components/esp32_ble_server/ble_server.cpp @@ -81,6 +81,7 @@ void BLEServer::loop() { if (this->device_information_service_->is_running()) { this->state_ = RUNNING; this->restart_advertising_(); + this->request_advertising_(); ESP_LOGD(TAG, "BLE server setup successfully"); } else if (this->device_information_service_->is_created()) { this->device_information_service_->start(); @@ -98,6 +99,20 @@ void BLEServer::restart_advertising_() { } } +void BLEServer::request_advertising_() { + if (!this->advertising_required_ || this->advertising_requested_) + return; + this->advertising_requested_ = true; + this->parent_->advertising_start(); +} + +void BLEServer::release_advertising_() { + if (!this->advertising_requested_) + return; + this->advertising_requested_ = false; + this->parent_->advertising_stop(); +} + BLEService *BLEServer::create_service(ESPBTUUID uuid, bool advertise, uint16_t num_handles) { #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE char uuid_buf[esp32_ble::UUID_STR_LEN]; @@ -170,7 +185,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga this->add_client_(param->connect.conn_id); // Resume advertising so additional clients can discover and connect if (this->client_count_ < this->max_clients_) { - this->parent_->advertising_start(); + this->parent_->advertising_refresh(); } this->dispatch_callbacks_(CallbackType::ON_CONNECT, param->connect.conn_id); break; @@ -178,7 +193,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga case ESP_GATTS_DISCONNECT_EVT: { ESP_LOGD(TAG, "BLE Client disconnected"); this->remove_client_(param->disconnect.conn_id); - this->parent_->advertising_start(); + this->parent_->advertising_refresh(); this->dispatch_callbacks_(CallbackType::ON_DISCONNECT, param->disconnect.conn_id); break; } @@ -226,6 +241,8 @@ void BLEServer::remove_client_(uint16_t conn_id) { } void BLEServer::ble_before_disabled_event_handler() { + // Advertising is re-requested once the server is running again after BLE is re-enabled + this->release_advertising_(); // Delete all clients this->client_count_ = 0; // Delete all services diff --git a/esphome/components/esp32_ble_server/ble_server.h b/esphome/components/esp32_ble_server/ble_server.h index fdd92812cd..7869c73cc5 100644 --- a/esphome/components/esp32_ble_server/ble_server.h +++ b/esphome/components/esp32_ble_server/ble_server.h @@ -38,6 +38,13 @@ class BLEServer final : public Component, public Parented { this->restart_advertising_(); } + /** Whether this server needs the device to advertise so clients can find and connect to it. + * + * False for a server that only hosts services created at runtime (e.g. esp32_improv), which + * request advertising themselves for as long as they need it. + */ + void set_advertising_required(bool required) { this->advertising_required_ = required; } + void set_max_clients(uint8_t max_clients) { this->max_clients_ = max_clients; } uint8_t get_max_clients() const { return this->max_clients_; } @@ -82,6 +89,8 @@ class BLEServer final : public Component, public Parented { }; void restart_advertising_(); + void request_advertising_(); + void release_advertising_(); int8_t find_client_index_(uint16_t conn_id) const; void add_client_(uint16_t conn_id); @@ -93,6 +102,8 @@ class BLEServer final : public Component, public Parented { std::vector manufacturer_data_{}; esp_gatt_if_t gatts_if_{0}; bool registered_{false}; + bool advertising_required_{true}; + bool advertising_requested_{false}; uint16_t clients_[USE_ESP32_BLE_MAX_CONNECTIONS]{}; uint8_t client_count_{0}; diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp index 4756fba637..9ec6eb7bab 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.cpp +++ b/esphome/components/esp32_improv/esp32_improv_component.cpp @@ -112,6 +112,7 @@ void ESP32ImprovComponent::loop() { this->state_callback_.call(this->state_, this->error_state_); #endif } + this->release_advertising_(); this->incoming_data_.clear(); return; } @@ -143,8 +144,9 @@ void ESP32ImprovComponent::loop() { ESP_LOGV(TAG, "Starting with device name advertising"); this->advertising_device_name_ = true; this->last_name_adv_time_ = App.get_loop_component_start_time(); + // Set the payload before requesting, so advertising starts exactly once esp32_ble::global_ble->advertising_set_service_data_and_name(std::span{}, true); - esp32_ble::global_ble->advertising_start(); + this->request_advertising_(); // Set initial state based on whether we have an authorizer this->set_state_(this->get_initial_state_(), false); @@ -326,6 +328,8 @@ void ESP32ImprovComponent::stop() { this->set_timeout("end-service", STOP_ADVERTISING_DELAY, [this] { if (this->state_ == improv::STATE_STOPPED || this->service_ == nullptr) return; + // Release first so removing the service UUID does not restart advertising on the way out + this->release_advertising_(); this->service_->stop(); this->set_state_(improv::STATE_STOPPED); }); @@ -520,6 +524,20 @@ void ESP32ImprovComponent::update_advertising_type_() { } } +void ESP32ImprovComponent::request_advertising_() { + if (this->advertising_requested_) + return; + this->advertising_requested_ = true; + esp32_ble::global_ble->advertising_start(); +} + +void ESP32ImprovComponent::release_advertising_() { + if (!this->advertising_requested_) + return; + this->advertising_requested_ = false; + esp32_ble::global_ble->advertising_stop(); +} + improv::State ESP32ImprovComponent::get_initial_state_() const { #ifdef USE_BINARY_SENSOR // If we have an authorizer, start in awaiting authorization state diff --git a/esphome/components/esp32_improv/esp32_improv_component.h b/esphome/components/esp32_improv/esp32_improv_component.h index 414948c977..a40d60552a 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.h +++ b/esphome/components/esp32_improv/esp32_improv_component.h @@ -104,8 +104,11 @@ class ESP32ImprovComponent final : public Component, public improv_base::ImprovB bool status_indicator_state_{false}; uint32_t last_name_adv_time_{0}; bool advertising_device_name_{false}; + bool advertising_requested_{false}; void set_status_indicator_state_(bool state); void update_advertising_type_(); + void request_advertising_(); + void release_advertising_(); void set_state_(improv::State state, bool update_advertising = true); void set_error_(improv::Error error); diff --git a/tests/component_tests/esp32_ble_server/config/improv_only.yaml b/tests/component_tests/esp32_ble_server/config/improv_only.yaml new file mode 100644 index 0000000000..8a5c3ba638 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/improv_only.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + variant: esp32 + +wifi: + ssid: MySSID + password: password1 + +# esp32_ble_server is only auto-loaded here, so it has no services of its own. +esp32_improv: + authorizer: none diff --git a/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml b/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml new file mode 100644 index 0000000000..b7bdae4af7 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + variant: esp32 + +esp32_ble_server: + id: ble_server + manufacturer_data: [0x72, 0x04, 0x00, 0x23] diff --git a/tests/component_tests/esp32_ble_server/config/own_service.yaml b/tests/component_tests/esp32_ble_server/config/own_service.yaml new file mode 100644 index 0000000000..c7ef0287b0 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/own_service.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp32: + variant: esp32 + +esp32_ble_server: + id: ble_server + services: + - uuid: 2a24b789-7aab-4535-af3e-ee76a35cc12d + characteristics: + - uuid: cad48e28-7fbe-41cf-bae9-d77a6c233423 + read: true + value: [1, 2, 3, 4] diff --git a/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py b/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py index 88307d0dcf..4b7ab79a81 100644 --- a/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py +++ b/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py @@ -1,5 +1,10 @@ """Tests for esp32_ble_server configuration helpers.""" +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + import pytest from esphome.components.esp32_ble_server import ( @@ -45,3 +50,26 @@ def test_uuid_is_matches_descriptor_short_strings(uuid16) -> None: assert uuid_is(uuid16, uuid16) assert uuid_is(f"{uuid16:04X}", uuid16) assert uuid_is(f"{uuid16:08X}", uuid16) + + +@pytest.mark.parametrize( + ("config_file", "required"), + [ + # Auto-loaded by esp32_improv only: nothing to find until Improv asks for it + ("improv_only.yaml", False), + # The configuration defines a service clients are meant to connect to + ("own_service.yaml", True), + # Manufacturer data is only useful if it is actually broadcast + ("manufacturer_data_only.yaml", True), + ], +) +def test_advertising_required( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + required: bool, +) -> None: + """The server only requests advertising when the configuration needs it.""" + main_cpp = generate_main(component_config_path(config_file)) + + assert f"set_advertising_required({str(required).lower()})" in main_cpp From ce87bf9b17f5f93171e62a1f6ecbc1ade6ce1132 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:05:26 -0400 Subject: [PATCH 095/147] Bump platformdirs from 4.11.5 to 4.11.7 (#18976) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8a510a2c60..cd3f7446f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ bleak==3.0.2 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 -platformdirs==4.11.5 # native esp-idf toolchain global cache dir +platformdirs==4.11.7 # native esp-idf toolchain global cache dir ninja==1.13.2 # native esp8266 arduino toolchain build driver filelock==3.32.5 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg From d1829c495d2c982eb2f2845406ccfe5b74bd2f64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:05:36 -0400 Subject: [PATCH 096/147] Bump prek from 0.5.0 to 0.5.1 (#18977) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index b1309ec63b..897445a4cb 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,7 @@ pylint==4.0.8 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating ruff==0.16.5 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating -prek==0.5.0 # also change in .github/workflows/ci.yml when updating +prek==0.5.1 # also change in .github/workflows/ci.yml when updating # Unit tests pytest==9.1.1 From b66822d9bd0f741b8d40264e019264d9910fc377 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:21:47 +1000 Subject: [PATCH 097/147] [ai] Advice to agents to limit verbiage (#18980) --- AGENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AGENTS.md b/AGENTS.md index e932c50f32..15b92c4deb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -553,6 +553,7 @@ file does, and it is the authority when they disagree. The most useful starting 4. **Lint:** Run `prek` to ensure code is compliant. 5. **Commit:** Commit your changes. There is no strict format for commit messages. 6. **Pull Request:** Submit a PR against the `dev` branch. The Pull Request title must start with a `[tag]` prefix. For component work, use the component name (e.g., `[display] Fix bug`, `[abc123] Add new component`); for changes to shared/core code that isn't tied to a single component, use `[core]` (e.g., `[core] Add validator`). Update documentation, examples, and add `CODEOWNERS` entries as needed. Pull requests should always be made using the `.github/PULL_REQUEST_TEMPLATE.md` template - fill out all sections completely without removing any parts of the template. + 7. **Comments:** When commenting on GitHub PRs or issues, don't tag contributors, especially bots. Avoid referring to list items (e.g. from reviews) with the form #nn - this will be interpreted by GitHub as a reference to issue or PR nn. Keep comments short and exclude irrelevant details, backstories, restatement of previous comments and anything that is already obvious to the reader. * **Documentation Contributions:** * Documentation is hosted in the separate `esphome/esphome.io` repository. From 13dbbcaa32e94423ff5bf9fe62b6f56cb073a6c5 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Sat, 5 Sep 2026 06:00:25 -0500 Subject: [PATCH 098/147] [usb_uart] Keep the comm interface number valid when its claim fails (#18968) --- esphome/components/usb_uart/usb_uart.cpp | 12 +++++++----- esphome/components/usb_uart/usb_uart.h | 3 +++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/esphome/components/usb_uart/usb_uart.cpp b/esphome/components/usb_uart/usb_uart.cpp index cf66e4c369..60b7fe4e9c 100644 --- a/esphome/components/usb_uart/usb_uart.cpp +++ b/esphome/components/usb_uart/usb_uart.cpp @@ -434,11 +434,12 @@ void USBUartTypeCdcAcm::on_connected() { auto err_comm = usb_host_interface_claim(this->handle_, this->device_handle_, channel->cdc_dev_.interrupt_interface_number, 0); if (err_comm != ESP_OK) { + // Continue anyway: the interface number stays valid for CDC request addressing ESP_LOGW(TAG, "Could not claim comm interface %d: %s", channel->cdc_dev_.interrupt_interface_number, esp_err_to_name(err_comm)); - channel->cdc_dev_.interrupt_interface_number = 0xFF; // Mark as unavailable, but continue anyway } else { ESP_LOGD(TAG, "Claimed comm interface %d", channel->cdc_dev_.interrupt_interface_number); + channel->cdc_dev_.interrupt_interface_claimed = true; } } auto err = @@ -465,14 +466,15 @@ void USBUartTypeCdcAcm::on_disconnected() { usb_host_endpoint_halt(this->device_handle_, channel->cdc_dev_.out_ep->bEndpointAddress); usb_host_endpoint_flush(this->device_handle_, channel->cdc_dev_.out_ep->bEndpointAddress); } - if (channel->cdc_dev_.notify_ep != nullptr) { + // Only tear down the notify pipe when we claimed its interface ourselves; + // no transfer is ever submitted on it, so there is nothing else to cancel. + if (channel->cdc_dev_.notify_ep != nullptr && channel->cdc_dev_.interrupt_interface_claimed) { usb_host_endpoint_halt(this->device_handle_, channel->cdc_dev_.notify_ep->bEndpointAddress); usb_host_endpoint_flush(this->device_handle_, channel->cdc_dev_.notify_ep->bEndpointAddress); } - if (channel->cdc_dev_.interrupt_interface_number != 0xFF && - channel->cdc_dev_.interrupt_interface_number != channel->cdc_dev_.bulk_interface_number) { + if (channel->cdc_dev_.interrupt_interface_claimed) { usb_host_interface_release(this->handle_, this->device_handle_, channel->cdc_dev_.interrupt_interface_number); - channel->cdc_dev_.interrupt_interface_number = 0xFF; + channel->cdc_dev_.interrupt_interface_claimed = false; } usb_host_interface_release(this->handle_, this->device_handle_, channel->cdc_dev_.bulk_interface_number); // Reset the input and output started flags to their initial state to avoid the possibility of spurious restarts diff --git a/esphome/components/usb_uart/usb_uart.h b/esphome/components/usb_uart/usb_uart.h index 00b34fb942..9d87bf964c 100644 --- a/esphome/components/usb_uart/usb_uart.h +++ b/esphome/components/usb_uart/usb_uart.h @@ -34,7 +34,10 @@ struct CdcEps { const usb_ep_desc_t *in_ep; const usb_ep_desc_t *out_ep; uint8_t bulk_interface_number; + // Also the wIndex target for CDC class requests (SET_LINE_CODING etc.), so it + // must remain valid even when the interface itself is not claimed. uint8_t interrupt_interface_number; + bool interrupt_interface_claimed{false}; }; enum CH34xChipType : uint8_t { From 84f78831f95442f124c2f652b644611b15143fbe Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:07:15 +0200 Subject: [PATCH 099/147] Bump bundled esphome-device-builder to 1.14.1 (#18981) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7952616496..2d4ddbef5d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.1 RUN \ platformio settings set enable_telemetry No \ From ae187f81f25fcce1869a8f128c3a929ec07c7f89 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:44:37 +1000 Subject: [PATCH 100/147] [wifi] Allow a forced roam check (#17349) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Claude --- esphome/components/wifi/__init__.py | 16 ++++++++- esphome/components/wifi/automation.h | 5 +++ esphome/components/wifi/wifi_component.cpp | 38 +++++++++++++++------- esphome/components/wifi/wifi_component.h | 6 ++++ tests/components/wifi/common.yaml | 1 + 5 files changed, 54 insertions(+), 12 deletions(-) diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index b8c6d774ac..1691dcc293 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -66,13 +66,14 @@ from esphome.const import ( ) from esphome.core import ( CORE, + ID, CoroPriority, EsphomeError, HexInt, coroutine_with_priority, ) import esphome.final_validate as fv -from esphome.types import ConfigType +from esphome.types import ConfigType, TemplateArgsType from . import wpa2_eap @@ -208,6 +209,7 @@ WiFiEnabledCondition = wifi_ns.class_("WiFiEnabledCondition", Condition) WiFiAPActiveCondition = wifi_ns.class_("WiFiAPActiveCondition", Condition) WiFiEnableAction = wifi_ns.class_("WiFiEnableAction", automation.Action) WiFiDisableAction = wifi_ns.class_("WiFiDisableAction", automation.Action) +WiFiRoamAction = wifi_ns.class_("WiFiRoamAction", automation.Action) WiFiConfigureAction = wifi_ns.class_( "WiFiConfigureAction", automation.Action, cg.Component ) @@ -820,6 +822,18 @@ async def wifi_disable_to_code(config, action_id, template_arg, args): return cg.new_Pvariable(action_id, template_arg) +@automation.register_action( + "wifi.roam", WiFiRoamAction, cv.Schema({}), synchronous=True +) +async def wifi_roam_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> cg.MockObj: + return cg.new_Pvariable(action_id, template_arg) + + KEEP_SCAN_RESULTS_KEY = "wifi_keep_scan_results" RUNTIME_POWER_SAVE_KEY = "wifi_runtime_power_save" RUNTIME_ROAMING_SUPPRESSION_KEY = "wifi_runtime_roaming_suppression" diff --git a/esphome/components/wifi/automation.h b/esphome/components/wifi/automation.h index e63faa18ab..c14341330f 100644 --- a/esphome/components/wifi/automation.h +++ b/esphome/components/wifi/automation.h @@ -31,6 +31,11 @@ template class WiFiDisableAction final : public Action { void play(const Ts &...x) override { global_wifi_component->disable(); } }; +template class WiFiRoamAction final : public Action { + public: + void play(const Ts &...x) override { global_wifi_component->force_roam_check(); } +}; + template class WiFiConfigureAction final : public Action, public Component { public: TEMPLATABLE_VALUE(std::string, ssid) diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index 694e616476..f9e80995e1 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -846,17 +846,18 @@ void WiFiComponent::loop() { this->notify_connect_state_listeners_(); #endif - // Post-connect roaming: check for better AP - if (this->post_connect_roaming_) { - if (this->is_roaming_scan_active()) { - if (this->scan_done_) { - this->process_roaming_scan_(); - } - // else: scan in progress, wait - } else if (this->roaming_state_ == RoamingState::IDLE && this->roaming_attempts_ < ROAMING_MAX_ATTEMPTS && - now - this->roaming_last_check_ >= ROAMING_CHECK_INTERVAL && !this->roaming_suppressed_()) { - this->check_roaming_(now); + // Post-connect roaming: check for better AP. A scan may have been started by an + // explicit force_roam_check() even when post_connect_roaming_ is disabled, so the + // scan must always be consumed here to avoid leaving roaming_state_ stuck. + if (this->is_roaming_scan_active()) { + if (this->scan_done_) { + this->process_roaming_scan_(); } + // else: scan in progress, wait + } else if (this->post_connect_roaming_ && this->roaming_state_ == RoamingState::IDLE && + this->roaming_attempts_ < ROAMING_MAX_ATTEMPTS && + now - this->roaming_last_check_ >= ROAMING_CHECK_INTERVAL && !this->roaming_suppressed_()) { + this->check_roaming_(now); } } break; @@ -2463,6 +2464,17 @@ void WiFiComponent::notify_scan_results_listeners_() { } #endif // USE_WIFI_SCAN_RESULTS_LISTENERS +void WiFiComponent::force_roam_check() { + if (!this->is_connected() || this->roaming_state_ != RoamingState::IDLE || this->roaming_suppressed_()) { + ESP_LOGD(TAG, "Roam check requested, but not able to check now"); + return; + } + // Reset the attempt counter so a prior run of failed roams doesn't block this explicit request + // Note that this re-arms automatic roaming if enabled. + this->roaming_attempts_ = 0; + this->check_roaming_(millis()); +} + void WiFiComponent::check_roaming_(uint32_t now) { // Guard: not for hidden networks (may not appear in scan) const WiFiAP *selected = this->get_selected_sta_(); @@ -2484,7 +2496,11 @@ void WiFiComponent::check_roaming_(uint32_t now) { ESP_LOGD(TAG, "Roam scan (%d dBm, attempt %u/%u)", rssi, this->roaming_attempts_, ROAMING_MAX_ATTEMPTS); this->roaming_state_ = RoamingState::SCANNING; - this->wifi_scan_start_(this->passive_scan_); + if (!this->wifi_scan_start_(this->passive_scan_)) { + // Scan failed to start (e.g. busy) - don't get stuck in SCANNING forever + ESP_LOGD(TAG, "Roam scan failed to start"); + this->roaming_state_ = RoamingState::IDLE; + } } void WiFiComponent::process_roaming_scan_() { diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index 63df9fbfa5..94fdd9bc14 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -565,6 +565,12 @@ class WiFiComponent final : public Component { void set_keep_scan_results(bool keep_scan_results) { this->keep_scan_results_ = keep_scan_results; } void set_post_connect_roaming(bool enabled) { this->post_connect_roaming_ = enabled; } + /** Force an immediate post-connect roaming check, bypassing the periodic interval and the + * per-connection attempt limit. Does nothing (besides a debug log) if not connected, if a + * roam scan or connect is already in progress, or if roaming is currently suppressed. + */ + void force_roam_check(); + #ifdef USE_WIFI_CONNECT_TRIGGER Trigger<> *get_connect_trigger() { return &this->connect_trigger_; } #endif diff --git a/tests/components/wifi/common.yaml b/tests/components/wifi/common.yaml index 10b68347eb..10a8a61c66 100644 --- a/tests/components/wifi/common.yaml +++ b/tests/components/wifi/common.yaml @@ -14,6 +14,7 @@ esphome: condition: wifi.ap_active then: - logger.log: "WiFi AP is active!" + - wifi.roam wifi: networks: From 3ef7460fca9e5326b5d51e0e3bf51c1bcb8abde6 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:25:58 +0000 Subject: [PATCH 101/147] Bump bundled esphome-device-builder to 1.14.2 (#18988) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2d4ddbef5d..b5170864a3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.2 RUN \ platformio settings set enable_telemetry No \ From e3dd2f44a45bc7200566393db51fa17eb0a5edf1 Mon Sep 17 00:00:00 2001 From: elwin loomis Date: Sat, 5 Sep 2026 16:08:21 -0500 Subject: [PATCH 102/147] [mipi_dsi] Let IDF pick the DPHY PLL reference clock (#18984) Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> --- esphome/components/mipi_dsi/mipi_dsi.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/mipi_dsi/mipi_dsi.cpp b/esphome/components/mipi_dsi/mipi_dsi.cpp index 0850b50c85..0150cc2544 100644 --- a/esphome/components/mipi_dsi/mipi_dsi.cpp +++ b/esphome/components/mipi_dsi/mipi_dsi.cpp @@ -35,8 +35,8 @@ void MipiDsi::setup() { .bus_id = 0, // index from 0, specify the DSI host to use .num_data_lanes = this->lanes_, // Number of data lanes to use, can't set a value that exceeds the chip's capability - .phy_clk_src = MIPI_DSI_PHY_CLK_SRC_DEFAULT, // Clock source for the DPHY - .lane_bit_rate_mbps = this->lane_bit_rate_, // Bit rate of the data lanes, in Mbps + // phy_clk_src left at 0 to enable runtime auto-select. + .lane_bit_rate_mbps = this->lane_bit_rate_, // Bit rate of the data lanes, in Mbps }; auto err = esp_lcd_new_dsi_bus(&bus_config, &this->bus_handle_); if (err != ESP_OK) { From e5200db6fd6008da8a1e4b88d8b99e463aae0759 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:28:02 +0200 Subject: [PATCH 103/147] Bump bundled esphome-device-builder to 1.14.3 (#18996) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b5170864a3..e875851bfb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.2 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.3 RUN \ platformio settings set enable_telemetry No \ From 8e1044e8ea35aa959121170d7a8000fcbf90aed2 Mon Sep 17 00:00:00 2001 From: Ricardo Sanz Date: Sun, 6 Sep 2026 23:03:07 +0200 Subject: [PATCH 104/147] [climate][template] New template climate component (#14455) --- esphome/components/climate/__init__.py | 13 + .../components/template/climate/__init__.py | 465 ++++++++++++++++++ .../components/template/climate/automation.h | 57 +++ .../template/climate/template_climate.cpp | 164 ++++++ .../template/climate/template_climate.h | 92 ++++ esphome/config_validation.py | 1 + .../template/test_template_climate.py | 145 ++++++ tests/components/climate/common.yaml | 3 +- tests/components/template/common-base.yaml | 113 +++++ .../fixtures/template_climate_basic.yaml | 72 +++ .../template_climate_custom_modes.yaml | 47 ++ .../template_climate_nonoptimistic.yaml | 56 +++ .../template_climate_on_control_ordering.yaml | 26 + .../template_climate_publish_all_fields.yaml | 63 +++ .../template_climate_sensor_push.yaml | 49 ++ .../template_climate_set_actions.yaml | 89 ++++ ...emplate_climate_two_point_temperature.yaml | 52 ++ .../test_template_climate_basic.py | 146 ++++++ .../test_template_climate_custom_modes.py | 98 ++++ .../test_template_climate_nonoptimistic.py | 107 ++++ ...st_template_climate_on_control_ordering.py | 83 ++++ ...est_template_climate_publish_all_fields.py | 96 ++++ .../test_template_climate_sensor_push.py | 88 ++++ .../test_template_climate_set_actions.py | 114 +++++ ..._template_climate_two_point_temperature.py | 118 +++++ 25 files changed, 2355 insertions(+), 2 deletions(-) create mode 100644 esphome/components/template/climate/__init__.py create mode 100644 esphome/components/template/climate/automation.h create mode 100644 esphome/components/template/climate/template_climate.cpp create mode 100644 esphome/components/template/climate/template_climate.h create mode 100644 tests/component_tests/template/test_template_climate.py create mode 100644 tests/integration/fixtures/template_climate_basic.yaml create mode 100644 tests/integration/fixtures/template_climate_custom_modes.yaml create mode 100644 tests/integration/fixtures/template_climate_nonoptimistic.yaml create mode 100644 tests/integration/fixtures/template_climate_on_control_ordering.yaml create mode 100644 tests/integration/fixtures/template_climate_publish_all_fields.yaml create mode 100644 tests/integration/fixtures/template_climate_sensor_push.yaml create mode 100644 tests/integration/fixtures/template_climate_set_actions.yaml create mode 100644 tests/integration/fixtures/template_climate_two_point_temperature.yaml create mode 100644 tests/integration/test_template_climate_basic.py create mode 100644 tests/integration/test_template_climate_custom_modes.py create mode 100644 tests/integration/test_template_climate_nonoptimistic.py create mode 100644 tests/integration/test_template_climate_on_control_ordering.py create mode 100644 tests/integration/test_template_climate_publish_all_fields.py create mode 100644 tests/integration/test_template_climate_sensor_push.py create mode 100644 tests/integration/test_template_climate_set_actions.py create mode 100644 tests/integration/test_template_climate_two_point_temperature.py diff --git a/esphome/components/climate/__init__.py b/esphome/components/climate/__init__.py index 80dd913fba..3fbca1a6d0 100644 --- a/esphome/components/climate/__init__.py +++ b/esphome/components/climate/__init__.py @@ -125,6 +125,19 @@ CLIMATE_SWING_MODES = { validate_climate_swing_mode = cv.enum(CLIMATE_SWING_MODES, upper=True) +ClimateAction = climate_ns.enum("ClimateAction") +CLIMATE_ACTIONS = { + "OFF": ClimateAction.CLIMATE_ACTION_OFF, + "COOLING": ClimateAction.CLIMATE_ACTION_COOLING, + "HEATING": ClimateAction.CLIMATE_ACTION_HEATING, + "IDLE": ClimateAction.CLIMATE_ACTION_IDLE, + "DRYING": ClimateAction.CLIMATE_ACTION_DRYING, + "FAN": ClimateAction.CLIMATE_ACTION_FAN, + "DEFROSTING": ClimateAction.CLIMATE_ACTION_DEFROSTING, +} + +validate_climate_action = cv.enum(CLIMATE_ACTIONS, upper=True) + CONF_MIN_HUMIDITY = "min_humidity" CONF_MAX_HUMIDITY = "max_humidity" CONF_TARGET_HUMIDITY = "target_humidity" diff --git a/esphome/components/template/climate/__init__.py b/esphome/components/template/climate/__init__.py new file mode 100644 index 0000000000..c39ea8f80e --- /dev/null +++ b/esphome/components/template/climate/__init__.py @@ -0,0 +1,465 @@ +from esphome import automation +import esphome.codegen as cg +from esphome.components import climate, sensor +from esphome.components.climate import climate_ns +import esphome.config_validation as cv +from esphome.const import ( + CONF_ACTION, + CONF_CURRENT_TEMPERATURE, + CONF_CUSTOM_FAN_MODE, + CONF_CUSTOM_FAN_MODES, + CONF_CUSTOM_PRESET, + CONF_CUSTOM_PRESETS, + CONF_FAN_MODE, + CONF_HUMIDITY_SENSOR, + CONF_ID, + CONF_INITIAL_STATE, + CONF_MODE, + CONF_OPTIMISTIC, + CONF_PRESET, + CONF_RESTORE_MODE, + CONF_SENSOR, + CONF_SUPPORTED_FAN_MODES, + CONF_SUPPORTED_MODES, + CONF_SUPPORTED_PRESETS, + CONF_SUPPORTED_SWING_MODES, + CONF_SWING_MODE, + CONF_TARGET_TEMPERATURE, + CONF_TARGET_TEMPERATURE_HIGH, + CONF_TARGET_TEMPERATURE_LOW, +) +from esphome.core import ID +from esphome.cpp_generator import MockObj, TemplateArgsType +from esphome.types import ConfigType + +from .. import template_ns + +CONF_CURRENT_HUMIDITY = "current_humidity" +CONF_TARGET_HUMIDITY = "target_humidity" +CONF_SUPPORTS_ACTION = "supports_action" +CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE = "supports_two_point_target_temperature" +CONF_SUPPORTS_TARGET_HUMIDITY = "supports_target_humidity" +CONF_SUPPORTS_CURRENT_TEMPERATURE = "supports_current_temperature" +CONF_SUPPORTS_CURRENT_HUMIDITY = "supports_current_humidity" +CONF_SET_MODE_ACTION = "set_mode_action" +CONF_SET_TARGET_TEMPERATURE_ACTION = "set_target_temperature_action" +CONF_SET_TARGET_TEMPERATURE_LOW_ACTION = "set_target_temperature_low_action" +CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION = "set_target_temperature_high_action" +CONF_SET_TARGET_HUMIDITY_ACTION = "set_target_humidity_action" +CONF_SET_FAN_MODE_ACTION = "set_fan_mode_action" +CONF_SET_CUSTOM_FAN_MODE_ACTION = "set_custom_fan_mode_action" +CONF_SET_SWING_MODE_ACTION = "set_swing_mode_action" +CONF_SET_PRESET_ACTION = "set_preset_action" +CONF_SET_CUSTOM_PRESET_ACTION = "set_custom_preset_action" + +TemplateClimate = template_ns.class_("TemplateClimate", climate.Climate, cg.Component) +TemplateClimatePublishAction = template_ns.class_( + "TemplateClimatePublishAction", + automation.Action, + cg.Parented.template(TemplateClimate), +) + +TemplateClimateRestoreMode = template_ns.enum( + "TemplateClimateRestoreMode", is_class=True +) +CLIMATE_RESTORE_MODES = { + "NO_RESTORE": TemplateClimateRestoreMode.TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE, + "RESTORE": TemplateClimateRestoreMode.TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE, +} + +# Per-field actions that forward a requested value on. The third item is the type of `x`. +SET_ACTIONS = ( + (CONF_SET_MODE_ACTION, "get_set_mode_trigger", climate.ClimateMode), + ( + CONF_SET_TARGET_TEMPERATURE_ACTION, + "get_set_target_temperature_trigger", + cg.float_, + ), + ( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + "get_set_target_temperature_low_trigger", + cg.float_, + ), + ( + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + "get_set_target_temperature_high_trigger", + cg.float_, + ), + (CONF_SET_TARGET_HUMIDITY_ACTION, "get_set_target_humidity_trigger", cg.float_), + (CONF_SET_FAN_MODE_ACTION, "get_set_fan_mode_trigger", climate.ClimateFanMode), + ( + CONF_SET_CUSTOM_FAN_MODE_ACTION, + "get_set_custom_fan_mode_trigger", + cg.StringRef, + ), + ( + CONF_SET_SWING_MODE_ACTION, + "get_set_swing_mode_trigger", + climate.ClimateSwingMode, + ), + (CONF_SET_PRESET_ACTION, "get_set_preset_trigger", climate.ClimatePreset), + (CONF_SET_CUSTOM_PRESET_ACTION, "get_set_custom_preset_trigger", cg.StringRef), +) + +# supports_* keys have no default so that an omitted key can mean "derive it from the sensor or +# set action that makes the trait useful", which is not expressible once a default fills it in. +DERIVED_SUPPORTS = ( + (CONF_SUPPORTS_CURRENT_TEMPERATURE, (CONF_SENSOR,)), + (CONF_SUPPORTS_CURRENT_HUMIDITY, (CONF_HUMIDITY_SENSOR,)), + ( + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + ( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + ), + ), + (CONF_SUPPORTS_TARGET_HUMIDITY, (CONF_SET_TARGET_HUMIDITY_ACTION,)), +) + + +# Custom fan modes/presets are opaque user-defined strings with no build-time correctness check +# elsewhere (Climate::set_supported_custom_fan_modes()/set_supported_custom_presets() don't block +# empty entries), so reject empty ones here -- they could never be selected at runtime anyway. +validate_custom_climate_string = cv.All(cv.string_strict, cv.Length(min=1)) + + +def _validate_two_point(config: ConfigType) -> ConfigType: + has_low = CONF_TARGET_TEMPERATURE_LOW in config + has_high = CONF_TARGET_TEMPERATURE_HIGH in config + if has_low != has_high: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE_LOW}' and '{CONF_TARGET_TEMPERATURE_HIGH}' must be used together" + ) + if (has_low or has_high) and CONF_TARGET_TEMPERATURE in config: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE}' cannot be used together with " + f"'{CONF_TARGET_TEMPERATURE_LOW}'/'{CONF_TARGET_TEMPERATURE_HIGH}'" + ) + return config + + +def _validate_set_actions(config: ConfigType) -> ConfigType: + has_low = CONF_SET_TARGET_TEMPERATURE_LOW_ACTION in config + has_high = CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION in config + if has_low != has_high: + raise cv.Invalid( + f"'{CONF_SET_TARGET_TEMPERATURE_LOW_ACTION}' and " + f"'{CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION}' must be used together" + ) + if (has_low or has_high) and CONF_SET_TARGET_TEMPERATURE_ACTION in config: + raise cv.Invalid( + f"'{CONF_SET_TARGET_TEMPERATURE_ACTION}' cannot be used together with " + f"'{CONF_SET_TARGET_TEMPERATURE_LOW_ACTION}'/'{CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION}'" + ) + return config + + +def _resolve_supports(config: ConfigType) -> ConfigType: + # An explicit true stays valid without either, since climate.template.publish can report the + # value; an explicit false that contradicts the configuration is an error, not a silent override. + for key, sources in DERIVED_SUPPORTS: + configured = [source for source in sources if source in config] + if key not in config: + config[key] = bool(configured) + elif not config[key] and configured: + raise cv.Invalid( + f"'{key}' cannot be false while '{configured[0]}' is configured", + path=[key], + ) + return config + + +def _validate_initial_state(config: ConfigType) -> ConfigType: + # Climate keeps target_temperature and target_temperature_low in a union, so writing the wrong + # one of the pair corrupts the setpoint with no runtime complaint. + if (initial_state := config.get(CONF_INITIAL_STATE)) is None: + return config + + two_point = config[CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE] + if two_point and CONF_TARGET_TEMPERATURE in initial_state: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE}' is not available while " + f"'{CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE}' is enabled; use " + f"'{CONF_TARGET_TEMPERATURE_LOW}'/'{CONF_TARGET_TEMPERATURE_HIGH}' instead", + path=[CONF_INITIAL_STATE, CONF_TARGET_TEMPERATURE], + ) + if not two_point: + for key in (CONF_TARGET_TEMPERATURE_LOW, CONF_TARGET_TEMPERATURE_HIGH): + if key in initial_state: + raise cv.Invalid( + f"'{key}' requires '{CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE}' to be enabled", + path=[CONF_INITIAL_STATE, key], + ) + if ( + CONF_TARGET_HUMIDITY in initial_state + and not config[CONF_SUPPORTS_TARGET_HUMIDITY] + ): + raise cv.Invalid( + f"'{CONF_TARGET_HUMIDITY}' requires '{CONF_SUPPORTS_TARGET_HUMIDITY}' to be enabled", + path=[CONF_INITIAL_STATE, CONF_TARGET_HUMIDITY], + ) + return config + + +# Same settable fields as climate.template.publish, minus current_temperature/current_humidity/ +# action: those are reported values (from a sensor or the device), not meaningful static defaults. +INITIAL_STATE_SCHEMA = cv.All( + cv.Schema( + { + cv.Optional(CONF_MODE): climate.validate_climate_mode, + cv.Optional(CONF_TARGET_TEMPERATURE): cv.temperature, + cv.Optional(CONF_TARGET_TEMPERATURE_LOW): cv.temperature, + cv.Optional(CONF_TARGET_TEMPERATURE_HIGH): cv.temperature, + cv.Optional(CONF_TARGET_HUMIDITY): cv.percentage_int, + cv.Exclusive(CONF_FAN_MODE, "fan_mode"): climate.validate_climate_fan_mode, + cv.Exclusive( + CONF_CUSTOM_FAN_MODE, "fan_mode" + ): validate_custom_climate_string, + cv.Optional(CONF_SWING_MODE): climate.validate_climate_swing_mode, + cv.Exclusive(CONF_PRESET, "preset"): climate.validate_climate_preset, + cv.Exclusive(CONF_CUSTOM_PRESET, "preset"): validate_custom_climate_string, + } + ), + _validate_two_point, +) + +CONFIG_SCHEMA = cv.All( + climate.climate_schema(TemplateClimate) + .extend( + { + cv.Optional(CONF_SENSOR): cv.use_id(sensor.Sensor), + cv.Optional(CONF_HUMIDITY_SENSOR): cv.use_id(sensor.Sensor), + # action only ever arrives through climate.template.publish, so unlike the other + # supports_* keys there is no set action to derive it from. + cv.Optional(CONF_SUPPORTS_ACTION, default=False): cv.boolean, + cv.Optional(CONF_SUPPORTS_CURRENT_TEMPERATURE): cv.boolean, + cv.Optional(CONF_SUPPORTS_CURRENT_HUMIDITY): cv.boolean, + cv.Optional(CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE): cv.boolean, + cv.Optional(CONF_SUPPORTS_TARGET_HUMIDITY): cv.boolean, + cv.Required(CONF_SUPPORTED_MODES): cv.All( + cv.ensure_list(climate.validate_climate_mode), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_FAN_MODES): cv.All( + cv.ensure_list(climate.validate_climate_fan_mode), cv.Unique() + ), + cv.Optional(CONF_CUSTOM_FAN_MODES): cv.All( + cv.ensure_list(validate_custom_climate_string), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_SWING_MODES): cv.All( + cv.ensure_list(climate.validate_climate_swing_mode), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_PRESETS): cv.All( + cv.ensure_list(climate.validate_climate_preset), cv.Unique() + ), + cv.Optional(CONF_CUSTOM_PRESETS): cv.All( + cv.ensure_list(validate_custom_climate_string), cv.Unique() + ), + cv.Optional(CONF_OPTIMISTIC, default=True): cv.boolean, + cv.Optional(CONF_RESTORE_MODE, default="RESTORE"): cv.enum( + CLIMATE_RESTORE_MODES, upper=True + ), + cv.Optional(CONF_INITIAL_STATE): INITIAL_STATE_SCHEMA, + cv.Optional(CONF_SET_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_HUMIDITY_ACTION + ): automation.validate_automation(single=True), + cv.Optional(CONF_SET_FAN_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional( + CONF_SET_CUSTOM_FAN_MODE_ACTION + ): automation.validate_automation(single=True), + cv.Optional(CONF_SET_SWING_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional(CONF_SET_PRESET_ACTION): automation.validate_automation( + single=True + ), + cv.Optional(CONF_SET_CUSTOM_PRESET_ACTION): automation.validate_automation( + single=True + ), + } + ) + .extend(cv.COMPONENT_SCHEMA), + _validate_set_actions, + _resolve_supports, + _validate_initial_state, +) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await climate.register_climate(var, config) + + if (sens := config.get(CONF_SENSOR)) is not None: + cg.add(var.set_sensor(await cg.get_variable(sens))) + + if (sens := config.get(CONF_HUMIDITY_SENSOR)) is not None: + cg.add(var.set_humidity_sensor(await cg.get_variable(sens))) + + for key, flag in ( + (CONF_SUPPORTS_ACTION, climate_ns.CLIMATE_SUPPORTS_ACTION), + ( + CONF_SUPPORTS_CURRENT_TEMPERATURE, + climate_ns.CLIMATE_SUPPORTS_CURRENT_TEMPERATURE, + ), + (CONF_SUPPORTS_CURRENT_HUMIDITY, climate_ns.CLIMATE_SUPPORTS_CURRENT_HUMIDITY), + ( + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + climate_ns.CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + ), + (CONF_SUPPORTS_TARGET_HUMIDITY, climate_ns.CLIMATE_SUPPORTS_TARGET_HUMIDITY), + ): + if config[key]: + cg.add(var.add_feature_flags(flag)) + + for mode in config[CONF_SUPPORTED_MODES]: + cg.add(var.add_supported_mode(mode)) + + for mode in config.get(CONF_SUPPORTED_FAN_MODES, []): + cg.add(var.add_supported_fan_mode(mode)) + + if CONF_CUSTOM_FAN_MODES in config: + cg.add( + var.set_supported_custom_fan_modes( + cg.ArrayInitializer(*config[CONF_CUSTOM_FAN_MODES]) + ) + ) + + for mode in config.get(CONF_SUPPORTED_SWING_MODES, []): + cg.add(var.add_supported_swing_mode(mode)) + + for preset in config.get(CONF_SUPPORTED_PRESETS, []): + cg.add(var.add_supported_preset(preset)) + + if CONF_CUSTOM_PRESETS in config: + cg.add( + var.set_supported_custom_presets( + cg.ArrayInitializer(*config[CONF_CUSTOM_PRESETS]) + ) + ) + + for key, trigger_getter, arg_type in SET_ACTIONS: + if (conf := config.get(key)) is not None: + await automation.build_automation( + getattr(var, trigger_getter)(), [(arg_type, "x")], conf + ) + + cg.add(var.set_optimistic(config[CONF_OPTIMISTIC])) + cg.add(var.set_restore_mode(config[CONF_RESTORE_MODE])) + + if (initial_state := config.get(CONF_INITIAL_STATE)) is not None: + if (v := initial_state.get(CONF_MODE)) is not None: + cg.add(var.set_mode(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE)) is not None: + cg.add(var.set_target_temperature(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE_LOW)) is not None: + cg.add(var.set_target_temperature_low(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE_HIGH)) is not None: + cg.add(var.set_target_temperature_high(v)) + if (v := initial_state.get(CONF_TARGET_HUMIDITY)) is not None: + cg.add(var.set_target_humidity(v)) + if (v := initial_state.get(CONF_FAN_MODE)) is not None: + cg.add(var.set_fan_mode(v)) + if (v := initial_state.get(CONF_CUSTOM_FAN_MODE)) is not None: + cg.add(var.set_custom_fan_mode(v)) + if (v := initial_state.get(CONF_SWING_MODE)) is not None: + cg.add(var.set_swing_mode(v)) + if (v := initial_state.get(CONF_PRESET)) is not None: + cg.add(var.set_preset(v)) + if (v := initial_state.get(CONF_CUSTOM_PRESET)) is not None: + cg.add(var.set_custom_preset(v)) + + +CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.use_id(TemplateClimate), + cv.Optional(CONF_CURRENT_TEMPERATURE): cv.templatable(cv.temperature), + cv.Optional(CONF_CURRENT_HUMIDITY): cv.templatable(cv.percentage_int), + cv.Optional(CONF_TARGET_TEMPERATURE): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_TEMPERATURE_LOW): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_TEMPERATURE_HIGH): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_HUMIDITY): cv.templatable(cv.percentage_int), + cv.Optional(CONF_MODE): cv.templatable(climate.validate_climate_mode), + cv.Optional(CONF_ACTION): cv.templatable(climate.validate_climate_action), + cv.Exclusive(CONF_FAN_MODE, "fan_mode"): cv.templatable( + climate.validate_climate_fan_mode + ), + cv.Exclusive(CONF_CUSTOM_FAN_MODE, "fan_mode"): cv.templatable( + validate_custom_climate_string + ), + cv.Optional(CONF_SWING_MODE): cv.templatable( + climate.validate_climate_swing_mode + ), + cv.Exclusive(CONF_PRESET, "preset"): cv.templatable( + climate.validate_climate_preset + ), + cv.Exclusive(CONF_CUSTOM_PRESET, "preset"): cv.templatable( + validate_custom_climate_string + ), + } + ), + _validate_two_point, +) + + +@automation.register_action( + "climate.template.publish", + TemplateClimatePublishAction, + CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA, + synchronous=True, +) +async def climate_template_publish_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + + if (v := config.get(CONF_CURRENT_TEMPERATURE)) is not None: + cg.add(var.set_current_temperature(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_CURRENT_HUMIDITY)) is not None: + cg.add(var.set_current_humidity(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE)) is not None: + cg.add(var.set_target_temperature(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE_LOW)) is not None: + cg.add(var.set_target_temperature_low(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE_HIGH)) is not None: + cg.add( + var.set_target_temperature_high(await cg.templatable(v, args, cg.float_)) + ) + if (v := config.get(CONF_TARGET_HUMIDITY)) is not None: + cg.add(var.set_target_humidity(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_MODE)) is not None: + cg.add(var.set_mode(await cg.templatable(v, args, climate.ClimateMode))) + if (v := config.get(CONF_ACTION)) is not None: + cg.add(var.set_action(await cg.templatable(v, args, climate.ClimateAction))) + if (v := config.get(CONF_FAN_MODE)) is not None: + cg.add(var.set_fan_mode(await cg.templatable(v, args, climate.ClimateFanMode))) + if (v := config.get(CONF_CUSTOM_FAN_MODE)) is not None: + cg.add(var.set_custom_fan_mode(await cg.templatable(v, args, cg.std_string))) + if (v := config.get(CONF_SWING_MODE)) is not None: + cg.add( + var.set_swing_mode(await cg.templatable(v, args, climate.ClimateSwingMode)) + ) + if (v := config.get(CONF_PRESET)) is not None: + cg.add(var.set_preset(await cg.templatable(v, args, climate.ClimatePreset))) + if (v := config.get(CONF_CUSTOM_PRESET)) is not None: + cg.add(var.set_custom_preset(await cg.templatable(v, args, cg.std_string))) + + return var diff --git a/esphome/components/template/climate/automation.h b/esphome/components/template/climate/automation.h new file mode 100644 index 0000000000..49a79ace2f --- /dev/null +++ b/esphome/components/template/climate/automation.h @@ -0,0 +1,57 @@ +#pragma once + +#include "template_climate.h" +#include "esphome/core/automation.h" + +namespace esphome::template_ { + +template +class TemplateClimatePublishAction final : public Action, public Parented { + public: + TEMPLATABLE_VALUE(float, current_temperature) + TEMPLATABLE_VALUE(float, current_humidity) + TEMPLATABLE_VALUE(float, target_temperature) + TEMPLATABLE_VALUE(float, target_temperature_low) + TEMPLATABLE_VALUE(float, target_temperature_high) + TEMPLATABLE_VALUE(float, target_humidity) + TEMPLATABLE_VALUE(climate::ClimateMode, mode) + TEMPLATABLE_VALUE(climate::ClimateAction, action) + TEMPLATABLE_VALUE(climate::ClimateFanMode, fan_mode) + TEMPLATABLE_VALUE(std::string, custom_fan_mode) + TEMPLATABLE_VALUE(climate::ClimateSwingMode, swing_mode) + TEMPLATABLE_VALUE(climate::ClimatePreset, preset) + TEMPLATABLE_VALUE(std::string, custom_preset) + + void play(const Ts &...x) override { + if (this->current_temperature_.has_value()) + this->parent_->current_temperature = this->current_temperature_.value(x...); + if (this->current_humidity_.has_value()) + this->parent_->current_humidity = this->current_humidity_.value(x...); + if (this->target_temperature_.has_value()) + this->parent_->set_target_temperature(this->target_temperature_.value(x...)); + if (this->target_temperature_low_.has_value()) + this->parent_->set_target_temperature_low(this->target_temperature_low_.value(x...)); + if (this->target_temperature_high_.has_value()) + this->parent_->set_target_temperature_high(this->target_temperature_high_.value(x...)); + if (this->target_humidity_.has_value()) + this->parent_->set_target_humidity(this->target_humidity_.value(x...)); + if (this->mode_.has_value()) + this->parent_->set_mode(this->mode_.value(x...)); + if (this->action_.has_value()) + this->parent_->action = this->action_.value(x...); + if (this->fan_mode_.has_value()) + this->parent_->set_fan_mode(this->fan_mode_.value(x...)); + if (this->custom_fan_mode_.has_value()) + this->parent_->set_custom_fan_mode(StringRef(this->custom_fan_mode_.value(x...))); + if (this->swing_mode_.has_value()) + this->parent_->set_swing_mode(this->swing_mode_.value(x...)); + if (this->preset_.has_value()) + this->parent_->set_preset(this->preset_.value(x...)); + if (this->custom_preset_.has_value()) + this->parent_->set_custom_preset(StringRef(this->custom_preset_.value(x...))); + + this->parent_->publish_state(); + } +}; + +} // namespace esphome::template_ diff --git a/esphome/components/template/climate/template_climate.cpp b/esphome/components/template/climate/template_climate.cpp new file mode 100644 index 0000000000..a7a4d2ccab --- /dev/null +++ b/esphome/components/template/climate/template_climate.cpp @@ -0,0 +1,164 @@ +#include "template_climate.h" +#include "esphome/core/log.h" + +namespace esphome::template_ { + +static const char *const TAG = "template.climate"; + +void TemplateClimate::setup() { + if (this->restore_mode_ == TemplateClimateRestoreMode::TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE) { + auto restore = this->restore_state_(); + if (restore.has_value()) { + restore->apply(this); + } + } + + // Sensors publish every reading, not just changes, so only re-publish when the value moved. + // NAN means the sensor went unavailable and is passed through rather than dropped; the second + // check stops an unavailable sensor re-publishing forever, since NAN never equals NAN. +#ifdef USE_SENSOR + if (this->sensor_ != nullptr) { + this->current_temperature = this->sensor_->state; + this->sensor_->add_on_state_callback([this](float state) { + if (state != this->current_temperature && !(std::isnan(state) && std::isnan(this->current_temperature))) { + this->current_temperature = state; + this->publish_state(); + } + }); + } + + if (this->humidity_sensor_ != nullptr) { + this->current_humidity = this->humidity_sensor_->state; + this->humidity_sensor_->add_on_state_callback([this](float state) { + if (state != this->current_humidity && !(std::isnan(state) && std::isnan(this->current_humidity))) { + this->current_humidity = state; + this->publish_state(); + } + }); + } +#endif +} + +void TemplateClimate::dump_config() { + LOG_CLIMATE("", "Template Climate", this); + ESP_LOGCONFIG(TAG, " Optimistic: %s", YESNO(this->optimistic_)); +} + +void TemplateClimate::control(const climate::ClimateCall &call) { + // Each field present fires its set_*_action; on_control sees the whole call. optimistic: true + // also applies the values right away, false waits for a climate.template.publish report. + if (auto mode = call.get_mode()) { + if (this->optimistic_) + this->mode = *mode; + this->set_mode_trigger_.trigger(*mode); + } + + if (auto target_temp = call.get_target_temperature()) { + if (this->optimistic_) + this->target_temperature = *target_temp; + this->set_target_temperature_trigger_.trigger(*target_temp); + } + + if (auto target_temp_low = call.get_target_temperature_low()) { + if (this->optimistic_) + this->target_temperature_low = *target_temp_low; + this->set_target_temperature_low_trigger_.trigger(*target_temp_low); + } + + if (auto target_temp_high = call.get_target_temperature_high()) { + if (this->optimistic_) + this->target_temperature_high = *target_temp_high; + this->set_target_temperature_high_trigger_.trigger(*target_temp_high); + } + + if (auto target_humidity = call.get_target_humidity()) { + if (this->optimistic_) + this->target_humidity = *target_humidity; + this->set_target_humidity_trigger_.trigger(*target_humidity); + } + + if (auto fan_mode = call.get_fan_mode()) { + if (this->optimistic_) + this->set_fan_mode_(*fan_mode); + this->set_fan_mode_trigger_.trigger(*fan_mode); + } + + if (call.has_custom_fan_mode()) { + if (this->optimistic_) + this->set_custom_fan_mode_(call.get_custom_fan_mode()); + this->set_custom_fan_mode_trigger_.trigger(call.get_custom_fan_mode()); + } + + if (auto swing_mode = call.get_swing_mode()) { + if (this->optimistic_) + this->swing_mode = *swing_mode; + this->set_swing_mode_trigger_.trigger(*swing_mode); + } + + if (auto preset = call.get_preset()) { + if (this->optimistic_) + this->set_preset_(*preset); + this->set_preset_trigger_.trigger(*preset); + } + + if (call.has_custom_preset()) { + if (this->optimistic_) + this->set_custom_preset_(call.get_custom_preset()); + this->set_custom_preset_trigger_.trigger(call.get_custom_preset()); + } + + if (this->optimistic_) + this->publish_state(); +} + +// A climate.template.publish report (and initial_state:) never goes through ClimateCall::validate_(), +// so check here instead -- otherwise a typo is published as state the receiving end will reject. +void TemplateClimate::set_mode(climate::ClimateMode mode) { + if (!this->traits_.supports_mode(mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported mode %u", this->get_name().c_str(), static_cast(mode)); + return; + } + this->mode = mode; +} + +void TemplateClimate::set_swing_mode(climate::ClimateSwingMode swing_mode) { + if (!this->traits_.supports_swing_mode(swing_mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported swing mode %u", this->get_name().c_str(), static_cast(swing_mode)); + return; + } + this->swing_mode = swing_mode; +} + +void TemplateClimate::set_fan_mode(climate::ClimateFanMode fan_mode) { + if (!this->traits_.supports_fan_mode(fan_mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported fan mode %u", this->get_name().c_str(), static_cast(fan_mode)); + return; + } + this->set_fan_mode_(fan_mode); +} + +void TemplateClimate::set_preset(climate::ClimatePreset preset) { + if (!this->traits_.supports_preset(preset)) { + ESP_LOGW(TAG, "'%s' - Unsupported preset %u", this->get_name().c_str(), static_cast(preset)); + return; + } + this->set_preset_(preset); +} + +void TemplateClimate::set_custom_fan_mode(StringRef mode) { + if (this->find_custom_fan_mode_(mode.c_str(), mode.size()) == nullptr) { + ESP_LOGW(TAG, "'%s' - Unsupported custom fan mode '%s'", this->get_name().c_str(), mode.c_str()); + return; + } + this->set_custom_fan_mode_(mode); +} + +void TemplateClimate::set_custom_preset(StringRef preset) { + if (this->find_custom_preset_(preset.c_str(), preset.size()) == nullptr) { + ESP_LOGW(TAG, "'%s' - Unsupported custom preset '%s'", this->get_name().c_str(), preset.c_str()); + return; + } + this->set_custom_preset_(preset); +} + +} // namespace esphome::template_ diff --git a/esphome/components/template/climate/template_climate.h b/esphome/components/template/climate/template_climate.h new file mode 100644 index 0000000000..5448488c34 --- /dev/null +++ b/esphome/components/template/climate/template_climate.h @@ -0,0 +1,92 @@ +#pragma once + +#include "esphome/core/automation.h" +#include "esphome/core/component.h" +#include "esphome/components/climate/climate.h" +#ifdef USE_SENSOR +#include "esphome/components/sensor/sensor.h" +#endif + +namespace esphome::template_ { + +enum class TemplateClimateRestoreMode { + TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE, + TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE, +}; + +class TemplateClimate final : public climate::Climate, public Component { + public: + void setup() override; + void dump_config() override; + + climate::ClimateTraits traits() override { return this->traits_; } + + void add_feature_flags(uint32_t flags) { this->traits_.add_feature_flags(flags); } + +#ifdef USE_SENSOR + // The matching feature flag is added from codegen, so the configuration alone decides it. + void set_sensor(sensor::Sensor *sensor) { this->sensor_ = sensor; } + void set_humidity_sensor(sensor::Sensor *sensor) { this->humidity_sensor_ = sensor; } +#endif + + void add_supported_mode(climate::ClimateMode mode) { this->traits_.add_supported_mode(mode); } + void add_supported_fan_mode(climate::ClimateFanMode mode) { this->traits_.add_supported_fan_mode(mode); } + void add_supported_swing_mode(climate::ClimateSwingMode mode) { this->traits_.add_supported_swing_mode(mode); } + void add_supported_preset(climate::ClimatePreset preset) { this->traits_.add_supported_preset(preset); } + + void set_optimistic(bool optimistic) { this->optimistic_ = optimistic; } + void set_restore_mode(TemplateClimateRestoreMode restore_mode) { this->restore_mode_ = restore_mode; } + + // Fired from control() for each field the call carries, so a device-backed config can forward + // it on. Which of these are configured also decides the two-point/target-humidity traits. + Trigger *get_set_mode_trigger() { return &this->set_mode_trigger_; } + Trigger *get_set_target_temperature_trigger() { return &this->set_target_temperature_trigger_; } + Trigger *get_set_target_temperature_low_trigger() { return &this->set_target_temperature_low_trigger_; } + Trigger *get_set_target_temperature_high_trigger() { return &this->set_target_temperature_high_trigger_; } + Trigger *get_set_target_humidity_trigger() { return &this->set_target_humidity_trigger_; } + Trigger *get_set_fan_mode_trigger() { return &this->set_fan_mode_trigger_; } + Trigger *get_set_custom_fan_mode_trigger() { return &this->set_custom_fan_mode_trigger_; } + Trigger *get_set_swing_mode_trigger() { return &this->set_swing_mode_trigger_; } + Trigger *get_set_preset_trigger() { return &this->set_preset_trigger_; } + Trigger *get_set_custom_preset_trigger() { return &this->set_custom_preset_trigger_; } + + // Used by TemplateClimatePublishAction, which is not a Climate subclass and so cannot reach the + // protected setters, and by codegen to apply `initial_state:` before setup() runs. + void set_target_temperature(float value) { this->target_temperature = value; } + void set_target_temperature_low(float value) { this->target_temperature_low = value; } + void set_target_temperature_high(float value) { this->target_temperature_high = value; } + void set_target_humidity(float value) { this->target_humidity = value; } + void set_mode(climate::ClimateMode mode); + void set_swing_mode(climate::ClimateSwingMode mode); + void set_fan_mode(climate::ClimateFanMode mode); + void set_custom_fan_mode(const char *mode) { this->set_custom_fan_mode(StringRef(mode)); } + void set_custom_fan_mode(StringRef mode); + void set_preset(climate::ClimatePreset preset); + void set_custom_preset(const char *preset) { this->set_custom_preset(StringRef(preset)); } + void set_custom_preset(StringRef preset); + + protected: + void control(const climate::ClimateCall &call) override; + + climate::ClimateTraits traits_; + bool optimistic_{false}; + TemplateClimateRestoreMode restore_mode_{TemplateClimateRestoreMode::TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE}; + +#ifdef USE_SENSOR + sensor::Sensor *sensor_{nullptr}; + sensor::Sensor *humidity_sensor_{nullptr}; +#endif + + Trigger set_mode_trigger_; + Trigger set_target_temperature_trigger_; + Trigger set_target_temperature_low_trigger_; + Trigger set_target_temperature_high_trigger_; + Trigger set_target_humidity_trigger_; + Trigger set_fan_mode_trigger_; + Trigger set_custom_fan_mode_trigger_; + Trigger set_swing_mode_trigger_; + Trigger set_preset_trigger_; + Trigger set_custom_preset_trigger_; +}; + +} // namespace esphome::template_ diff --git a/esphome/config_validation.py b/esphome/config_validation.py index aff39201e8..685a9d04b3 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -133,6 +133,7 @@ Upper = vol.Upper Length = vol.Length Exclusive = vol.Exclusive Inclusive = vol.Inclusive +Unique = vol.Unique ALLOW_EXTRA = vol.ALLOW_EXTRA UNDEFINED = vol.UNDEFINED RequiredFieldInvalid = vol.RequiredFieldInvalid diff --git a/tests/component_tests/template/test_template_climate.py b/tests/component_tests/template/test_template_climate.py new file mode 100644 index 0000000000..304991ea64 --- /dev/null +++ b/tests/component_tests/template/test_template_climate.py @@ -0,0 +1,145 @@ +"""Tests for template climate config validation.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components.template.climate import ( + CONF_SET_TARGET_HUMIDITY_ACTION, + CONF_SET_TARGET_TEMPERATURE_ACTION, + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + CONF_SUPPORTS_CURRENT_HUMIDITY, + CONF_SUPPORTS_CURRENT_TEMPERATURE, + CONF_SUPPORTS_TARGET_HUMIDITY, + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + CONF_TARGET_HUMIDITY, + _resolve_supports, + _validate_initial_state, + _validate_set_actions, +) +from esphome.const import ( + CONF_HUMIDITY_SENSOR, + CONF_INITIAL_STATE, + CONF_SENSOR, + CONF_TARGET_TEMPERATURE, + CONF_TARGET_TEMPERATURE_HIGH, + CONF_TARGET_TEMPERATURE_LOW, +) +from esphome.types import ConfigType + + +def test_supports_current_temperature_derived_from_sensor() -> None: + config: ConfigType = {CONF_SENSOR: "some_sensor"} + assert _resolve_supports(config)[CONF_SUPPORTS_CURRENT_TEMPERATURE] is True + + +def test_supports_current_temperature_false_without_sensor() -> None: + assert _resolve_supports({})[CONF_SUPPORTS_CURRENT_TEMPERATURE] is False + + +def test_supports_current_temperature_explicit_true_without_sensor_allowed() -> None: + # The value can still be reported with climate.template.publish. + config: ConfigType = {CONF_SUPPORTS_CURRENT_TEMPERATURE: True} + assert _resolve_supports(config)[CONF_SUPPORTS_CURRENT_TEMPERATURE] is True + + +def test_supports_current_temperature_false_with_sensor_rejected() -> None: + config: ConfigType = { + CONF_SENSOR: "some_sensor", + CONF_SUPPORTS_CURRENT_TEMPERATURE: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_supports_current_humidity_false_with_sensor_rejected() -> None: + config: ConfigType = { + CONF_HUMIDITY_SENSOR: "some_sensor", + CONF_SUPPORTS_CURRENT_HUMIDITY: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_two_point_derived_from_set_actions() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION: [{}], + } + assert _resolve_supports(config)[CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE] is True + + +def test_two_point_false_with_set_action_rejected() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_target_humidity_derived_from_set_action() -> None: + config: ConfigType = {CONF_SET_TARGET_HUMIDITY_ACTION: [{}]} + assert _resolve_supports(config)[CONF_SUPPORTS_TARGET_HUMIDITY] is True + + +def test_set_target_temperature_low_requires_high() -> None: + config: ConfigType = {CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}]} + with pytest.raises(cv.Invalid, match="must be used together"): + _validate_set_actions(config) + + +def test_set_target_temperature_conflicts_with_two_point_actions() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION: [{}], + } + with pytest.raises(cv.Invalid, match="cannot be used together"): + _validate_set_actions(config) + + +def test_initial_state_target_temperature_rejected_with_two_point() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: True, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: {CONF_TARGET_TEMPERATURE: 21.0}, + } + with pytest.raises(cv.Invalid, match="is not available"): + _validate_initial_state(config) + + +def test_initial_state_two_point_values_rejected_without_two_point() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: { + CONF_TARGET_TEMPERATURE_LOW: 18.0, + CONF_TARGET_TEMPERATURE_HIGH: 24.0, + }, + } + with pytest.raises(cv.Invalid, match="requires"): + _validate_initial_state(config) + + +def test_initial_state_target_humidity_rejected_without_support() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: {CONF_TARGET_HUMIDITY: 50}, + } + with pytest.raises(cv.Invalid, match="requires"): + _validate_initial_state(config) + + +def test_initial_state_matching_two_point_accepted() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: True, + CONF_SUPPORTS_TARGET_HUMIDITY: True, + CONF_INITIAL_STATE: { + CONF_TARGET_TEMPERATURE_LOW: 18.0, + CONF_TARGET_TEMPERATURE_HIGH: 24.0, + CONF_TARGET_HUMIDITY: 50, + }, + } + assert _validate_initial_state(config) is config diff --git a/tests/components/climate/common.yaml b/tests/components/climate/common.yaml index c28fde8eeb..49386a16d5 100644 --- a/tests/components/climate/common.yaml +++ b/tests/components/climate/common.yaml @@ -30,8 +30,7 @@ climate: - switch.turn_on: climate_heater_switch - switch.turn_off: climate_cooler_switch # Thermostat-based climate so climate.control: action variants get build - # coverage (bang_bang doesn't support fan modes, presets, etc.). Climate - # has no template platform, so thermostat is the right vehicle. + # coverage (bang_bang doesn't support fan modes, presets, etc.). - platform: thermostat id: climate_test_thermostat name: Test Thermostat diff --git a/tests/components/template/common-base.yaml b/tests/components/template/common-base.yaml index 92a1fc8eda..02aedaf167 100644 --- a/tests/components/template/common-base.yaml +++ b/tests/components/template/common-base.yaml @@ -25,6 +25,27 @@ esphome: away: !lambda "return true;" is_on: !lambda "return false;" + - climate.template.publish: + id: template_climate + current_temperature: 21.0 + mode: HEAT + fan_mode: AUTO + swing_mode: "OFF" + preset: NONE + target_temperature: 22.0 + + # Templated + - climate.template.publish: + id: template_climate + current_temperature: !lambda "return 21.5f;" + mode: !lambda "return climate::CLIMATE_MODE_COOL;" + target_temperature: !lambda "return 23.0f;" + + - climate.template.publish: + id: template_climate_custom_modes + custom_fan_mode: "turbo" + custom_preset: "eco_plus" + # Test C++ API: set_template() with stateless lambda (no captures) # NOTE: set_template() is not intended to be a public API, but we test it to ensure it doesn't break. - lambda: |- @@ -513,6 +534,98 @@ alarm_control_panel: codes: - "1234" +climate: + - platform: template + id: template_climate + name: "Template Climate" + optimistic: true + sensor: template_template_sens + supports_action: true + supports_current_humidity: true + restore_mode: NO_RESTORE + initial_state: + mode: HEAT + target_temperature: 21.0 + fan_mode: LOW + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + set_mode_action: + - logger.log: + format: "set_mode_action %d" + args: ["(int) x"] + set_target_temperature_action: + - logger.log: + format: "set_target_temperature_action %.1f" + args: ["x"] + set_target_humidity_action: + - logger.log: + format: "set_target_humidity_action %.1f" + args: ["x"] + set_fan_mode_action: + - logger.log: + format: "set_fan_mode_action %d" + args: ["(int) x"] + set_swing_mode_action: + - logger.log: + format: "set_swing_mode_action %d" + args: ["(int) x"] + set_preset_action: + - logger.log: + format: "set_preset_action %d" + args: ["(int) x"] + on_control: + - logger.log: "on_control fired" + on_state: + - logger.log: "on_state fired" + + - platform: template + id: template_climate_custom_modes + name: "Template Climate Custom Modes" + optimistic: true + sensor: template_template_sens + supported_modes: + - "OFF" + - HEAT + custom_fan_modes: + - turbo + - silent + - eco + custom_presets: + - eco_plus + - power_save + - max + set_custom_fan_mode_action: + - logger.log: + format: "set_custom_fan_mode_action %s" + args: ["x.c_str()"] + set_custom_preset_action: + - logger.log: + format: "set_custom_preset_action %s" + args: ["x.c_str()"] + initial_state: + custom_fan_mode: eco + custom_preset: max + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + water_heater: - platform: template id: template_water_heater diff --git a/tests/integration/fixtures/template_climate_basic.yaml b/tests/integration/fixtures/template_climate_basic.yaml new file mode 100644 index 0000000000..51558b4875 --- /dev/null +++ b/tests/integration/fixtures/template_climate_basic.yaml @@ -0,0 +1,72 @@ +esphome: + name: tmpl-clim-basic + on_boot: + - climate.template.publish: + id: test_climate + action: IDLE +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Basic Climate + optimistic: true + sensor: test_climate_current_temperature + humidity_sensor: test_climate_current_humidity + supports_action: true + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature().has_value()) + ESP_LOGD("test", "on_control target_temperature=%.1f", *x.get_target_temperature()); + if (x.get_fan_mode().has_value()) + ESP_LOGD("test", "on_control fan_mode=%d", (int) *x.get_fan_mode()); + if (x.get_swing_mode().has_value()) + ESP_LOGD("test", "on_control swing_mode=%d", (int) *x.get_swing_mode()); + if (x.get_preset().has_value()) + ESP_LOGD("test", "on_control preset=%d", (int) *x.get_preset()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 22.5f;" + update_interval: 10ms + - platform: template + id: test_climate_current_humidity + name: Test Climate Current Humidity + lambda: "return 55.0f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + mode: "OFF" + fan_mode: AUTO + swing_mode: "OFF" + preset: NONE diff --git a/tests/integration/fixtures/template_climate_custom_modes.yaml b/tests/integration/fixtures/template_climate_custom_modes.yaml new file mode 100644 index 0000000000..9dbfe60cb9 --- /dev/null +++ b/tests/integration/fixtures/template_climate_custom_modes.yaml @@ -0,0 +1,47 @@ +esphome: + name: tmpl-clim-custom +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Custom Mode Climate + optimistic: true + sensor: test_climate_current_temperature + supported_modes: + - "OFF" + - HEAT + - COOL + custom_fan_modes: + - turbo + - silent + - eco + custom_presets: + - eco_plus + - power_save + - max + on_control: + - lambda: |- + if (x.has_custom_fan_mode()) + ESP_LOGD("test", "on_control custom_fan_mode=%s", x.get_custom_fan_mode().c_str()); + if (x.has_custom_preset()) + ESP_LOGD("test", "on_control custom_preset=%s", x.get_custom_preset().c_str()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 22.5f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + custom_fan_mode: "eco" + custom_preset: "max" diff --git a/tests/integration/fixtures/template_climate_nonoptimistic.yaml b/tests/integration/fixtures/template_climate_nonoptimistic.yaml new file mode 100644 index 0000000000..2b0c7ee132 --- /dev/null +++ b/tests/integration/fixtures/template_climate_nonoptimistic.yaml @@ -0,0 +1,56 @@ +esphome: + name: tmpl-clim-nonopt +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Template Climate Nonoptimistic + optimistic: false + supported_modes: + - "OFF" + - HEAT + - COOL + - FAN_ONLY + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + - AWAY + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature().has_value()) + ESP_LOGD("test", "on_control target_temperature=%.1f", *x.get_target_temperature()); + if (x.get_fan_mode().has_value()) + ESP_LOGD("test", "on_control fan_mode=%d", (int) *x.get_fan_mode()); + if (x.get_swing_mode().has_value()) + ESP_LOGD("test", "on_control swing_mode=%d", (int) *x.get_swing_mode()); + if (x.get_preset().has_value()) + ESP_LOGD("test", "on_control preset=%d", (int) *x.get_preset()); + +button: + - platform: template + id: simulate_device_confirmation + name: Simulate Device Confirmation + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT + target_temperature: 22.5 + fan_mode: HIGH + swing_mode: VERTICAL + preset: AWAY diff --git a/tests/integration/fixtures/template_climate_on_control_ordering.yaml b/tests/integration/fixtures/template_climate_on_control_ordering.yaml new file mode 100644 index 0000000000..8366a6d21e --- /dev/null +++ b/tests/integration/fixtures/template_climate_on_control_ordering.yaml @@ -0,0 +1,26 @@ +esphome: + name: tmpl-clim-oc-order +host: +api: +logger: + +# on_control fires with the full ClimateCall (arg `x`) from the base Climate component's +# ClimateCall::perform(), before validate_()/control() run -- so when the lambda action below +# runs, the entity's own .mode is still the OLD value, even though x.get_mode() already reports +# the NEW requested value. on_state fires afterward, once control() has applied it. +climate: + - platform: template + id: test_climate + name: Test On Control Ordering + optimistic: true + supported_modes: + - "OFF" + - HEAT + on_control: + - lambda: |- + ESP_LOGD("test", "on_control requested_mode=%d current_mode_before_apply=%d", + x.get_mode().has_value() ? (int) *x.get_mode() : -1, + (int) id(test_climate).mode); + on_state: + - lambda: |- + ESP_LOGD("test", "on_state mode=%d", (int) x.mode); diff --git a/tests/integration/fixtures/template_climate_publish_all_fields.yaml b/tests/integration/fixtures/template_climate_publish_all_fields.yaml new file mode 100644 index 0000000000..e57fcc4508 --- /dev/null +++ b/tests/integration/fixtures/template_climate_publish_all_fields.yaml @@ -0,0 +1,63 @@ +esphome: + name: tmpl-clim-publish-all +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Publish All Fields + optimistic: true + # current_temperature/current_humidity/action are only sent over the API at all if their + # trait is advertised: current_temperature/current_humidity because a sensor/humidity_sensor + # is referenced below, action because supports_action is set. The sensors' fixed readings + # match what climate.template.publish pushes, so the sensor callback (guarded to only publish + # on an actual change) doesn't produce an extra, unexpected state update of its own. + sensor: test_climate_current_temperature + humidity_sensor: test_climate_current_humidity + supports_action: true + supported_modes: + - "OFF" + - HEAT + supported_fan_modes: + - AUTO + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + on_control: + # Should never fire in this test: climate.template.publish is a pure bypass and must not + # re-trigger on_control as if the entity were freshly commanded. + - logger.log: "on_control fired" + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 20.0f;" + update_interval: 10ms + - platform: template + id: test_climate_current_humidity + name: Test Climate Current Humidity + lambda: "return 60.0f;" + update_interval: 10ms + +button: + - platform: template + id: publish_all + name: Publish All + on_press: + - climate.template.publish: + id: test_climate + current_temperature: 20.0 + current_humidity: 60.0 + target_temperature: 23.0 + mode: HEAT + action: HEATING + fan_mode: HIGH + swing_mode: VERTICAL + preset: ECO diff --git a/tests/integration/fixtures/template_climate_sensor_push.yaml b/tests/integration/fixtures/template_climate_sensor_push.yaml new file mode 100644 index 0000000000..1fc004335d --- /dev/null +++ b/tests/integration/fixtures/template_climate_sensor_push.yaml @@ -0,0 +1,49 @@ +esphome: + name: tmpl-clim-sensor-push +host: +api: +logger: + +# No lambda/update_interval: these sensors only ever report a value when a button below +# publishes one (standing in for e.g. a BLE scan callback in a real config). +sensor: + - platform: template + id: room_temperature + name: Room Temperature + - platform: template + id: room_humidity + name: Room Humidity + +climate: + - platform: template + id: test_climate + name: Test Sensor Push Climate + optimistic: true + sensor: room_temperature + humidity_sensor: room_humidity + supported_modes: + - "OFF" + - HEAT + +button: + - platform: template + id: publish_temperature + name: Publish Temperature + on_press: + - sensor.template.publish: + id: room_temperature + state: 24.0 + - platform: template + id: publish_temperature_same + name: Publish Temperature Same Value + on_press: + - sensor.template.publish: + id: room_temperature + state: 24.0 + - platform: template + id: publish_humidity + name: Publish Humidity + on_press: + - sensor.template.publish: + id: room_humidity + state: 65.0 diff --git a/tests/integration/fixtures/template_climate_set_actions.yaml b/tests/integration/fixtures/template_climate_set_actions.yaml new file mode 100644 index 0000000000..b247367f64 --- /dev/null +++ b/tests/integration/fixtures/template_climate_set_actions.yaml @@ -0,0 +1,89 @@ +esphome: + name: tmpl-clim-set-act +host: +api: +logger: + +# Every settable field forwards its requested value to a set_*_action. supports_two_point and +# supports_target_humidity are not declared here: they are derived from the low/high and humidity +# set actions being present. +climate: + - platform: template + id: test_climate + name: Test Set Actions + optimistic: false + restore_mode: NO_RESTORE + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + custom_fan_modes: + - turbo + custom_presets: + - eco_plus + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + set_mode_action: + - logger.log: + format: "set_mode_action %d" + args: ["(int) x"] + set_target_temperature_low_action: + - logger.log: + format: "set_target_temperature_low_action %.1f" + args: ["x"] + set_target_temperature_high_action: + - logger.log: + format: "set_target_temperature_high_action %.1f" + args: ["x"] + set_target_humidity_action: + - logger.log: + format: "set_target_humidity_action %.0f" + args: ["x"] + set_fan_mode_action: + - logger.log: + format: "set_fan_mode_action %d" + args: ["(int) x"] + set_custom_fan_mode_action: + - logger.log: + format: "set_custom_fan_mode_action %s" + args: ["x.c_str()"] + set_swing_mode_action: + - logger.log: + format: "set_swing_mode_action %d" + args: ["(int) x"] + set_preset_action: + - logger.log: + format: "set_preset_action %d" + args: ["(int) x"] + set_custom_preset_action: + - logger.log: + format: "set_custom_preset_action %s" + args: ["x.c_str()"] + +button: + - platform: template + id: report_device_state + name: Report Device State + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT + + - platform: template + id: report_unsupported_mode + name: Report Unsupported Mode + on_press: + - climate.template.publish: + id: test_climate + mode: DRY diff --git a/tests/integration/fixtures/template_climate_two_point_temperature.yaml b/tests/integration/fixtures/template_climate_two_point_temperature.yaml new file mode 100644 index 0000000000..ec10785ee8 --- /dev/null +++ b/tests/integration/fixtures/template_climate_two_point_temperature.yaml @@ -0,0 +1,52 @@ +esphome: + name: tmpl-clim-two-point +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Two-Point Heatpump + optimistic: true + sensor: test_climate_current_temperature + supports_two_point_target_temperature: true + supports_target_humidity: true + supported_modes: + - "OFF" + - HEAT_COOL + - HEAT + - COOL + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature_low().has_value()) + ESP_LOGD("test", "on_control target_temperature_low=%.1f", *x.get_target_temperature_low()); + if (x.get_target_temperature_high().has_value()) + ESP_LOGD("test", "on_control target_temperature_high=%.1f", *x.get_target_temperature_high()); + if (x.get_target_humidity().has_value()) + ESP_LOGD("test", "on_control target_humidity=%.1f", *x.get_target_humidity()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 21.0f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT_COOL + target_temperature_low: 18.0 + target_temperature_high: 24.0 + target_humidity: 50.0 diff --git a/tests/integration/test_template_climate_basic.py b/tests/integration/test_template_climate_basic.py new file mode 100644 index 0000000000..431fd4e3e8 --- /dev/null +++ b/tests/integration/test_template_climate_basic.py @@ -0,0 +1,146 @@ +"""Integration test for template climate: sensor-pushed measured values, on_control + publish +for the settable ones. + +current_temperature/current_humidity are pushed by a referenced sensor/humidity_sensor (no +polling); action is set once at boot via climate.template.publish, since it has no sensor +equivalent. mode/target_temperature/fan_mode/swing_mode/preset are plain internal state: +on_control fires exactly once per command (never before the first one), and +climate.template.publish simulates the device reporting its own state independent of any prior +command -- that report is authoritative, overriding whatever was optimistically applied earlier. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateAction, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-basic" + + +@pytest.mark.asyncio +async def test_template_climate_basic( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Sensor-pushed measured values, on_control + publish for settable ones.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + # Advertised capabilities come straight from the supported_*/custom_* config lists. + assert ClimateMode.OFF in test_climate.supported_modes + assert ClimateMode.HEAT in test_climate.supported_modes + assert ClimateMode.COOL in test_climate.supported_modes + + assert ClimateFanMode.AUTO in test_climate.supported_fan_modes + assert ClimateFanMode.LOW in test_climate.supported_fan_modes + assert ClimateFanMode.HIGH in test_climate.supported_fan_modes + + assert ClimateSwingMode.OFF in test_climate.supported_swing_modes + assert ClimateSwingMode.VERTICAL in test_climate.supported_swing_modes + + assert ClimatePreset.NONE in test_climate.supported_presets + assert ClimatePreset.ECO in test_climate.supported_presets + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.current_temperature == pytest.approx(22.5, abs=0.1) + assert initial.current_humidity == pytest.approx(55.0, abs=0.1) + assert initial.action == ClimateAction.IDLE + assert initial.mode == ClimateMode.OFF + # Nothing was commanded yet: on_control must not have fired. + assert not log_lines + + # Commands apply optimistically and on_control fires with the same values. + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT + + client.climate_command(test_climate.key, target_temperature=22.5) + state = await wait_for_climate_state() + assert state.target_temperature == pytest.approx(22.5, abs=0.1) + + client.climate_command(test_climate.key, fan_mode=ClimateFanMode.HIGH) + state = await wait_for_climate_state() + assert state.fan_mode == ClimateFanMode.HIGH + + client.climate_command(test_climate.key, swing_mode=ClimateSwingMode.VERTICAL) + state = await wait_for_climate_state() + assert state.swing_mode == ClimateSwingMode.VERTICAL + + client.climate_command(test_climate.key, preset=ClimatePreset.ECO) + state = await wait_for_climate_state() + assert state.preset == ClimatePreset.ECO + + await asyncio.sleep(0.2) + assert any( + "on_control mode=3" in line for line in log_lines + ) # CLIMATE_MODE_HEAT + assert any("on_control target_temperature=22.5" in line for line in log_lines) + assert any("on_control fan_mode=" in line for line in log_lines) + assert any("on_control swing_mode=" in line for line in log_lines) + assert any("on_control preset=" in line for line in log_lines) + # Exactly one on_control log line per command, none extra (e.g. from a stray republish). + assert len(log_lines) == 5 + + # measured values are untouched by any of the above (no set action exists for them). + assert state.current_temperature == pytest.approx(22.5, abs=0.1) + assert state.current_humidity == pytest.approx(55.0, abs=0.1) + assert state.action == ClimateAction.IDLE + + # The device's report is authoritative and overrides everything commanded above. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.OFF + assert state.fan_mode == ClimateFanMode.AUTO + assert state.swing_mode == ClimateSwingMode.OFF + assert state.preset == ClimatePreset.NONE diff --git a/tests/integration/test_template_climate_custom_modes.py b/tests/integration/test_template_climate_custom_modes.py new file mode 100644 index 0000000000..4817fe1ddf --- /dev/null +++ b/tests/integration/test_template_climate_custom_modes.py @@ -0,0 +1,98 @@ +"""Integration test for template climate: custom fan modes and presets. + +Same on_control (forward) + climate.template.publish (device report, authoritative) pattern as +the enum-based mode/preset fields, but for the custom string variants. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-custom" + + +@pytest.mark.asyncio +async def test_template_climate_custom_modes( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Custom fan mode/preset: traits, on_control forwarding, and publish precedence.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + assert set(test_climate.supported_custom_fan_modes) == { + "turbo", + "silent", + "eco", + } + assert set(test_climate.supported_custom_presets) == { + "eco_plus", + "power_save", + "max", + } + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.custom_fan_mode == "" + assert initial.custom_preset == "" + + client.climate_command(test_climate.key, custom_fan_mode="turbo") + state = await wait_for_climate_state() + assert state.custom_fan_mode == "turbo" + + client.climate_command(test_climate.key, custom_preset="power_save") + state = await wait_for_climate_state() + assert state.custom_preset == "power_save" + + await asyncio.sleep(0.2) + assert any("on_control custom_fan_mode=turbo" in line for line in log_lines) + assert any("on_control custom_preset=power_save" in line for line in log_lines) + + # The device's report is authoritative and overrides what was commanded above. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.custom_fan_mode == "eco" + assert state.custom_preset == "max" diff --git a/tests/integration/test_template_climate_nonoptimistic.py b/tests/integration/test_template_climate_nonoptimistic.py new file mode 100644 index 0000000000..e922ec31b9 --- /dev/null +++ b/tests/integration/test_template_climate_nonoptimistic.py @@ -0,0 +1,107 @@ +"""Integration test for template climate: optimistic: false. + +A command still fires on_control (so a real device-backed config can forward it out), but must +NOT change the entity's own state -- only an explicit climate.template.publish call (standing in +for the device confirming the command actually took effect) does that. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-nonopt" + + +@pytest.mark.asyncio +async def test_template_climate_nonoptimistic( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Nonoptimistic: a command doesn't change state until explicitly published.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + state_updates: list[aioesphomeapi.ClimateState] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + confirm_button = require_entity( + entities, "simulate_device_confirmation", ButtonInfo + ) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.mode == ClimateMode.OFF + + # Send every settable field in one command. on_control must fire with all of them, but + # nothing may be applied to the entity's own state -- no ClimateState update at all. + client.climate_command( + test_climate.key, + mode=ClimateMode.HEAT, + target_temperature=22.5, + fan_mode=ClimateFanMode.HIGH, + swing_mode=ClimateSwingMode.VERTICAL, + preset=ClimatePreset.AWAY, + ) + await asyncio.sleep(0.3) + assert any( + "on_control mode=3" in line for line in log_lines + ) # CLIMATE_MODE_HEAT + assert any("on_control target_temperature=22.5" in line for line in log_lines) + assert any("on_control fan_mode=" in line for line in log_lines) + assert any("on_control swing_mode=" in line for line in log_lines) + assert any("on_control preset=" in line for line in log_lines) + assert not state_updates, ( + "optimistic: false must not publish a state until climate.template.publish reports it" + ) + + # The device confirms the command actually took effect. + client.button_command(confirm_button.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.mode == ClimateMode.HEAT + assert state.target_temperature == pytest.approx(22.5, abs=0.1) + assert state.fan_mode == ClimateFanMode.HIGH + assert state.swing_mode == ClimateSwingMode.VERTICAL + assert state.preset == ClimatePreset.AWAY diff --git a/tests/integration/test_template_climate_on_control_ordering.py b/tests/integration/test_template_climate_on_control_ordering.py new file mode 100644 index 0000000000..8d212b3ccb --- /dev/null +++ b/tests/integration/test_template_climate_on_control_ordering.py @@ -0,0 +1,83 @@ +"""Integration test: on_control fires before control()/on_state, with the full ClimateCall. + +on_control's lambda argument exposes get_mode()/etc. on the *requested* ClimateCall, while the +entity's own .mode field still reflects the state *before* control() applies the change -- +proving the firing order is on_control, then control(), then on_state. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ClimateInfo, ClimateMode +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-oc-order" + + +@pytest.mark.asyncio +async def test_template_climate_on_control_ordering( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """on_control sees the requested value while the entity's own state is still the old one.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line or "on_state " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT + + await asyncio.sleep(0.2) + + # on_control saw the new requested mode (3 == CLIMATE_MODE_HEAT) while the entity's own + # state was still the old one (0 == CLIMATE_MODE_OFF) -- proving it fired before control(). + assert any( + "on_control requested_mode=3 current_mode_before_apply=0" in line + for line in log_lines + ) + # on_state fired afterward, reporting the now-applied mode. + assert any("on_state mode=3" in line for line in log_lines) + + control_index = next( + i for i, line in enumerate(log_lines) if "on_control " in line + ) + state_index = next(i for i, line in enumerate(log_lines) if "on_state " in line) + assert control_index < state_index, "on_control must fire before on_state" diff --git a/tests/integration/test_template_climate_publish_all_fields.py b/tests/integration/test_template_climate_publish_all_fields.py new file mode 100644 index 0000000000..9c4262b311 --- /dev/null +++ b/tests/integration/test_template_climate_publish_all_fields.py @@ -0,0 +1,96 @@ +"""Integration test for template climate: climate.template.publish covering every field at once. + +A single climate.template.publish call resolves into exactly one ClimateState update, and never +triggers on_control (which would misrepresent a device state report as a fresh command). This also +exercises that a sensor/humidity_sensor whose reading matches what's about to be published doesn't +sneak in an extra state update of its own (the sensor callback only re-publishes on an actual +change). +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateAction, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-publish-all" + + +@pytest.mark.asyncio +async def test_template_climate_publish_all_fields( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """One climate.template.publish call setting every field resolves to one state update.""" + clear_host_prefs(DEVICE_NAME) + + state_updates: list[aioesphomeapi.ClimateState] = [] + on_control_count = 0 + + def on_log_line(line: str) -> None: + nonlocal on_control_count + if "on_control fired" in line: + on_control_count += 1 + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + + publish_button = require_entity(entities, "publish_all", ButtonInfo) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + client.button_command(publish_button.key) + try: + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + except TimeoutError: + pytest.fail("Timeout waiting for the published climate state") + + assert state.current_temperature == pytest.approx(20.0, abs=0.1) + assert state.current_humidity == pytest.approx(60.0, abs=0.1) + assert state.target_temperature == pytest.approx(23.0, abs=0.1) + assert state.mode == ClimateMode.HEAT + assert state.action == ClimateAction.HEATING + assert state.fan_mode == ClimateFanMode.HIGH + assert state.swing_mode == ClimateSwingMode.VERTICAL + assert state.preset == ClimatePreset.ECO + + # Give any stray extra update (there shouldn't be one) a moment to arrive. + await asyncio.sleep(0.2) + assert len(state_updates) == 1, ( + f"Expected exactly one ClimateState update, got {len(state_updates)}" + ) + assert on_control_count == 0, ( + "climate.template.publish must not trigger on_control" + ) diff --git a/tests/integration/test_template_climate_sensor_push.py b/tests/integration/test_template_climate_sensor_push.py new file mode 100644 index 0000000000..1db4da81ed --- /dev/null +++ b/tests/integration/test_template_climate_sensor_push.py @@ -0,0 +1,88 @@ +"""Integration test for template climate: current_temperature/current_humidity live sensor push. + +A *later* change to a backing sensor's value -- not just its initial reading at boot -- propagates +into a new climate state via add_on_state_callback. Re-publishing the same sensor value again must +not cause a redundant climate state update. +""" + +from __future__ import annotations + +import asyncio +import math + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-sensor-push" + + +@pytest.mark.asyncio +async def test_template_climate_sensor_push( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """A later change to the backing sensor pushes a new climate state; an unchanged republish does not.""" + clear_host_prefs(DEVICE_NAME) + + state_updates: list[aioesphomeapi.ClimateState] = [] + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + publish_temp = require_entity(entities, "publish_temperature", ButtonInfo) + publish_temp_same = require_entity( + entities, "publish_temperature_same", ButtonInfo + ) + publish_humidity = require_entity(entities, "publish_humidity", ButtonInfo) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + # Neither backing sensor has published anything yet. + assert math.isnan(initial.current_temperature) + assert math.isnan(initial.current_humidity) + + # A later sensor reading -- not the initial one -- pushes a new climate state. + client.button_command(publish_temp.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.current_temperature == pytest.approx(24.0, abs=0.1) + + client.button_command(publish_humidity.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.current_humidity == pytest.approx(65.0, abs=0.1) + + # Re-publishing the same temperature must not cause a redundant climate state update. + updates_before = len(state_updates) + client.button_command(publish_temp_same.key) + await asyncio.sleep(0.3) + assert len(state_updates) == updates_before, ( + "Re-publishing an unchanged sensor reading must not republish the climate state" + ) diff --git a/tests/integration/test_template_climate_set_actions.py b/tests/integration/test_template_climate_set_actions.py new file mode 100644 index 0000000000..0b1eb80874 --- /dev/null +++ b/tests/integration/test_template_climate_set_actions.py @@ -0,0 +1,114 @@ +"""Integration test: each settable field forwards its value to the matching set_*_action. + +With optimistic: false the entity state stays put until climate.template.publish reports the +device's actual state back, so the actions are the only thing that reacts to a command. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-set-act" + + +@pytest.mark.asyncio +async def test_template_climate_set_actions( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Every set_*_action fires with the requested value; state waits for a publish.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "_action " in line or "Unsupported" in line: + log_lines.append(line) + + def logged(fragment: str) -> bool: + return any(fragment in line for line in log_lines) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + report_button = require_entity(entities, "report_device_state", ButtonInfo) + unsupported_button = require_entity( + entities, "report_unsupported_mode", ButtonInfo + ) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + # Both traits are derived from the low/high and humidity set actions, not declared. + assert test_climate.supports_two_point_target_temperature + assert test_climate.supports_target_humidity + + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + client.climate_command( + test_climate.key, target_temperature_low=18.0, target_temperature_high=24.0 + ) + client.climate_command(test_climate.key, target_humidity=55) + client.climate_command(test_climate.key, fan_mode=ClimateFanMode.LOW) + client.climate_command(test_climate.key, custom_fan_mode="turbo") + client.climate_command(test_climate.key, swing_mode=ClimateSwingMode.VERTICAL) + client.climate_command(test_climate.key, preset=ClimatePreset.ECO) + client.climate_command(test_climate.key, custom_preset="eco_plus") + + for _ in range(50): + await asyncio.sleep(0.1) + if logged("set_custom_preset_action eco_plus"): + break + + assert logged("set_mode_action 3") # CLIMATE_MODE_HEAT + assert logged("set_target_temperature_low_action 18.0") + assert logged("set_target_temperature_high_action 24.0") + assert logged("set_target_humidity_action 55") + assert logged("set_fan_mode_action 3") # CLIMATE_FAN_LOW + assert logged("set_custom_fan_mode_action turbo") + assert logged("set_swing_mode_action 2") # CLIMATE_SWING_VERTICAL + assert logged("set_preset_action 5") # CLIMATE_PRESET_ECO + assert logged("set_custom_preset_action eco_plus") + + # optimistic: false, so none of the commands above touched the entity's own state -- + # a device report is what actually moves it. + client.button_command(report_button.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.mode == ClimateMode.HEAT + + # A publish naming a mode outside supported_modes warns instead of publishing it. + client.button_command(unsupported_button.key) + for _ in range(50): + await asyncio.sleep(0.1) + if logged("Unsupported mode"): + break + assert logged("Unsupported mode") diff --git a/tests/integration/test_template_climate_two_point_temperature.py b/tests/integration/test_template_climate_two_point_temperature.py new file mode 100644 index 0000000000..9270b59ffc --- /dev/null +++ b/tests/integration/test_template_climate_two_point_temperature.py @@ -0,0 +1,118 @@ +"""Integration tests for template climate: two-point target temperature + humidity. + +Covers the supports_two_point_target_temperature/supports_target_humidity boolean flags plus +on_control (forwarding commands out) and climate.template.publish (the device reporting its own +authoritative state, independent of any prior command -- e.g. a device that owns its own setpoint, +changed via a physical remote). +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo, ClimateMode +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-two-point" + + +@pytest.mark.asyncio +async def test_template_climate_two_point_temperature( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Two-point target temperature + humidity: booleans, on_control, and publish precedence.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + + test_climate = climate_infos[0] + assert test_climate.name == "Test Two-Point Heatpump" + assert test_climate.supports_two_point_target_temperature + assert test_climate.supports_target_humidity + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + # Nothing has been published yet: settable fields have no sensor to seed them from, so + # the entity starts at ESPHome's plain defaults. current_temperature is pushed by the + # referenced sensor, which has already settled by the time we get here. + assert initial.mode == ClimateMode.OFF + assert initial.current_temperature == pytest.approx(21.0, abs=0.1) + + # The device reports its actual state for the first time. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT_COOL + assert state.target_temperature_low == pytest.approx(18.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(24.0, abs=0.1) + assert state.target_humidity == pytest.approx(50.0, abs=0.1) + + # Commands apply optimistically (settable fields are plain internal state), and on_control + # fires with the same values so a real config could forward them to the device. + client.climate_command( + test_climate.key, target_temperature_low=19.0, target_temperature_high=25.0 + ) + state = await wait_for_climate_state() + assert state.target_temperature_low == pytest.approx(19.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(25.0, abs=0.1) + await asyncio.sleep(0.2) + assert any( + "on_control target_temperature_low=19.0" in line for line in log_lines + ) + assert any( + "on_control target_temperature_high=25.0" in line for line in log_lines + ) + + client.climate_command(test_climate.key, target_humidity=45.0) + state = await wait_for_climate_state() + assert state.target_humidity == pytest.approx(45.0, abs=0.1) + await asyncio.sleep(0.2) + assert any("on_control target_humidity=45.0" in line for line in log_lines) + + # The device's next report is authoritative and overrides whatever was optimistically + # applied above -- this is the whole point of climate.template.publish: a device that owns + # its own state (e.g. changed by a physical remote) always wins. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.target_temperature_low == pytest.approx(18.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(24.0, abs=0.1) + assert state.target_humidity == pytest.approx(50.0, abs=0.1) From 833dd0e812ecf6e413022bd23b9d4e098245d715 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 6 Sep 2026 23:59:40 +0200 Subject: [PATCH 105/147] [ota] Offer encryption with the api key so enabling it works over OTA (#18979) --- THREAT_MODEL.md | 55 ++- esphome/__main__.py | 14 +- esphome/components/api/__init__.py | 4 +- esphome/components/api/api_connection.cpp | 6 +- .../components/api/api_frame_helper_noise.cpp | 2 +- esphome/components/api/api_server.cpp | 37 +- esphome/components/api/api_server.h | 12 +- esphome/components/esphome/ota/__init__.py | 130 +++--- .../components/esphome/ota/ota_esphome.cpp | 83 ++-- esphome/components/esphome/ota/ota_esphome.h | 13 +- .../esphome/ota/ota_esphome_noise.cpp | 91 +++-- esphome/components/noise/__init__.py | 36 +- esphome/components/noise/noise.cpp | 9 + esphome/components/noise/noise.h | 16 +- esphome/components/noise/noise_handshake.cpp | 5 +- esphome/components/noise/noise_handshake.h | 6 +- esphome/core/defines.h | 3 + esphome/espota2.py | 122 +++++- esphome/wizard.py | 18 +- .../noise/test_encryption_key.py | 14 +- tests/component_tests/ota/test_esphome_ota.py | 242 +++++++++--- .../ota/test_esphome_ota_api_key_offer.yaml | 11 + ...st_esphome_ota_api_key_offer_password.yaml | 12 + .../test_esphome_ota_encryption_required.yaml | 12 + .../ota/test_esphome_ota_own_key.yaml | 11 + .../ota/test_esphome_ota_plain.yaml | 9 + .../ota/test_esphome_ota_runtime_api_key.yaml | 10 + .../components/noise/test_noise_handshake.cpp | 18 +- .../noise/test_noise_primitives.cpp | 13 +- tests/components/ota/api_key_offer.yaml | 12 + tests/components/ota/api_runtime_key.yaml | 10 + .../ota/test-api_key_offer.esp32-idf.yaml | 2 + .../ota/test-api_key_offer.esp8266-ard.yaml | 2 + .../ota/test-api_runtime_key.esp32-idf.yaml | 2 + .../ota/test-api_runtime_key.esp8266-ard.yaml | 2 + tests/integration/conftest.py | 7 + tests/integration/const.py | 7 + .../host_ota_api_key_offer_with_password.yaml | 12 + .../host_ota_provisioned_api_key.yaml | 10 + .../test_api_zero_psk_provisioning.py | 51 ++- tests/integration/test_host_ota.py | 373 ++++++++++++------ tests/unit_tests/test_espota2_noise.py | 136 ++++++- tests/unit_tests/test_main.py | 114 +++++- tests/unit_tests/test_wizard.py | 31 +- 44 files changed, 1342 insertions(+), 443 deletions(-) create mode 100644 tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_encryption_required.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_own_key.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_plain.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml create mode 100644 tests/components/ota/api_key_offer.yaml create mode 100644 tests/components/ota/api_runtime_key.yaml create mode 100644 tests/components/ota/test-api_key_offer.esp32-idf.yaml create mode 100644 tests/components/ota/test-api_key_offer.esp8266-ard.yaml create mode 100644 tests/components/ota/test-api_runtime_key.esp32-idf.yaml create mode 100644 tests/components/ota/test-api_runtime_key.esp8266-ard.yaml create mode 100644 tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml create mode 100644 tests/integration/fixtures/host_ota_provisioned_api_key.yaml diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index b4f557e55b..11656ff0b7 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -125,30 +125,47 @@ design is optimal or that it will not change. ## OTA update encryption The `esphome` OTA platform optionally encrypts updates with the same Noise -`NNpsk0` pattern the native API uses; one key protects the device. With an -`encryption:` block configured the guarantees are: the firmware image is -confidential in transit, the uploader is authenticated by the pre-shared key, -and the plaintext negotiation preceding the handshake is bound into the -handshake prologue, so stripping or tampering with it fails the first MAC. -Both ends fail closed with no override: a device built with a key refuses +`NNpsk0` pattern the native API uses; one key protects the device. A device +whose `api:` block has an encryption key, static in the YAML or provisioned at +runtime, compiles in the transport and offers it on every OTA connection once +it holds a key, so an uploader presenting that key gets the guarantees below +even without an `ota: encryption:` block; only that block makes the device +require encryption. The guarantees are: the firmware image is confidential in +transit, the uploader is authenticated by the pre-shared key, and the plaintext +negotiation preceding the handshake is bound into the handshake prologue, so +stripping or tampering with it fails the first MAC. With `ota: encryption:` +configured both ends fail closed with no override: the device refuses plaintext uploads, and the CLI refuses to send plaintext when a key is -configured. +configured. Without that block the CLI tries a static api key when the device +offers and, until 2027.3.0, falls back to plaintext with a warning when the +offer is missing or the handshake fails; a runtime provisioned key never +reaches the CLI, so those uploads stay plaintext. -Defeating any of that without the key is in scope: a keyed device accepting a -plaintext or downgraded upload, getting past the MAC, or recovering image -contents from captured traffic. +Defeating any of that without the key is in scope: a device that requires +encryption accepting a plaintext or downgraded upload, getting past the MAC, +or recovering image contents from captured traffic. The following are **not** vulnerabilities, by design: -- Plaintext OTA on a device with no `encryption:` block. That is the - documented default, authenticated (if at all) by the OTA password. -- The enablement window: turning encryption on takes one last upload of the - encryption-enabled firmware over the existing plaintext channel, with the - pre-existing plaintext exposure. -- The web OTA `/update` endpoint alongside encryption. The `web_server` - component keeps it always reachable, and `captive_portal:` auto-loads it - for the fallback AP window; validation warns about both combinations, and - the operator keeps the recovery path. +- Plaintext OTA on a device with no `ota: encryption:` block, including one + that offers encryption because it has an api key. That is the documented + default, authenticated (if at all) by the OTA password. An uploader that + takes the offer skips the password; the key authenticates it. With a + runtime provisioned key and no `provisioning:` window, whoever provisions + the key gains that upload path too; validation warns about the pair. +- The CLI plaintext fallback until 2027.3.0: without `ota: encryption:` an + active attacker who strips the offer or breaks the handshake can make a + keyed CLI upload plaintext, with the pre-existing plaintext exposure. A + device that requires encryption still refuses that upload. +- The enablement window: firmware built with a static api key already offers + encryption, so turning on `ota: encryption:` is itself an encrypted upload. + Older firmware needs one last plaintext upload of an offering build, with + the pre-existing plaintext exposure. +- The web OTA `/update` endpoint alongside encryption. With the `web_server` + or `prometheus` component the shared listener is always up, so the endpoint + stays reachable and validation warns about that combination; + `captive_portal:` alone brings the listener up only for the fallback AP + window, which is the intended recovery path, so that is not warned about. - CLI retry behavior on transport or MAC failures; every attempt renegotiates a fresh handshake with fresh ephemerals, so retrying does not weaken authentication. diff --git a/esphome/__main__.py b/esphome/__main__.py index b3d58ad13b..30e97f55eb 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1335,12 +1335,14 @@ def _upload_via_native_api( break from esphome import espota2 + from esphome.components.noise import static_encryption_key remote_port = int(ota_conf[CONF_PORT]) password = ota_conf.get(CONF_PASSWORD) # Fail closed: an encryption block whose key did not resolve must never # fall back to a plaintext upload noise_psk = None + plaintext_fallback = False if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: noise_psk = encryption_conf.get(CONF_KEY) if not noise_psk: @@ -1351,6 +1353,10 @@ def _upload_via_native_api( # Ensure the key is a string, as required by the underlying OTA implementation. # It arrives here as a SensitiveStr which aioesphomeapi rejects. noise_psk = str(noise_psk) + elif api_key := static_encryption_key(config.get(CONF_API) or {}): + # Remove before 2027.3.0: the api key is tried, falling back to plaintext + noise_psk = str(api_key) + plaintext_fallback = True def check_partition_access(option_string: str) -> None: if not ota_conf.get("allow_partition_access"): @@ -1382,7 +1388,13 @@ def _upload_via_native_api( _validate_bootloader_binary(binary) return espota2.run_ota( - network_devices, remote_port, password, binary, ota_type, noise_psk + network_devices, + remote_port, + password, + binary, + ota_type, + noise_psk, + plaintext_fallback=plaintext_fallback, ) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 3568318dad..6202e127bf 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -14,6 +14,7 @@ from esphome.components.noise import ( # noqa: F401 ENCRYPTION_SCHEMA, decode_encryption_key, encryption_schema, + new_psk_progmem, validate_encryption_key, ) from esphome.config_helpers import filter_source_files_from_defines, get_logger_level @@ -589,8 +590,7 @@ async def to_code(config: ConfigType) -> None: if (encryption_config := config.get(CONF_ENCRYPTION, None)) is not None: if key := encryption_config.get(CONF_KEY): - decoded = decode_encryption_key(key) - cg.add(var.set_noise_psk(list(decoded))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], key))) cg.add_define("USE_API_NOISE_PSK_FROM_YAML") else: # No key provided, but encryption desired diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 9c609aa047..da4b7d7702 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -2161,7 +2161,10 @@ void APIConnection::on_homeassistant_action_response(const HomeassistantActionRe bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptionSetKeyRequest &msg) { NoiseEncryptionSetKeyResponse resp; resp.success = false; - +#ifdef USE_API_NOISE_PSK_FROM_YAML + // A yaml key cannot be changed at runtime, so no decode or save path is built + ESP_LOGW(TAG, "Key set in YAML"); +#else #ifdef USE_PROVISIONING // Refuse to set a key once the provisioning window has closed (defense in depth; // such connections are already rejected at hello). @@ -2196,6 +2199,7 @@ bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptio } #endif } +#endif // USE_API_NOISE_PSK_FROM_YAML return this->send_message(resp); } diff --git a/esphome/components/api/api_frame_helper_noise.cpp b/esphome/components/api/api_frame_helper_noise.cpp index 138dbdddba..29b2858aee 100644 --- a/esphome/components/api/api_frame_helper_noise.cpp +++ b/esphome/components/api/api_frame_helper_noise.cpp @@ -548,7 +548,7 @@ APIError APINoiseFrameHelper::write_frame_(const uint8_t *data, uint16_t len) { * @return 0 on success, -1 on error (check errno) */ APIError APINoiseFrameHelper::init_handshake_() { - int err = this->handshake_.init(this->ctx_.get_psk(), prologue_.data(), prologue_.size()); + int err = this->handshake_.init(this->ctx_, prologue_.data(), prologue_.size()); APIError aerr = handle_noise_error_(err, LOG_STR("noise_handshake_init"), APIError::HANDSHAKESTATE_SETUP_FAILED); if (aerr != APIError::OK) return aerr; diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 43d35363d3..78ebe5c38e 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -41,13 +41,13 @@ void APIServer::setup() { ControllerRegistry::register_controller(this); #ifdef USE_API_NOISE + // Always reserve the slot: flash preferences are positional on esp8266, so + // a yaml key build must keep the layout of a runtime key build uint32_t hash = 88491486UL; - this->noise_pref_ = global_preferences->make_preference(hash, true); - #ifndef USE_API_NOISE_PSK_FROM_YAML - // Only load saved PSK if not set from YAML - if (this->load_and_apply_noise_psk_()) { + // A cleared record loads fine but holds no key + if (this->load_and_apply_noise_psk_() && this->noise_ctx_.has_psk()) { ESP_LOGD(TAG, "Loaded saved Noise PSK"); } #endif @@ -550,6 +550,7 @@ const std::vector &APIServer::get_sta #endif #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg, bool make_active) { if (!this->noise_pref_.save(&new_psk)) { @@ -583,22 +584,19 @@ bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString } bool APIServer::load_and_apply_noise_psk_() { - SavedNoisePsk saved{}; - if (!this->noise_pref_.load(&saved)) + // Load into a temp so a failed read cannot disturb the key in use + SavedNoisePsk loaded{}; + if (!this->noise_pref_.load(&loaded)) return false; - this->set_noise_psk(saved.psk); + this->saved_psk_ = loaded; + // An unprovisioned device stores the reserved all-zeros key, which is no key + const bool has_key = !noise::NoiseContext::is_all_zeros(this->saved_psk_.psk); + this->noise_ctx_.set_psk(has_key ? this->saved_psk_.psk.data() : nullptr); return true; } bool APIServer::save_noise_psk(noise::psk_t psk, bool make_active) { -#ifdef USE_API_NOISE_PSK_FROM_YAML - // When PSK is set from YAML, this function should never be called - // but if it is, reject the change - ESP_LOGW(TAG, "Key set in YAML"); - return false; -#else - auto &old_psk = this->noise_ctx_.get_psk(); - if (std::equal(old_psk.begin(), old_psk.end(), psk.begin())) { + if (this->saved_psk_.psk == psk) { ESP_LOGW(TAG, "New PSK matches old"); return true; } @@ -614,15 +612,8 @@ bool APIServer::save_noise_psk(noise::psk_t psk, bool make_active) { } #endif return result; -#endif } bool APIServer::clear_noise_psk(bool make_active) { -#ifdef USE_API_NOISE_PSK_FROM_YAML - // When PSK is set from YAML, this function should never be called - // but if it is, reject the change - ESP_LOGW(TAG, "Key set in YAML"); - return false; -#else SavedNoisePsk empty_psk{}; bool result = this->update_noise_psk_(empty_psk, LOG_STR("Noise PSK cleared"), LOG_STR("Failed to clear Noise PSK"), make_active); @@ -634,8 +625,8 @@ bool APIServer::clear_noise_psk(bool make_active) { } #endif return result; -#endif } +#endif // USE_API_NOISE_PSK_FROM_YAML #endif #ifdef USE_HOMEASSISTANT_TIME diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 072a583901..618ea4eb11 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -76,9 +76,14 @@ class APIServer final : public Component, APIBuffer &get_shared_buffer_ref() { return shared_write_buffer_; } #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML + // Runtime key changes exist for the provisioning path only (not lambdas); + // with a yaml key they compile out bool save_noise_psk(noise::psk_t psk, bool make_active = true); bool clear_noise_psk(bool make_active = true); - void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#endif + /// psk points at 32 bytes that live in flash for the life of the program + void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } noise::NoiseContext &get_noise_ctx() { return this->noise_ctx_; } #endif // USE_API_NOISE @@ -275,10 +280,12 @@ class APIServer final : public Component, #endif #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML bool update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg, bool make_active); // Load saved PSK from preferences and apply it. Returns true on success. bool load_and_apply_noise_psk_(); +#endif // USE_API_NOISE_PSK_FROM_YAML #endif // USE_API_NOISE #ifdef USE_API_HOMEASSISTANT_STATES // Helper methods to reduce code duplication @@ -358,6 +365,9 @@ class APIServer final : public Component, #ifdef USE_API_NOISE noise::NoiseContext noise_ctx_; +#ifndef USE_API_NOISE_PSK_FROM_YAML + SavedNoisePsk saved_psk_{}; // backs noise_ctx_ for a runtime provisioned key +#endif ESPPreferenceObject noise_pref_; #endif // USE_API_NOISE }; diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index 1fec9e5c9b..f5eb878260 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -2,12 +2,12 @@ import logging import esphome.codegen as cg from esphome.components.noise import ( - decode_encryption_key, encryption_schema, - is_reserved_key, + new_psk_progmem, + static_encryption_key, ) from esphome.components.ota import BASE_OTA_SCHEMA, OTAComponent, ota_to_code -from esphome.config_helpers import merge_config +from esphome.config_helpers import filter_source_files_from_defines, merge_config import esphome.config_validation as cv from esphome.const import ( CONF_API, @@ -31,7 +31,6 @@ import esphome.final_validate as fv from esphome.types import ConfigType CONF_ALLOW_PARTITION_ACCESS = "allow_partition_access" -CONF_CAPTIVE_PORTAL = "captive_portal" _LOGGER = logging.getLogger(__name__) @@ -41,11 +40,10 @@ DEPENDENCIES = ["network"] def AUTO_LOAD(config: ConfigType) -> list[str]: - """Auto-load noise only when encryption is configured.""" + """Auto-load noise only when encryption is configured; the api key offer + inherits it from the api component.""" base = ["sha256", "socket"] - # A falsy config is a tooling probe for the maximal set (None from - # dependency resolution, {} from the components-graph platform probe); - # a validated config always carries defaults, never empty + # A falsy config is a tooling probe for the maximal set if not config or CONF_ENCRYPTION in config: return base + ["noise"] return base @@ -132,12 +130,56 @@ def ota_esphome_final_validate(config: ConfigType) -> None: _validate_no_password_with_encryption(ota_conf) if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: _resolve_encryption_key(encryption_conf, api_conf) - if any( - conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf - ) and any( - CONF_ENCRYPTION in conf for conf in merged_ota_esphome_configs_by_port.values() + elif CONF_PASSWORD in ota_conf and static_encryption_key(api_conf) is not None: + _LOGGER.warning( + "'%s' %s wastes significant flash and RAM (about 3.5 KB and 60 " + "bytes plus the password on the heap): the device already offers " + "encryption with the '%s' %s %s, which authenticates any uploader " + "that takes it, and a password only matters for uploaders without " + "encryption support; remove '%s' and add '%s' under '%s' so " + "uploads use the key and encryption is required", + CONF_OTA, + CONF_PASSWORD, + CONF_API, + CONF_ENCRYPTION, + CONF_KEY, + CONF_PASSWORD, + CONF_ENCRYPTION, + CONF_OTA, + ) + elif ( + CONF_PASSWORD in ota_conf + and CONF_ENCRYPTION in api_conf + and not api_conf[CONF_ENCRYPTION].get(CONF_KEY) + ): + # The CLI still needs the password; whoever provisions the key skips it + _LOGGER.warning( + "The '%s' %s %s provisioned at runtime also authenticates OTA " + "uploads once provisioned; '%s' %s then only guards plaintext " + "uploads. Whoever provisions the key can upload firmware " + "without the password, so add a 'provisioning:' block to limit " + "when that is possible", + CONF_API, + CONF_ENCRYPTION, + CONF_KEY, + CONF_OTA, + CONF_PASSWORD, + ) + # web_server and prometheus keep the shared listener up; the captive + # portal's copy only exists on the fallback AP and is the recovery path + if ( + (CONF_WEB_SERVER in full_conf or "prometheus" in full_conf) + and any(conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf) + and any( + CONF_ENCRYPTION in conf + for conf in merged_ota_esphome_configs_by_port.values() + ) ): - _warn_web_server_ota(full_conf) + _LOGGER.warning( + "OTA encryption does not cover the %s OTA platform; its " + "plaintext /update endpoint accepts the same image", + CONF_WEB_SERVER, + ) full_conf[CONF_OTA] = new_ota_conf fv.full_config.set(full_conf) @@ -152,33 +194,11 @@ def ota_esphome_final_validate(config: ConfigType) -> None: ) -def _warn_web_server_ota(full_conf: ConfigType) -> None: - """The web_server ota platform accepts the same image over plaintext HTTP - with basic auth, bypassing the encryption; warn rather than fail so the - operator keeps the recovery path.""" - if CONF_CAPTIVE_PORTAL in full_conf and CONF_WEB_SERVER not in full_conf: - # The captive_portal auto-load: the endpoint only exists while the - # fallback AP is active - _LOGGER.warning( - "OTA encryption does not cover the %s OTA platform (auto-loaded " - "by captive_portal); the plaintext /update endpoint stays " - "reachable while the fallback AP is active", - CONF_WEB_SERVER, - ) - else: - _LOGGER.warning( - "OTA encryption does not cover the %s OTA platform; its " - "plaintext /update endpoint accepts the same image", - CONF_WEB_SERVER, - ) - - def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) -> None: """Resolve the one encryption key per device into the ota block. An explicit ota key must match the api key, a bare block inherits it, - a runtime provisioned api key cannot be inherited, and the all-zeros - provisioning sentinel is rejected (the device treats it as no key). + a runtime provisioned api key cannot be inherited. """ api_key = api_conf.get(CONF_ENCRYPTION, {}).get(CONF_KEY) if ota_key := encryption_conf.get(CONF_KEY): @@ -201,11 +221,6 @@ def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) - ) else: encryption_conf[CONF_KEY] = api_key - if is_reserved_key(encryption_conf[CONF_KEY]): - raise cv.Invalid( - f"The all-zeros {CONF_KEY} is reserved and provides no protection; " - f"generate a real key with: openssl rand -base64 32" - ) # Also called on merged same-port configs in final validate, where schemas @@ -267,15 +282,9 @@ CONFIG_SCHEMA = cv.All( FINAL_VALIDATE_SCHEMA = ota_esphome_final_validate -def FILTER_SOURCE_FILES() -> list[str]: - """Filter out the noise transport when no ota entry configures encryption.""" - for ota_conf in CORE.config.get(CONF_OTA, []): - if ( - ota_conf.get(CONF_PLATFORM) == CONF_ESPHOME - and ota_conf.get(CONF_ENCRYPTION) is not None - ): - return [] - return ["ota_esphome_noise.cpp"] +FILTER_SOURCE_FILES = filter_source_files_from_defines( + {"ota_esphome_noise.cpp": "USE_OTA_ENCRYPTION"} +) @coroutine_with_priority(CoroPriority.OTA_UPDATES) @@ -296,11 +305,24 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_ALLOW_PARTITION_ACCESS): cg.add_define("USE_OTA_PARTITIONS") - if (encryption_conf := config.get(CONF_ENCRYPTION)) is not None: - # A missing key was resolved from the api component in final validate. - key = encryption_conf[CONF_KEY] + # One key per device: an api encryption block supplies it (static or + # runtime) and offers; the ota block only adds the requirement + api_conf = CORE.config.get(CONF_API) or {} + encryption_conf = config.get(CONF_ENCRYPTION) + own_key = None + if encryption_conf is not None and static_encryption_key(api_conf) is None: + own_key = encryption_conf[CONF_KEY] + if own_key is not None: cg.add_define("USE_OTA_ENCRYPTION") - cg.add(var.set_noise_psk(list(decode_encryption_key(key)))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], own_key))) + elif CONF_ENCRYPTION in api_conf: + cg.add_define("USE_OTA_ENCRYPTION") + cg.add_define("USE_OTA_ENCRYPTION_FROM_API") + if static_encryption_key(api_conf) is None: + # The key arrives at runtime, so the offer has to look for it + cg.add_define("USE_OTA_ENCRYPTION_PROVISIONED") + if encryption_conf is not None: + cg.add_define("USE_OTA_ENCRYPTION_REQUIRED") # Build flag so lwip_fast_select.c (a .c file that can't include defines.h) sees it. cg.add_build_flag("-DUSE_OTA_PLATFORM_ESPHOME") diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 396a47bc52..1005ed214b 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -1,4 +1,7 @@ #include "ota_esphome.h" +#ifdef USE_OTA_ENCRYPTION_FROM_API +#include "esphome/components/api/api_server.h" +#endif #ifdef USE_OTA #ifdef USE_OTA_PASSWORD #include "esphome/components/sha256/sha256.h" @@ -26,6 +29,16 @@ namespace esphome { static const char *const TAG = "esphome.ota"; + +#ifdef USE_OTA_ENCRYPTION +const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { +#ifdef USE_OTA_ENCRYPTION_FROM_API + return api::global_api_server->get_noise_ctx(); +#else + return this->noise_ctx_; +#endif +} +#endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer @@ -97,18 +110,30 @@ void ESPHomeOTAComponent::dump_config() { ESP_LOGCONFIG(TAG, "Over-The-Air updates:\n" " Address: %s:%u\n" - " Version: %d", - network::get_use_address_to(addr_buf), this->port_, USE_OTA_VERSION); + " Version: %d" +#ifdef USE_OTA_ENCRYPTION + "\n Encryption: %s" +#endif + , + network::get_use_address_to(addr_buf), this->port_, USE_OTA_VERSION +#ifdef USE_OTA_ENCRYPTION_REQUIRED + , + LOG_STR_LITERAL("required") +#elif defined(USE_OTA_ENCRYPTION_PROVISIONED) + // A runtime provisioned key may not exist yet + , + this->noise_context_().has_psk() ? LOG_STR_LITERAL("offered, plaintext accepted") + : LOG_STR_LITERAL("offered once the api key is provisioned") +#elif defined(USE_OTA_ENCRYPTION) + , + LOG_STR_LITERAL("offered, plaintext accepted") +#endif + ); #ifdef USE_OTA_PASSWORD if (!this->password_.empty()) { ESP_LOGCONFIG(TAG, " Password configured"); } #endif -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ctx_.has_psk()) { - ESP_LOGCONFIG(TAG, " Encryption configured"); - } -#endif #ifdef USE_OTA_PARTITIONS ESP_LOGCONFIG(TAG, " Partition access allowed\n" @@ -154,10 +179,22 @@ static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_NOISE = 0x08; +// Noise needs the extended protocol: the prologue binds the 2-byte feature ack +static constexpr uint8_t CLIENT_NOISE_FEATURES = + CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_NOISE = 0x04; +inline bool ESPHomeOTAComponent::extended_proto_() const { +#ifdef USE_OTA_ENCRYPTION_REQUIRED + // FEATURE_READ already refused every client without the extended protocol + return true; +#else + return (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL) != 0; +#endif +} + void ESPHomeOTAComponent::handle_handshake_() { /// Handle the OTA handshake and authentication. /// @@ -241,12 +278,9 @@ void ESPHomeOTAComponent::handle_handshake_() { this->ota_features_ = this->handshake_buf_[0]; ESP_LOGV(TAG, "Features: 0x%02X", this->ota_features_); -#ifdef USE_OTA_ENCRYPTION - // Fail closed: with a PSK configured the client must negotiate encryption - // (which requires the extended protocol); refuse plaintext uploads. - static constexpr uint8_t NOISE_REQUIRED_FEATURES = - CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; - if (this->noise_ctx_.has_psk() && (this->ota_features_ & NOISE_REQUIRED_FEATURES) != NOISE_REQUIRED_FEATURES) { +#ifdef USE_OTA_ENCRYPTION_REQUIRED + // `ota: encryption:` requires the client to negotiate encryption + if ((this->ota_features_ & CLIENT_NOISE_FEATURES) != CLIENT_NOISE_FEATURES) { ESP_LOGW(TAG, "Client does not support encryption"); this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_ENCRYPTION_REQUIRED); return; @@ -261,18 +295,21 @@ void ESPHomeOTAComponent::handle_handshake_() { // Compose the feature-ack response. When the client negotiates the extended protocol we emit // a 2-byte response (marker + server feature flags); otherwise we emit the single-byte // legacy response. - this->extended_proto_ = (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL) != 0; - if (this->extended_proto_) { + if (this->extended_proto_()) { static_assert(HANDSHAKE_BUF_SIZE >= 2, "handshake_buf_ must hold the 2-byte extended-protocol feature ack"); this->handshake_buf_[0] = ota::OTA_RESPONSE_FEATURE_FLAGS; this->handshake_buf_[1] = (supports_compression ? SERVER_FEATURE_SUPPORTS_COMPRESSION : 0); #ifdef USE_OTA_PARTITIONS this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS; #endif -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ctx_.has_psk()) { +#ifdef USE_OTA_ENCRYPTION_PROVISIONED + // A runtime provisioned key may not exist yet + if (this->noise_context_().has_psk()) { this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; } +#elif defined(USE_OTA_ENCRYPTION) + // A yaml key always exists: validation rejects the all-zeros key + this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; #endif } else { this->handshake_buf_[0] = @@ -284,15 +321,15 @@ void ESPHomeOTAComponent::handle_handshake_() { case OTAState::FEATURE_ACK: { static constexpr size_t STANDARD_PROTO_ACK_SIZE = 1; static constexpr size_t EXTENDED_PROTO_ACK_SIZE = 2; - const size_t ack_size = this->extended_proto_ ? EXTENDED_PROTO_ACK_SIZE : STANDARD_PROTO_ACK_SIZE; + const size_t ack_size = this->extended_proto_() ? EXTENDED_PROTO_ACK_SIZE : STANDARD_PROTO_ACK_SIZE; if (!this->try_write_(ack_size, LOG_STR("ack feature"))) { return; } #ifdef USE_OTA_ENCRYPTION - // With a PSK configured the rest of the session runs inside the noise - // transport; the client sends the first handshake frame next, so there - // is nothing to do until data arrives. - if (this->noise_ctx_.has_psk()) { + // Latch the offer actually sent: a key activating between the two + // states must not start a session the client never expects + if ((this->handshake_buf_[1] & SERVER_FEATURE_SUPPORTS_NOISE) != 0 && + (this->ota_features_ & CLIENT_NOISE_FEATURES) == CLIENT_NOISE_FEATURES) { // handshake_buf_ still holds the feature ack composed above; a // would-block re-entry lands here without rebuilding it if (!this->noise_start_session_(this->handshake_buf_[1])) { @@ -412,7 +449,7 @@ void ESPHomeOTAComponent::handle_data_() { // Acknowledge auth OK - 1 byte this->data_write_byte_(ota::OTA_RESPONSE_AUTH_OK); - if (this->extended_proto_) { + if (this->extended_proto_()) { // Read ota type, 1 byte if (!this->data_readall_(buf, 1)) { this->log_read_error_(LOG_STR("OTA type")); diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index fd164b8138..c6f710b3fc 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -44,8 +44,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { } #endif // USE_OTA_PASSWORD -#ifdef USE_OTA_ENCRYPTION - void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#if defined(USE_OTA_ENCRYPTION) && !defined(USE_OTA_ENCRYPTION_FROM_API) + /// psk points at 32 bytes that live in flash for the life of the program + void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } #endif /// Manually set the port OTA should listen on @@ -85,9 +86,12 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { bool writing{false}; // a produced handshake frame is still being flushed uint8_t frame_buf[noise::FRAME_HEADER_SIZE + 1 + noise::MAX_HANDSHAKE_SIZE]; }; + // The api server's live context when the api has encryption, else our own + const noise::NoiseContext &noise_context_() const; bool noise_start_session_(uint8_t server_feature_flags); bool handle_noise_handshake_(); bool noise_try_read_frame_(); + size_t noise_frame_payload_len_(const uint8_t *header, size_t min_len, size_t max_len); bool noise_try_write_frame_(); void noise_send_reject_(const LogString *reason); ssize_t noise_decrypt_(uint8_t *buf, size_t len); @@ -144,7 +148,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { std::unique_ptr auth_buf_; #endif // USE_OTA_PASSWORD #ifdef USE_OTA_ENCRYPTION +#ifndef USE_OTA_ENCRYPTION_FROM_API noise::NoiseContext noise_ctx_; +#endif std::unique_ptr noise_; #endif // USE_OTA_ENCRYPTION @@ -166,6 +172,8 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { "OTA_BUFFER_SIZE must fit a full encrypted data frame"); #endif static constexpr uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45}; + // Derived from the feature byte; storing it would pad the trailing bytes + bool extended_proto_() const; #ifdef USE_OTA_PARTITIONS uint32_t running_app_offset_{0}; size_t running_app_size_{0}; @@ -179,7 +187,6 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { uint8_t auth_buf_pos_{0}; uint8_t auth_type_{0}; // Store auth type to know which hasher to use #endif // USE_OTA_PASSWORD - bool extended_proto_{false}; }; } // namespace esphome diff --git a/esphome/components/esphome/ota/ota_esphome_noise.cpp b/esphome/components/esphome/ota/ota_esphome_noise.cpp index 7f8331cf96..7401413d6d 100644 --- a/esphome/components/esphome/ota/ota_esphome_noise.cpp +++ b/esphome/components/esphome/ota/ota_esphome_noise.cpp @@ -3,6 +3,7 @@ #ifdef USE_OTA_ENCRYPTION #include "esphome/components/noise/noise.h" #include "esphome/components/ota/ota_backend.h" +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include @@ -40,24 +41,17 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() { * "NoiseOTAInit" | magic(5) | OK,version | client_features | FEATURE_FLAGS,server_flags */ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { + // A provisioned key cleared between the offer and here is not guarded: the + // session runs on the zero key load_psk fills in and fails the client's MAC. + // Default-init: the frame buffer is written before it is read // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession()); - if (this->noise_ == nullptr) { - ESP_LOGW(TAG, "Session allocation failed"); - this->cleanup_connection_(); - return false; - } - + this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession); static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1; static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags uint8_t prologue[OTA_NOISE_PROLOGUE_INIT_LEN + sizeof(MAGIC_BYTES) + PROLOGUE_ACK_LEN + PROLOGUE_CLIENT_FEATURES_LEN + PROLOGUE_FEATURE_ACK_LEN]; -#ifdef USE_ESP8266 - memcpy_P(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); -#else - std::memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); -#endif + progmem_memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); uint8_t *p = prologue + OTA_NOISE_PROLOGUE_INIT_LEN; // Magic bytes, already validated in MAGIC_READ std::memcpy(p, MAGIC_BYTES, sizeof(MAGIC_BYTES)); @@ -71,9 +65,13 @@ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { *p++ = ota::OTA_RESPONSE_FEATURE_FLAGS; *p++ = server_feature_flags; - int err = this->noise_->handshake.init(this->noise_ctx_.get_psk(), prologue, sizeof(prologue)); + // The caller only starts a session when the context holds a key + int err = this->noise_ == nullptr ? NOISE_ERROR_NO_MEMORY + : this->noise_->handshake.init(this->noise_context_(), prologue, sizeof(prologue)); if (err != 0) { - ESP_LOGW(TAG, "Handshake init: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + // Raw noise codes throughout: the name table would cost flash in builds + // where only the OTA uses noise + ESP_LOGW(TAG, "Session init: %d", err); this->cleanup_connection_(); return false; } @@ -105,14 +103,16 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { s.frame_pos = 0; s.frame_len = 0; if (s.frame_buf[noise::FRAME_HEADER_SIZE] != noise::HANDSHAKE_STATUS_OK) { - ESP_LOGW(TAG, "Bad handshake error byte: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); + ESP_LOGW(TAG, "Client rejected the handshake: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); this->cleanup_connection_(); return false; } int err = s.handshake.read_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, payload_len - 1); if (err != 0) { - ESP_LOGW(TAG, "Handshake read: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); - this->noise_send_reject_(noise::reject_reason_for(err)); + // A MAC failure here almost always means the uploader has a different key + const LogString *reason = noise::reject_reason_for(err); + ESP_LOGW(TAG, "Handshake read: %s (%d)", LOG_STR_ARG(reason), err); + this->noise_send_reject_(reason); this->cleanup_connection_(); return false; } @@ -123,7 +123,7 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { int err = s.handshake.write_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, noise::MAX_HANDSHAKE_SIZE, msg_len); if (err != 0) { - ESP_LOGW(TAG, "Handshake write: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Handshake write: %d", err); this->cleanup_connection_(); return false; } @@ -138,7 +138,7 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { case noise::NoiseResponderHandshake::Action::ACTION_SPLIT: { int err = s.handshake.split(s.send_cipher, s.recv_cipher); if (err != 0) { - ESP_LOGW(TAG, "Handshake split: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Handshake split: %d", err); this->cleanup_connection_(); return false; } @@ -154,33 +154,41 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { } } +/// Payload length from a frame header, or 0 (logged) when the indicator or +/// the length is out of range. Callers pass min_len >= 1 so 0 is never valid. +size_t ESPHomeOTAComponent::noise_frame_payload_len_(const uint8_t *header, size_t min_len, size_t max_len) { + const size_t payload_len = encode_uint16(header[1], header[2]); + if (header[0] != noise::FRAME_INDICATOR || payload_len < min_len || payload_len > max_len) { + ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], payload_len); + return 0; + } + return payload_len; +} + /// Non-blocking read of one handshake frame into the session buffer. bool ESPHomeOTAComponent::noise_try_read_frame_() { NoiseSession &s = *this->noise_; - while (s.frame_pos < noise::FRAME_HEADER_SIZE) { - ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, noise::FRAME_HEADER_SIZE - s.frame_pos); - if (!this->handle_read_error_(read, LOG_STR("read noise header"))) { - return false; + while (true) { + // The header first, then the body once the header says how long it is + const uint16_t want = s.frame_len == 0 ? noise::FRAME_HEADER_SIZE : s.frame_len; + if (s.frame_pos < want) { + ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, want - s.frame_pos); + if (!this->handle_read_error_(read, LOG_STR("read noise"))) { + return false; + } + s.frame_pos += read; + continue; } - s.frame_pos += read; - } - if (s.frame_len == 0) { - const uint16_t payload_len = encode_uint16(s.frame_buf[1], s.frame_buf[2]); - if (s.frame_buf[0] != noise::FRAME_INDICATOR || payload_len < 1 || payload_len > 1 + noise::MAX_HANDSHAKE_SIZE) { - ESP_LOGW(TAG, "Bad handshake frame: 0x%02X, %u bytes", s.frame_buf[0], payload_len); + if (s.frame_len != 0) { + return true; + } + const size_t payload_len = this->noise_frame_payload_len_(s.frame_buf, 1, 1 + noise::MAX_HANDSHAKE_SIZE); + if (payload_len == 0) { this->cleanup_connection_(); return false; } s.frame_len = noise::FRAME_HEADER_SIZE + payload_len; } - while (s.frame_pos < s.frame_len) { - ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, s.frame_len - s.frame_pos); - if (!this->handle_read_error_(read, LOG_STR("read noise frame"))) { - return false; - } - s.frame_pos += read; - } - return true; } /// Non-blocking write of the pending session-buffer frame. @@ -214,7 +222,7 @@ ssize_t ESPHomeOTAComponent::noise_decrypt_(uint8_t *buf, size_t len) { noise_buffer_set_inout(mbuf, buf, len, len); int err = noise_cipherstate_decrypt(this->noise_->recv_cipher, &mbuf); if (err != 0) { - ESP_LOGW(TAG, "Decrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Decrypt: %d", err); return -1; } return mbuf.size; @@ -229,9 +237,8 @@ ssize_t ESPHomeOTAComponent::noise_read_frame_blocking_(uint8_t *buf, size_t min if (!this->readall_(header, sizeof(header))) { return -1; } - const size_t ciphertext_len = encode_uint16(header[1], header[2]); - if (header[0] != noise::FRAME_INDICATOR || ciphertext_len < min_ciphertext || ciphertext_len > max_ciphertext) { - ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], ciphertext_len); + const size_t ciphertext_len = this->noise_frame_payload_len_(header, min_ciphertext, max_ciphertext); + if (ciphertext_len == 0) { return -1; } if (!this->readall_(buf, ciphertext_len)) { @@ -267,7 +274,7 @@ bool ESPHomeOTAComponent::noise_write_byte_(uint8_t byte) { noise_buffer_set_inout(mbuf, frame + noise::FRAME_HEADER_SIZE, 1, 1 + noise::MAC_SIZE); int err = noise_cipherstate_encrypt(this->noise_->send_cipher, &mbuf); if (err != 0) { - ESP_LOGW(TAG, "Encrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Encrypt: %d", err); return false; } noise::write_frame_header(frame, mbuf.size); diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 0f9328a482..a1d9444fc0 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -4,7 +4,9 @@ from typing import Any import esphome.codegen as cg import esphome.config_validation as cv -from esphome.const import CONF_KEY +from esphome.const import CONF_ENCRYPTION, CONF_KEY +from esphome.core import ID +from esphome.cpp_generator import MockObj from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] @@ -23,6 +25,14 @@ def validate_encryption_key(value: Any) -> str: if len(decoded) != 32: raise cv.Invalid("Encryption key must be base64 and 32 bytes long") + if not any(decoded): + # The device treats the all-zeros key as no key at all (it is the + # provisioning sentinel), so it must never reach a build + raise cv.Invalid( + f"The all-zeros {CONF_KEY} is reserved and provides no protection; " + f"omit the {CONF_KEY} to provision it at runtime, or generate a real " + "key with: openssl rand -base64 32" + ) # Return original data for roundtrip conversion return value @@ -45,15 +55,6 @@ def decode_encryption_key(value: str) -> bytes: return decoded -def is_reserved_key(value: str) -> bool: - """Whether the key is the reserved all-zeros provisioning sentinel. - - The device treats it as no key configured, so consumers that require a - real key must reject it. - """ - return not any(decode_encryption_key(value)) - - ENCRYPTION_SCHEMA = cv.Schema( { cv.Optional(CONF_KEY): cv.sensitive(validate_encryption_key), @@ -61,6 +62,21 @@ ENCRYPTION_SCHEMA = cv.Schema( ) +def static_encryption_key(conf: ConfigType) -> str | None: + """The build time key of a component config; None without one or when + the key is provisioned at runtime.""" + return (conf.get(CONF_ENCRYPTION) or {}).get(CONF_KEY) or None + + +def new_psk_progmem(parent_id: ID, key: str) -> MockObj: + """Emit the decoded key as a PROGMEM array; the component keeps a pointer + so the key never occupies RAM.""" + return cg.progmem_array( + ID(f"{parent_id.id}_psk", is_declaration=True, type=cg.uint8), + list(decode_encryption_key(key)), + ) + + def encryption_schema(config: ConfigType | None) -> ConfigType: # A bare `encryption:` block is valid; a missing key means the consumer # falls back to its keyless behavior (api provisioning, ota inheriting diff --git a/esphome/components/noise/noise.cpp b/esphome/components/noise/noise.cpp index 95fab322db..4806706167 100644 --- a/esphome/components/noise/noise.cpp +++ b/esphome/components/noise/noise.cpp @@ -1,5 +1,6 @@ #include "noise.h" #ifdef USE_NOISE +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include @@ -15,6 +16,14 @@ namespace esphome::noise { static const char *const TAG = "noise"; +void NoiseContext::load_psk(psk_t &out) const { + if (this->psk_ == nullptr) { + out.fill(0); + return; + } + progmem_memcpy(out.data(), this->psk_, out.size()); +} + const LogString *noise_err_to_logstr(int err) { if (err == NOISE_ERROR_NO_MEMORY) return LOG_STR("NO_MEMORY"); diff --git a/esphome/components/noise/noise.h b/esphome/components/noise/noise.h index f9da8d35b8..1033d5423c 100644 --- a/esphome/components/noise/noise.h +++ b/esphome/components/noise/noise.h @@ -23,16 +23,16 @@ class NoiseContext { } return acc == 0; } - void set_psk(psk_t psk) { - this->psk_ = psk; - this->has_psk_ = !is_all_zeros(psk); - } - const psk_t &get_psk() const { return this->psk_; } - bool has_psk() const { return this->has_psk_; } + /// psk points at 32 bytes that outlive the context (PROGMEM or caller owned + /// RAM); nullptr means no key. Runtime callers map the all-zeros key to + /// nullptr themselves; validation keeps it out of yaml. + void set_psk(const uint8_t *psk) { this->psk_ = psk; } + /// Copy the key out (flash-aware on ESP8266); all zeros when none is set. + void load_psk(psk_t &out) const; + bool has_psk() const { return this->psk_ != nullptr; } protected: - psk_t psk_{}; - bool has_psk_{false}; + const uint8_t *psk_{nullptr}; }; /// Convert a noise error code to a readable error diff --git a/esphome/components/noise/noise_handshake.cpp b/esphome/components/noise/noise_handshake.cpp index 6d426de012..cc7fa603c4 100644 --- a/esphome/components/noise/noise_handshake.cpp +++ b/esphome/components/noise/noise_handshake.cpp @@ -20,7 +20,7 @@ NoiseResponderHandshake::~NoiseResponderHandshake() { } } -int NoiseResponderHandshake::init(const psk_t &psk, const uint8_t *prologue, size_t prologue_len) { +int NoiseResponderHandshake::init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len) { if (this->handshake_ != nullptr) { noise_handshakestate_free(this->handshake_); this->handshake_ = nullptr; @@ -44,6 +44,9 @@ int NoiseResponderHandshake::init(const psk_t &psk, const uint8_t *prologue, siz HANDSHAKE_STEP_LOG("noise_handshakestate_new_by_id", err); return err; } + // noise-c keeps its own copy, so the key only passes through the stack here + psk_t psk; + ctx.load_psk(psk); err = noise_handshakestate_set_pre_shared_key(this->handshake_, psk.data(), psk.size()); if (err != 0) { HANDSHAKE_STEP_LOG("noise_handshakestate_set_pre_shared_key", err); diff --git a/esphome/components/noise/noise_handshake.h b/esphome/components/noise/noise_handshake.h index 30596f35c2..bf1aa8cb7f 100644 --- a/esphome/components/noise/noise_handshake.h +++ b/esphome/components/noise/noise_handshake.h @@ -36,9 +36,9 @@ class NoiseResponderHandshake { NoiseResponderHandshake(const NoiseResponderHandshake &) = delete; NoiseResponderHandshake &operator=(const NoiseResponderHandshake &) = delete; - /// Create and start the handshake with the given PSK and prologue. A - /// repeated call frees the previous handshake state and starts over. - [[nodiscard]] int init(const psk_t &psk, const uint8_t *prologue, size_t prologue_len); + /// Create and start the handshake with the context's PSK and the prologue. + /// A repeated call frees the previous handshake state and starts over. + [[nodiscard]] int init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len); /// ACTION_FAILED is the catch-all: returned before init(), after split() /// has released the state, and when noise-c reports a failed handshake. [[nodiscard]] Action action() const; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 526adf74f0..9dd1e0ced6 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -244,6 +244,9 @@ #define USE_RUNTIME_STATS #define USE_OTA #define USE_OTA_ENCRYPTION +#define USE_OTA_ENCRYPTION_FROM_API +#define USE_OTA_ENCRYPTION_PROVISIONED +#define USE_OTA_ENCRYPTION_REQUIRED #define USE_OTA_PASSWORD #define USE_OTA_VERSION 2 #define USE_TIME_TIMEZONE diff --git a/esphome/espota2.py b/esphome/espota2.py index ac4cbeeb7c..ce403c398d 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -202,6 +202,49 @@ class OTANetworkError(OTAError): """Network-level OTA failure (timeout, reset, closed connection); retrying may succeed.""" +# Remove before 2027.3.0 +class OTAEncryptionFallback(OTAError): + """The encrypted attempt failed and the caller may retry in plaintext.""" + + +# Remove before 2027.3.0 +PLAINTEXT_FALLBACK_NOTICE = ( + "A device with an api encryption key offers encryption after this " + "install; add 'encryption:' under 'ota: platform: esphome' to require it. " + "This plaintext fallback is removed in 2027.3.0." +) + + +# Remove before 2027.3.0 +class _EncryptionAttempt: + """The key an upload tries and whether it may fall back to plaintext; + a rejected handshake falls back at once, a transport fault only on repeat.""" + + def __init__(self, noise_psk: str | None, plaintext_fallback: bool) -> None: + self.noise_psk = noise_psk + self.plaintext_fallback = plaintext_fallback + self.handshake_faults = 0 + + def handshake_fault_falls_back(self) -> bool: + self.handshake_faults += 1 + return self.plaintext_fallback and self.handshake_faults >= 2 + + def downgrade(self, reason: str) -> None: + _LOGGER.warning( + "%s. Retrying in plaintext; a device that requires encryption " + "refuses it. %s", + reason, + PLAINTEXT_FALLBACK_NOTICE, + ) + self.noise_psk = None + self.plaintext_fallback = False + + +# Remove before 2027.3.0: only the fallback decision needs this distinction +class OTAHandshakeNetworkError(OTANetworkError): + """A transport failure inside the noise handshake; retrying encrypted may succeed.""" + + def _committed_error(err: OTANetworkError) -> OTAError: """Wrap a network failure that happened once the device had the full image. @@ -464,6 +507,7 @@ def perform_ota( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> None: # Validate up front; an out-of-range value would only surface as a # ValueError deep inside send_check, bypassing OTAError handling @@ -528,19 +572,28 @@ def perform_ota( else: features = 0 - if noise_psk: - # Fail closed: never fall back to a plaintext upload when an - # encryption key is configured, an active attacker could otherwise - # strip the feature flag and capture the image (it contains the wifi - # credentials and the api encryption key). - if not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + if noise_psk and not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + if plaintext_fallback: + # Remove before 2027.3.0: older firmware that cannot encrypt still + # gets its update on this connection + _LOGGER.warning( + "The device did not offer OTA encryption; continuing in plaintext. %s", + PLAINTEXT_FALLBACK_NOTICE, + ) + noise_psk = None + else: + # Fail closed: an attacker could otherwise strip the offer and + # capture the image (wifi credentials, api key) raise OTAError( "An OTA encryption key is configured but the device did not " "offer encryption; refusing to send the image in plaintext. " - "If the running firmware predates OTA encryption, first update " - "it without the 'ota: encryption:' block (over a trusted " - "network or via USB), then restore the block and upload again." + "The running firmware predates ESPHome 2026.9.0 or has no " + "'api: encryption: key'. With an api key, install once " + "without the 'ota: encryption:' block (that build offers " + "encryption), then restore it; otherwise flash by serial or " + "the web_server OTA platform." ) + if noise_psk: # The prologue binds every negotiation byte both sides saw, so any # tampering with the plaintext preamble breaks the handshake. prologue = ( @@ -549,8 +602,18 @@ def perform_ota( + bytes([RESPONSE_OK, version, features_to_send]) + bytes([RESPONSE_FEATURE_FLAGS, features]) ) + # Built outside the try: a local failure must never downgrade the upload sock = NoiseSocketWrapper(sock, noise_psk, prologue) - sock.do_handshake() + try: + sock.do_handshake() + except OTANetworkError as err: + # A transport fault: retry encrypted before considering plaintext + raise OTAHandshakeNetworkError(str(err)) from err + except OTAError as err: + # Remove before 2027.3.0 + if plaintext_fallback: + raise OTAEncryptionFallback(str(err)) from err + raise _LOGGER.info("Encrypted connection established") if ota_type != OTA_TYPE_UPDATE_APP: @@ -757,6 +820,7 @@ def run_ota_impl_( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> tuple[int, str | None]: from esphome.core import CORE @@ -795,7 +859,9 @@ def run_ota_impl_( total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS last_error = "" reached_device = False - for attempt in range(total_attempts): + attempt = 0 + encryption = _EncryptionAttempt(noise_psk, plaintext_fallback) + while attempt < total_attempts: af, socktype, _, _, sa = res[attempt % len(res)] if reached_device or attempt >= len(res): _LOGGER.info( @@ -815,17 +881,40 @@ def run_ota_impl_( sock.close() _LOGGER.warning("Connecting to %s port %s failed: %s", sa[0], sa[1], err) last_error = f"connecting to {sa[0]} failed: {err}" + attempt += 1 continue _LOGGER.info("Connected to %s", sa[0]) reached_device = True with contextlib.closing(sock), Path(filename).open("rb") as file_handle: try: - perform_ota(sock, password, file_handle, filename, ota_type, noise_psk) + perform_ota( + sock, + password, + file_handle, + filename, + ota_type, + encryption.noise_psk, + encryption.plaintext_fallback, + ) + except OTAEncryptionFallback as err: + # Same address and attempt budget: not a network retry + last_error = str(err) + encryption.downgrade(last_error) + continue + except OTAHandshakeNetworkError as err: + last_error = str(err) + if encryption.handshake_fault_falls_back(): + encryption.downgrade(last_error) + continue + _LOGGER.warning("%s", last_error) + attempt += 1 + continue except OTANetworkError as err: # Transient network failure; retry last_error = str(err) _LOGGER.warning("%s", last_error) + attempt += 1 continue except OTAError as err: # Device-reported error (wrong password, wrong flash size, ...); @@ -847,10 +936,17 @@ def run_ota( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> tuple[int, str | None]: try: return run_ota_impl_( - remote_host, remote_port, password, filename, ota_type, noise_psk + remote_host, + remote_port, + password, + filename, + ota_type, + noise_psk, + plaintext_fallback, ) except OTAError as err: _LOGGER.error(err) diff --git a/esphome/wizard.py b/esphome/wizard.py index f7706928e9..897d5f60a1 100644 --- a/esphome/wizard.py +++ b/esphome/wizard.py @@ -148,11 +148,13 @@ def wizard_file(**kwargs: Unpack[WizardFileKwargs]) -> str: if "api_encryption_key" in kwargs: config += f' encryption:\n key: "{kwargs["api_encryption_key"]}"\n' - # Configure OTA + # The api key also secures OTA; a password only serves older uploaders config += "\nota:\n" config += " - platform: esphome\n" if "ota_password" in kwargs: config += f' password: "{kwargs["ota_password"]}"' + elif "api_encryption_key" in kwargs: + config += " encryption:" # Configuring wifi config += "\n\nwifi:\n" @@ -529,20 +531,9 @@ def wizard(path: Path) -> int: safe_print() safe_print("You'll need this key when adding the device to Home Assistant.") sleep(1) - - safe_print() - safe_print( - f"Do you want to set a {color(AnsiFore.GREEN, 'password')} for OTA updates? " - "This can be insecure if you do not trust the WiFi network." - ) - safe_print() - sleep(0.25) - safe_print("Press ENTER for no password") - ota_password = safe_input(color(AnsiFore.BOLD_WHITE, "(password): ")) else: ssid, psk = "", "" api_encryption_key = None - ota_password = "" kwargs = { "path": path, @@ -553,10 +544,9 @@ def wizard(path: Path) -> int: "psk": psk, "type": "basic", } + # The api key also secures OTA updates, so the wizard sets no OTA password if api_encryption_key: kwargs["api_encryption_key"] = api_encryption_key - if ota_password: - kwargs["ota_password"] = ota_password if not wizard_write(**kwargs): return 1 diff --git a/tests/component_tests/noise/test_encryption_key.py b/tests/component_tests/noise/test_encryption_key.py index 10f1eb3d4c..2b79bd5464 100644 --- a/tests/component_tests/noise/test_encryption_key.py +++ b/tests/component_tests/noise/test_encryption_key.py @@ -5,11 +5,7 @@ from __future__ import annotations import pytest from esphome import config_validation as cv -from esphome.components.noise import ( - decode_encryption_key, - is_reserved_key, - validate_encryption_key, -) +from esphome.components.noise import decode_encryption_key, validate_encryption_key KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @@ -41,6 +37,8 @@ def test_decode_encryption_key_rejects_short_decode() -> None: decode_encryption_key("AAECAw==") -def test_is_reserved_key() -> None: - assert is_reserved_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") - assert not is_reserved_key(KEY) +def test_validate_encryption_key_rejects_all_zeros() -> None: + """The all-zeros key is the provisioning sentinel the device treats as no + key, so it never reaches a build.""" + with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): + validate_encryption_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") diff --git a/tests/component_tests/ota/test_esphome_ota.py b/tests/component_tests/ota/test_esphome_ota.py index 873f162555..d3092294dc 100644 --- a/tests/component_tests/ota/test_esphome_ota.py +++ b/tests/component_tests/ota/test_esphome_ota.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable import logging from typing import Any @@ -14,6 +15,7 @@ from esphome.components.esphome.ota import ( _validate_no_password_with_encryption, ota_esphome_final_validate, ) +from esphome.components.noise import static_encryption_key from esphome.const import ( CONF_API, CONF_ENCRYPTION, @@ -115,7 +117,6 @@ def test_non_esphome_ota_unaffected() -> None: API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" OTHER_KEY = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" -ZEROS_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" def test_encryption_key_inherited_from_api() -> None: @@ -197,36 +198,6 @@ def test_encryption_without_any_key_rejected() -> None: fv.full_config.reset(token) -def test_encryption_explicit_all_zeros_key_rejected() -> None: - """The all-zeros key is the provisioning sentinel; the device would treat - it as no PSK and accept plaintext, so it must fail validation.""" - full_conf = { - CONF_OTA: [ - _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}) - ], - } - token = fv.full_config.set(full_conf) - try: - with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): - ota_esphome_final_validate({}) - finally: - fv.full_config.reset(token) - - -def test_encryption_inherited_all_zeros_key_rejected() -> None: - """An all-zeros api key must not silently disable ota encryption either.""" - full_conf = { - CONF_API: {CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}, - CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], - } - token = fv.full_config.set(full_conf) - try: - with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): - ota_esphome_final_validate({}) - finally: - fv.full_config.reset(token) - - def test_encryption_key_mismatch_between_merged_configs_rejected() -> None: """Same-port configs with different encryption keys raise.""" full_conf = { @@ -295,13 +266,14 @@ def test_encryption_explicit_key_with_runtime_provisioned_api_accepted() -> None fv.full_config.reset(token) +@pytest.mark.parametrize("component", ["web_server", "prometheus"]) def test_encryption_with_web_server_ota_warns( - caplog: pytest.LogCaptureFixture, + caplog: pytest.LogCaptureFixture, component: str ) -> None: - """With the web_server component the plaintext /update endpoint is always - on; the combination validates with a warning.""" + """web_server and prometheus keep the shared listener up, so the + plaintext /update endpoint is always on and the combination warns.""" full_conf = { - "web_server": {}, + component: {}, CONF_OTA: [ _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, @@ -316,12 +288,12 @@ def test_encryption_with_web_server_ota_warns( fv.full_config.reset(token) -def test_encryption_with_captive_portal_web_server_ota_warns( +def test_encryption_with_captive_portal_does_not_warn( caplog: pytest.LogCaptureFixture, ) -> None: """captive_portal auto-loads the web_server ota platform without the - web_server component; encryption stays usable and only warns, so the - fallback AP recovery path is not lost.""" + web_server component; its endpoint only exists while the fallback AP is + active and is the intended recovery path, so there is no warning.""" full_conf = { "captive_portal": {}, CONF_OTA: [ @@ -333,7 +305,10 @@ def test_encryption_with_captive_portal_web_server_ota_warns( try: with caplog.at_level(logging.WARNING): ota_esphome_final_validate({}) - assert any("captive_portal" in record.message for record in caplog.records) + assert not any( + "OTA encryption does not cover" in record.message + for record in caplog.records + ) esphome_conf = next( conf for conf in fv.full_config.get()[CONF_OTA] @@ -344,6 +319,100 @@ def test_encryption_with_captive_portal_web_server_ota_warns( fv.full_config.reset(token) +def test_password_with_api_key_warns(caplog: pytest.LogCaptureFixture) -> None: + """A static api key makes the device offer encryption and the CLI take + it, so the password is dead weight; the config validates with a warning.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: API_KEY}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert any("wastes significant flash" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_password_with_runtime_api_key_warns_differently( + caplog: pytest.LogCaptureFixture, +) -> None: + """The CLI still needs the password, but the provisioned key also + authenticates uploads; the warning says so without the flash advice.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + messages = [r.message for r in caplog.records] + assert any("provisioned at runtime also authenticates" in m for m in messages) + assert not any("wastes significant flash" in m for m in messages) + finally: + fv.full_config.reset(token) + + +def test_password_without_api_key_no_warning( + caplog: pytest.LogCaptureFixture, +) -> None: + """Without an api key there is no offer, so nothing to warn about.""" + full_conf = { + CONF_API: {}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any("authenticates" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_web_server_component_without_ota_platform_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + """The web_server component alone has no /update endpoint.""" + full_conf = { + "web_server": {}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any( + "OTA encryption does not cover" in r.message for r in caplog.records + ) + finally: + fv.full_config.reset(token) + + +def test_web_server_ota_platform_alone_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + """Only the web_server component starts the shared listener, so the ota + platform on its own never exposes /update.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any("plaintext /update" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + def test_web_server_ota_without_encryption_unaffected() -> None: """web_server ota stays valid alongside an unencrypted esphome entry.""" full_conf = { @@ -370,20 +439,87 @@ def test_auto_load_pulls_noise_only_for_encryption() -> None: assert "noise" in AUTO_LOAD({}) -def test_filter_source_files_excludes_noise_without_encryption() -> None: - """The noise transport source compiles only for encrypted builds.""" - old_config = CORE.config - try: - CORE.config = {CONF_OTA: [_make_ota_config(port=3232)]} - assert FILTER_SOURCE_FILES() == ["ota_esphome_noise.cpp"] - CORE.config = { - CONF_OTA: [ - _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) - ] - } - assert FILTER_SOURCE_FILES() == [] - finally: - CORE.config = old_config +def test_static_encryption_key() -> None: + """Only a build-time key counts; a runtime provisioned one does not.""" + assert static_encryption_key({}) is None + assert static_encryption_key({CONF_ENCRYPTION: {}}) is None + assert static_encryption_key({CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) == API_KEY + + +@pytest.mark.parametrize( + ("yaml_name", "defines_present", "defines_absent"), + [ + # An api key alone compiles the transport in without requiring it; + # the device uses the api server's key, not a copy + ( + "api_key_offer", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API"}, + {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # A password still guards plaintext uploads on an offering device + ( + "api_key_offer_password", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_PASSWORD"}, + {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # The ota encryption block is what makes the device refuse plaintext + ( + "encryption_required", + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_REQUIRED", + "USE_OTA_ENCRYPTION_FROM_API", + }, + {"USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # Without api encryption the ota key is the device's own + ( + "own_key", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_REQUIRED"}, + {"USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # A key provisioned at runtime lives in the api server; the device + # offers with it once provisioned and never requires it + ( + "runtime_api_key", + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_FROM_API", + "USE_OTA_ENCRYPTION_PROVISIONED", + }, + {"USE_OTA_ENCRYPTION_REQUIRED"}, + ), + # No api encryption at all keeps the noise glue out of the build + ( + "plain", + set(), + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_REQUIRED", + "USE_OTA_ENCRYPTION_FROM_API", + "USE_OTA_ENCRYPTION_PROVISIONED", + }, + ), + ], +) +def test_encryption_offer_codegen( + generate_main: Callable[[str], str], + yaml_name: str, + defines_present: set[str], + defines_absent: set[str], +) -> None: + main_cpp = generate_main( + f"tests/component_tests/ota/test_esphome_ota_{yaml_name}.yaml" + ) + defines = {define.name for define in CORE.defines} + assert defines_present <= defines + assert not (defines_absent & defines) + encrypted = "USE_OTA_ENCRYPTION" in defines_present + own_key = encrypted and "USE_OTA_ENCRYPTION_FROM_API" not in defines_present + assert ("esphome_esphomeotacomponent_id->set_noise_psk(" in main_cpp) is own_key + assert ("set_auth_password(" in main_cpp) is ("USE_OTA_PASSWORD" in defines_present) + # The noise transport source compiles only when the define is set + assert FILTER_SOURCE_FILES() == ([] if encrypted else ["ota_esphome_noise.cpp"]) def test_password_with_encryption_rejected() -> None: diff --git a/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml b/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml new file mode 100644 index 0000000000..ca26eb9f46 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml @@ -0,0 +1,11 @@ +esphome: + name: ota-offer + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome diff --git a/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml b/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml new file mode 100644 index 0000000000..1e23975690 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml @@ -0,0 +1,12 @@ +esphome: + name: ota-offer-password + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + password: "superlongpasswordthatnoonewillknow" diff --git a/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml b/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml new file mode 100644 index 0000000000..36690038d8 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml @@ -0,0 +1,12 @@ +esphome: + name: ota-encryption-required + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + encryption: diff --git a/tests/component_tests/ota/test_esphome_ota_own_key.yaml b/tests/component_tests/ota/test_esphome_ota_own_key.yaml new file mode 100644 index 0000000000..b6d1e4200d --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_own_key.yaml @@ -0,0 +1,11 @@ +esphome: + name: ota-own-key + +host: + +api: + +ota: + - platform: esphome + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" diff --git a/tests/component_tests/ota/test_esphome_ota_plain.yaml b/tests/component_tests/ota/test_esphome_ota_plain.yaml new file mode 100644 index 0000000000..c5ca7afcf0 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_plain.yaml @@ -0,0 +1,9 @@ +esphome: + name: ota-plain + +host: + +api: + +ota: + - platform: esphome diff --git a/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml b/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml new file mode 100644 index 0000000000..8825335141 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml @@ -0,0 +1,10 @@ +esphome: + name: ota-runtime-key + +host: + +api: + encryption: + +ota: + - platform: esphome diff --git a/tests/components/noise/test_noise_handshake.cpp b/tests/components/noise/test_noise_handshake.cpp index d879a26c43..f2081f2965 100644 --- a/tests/components/noise/test_noise_handshake.cpp +++ b/tests/components/noise/test_noise_handshake.cpp @@ -68,6 +68,14 @@ class Initiator { static const uint8_t PROLOGUE[] = {'t', 'e', 's', 't', 'p', 'r', 'o', 'l', 'o', 'g', 'u', 'e'}; +// The context only points at the key and init() copies it before returning, +// so a temporary context over a temporary key is safe within one call +static NoiseContext ctx_for(const psk_t &psk) { + NoiseContext ctx; + ctx.set_psk(psk.data()); + return ctx; +} + static psk_t make_psk(uint8_t seed) { psk_t psk; for (size_t i = 0; i < psk.size(); i++) { @@ -102,7 +110,7 @@ TEST(NoiseResponderHandshakeTest, MessageMethodsErrorBeforeInit) { TEST(NoiseResponderHandshakeTest, FullHandshakeAndTransportRoundTrip) { const psk_t psk = make_psk(7); NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(psk, PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); EXPECT_EQ(responder.action(), Action::ACTION_READ); Initiator initiator(psk, PROLOGUE, sizeof(PROLOGUE)); @@ -155,8 +163,8 @@ TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { // proves the restart took effect; the old state surviving would fail the // MAC here. NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(make_psk(7), PROLOGUE, sizeof(PROLOGUE)), 0); - ASSERT_EQ(responder.init(make_psk(9), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(7)), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(9)), PROLOGUE, sizeof(PROLOGUE)), 0); EXPECT_EQ(responder.action(), Action::ACTION_READ); Initiator initiator(make_psk(9), PROLOGUE, sizeof(PROLOGUE)); @@ -168,7 +176,7 @@ TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { TEST(NoiseResponderHandshakeTest, WrongPskFailsWithMacFailure) { NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(make_psk(7), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(7)), PROLOGUE, sizeof(PROLOGUE)), 0); Initiator initiator(make_psk(200), PROLOGUE, sizeof(PROLOGUE)); uint8_t msg[MAX_HANDSHAKE_SIZE]; @@ -185,7 +193,7 @@ TEST(NoiseResponderHandshakeTest, MismatchedPrologueFailsWithMacFailure) { // tampered preamble must fail even with the right key. const psk_t psk = make_psk(7); NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(psk, PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); static const uint8_t TAMPERED[] = {'x'}; Initiator initiator(psk, TAMPERED, sizeof(TAMPERED)); diff --git a/tests/components/noise/test_noise_primitives.cpp b/tests/components/noise/test_noise_primitives.cpp index 018be9f717..8687c4b963 100644 --- a/tests/components/noise/test_noise_primitives.cpp +++ b/tests/components/noise/test_noise_primitives.cpp @@ -17,12 +17,17 @@ TEST(NoiseContextTest, AllZerosPskIsReserved) { EXPECT_FALSE(NoiseContext::is_all_zeros(psk)); NoiseContext ctx; + psk_t loaded; EXPECT_FALSE(ctx.has_psk()); - ctx.set_psk(zeros); - EXPECT_FALSE(ctx.has_psk()); - ctx.set_psk(psk); + ctx.load_psk(loaded); + EXPECT_EQ(loaded, zeros); + ctx.set_psk(psk.data()); EXPECT_TRUE(ctx.has_psk()); - EXPECT_EQ(ctx.get_psk(), psk); + ctx.load_psk(loaded); + EXPECT_EQ(loaded, psk); + // Callers map the reserved key to nullptr; the context just stores what it is given + ctx.set_psk(nullptr); + EXPECT_FALSE(ctx.has_psk()); } TEST(WireFormatTest, FrameHeaderIsIndicatorPlusBigEndianLength) { diff --git a/tests/components/ota/api_key_offer.yaml b/tests/components/ota/api_key_offer.yaml new file mode 100644 index 0000000000..8d1814bf7e --- /dev/null +++ b/tests/components/ota/api_key_offer.yaml @@ -0,0 +1,12 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + port: 3290 + password: "superlongpasswordthatnoonewillknow" diff --git a/tests/components/ota/api_runtime_key.yaml b/tests/components/ota/api_runtime_key.yaml new file mode 100644 index 0000000000..8976c92f96 --- /dev/null +++ b/tests/components/ota/api_runtime_key.yaml @@ -0,0 +1,10 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + +ota: + - platform: esphome + port: 3291 diff --git a/tests/components/ota/test-api_key_offer.esp32-idf.yaml b/tests/components/ota/test-api_key_offer.esp32-idf.yaml new file mode 100644 index 0000000000..ecda625521 --- /dev/null +++ b/tests/components/ota/test-api_key_offer.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_key_offer.yaml diff --git a/tests/components/ota/test-api_key_offer.esp8266-ard.yaml b/tests/components/ota/test-api_key_offer.esp8266-ard.yaml new file mode 100644 index 0000000000..ecda625521 --- /dev/null +++ b/tests/components/ota/test-api_key_offer.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_key_offer.yaml diff --git a/tests/components/ota/test-api_runtime_key.esp32-idf.yaml b/tests/components/ota/test-api_runtime_key.esp32-idf.yaml new file mode 100644 index 0000000000..4709a9e45c --- /dev/null +++ b/tests/components/ota/test-api_runtime_key.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_runtime_key.yaml diff --git a/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml b/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml new file mode 100644 index 0000000000..4709a9e45c --- /dev/null +++ b/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_runtime_key.yaml diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6777e6cabc..15c5860879 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -162,6 +162,13 @@ def integration_test_dir() -> Generator[Path]: yield Path(tmpdir) +@pytest.fixture +def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Host preferences persist per device name; give the test its own so a + provisioned key never leaks into another run.""" + monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) + + @pytest.fixture def reserved_tcp_port() -> Generator[tuple[int, socket.socket]]: """Reserve an unused TCP port by holding the socket open.""" diff --git a/tests/integration/const.py b/tests/integration/const.py index 6876bbd443..e35d4673af 100644 --- a/tests/integration/const.py +++ b/tests/integration/const.py @@ -9,6 +9,13 @@ API_CONNECTION_TIMEOUT = 30.0 # seconds PORT_WAIT_TIMEOUT = 30.0 # seconds PORT_POLL_INTERVAL = 0.1 # seconds +# The well-known all-zeros provisioning PSK, a key to provision over it, and +# the time the device takes to activate a newly saved key (100 ms timer plus +# margin) +ZERO_PSK = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +PROVISIONING_PSK = b"bm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm4=" +KEY_ACTIVATION_DELAY = 0.5 # seconds + # Process shutdown timeouts SIGINT_TIMEOUT = 5.0 # seconds SIGTERM_TIMEOUT = 2.0 # seconds diff --git a/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml b/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml new file mode 100644 index 0000000000..1dedcc9ee1 --- /dev/null +++ b/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml @@ -0,0 +1,12 @@ +esphome: + name: host-ota-test +host: +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +ota: + - platform: esphome + port: __OTA_PORT__ + password: "hunter2" +logger: + level: DEBUG diff --git a/tests/integration/fixtures/host_ota_provisioned_api_key.yaml b/tests/integration/fixtures/host_ota_provisioned_api_key.yaml new file mode 100644 index 0000000000..aa0a9a66c9 --- /dev/null +++ b/tests/integration/fixtures/host_ota_provisioned_api_key.yaml @@ -0,0 +1,10 @@ +esphome: + name: host-ota-test +host: +api: + encryption: +ota: + - platform: esphome + port: __OTA_PORT__ +logger: + level: DEBUG diff --git a/tests/integration/test_api_zero_psk_provisioning.py b/tests/integration/test_api_zero_psk_provisioning.py index bcea2a2471..f315335d1b 100644 --- a/tests/integration/test_api_zero_psk_provisioning.py +++ b/tests/integration/test_api_zero_psk_provisioning.py @@ -10,34 +10,40 @@ from __future__ import annotations import asyncio import base64 +import socket from aioesphomeapi import InvalidEncryptionKeyAPIError, RequiresEncryptionAPIError import pytest -from .types import APIClientConnectedFactory, RunCompiledFunction +from .conftest import run_binary_and_wait_for_port +from .const import KEY_ACTIVATION_DELAY, LOCALHOST, PROVISIONING_PSK, ZERO_PSK +from .types import ( + APIClientConnectedFactory, + CompileFunction, + ConfigWriter, + RunCompiledFunction, +) -# The well-known provisioning PSK: base64 of 32 zero bytes -ZERO_PSK = base64.b64encode(bytes(32)).decode() -# A real key to provision -NEW_KEY = base64.b64encode(b"n" * 32) -# Time for the device to activate a newly saved key (100ms timer plus margin) -KEY_ACTIVATION_DELAY = 0.5 - - -@pytest.fixture(autouse=True) -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: - """Keep host preferences per-test so every run starts unprovisioned.""" - monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) +pytestmark = pytest.mark.usefixtures("isolated_preferences") +NEW_KEY = PROVISIONING_PSK @pytest.mark.asyncio async def test_api_zero_psk_provisioning( yaml_config: str, - run_compiled: RunCompiledFunction, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], api_client_connected: APIClientConnectedFactory, ) -> None: - """Exercise the reject paths, then provision a key over the zero-PSK channel.""" - async with run_compiled(yaml_config): + """Exercise the reject paths, provision a key over the zero-PSK channel, + and check the key comes back from preferences on the next boot.""" + port, port_socket = reserved_tcp_port + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + port_socket.close() + + async with run_binary_and_wait_for_port(binary_path, LOCALHOST, port): # --- Pre-provisioning reject paths (device state is unchanged) --- # A wrong (non-zero) PSK fails against the zero provisioning PSK @@ -97,6 +103,19 @@ async def test_api_zero_psk_provisioning( async with api_client_connected(timeout=5) as client: await client.device_info() + # The key is loaded from preferences on the next boot + lines: list[str] = [] + async with run_binary_and_wait_for_port( + binary_path, LOCALHOST, port, line_callback=lines.append + ): + async with api_client_connected(noise_psk=NEW_KEY.decode()) as client: + device_info = await client.device_info() + assert device_info.api_encryption_provisionable is False + with pytest.raises(InvalidEncryptionKeyAPIError): + async with api_client_connected(noise_psk=ZERO_PSK, timeout=5) as client: + await client.device_info() + assert any("Loaded saved Noise PSK" in line for line in lines) + @pytest.mark.asyncio async def test_api_zero_psk_provisioning_plaintext( diff --git a/tests/integration/test_host_ota.py b/tests/integration/test_host_ota.py index 4e74814534..f8c122c6e1 100644 --- a/tests/integration/test_host_ota.py +++ b/tests/integration/test_host_ota.py @@ -8,9 +8,12 @@ instance covers the FD_CLOEXEC path. from __future__ import annotations import asyncio +import base64 from collections.abc import Generator from contextlib import contextmanager +from dataclasses import dataclass import functools +from pathlib import Path import socket import pytest @@ -18,10 +21,18 @@ import pytest from esphome import espota2 from .conftest import run_binary, wait_and_connect_api_client -from .const import LOCALHOST, PORT_POLL_INTERVAL, PORT_WAIT_TIMEOUT -from .types import CompileFunction, ConfigWriter +from .const import ( + KEY_ACTIVATION_DELAY, + LOCALHOST, + PORT_POLL_INTERVAL, + PORT_WAIT_TIMEOUT, + PROVISIONING_PSK, + ZERO_PSK, +) +from .types import APIClientConnectedFactory, CompileFunction, ConfigWriter DEVICE_NAME = "host-ota-test" +API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @contextmanager @@ -35,6 +46,14 @@ def _reserve_port() -> Generator[tuple[int, socket.socket]]: s.close() +async def _wait_for_line(lines: list[str], needle: str, timeout: float = 5.0) -> None: + """The config dump prints after every setup, a little after the api port + opens, so wait for it rather than assert on the lines seen so far.""" + async with asyncio.timeout(timeout): + while not any(needle in line for line in lines): + await asyncio.sleep(PORT_POLL_INTERVAL) + + async def _wait_for_port(host: str, port: int, timeout: float) -> None: """Poll until a TCP port accepts connections, or raise TimeoutError.""" loop = asyncio.get_running_loop() @@ -51,6 +70,102 @@ async def _wait_for_port(host: str, port: int, timeout: float) -> None: raise TimeoutError(f"Port {port} on {host} did not open within {timeout}s") +async def _build( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> tuple[int, int, Path]: + """Reserve an OTA port, compile the fixture with it, and release both + ports right before the binary is started.""" + api_port, api_socket = reserved_tcp_port + with _reserve_port() as (ota_port, ota_socket): + config_path = await write_yaml_config( + yaml_config.replace("__OTA_PORT__", str(ota_port)) + ) + binary_path = await compile_esphome(config_path) + api_socket.close() + ota_socket.close() + return api_port, ota_port, binary_path + + +async def _run_ota( + ota_port: int, + password: str | None, + binary_path: Path, + noise_psk: str | None, + plaintext_fallback: bool = False, +) -> int: + """espota2 is blocking; run it in the executor and return its exit code.""" + rc, _ = await asyncio.get_running_loop().run_in_executor( + None, + functools.partial( + espota2.run_ota, + LOCALHOST, + ota_port, + password, + binary_path, + noise_psk=noise_psk, + plaintext_fallback=plaintext_fallback, + ), + ) + return rc + + +@dataclass +class _Device: + """A running host binary and the checks every successful OTA repeats: + a safe reboot, the api port back up, and the pid preserved by execv.""" + + api_port: int + ota_port: int + binary_path: Path + proc: asyncio.subprocess.Process | None = None + reboots: int = 0 + + def __post_init__(self) -> None: + self._rebooted = asyncio.Event() + + def on_log(self, line: str) -> None: + if "Rebooting safely" in line: + self.reboots += 1 + self._rebooted.set() + + async def wait_reboot(self, count: int, timeout: float = 10.0) -> None: + async with asyncio.timeout(timeout): + while self.reboots < count: + self._rebooted.clear() + await self._rebooted.wait() + + async def ota( + self, + password: str | None, + noise_psk: str | None, + msg: str, + plaintext_fallback: bool = False, + ) -> None: + """Upload, then expect the re-exec with the pid preserved.""" + pid_before = self.proc.pid + expected_reboots = self.reboots + 1 + rc = await _run_ota( + self.ota_port, password, self.binary_path, noise_psk, plaintext_fallback + ) + assert rc == 0, msg + await self.wait_reboot(expected_reboots) + await _wait_for_port(LOCALHOST, self.api_port, PORT_WAIT_TIMEOUT) + assert self.proc.returncode is None, "process exited instead of execing" + assert self.proc.pid == pid_before + + async def refused_ota( + self, password: str | None, noise_psk: str | None, msg: str + ) -> None: + """Upload must fail and the device must keep running.""" + rc = await _run_ota(self.ota_port, password, self.binary_path, noise_psk) + assert rc == 1, msg + await asyncio.sleep(0.5) + assert self.proc.returncode is None, "process died on rejected OTA" + + @pytest.mark.asyncio async def test_host_ota_self_update( yaml_config: str, @@ -59,57 +174,34 @@ async def test_host_ota_self_update( reserved_tcp_port: tuple[int, socket.socket], ) -> None: """Self-OTA: upload the running binary back to itself, expect re-exec.""" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - api_socket.close() - ota_socket.close() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + staged = asyncio.Event() - loop = asyncio.get_running_loop() - ota_staged = loop.create_future() - rebooted = loop.create_future() + def on_log(line: str) -> None: + if "OTA staged at" in line: + staged.set() + dev.on_log(line) - def on_log(line: str) -> None: - if not ota_staged.done() and "OTA staged at" in line: - ota_staged.set_result(True) - if not rebooted.done() and "Rebooting safely" in line: - rebooted.set_result(True) + async with run_binary(dev.binary_path, line_callback=on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + async with wait_and_connect_api_client(port=dev.api_port) as client: + info_before = await client.device_info() + assert info_before.name == DEVICE_NAME - async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid - async with wait_and_connect_api_client(port=api_port) as client: - info_before = await client.device_info() - assert info_before.name == DEVICE_NAME + await dev.ota(None, None, "espota2 reported failure") + assert staged.is_set() - # espota2 is blocking; run in executor. - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path - ) - assert rc == 0, "espota2 reported failure" + async with wait_and_connect_api_client(port=dev.api_port) as client: + info_after = await client.device_info() + assert info_after.name == info_before.name - await asyncio.wait_for(ota_staged, timeout=10.0) - await asyncio.wait_for(rebooted, timeout=10.0) - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - - # execv preserves pid; mismatch means external respawn. - assert proc.returncode is None, "process exited instead of execing" - assert proc.pid == pid_before - - async with wait_and_connect_api_client(port=api_port) as client: - info_after = await client.device_info() - assert info_after.name == DEVICE_NAME - assert info_after.name == info_before.name - - # Second OTA: catches FD_CLOEXEC regressions (EADDRINUSE on rebind). - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path - ) - assert rc == 0, "second OTA failed -- listener leaked across execv" - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - assert proc.pid == pid_before + # Second OTA: catches FD_CLOEXEC regressions (EADDRINUSE on rebind). + await dev.ota(None, None, "second OTA failed -- listener leaked across execv") @pytest.mark.asyncio @@ -121,51 +213,110 @@ async def test_host_ota_encrypted( ) -> None: """Encrypted self-OTA succeeds; a plaintext upload to the same device fails.""" pytest.importorskip("aioesphomeapi.noise") - noise_psk = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - api_socket.close() - ota_socket.close() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await dev.refused_ota( + None, None, "plaintext upload to an encrypted device must fail" + ) + await dev.ota(None, API_KEY, "encrypted OTA reported failure") - loop = asyncio.get_running_loop() - rebooted = loop.create_future() - def on_log(line: str) -> None: - if not rebooted.done() and "Rebooting safely" in line: - rebooted.set_result(True) +@pytest.mark.asyncio +async def test_host_ota_api_key_offer_with_password( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + caplog: pytest.LogCaptureFixture, +) -> None: + """With only an api key the device offers encryption without requiring + it: the password still guards plaintext uploads, the key alone + authenticates an encrypted one, and until 2027.3.0 a failed encrypted + attempt falls back to plaintext.""" + pytest.importorskip("aioesphomeapi.noise") + wrong_key = base64.b64encode(b"w" * 32).decode() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _wait_for_line(lines, "Encryption: offered") - async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid + await dev.refused_ota( + None, None, "plaintext upload without the password must fail" + ) + await dev.ota( + "hunter2", None, "plaintext upload with the password must succeed" + ) + await dev.ota(None, API_KEY, "encrypted upload with the api key must succeed") - # A plaintext upload must be refused with the device unharmed - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path + # Remove before 2027.3.0: a wrong key falls back to plaintext, which + # the password still guards + with caplog.at_level("WARNING", logger="esphome.espota2"): + await dev.ota( + "hunter2", + wrong_key, + "the plaintext retry with the password must succeed", + plaintext_fallback=True, ) - assert rc == 1, "plaintext upload to an encrypted device must fail" - await asyncio.sleep(0.5) - assert proc.returncode is None, "process died on rejected plaintext OTA" + assert any("Retrying in plaintext" in r.message for r in caplog.records) + await dev.ota( + None, + API_KEY, + "the right api key encrypts without touching the fallback", + plaintext_fallback=True, + ) - # The encrypted upload goes through and the device re-execs - rc, _ = await loop.run_in_executor( - None, - functools.partial( - espota2.run_ota, - LOCALHOST, - ota_port, - None, - binary_path, - noise_psk=noise_psk, - ), - ) - assert rc == 0, "encrypted OTA reported failure" - await asyncio.wait_for(rebooted, timeout=10.0) - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - assert proc.returncode is None, "process exited instead of execing" - assert proc.pid == pid_before + +@pytest.mark.asyncio +@pytest.mark.usefixtures("isolated_preferences") +async def test_host_ota_provisioned_api_key( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + api_client_connected: APIClientConnectedFactory, +) -> None: + """A key provisioned over the api feeds the OTA offer: plaintext works + while unprovisioned, the provisioned key encrypts, the key loaded from + preferences on the next boot keeps encrypting, and plaintext stays + accepted because only the ota block requires encryption.""" + pytest.importorskip("aioesphomeapi.noise") + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _wait_for_line(lines, "once the api key is provisioned") + + await dev.ota( + None, None, "plaintext upload to an unprovisioned device must succeed" + ) + + async with api_client_connected( + port=dev.api_port, noise_psk=ZERO_PSK + ) as client: + assert await client.noise_encryption_set_key(PROVISIONING_PSK) is True + await asyncio.sleep(KEY_ACTIVATION_DELAY) + + key = PROVISIONING_PSK.decode() + await dev.ota( + None, key, "encrypted upload with the provisioned key must succeed" + ) + await dev.ota(None, key, "the key loaded at boot must feed the OTA offer") + await dev.ota(None, None, "plaintext must stay accepted on an offering device") @pytest.mark.asyncio @@ -177,33 +328,25 @@ async def test_host_ota_rejects_garbage( integration_test_dir, ) -> None: """Bogus payload is rejected and the device keeps running.""" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + # 192 bytes that are neither ELF nor Mach-O. + bogus_path = integration_test_dir / "bogus.bin" + bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8) - # 192 bytes that are neither ELF nor Mach-O. - bogus_path = integration_test_dir / "bogus.bin" - bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8) + async with run_binary(dev.binary_path) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + pid_before = proc.pid + rc = await _run_ota(dev.ota_port, None, bogus_path, None) + assert rc == 1 + await asyncio.sleep(0.5) + assert proc.returncode is None, "process died on rejected OTA" + assert proc.pid == pid_before - api_socket.close() - ota_socket.close() - - async with run_binary(binary_path) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid - - loop = asyncio.get_running_loop() - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, bogus_path - ) - assert rc == 1 - - await asyncio.sleep(0.5) - assert proc.returncode is None, "process died on rejected OTA" - assert proc.pid == pid_before - - async with wait_and_connect_api_client(port=api_port) as client: - info = await client.device_info() - assert info.name == DEVICE_NAME + async with wait_and_connect_api_client(port=dev.api_port) as client: + info = await client.device_info() + assert info.name == DEVICE_NAME diff --git a/tests/unit_tests/test_espota2_noise.py b/tests/unit_tests/test_espota2_noise.py index 5b43d05530..439220f09c 100644 --- a/tests/unit_tests/test_espota2_noise.py +++ b/tests/unit_tests/test_espota2_noise.py @@ -10,12 +10,15 @@ when the installed aioesphomeapi predates the noise module. from __future__ import annotations import base64 +from collections.abc import Callable import hashlib import io +import logging from pathlib import Path import socket import sys import threading +from typing import Any from unittest.mock import Mock, patch import pytest @@ -65,8 +68,12 @@ class FakeEncryptedDevice(threading.Thread): offer_noise: bool = True, require_noise: bool = True, prologue_features_override: int | None = None, + connections: int = 1, + drop_handshakes: int = 0, ) -> None: super().__init__(daemon=True) + self.connections = connections + self.drop_handshakes = drop_handshakes # hang up mid-handshake this many times self.psk = psk self.version = version self.offer_noise = offer_noise @@ -81,10 +88,11 @@ class FakeEncryptedDevice(threading.Thread): def run(self) -> None: try: - sock, _ = self.listener.accept() - sock.settimeout(10) - with sock: - self._serve(sock) + for _ in range(self.connections): + sock, _ = self.listener.accept() + sock.settimeout(10) + with sock: + self._serve(sock) except Exception as err: # noqa: BLE001 - surfaced via join_and_check self.error = err finally: @@ -109,8 +117,23 @@ class FakeEncryptedDevice(threading.Thread): return server_flags = espota2.SERVER_FEATURE_SUPPORTS_NOISE if self.offer_noise else 0 sock.sendall(bytes([espota2.RESPONSE_FEATURE_FLAGS, server_flags])) - if not (self.offer_noise and noise_negotiated): - return # the client fails closed; nothing further arrives + if not (noise_negotiated and self.offer_noise): + # A device that does not require encryption continues in + # plaintext whatever the client asked for, like older firmware + try: + self._transfer( + lambda byte: sock.sendall(bytes([byte])), + lambda length: _recv_exact(sock, length), + lambda remaining: _recv_exact( + sock, min(remaining, espota2.UPLOAD_BLOCK_SIZE) + ), + ) + except ConnectionError: + # A keyed client without fallback fails closed and hangs up + if noise_negotiated and not self.offer_noise: + return + raise + return from cryptography.exceptions import InvalidTag from noise.connection import NoiseConnection @@ -134,6 +157,9 @@ class FakeEncryptedDevice(threading.Thread): msg1 = _recv_frame(sock) assert msg1[0] == 0x00 + if self.drop_handshakes > 0: + self.drop_handshakes -= 1 + return # a transport fault: the socket closes with no reply try: proto.read_message(msg1[1:]) except InvalidTag: @@ -149,6 +175,20 @@ class FakeEncryptedDevice(threading.Thread): assert len(plaintext) == length, "control units must be one per frame" return plaintext + def recv_data(_remaining: int) -> bytes: + plaintext = proto.decrypt(_recv_frame(sock)) + assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT + return plaintext + + self._transfer(send_byte, recv_unit, recv_data) + + def _transfer( + self, + send_byte: Callable[[int], None], + recv_unit: Callable[[int], bytes], + recv_data: Callable[[int], bytes], + ) -> None: + """The post-handshake exchange, identical over both transports.""" send_byte(espota2.RESPONSE_AUTH_OK) recv_unit(1) # ota type size = int.from_bytes(recv_unit(4), "big") @@ -159,9 +199,7 @@ class FakeEncryptedDevice(threading.Thread): received = b"" acked = 0 while len(received) < size: - plaintext = proto.decrypt(_recv_frame(sock)) - assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT - received += plaintext + received += recv_data(size - len(received)) if self.version >= espota2.OTA_VERSION_2_0: while acked + espota2.UPLOAD_BLOCK_SIZE <= len(received) or ( len(received) == size and acked < size @@ -176,7 +214,10 @@ class FakeEncryptedDevice(threading.Thread): def _upload( - device: FakeEncryptedDevice, firmware: bytes, noise_psk: str | None + device: FakeEncryptedDevice, + firmware: bytes, + noise_psk: str | None, + plaintext_fallback: bool = False, ) -> None: device.start() sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -184,12 +225,35 @@ def _upload( sock.connect(("127.0.0.1", device.port)) try: espota2.perform_ota( - sock, None, io.BytesIO(firmware), Path("firmware.bin"), noise_psk=noise_psk + sock, + None, + io.BytesIO(firmware), + Path("firmware.bin"), + noise_psk=noise_psk, + plaintext_fallback=plaintext_fallback, ) finally: sock.close() +def _run_ota( + device: FakeEncryptedDevice, firmware: bytes, tmp_path: Path, noise_psk: str +) -> int: + """Drive the retry loop, which is where the plaintext fallback reconnects.""" + path = tmp_path / "firmware.bin" + path.write_bytes(firmware) + device.start() + rc, _ = espota2.run_ota( + "127.0.0.1", + device.port, + None, + path, + noise_psk=noise_psk, + plaintext_fallback=True, + ) + return rc + + def test_encrypted_upload_success() -> None: """A full encrypted v2 upload spanning several 8192-byte blocks.""" pytest.importorskip("aioesphomeapi.noise") @@ -240,6 +304,56 @@ def test_client_fails_closed_when_device_lacks_encryption() -> None: device.join_and_check() +# Remove before 2027.3.0 +def test_fallback_when_device_does_not_offer(caplog: pytest.LogCaptureFixture) -> None: + """The api key is tried opportunistically; an older device that cannot + encrypt still gets its update, with a warning.""" + firmware = b"firmware" + device = FakeEncryptedDevice(offer_noise=False, require_noise=False) + with patch("time.sleep"), caplog.at_level(logging.WARNING): + _upload(device, firmware, PSK, plaintext_fallback=True) + device.join_and_check() + assert device.received == firmware + assert any("fallback is removed in 2027.3.0" in r.message for r in caplog.records) + + +# Remove before 2027.3.0 +@pytest.mark.parametrize( + ("device_kwargs", "expected_rc", "fell_back"), + [ + # A wrong key against an offering device reconnects in plaintext + ({"psk": OTHER_PSK, "require_noise": False, "connections": 2}, 0, True), + # The plaintext retry is refused by a device that requires encryption + ({"psk": OTHER_PSK, "require_noise": True, "connections": 2}, 1, True), + # A dropped connection inside the handshake is retried encrypted + ({"require_noise": False, "connections": 2, "drop_handshakes": 1}, 0, False), + # A second transport fault inside the handshake falls back + ({"require_noise": False, "connections": 3, "drop_handshakes": 2}, 0, True), + ], + ids=["wrong_key", "wrong_key_required", "one_fault", "two_faults"], +) +def test_fallback_through_the_retry_loop( + caplog: pytest.LogCaptureFixture, + tmp_path: Path, + device_kwargs: dict[str, Any], + expected_rc: int, + fell_back: bool, +) -> None: + pytest.importorskip("aioesphomeapi.noise") + firmware = b"firmware" + device = FakeEncryptedDevice(**device_kwargs) + with patch("time.sleep"), caplog.at_level(logging.WARNING): + rc = _run_ota(device, firmware, tmp_path, PSK) + device.join_and_check() + assert rc == expected_rc + assert (device.received == firmware) is (expected_rc == 0) + assert ( + any("Retrying in plaintext" in r.message for r in caplog.records) is fell_back + ) + if expected_rc == 1: + assert any("requires an encrypted OTA" in r.message for r in caplog.records) + + def test_plaintext_client_gets_encryption_required_error() -> None: """A client without a key gets the device's 0x94 error message.""" device = FakeEncryptedDevice() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 5372a7203d..8fb9b7376e 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -2108,7 +2108,13 @@ def test_upload_program_ota_success( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, "secret", expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + "secret", + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -2140,10 +2146,77 @@ def test_upload_program_ota_encryption_key( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, key + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + key, + plaintext_fallback=False, ) +def test_upload_program_ota_api_key_opportunistic( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """Without an ota encryption block the api key is tried with a plaintext + fallback (removed in 2027.3.0).""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + key = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + config = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: key}}, + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}], + } + exit_code, _ = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + expected_firmware = ( + tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" + ) + mock_run_ota.assert_called_once_with( + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + key, + plaintext_fallback=True, + ) + + +@pytest.mark.parametrize( + "api_conf", + [{}, {CONF_ENCRYPTION: {}}], + ids=["no_encryption", "runtime_key"], +) +def test_upload_program_ota_no_usable_api_key_stays_plaintext( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, + api_conf: dict[str, Any], +) -> None: + """A missing or runtime provisioned api key gives the uploader nothing + to try.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_API: api_conf, + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}], + } + exit_code, _ = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + assert mock_run_ota.call_args.args[5] is None + assert mock_run_ota.call_args.kwargs == {"plaintext_fallback": False} + + def test_upload_program_ota_encryption_without_key_fails_closed( mock_run_ota: Mock, mock_get_port_type: Mock, @@ -2194,7 +2267,13 @@ def test_upload_program_ota_with_file_arg( assert exit_code == 0 assert host == "192.168.1.100" mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, Path("custom.bin"), OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + Path("custom.bin"), + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -2250,6 +2329,7 @@ def test_upload_program_ota_partition_table_with_file_arg( partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, None, + plaintext_fallback=False, ) @@ -2312,6 +2392,7 @@ def test_upload_program_ota_partition_table_mqttip( partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, None, + plaintext_fallback=False, ) @@ -2500,6 +2581,7 @@ def test_upload_program_ota_bootloader_with_file_arg( bootloader_file, OTA_TYPE_UPDATE_BOOTLOADER, None, + plaintext_fallback=False, ) @@ -2988,7 +3070,13 @@ def test_upload_program_ota_with_mqtt_resolution( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -3038,7 +3126,13 @@ def test_upload_program_ota_with_mqtt_empty_broker( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.50"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.50"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) # Verify warning was logged assert "MQTT IP discovery failed" in caplog.text @@ -5211,6 +5305,7 @@ def test_upload_program_ota_static_ip_with_mqttip( expected_firmware, OTA_TYPE_UPDATE_APP, None, + plaintext_fallback=False, ) @@ -5261,6 +5356,7 @@ def test_upload_program_ota_multiple_mqttip_resolves_once( expected_firmware, OTA_TYPE_UPDATE_APP, None, + plaintext_fallback=False, ) @@ -5438,7 +5534,13 @@ def test_upload_program_ota_mqtt_timeout_fallback( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) diff --git a/tests/unit_tests/test_wizard.py b/tests/unit_tests/test_wizard.py index 244e4eb5a1..f57ae71ae6 100644 --- a/tests/unit_tests/test_wizard.py +++ b/tests/unit_tests/test_wizard.py @@ -37,7 +37,6 @@ def wizard_answers() -> list[str]: "nodemcuv2", # board "SSID", # ssid "psk", # wifi password - "", # ota password (empty for no password) ] @@ -101,6 +100,25 @@ def test_config_file_should_include_ota(default_config: dict[str, Any]): assert "ota:" in config +def test_config_file_should_use_encryption_when_api_key_set( + default_config: dict[str, Any], +): + """ + With an API encryption key and no OTA password the OTA block reuses the key + """ + # Given + default_config["api_encryption_key"] = ( + "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + ) + + # When + config = wz.wizard_file(**default_config) + + # Then + assert "ota:\n - platform: esphome\n encryption:" in config + assert "password" not in config.split("ota:")[1].split("wifi:")[0] + + def test_config_file_should_include_ota_when_password_set( default_config: dict[str, Any], ): @@ -630,15 +648,15 @@ def test_wizard_write_protects_existing_config( assert config_file.read_text() == original_content -def test_wizard_accepts_ota_password( +def test_wizard_uses_the_api_key_for_ota( tmp_path: Path, monkeypatch: MonkeyPatch, wizard_answers: list[str] ): """ - The wizard should pass ota_password to wizard_write when the user provides one + The wizard generates an api key and does not ask for an OTA password; + the key secures OTA updates """ # Given - wizard_answers[5] = "my_ota_password" # Set OTA password config_file = tmp_path / "test.yaml" input_mock = MagicMock(side_effect=wizard_answers) monkeypatch.setattr("builtins.input", input_mock) @@ -653,8 +671,9 @@ def test_wizard_accepts_ota_password( # Then assert retval == 0 call_kwargs = wizard_write_mock.call_args.kwargs - assert "ota_password" in call_kwargs - assert call_kwargs["ota_password"] == "my_ota_password" + assert "api_encryption_key" in call_kwargs + assert "ota_password" not in call_kwargs + assert input_mock.call_count == len(wizard_answers) def test_wizard_accepts_rpipico_board(tmp_path: Path, monkeypatch: MonkeyPatch): From 9c00f13606886643a5e9ff8195a08621dd10e9af Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:05:16 +0000 Subject: [PATCH 106/147] Bump bundled esphome-device-builder to 1.14.4 (#19006) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e875851bfb..da76ab7b6a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.3 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.4 RUN \ platformio settings set enable_telemetry No \ From 688af60cbfa4289fb3e13b0284622d34dbf77366 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:12:52 +0200 Subject: [PATCH 107/147] [noise] Bump noise-c to 0.1.24 and libsodium to 1.10021.6 (#18989) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index a1d9444fc0..4de706120e 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.21") + cg.add_library("esphome/noise-c", "0.1.24") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.4") + cg.add_library("esphome/libsodium", "1.10021.6") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index fcf7caa7c7..779a05e7de 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.21 ; noise (api, ota) + esphome/noise-c@0.1.24 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.21 ; noise (api, ota) + esphome/noise-c@0.1.24 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.21 ; used by noise (api, ota) + esphome/noise-c@0.1.24 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index a263d7937f..4f7f5a4a4c 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.21") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.21") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 0.1.24") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.24") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.21\n" + " esphome/noise-c @ 0.1.24\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.21\n" + " esphome/noise-c @ 0.1.24\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.21"] + assert libs == ["esphome/noise-c @ 0.1.24"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.24", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.21", - "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.24", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.21"] + assert cls.calls == ["esphome/noise-c @ 0.1.24"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.21"] is None + assert compats["esphome/noise-c @ 0.1.24"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.21"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.24"}) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 379ef52ebd..fb79885736 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1576,7 +1576,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1596,7 +1596,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 20c7dcb1ddf6a70aaf75ff418499835c3e99228e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:26:44 +0200 Subject: [PATCH 108/147] [mdns] Guard LEAmDNS main loop calls against lwIP re-entrancy on ESP8266 (#18990) --- esphome/components/mdns/__init__.py | 2 + esphome/components/mdns/mdns_esp8266.cpp | 51 +++++++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index f039bb69f0..c8020104b3 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -192,6 +192,8 @@ async def to_code(config: ConfigType) -> None: if CORE.using_arduino: if CORE.is_esp8266: cg.add_library("ESP8266mDNS", None) + # No MDNS global in the build; mdns_esp8266.cpp owns a guarded MDNSResponder + cg.add_build_flag("-DNO_GLOBAL_MDNS") elif CORE.is_rp2: cg.add_library("LEAmDNS", None) diff --git a/esphome/components/mdns/mdns_esp8266.cpp b/esphome/components/mdns/mdns_esp8266.cpp index 1f0b3c9519..0e600d3bac 100644 --- a/esphome/components/mdns/mdns_esp8266.cpp +++ b/esphome/components/mdns/mdns_esp8266.cpp @@ -13,8 +13,47 @@ namespace esphome::mdns { +// Main-loop calls into LEAmDNS that send (update() and close(); begin(), addService() and +// the scheduled restart never reach a send) can yield inside UdpContext::sendTimeout(); a +// packet arriving then re-enters LEAmDNS from lwIP on the same UdpContext and both sides +// free the same tx pbufs (#18760). Received packets stay queued during such a call and are +// processed from the main loop afterwards. +class GuardedMDNSResponder : public ::esp8266::MDNSImplementation::MDNSResponder { + public: + void update_guarded() { this->run_guarded_(&GuardedMDNSResponder::update); } + void close_guarded() { this->run_guarded_(&GuardedMDNSResponder::close); } + + private: + void run_guarded_(bool (GuardedMDNSResponder::*fn)()) { + UdpContext *ctx = this->m_pUDPContext; + if (ctx == nullptr) { + (this->*fn)(); + return; + } + // Set every time: a restart replaces the context together with its stock handler. Only + // begin() and the scheduled netif callback restart, never update() or close(), so the + // context cannot change underneath this call. + ctx->onRx([this]() { + if (!this->in_loop_call_) { + this->_callProcess(); + } + }); + this->in_loop_call_ = true; + (this->*fn)(); + // close() releases the context; a yield in here queues further packets for this loop too + while (this->m_pUDPContext != nullptr && this->m_pUDPContext->next()) { + this->_parseMessage(); + } + this->in_loop_call_ = false; + } + + volatile bool in_loop_call_{false}; +}; + +static GuardedMDNSResponder mdns_responder; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + static void register_esp8266(MDNSComponent *, StaticVector &services) { - MDNS.begin(App.get_name().c_str()); + mdns_responder.begin(App.get_name().c_str()); for (const auto &service : services) { // Strip the leading underscore from the proto and service_type. While it is @@ -30,10 +69,10 @@ static void register_esp8266(MDNSComponent *, StaticVectoris_roaming() || (!wifi->is_connected() && !wifi->is_ap_active())) return; #endif - MDNS.update(); + mdns_responder.update_guarded(); }); this->set_timeout(MDNS_POLL_STOP_ID, MDNS_POLL_WINDOW_MS, [this]() { this->cancel_interval(MDNS_POLL_ID); }); } @@ -81,7 +120,7 @@ void MDNSComponent::on_ip_state(const network::IPAddresses &ips, const network:: #endif void MDNSComponent::on_shutdown() { - MDNS.close(); + mdns_responder.close_guarded(); delay(10); } From 8966567be072926e211b1ca717b7b1d628be7b15 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:28:31 +0200 Subject: [PATCH 109/147] [core] Show the other downloader's progress while a prefetch job waits on its lock (#18983) --- esphome/framework_helpers.py | 61 +++++++++++++- esphome/platformio/prefetch.py | 87 ++++++++++---------- esphome/platformio/registry.py | 67 ++++++++++----- tests/unit_tests/conftest.py | 39 ++++++++- tests/unit_tests/test_framework_helpers.py | 17 ++++ tests/unit_tests/test_platformio_prefetch.py | 87 ++++++++++++++++++-- tests/unit_tests/test_platformio_registry.py | 73 ++++++++++++++-- 7 files changed, 348 insertions(+), 83 deletions(-) diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 82bc0d3727..fc2a18a6ec 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -23,6 +23,7 @@ from esphome.net_retry import ( ) if TYPE_CHECKING: + from filelock import FileLock import requests PathType = str | os.PathLike @@ -909,6 +910,61 @@ def _part_path(dest: Path) -> Path: return dest.with_name(dest.name + ".part") +def downloaded_bytes(dest: Path, size: int | None = None) -> int: + """Bytes of ``dest`` on disk (its ``.part`` while streaming), capped at ``size``.""" + done = 0 + for candidate in (_part_path(dest), dest): + try: + done = candidate.stat().st_size + break + except FileNotFoundError: + continue + return done if size is None else min(done, size) + + +# Short lock-acquire slices so a waiting worker still observes Ctrl-C +_DOWNLOAD_LOCK_POLL = 1 + +# Waiting on another process's download; past this the caller leaves the +# file to its holder (the later sequential install waits on the same lock) +DOWNLOAD_LOCK_TIMEOUT = 60 + + +class DownloadLockUnavailable(OSError): + """The lock file cannot be used at all (a lock-less filesystem).""" + + +def wait_for_download_lock( + lock: "FileLock", + tracker: Callable[[int], None], + on_disk: Callable[[], int], + name: str, +) -> None: + """Acquire ``lock``, reporting ``on_disk()`` to ``tracker`` each poll so the + bar follows the holder's download. Raises filelock's ``Timeout`` once + ``DOWNLOAD_LOCK_TIMEOUT`` seconds pass.""" + from filelock import Timeout + + deadline = time.monotonic() + DOWNLOAD_LOCK_TIMEOUT + waiting = False + while True: + try: + lock.acquire(timeout=_DOWNLOAD_LOCK_POLL) + return + except Timeout: + pass + except OSError as err: + # Distinct from an OSError out of on_disk(), which must not + # read as "locks unsupported" + raise DownloadLockUnavailable(*err.args) from err + if not waiting: + waiting = True + _LOGGER.info("Waiting for another process downloading %s", name) + tracker(on_disk()) # raises when the batch is cancelled + if time.monotonic() >= deadline: + raise Timeout(lock.lock_file) + + def discard_partial_download(dest: Path) -> None: """Remove ``dest`` and the resume sidecars of an abandoned download.""" part = _part_path(dest) @@ -1319,10 +1375,7 @@ def download_from_mirrors( ) # Tick with the bytes already on disk so a combined bar holds # steady during the backoff instead of rewinding to zero - done = 0 - if progress is not None: - part = _part_path(path_target) - done = part.stat().st_size if part.is_file() else 0 + done = downloaded_bytes(path_target) if progress is not None else 0 _cancellable_sleep(delay, progress, done) # 3. Report every attempted URL if all mirrors failed. failures spans diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 5097239065..17a06cb9c1 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -33,11 +33,14 @@ import time from typing import Any, NamedTuple from esphome.framework_helpers import ( + DownloadLockUnavailable, content_length, discard_partial_download, + downloaded_bytes, failure_reason, resume_fetch_job, run_batch_downloads, + wait_for_download_lock, warn_prefetch_failures, ) from esphome.helpers import get_bool_env, get_usable_cpu_count, rmtree @@ -61,16 +64,10 @@ _RESOLVE_WORKERS = 8 # A hung child must not block the build; downloads resume on the next run _PREFETCH_TIMEOUT = 20 * 60 -# Waiting on another process's URL download; past this, leave it to pio -_DOWNLOAD_LOCK_TIMEOUT = 60 - # Child exit for a handled, already-warned failure; 1 would collide with # the interpreter's own import-failure exit _EXIT_HANDLED = 3 -# Short lock-acquire slices so a waiting worker still observes Ctrl-C -_URI_LOCK_POLL = 1 - # Resolution errored (vs a clean skip); suppresses the warm sentinel _RESOLVE_FAILED = object() @@ -462,51 +459,54 @@ def _uri_jobs( def _serialized_fetch_job( - dl_path: Path, lock_path: str, body: Any, unlocked_ok: bool = True + dl_path: Path, + lock_path: str, + body: Any, + size: int, + stream_dest: Path | None = None, + unlocked_ok: bool = True, ) -> Any: - """Wrap ``body`` so the shared destination is single-writer. - - Interleaved writers truncate each other's ``.part`` bytes (see - registry.py). The bounded poll observes Ctrl-C via the tracker; a - blown deadline is a clean skip (the holder's copy is what the build - needs). On a lock-less filesystem a sha256-verified body runs - unlocked with one warning; a checksum-less one - (``unlocked_ok=False``) is a counted failure instead. + """Wrap ``body`` so the shared destination is single-writer (interleaved + writers truncate each other's ``.part``, see registry.py). A blown deadline + is a clean skip. On a lock-less filesystem a sha256-verified body runs + unlocked with one warning; a checksum-less one (``unlocked_ok=False``) fails. """ + def on_disk() -> int: + # A URL job's holder streams beside the staging path until it + # promotes; after that only dl_path is left + done = downloaded_bytes(dl_path, size) + if not done and stream_dest is not None: + done = downloaded_bytes(stream_dest, size) + return done + def run(tracker: Any) -> None: from filelock import FileLock, Timeout # fallback_to_soft would leave a stale marker on lock-less # filesystems that blocks every later build (see git.py) lock = FileLock(lock_path, fallback_to_soft=False) - deadline = time.monotonic() + _DOWNLOAD_LOCK_TIMEOUT - while True: - try: - lock.acquire(timeout=_URI_LOCK_POLL) - break - except Timeout: - tracker(0) # raises when the batch is cancelled - if time.monotonic() >= deadline: - # Another process is fetching this same file; its copy - # is what the build needs (a large framework archive - # can hold the lock far longer than this deadline) - _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) - return - except OSError as err: - if not unlocked_ok: - # A body with no checksum to catch interleaved corruption - raise - lock = None - _LOGGER.warning( - "Could not lock %s (%s); downloading unlocked", - dl_path.name, - err, - ) - break + try: + wait_for_download_lock(lock, tracker, on_disk, dl_path.name) + except Timeout: + # The holder's copy is what the build needs (a large + # framework archive can outlast this deadline) + _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) + return + except DownloadLockUnavailable as err: + if not unlocked_ok: + # A body with no checksum to catch interleaved corruption + raise + lock = None + _LOGGER.warning( + "Could not lock %s (%s); downloading unlocked", + dl_path.name, + err, + ) try: if dl_path.is_file(): - return # another process finished it while we waited + tracker(size) # another process finished it while we waited + return body(tracker) finally: if lock is not None: @@ -540,6 +540,7 @@ def _registry_fetch_job( dl_path, f"{dl_path}.esphome.lock", resume_fetch_job(url, dl_path, sha256=checksum, size=size), + size, ) def run(tracker: Any) -> None: @@ -571,9 +572,9 @@ def _uri_fetch_job(manager: Any, url: str, dl_path: Path, size: int) -> Any: tmp.replace(dl_path) def run(tracker: Any) -> None: - _serialized_fetch_job(dl_path, f"{tmp}.lock", promote, unlocked_ok=False)( - tracker - ) + _serialized_fetch_job( + dl_path, f"{tmp}.lock", promote, size, tmp, unlocked_ok=False + )(tracker) if dl_path.is_file(): # Won or lost, the race is over; staging files left behind # are dead weight PlatformIO's cache never prunes diff --git a/esphome/platformio/registry.py b/esphome/platformio/registry.py index 9538a28ff4..75df82da0e 100644 --- a/esphome/platformio/registry.py +++ b/esphome/platformio/registry.py @@ -17,8 +17,10 @@ from esphome.framework_helpers import ( archive_extract_all, download_from_mirrors, download_with_resume, + downloaded_bytes, rmdir, run_batch_downloads, + wait_for_download_lock, ) from esphome.net_retry import fetch_with_retry, http_request @@ -164,11 +166,17 @@ class _PendingArchive(NamedTuple): name: str version: str dest: Path + archive: Path url: str sha256: str size: int +def _archive_path(downloads_dir: Path, name: str, version: str) -> Path: + """The one archive path the prefetch and the sequential install share.""" + return downloads_dir / f"{name}-{version}" + + def _already_installed(dest: Path) -> bool: """Whether ``dest`` holds a completed install (extraction marker).""" return (dest / ".esphome_extracted").is_file() @@ -187,18 +195,18 @@ def prefetch_packages( lock as ``install_package``: the archive's ``.part`` file is shared, and two concurrent writers would truncate each other's bytes. """ - from filelock import FileLock + from filelock import FileLock, Timeout pending: list[_PendingArchive] = [] - seen: set[str] = set() + seen: set[Path] = set() for name, version, dest, mirrors in packages: if mirrors or (dest / ".esphome_extracted").is_file(): continue - archive_name = f"{name}-{version}" - if archive_name in seen: + archive = _archive_path(downloads_dir, name, version) + if archive in seen: # A duplicate entry would race itself between two workers continue - seen.add(archive_name) + seen.add(archive) try: url, sha256, size = registry_download(name, version) except EsphomeError as err: @@ -207,10 +215,9 @@ def prefetch_packages( continue if not size: continue - archive = downloads_dir / archive_name if archive.is_file() and archive.stat().st_size == size: continue - pending.append(_PendingArchive(name, version, dest, url, sha256, size)) + pending.append(_PendingArchive(name, version, dest, archive, url, sha256, size)) if len(pending) < 2: return downloads_dir.mkdir(parents=True, exist_ok=True) @@ -222,20 +229,36 @@ def prefetch_packages( def _fetch(entry: _PendingArchive, tracker: Callable[[int], None]) -> None: entry.dest.parent.mkdir(parents=True, exist_ok=True) - with FileLock(f"{entry.dest}.lock", fallback_to_soft=False): - # Marker re-check: a concurrent build may have installed (and - # deleted the archive of) this package while we waited; - # re-downloading would orphan a fresh copy in downloads_dir - # no branch: the thread tracer misses the skip edge; both - # arms of _already_installed are pinned directly - if not _already_installed(entry.dest): # pragma: no branch - download_with_resume( - entry.url, - downloads_dir / f"{entry.name}-{entry.version}", - sha256=entry.sha256, - size=entry.size, - progress=tracker, - ) + + def on_disk() -> int: + if done := downloaded_bytes(entry.archive, entry.size): + return done + # The holder deletes the archive once it has installed it + return entry.size if _already_installed(entry.dest) else 0 + + lock = FileLock(f"{entry.dest}.lock", fallback_to_soft=False) + try: + wait_for_download_lock(lock, tracker, on_disk, entry.name) + except Timeout: + # install_package waits on this same lock and verifies the + # holder's copy + _LOGGER.debug("Leaving %s to its current downloader", entry.name) + return + try: + if _already_installed(entry.dest): + # A concurrent build installed it while we waited; a + # re-download would orphan a fresh copy in downloads_dir + tracker(entry.size) + return + download_with_resume( + entry.url, + entry.archive, + sha256=entry.sha256, + size=entry.size, + progress=tracker, + ) + finally: + lock.release() failures = run_batch_downloads( "Downloading packages", @@ -288,7 +311,7 @@ def install_package( rmdir(dest, msg=f"Clean up incomplete {name} install") # Persistent location so an interrupted download resumes across runs. downloads_dir.mkdir(parents=True, exist_ok=True) - archive = downloads_dir / f"{name}-{version}" + archive = _archive_path(downloads_dir, name, version) _LOGGER.info("Downloading %s %s ...", name, version) if mirrors: _LOGGER.warning( diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index 9de8f715ef..ad9c0bb11f 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -9,7 +9,7 @@ not be part of a unit test suite. """ -from collections.abc import Generator +from collections.abc import Callable, Generator import os from pathlib import Path import sys @@ -137,3 +137,40 @@ def mock_get_component() -> Generator[Mock, None, None]: """Mock get_component for config module.""" with patch("esphome.config.get_component") as mock: yield mock + + +@pytest.fixture +def held_lock() -> Callable[..., Callable[..., None]]: + """Factory for a ``FileLock.acquire`` fake held by another downloader. + + Each poll writes the next chunk to ``part`` (or runs it, for a callable) + and raises ``Timeout``; when the chunks run out the part is removed, + ``land()`` runs, and the acquire succeeds (also for any later job, so + ``land`` must be idempotent). + """ + from filelock import Timeout + + def make( + part: Path, + chunks: list[bytes | Callable[[], None]], + land: Callable[[], None], + ) -> Callable[..., None]: + polls = iter(chunks) + + def acquire(*args, **kwargs) -> None: + try: + chunk = next(polls) + except StopIteration: + part.unlink(missing_ok=True) + land() + return + if callable(chunk): + chunk() + else: + part.parent.mkdir(parents=True, exist_ok=True) + part.write_bytes(chunk) + raise Timeout("held") + + return acquire + + return make diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index fcc5572f51..22b34c9df5 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -2353,3 +2353,20 @@ def test_discard_partial_download_logs_undeletable( ): framework_helpers.discard_partial_download(dest) assert "Could not remove" in caplog.text + + +def test_downloaded_bytes_reports_what_is_on_disk(tmp_path: Path) -> None: + """Part file first, then the landed file, both capped at size; else 0.""" + dest = tmp_path / "archive" + assert framework_helpers.downloaded_bytes(dest, 4) == 0 + part = tmp_path / "archive.part" + part.write_bytes(b"ab") + assert framework_helpers.downloaded_bytes(dest, 4) == 2 + part.write_bytes(b"abcdef") + assert framework_helpers.downloaded_bytes(dest, 4) == 4 + part.unlink() + dest.write_bytes(b"abc") + assert framework_helpers.downloaded_bytes(dest, 4) == 3 + assert framework_helpers.downloaded_bytes(dest) == 3 + dest.write_bytes(b"abcdef") + assert framework_helpers.downloaded_bytes(dest, 4) == 4 diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index fb79885736..77490fd861 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -454,23 +454,96 @@ def test_uri_fetch_job_waits_out_a_briefly_held_lock(tmp_path: Path) -> None: assert dl_path.read_bytes() == b"data" -def test_lock_deadline_leaves_download_to_the_holder(tmp_path: Path) -> None: - """A lock held past the deadline means another process is fetching the - same file; skipping cleanly beats a misleading failure warning. The - tracker is still polled so a parked worker observes cancellation.""" +@pytest.mark.parametrize("staged", [b"", b"ab"]) +def test_lock_deadline_leaves_download_to_the_holder( + tmp_path: Path, staged: bytes +) -> None: + """A lock held past the deadline is another process's download; skip + cleanly, polling the tracker with what the holder has staged so far.""" dl_path = tmp_path / "archive" + (tmp_path / "archive.prefetch.part").write_bytes(staged) ticks: list[int] = [] with ( patch("esphome.framework_helpers.download_with_resume") as mock_download, patch("filelock.FileLock.acquire", side_effect=Timeout("held")), - patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), ): pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) mock_download.assert_not_called() - assert ticks == [0] + assert ticks == [len(staged)] assert not dl_path.exists() +@pytest.mark.parametrize( + ("job", "part_name", "chunks", "expected"), + [ + ( + lambda dl_path: pf._registry_fetch_job( + MagicMock(), "https://x/a.tar.gz", dl_path, "ab" * 32, 4 + ), + "archive.part", + [b"a", b"abc"], + [1, 3, 4], + ), + ( + lambda dl_path: pf._uri_fetch_job( + MagicMock(), "https://x/a.zip", dl_path, 4 + ), + "archive.prefetch.part", + [b"ab"], + [2, 4], + ), + ], + ids=["registry", "uri"], +) +def test_lock_wait_reports_the_holders_progress( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + held_lock, + job, + part_name: str, + chunks: list[bytes], + expected: list[int], +) -> None: + """A waiting job reports the holder's part file (the staging one for a + URL job), then the full size once the holder lands the archive.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + acquire = held_lock( + tmp_path / part_name, chunks, lambda: dl_path.write_bytes(b"abcd") + ) + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + caplog.at_level(logging.INFO), + ): + job(dl_path)(ticks.append) + mock_download.assert_not_called() + assert ticks == expected + assert caplog.text.count("Waiting for another process downloading archive") == 1 + + +def test_uri_lock_wait_prefers_the_landed_archive(tmp_path: Path, held_lock) -> None: + """Between the holder's promotion rename and its release the staging + part is gone; the landed cache file is credited instead of 0.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + acquire = held_lock( + tmp_path / "archive.prefetch.part", + [b"ab", lambda: dl_path.write_bytes(b"abcd")], + lambda: None, + ) + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + ): + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) + mock_download.assert_not_called() + assert ticks == [2, 4, 4] + + def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: """A registry job that lost the download race to another process must not stamp a nonexistent archive into pio's usage.db.""" @@ -479,7 +552,7 @@ def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: with ( patch("esphome.framework_helpers.download_with_resume") as mock_download, patch("filelock.FileLock.acquire", side_effect=Timeout("held")), - patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), ): pf._registry_fetch_job(manager, "https://x/a.tar.gz", dl_path, "ab" * 32, 4)( lambda done: None diff --git a/tests/unit_tests/test_platformio_registry.py b/tests/unit_tests/test_platformio_registry.py index 6ba8691c4e..9d5f6c4ce5 100644 --- a/tests/unit_tests/test_platformio_registry.py +++ b/tests/unit_tests/test_platformio_registry.py @@ -8,6 +8,7 @@ import os from pathlib import Path from unittest.mock import MagicMock, patch +from filelock import Timeout import pytest from esphome.core import EsphomeError @@ -540,16 +541,13 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: dest = tmp_path / "a" dest.mkdir() - from contextlib import contextmanager - - @contextmanager - def marker_appears_under_lock(path, **kwargs): + def marker_appears_under_lock(*args, **kwargs): # Simulates the concurrent build finishing while we waited (dest / ".esphome_extracted").touch() - yield with ( - patch("filelock.FileLock", side_effect=marker_appears_under_lock), + patch("filelock.FileLock.acquire", side_effect=marker_appears_under_lock), + patch("filelock.FileLock.release"), patch.object(registry, "download_with_resume") as mock_download, patch.object( registry, "registry_download", side_effect=_resolve_for({"a": 10}) @@ -559,6 +557,69 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: mock_download.assert_not_called() +def test_prefetch_packages_waits_with_the_holders_progress( + tmp_path: Path, held_lock +) -> None: + """A worker parked on another build's lock reports that build's part + file, then the full size once the marker appears.""" + dest = tmp_path / "a" + dest.mkdir() + ticks: list[int] = [] + part = tmp_path / "dl" / "a-1.0.part" + + def installed_and_pruned() -> None: + # install_package touches the marker, then unlinks the archive + (dest / ".esphome_extracted").touch() + part.unlink() + + acquire = held_lock( + part, + [lambda: None, b"abc", installed_and_pruned], + (dest / ".esphome_extracted").touch, + ) + + def fake_batch(header, jobs): + for _name, _size, fetch in jobs: + fetch(ticks.append) + return [] + + with ( + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + patch.object(registry, "run_batch_downloads", side_effect=fake_batch), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 5}) + ), + ): + registry.prefetch_packages( + [("a", "1.0", dest, []), ("b", "2.0", tmp_path / "b", [])], + tmp_path / "dl", + ) + assert ticks == [0, 3, 10, 10] + mock_download.assert_called_once() + + +def test_prefetch_packages_leaves_a_long_held_lock_to_its_holder( + tmp_path: Path, +) -> None: + """Past the deadline the worker skips; install_package waits on the same + lock later and verifies whatever the holder produced.""" + with ( + patch("filelock.FileLock.acquire", side_effect=Timeout("held")), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 5}) + ), + ): + registry.prefetch_packages( + [("a", "1.0", tmp_path / "a", []), ("b", "2.0", tmp_path / "b", [])], + tmp_path / "dl", + ) + mock_download.assert_not_called() + + def test_already_installed_probe(tmp_path: Path) -> None: """Both arms of the marker probe the prefetch worker keys on.""" dest = tmp_path / "pkg" From d58b37faa1eff3324dd6c9389c864d5c3576eadf Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:29:20 +1200 Subject: [PATCH 110/147] [esp32_hosted] Add ESP-NOW-over-hosted shim for the ESP32-P4 (#17712) --- esphome/components/esp32_hosted/__init__.py | 36 ++ .../esp32_hosted/esp_now_hosted.cpp | 467 ++++++++++++++++++ .../esp32_hosted/esp_now_hosted_rpc.h | 128 +++++ esphome/components/espnow/__init__.py | 20 + esphome/core/defines.h | 1 + script/ci-custom.py | 17 +- .../test-espnow.esp32-p4-idf.yaml | 5 + tests/unit_tests/components/test_espnow.py | 48 ++ 8 files changed, 721 insertions(+), 1 deletion(-) create mode 100644 esphome/components/esp32_hosted/esp_now_hosted.cpp create mode 100644 esphome/components/esp32_hosted/esp_now_hosted_rpc.h create mode 100644 tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml create mode 100644 tests/unit_tests/components/test_espnow.py diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index ab9455250c..21626e432b 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -37,6 +37,25 @@ CONF_HANDSHAKE_PIN = "handshake_pin" CONF_SDIO_FREQUENCY = "sdio_frequency" CONF_SPI_MODE = "spi_mode" +# ESP-NOW-over-hosted shim (esp_now_hosted.cpp). esp-hosted proxies esp_wifi.h +# but not esp_now.h (espressif/esp-hosted-mcu#19), and esp_wifi_remote injects +# the esp_now.h header on the ESP32-P4 host with no implementation, leaving the +# esp_now_* symbols undefined at link. On a P4 host, esp_now_hosted.cpp DEFINES +# those symbols and forwards each call to the co-processor over esp-hosted's +# CustomRpc "peer data transfer" channel, so ESPHome's `espnow` component links +# and runs unchanged (proven on a Tab5, 2026-07-20). The .cpp is guarded to +# CONFIG_IDF_TARGET_ESP32P4 so it compiles to nothing on hosts with a native +# ESP-NOW stack. CustomRpc needs these two host-side Kconfig options. Host +# registers 3 handlers (RESP, RECV, SEND); the coprocessor registers 1 (REQ); +# we ask for 8 to leave room for other CustomRpc extensions alongside. +# +# The coprocessor must run the matching custom firmware (a parallel effort in +# esphome/esp-hosted-firmware). esp_now_hosted_rpc.h here is the canonical copy +# of the wire contract and MUST stay byte-identical to the copy that coprocessor +# firmware uses — the packed structs are the on-wire layout, so any divergence +# silently corrupts every ESP-NOW frame. +_MAX_CUSTOM_MSG_HANDLERS = 8 + # Shared fields for both transport modes BASE_SCHEMA = cv.Schema( { @@ -262,6 +281,23 @@ async def to_code(config: ConfigType) -> None: else: _configure_spi(config) + # ESP-NOW-over-hosted shim: only the radio-less ESP32-P4 host needs it (see + # the note by _MAX_CUSTOM_MSG_HANDLERS). Enabled for every P4 host, not + # gated on the `espnow` component being present: the shim is tiny and the + # esp_now_* symbols/CustomRpc calls it defines require these Kconfig options + # to link whenever esp_now_hosted.cpp compiles (which is on any P4 host), so + # coupling the two keeps the build consistent. When `espnow` is absent the + # symbols are simply unused and never register a callback at runtime. + if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4: + add_define("USE_ESP_NOW_HOSTED") + # esp-hosted's CustomRpc ("peer data transfer") path — off by default. + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_ENABLE_PEER_DATA_TRANSFER", True + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_MAX_CUSTOM_MSG_HANDLERS", _MAX_CUSTOM_MSG_HANDLERS + ) + # Place the transport mempool in PSRAM. Required on memory-tight host # configurations (e.g. P4 with a large LVGL UI) where the internal-RAM # mempool allocation fails at boot with `sdio_mempool_create` assert. diff --git a/esphome/components/esp32_hosted/esp_now_hosted.cpp b/esphome/components/esp32_hosted/esp_now_hosted.cpp new file mode 100644 index 0000000000..ad29b208fe --- /dev/null +++ b/esphome/components/esp32_hosted/esp_now_hosted.cpp @@ -0,0 +1,467 @@ +/* + * esp_now_hosted — host-side shim implementing over esp-hosted + * CustomRpc, so ESPHome's `espnow` component can run on a radio-less host + * (e.g. the ESP32-P4) whose radio lives on an esp-hosted co-processor. + * + * A radio-less host has no native ESP-NOW. esp_wifi_remote INJECTS the full + * esp_now.h header (types + declarations) but ships NO implementation, so every + * esp_now_* symbol is an undefined reference at link time. This translation + * unit provides those definitions; each forwards to the co-processor over + * CustomRpc (see esphome/esp-hosted-firmware for the matching coprocessor + * handlers). No esp-hosted or esp_wifi_remote source is patched, and there is no + * duplicate-symbol clash because nothing else defines these symbols here. + * + * See esp_now_hosted_rpc.h for the wire protocol. + */ + +#include "sdkconfig.h" + +// Only build the shim on the radio-less host. On chips with a native ESP-NOW +// stack (S3, C6, …) the real symbols exist and this file must stay empty to +// avoid duplicate definitions. +#if defined(CONFIG_IDF_TARGET_ESP32P4) + +#include + +#include "freertos/FreeRTOS.h" +#include "freertos/semphr.h" + +#include "esp_idf_version.h" +#include "esp_log.h" +#include "esp_timer.h" + +#include // injected declarations we are now DEFINING +#include // wifi_pkt_rx_ctrl_t, wifi_tx_info_t + +// esp_hosted_misc.h (host) ships WITHOUT an extern "C" guard, so including it +// from C++ would give its declarations C++ linkage and the real C symbols in +// libesp_hosted would go unresolved at link. Wrap it. (Verified vs +// esp_hosted 2.12.9.) +extern "C" { +#include "esp_hosted_misc.h" // esp_hosted_{send_custom_data,register_custom_callback} +} + +#include "esp_now_hosted_rpc.h" + +namespace { + +const char *const TAG = "esp_now_hosted"; + +// One outstanding request at a time. ESPHome drives esp_now_* from the main +// loop; the matching response and the async RECV/SEND events all arrive on the +// single esp-hosted RPC RX thread. Serializing requests keeps the shared +// response slot race-free; a sequence number stops a late/stale response from +// being mistaken for ours. +SemaphoreHandle_t g_req_mutex = nullptr; +SemaphoreHandle_t g_resp_sem = nullptr; // given when the matching RESP lands +bool g_setup_done = false; // set only after setup fully succeeds +uint8_t g_seq = 0; +volatile uint8_t g_expect_seq = 0; +volatile int32_t g_resp_status = 0; +uint8_t g_resp_ret[16]; +volatile uint16_t g_resp_ret_len = 0; + +// Written from the main loop (register/unregister/deinit), read from the +// esp-hosted RX thread (on_recv/on_send). volatile for the same reason the +// g_resp_* globals are: force the RX thread to observe an updated pointer +// (e.g. a nulling by esp_now_deinit) rather than a cached one. +volatile esp_now_recv_cb_t g_recv_cb = nullptr; +volatile esp_now_send_cb_t g_send_cb = nullptr; + +// Local mirror of the co-processor's peer table. ESPHome's espnow component +// calls esp_now_is_peer_exist() on the main loop for every received frame +// (twice) and every send; forwarding each as a blocking RPC round-trip stalls +// the loop. The shim is the only path that mutates the co-processor peer table +// (add/del/deinit all go through here), so this mirror is authoritative and +// esp_now_is_peer_exist() can answer from it with no round-trip. +// +// esp_now_* are public C symbols: any component or user lambda may call them, +// and although ESPHome's espnow touches peers only from the main loop today +// (its RX/TX callbacks merely enqueue), the shim cannot rely on that. A short +// spinlock keeps the mirror consistent from any task/core, matching native +// esp_now_*'s own internal thread-safety. The critical sections are a bounded +// (<=20-entry) scan, so they stay tiny. ESP_NOW_MAX_TOTAL_PEER_NUM is 20. +constexpr size_t ESP_NOW_HOSTED_MAX_PEERS = 20; +uint8_t g_peer_cache[ESP_NOW_HOSTED_MAX_PEERS][6]; +size_t g_peer_count = 0; +portMUX_TYPE g_peer_lock = portMUX_INITIALIZER_UNLOCKED; + +// Caller must hold g_peer_lock. +int peer_cache_find_locked(const uint8_t *mac) { + for (size_t i = 0; i < g_peer_count; i++) { + if (memcmp(g_peer_cache[i], mac, 6) == 0) + return static_cast(i); + } + return -1; +} + +bool peer_cache_contains(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + const bool found = peer_cache_find_locked(mac) >= 0; + portEXIT_CRITICAL(&g_peer_lock); + return found; +} + +void peer_cache_add(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + if (peer_cache_find_locked(mac) < 0 && g_peer_count < ESP_NOW_HOSTED_MAX_PEERS) + memcpy(g_peer_cache[g_peer_count++], mac, 6); + portEXIT_CRITICAL(&g_peer_lock); +} + +void peer_cache_remove(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + const int idx = peer_cache_find_locked(mac); + if (idx >= 0) { + g_peer_count--; + if (static_cast(idx) != g_peer_count) // move the last entry into the gap + memcpy(g_peer_cache[idx], g_peer_cache[g_peer_count], 6); + } + portEXIT_CRITICAL(&g_peer_lock); +} + +void peer_cache_clear() { + portENTER_CRITICAL(&g_peer_lock); + g_peer_count = 0; + portEXIT_CRITICAL(&g_peer_lock); +} + +// ── CustomRpc event handlers (run on the esp-hosted RPC RX thread) ────────── +// Keep them short and non-blocking. In particular they MUST NOT call back into +// any esp_now_* shim function: that would try to take g_req_mutex / wait on the +// RX thread that delivers the response, and deadlock. + +void on_resp(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + if (len < sizeof(esp_now_hosted_resp_t)) { + ESP_LOGW(TAG, "RESP too short: %u bytes", static_cast(len)); + return; + } + const auto *r = reinterpret_cast(data); + if (r->seq != g_expect_seq) { // late response from a timed-out request (expected) + ESP_LOGV(TAG, "dropping stale RESP seq %u (want %u)", r->seq, g_expect_seq); + return; + } + g_resp_status = r->status; + uint16_t rl = r->ret_len; + if (rl > sizeof(g_resp_ret)) { + // Larger than any real opcode return — a likely wire-format drift signal. + ESP_LOGW(TAG, "RESP ret_len %u exceeds buffer, clamping (wire drift?)", rl); + rl = sizeof(g_resp_ret); + } + if (len >= sizeof(esp_now_hosted_resp_t) + rl) { + memcpy(g_resp_ret, r->ret, rl); + } else { + // Truncated frame: fail closed. Never hand the caller stale bytes left in + // g_resp_ret by a previous response, and don't let request() report a + // zeroed payload as success — override the status to an error. + ESP_LOGW(TAG, "RESP truncated: claims %u ret bytes, frame too short", rl); + rl = 0; + g_resp_status = ESP_ERR_INVALID_RESPONSE; + } + g_resp_ret_len = rl; + xSemaphoreGive(g_resp_sem); +} + +void on_recv(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + // Read the volatile pointer once: esp_now_unregister_recv_cb()/deinit() (via + // the espnow component's disable()) can null it on the main loop between the + // guard and the call, which would otherwise turn the call into a null-deref. + const esp_now_recv_cb_t cb = g_recv_cb; + if (cb == nullptr) + return; + if (len < sizeof(esp_now_hosted_recv_evt_t)) { + ESP_LOGW(TAG, "RECV too short: %u bytes", static_cast(len)); + return; + } + const auto *e = reinterpret_cast(data); + if (len < sizeof(esp_now_hosted_recv_evt_t) + e->data_len) { + ESP_LOGW(TAG, "RECV data_len %u exceeds frame", e->data_len); + return; + } + + // ESPHome dereferences info->rx_ctrl->{rssi,timestamp}; give it a real one. + wifi_pkt_rx_ctrl_t rx_ctrl; + memset(&rx_ctrl, 0, sizeof(rx_ctrl)); + rx_ctrl.rssi = e->rssi; + rx_ctrl.channel = e->channel; + rx_ctrl.timestamp = static_cast(esp_timer_get_time()); + + esp_now_recv_info_t info; + info.src_addr = const_cast(e->src_addr); + info.des_addr = const_cast(e->des_addr); + info.rx_ctrl = &rx_ctrl; + cb(&info, e->data, static_cast(e->data_len)); +} + +void on_send(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + // Read the volatile pointer once (see on_recv): disable()/deinit() can null it + // on the main loop concurrently with this RX-thread callback. + const esp_now_send_cb_t cb = g_send_cb; + if (cb == nullptr) + return; + if (len < sizeof(esp_now_hosted_send_evt_t)) { + ESP_LOGW(TAG, "SEND evt too short: %u bytes", static_cast(len)); + return; + } + const auto *e = reinterpret_cast(data); +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 5, 0) + // IDF >= 5.5: esp_now_send_cb_t takes esp_now_send_info_t (== wifi_tx_info_t), + // whose des_addr is a POINTER (not an inline array). Point it at the event's + // MAC (valid for this callback) — do NOT memcpy into it (that writes NULL and + // faults). ESPHome reads only info->des_addr. + esp_now_send_info_t si; + memset(&si, 0, sizeof(si)); + si.des_addr = const_cast(e->des_addr); + cb(&si, static_cast(e->status)); +#else + cb(e->des_addr, static_cast(e->status)); +#endif +} + +esp_err_t ensure_setup() { + // Gate on g_setup_done, not on g_req_mutex: a failure part-way through (a + // semaphore that did not allocate, a callback that did not register) must not + // leave a later call thinking setup completed. Semaphore creation is guarded + // so a retry after a partial failure does not leak the earlier handles. + if (g_setup_done) + return ESP_OK; + if (g_req_mutex == nullptr) + g_req_mutex = xSemaphoreCreateMutex(); + if (g_resp_sem == nullptr) + g_resp_sem = xSemaphoreCreateBinary(); + if (g_req_mutex == nullptr || g_resp_sem == nullptr) + return ESP_ERR_NO_MEM; + esp_err_t err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_RESP, on_resp, nullptr)) != ESP_OK) + return err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_RECV, on_recv, nullptr)) != ESP_OK) + return err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_SEND, on_send, nullptr)) != ESP_OK) + return err; + g_setup_done = true; + return ESP_OK; +} + +// Send one request envelope. With wait=true (default) block until the matching +// response (or timeout); with wait=false return as soon as the frame is handed +// to the transport (fire-and-forget, used by esp_now_send). +// +// `tail` is an optional second chunk written straight after `payload`. Callers +// with a fixed header plus a bulk body (esp_now_send) pass the two separately +// so they never need a build buffer of their own: both chunks are laid into the +// request buffer here, under g_req_mutex, which keeps concurrent callers from +// racing and saves a full copy of the body on every transmit. +esp_err_t request(uint8_t opcode, const void *payload, uint16_t plen, void *ret, uint16_t ret_cap, uint16_t *ret_len, + bool wait = true, const void *tail = nullptr, uint16_t tail_len = 0) { + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + if (plen > ESP_NOW_HOSTED_MAX_PAYLOAD || tail_len > ESP_NOW_HOSTED_MAX_PAYLOAD - plen) + return ESP_ERR_INVALID_SIZE; + const uint16_t total_len = static_cast(plen + tail_len); + + if (xSemaphoreTake(g_req_mutex, portMAX_DELAY) != pdTRUE) + return ESP_FAIL; + + static uint8_t buf[sizeof(esp_now_hosted_req_t) + ESP_NOW_HOSTED_MAX_PAYLOAD]; // guarded by g_req_mutex + auto *req = reinterpret_cast(buf); + req->opcode = opcode; + req->seq = ++g_seq; + req->payload_len = total_len; + if (plen != 0) + memcpy(req->payload, payload, plen); + if (tail_len != 0) + memcpy(req->payload + plen, tail, tail_len); + g_expect_seq = req->seq; + + xSemaphoreTake(g_resp_sem, 0); // drain any stale signal before sending + err = esp_hosted_send_custom_data(ESP_NOW_HOSTED_MSG_REQ, buf, sizeof(esp_now_hosted_req_t) + total_len); + if (err != ESP_OK) { + xSemaphoreGive(g_req_mutex); + return err; + } + if (!wait) { + // Fire-and-forget (esp_now_send): the co-processor enqueues the frame and + // reports the real TX result later via the async SEND event, exactly like + // native esp_now_send. Returning here keeps the main loop off the ~100 ms+ + // RPC round-trip. The matching RESP is ignored (seq won't match the next + // waited request, so on_resp drops it). + xSemaphoreGive(g_req_mutex); + return ESP_OK; + } + if (xSemaphoreTake(g_resp_sem, pdMS_TO_TICKS(ESP_NOW_HOSTED_TIMEOUT_MS)) != pdTRUE) { + ESP_LOGW(TAG, "opcode %u timed out", opcode); + xSemaphoreGive(g_req_mutex); + return ESP_ERR_TIMEOUT; + } + + const int32_t status = g_resp_status; + if (ret != nullptr && ret_cap != 0) { + uint16_t n = g_resp_ret_len < ret_cap ? g_resp_ret_len : ret_cap; + memcpy(ret, const_cast(g_resp_ret), n); + if (ret_len != nullptr) + *ret_len = n; + } + xSemaphoreGive(g_req_mutex); + return static_cast(status); +} + +} // namespace + +// ── The surface, defined for the radio-less host ──────────────── +extern "C" { + +esp_err_t esp_now_init(void) { return request(ESP_NOW_HOSTED_OP_INIT, nullptr, 0, nullptr, 0, nullptr); } + +esp_err_t esp_now_deinit(void) { + g_recv_cb = nullptr; + g_send_cb = nullptr; + peer_cache_clear(); // the co-processor drops all peers on deinit + return request(ESP_NOW_HOSTED_OP_DEINIT, nullptr, 0, nullptr, 0, nullptr); +} + +esp_err_t esp_now_get_version(uint32_t *version) { + uint32_t v = 0; + uint16_t rl = 0; + esp_err_t err = request(ESP_NOW_HOSTED_OP_GET_VERSION, nullptr, 0, &v, sizeof(v), &rl); + if (version != nullptr) + *version = v; + return err; +} + +esp_err_t esp_now_register_recv_cb(esp_now_recv_cb_t cb) { + // Only arm the callback once the CustomRpc handlers are actually registered, + // so a failed setup leaves g_recv_cb null rather than falsely "registered". + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + g_recv_cb = cb; + return ESP_OK; +} +esp_err_t esp_now_unregister_recv_cb(void) { + g_recv_cb = nullptr; + return ESP_OK; +} +esp_err_t esp_now_register_send_cb(esp_now_send_cb_t cb) { + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + g_send_cb = cb; + return ESP_OK; +} +esp_err_t esp_now_unregister_send_cb(void) { + g_send_cb = nullptr; + return ESP_OK; +} + +static esp_err_t add_or_mod_peer(uint8_t opcode, const esp_now_peer_info_t *peer, bool wait) { + if (peer == nullptr) + return ESP_ERR_ESPNOW_ARG; + esp_now_hosted_peer_t p; + memset(&p, 0, sizeof(p)); + memcpy(p.peer_addr, peer->peer_addr, 6); + memcpy(p.lmk, peer->lmk, 16); + p.channel = peer->channel; + p.ifidx = static_cast(peer->ifidx); + p.encrypt = peer->encrypt ? 1 : 0; + return request(opcode, &p, sizeof(p), nullptr, 0, nullptr, wait); +} +esp_err_t esp_now_add_peer(const esp_now_peer_info_t *peer) { + // Fire-and-forget (wait=false): adding a peer is a blocking RPC round-trip, + // and ESPHome's espnow calls it on the main loop when a device joins the mesh + // — under co-processor load that stalls the UI (peer-churn stutter). Issue it + // without waiting and mirror it locally. Safe against a following + // esp_now_send to the same peer: both ride the same in-order CustomRpc + // channel (mutex-serialized on the host) and the co-processor processes REQs + // FIFO, so ADD_PEER is applied before the SEND. Trade-off: a co-processor-side + // failure (e.g. peer table full) is no longer reported synchronously — the + // same limitation as esp_now_send — but ESPHome only adds peers it validated. + esp_err_t err = add_or_mod_peer(ESP_NOW_HOSTED_OP_ADD_PEER, peer, /*wait=*/false); + if (err == ESP_OK) + peer_cache_add(peer->peer_addr); // keep the local mirror in sync + return err; +} +esp_err_t esp_now_mod_peer(const esp_now_peer_info_t *peer) { + // mod_peer changes a peer's parameters, not its existence, so the cache is + // unaffected. Kept synchronous — it is not on any hot path (espnow never + // calls it), so the extra round-trip does not matter and the status is useful. + return add_or_mod_peer(ESP_NOW_HOSTED_OP_MOD_PEER, peer, /*wait=*/true); +} + +esp_err_t esp_now_del_peer(const uint8_t *peer_addr) { + if (peer_addr == nullptr) + return ESP_ERR_ESPNOW_ARG; + // Fire-and-forget for the same reason as add_peer (peer churn on the main + // loop). Removal is order-independent, so this is strictly safe. + esp_err_t err = request(ESP_NOW_HOSTED_OP_DEL_PEER, peer_addr, 6, nullptr, 0, nullptr, /*wait=*/false); + if (err == ESP_OK) + peer_cache_remove(peer_addr); // keep the local mirror in sync + return err; +} + +bool esp_now_is_peer_exist(const uint8_t *peer_addr) { + if (peer_addr == nullptr) + return false; + // Answered from the local mirror — no RPC round-trip. ESPHome's espnow calls + // this on the main loop for every received frame and every send, so a + // blocking round-trip here would stall rendering under mesh traffic. + return peer_cache_contains(peer_addr); +} + +esp_err_t esp_now_send(const uint8_t *peer_addr, const uint8_t *data, size_t len) { + if (len > ESP_NOW_HOSTED_MAX_FRAME) + return ESP_ERR_ESPNOW_ARG; + if (data == nullptr && len != 0) // native esp_now_send treats this as an arg error + return ESP_ERR_ESPNOW_ARG; + // Only the small fixed header is built here; the caller's frame goes over as + // the request tail, so request() lays both into its own buffer under + // g_req_mutex. esp_now_send is a public C symbol and may be called from any + // task, and a shared build buffer here would let two callers corrupt each + // other's frame. Passing the body through also drops a full-frame copy per + // transmit, on the path this shim exists to keep quick. + uint8_t hdr[sizeof(esp_now_hosted_send_req_t)]; + auto *s = reinterpret_cast(hdr); + s->has_addr = peer_addr != nullptr ? 1 : 0; + if (peer_addr != nullptr) + memcpy(s->peer_addr, peer_addr, 6); + else + memset(s->peer_addr, 0, 6); + s->data_len = static_cast(len); + // Fire-and-forget (wait=false): native esp_now_send returns once the frame is + // queued, with the real TX result delivered later through the send callback. + // The co-processor mirrors that — it acks enqueue immediately and reports the + // outcome via the async SEND event (on_send -> on_send_report). Waiting for + // the RPC RESP here would block the main loop for the full round-trip on + // every transmit. + return request(ESP_NOW_HOSTED_OP_SEND, hdr, sizeof(hdr), nullptr, 0, nullptr, /*wait=*/false, data, + static_cast(len)); +} + +esp_err_t esp_now_set_pmk(const uint8_t *pmk) { + if (pmk == nullptr) + return ESP_ERR_ESPNOW_ARG; + return request(ESP_NOW_HOSTED_OP_SET_PMK, pmk, 16, nullptr, 0, nullptr); +} + +// Remainder of the surface. Not used by ESPHome's espnow component +// today; provided so the whole header links and future callers get a defined +// (if unimplemented) symbol rather than a link error. Wire them through +// CustomRpc if a use case appears. +esp_err_t esp_now_get_peer(const uint8_t * /*peer_addr*/, esp_now_peer_info_t * /*peer*/) { + return ESP_ERR_NOT_SUPPORTED; +} +esp_err_t esp_now_fetch_peer(bool /*from_head*/, esp_now_peer_info_t * /*peer*/) { return ESP_ERR_NOT_SUPPORTED; } +esp_err_t esp_now_get_peer_num(esp_now_peer_num_t * /*num*/) { return ESP_ERR_NOT_SUPPORTED; } +esp_err_t esp_now_set_wake_window(uint16_t /*window*/) { + return ESP_ERR_NOT_SUPPORTED; // power-save wake window is not forwarded; don't claim success +} +esp_err_t esp_now_set_peer_rate_config(const uint8_t * /*peer_addr*/, esp_now_rate_config_t * /*cfg*/) { + return ESP_ERR_NOT_SUPPORTED; +} +esp_err_t esp_wifi_config_espnow_rate(wifi_interface_t /*ifx*/, wifi_phy_rate_t /*rate*/) { + return ESP_ERR_NOT_SUPPORTED; +} + +} // extern "C" + +#endif // CONFIG_IDF_TARGET_ESP32P4 diff --git a/esphome/components/esp32_hosted/esp_now_hosted_rpc.h b/esphome/components/esp32_hosted/esp_now_hosted_rpc.h new file mode 100644 index 0000000000..bf68c759ee --- /dev/null +++ b/esphome/components/esp32_hosted/esp_now_hosted_rpc.h @@ -0,0 +1,128 @@ +/* + * esp_now_hosted — ESP-NOW-over-CustomRpc wire protocol. + * + * Shared, byte-for-byte-identical contract between: + * - the host shim (esphome/components/esp32_hosted/esp_now_hosted.cpp) + * - the coprocessor firmware (esphome/esp-hosted-firmware) + * + * It rides esp-hosted's CustomRpc channel (RPC ID 388, "peer data transfer", + * available since esp-hosted v2.8.1), teaching the radio-less host <-> radio + * co-processor link to carry esp_now.h, which esp-hosted itself does not proxy + * (Espressif issue espressif/esp-hosted-mcu#19). + * + * KEEP THE TWO COPIES IN SYNC. The canonical copy lives here; the coprocessor + * firmware uses a verbatim copy. Both sides are little-endian, so these packed + * structs are wire-compatible with no byte-swapping. + */ + +#ifndef ESP_NOW_HOSTED_RPC_H +#define ESP_NOW_HOSTED_RPC_H + +#ifdef __cplusplus +#include +#else +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* ── CustomRpc message IDs (any uint32_t except 0xFFFFFFFF) ────────────────── + * One REQ handler slot on the device; three event handler slots on the host. + * The bytes spell "now" + index, a private range unlikely to clash with other + * CustomRpc users (e.g. the stock peer_data_transfer example's 1..6). */ +#define ESP_NOW_HOSTED_MSG_REQ 0x6E6F7701u /* host -> device : request envelope */ +#define ESP_NOW_HOSTED_MSG_RESP 0x6E6F7702u /* device -> host : reply to a REQ */ +#define ESP_NOW_HOSTED_MSG_RECV 0x6E6F7703u /* device -> host : async RX frame */ +#define ESP_NOW_HOSTED_MSG_SEND 0x6E6F7704u /* device -> host : async TX status */ + +/* ── Request opcodes ────────────────────────────────────────────────────── */ +enum { + ESP_NOW_HOSTED_OP_INIT = 1, /* esp_now_init + register device recv/send cbs */ + ESP_NOW_HOSTED_OP_DEINIT = 2, /* unregister cbs + esp_now_deinit */ + ESP_NOW_HOSTED_OP_ADD_PEER = 3, /* payload: esp_now_hosted_peer_t */ + ESP_NOW_HOSTED_OP_DEL_PEER = 4, /* payload: 6-byte peer MAC */ + ESP_NOW_HOSTED_OP_IS_PEER_EXIST = 5, /* payload: 6-byte MAC; ret: 1 byte bool */ + ESP_NOW_HOSTED_OP_SEND = 6, /* payload: esp_now_hosted_send_req_t */ + ESP_NOW_HOSTED_OP_GET_VERSION = 7, /* ret: uint32 version */ + ESP_NOW_HOSTED_OP_SET_PMK = 8, /* payload: 16-byte PMK */ + ESP_NOW_HOSTED_OP_MOD_PEER = 9, /* payload: esp_now_hosted_peer_t */ +}; + +/* Largest ESP-NOW payload we forward. ESP-NOW v2 (IDF >= 5.4) is 1470 B; well + * under esp-hosted's 8166 B CustomRpc cap, so the shim never truncates. */ +#define ESP_NOW_HOSTED_MAX_FRAME 1470u +/* Envelope slack for the largest opcode payload (a SEND req wrapping a frame). */ +#define ESP_NOW_HOSTED_MAX_PAYLOAD (ESP_NOW_HOSTED_MAX_FRAME + 16u) +/* Host request/response round-trip timeout over the transport. Generous: + * normal RTT is sub-millisecond, but Wi-Fi/BLE contention on the co-processor + * can stall the RX thread. */ +#define ESP_NOW_HOSTED_TIMEOUT_MS 2000 + +/* ── Envelopes ──────────────────────────────────────────────────────────── */ + +/* These payloads are shared verbatim with the C co-processor firmware, so they + * use C's `typedef struct {...} name;` idiom rather than C++ `using` aliases, + * which would not compile there. Silence clang-tidy's modernize-use-using for + * the shared struct block. */ +// NOLINTBEGIN(modernize-use-using) +typedef struct { + uint8_t opcode; /* one of ESP_NOW_HOSTED_OP_* */ + uint8_t seq; /* wraps 0..255; echoed in the response for matching */ + uint16_t payload_len; /* bytes of opcode-specific payload that follow */ + uint8_t payload[]; /* flexible */ +} __attribute__((packed)) esp_now_hosted_req_t; + +typedef struct { + uint8_t opcode; /* echoes the request opcode */ + uint8_t seq; /* echoes the request seq */ + int32_t status; /* esp_err_t from the native call on the co-processor */ + uint16_t ret_len; /* bytes of return payload that follow */ + uint8_t ret[]; /* flexible (e.g. version u32, is_peer_exist bool) */ +} __attribute__((packed)) esp_now_hosted_resp_t; + +/* ── Opcode payloads ────────────────────────────────────────────────────── */ + +/* esp_now_peer_info_t minus the host-only `priv` pointer, which is meaningless + * across the transport and never set by ESPHome's espnow component. */ +typedef struct { + uint8_t peer_addr[6]; + uint8_t lmk[16]; + uint8_t channel; /* 0 = current channel */ + uint8_t ifidx; /* wifi_interface_t (0=STA, 1=AP) */ + uint8_t encrypt; /* bool */ +} __attribute__((packed)) esp_now_hosted_peer_t; + +typedef struct { + uint8_t has_addr; /* 0 => peer_addr is NULL (broadcast to all peers) */ + uint8_t peer_addr[6]; + uint16_t data_len; + uint8_t data[]; /* flexible, up to ESP_NOW_HOSTED_MAX_FRAME */ +} __attribute__((packed)) esp_now_hosted_send_req_t; + +/* ── Async events (device -> host) ──────────────────────────────────────── */ + +/* Reconstructed on the host into an esp_now_recv_info_t + a minimal + * wifi_pkt_rx_ctrl_t. ESPHome's espnow reads info->src_addr, info->des_addr, + * info->rx_ctrl->rssi and info->rx_ctrl->timestamp. */ +typedef struct { + uint8_t src_addr[6]; + uint8_t des_addr[6]; + int8_t rssi; + uint8_t channel; + uint16_t data_len; + uint8_t data[]; /* flexible */ +} __attribute__((packed)) esp_now_hosted_recv_evt_t; + +typedef struct { + uint8_t des_addr[6]; + uint8_t status; /* esp_now_send_status_t (0 = success) */ +} __attribute__((packed)) esp_now_hosted_send_evt_t; +// NOLINTEND(modernize-use-using) + +#ifdef __cplusplus +} +#endif + +#endif /* ESP_NOW_HOSTED_RPC_H */ diff --git a/esphome/components/espnow/__init__.py b/esphome/components/espnow/__init__.py index 5541a6ee97..14d099ec06 100644 --- a/esphome/components/espnow/__init__.py +++ b/esphome/components/espnow/__init__.py @@ -3,6 +3,7 @@ from typing import Any from esphome import automation, core import esphome.codegen as cg from esphome.components import wifi +from esphome.components.esp32 import VARIANT_ESP32P4, get_esp32_variant from esphome.components.udp import CONF_ON_RECEIVE import esphome.config_validation as cv from esphome.const import ( @@ -17,6 +18,7 @@ from esphome.const import ( ) from esphome.core import CORE, HexInt from esphome.cpp_generator import MockObj, TemplateArgsType +import esphome.final_validate as fv from esphome.types import ConfigType CODEOWNERS = ["@jesserockz"] @@ -132,6 +134,24 @@ CONFIG_SCHEMA = cv.All( ) +def _validate_variant(config: ConfigType) -> ConfigType: + # ESP-NOW rides the Wi-Fi PHY. Radio-less esp32 variants have no native + # ESP-NOW; only the ESP32-P4 has a path, via the esp32_hosted shim that + # supplies the esp_now_* symbols. Fail here with a clear message instead of + # letting the build reach an "undefined reference to esp_now_*" link error. + variant = get_esp32_variant() + if wifi.variant_has_wifi(variant): + return config + if variant != VARIANT_ESP32P4: + raise cv.Invalid(f"ESP-NOW is not supported on {variant} (no Wi-Fi radio)") + if "esp32_hosted" not in fv.full_config.get(): + raise cv.Invalid(f"ESP-NOW on {variant} requires the esp32_hosted component") + return config + + +FINAL_VALIDATE_SCHEMA = _validate_variant + + async def _trigger_to_code(config: ConfigType) -> MockObj: if address := config.get(CONF_ADDRESS): address = address.parts diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 9dd1e0ced6..eaece6d5ff 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -71,6 +71,7 @@ #define USE_ESP32_HOSTED #define USE_ESP32_HOSTED_HTTP_UPDATE #define USE_ESP32_IMPROV_STATE_CALLBACK +#define USE_ESP_NOW_HOSTED #define USE_EVENT #define USE_FAN #define USE_GPIO_BINARY_SENSOR_INTERRUPT diff --git a/script/ci-custom.py b/script/ci-custom.py index f481fda860..e2b7cd8d37 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -294,6 +294,9 @@ def highlight(s): "esphome/components/socket/headers.h", "esphome/core/defines.h", "esphome/components/http_request/httplib.h", + # Shared C wire header (byte-identical with the co-processor firmware); + # these are protocol constants and constexpr is C++-only. + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", ], ) def lint_no_defines(fname, match): @@ -816,6 +819,10 @@ def lint_relative_py_import(fname: Path, line, col, content): "esphome/components/host/helpers.cpp", "esphome/components/zephyr/helpers.cpp", "esphome/components/http_request/httplib.h", + # Global extern "C" esp_now_* linker symbols + shared C wire header; + # neither can live in a C++ namespace. + "esphome/components/esp32_hosted/esp_now_hosted.cpp", + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", ], ) def lint_namespace(fname: Path, content: str) -> str | None: @@ -841,7 +848,15 @@ def lint_esphome_h(fname, line, col, content): ) -@lint_content_check(include=["*.h"], exclude=["esphome/core/entity_types.h"]) +@lint_content_check( + include=["*.h"], + exclude=[ + "esphome/core/entity_types.h", + # Shared C wire header; uses a classic #ifndef guard for portability + # across the co-processor firmware repo it stays byte-identical with. + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", + ], +) def lint_pragma_once(fname, content): if "#pragma once" not in content: return ( diff --git a/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml b/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml new file mode 100644 index 0000000000..fab0a64ab8 --- /dev/null +++ b/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml @@ -0,0 +1,5 @@ +# Exercises the ESP-NOW-over-hosted shim: on the ESP32-P4 host, esp32_hosted +# supplies the esp_now_* symbols that the espnow component links against. +packages: + esp32_hosted: !include common.yaml + espnow: !include ../espnow/common.yaml diff --git a/tests/unit_tests/components/test_espnow.py b/tests/unit_tests/components/test_espnow.py new file mode 100644 index 0000000000..21305c2b33 --- /dev/null +++ b/tests/unit_tests/components/test_espnow.py @@ -0,0 +1,48 @@ +"""Tests for the espnow component's final validation.""" + +import pytest + +from esphome.components.esp32.const import ( + VARIANT_ESP32C3, + VARIANT_ESP32H2, + VARIANT_ESP32P4, +) +from esphome.components.espnow import _validate_variant +import esphome.config_validation as cv +import esphome.final_validate as fv +from esphome.types import ConfigType + + +def _run( + monkeypatch, variant: str, full_config: dict, config: ConfigType +) -> ConfigType: + monkeypatch.setattr("esphome.components.espnow.get_esp32_variant", lambda: variant) + token = fv.full_config.set(full_config) + try: + return _validate_variant(config) + finally: + fv.full_config.reset(token) + + +def test_variant_with_native_wifi_passes(monkeypatch) -> None: + """A variant with a native Wi-Fi PHY needs no shim; config passes through.""" + config = {"id": "espnow"} + assert _run(monkeypatch, VARIANT_ESP32C3, {}, config) is config + + +def test_radioless_non_p4_variant_rejected(monkeypatch) -> None: + """Radio-less variants without any ESP-NOW path are rejected outright.""" + with pytest.raises(cv.Invalid, match="not supported"): + _run(monkeypatch, VARIANT_ESP32H2, {}, {}) + + +def test_p4_without_esp32_hosted_rejected(monkeypatch) -> None: + """The P4 needs the esp32_hosted shim to supply the esp_now_* symbols.""" + with pytest.raises(cv.Invalid, match="esp32_hosted"): + _run(monkeypatch, VARIANT_ESP32P4, {}, {}) + + +def test_p4_with_esp32_hosted_passes(monkeypatch) -> None: + """The P4 with esp32_hosted present validates; config passes through.""" + config = {"id": "espnow"} + assert _run(monkeypatch, VARIANT_ESP32P4, {"esp32_hosted": {}}, config) is config From 3321566cc010c3a4e774a78e1d81d887c8e02879 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Thu, 3 Sep 2026 07:12:36 -0500 Subject: [PATCH 111/147] [remote_transmitter] Fix BK7231N build by limiting the PWM path to BK7238 (#18958) --- esphome/components/remote_transmitter/__init__.py | 14 +++++--------- .../remote_transmitter/remote_transmitter.h | 9 +++++---- .../remote_transmitter_bk72xx.cpp | 11 +++++++---- .../remote_transmitter_libretiny_isr.cpp | 10 +++++----- .../remote_transmitter/test_non_blocking_gate.py | 2 +- .../remote_transmitter/test.bk72xx-ard.yaml | 2 +- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/remote_transmitter/__init__.py b/esphome/components/remote_transmitter/__init__.py index cb2aebec91..58392c48ab 100644 --- a/esphome/components/remote_transmitter/__init__.py +++ b/esphome/components/remote_transmitter/__init__.py @@ -4,11 +4,7 @@ from esphome import automation, pins import esphome.codegen as cg from esphome.components import esp32, esp32_rmt, remote_base from esphome.components.libretiny import get_libretiny_family -from esphome.components.libretiny.const import ( - FAMILY_BK7231N, - FAMILY_BK7238, - FAMILY_RTL8720C, -) +from esphome.components.libretiny.const import FAMILY_BK7238, FAMILY_RTL8720C from esphome.config_helpers import filter_source_files_from_platform import esphome.config_validation as cv from esphome.const import ( @@ -49,7 +45,9 @@ DigitalWriteAction = remote_transmitter_ns.class_( ) -_NON_BLOCKING_LIBRETINY_FAMILIES = (FAMILY_RTL8720C, FAMILY_BK7231N, FAMILY_BK7238) +# Keep in sync with the USE_LIBRETINY_VARIANT_RTL8720C / REMOTE_TRANSMITTER_BK_PWM gates in +# remote_transmitter.h, which decide where set_non_blocking() is declared +_NON_BLOCKING_LIBRETINY_FAMILIES = (FAMILY_RTL8720C, FAMILY_BK7238) def _validate_non_blocking_platform(value: bool) -> bool: @@ -59,9 +57,7 @@ def _validate_non_blocking_platform(value: bool) -> bool: return cv.boolean(value) if CORE.is_libretiny and get_libretiny_family() in _NON_BLOCKING_LIBRETINY_FAMILIES: return cv.boolean(value) - raise cv.Invalid( - "non_blocking is only supported on ESP32, RTL8720C, BK7231N and BK7238" - ) + raise cv.Invalid("non_blocking is only supported on ESP32, RTL8720C and BK7238") MULTI_CONF = True diff --git a/esphome/components/remote_transmitter/remote_transmitter.h b/esphome/components/remote_transmitter/remote_transmitter.h index 313b26364d..4db4e80a60 100644 --- a/esphome/components/remote_transmitter/remote_transmitter.h +++ b/esphome/components/remote_transmitter/remote_transmitter.h @@ -12,10 +12,11 @@ #endif // SOC_RMT_SUPPORTED #endif // USE_ESP32 -// The BK7231N-style PWM block (hardware shadow-load duty updates) enables the ISR-driven -// transmitter on these families; family-level proxy for the SDK's CFG_SOC_NAME gate. -// See remote_transmitter_bk72xx.cpp. -#if defined(USE_LIBRETINY_VARIANT_BK7231N) || defined(USE_LIBRETINY_VARIANT_BK7238) +// Enables the ISR-driven transmitter on Beken. Gated on BK7238 alone: the shadow-load PWM +// block is shared with BK7231N, but LibreTiny builds that family against an older BDK whose +// PWM driver has no pwm_init_param()/pwm_start(). See remote_transmitter_bk72xx.cpp. +// Keep in sync with _NON_BLOCKING_LIBRETINY_FAMILIES in __init__.py. +#ifdef USE_LIBRETINY_VARIANT_BK7238 #define REMOTE_TRANSMITTER_BK_PWM #endif diff --git a/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp index 0081ae47b3..822389ccf9 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_bk72xx.cpp @@ -9,10 +9,13 @@ // with the core's fixes for type-name collisions between the two #include -// Only the BK7231N-style PWM block (shadow registers with a hardware CFG_UPDATA load bit) -// supports glitch-free per-edge duty updates; older SoCs compile the generic bit-bang -// implementation (remote_transmitter.cpp) instead, and this file compiles to nothing. -// REMOTE_TRANSMITTER_BK_PWM is set per-family in remote_transmitter.h. +// Needs the BK7231N-style PWM block (shadow registers with a hardware CFG_UPDATA load bit) +// for glitch-free per-edge duty updates, and an SDK exposing pwm_init_param()/pwm_start(). +// BK7231N has the block but LibreTiny builds it against an older BDK offering only the +// sddev_control API (CMD_PWM_INIT_PARAM), so it stays on the generic bit-bang path until +// someone can add and validate that path on real hardware. Every other Beken SoC lacks the +// block. REMOTE_TRANSMITTER_BK_PWM is set per-family in remote_transmitter.h; when it is +// unset this file compiles to nothing and remote_transmitter.cpp is used instead. namespace esphome::remote_transmitter { diff --git a/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp index 003cdfa986..fad91f593f 100644 --- a/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp +++ b/esphome/components/remote_transmitter/remote_transmitter_libretiny_isr.cpp @@ -3,11 +3,11 @@ #include "esphome/core/hal.h" #include "esphome/core/log.h" -// Envelope chain shared by the LibreTiny families that pace transmission from a hardware -// timer interrupt: RTL8720C (gtimer) and the BK7231N-style PWM block (BKTIMER1). Everything -// platform-specific sits behind five hooks implemented in the per-family files -- carrier -// setup, duty writes, one-shot arming and timer stop. Families without a usable timer keep -// the generic bit-bang implementation and compile none of this. +// Envelope chain shared by the LibreTiny families that pace transmission from a hardware timer +// interrupt: RTL8720C (gtimer) and BK7238 (BKTIMER1). Everything platform-specific sits behind +// five hooks implemented in the per-family files -- carrier setup, duty writes, one-shot arming +// and timer stop. Families without a usable timer keep the generic bit-bang implementation and +// compile none of this. #if defined(USE_LIBRETINY_VARIANT_RTL8720C) || defined(REMOTE_TRANSMITTER_BK_PWM) namespace esphome::remote_transmitter { diff --git a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py index ee2769e177..525ab3329e 100644 --- a/tests/component_tests/remote_transmitter/test_non_blocking_gate.py +++ b/tests/component_tests/remote_transmitter/test_non_blocking_gate.py @@ -26,7 +26,7 @@ from ..types import SetCoreConfigCallable (PlatformFramework.ESP32_IDF, None, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8720C, True), (PlatformFramework.RTL87XX_ARDUINO, FAMILY_RTL8710B, False), - (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231N, True), + (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231N, False), (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7238, True), (PlatformFramework.BK72XX_ARDUINO, FAMILY_BK7231T, False), (PlatformFramework.ESP8266_ARDUINO, None, False), diff --git a/tests/components/remote_transmitter/test.bk72xx-ard.yaml b/tests/components/remote_transmitter/test.bk72xx-ard.yaml index ea2feafda9..f3e2da9daf 100644 --- a/tests/components/remote_transmitter/test.bk72xx-ard.yaml +++ b/tests/components/remote_transmitter/test.bk72xx-ard.yaml @@ -2,7 +2,7 @@ remote_transmitter: id: xmitr pin: GPIO26 carrier_duty_percent: 50% - # non_blocking is bk7231n/bk7238-only; the CI board is a BK7252 + # non_blocking is bk7238-only; the CI board is a BK7252, so this builds the bit-bang path packages: buttons: !include common-buttons.yaml From 657116a213de452fc191772c06e20aec09d16446 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:15:06 +0000 Subject: [PATCH 112/147] Bump bundled esphome-device-builder to 1.14.0 (#18960) Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0da8048c57..7952616496 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.13.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.0 RUN \ platformio settings set enable_telemetry No \ From e47247486ba238f16f958a3298e08c43d309e6c7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 3 Sep 2026 21:16:36 +0200 Subject: [PATCH 113/147] [esp8266] Drop Arduino framework versions before 3.0.0 (#18917) to Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/arduino8266/framework.py | 13 ++--- esphome/components/climate/climate.cpp | 4 +- esphome/components/debug/debug_component.cpp | 4 +- esphome/components/debug/debug_component.h | 4 +- esphome/components/debug/debug_esp8266.cpp | 2 - esphome/components/debug/sensor.py | 7 +-- esphome/components/esp8266/__init__.py | 57 ++++++------------- .../nextion/nextion_upload_arduino.cpp | 6 -- esphome/components/wifi/wifi_component.h | 5 -- .../wifi/wifi_component_esp8266.cpp | 10 +--- esphome/core/log.h | 14 ----- .../components/esp8266/test_boards.py | 17 +----- .../esp8266/test_framework_version.py | 23 ++++++++ .../unit_tests/test_arduino8266_framework.py | 17 ++---- 14 files changed, 62 insertions(+), 121 deletions(-) create mode 100644 tests/unit_tests/components/esp8266/test_framework_version.py diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 1edbe4b36f..663002b3b1 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -44,8 +44,7 @@ def get_arduino8266_tools_path() -> Path: return tools_cache_path(*ARDUINO8266_TOOLS_CACHE) -# 3.1.1 rather than 3.1.0: the registry has no package for 3.1.0, and the -# encoder below cannot name 3.0.0/3.0.1 either (see its docstring) +# 3.1.1 rather than 3.1.0: the registry has no packages for 3.0.0, 3.0.1 or 3.1.0 MIN_FRAMEWORK_VERSION = Version(3, 1, 1) @@ -53,20 +52,16 @@ def framework_package_version(ver: Version) -> str: """Map an Arduino core version to its registry package version (3.1.2 -> 3.30102.0; the leading 3 is the package major). - Exact registry names only for cores > 2.6.2 and >= 3.0.2; callers floor - at MIN_FRAMEWORK_VERSION. + Exact registry names for 3.x cores; callers floor at MIN_FRAMEWORK_VERSION. """ if ver.major > 3: raise EsphomeError( f"Arduino core {ver} is not supported yet; " "the newest known core series is 3.x" ) - if ver <= Version(2, 6, 2): - # Cores <= 2.6.2 use the older 1.x/2.x package-major encodings (same - # boundary as _format_framework_arduino_version's era guard) + if ver.major < 3: raise EsphomeError( - f"Arduino core {ver} uses an older package encoding than this " - "helper implements (newer than 2.6.2)" + f"Arduino core {ver} is not supported; ESPHome requires core 3.x" ) return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" diff --git a/esphome/components/climate/climate.cpp b/esphome/components/climate/climate.cpp index 34684a87e1..f80de151b1 100644 --- a/esphome/components/climate/climate.cpp +++ b/esphome/components/climate/climate.cpp @@ -368,8 +368,8 @@ optional Climate::restore_state_() { } void Climate::save_state_(const ClimateTraits &traits) { -#if (defined(USE_ESP32) || (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0))) && \ - !defined(CLANG_TIDY) +#if (defined(USE_ESP32) || defined(USE_ESP8266)) && !defined(CLANG_TIDY) +#pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wclass-memaccess" #define TEMP_IGNORE_MEMACCESS #endif diff --git a/esphome/components/debug/debug_component.cpp b/esphome/components/debug/debug_component.cpp index 9020c261c2..97f4522c62 100644 --- a/esphome/components/debug/debug_component.cpp +++ b/esphome/components/debug/debug_component.cpp @@ -22,9 +22,9 @@ void DebugComponent::dump_config() { LOG_SENSOR(" ", "Free space on heap", this->free_sensor_); LOG_SENSOR(" ", "Largest free heap block", this->block_sensor_); LOG_SENSOR(" ", "CPU frequency", this->cpu_frequency_sensor_); -#if defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) +#ifdef USE_ESP8266 LOG_SENSOR(" ", "Heap fragmentation", this->fragmentation_sensor_); -#endif // defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) +#endif // USE_ESP8266 #endif // USE_SENSOR char device_info_buffer[DEVICE_INFO_BUFFER_SIZE]; diff --git a/esphome/components/debug/debug_component.h b/esphome/components/debug/debug_component.h index 20798cf600..b05029f878 100644 --- a/esphome/components/debug/debug_component.h +++ b/esphome/components/debug/debug_component.h @@ -35,7 +35,7 @@ class DebugComponent final : public PollingComponent { #ifdef USE_SENSOR void set_free_sensor(sensor::Sensor *free_sensor) { free_sensor_ = free_sensor; } void set_block_sensor(sensor::Sensor *block_sensor) { block_sensor_ = block_sensor; } -#if (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2)) || defined(USE_ESP32) +#if defined(USE_ESP8266) || defined(USE_ESP32) void set_fragmentation_sensor(sensor::Sensor *fragmentation_sensor) { fragmentation_sensor_ = fragmentation_sensor; } #endif #if defined(USE_ESP32) || defined(USE_LIBRETINY) @@ -61,7 +61,7 @@ class DebugComponent final : public PollingComponent { sensor::Sensor *free_sensor_{nullptr}; sensor::Sensor *block_sensor_{nullptr}; -#if (defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2)) || defined(USE_ESP32) +#if defined(USE_ESP8266) || defined(USE_ESP32) sensor::Sensor *fragmentation_sensor_{nullptr}; #endif #if defined(USE_ESP32) || defined(USE_LIBRETINY) diff --git a/esphome/components/debug/debug_esp8266.cpp b/esphome/components/debug/debug_esp8266.cpp index 272123dfc0..acce28818c 100644 --- a/esphome/components/debug/debug_esp8266.cpp +++ b/esphome/components/debug/debug_esp8266.cpp @@ -159,12 +159,10 @@ void DebugComponent::update_platform_() { // NOLINTNEXTLINE(readability-static-accessed-through-instance) this->block_sensor_->publish_state(ESP.getMaxFreeBlockSize()); } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 2) if (this->fragmentation_sensor_ != nullptr) { // NOLINTNEXTLINE(readability-static-accessed-through-instance) this->fragmentation_sensor_->publish_state(ESP.getHeapFragmentation()); } -#endif #endif } diff --git a/esphome/components/debug/sensor.py b/esphome/components/debug/sensor.py index 72e2efebc2..e53cb0d1e4 100644 --- a/esphome/components/debug/sensor.py +++ b/esphome/components/debug/sensor.py @@ -52,12 +52,9 @@ CONFIG_SCHEMA = { ), cv.Optional(CONF_FRAGMENTATION): cv.All( cv.Any( - cv.All( - cv.only_on_esp8266, - cv.require_framework_version(esp8266_arduino=cv.Version(2, 5, 2)), - ), + cv.only_on_esp8266, cv.only_on_esp32, - msg="This feature is only available on ESP8266 (Arduino 2.5.2+) and ESP32", + msg="This feature is only available on ESP8266 and ESP32", ), sensor.sensor_schema( unit_of_measurement=UNIT_PERCENT, diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 63665e7681..19dbb68f29 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -35,7 +35,7 @@ from esphome.platformio.toolchain import copy_ccache_script from esphome.storage_json import StorageJSON from esphome.types import ConfigType -from .boards import BOARDS, ESP8266_LD_SCRIPTS, board_ld_script +from .boards import BOARDS, board_ld_script from .const import ( CONF_EARLY_PIN_INIT, CONF_ENABLE_SERIAL, @@ -43,8 +43,6 @@ from .const import ( CONF_RESTORE_FROM_FLASH, KEY_BOARD, KEY_ESP8266, - KEY_FLASH_SIZE, - KEY_LDSCRIPT, KEY_PIN_INITIAL_STATES, KEY_SERIAL1_REQUIRED, KEY_SERIAL_REQUIRED, @@ -133,10 +131,6 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # format the given arduino (https://github.com/esp8266/Arduino/releases) version to # a PIO platformio/framework-arduinoespressif8266 value # List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 - if ver <= cv.Version(2, 4, 1): - return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" - if ver <= cv.Version(2, 6, 2): - return f"~2.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" # Same encoding the native toolchain uses for its package download, so a # version bump cannot drift between the two paths. from esphome.arduino8266.framework import framework_package_version @@ -159,11 +153,9 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # - https://github.com/esp8266/Arduino/releases # - https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 RECOMMENDED_ARDUINO_FRAMEWORK_VERSION = cv.Version(3, 1, 2) -# The platformio/espressif8266 version to use for arduino 2 framework versions +# The platformio/espressif8266 version to use for arduino 3 framework versions # - https://github.com/platformio/platform-espressif8266/releases # - https://api.registry.platformio.org/v3/packages/platformio/platform/espressif8266 -ARDUINO_2_PLATFORM_VERSION = cv.Version(2, 6, 3) -# for arduino 3 framework versions ARDUINO_3_PLATFORM_VERSION = cv.Version(3, 2, 0) # for arduino 4 framework versions ARDUINO_4_PLATFORM_VERSION = cv.Version(4, 2, 1) @@ -188,6 +180,14 @@ def _arduino_check_versions(value: ConfigType) -> ConfigType: version = cv.Version.parse(cv.version_number(value[CONF_VERSION])) source = value.get(CONF_SOURCE, None) + if version < cv.Version(3, 0, 0): + raise cv.Invalid( + f"Arduino framework {version} is no longer supported; ESPHome requires " + f"C++20, which needs Arduino core 3.x. Use the recommended version " + f"({RECOMMENDED_ARDUINO_FRAMEWORK_VERSION}).", + path=[CONF_VERSION], + ) + value[CONF_VERSION] = str(version) value[CONF_SOURCE] = source or _format_framework_arduino_version(version) @@ -195,12 +195,8 @@ def _arduino_check_versions(value: ConfigType) -> ConfigType: if platform_version is None: if version >= cv.Version(3, 1, 0): platform_version = _parse_platform_version(str(ARDUINO_4_PLATFORM_VERSION)) - elif version >= cv.Version(3, 0, 0): - platform_version = _parse_platform_version(str(ARDUINO_3_PLATFORM_VERSION)) - elif version >= cv.Version(2, 5, 0): - platform_version = _parse_platform_version(str(ARDUINO_2_PLATFORM_VERSION)) else: - platform_version = _parse_platform_version(str(cv.Version(1, 8, 0))) + platform_version = _parse_platform_version(str(ARDUINO_3_PLATFORM_VERSION)) value[CONF_PLATFORM_VERSION] = platform_version if version != RECOMMENDED_ARDUINO_FRAMEWORK_VERSION: @@ -289,29 +285,11 @@ def check_rosetta() -> None: ) -def _choose_ld_script(board: str, ver: cv.Version) -> str | None: - """The flash ld to pin for this board and core, or None for cores - without ld-script support.""" - board_data = BOARDS[board] - ld_scripts = ESP8266_LD_SCRIPTS[board_data[KEY_FLASH_SIZE]] - if ver <= cv.Version(2, 3, 0): - # No ld script support - return None - if ver <= cv.Version(2, 4, 2): - # Old ld script path; the modern per-board override names do not - # exist in this core's SDK, so the override cannot be honored. - # Substituting the size default would move _FS_end and the - # preferences sector, wiping flash-backed state on flash. - if KEY_LDSCRIPT in board_data: - raise EsphomeError( - f"Board {board} requires its {board_data[KEY_LDSCRIPT]} " - f"flash layout, which Arduino core {ver} cannot honor; " - "use a core newer than 2.4.2" - ) - return ld_scripts[0] +def _choose_ld_script(board: str) -> str: + """The flash ld to pin for this board.""" # A per-board override preserves a layout the board shipped with # (see d1_wroom_02 in boards.py) - return board_ld_script(board_data) + return board_ld_script(BOARDS[board]) @coroutine_with_priority(CoroPriority.PLATFORM) @@ -435,10 +413,9 @@ async def to_code(config: ConfigType) -> None: ) if config[CONF_BOARD] in BOARDS: - ld_script = _choose_ld_script(config[CONF_BOARD], ver) - - if ld_script is not None: - cg.add_platformio_option("board_build.ldscript", ld_script) + cg.add_platformio_option( + "board_build.ldscript", _choose_ld_script(config[CONF_BOARD]) + ) CORE.add_job(add_pin_initial_states_array) CORE.add_job(finalize_waveform_config) diff --git a/esphome/components/nextion/nextion_upload_arduino.cpp b/esphome/components/nextion/nextion_upload_arduino.cpp index f02f32d5ca..944fa1db47 100644 --- a/esphome/components/nextion/nextion_upload_arduino.cpp +++ b/esphome/components/nextion/nextion_upload_arduino.cpp @@ -209,14 +209,8 @@ bool Nextion::upload_tft(uint32_t baud_rate, bool exit_reparse) { http_client.setTimeout(this->tft_upload_http_timeout_); bool begin_status = false; -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 7, 0) http_client.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS); -#elif USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 6, 0) - http_client.setFollowRedirects(true); -#endif -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 6, 0) http_client.setRedirectLimit(3); -#endif begin_status = http_client.begin(*this->get_wifi_client_(), this->tft_url_.c_str()); if (!begin_status) { this->connection_state_.is_updating_ = false; diff --git a/esphome/components/wifi/wifi_component.h b/esphome/components/wifi/wifi_component.h index cfdbc1a968..63df9fbfa5 100644 --- a/esphome/components/wifi/wifi_component.h +++ b/esphome/components/wifi/wifi_component.h @@ -40,11 +40,6 @@ #include #include -#if defined(USE_ESP8266) && USE_ARDUINO_VERSION_CODE < VERSION_CODE(2, 4, 0) -extern "C" { -#include -}; -#endif #endif #ifdef USE_RP2 diff --git a/esphome/components/wifi/wifi_component_esp8266.cpp b/esphome/components/wifi/wifi_component_esp8266.cpp index b4a91fb3cd..031da1b355 100644 --- a/esphome/components/wifi/wifi_component_esp8266.cpp +++ b/esphome/components/wifi/wifi_component_esp8266.cpp @@ -21,7 +21,6 @@ extern "C" { #include "lwip/apps/sntp.h" #include "lwip/netif.h" // struct netif #include -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0) #include "LwipDhcpServer.h" #if USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) #include @@ -30,7 +29,6 @@ extern "C" { #define wifi_softap_set_dhcps_lease_time(time) dhcpSoftAP.set_dhcps_lease_time(time) #define wifi_softap_set_dhcps_offer_option(offer, mode) dhcpSoftAP.set_dhcps_offer_option(offer, mode) #endif -#endif } #include "esphome/core/application.h" @@ -293,7 +291,6 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { conf.bssid_set = 0; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) if (ap.password_.empty()) { conf.threshold.authmode = AUTH_OPEN; } else { @@ -310,7 +307,6 @@ bool WiFiComponent::wifi_sta_connect_(const WiFiAP &ap) { } } conf.threshold.rssi = -127; -#endif ETS_UART_INTR_DISABLE(); bool ret = wifi_station_set_config_current(&conf); @@ -602,7 +598,6 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { #endif break; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) case EVENT_OPMODE_CHANGED: { auto it = event->event_info.opmode_changed; ESP_LOGV(TAG, "Changed Mode old=%s new=%s", LOG_STR_ARG(get_op_mode_str(it.old_opmode)), @@ -620,7 +615,6 @@ void WiFiComponent::wifi_event_callback(System_Event_t *event) { #endif break; } -#endif default: break; } @@ -705,7 +699,6 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { config.bssid = nullptr; config.channel = 0; config.show_hidden = 1; -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 4, 0) config.scan_type = passive ? WIFI_SCAN_TYPE_PASSIVE : WIFI_SCAN_TYPE_ACTIVE; // Use shorter dwell times for roaming scans - we only need to detect strong // nearby APs, not do a thorough survey. This also reduces off-channel time @@ -724,7 +717,6 @@ bool WiFiComponent::wifi_scan_start_(bool passive) { config.scan_time.active.min = roaming ? SCAN_ACTIVE_MIN_ROAMING_MS : SCAN_ACTIVE_MIN_DEFAULT_MS; config.scan_time.active.max = roaming ? SCAN_ACTIVE_MAX_ROAMING_MS : SCAN_ACTIVE_MAX_DEFAULT_MS; } -#endif bool ret = wifi_station_scan(&config, &WiFiComponent::s_wifi_scan_done_callback); if (!ret) { ESP_LOGV(TAG, "wifi_station_scan failed"); @@ -830,7 +822,7 @@ bool WiFiComponent::wifi_ap_ip_config_(const optional &manual_ip) { return false; } -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(3, 0, 0) && USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) +#if USE_ARDUINO_VERSION_CODE < VERSION_CODE(3, 1, 0) dhcpSoftAP.begin(&info); #endif diff --git a/esphome/core/log.h b/esphome/core/log.h index 272e516808..14d24412ef 100644 --- a/esphome/core/log.h +++ b/esphome/core/log.h @@ -18,7 +18,6 @@ #ifdef USE_STORE_LOG_STR_IN_FLASH #include "WString.h" -#include "esphome/core/defines.h" // for USE_ARDUINO_VERSION_CODE #endif // Include ESP-IDF/Arduino based logging methods here so they don't undefine ours later @@ -177,20 +176,7 @@ struct LogString; #include -#if USE_ARDUINO_VERSION_CODE >= VERSION_CODE(2, 5, 0) #define LOG_STR_ARG(s) ((PGM_P) (s)) -#else -// Pre-Arduino 2.5, we can't pass a PSTR() to printf(). Emulate support by copying the message to a -// local buffer first. String length is limited to 63 characters. -// https://github.com/esp8266/Arduino/commit/6280e98b0360f85fdac2b8f10707fffb4f6e6e31 -#define LOG_STR_ARG(s) \ - ({ \ - char __buf[64]; \ - __buf[63] = '\0'; \ - strncpy_P(__buf, (PGM_P) (s), 63); \ - __buf; \ - }) -#endif #define LOG_STR(s) (reinterpret_cast(PSTR(s))) #define LOG_STR_LITERAL(s) LOG_STR_ARG(LOG_STR(s)) diff --git a/tests/unit_tests/components/esp8266/test_boards.py b/tests/unit_tests/components/esp8266/test_boards.py index df0e536d42..78213a762a 100644 --- a/tests/unit_tests/components/esp8266/test_boards.py +++ b/tests/unit_tests/components/esp8266/test_boards.py @@ -1,11 +1,7 @@ """Tests for the per-board linker-script rule.""" -import pytest - from esphome.components.esp8266 import _choose_ld_script from esphome.components.esp8266.boards import BOARDS, board_ld_script -import esphome.config_validation as cv -from esphome.core import EsphomeError def test_d1_wroom_02_keeps_its_shipped_layout() -> None: @@ -21,13 +17,6 @@ def test_default_boards_use_the_flash_size_layout() -> None: def test_choose_ld_script_paths() -> None: - """Old cores get the size default, overriding boards hard-error there - (a substituted layout would wipe flash-backed state), modern cores - honor the override.""" - assert _choose_ld_script("nodemcuv2", cv.Version(2, 3, 0)) is None - assert _choose_ld_script("nodemcuv2", cv.Version(2, 4, 2)) == "eagle.flash.4m.ld" - assert _choose_ld_script("d1_wroom_02", cv.Version(2, 7, 4)) == ( - "eagle.flash.2m64.ld" - ) - with pytest.raises(EsphomeError, match="cannot honor"): - _choose_ld_script("d1_wroom_02", cv.Version(2, 4, 2)) + """Default boards get the size layout, overriding boards keep theirs.""" + assert _choose_ld_script("nodemcuv2") == "eagle.flash.4m.ld" + assert _choose_ld_script("d1_wroom_02") == "eagle.flash.2m64.ld" diff --git a/tests/unit_tests/components/esp8266/test_framework_version.py b/tests/unit_tests/components/esp8266/test_framework_version.py new file mode 100644 index 0000000000..0107aff8dd --- /dev/null +++ b/tests/unit_tests/components/esp8266/test_framework_version.py @@ -0,0 +1,23 @@ +"""Tests for the Arduino framework version floor.""" + +import pytest + +from esphome.components.esp8266 import _arduino_check_versions +import esphome.config_validation as cv +from esphome.const import CONF_PLATFORM_VERSION, CONF_VERSION + + +def test_versions_before_3_are_rejected() -> None: + with pytest.raises(cv.Invalid, match="no longer supported") as excinfo: + _arduino_check_versions({CONF_VERSION: "2.7.4"}) + assert excinfo.value.path == [CONF_VERSION] + + +def test_supported_versions_pass() -> None: + value = _arduino_check_versions({CONF_VERSION: "3.0.2"}) + assert value[CONF_VERSION] == "3.0.2" + assert "espressif8266@3.2.0" in value[CONF_PLATFORM_VERSION] + + value = _arduino_check_versions({CONF_VERSION: "recommended"}) + assert value[CONF_VERSION] == "3.1.2" + assert "espressif8266@4.2.1" in value[CONF_PLATFORM_VERSION] diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index bd0a620e10..9f415344ae 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -21,17 +21,12 @@ def _build_path(tmp_path: Path) -> None: def test_framework_package_version() -> None: assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0" assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0" - # 2.6.3+ cores use the same package-major-3 encoding (PlatformIO path) - assert framework.framework_package_version(cv.Version(2, 7, 4)) == "3.20704.0" # A future major bump needs its own encoding, not a doomed registry lookup with pytest.raises(EsphomeError, match="not supported yet"): framework.framework_package_version(cv.Version(4, 0, 0)) - # The boundary matches the PlatformIO era guard; a 2.6.2 pre-release - # keeps this encoding - with pytest.raises(EsphomeError, match="older package encoding"): - framework.framework_package_version(cv.Version(2, 6, 2)) - assert framework.framework_package_version(cv.Version(2, 6, 2, "b1")) == "3.20602.0" - assert framework.framework_package_version(cv.Version(2, 6, 3)) == "3.20603.0" + # Cores before 3.x cannot build ESPHome (C++20) and are rejected + with pytest.raises(EsphomeError, match="requires core 3"): + framework.framework_package_version(cv.Version(2, 7, 4)) def test_format_framework_arduino_version_pins_all_series() -> None: @@ -39,10 +34,10 @@ def test_format_framework_arduino_version_pins_all_series() -> None: era, including the 4.x rejection it now shares with the installer.""" from esphome.components.esp8266 import _format_framework_arduino_version as fmt - assert fmt(cv.Version(2, 4, 1)) == "~1.20401.0" - assert fmt(cv.Version(2, 6, 2)) == "~2.20602.0" - assert fmt(cv.Version(2, 7, 4)) == "~3.20704.0" assert fmt(cv.Version(3, 1, 2)) == "~3.30102.0" + # Pre-3 cores are rejected with the version line anchored + with pytest.raises(cv.Invalid, match="requires core 3"): + fmt(cv.Version(2, 7, 4)) # Anchored to the framework version line, not a bare EsphomeError with pytest.raises(cv.Invalid, match="not supported yet") as excinfo: fmt(cv.Version(4, 0, 0)) From cb0c2bdaca67440249b415f4bb831c3906b83bbe Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Sat, 5 Sep 2026 08:38:55 +1200 Subject: [PATCH 114/147] [esp32_ble] Reference count BLE advertising (#18943) --- esphome/components/esp32_ble/ble.cpp | 35 +++++++++++++++---- esphome/components/esp32_ble/ble.h | 13 +++++++ .../esp32_ble_beacon/esp32_ble_beacon.cpp | 2 ++ .../components/esp32_ble_server/__init__.py | 12 +++++++ .../esp32_ble_server/ble_server.cpp | 21 +++++++++-- .../components/esp32_ble_server/ble_server.h | 11 ++++++ .../esp32_improv/esp32_improv_component.cpp | 20 ++++++++++- .../esp32_improv/esp32_improv_component.h | 3 ++ .../esp32_ble_server/config/improv_only.yaml | 13 +++++++ .../config/manufacturer_data_only.yaml | 9 +++++ .../esp32_ble_server/config/own_service.yaml | 14 ++++++++ .../esp32_ble_server/test_esp32_ble_server.py | 28 +++++++++++++++ 12 files changed, 171 insertions(+), 10 deletions(-) create mode 100644 tests/component_tests/esp32_ble_server/config/improv_only.yaml create mode 100644 tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml create mode 100644 tests/component_tests/esp32_ble_server/config/own_service.yaml diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index 6e6fb0e30d..fc95760cf8 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -100,21 +100,38 @@ void ESP32BLE::disable() { #ifdef USE_ESP32_BLE_ADVERTISING void ESP32BLE::advertising_start() { this->advertising_init_(); - if (!this->is_active()) + this->advertising_ref_count_++; + this->advertising_refresh(); +} + +void ESP32BLE::advertising_stop() { + if (this->advertising_ref_count_ == 0) return; - this->advertising_->start(); + this->advertising_ref_count_--; + this->advertising_refresh(); +} + +void ESP32BLE::advertising_refresh() { + if (this->advertising_ == nullptr || !this->is_active()) + return; + // Advertise while any component still needs it, otherwise stop + if (this->advertising_ref_count_ == 0) { + this->advertising_->stop(); + } else { + this->advertising_->start(); + } } void ESP32BLE::advertising_set_service_data(const std::vector &data) { this->advertising_init_(); this->advertising_->set_service_data(data); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_set_manufacturer_data(const std::vector &data) { this->advertising_init_(); this->advertising_->set_manufacturer_data(data); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_set_service_data_and_name(std::span data, bool include_name) { @@ -136,7 +153,7 @@ void ESP32BLE::advertising_set_service_data_and_name(std::span da this->advertising_->set_service_data(data); } - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_register_raw_advertisement_callback(std::function &&callback) { @@ -147,13 +164,13 @@ void ESP32BLE::advertising_register_raw_advertisement_callback(std::functionadvertising_init_(); this->advertising_->add_service_uuid(uuid); - this->advertising_start(); + this->advertising_refresh(); } void ESP32BLE::advertising_remove_service_uuid(ESPBTUUID uuid) { this->advertising_init_(); this->advertising_->remove_service_uuid(uuid); - this->advertising_start(); + this->advertising_refresh(); } #endif @@ -575,6 +592,10 @@ void ESP32BLE::loop_handle_state_transition_not_active_() { } this->state_ = BLE_COMPONENT_STATE_ACTIVE; +#ifdef USE_ESP32_BLE_ADVERTISING + // Requests made before the stack was up (or before it was re-enabled) take effect now + this->advertising_refresh(); +#endif } } diff --git a/esphome/components/esp32_ble/ble.h b/esphome/components/esp32_ble/ble.h index 2a355a6c8b..7d2d0438a4 100644 --- a/esphome/components/esp32_ble/ble.h +++ b/esphome/components/esp32_ble/ble.h @@ -114,7 +114,17 @@ class ESP32BLE final : public Component { void set_name(const char *name) { this->name_ = name; } #ifdef USE_ESP32_BLE_ADVERTISING + /** Request advertising on behalf of a component. + * + * Requests are reference counted: advertising runs until every component that called + * advertising_start() has released it again with advertising_stop(). Each component must + * pair its calls, so nothing advertises until something actually asks for it. + */ void advertising_start(); + /// Release a request made with advertising_start(); advertising stops at the last release. + void advertising_stop(); + /// Apply the current payload and request count: advertise while requested, otherwise stop. + void advertising_refresh(); void advertising_set_service_data(const std::vector &data); void advertising_set_manufacturer_data(const std::vector &data); void advertising_set_appearance(uint16_t appearance) { this->appearance_ = appearance; } @@ -226,6 +236,9 @@ class ESP32BLE final : public Component { // 1-byte aligned members (grouped together to minimize padding) BLEComponentState state_{BLE_COMPONENT_STATE_OFF}; // 1 byte (uint8_t enum) bool enable_on_boot_{}; // 1 byte +#ifdef USE_ESP32_BLE_ADVERTISING + uint8_t advertising_ref_count_{0}; // 1 byte, number of components requesting advertising +#endif #ifdef ESPHOME_ESP32_BLE_EXTENDED_AUTH_PARAMS optional auth_req_mode_; diff --git a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp index 9f1723430b..ab728f9f6f 100644 --- a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp +++ b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp @@ -67,6 +67,8 @@ void ESP32BLEBeacon::setup() { this->on_advertise_(); } }); + // A beacon always needs the device to advertise, and never releases the request + global_ble->advertising_start(); } void ESP32BLEBeacon::on_advertise_() { diff --git a/esphome/components/esp32_ble_server/__init__.py b/esphome/components/esp32_ble_server/__init__.py index 855a3be29b..d8095cd702 100644 --- a/esphome/components/esp32_ble_server/__init__.py +++ b/esphome/components/esp32_ble_server/__init__.py @@ -596,6 +596,18 @@ async def to_code(config): cg.add(var.set_parent(parent)) cg.add(parent.advertising_set_appearance(config[CONF_APPEARANCE])) cg.add(var.set_max_clients(config[CONF_MAX_CLIENTS])) + # Only advertise for the server itself when the configuration gives clients something to + # find. A server that is auto-loaded purely to host a runtime service (esp32_improv) stays + # silent until that service asks for advertising. + cg.add( + var.set_advertising_required( + CONF_MANUFACTURER_DATA in config + or any( + not uuid_is(service_config[CONF_UUID], DEVICE_INFORMATION_SERVICE_UUID) + for service_config in config[CONF_SERVICES] + ) + ) + ) if CONF_MANUFACTURER_DATA in config: cg.add(var.set_manufacturer_data(config[CONF_MANUFACTURER_DATA])) for service_config in config[CONF_SERVICES]: diff --git a/esphome/components/esp32_ble_server/ble_server.cpp b/esphome/components/esp32_ble_server/ble_server.cpp index 2dea1666bb..45679b9b98 100644 --- a/esphome/components/esp32_ble_server/ble_server.cpp +++ b/esphome/components/esp32_ble_server/ble_server.cpp @@ -81,6 +81,7 @@ void BLEServer::loop() { if (this->device_information_service_->is_running()) { this->state_ = RUNNING; this->restart_advertising_(); + this->request_advertising_(); ESP_LOGD(TAG, "BLE server setup successfully"); } else if (this->device_information_service_->is_created()) { this->device_information_service_->start(); @@ -98,6 +99,20 @@ void BLEServer::restart_advertising_() { } } +void BLEServer::request_advertising_() { + if (!this->advertising_required_ || this->advertising_requested_) + return; + this->advertising_requested_ = true; + this->parent_->advertising_start(); +} + +void BLEServer::release_advertising_() { + if (!this->advertising_requested_) + return; + this->advertising_requested_ = false; + this->parent_->advertising_stop(); +} + BLEService *BLEServer::create_service(ESPBTUUID uuid, bool advertise, uint16_t num_handles) { #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE char uuid_buf[esp32_ble::UUID_STR_LEN]; @@ -170,7 +185,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga this->add_client_(param->connect.conn_id); // Resume advertising so additional clients can discover and connect if (this->client_count_ < this->max_clients_) { - this->parent_->advertising_start(); + this->parent_->advertising_refresh(); } this->dispatch_callbacks_(CallbackType::ON_CONNECT, param->connect.conn_id); break; @@ -178,7 +193,7 @@ void BLEServer::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt_if_t ga case ESP_GATTS_DISCONNECT_EVT: { ESP_LOGD(TAG, "BLE Client disconnected"); this->remove_client_(param->disconnect.conn_id); - this->parent_->advertising_start(); + this->parent_->advertising_refresh(); this->dispatch_callbacks_(CallbackType::ON_DISCONNECT, param->disconnect.conn_id); break; } @@ -226,6 +241,8 @@ void BLEServer::remove_client_(uint16_t conn_id) { } void BLEServer::ble_before_disabled_event_handler() { + // Advertising is re-requested once the server is running again after BLE is re-enabled + this->release_advertising_(); // Delete all clients this->client_count_ = 0; // Delete all services diff --git a/esphome/components/esp32_ble_server/ble_server.h b/esphome/components/esp32_ble_server/ble_server.h index fdd92812cd..7869c73cc5 100644 --- a/esphome/components/esp32_ble_server/ble_server.h +++ b/esphome/components/esp32_ble_server/ble_server.h @@ -38,6 +38,13 @@ class BLEServer final : public Component, public Parented { this->restart_advertising_(); } + /** Whether this server needs the device to advertise so clients can find and connect to it. + * + * False for a server that only hosts services created at runtime (e.g. esp32_improv), which + * request advertising themselves for as long as they need it. + */ + void set_advertising_required(bool required) { this->advertising_required_ = required; } + void set_max_clients(uint8_t max_clients) { this->max_clients_ = max_clients; } uint8_t get_max_clients() const { return this->max_clients_; } @@ -82,6 +89,8 @@ class BLEServer final : public Component, public Parented { }; void restart_advertising_(); + void request_advertising_(); + void release_advertising_(); int8_t find_client_index_(uint16_t conn_id) const; void add_client_(uint16_t conn_id); @@ -93,6 +102,8 @@ class BLEServer final : public Component, public Parented { std::vector manufacturer_data_{}; esp_gatt_if_t gatts_if_{0}; bool registered_{false}; + bool advertising_required_{true}; + bool advertising_requested_{false}; uint16_t clients_[USE_ESP32_BLE_MAX_CONNECTIONS]{}; uint8_t client_count_{0}; diff --git a/esphome/components/esp32_improv/esp32_improv_component.cpp b/esphome/components/esp32_improv/esp32_improv_component.cpp index 4756fba637..9ec6eb7bab 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.cpp +++ b/esphome/components/esp32_improv/esp32_improv_component.cpp @@ -112,6 +112,7 @@ void ESP32ImprovComponent::loop() { this->state_callback_.call(this->state_, this->error_state_); #endif } + this->release_advertising_(); this->incoming_data_.clear(); return; } @@ -143,8 +144,9 @@ void ESP32ImprovComponent::loop() { ESP_LOGV(TAG, "Starting with device name advertising"); this->advertising_device_name_ = true; this->last_name_adv_time_ = App.get_loop_component_start_time(); + // Set the payload before requesting, so advertising starts exactly once esp32_ble::global_ble->advertising_set_service_data_and_name(std::span{}, true); - esp32_ble::global_ble->advertising_start(); + this->request_advertising_(); // Set initial state based on whether we have an authorizer this->set_state_(this->get_initial_state_(), false); @@ -326,6 +328,8 @@ void ESP32ImprovComponent::stop() { this->set_timeout("end-service", STOP_ADVERTISING_DELAY, [this] { if (this->state_ == improv::STATE_STOPPED || this->service_ == nullptr) return; + // Release first so removing the service UUID does not restart advertising on the way out + this->release_advertising_(); this->service_->stop(); this->set_state_(improv::STATE_STOPPED); }); @@ -520,6 +524,20 @@ void ESP32ImprovComponent::update_advertising_type_() { } } +void ESP32ImprovComponent::request_advertising_() { + if (this->advertising_requested_) + return; + this->advertising_requested_ = true; + esp32_ble::global_ble->advertising_start(); +} + +void ESP32ImprovComponent::release_advertising_() { + if (!this->advertising_requested_) + return; + this->advertising_requested_ = false; + esp32_ble::global_ble->advertising_stop(); +} + improv::State ESP32ImprovComponent::get_initial_state_() const { #ifdef USE_BINARY_SENSOR // If we have an authorizer, start in awaiting authorization state diff --git a/esphome/components/esp32_improv/esp32_improv_component.h b/esphome/components/esp32_improv/esp32_improv_component.h index 414948c977..a40d60552a 100644 --- a/esphome/components/esp32_improv/esp32_improv_component.h +++ b/esphome/components/esp32_improv/esp32_improv_component.h @@ -104,8 +104,11 @@ class ESP32ImprovComponent final : public Component, public improv_base::ImprovB bool status_indicator_state_{false}; uint32_t last_name_adv_time_{0}; bool advertising_device_name_{false}; + bool advertising_requested_{false}; void set_status_indicator_state_(bool state); void update_advertising_type_(); + void request_advertising_(); + void release_advertising_(); void set_state_(improv::State state, bool update_advertising = true); void set_error_(improv::Error error); diff --git a/tests/component_tests/esp32_ble_server/config/improv_only.yaml b/tests/component_tests/esp32_ble_server/config/improv_only.yaml new file mode 100644 index 0000000000..8a5c3ba638 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/improv_only.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + variant: esp32 + +wifi: + ssid: MySSID + password: password1 + +# esp32_ble_server is only auto-loaded here, so it has no services of its own. +esp32_improv: + authorizer: none diff --git a/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml b/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml new file mode 100644 index 0000000000..b7bdae4af7 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/manufacturer_data_only.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + variant: esp32 + +esp32_ble_server: + id: ble_server + manufacturer_data: [0x72, 0x04, 0x00, 0x23] diff --git a/tests/component_tests/esp32_ble_server/config/own_service.yaml b/tests/component_tests/esp32_ble_server/config/own_service.yaml new file mode 100644 index 0000000000..c7ef0287b0 --- /dev/null +++ b/tests/component_tests/esp32_ble_server/config/own_service.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp32: + variant: esp32 + +esp32_ble_server: + id: ble_server + services: + - uuid: 2a24b789-7aab-4535-af3e-ee76a35cc12d + characteristics: + - uuid: cad48e28-7fbe-41cf-bae9-d77a6c233423 + read: true + value: [1, 2, 3, 4] diff --git a/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py b/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py index 88307d0dcf..4b7ab79a81 100644 --- a/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py +++ b/tests/component_tests/esp32_ble_server/test_esp32_ble_server.py @@ -1,5 +1,10 @@ """Tests for esp32_ble_server configuration helpers.""" +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + import pytest from esphome.components.esp32_ble_server import ( @@ -45,3 +50,26 @@ def test_uuid_is_matches_descriptor_short_strings(uuid16) -> None: assert uuid_is(uuid16, uuid16) assert uuid_is(f"{uuid16:04X}", uuid16) assert uuid_is(f"{uuid16:08X}", uuid16) + + +@pytest.mark.parametrize( + ("config_file", "required"), + [ + # Auto-loaded by esp32_improv only: nothing to find until Improv asks for it + ("improv_only.yaml", False), + # The configuration defines a service clients are meant to connect to + ("own_service.yaml", True), + # Manufacturer data is only useful if it is actually broadcast + ("manufacturer_data_only.yaml", True), + ], +) +def test_advertising_required( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + required: bool, +) -> None: + """The server only requests advertising when the configuration needs it.""" + main_cpp = generate_main(component_config_path(config_file)) + + assert f"set_advertising_required({str(required).lower()})" in main_cpp From e36445fa5feb4db65586186ec823a225339b8885 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Sat, 5 Sep 2026 06:00:25 -0500 Subject: [PATCH 115/147] [usb_uart] Keep the comm interface number valid when its claim fails (#18968) --- esphome/components/usb_uart/usb_uart.cpp | 12 +++++++----- esphome/components/usb_uart/usb_uart.h | 3 +++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/esphome/components/usb_uart/usb_uart.cpp b/esphome/components/usb_uart/usb_uart.cpp index cf66e4c369..60b7fe4e9c 100644 --- a/esphome/components/usb_uart/usb_uart.cpp +++ b/esphome/components/usb_uart/usb_uart.cpp @@ -434,11 +434,12 @@ void USBUartTypeCdcAcm::on_connected() { auto err_comm = usb_host_interface_claim(this->handle_, this->device_handle_, channel->cdc_dev_.interrupt_interface_number, 0); if (err_comm != ESP_OK) { + // Continue anyway: the interface number stays valid for CDC request addressing ESP_LOGW(TAG, "Could not claim comm interface %d: %s", channel->cdc_dev_.interrupt_interface_number, esp_err_to_name(err_comm)); - channel->cdc_dev_.interrupt_interface_number = 0xFF; // Mark as unavailable, but continue anyway } else { ESP_LOGD(TAG, "Claimed comm interface %d", channel->cdc_dev_.interrupt_interface_number); + channel->cdc_dev_.interrupt_interface_claimed = true; } } auto err = @@ -465,14 +466,15 @@ void USBUartTypeCdcAcm::on_disconnected() { usb_host_endpoint_halt(this->device_handle_, channel->cdc_dev_.out_ep->bEndpointAddress); usb_host_endpoint_flush(this->device_handle_, channel->cdc_dev_.out_ep->bEndpointAddress); } - if (channel->cdc_dev_.notify_ep != nullptr) { + // Only tear down the notify pipe when we claimed its interface ourselves; + // no transfer is ever submitted on it, so there is nothing else to cancel. + if (channel->cdc_dev_.notify_ep != nullptr && channel->cdc_dev_.interrupt_interface_claimed) { usb_host_endpoint_halt(this->device_handle_, channel->cdc_dev_.notify_ep->bEndpointAddress); usb_host_endpoint_flush(this->device_handle_, channel->cdc_dev_.notify_ep->bEndpointAddress); } - if (channel->cdc_dev_.interrupt_interface_number != 0xFF && - channel->cdc_dev_.interrupt_interface_number != channel->cdc_dev_.bulk_interface_number) { + if (channel->cdc_dev_.interrupt_interface_claimed) { usb_host_interface_release(this->handle_, this->device_handle_, channel->cdc_dev_.interrupt_interface_number); - channel->cdc_dev_.interrupt_interface_number = 0xFF; + channel->cdc_dev_.interrupt_interface_claimed = false; } usb_host_interface_release(this->handle_, this->device_handle_, channel->cdc_dev_.bulk_interface_number); // Reset the input and output started flags to their initial state to avoid the possibility of spurious restarts diff --git a/esphome/components/usb_uart/usb_uart.h b/esphome/components/usb_uart/usb_uart.h index 00b34fb942..9d87bf964c 100644 --- a/esphome/components/usb_uart/usb_uart.h +++ b/esphome/components/usb_uart/usb_uart.h @@ -34,7 +34,10 @@ struct CdcEps { const usb_ep_desc_t *in_ep; const usb_ep_desc_t *out_ep; uint8_t bulk_interface_number; + // Also the wIndex target for CDC class requests (SET_LINE_CODING etc.), so it + // must remain valid even when the interface itself is not claimed. uint8_t interrupt_interface_number; + bool interrupt_interface_claimed{false}; }; enum CH34xChipType : uint8_t { From 745eb3010910a400c14527d0dd8e1fd5fa7ac984 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:07:15 +0200 Subject: [PATCH 116/147] Bump bundled esphome-device-builder to 1.14.1 (#18981) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7952616496..2d4ddbef5d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.1 RUN \ platformio settings set enable_telemetry No \ From 7089dae3b63f57db6435202c14668001d0f0b595 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:25:58 +0000 Subject: [PATCH 117/147] Bump bundled esphome-device-builder to 1.14.2 (#18988) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2d4ddbef5d..b5170864a3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.2 RUN \ platformio settings set enable_telemetry No \ From 011497d6eeed511d55ec556db334f154f9dfc514 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 09:28:02 +0200 Subject: [PATCH 118/147] Bump bundled esphome-device-builder to 1.14.3 (#18996) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b5170864a3..e875851bfb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.2 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.3 RUN \ platformio settings set enable_telemetry No \ From 18220e0b3940727d2f0657247ba0b749403b9db4 Mon Sep 17 00:00:00 2001 From: Ricardo Sanz Date: Sun, 6 Sep 2026 23:03:07 +0200 Subject: [PATCH 119/147] [climate][template] New template climate component (#14455) --- esphome/components/climate/__init__.py | 13 + .../components/template/climate/__init__.py | 465 ++++++++++++++++++ .../components/template/climate/automation.h | 57 +++ .../template/climate/template_climate.cpp | 164 ++++++ .../template/climate/template_climate.h | 92 ++++ esphome/config_validation.py | 1 + .../template/test_template_climate.py | 145 ++++++ tests/components/climate/common.yaml | 3 +- tests/components/template/common-base.yaml | 113 +++++ .../fixtures/template_climate_basic.yaml | 72 +++ .../template_climate_custom_modes.yaml | 47 ++ .../template_climate_nonoptimistic.yaml | 56 +++ .../template_climate_on_control_ordering.yaml | 26 + .../template_climate_publish_all_fields.yaml | 63 +++ .../template_climate_sensor_push.yaml | 49 ++ .../template_climate_set_actions.yaml | 89 ++++ ...emplate_climate_two_point_temperature.yaml | 52 ++ .../test_template_climate_basic.py | 146 ++++++ .../test_template_climate_custom_modes.py | 98 ++++ .../test_template_climate_nonoptimistic.py | 107 ++++ ...st_template_climate_on_control_ordering.py | 83 ++++ ...est_template_climate_publish_all_fields.py | 96 ++++ .../test_template_climate_sensor_push.py | 88 ++++ .../test_template_climate_set_actions.py | 114 +++++ ..._template_climate_two_point_temperature.py | 118 +++++ 25 files changed, 2355 insertions(+), 2 deletions(-) create mode 100644 esphome/components/template/climate/__init__.py create mode 100644 esphome/components/template/climate/automation.h create mode 100644 esphome/components/template/climate/template_climate.cpp create mode 100644 esphome/components/template/climate/template_climate.h create mode 100644 tests/component_tests/template/test_template_climate.py create mode 100644 tests/integration/fixtures/template_climate_basic.yaml create mode 100644 tests/integration/fixtures/template_climate_custom_modes.yaml create mode 100644 tests/integration/fixtures/template_climate_nonoptimistic.yaml create mode 100644 tests/integration/fixtures/template_climate_on_control_ordering.yaml create mode 100644 tests/integration/fixtures/template_climate_publish_all_fields.yaml create mode 100644 tests/integration/fixtures/template_climate_sensor_push.yaml create mode 100644 tests/integration/fixtures/template_climate_set_actions.yaml create mode 100644 tests/integration/fixtures/template_climate_two_point_temperature.yaml create mode 100644 tests/integration/test_template_climate_basic.py create mode 100644 tests/integration/test_template_climate_custom_modes.py create mode 100644 tests/integration/test_template_climate_nonoptimistic.py create mode 100644 tests/integration/test_template_climate_on_control_ordering.py create mode 100644 tests/integration/test_template_climate_publish_all_fields.py create mode 100644 tests/integration/test_template_climate_sensor_push.py create mode 100644 tests/integration/test_template_climate_set_actions.py create mode 100644 tests/integration/test_template_climate_two_point_temperature.py diff --git a/esphome/components/climate/__init__.py b/esphome/components/climate/__init__.py index 80dd913fba..3fbca1a6d0 100644 --- a/esphome/components/climate/__init__.py +++ b/esphome/components/climate/__init__.py @@ -125,6 +125,19 @@ CLIMATE_SWING_MODES = { validate_climate_swing_mode = cv.enum(CLIMATE_SWING_MODES, upper=True) +ClimateAction = climate_ns.enum("ClimateAction") +CLIMATE_ACTIONS = { + "OFF": ClimateAction.CLIMATE_ACTION_OFF, + "COOLING": ClimateAction.CLIMATE_ACTION_COOLING, + "HEATING": ClimateAction.CLIMATE_ACTION_HEATING, + "IDLE": ClimateAction.CLIMATE_ACTION_IDLE, + "DRYING": ClimateAction.CLIMATE_ACTION_DRYING, + "FAN": ClimateAction.CLIMATE_ACTION_FAN, + "DEFROSTING": ClimateAction.CLIMATE_ACTION_DEFROSTING, +} + +validate_climate_action = cv.enum(CLIMATE_ACTIONS, upper=True) + CONF_MIN_HUMIDITY = "min_humidity" CONF_MAX_HUMIDITY = "max_humidity" CONF_TARGET_HUMIDITY = "target_humidity" diff --git a/esphome/components/template/climate/__init__.py b/esphome/components/template/climate/__init__.py new file mode 100644 index 0000000000..c39ea8f80e --- /dev/null +++ b/esphome/components/template/climate/__init__.py @@ -0,0 +1,465 @@ +from esphome import automation +import esphome.codegen as cg +from esphome.components import climate, sensor +from esphome.components.climate import climate_ns +import esphome.config_validation as cv +from esphome.const import ( + CONF_ACTION, + CONF_CURRENT_TEMPERATURE, + CONF_CUSTOM_FAN_MODE, + CONF_CUSTOM_FAN_MODES, + CONF_CUSTOM_PRESET, + CONF_CUSTOM_PRESETS, + CONF_FAN_MODE, + CONF_HUMIDITY_SENSOR, + CONF_ID, + CONF_INITIAL_STATE, + CONF_MODE, + CONF_OPTIMISTIC, + CONF_PRESET, + CONF_RESTORE_MODE, + CONF_SENSOR, + CONF_SUPPORTED_FAN_MODES, + CONF_SUPPORTED_MODES, + CONF_SUPPORTED_PRESETS, + CONF_SUPPORTED_SWING_MODES, + CONF_SWING_MODE, + CONF_TARGET_TEMPERATURE, + CONF_TARGET_TEMPERATURE_HIGH, + CONF_TARGET_TEMPERATURE_LOW, +) +from esphome.core import ID +from esphome.cpp_generator import MockObj, TemplateArgsType +from esphome.types import ConfigType + +from .. import template_ns + +CONF_CURRENT_HUMIDITY = "current_humidity" +CONF_TARGET_HUMIDITY = "target_humidity" +CONF_SUPPORTS_ACTION = "supports_action" +CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE = "supports_two_point_target_temperature" +CONF_SUPPORTS_TARGET_HUMIDITY = "supports_target_humidity" +CONF_SUPPORTS_CURRENT_TEMPERATURE = "supports_current_temperature" +CONF_SUPPORTS_CURRENT_HUMIDITY = "supports_current_humidity" +CONF_SET_MODE_ACTION = "set_mode_action" +CONF_SET_TARGET_TEMPERATURE_ACTION = "set_target_temperature_action" +CONF_SET_TARGET_TEMPERATURE_LOW_ACTION = "set_target_temperature_low_action" +CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION = "set_target_temperature_high_action" +CONF_SET_TARGET_HUMIDITY_ACTION = "set_target_humidity_action" +CONF_SET_FAN_MODE_ACTION = "set_fan_mode_action" +CONF_SET_CUSTOM_FAN_MODE_ACTION = "set_custom_fan_mode_action" +CONF_SET_SWING_MODE_ACTION = "set_swing_mode_action" +CONF_SET_PRESET_ACTION = "set_preset_action" +CONF_SET_CUSTOM_PRESET_ACTION = "set_custom_preset_action" + +TemplateClimate = template_ns.class_("TemplateClimate", climate.Climate, cg.Component) +TemplateClimatePublishAction = template_ns.class_( + "TemplateClimatePublishAction", + automation.Action, + cg.Parented.template(TemplateClimate), +) + +TemplateClimateRestoreMode = template_ns.enum( + "TemplateClimateRestoreMode", is_class=True +) +CLIMATE_RESTORE_MODES = { + "NO_RESTORE": TemplateClimateRestoreMode.TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE, + "RESTORE": TemplateClimateRestoreMode.TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE, +} + +# Per-field actions that forward a requested value on. The third item is the type of `x`. +SET_ACTIONS = ( + (CONF_SET_MODE_ACTION, "get_set_mode_trigger", climate.ClimateMode), + ( + CONF_SET_TARGET_TEMPERATURE_ACTION, + "get_set_target_temperature_trigger", + cg.float_, + ), + ( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + "get_set_target_temperature_low_trigger", + cg.float_, + ), + ( + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + "get_set_target_temperature_high_trigger", + cg.float_, + ), + (CONF_SET_TARGET_HUMIDITY_ACTION, "get_set_target_humidity_trigger", cg.float_), + (CONF_SET_FAN_MODE_ACTION, "get_set_fan_mode_trigger", climate.ClimateFanMode), + ( + CONF_SET_CUSTOM_FAN_MODE_ACTION, + "get_set_custom_fan_mode_trigger", + cg.StringRef, + ), + ( + CONF_SET_SWING_MODE_ACTION, + "get_set_swing_mode_trigger", + climate.ClimateSwingMode, + ), + (CONF_SET_PRESET_ACTION, "get_set_preset_trigger", climate.ClimatePreset), + (CONF_SET_CUSTOM_PRESET_ACTION, "get_set_custom_preset_trigger", cg.StringRef), +) + +# supports_* keys have no default so that an omitted key can mean "derive it from the sensor or +# set action that makes the trait useful", which is not expressible once a default fills it in. +DERIVED_SUPPORTS = ( + (CONF_SUPPORTS_CURRENT_TEMPERATURE, (CONF_SENSOR,)), + (CONF_SUPPORTS_CURRENT_HUMIDITY, (CONF_HUMIDITY_SENSOR,)), + ( + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + ( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + ), + ), + (CONF_SUPPORTS_TARGET_HUMIDITY, (CONF_SET_TARGET_HUMIDITY_ACTION,)), +) + + +# Custom fan modes/presets are opaque user-defined strings with no build-time correctness check +# elsewhere (Climate::set_supported_custom_fan_modes()/set_supported_custom_presets() don't block +# empty entries), so reject empty ones here -- they could never be selected at runtime anyway. +validate_custom_climate_string = cv.All(cv.string_strict, cv.Length(min=1)) + + +def _validate_two_point(config: ConfigType) -> ConfigType: + has_low = CONF_TARGET_TEMPERATURE_LOW in config + has_high = CONF_TARGET_TEMPERATURE_HIGH in config + if has_low != has_high: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE_LOW}' and '{CONF_TARGET_TEMPERATURE_HIGH}' must be used together" + ) + if (has_low or has_high) and CONF_TARGET_TEMPERATURE in config: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE}' cannot be used together with " + f"'{CONF_TARGET_TEMPERATURE_LOW}'/'{CONF_TARGET_TEMPERATURE_HIGH}'" + ) + return config + + +def _validate_set_actions(config: ConfigType) -> ConfigType: + has_low = CONF_SET_TARGET_TEMPERATURE_LOW_ACTION in config + has_high = CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION in config + if has_low != has_high: + raise cv.Invalid( + f"'{CONF_SET_TARGET_TEMPERATURE_LOW_ACTION}' and " + f"'{CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION}' must be used together" + ) + if (has_low or has_high) and CONF_SET_TARGET_TEMPERATURE_ACTION in config: + raise cv.Invalid( + f"'{CONF_SET_TARGET_TEMPERATURE_ACTION}' cannot be used together with " + f"'{CONF_SET_TARGET_TEMPERATURE_LOW_ACTION}'/'{CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION}'" + ) + return config + + +def _resolve_supports(config: ConfigType) -> ConfigType: + # An explicit true stays valid without either, since climate.template.publish can report the + # value; an explicit false that contradicts the configuration is an error, not a silent override. + for key, sources in DERIVED_SUPPORTS: + configured = [source for source in sources if source in config] + if key not in config: + config[key] = bool(configured) + elif not config[key] and configured: + raise cv.Invalid( + f"'{key}' cannot be false while '{configured[0]}' is configured", + path=[key], + ) + return config + + +def _validate_initial_state(config: ConfigType) -> ConfigType: + # Climate keeps target_temperature and target_temperature_low in a union, so writing the wrong + # one of the pair corrupts the setpoint with no runtime complaint. + if (initial_state := config.get(CONF_INITIAL_STATE)) is None: + return config + + two_point = config[CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE] + if two_point and CONF_TARGET_TEMPERATURE in initial_state: + raise cv.Invalid( + f"'{CONF_TARGET_TEMPERATURE}' is not available while " + f"'{CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE}' is enabled; use " + f"'{CONF_TARGET_TEMPERATURE_LOW}'/'{CONF_TARGET_TEMPERATURE_HIGH}' instead", + path=[CONF_INITIAL_STATE, CONF_TARGET_TEMPERATURE], + ) + if not two_point: + for key in (CONF_TARGET_TEMPERATURE_LOW, CONF_TARGET_TEMPERATURE_HIGH): + if key in initial_state: + raise cv.Invalid( + f"'{key}' requires '{CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE}' to be enabled", + path=[CONF_INITIAL_STATE, key], + ) + if ( + CONF_TARGET_HUMIDITY in initial_state + and not config[CONF_SUPPORTS_TARGET_HUMIDITY] + ): + raise cv.Invalid( + f"'{CONF_TARGET_HUMIDITY}' requires '{CONF_SUPPORTS_TARGET_HUMIDITY}' to be enabled", + path=[CONF_INITIAL_STATE, CONF_TARGET_HUMIDITY], + ) + return config + + +# Same settable fields as climate.template.publish, minus current_temperature/current_humidity/ +# action: those are reported values (from a sensor or the device), not meaningful static defaults. +INITIAL_STATE_SCHEMA = cv.All( + cv.Schema( + { + cv.Optional(CONF_MODE): climate.validate_climate_mode, + cv.Optional(CONF_TARGET_TEMPERATURE): cv.temperature, + cv.Optional(CONF_TARGET_TEMPERATURE_LOW): cv.temperature, + cv.Optional(CONF_TARGET_TEMPERATURE_HIGH): cv.temperature, + cv.Optional(CONF_TARGET_HUMIDITY): cv.percentage_int, + cv.Exclusive(CONF_FAN_MODE, "fan_mode"): climate.validate_climate_fan_mode, + cv.Exclusive( + CONF_CUSTOM_FAN_MODE, "fan_mode" + ): validate_custom_climate_string, + cv.Optional(CONF_SWING_MODE): climate.validate_climate_swing_mode, + cv.Exclusive(CONF_PRESET, "preset"): climate.validate_climate_preset, + cv.Exclusive(CONF_CUSTOM_PRESET, "preset"): validate_custom_climate_string, + } + ), + _validate_two_point, +) + +CONFIG_SCHEMA = cv.All( + climate.climate_schema(TemplateClimate) + .extend( + { + cv.Optional(CONF_SENSOR): cv.use_id(sensor.Sensor), + cv.Optional(CONF_HUMIDITY_SENSOR): cv.use_id(sensor.Sensor), + # action only ever arrives through climate.template.publish, so unlike the other + # supports_* keys there is no set action to derive it from. + cv.Optional(CONF_SUPPORTS_ACTION, default=False): cv.boolean, + cv.Optional(CONF_SUPPORTS_CURRENT_TEMPERATURE): cv.boolean, + cv.Optional(CONF_SUPPORTS_CURRENT_HUMIDITY): cv.boolean, + cv.Optional(CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE): cv.boolean, + cv.Optional(CONF_SUPPORTS_TARGET_HUMIDITY): cv.boolean, + cv.Required(CONF_SUPPORTED_MODES): cv.All( + cv.ensure_list(climate.validate_climate_mode), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_FAN_MODES): cv.All( + cv.ensure_list(climate.validate_climate_fan_mode), cv.Unique() + ), + cv.Optional(CONF_CUSTOM_FAN_MODES): cv.All( + cv.ensure_list(validate_custom_climate_string), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_SWING_MODES): cv.All( + cv.ensure_list(climate.validate_climate_swing_mode), cv.Unique() + ), + cv.Optional(CONF_SUPPORTED_PRESETS): cv.All( + cv.ensure_list(climate.validate_climate_preset), cv.Unique() + ), + cv.Optional(CONF_CUSTOM_PRESETS): cv.All( + cv.ensure_list(validate_custom_climate_string), cv.Unique() + ), + cv.Optional(CONF_OPTIMISTIC, default=True): cv.boolean, + cv.Optional(CONF_RESTORE_MODE, default="RESTORE"): cv.enum( + CLIMATE_RESTORE_MODES, upper=True + ), + cv.Optional(CONF_INITIAL_STATE): INITIAL_STATE_SCHEMA, + cv.Optional(CONF_SET_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION + ): automation.validate_automation(single=True), + cv.Optional( + CONF_SET_TARGET_HUMIDITY_ACTION + ): automation.validate_automation(single=True), + cv.Optional(CONF_SET_FAN_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional( + CONF_SET_CUSTOM_FAN_MODE_ACTION + ): automation.validate_automation(single=True), + cv.Optional(CONF_SET_SWING_MODE_ACTION): automation.validate_automation( + single=True + ), + cv.Optional(CONF_SET_PRESET_ACTION): automation.validate_automation( + single=True + ), + cv.Optional(CONF_SET_CUSTOM_PRESET_ACTION): automation.validate_automation( + single=True + ), + } + ) + .extend(cv.COMPONENT_SCHEMA), + _validate_set_actions, + _resolve_supports, + _validate_initial_state, +) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await climate.register_climate(var, config) + + if (sens := config.get(CONF_SENSOR)) is not None: + cg.add(var.set_sensor(await cg.get_variable(sens))) + + if (sens := config.get(CONF_HUMIDITY_SENSOR)) is not None: + cg.add(var.set_humidity_sensor(await cg.get_variable(sens))) + + for key, flag in ( + (CONF_SUPPORTS_ACTION, climate_ns.CLIMATE_SUPPORTS_ACTION), + ( + CONF_SUPPORTS_CURRENT_TEMPERATURE, + climate_ns.CLIMATE_SUPPORTS_CURRENT_TEMPERATURE, + ), + (CONF_SUPPORTS_CURRENT_HUMIDITY, climate_ns.CLIMATE_SUPPORTS_CURRENT_HUMIDITY), + ( + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + climate_ns.CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + ), + (CONF_SUPPORTS_TARGET_HUMIDITY, climate_ns.CLIMATE_SUPPORTS_TARGET_HUMIDITY), + ): + if config[key]: + cg.add(var.add_feature_flags(flag)) + + for mode in config[CONF_SUPPORTED_MODES]: + cg.add(var.add_supported_mode(mode)) + + for mode in config.get(CONF_SUPPORTED_FAN_MODES, []): + cg.add(var.add_supported_fan_mode(mode)) + + if CONF_CUSTOM_FAN_MODES in config: + cg.add( + var.set_supported_custom_fan_modes( + cg.ArrayInitializer(*config[CONF_CUSTOM_FAN_MODES]) + ) + ) + + for mode in config.get(CONF_SUPPORTED_SWING_MODES, []): + cg.add(var.add_supported_swing_mode(mode)) + + for preset in config.get(CONF_SUPPORTED_PRESETS, []): + cg.add(var.add_supported_preset(preset)) + + if CONF_CUSTOM_PRESETS in config: + cg.add( + var.set_supported_custom_presets( + cg.ArrayInitializer(*config[CONF_CUSTOM_PRESETS]) + ) + ) + + for key, trigger_getter, arg_type in SET_ACTIONS: + if (conf := config.get(key)) is not None: + await automation.build_automation( + getattr(var, trigger_getter)(), [(arg_type, "x")], conf + ) + + cg.add(var.set_optimistic(config[CONF_OPTIMISTIC])) + cg.add(var.set_restore_mode(config[CONF_RESTORE_MODE])) + + if (initial_state := config.get(CONF_INITIAL_STATE)) is not None: + if (v := initial_state.get(CONF_MODE)) is not None: + cg.add(var.set_mode(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE)) is not None: + cg.add(var.set_target_temperature(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE_LOW)) is not None: + cg.add(var.set_target_temperature_low(v)) + if (v := initial_state.get(CONF_TARGET_TEMPERATURE_HIGH)) is not None: + cg.add(var.set_target_temperature_high(v)) + if (v := initial_state.get(CONF_TARGET_HUMIDITY)) is not None: + cg.add(var.set_target_humidity(v)) + if (v := initial_state.get(CONF_FAN_MODE)) is not None: + cg.add(var.set_fan_mode(v)) + if (v := initial_state.get(CONF_CUSTOM_FAN_MODE)) is not None: + cg.add(var.set_custom_fan_mode(v)) + if (v := initial_state.get(CONF_SWING_MODE)) is not None: + cg.add(var.set_swing_mode(v)) + if (v := initial_state.get(CONF_PRESET)) is not None: + cg.add(var.set_preset(v)) + if (v := initial_state.get(CONF_CUSTOM_PRESET)) is not None: + cg.add(var.set_custom_preset(v)) + + +CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.use_id(TemplateClimate), + cv.Optional(CONF_CURRENT_TEMPERATURE): cv.templatable(cv.temperature), + cv.Optional(CONF_CURRENT_HUMIDITY): cv.templatable(cv.percentage_int), + cv.Optional(CONF_TARGET_TEMPERATURE): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_TEMPERATURE_LOW): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_TEMPERATURE_HIGH): cv.templatable(cv.temperature), + cv.Optional(CONF_TARGET_HUMIDITY): cv.templatable(cv.percentage_int), + cv.Optional(CONF_MODE): cv.templatable(climate.validate_climate_mode), + cv.Optional(CONF_ACTION): cv.templatable(climate.validate_climate_action), + cv.Exclusive(CONF_FAN_MODE, "fan_mode"): cv.templatable( + climate.validate_climate_fan_mode + ), + cv.Exclusive(CONF_CUSTOM_FAN_MODE, "fan_mode"): cv.templatable( + validate_custom_climate_string + ), + cv.Optional(CONF_SWING_MODE): cv.templatable( + climate.validate_climate_swing_mode + ), + cv.Exclusive(CONF_PRESET, "preset"): cv.templatable( + climate.validate_climate_preset + ), + cv.Exclusive(CONF_CUSTOM_PRESET, "preset"): cv.templatable( + validate_custom_climate_string + ), + } + ), + _validate_two_point, +) + + +@automation.register_action( + "climate.template.publish", + TemplateClimatePublishAction, + CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA, + synchronous=True, +) +async def climate_template_publish_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + + if (v := config.get(CONF_CURRENT_TEMPERATURE)) is not None: + cg.add(var.set_current_temperature(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_CURRENT_HUMIDITY)) is not None: + cg.add(var.set_current_humidity(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE)) is not None: + cg.add(var.set_target_temperature(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE_LOW)) is not None: + cg.add(var.set_target_temperature_low(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_TARGET_TEMPERATURE_HIGH)) is not None: + cg.add( + var.set_target_temperature_high(await cg.templatable(v, args, cg.float_)) + ) + if (v := config.get(CONF_TARGET_HUMIDITY)) is not None: + cg.add(var.set_target_humidity(await cg.templatable(v, args, cg.float_))) + if (v := config.get(CONF_MODE)) is not None: + cg.add(var.set_mode(await cg.templatable(v, args, climate.ClimateMode))) + if (v := config.get(CONF_ACTION)) is not None: + cg.add(var.set_action(await cg.templatable(v, args, climate.ClimateAction))) + if (v := config.get(CONF_FAN_MODE)) is not None: + cg.add(var.set_fan_mode(await cg.templatable(v, args, climate.ClimateFanMode))) + if (v := config.get(CONF_CUSTOM_FAN_MODE)) is not None: + cg.add(var.set_custom_fan_mode(await cg.templatable(v, args, cg.std_string))) + if (v := config.get(CONF_SWING_MODE)) is not None: + cg.add( + var.set_swing_mode(await cg.templatable(v, args, climate.ClimateSwingMode)) + ) + if (v := config.get(CONF_PRESET)) is not None: + cg.add(var.set_preset(await cg.templatable(v, args, climate.ClimatePreset))) + if (v := config.get(CONF_CUSTOM_PRESET)) is not None: + cg.add(var.set_custom_preset(await cg.templatable(v, args, cg.std_string))) + + return var diff --git a/esphome/components/template/climate/automation.h b/esphome/components/template/climate/automation.h new file mode 100644 index 0000000000..49a79ace2f --- /dev/null +++ b/esphome/components/template/climate/automation.h @@ -0,0 +1,57 @@ +#pragma once + +#include "template_climate.h" +#include "esphome/core/automation.h" + +namespace esphome::template_ { + +template +class TemplateClimatePublishAction final : public Action, public Parented { + public: + TEMPLATABLE_VALUE(float, current_temperature) + TEMPLATABLE_VALUE(float, current_humidity) + TEMPLATABLE_VALUE(float, target_temperature) + TEMPLATABLE_VALUE(float, target_temperature_low) + TEMPLATABLE_VALUE(float, target_temperature_high) + TEMPLATABLE_VALUE(float, target_humidity) + TEMPLATABLE_VALUE(climate::ClimateMode, mode) + TEMPLATABLE_VALUE(climate::ClimateAction, action) + TEMPLATABLE_VALUE(climate::ClimateFanMode, fan_mode) + TEMPLATABLE_VALUE(std::string, custom_fan_mode) + TEMPLATABLE_VALUE(climate::ClimateSwingMode, swing_mode) + TEMPLATABLE_VALUE(climate::ClimatePreset, preset) + TEMPLATABLE_VALUE(std::string, custom_preset) + + void play(const Ts &...x) override { + if (this->current_temperature_.has_value()) + this->parent_->current_temperature = this->current_temperature_.value(x...); + if (this->current_humidity_.has_value()) + this->parent_->current_humidity = this->current_humidity_.value(x...); + if (this->target_temperature_.has_value()) + this->parent_->set_target_temperature(this->target_temperature_.value(x...)); + if (this->target_temperature_low_.has_value()) + this->parent_->set_target_temperature_low(this->target_temperature_low_.value(x...)); + if (this->target_temperature_high_.has_value()) + this->parent_->set_target_temperature_high(this->target_temperature_high_.value(x...)); + if (this->target_humidity_.has_value()) + this->parent_->set_target_humidity(this->target_humidity_.value(x...)); + if (this->mode_.has_value()) + this->parent_->set_mode(this->mode_.value(x...)); + if (this->action_.has_value()) + this->parent_->action = this->action_.value(x...); + if (this->fan_mode_.has_value()) + this->parent_->set_fan_mode(this->fan_mode_.value(x...)); + if (this->custom_fan_mode_.has_value()) + this->parent_->set_custom_fan_mode(StringRef(this->custom_fan_mode_.value(x...))); + if (this->swing_mode_.has_value()) + this->parent_->set_swing_mode(this->swing_mode_.value(x...)); + if (this->preset_.has_value()) + this->parent_->set_preset(this->preset_.value(x...)); + if (this->custom_preset_.has_value()) + this->parent_->set_custom_preset(StringRef(this->custom_preset_.value(x...))); + + this->parent_->publish_state(); + } +}; + +} // namespace esphome::template_ diff --git a/esphome/components/template/climate/template_climate.cpp b/esphome/components/template/climate/template_climate.cpp new file mode 100644 index 0000000000..a7a4d2ccab --- /dev/null +++ b/esphome/components/template/climate/template_climate.cpp @@ -0,0 +1,164 @@ +#include "template_climate.h" +#include "esphome/core/log.h" + +namespace esphome::template_ { + +static const char *const TAG = "template.climate"; + +void TemplateClimate::setup() { + if (this->restore_mode_ == TemplateClimateRestoreMode::TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE) { + auto restore = this->restore_state_(); + if (restore.has_value()) { + restore->apply(this); + } + } + + // Sensors publish every reading, not just changes, so only re-publish when the value moved. + // NAN means the sensor went unavailable and is passed through rather than dropped; the second + // check stops an unavailable sensor re-publishing forever, since NAN never equals NAN. +#ifdef USE_SENSOR + if (this->sensor_ != nullptr) { + this->current_temperature = this->sensor_->state; + this->sensor_->add_on_state_callback([this](float state) { + if (state != this->current_temperature && !(std::isnan(state) && std::isnan(this->current_temperature))) { + this->current_temperature = state; + this->publish_state(); + } + }); + } + + if (this->humidity_sensor_ != nullptr) { + this->current_humidity = this->humidity_sensor_->state; + this->humidity_sensor_->add_on_state_callback([this](float state) { + if (state != this->current_humidity && !(std::isnan(state) && std::isnan(this->current_humidity))) { + this->current_humidity = state; + this->publish_state(); + } + }); + } +#endif +} + +void TemplateClimate::dump_config() { + LOG_CLIMATE("", "Template Climate", this); + ESP_LOGCONFIG(TAG, " Optimistic: %s", YESNO(this->optimistic_)); +} + +void TemplateClimate::control(const climate::ClimateCall &call) { + // Each field present fires its set_*_action; on_control sees the whole call. optimistic: true + // also applies the values right away, false waits for a climate.template.publish report. + if (auto mode = call.get_mode()) { + if (this->optimistic_) + this->mode = *mode; + this->set_mode_trigger_.trigger(*mode); + } + + if (auto target_temp = call.get_target_temperature()) { + if (this->optimistic_) + this->target_temperature = *target_temp; + this->set_target_temperature_trigger_.trigger(*target_temp); + } + + if (auto target_temp_low = call.get_target_temperature_low()) { + if (this->optimistic_) + this->target_temperature_low = *target_temp_low; + this->set_target_temperature_low_trigger_.trigger(*target_temp_low); + } + + if (auto target_temp_high = call.get_target_temperature_high()) { + if (this->optimistic_) + this->target_temperature_high = *target_temp_high; + this->set_target_temperature_high_trigger_.trigger(*target_temp_high); + } + + if (auto target_humidity = call.get_target_humidity()) { + if (this->optimistic_) + this->target_humidity = *target_humidity; + this->set_target_humidity_trigger_.trigger(*target_humidity); + } + + if (auto fan_mode = call.get_fan_mode()) { + if (this->optimistic_) + this->set_fan_mode_(*fan_mode); + this->set_fan_mode_trigger_.trigger(*fan_mode); + } + + if (call.has_custom_fan_mode()) { + if (this->optimistic_) + this->set_custom_fan_mode_(call.get_custom_fan_mode()); + this->set_custom_fan_mode_trigger_.trigger(call.get_custom_fan_mode()); + } + + if (auto swing_mode = call.get_swing_mode()) { + if (this->optimistic_) + this->swing_mode = *swing_mode; + this->set_swing_mode_trigger_.trigger(*swing_mode); + } + + if (auto preset = call.get_preset()) { + if (this->optimistic_) + this->set_preset_(*preset); + this->set_preset_trigger_.trigger(*preset); + } + + if (call.has_custom_preset()) { + if (this->optimistic_) + this->set_custom_preset_(call.get_custom_preset()); + this->set_custom_preset_trigger_.trigger(call.get_custom_preset()); + } + + if (this->optimistic_) + this->publish_state(); +} + +// A climate.template.publish report (and initial_state:) never goes through ClimateCall::validate_(), +// so check here instead -- otherwise a typo is published as state the receiving end will reject. +void TemplateClimate::set_mode(climate::ClimateMode mode) { + if (!this->traits_.supports_mode(mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported mode %u", this->get_name().c_str(), static_cast(mode)); + return; + } + this->mode = mode; +} + +void TemplateClimate::set_swing_mode(climate::ClimateSwingMode swing_mode) { + if (!this->traits_.supports_swing_mode(swing_mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported swing mode %u", this->get_name().c_str(), static_cast(swing_mode)); + return; + } + this->swing_mode = swing_mode; +} + +void TemplateClimate::set_fan_mode(climate::ClimateFanMode fan_mode) { + if (!this->traits_.supports_fan_mode(fan_mode)) { + ESP_LOGW(TAG, "'%s' - Unsupported fan mode %u", this->get_name().c_str(), static_cast(fan_mode)); + return; + } + this->set_fan_mode_(fan_mode); +} + +void TemplateClimate::set_preset(climate::ClimatePreset preset) { + if (!this->traits_.supports_preset(preset)) { + ESP_LOGW(TAG, "'%s' - Unsupported preset %u", this->get_name().c_str(), static_cast(preset)); + return; + } + this->set_preset_(preset); +} + +void TemplateClimate::set_custom_fan_mode(StringRef mode) { + if (this->find_custom_fan_mode_(mode.c_str(), mode.size()) == nullptr) { + ESP_LOGW(TAG, "'%s' - Unsupported custom fan mode '%s'", this->get_name().c_str(), mode.c_str()); + return; + } + this->set_custom_fan_mode_(mode); +} + +void TemplateClimate::set_custom_preset(StringRef preset) { + if (this->find_custom_preset_(preset.c_str(), preset.size()) == nullptr) { + ESP_LOGW(TAG, "'%s' - Unsupported custom preset '%s'", this->get_name().c_str(), preset.c_str()); + return; + } + this->set_custom_preset_(preset); +} + +} // namespace esphome::template_ diff --git a/esphome/components/template/climate/template_climate.h b/esphome/components/template/climate/template_climate.h new file mode 100644 index 0000000000..5448488c34 --- /dev/null +++ b/esphome/components/template/climate/template_climate.h @@ -0,0 +1,92 @@ +#pragma once + +#include "esphome/core/automation.h" +#include "esphome/core/component.h" +#include "esphome/components/climate/climate.h" +#ifdef USE_SENSOR +#include "esphome/components/sensor/sensor.h" +#endif + +namespace esphome::template_ { + +enum class TemplateClimateRestoreMode { + TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE, + TEMPLATE_CLIMATE_RESTORE_MODE_RESTORE, +}; + +class TemplateClimate final : public climate::Climate, public Component { + public: + void setup() override; + void dump_config() override; + + climate::ClimateTraits traits() override { return this->traits_; } + + void add_feature_flags(uint32_t flags) { this->traits_.add_feature_flags(flags); } + +#ifdef USE_SENSOR + // The matching feature flag is added from codegen, so the configuration alone decides it. + void set_sensor(sensor::Sensor *sensor) { this->sensor_ = sensor; } + void set_humidity_sensor(sensor::Sensor *sensor) { this->humidity_sensor_ = sensor; } +#endif + + void add_supported_mode(climate::ClimateMode mode) { this->traits_.add_supported_mode(mode); } + void add_supported_fan_mode(climate::ClimateFanMode mode) { this->traits_.add_supported_fan_mode(mode); } + void add_supported_swing_mode(climate::ClimateSwingMode mode) { this->traits_.add_supported_swing_mode(mode); } + void add_supported_preset(climate::ClimatePreset preset) { this->traits_.add_supported_preset(preset); } + + void set_optimistic(bool optimistic) { this->optimistic_ = optimistic; } + void set_restore_mode(TemplateClimateRestoreMode restore_mode) { this->restore_mode_ = restore_mode; } + + // Fired from control() for each field the call carries, so a device-backed config can forward + // it on. Which of these are configured also decides the two-point/target-humidity traits. + Trigger *get_set_mode_trigger() { return &this->set_mode_trigger_; } + Trigger *get_set_target_temperature_trigger() { return &this->set_target_temperature_trigger_; } + Trigger *get_set_target_temperature_low_trigger() { return &this->set_target_temperature_low_trigger_; } + Trigger *get_set_target_temperature_high_trigger() { return &this->set_target_temperature_high_trigger_; } + Trigger *get_set_target_humidity_trigger() { return &this->set_target_humidity_trigger_; } + Trigger *get_set_fan_mode_trigger() { return &this->set_fan_mode_trigger_; } + Trigger *get_set_custom_fan_mode_trigger() { return &this->set_custom_fan_mode_trigger_; } + Trigger *get_set_swing_mode_trigger() { return &this->set_swing_mode_trigger_; } + Trigger *get_set_preset_trigger() { return &this->set_preset_trigger_; } + Trigger *get_set_custom_preset_trigger() { return &this->set_custom_preset_trigger_; } + + // Used by TemplateClimatePublishAction, which is not a Climate subclass and so cannot reach the + // protected setters, and by codegen to apply `initial_state:` before setup() runs. + void set_target_temperature(float value) { this->target_temperature = value; } + void set_target_temperature_low(float value) { this->target_temperature_low = value; } + void set_target_temperature_high(float value) { this->target_temperature_high = value; } + void set_target_humidity(float value) { this->target_humidity = value; } + void set_mode(climate::ClimateMode mode); + void set_swing_mode(climate::ClimateSwingMode mode); + void set_fan_mode(climate::ClimateFanMode mode); + void set_custom_fan_mode(const char *mode) { this->set_custom_fan_mode(StringRef(mode)); } + void set_custom_fan_mode(StringRef mode); + void set_preset(climate::ClimatePreset preset); + void set_custom_preset(const char *preset) { this->set_custom_preset(StringRef(preset)); } + void set_custom_preset(StringRef preset); + + protected: + void control(const climate::ClimateCall &call) override; + + climate::ClimateTraits traits_; + bool optimistic_{false}; + TemplateClimateRestoreMode restore_mode_{TemplateClimateRestoreMode::TEMPLATE_CLIMATE_RESTORE_MODE_NO_RESTORE}; + +#ifdef USE_SENSOR + sensor::Sensor *sensor_{nullptr}; + sensor::Sensor *humidity_sensor_{nullptr}; +#endif + + Trigger set_mode_trigger_; + Trigger set_target_temperature_trigger_; + Trigger set_target_temperature_low_trigger_; + Trigger set_target_temperature_high_trigger_; + Trigger set_target_humidity_trigger_; + Trigger set_fan_mode_trigger_; + Trigger set_custom_fan_mode_trigger_; + Trigger set_swing_mode_trigger_; + Trigger set_preset_trigger_; + Trigger set_custom_preset_trigger_; +}; + +} // namespace esphome::template_ diff --git a/esphome/config_validation.py b/esphome/config_validation.py index aff39201e8..685a9d04b3 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -133,6 +133,7 @@ Upper = vol.Upper Length = vol.Length Exclusive = vol.Exclusive Inclusive = vol.Inclusive +Unique = vol.Unique ALLOW_EXTRA = vol.ALLOW_EXTRA UNDEFINED = vol.UNDEFINED RequiredFieldInvalid = vol.RequiredFieldInvalid diff --git a/tests/component_tests/template/test_template_climate.py b/tests/component_tests/template/test_template_climate.py new file mode 100644 index 0000000000..304991ea64 --- /dev/null +++ b/tests/component_tests/template/test_template_climate.py @@ -0,0 +1,145 @@ +"""Tests for template climate config validation.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components.template.climate import ( + CONF_SET_TARGET_HUMIDITY_ACTION, + CONF_SET_TARGET_TEMPERATURE_ACTION, + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION, + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION, + CONF_SUPPORTS_CURRENT_HUMIDITY, + CONF_SUPPORTS_CURRENT_TEMPERATURE, + CONF_SUPPORTS_TARGET_HUMIDITY, + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE, + CONF_TARGET_HUMIDITY, + _resolve_supports, + _validate_initial_state, + _validate_set_actions, +) +from esphome.const import ( + CONF_HUMIDITY_SENSOR, + CONF_INITIAL_STATE, + CONF_SENSOR, + CONF_TARGET_TEMPERATURE, + CONF_TARGET_TEMPERATURE_HIGH, + CONF_TARGET_TEMPERATURE_LOW, +) +from esphome.types import ConfigType + + +def test_supports_current_temperature_derived_from_sensor() -> None: + config: ConfigType = {CONF_SENSOR: "some_sensor"} + assert _resolve_supports(config)[CONF_SUPPORTS_CURRENT_TEMPERATURE] is True + + +def test_supports_current_temperature_false_without_sensor() -> None: + assert _resolve_supports({})[CONF_SUPPORTS_CURRENT_TEMPERATURE] is False + + +def test_supports_current_temperature_explicit_true_without_sensor_allowed() -> None: + # The value can still be reported with climate.template.publish. + config: ConfigType = {CONF_SUPPORTS_CURRENT_TEMPERATURE: True} + assert _resolve_supports(config)[CONF_SUPPORTS_CURRENT_TEMPERATURE] is True + + +def test_supports_current_temperature_false_with_sensor_rejected() -> None: + config: ConfigType = { + CONF_SENSOR: "some_sensor", + CONF_SUPPORTS_CURRENT_TEMPERATURE: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_supports_current_humidity_false_with_sensor_rejected() -> None: + config: ConfigType = { + CONF_HUMIDITY_SENSOR: "some_sensor", + CONF_SUPPORTS_CURRENT_HUMIDITY: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_two_point_derived_from_set_actions() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION: [{}], + } + assert _resolve_supports(config)[CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE] is True + + +def test_two_point_false_with_set_action_rejected() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + } + with pytest.raises(cv.Invalid, match="cannot be false"): + _resolve_supports(config) + + +def test_target_humidity_derived_from_set_action() -> None: + config: ConfigType = {CONF_SET_TARGET_HUMIDITY_ACTION: [{}]} + assert _resolve_supports(config)[CONF_SUPPORTS_TARGET_HUMIDITY] is True + + +def test_set_target_temperature_low_requires_high() -> None: + config: ConfigType = {CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}]} + with pytest.raises(cv.Invalid, match="must be used together"): + _validate_set_actions(config) + + +def test_set_target_temperature_conflicts_with_two_point_actions() -> None: + config: ConfigType = { + CONF_SET_TARGET_TEMPERATURE_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_LOW_ACTION: [{}], + CONF_SET_TARGET_TEMPERATURE_HIGH_ACTION: [{}], + } + with pytest.raises(cv.Invalid, match="cannot be used together"): + _validate_set_actions(config) + + +def test_initial_state_target_temperature_rejected_with_two_point() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: True, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: {CONF_TARGET_TEMPERATURE: 21.0}, + } + with pytest.raises(cv.Invalid, match="is not available"): + _validate_initial_state(config) + + +def test_initial_state_two_point_values_rejected_without_two_point() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: { + CONF_TARGET_TEMPERATURE_LOW: 18.0, + CONF_TARGET_TEMPERATURE_HIGH: 24.0, + }, + } + with pytest.raises(cv.Invalid, match="requires"): + _validate_initial_state(config) + + +def test_initial_state_target_humidity_rejected_without_support() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: False, + CONF_SUPPORTS_TARGET_HUMIDITY: False, + CONF_INITIAL_STATE: {CONF_TARGET_HUMIDITY: 50}, + } + with pytest.raises(cv.Invalid, match="requires"): + _validate_initial_state(config) + + +def test_initial_state_matching_two_point_accepted() -> None: + config: ConfigType = { + CONF_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE: True, + CONF_SUPPORTS_TARGET_HUMIDITY: True, + CONF_INITIAL_STATE: { + CONF_TARGET_TEMPERATURE_LOW: 18.0, + CONF_TARGET_TEMPERATURE_HIGH: 24.0, + CONF_TARGET_HUMIDITY: 50, + }, + } + assert _validate_initial_state(config) is config diff --git a/tests/components/climate/common.yaml b/tests/components/climate/common.yaml index c28fde8eeb..49386a16d5 100644 --- a/tests/components/climate/common.yaml +++ b/tests/components/climate/common.yaml @@ -30,8 +30,7 @@ climate: - switch.turn_on: climate_heater_switch - switch.turn_off: climate_cooler_switch # Thermostat-based climate so climate.control: action variants get build - # coverage (bang_bang doesn't support fan modes, presets, etc.). Climate - # has no template platform, so thermostat is the right vehicle. + # coverage (bang_bang doesn't support fan modes, presets, etc.). - platform: thermostat id: climate_test_thermostat name: Test Thermostat diff --git a/tests/components/template/common-base.yaml b/tests/components/template/common-base.yaml index 92a1fc8eda..02aedaf167 100644 --- a/tests/components/template/common-base.yaml +++ b/tests/components/template/common-base.yaml @@ -25,6 +25,27 @@ esphome: away: !lambda "return true;" is_on: !lambda "return false;" + - climate.template.publish: + id: template_climate + current_temperature: 21.0 + mode: HEAT + fan_mode: AUTO + swing_mode: "OFF" + preset: NONE + target_temperature: 22.0 + + # Templated + - climate.template.publish: + id: template_climate + current_temperature: !lambda "return 21.5f;" + mode: !lambda "return climate::CLIMATE_MODE_COOL;" + target_temperature: !lambda "return 23.0f;" + + - climate.template.publish: + id: template_climate_custom_modes + custom_fan_mode: "turbo" + custom_preset: "eco_plus" + # Test C++ API: set_template() with stateless lambda (no captures) # NOTE: set_template() is not intended to be a public API, but we test it to ensure it doesn't break. - lambda: |- @@ -513,6 +534,98 @@ alarm_control_panel: codes: - "1234" +climate: + - platform: template + id: template_climate + name: "Template Climate" + optimistic: true + sensor: template_template_sens + supports_action: true + supports_current_humidity: true + restore_mode: NO_RESTORE + initial_state: + mode: HEAT + target_temperature: 21.0 + fan_mode: LOW + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + set_mode_action: + - logger.log: + format: "set_mode_action %d" + args: ["(int) x"] + set_target_temperature_action: + - logger.log: + format: "set_target_temperature_action %.1f" + args: ["x"] + set_target_humidity_action: + - logger.log: + format: "set_target_humidity_action %.1f" + args: ["x"] + set_fan_mode_action: + - logger.log: + format: "set_fan_mode_action %d" + args: ["(int) x"] + set_swing_mode_action: + - logger.log: + format: "set_swing_mode_action %d" + args: ["(int) x"] + set_preset_action: + - logger.log: + format: "set_preset_action %d" + args: ["(int) x"] + on_control: + - logger.log: "on_control fired" + on_state: + - logger.log: "on_state fired" + + - platform: template + id: template_climate_custom_modes + name: "Template Climate Custom Modes" + optimistic: true + sensor: template_template_sens + supported_modes: + - "OFF" + - HEAT + custom_fan_modes: + - turbo + - silent + - eco + custom_presets: + - eco_plus + - power_save + - max + set_custom_fan_mode_action: + - logger.log: + format: "set_custom_fan_mode_action %s" + args: ["x.c_str()"] + set_custom_preset_action: + - logger.log: + format: "set_custom_preset_action %s" + args: ["x.c_str()"] + initial_state: + custom_fan_mode: eco + custom_preset: max + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + water_heater: - platform: template id: template_water_heater diff --git a/tests/integration/fixtures/template_climate_basic.yaml b/tests/integration/fixtures/template_climate_basic.yaml new file mode 100644 index 0000000000..51558b4875 --- /dev/null +++ b/tests/integration/fixtures/template_climate_basic.yaml @@ -0,0 +1,72 @@ +esphome: + name: tmpl-clim-basic + on_boot: + - climate.template.publish: + id: test_climate + action: IDLE +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Basic Climate + optimistic: true + sensor: test_climate_current_temperature + humidity_sensor: test_climate_current_humidity + supports_action: true + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature().has_value()) + ESP_LOGD("test", "on_control target_temperature=%.1f", *x.get_target_temperature()); + if (x.get_fan_mode().has_value()) + ESP_LOGD("test", "on_control fan_mode=%d", (int) *x.get_fan_mode()); + if (x.get_swing_mode().has_value()) + ESP_LOGD("test", "on_control swing_mode=%d", (int) *x.get_swing_mode()); + if (x.get_preset().has_value()) + ESP_LOGD("test", "on_control preset=%d", (int) *x.get_preset()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 22.5f;" + update_interval: 10ms + - platform: template + id: test_climate_current_humidity + name: Test Climate Current Humidity + lambda: "return 55.0f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + mode: "OFF" + fan_mode: AUTO + swing_mode: "OFF" + preset: NONE diff --git a/tests/integration/fixtures/template_climate_custom_modes.yaml b/tests/integration/fixtures/template_climate_custom_modes.yaml new file mode 100644 index 0000000000..9dbfe60cb9 --- /dev/null +++ b/tests/integration/fixtures/template_climate_custom_modes.yaml @@ -0,0 +1,47 @@ +esphome: + name: tmpl-clim-custom +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Custom Mode Climate + optimistic: true + sensor: test_climate_current_temperature + supported_modes: + - "OFF" + - HEAT + - COOL + custom_fan_modes: + - turbo + - silent + - eco + custom_presets: + - eco_plus + - power_save + - max + on_control: + - lambda: |- + if (x.has_custom_fan_mode()) + ESP_LOGD("test", "on_control custom_fan_mode=%s", x.get_custom_fan_mode().c_str()); + if (x.has_custom_preset()) + ESP_LOGD("test", "on_control custom_preset=%s", x.get_custom_preset().c_str()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 22.5f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + custom_fan_mode: "eco" + custom_preset: "max" diff --git a/tests/integration/fixtures/template_climate_nonoptimistic.yaml b/tests/integration/fixtures/template_climate_nonoptimistic.yaml new file mode 100644 index 0000000000..2b0c7ee132 --- /dev/null +++ b/tests/integration/fixtures/template_climate_nonoptimistic.yaml @@ -0,0 +1,56 @@ +esphome: + name: tmpl-clim-nonopt +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Template Climate Nonoptimistic + optimistic: false + supported_modes: + - "OFF" + - HEAT + - COOL + - FAN_ONLY + supported_fan_modes: + - AUTO + - LOW + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + - AWAY + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature().has_value()) + ESP_LOGD("test", "on_control target_temperature=%.1f", *x.get_target_temperature()); + if (x.get_fan_mode().has_value()) + ESP_LOGD("test", "on_control fan_mode=%d", (int) *x.get_fan_mode()); + if (x.get_swing_mode().has_value()) + ESP_LOGD("test", "on_control swing_mode=%d", (int) *x.get_swing_mode()); + if (x.get_preset().has_value()) + ESP_LOGD("test", "on_control preset=%d", (int) *x.get_preset()); + +button: + - platform: template + id: simulate_device_confirmation + name: Simulate Device Confirmation + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT + target_temperature: 22.5 + fan_mode: HIGH + swing_mode: VERTICAL + preset: AWAY diff --git a/tests/integration/fixtures/template_climate_on_control_ordering.yaml b/tests/integration/fixtures/template_climate_on_control_ordering.yaml new file mode 100644 index 0000000000..8366a6d21e --- /dev/null +++ b/tests/integration/fixtures/template_climate_on_control_ordering.yaml @@ -0,0 +1,26 @@ +esphome: + name: tmpl-clim-oc-order +host: +api: +logger: + +# on_control fires with the full ClimateCall (arg `x`) from the base Climate component's +# ClimateCall::perform(), before validate_()/control() run -- so when the lambda action below +# runs, the entity's own .mode is still the OLD value, even though x.get_mode() already reports +# the NEW requested value. on_state fires afterward, once control() has applied it. +climate: + - platform: template + id: test_climate + name: Test On Control Ordering + optimistic: true + supported_modes: + - "OFF" + - HEAT + on_control: + - lambda: |- + ESP_LOGD("test", "on_control requested_mode=%d current_mode_before_apply=%d", + x.get_mode().has_value() ? (int) *x.get_mode() : -1, + (int) id(test_climate).mode); + on_state: + - lambda: |- + ESP_LOGD("test", "on_state mode=%d", (int) x.mode); diff --git a/tests/integration/fixtures/template_climate_publish_all_fields.yaml b/tests/integration/fixtures/template_climate_publish_all_fields.yaml new file mode 100644 index 0000000000..e57fcc4508 --- /dev/null +++ b/tests/integration/fixtures/template_climate_publish_all_fields.yaml @@ -0,0 +1,63 @@ +esphome: + name: tmpl-clim-publish-all +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Publish All Fields + optimistic: true + # current_temperature/current_humidity/action are only sent over the API at all if their + # trait is advertised: current_temperature/current_humidity because a sensor/humidity_sensor + # is referenced below, action because supports_action is set. The sensors' fixed readings + # match what climate.template.publish pushes, so the sensor callback (guarded to only publish + # on an actual change) doesn't produce an extra, unexpected state update of its own. + sensor: test_climate_current_temperature + humidity_sensor: test_climate_current_humidity + supports_action: true + supported_modes: + - "OFF" + - HEAT + supported_fan_modes: + - AUTO + - HIGH + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + on_control: + # Should never fire in this test: climate.template.publish is a pure bypass and must not + # re-trigger on_control as if the entity were freshly commanded. + - logger.log: "on_control fired" + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 20.0f;" + update_interval: 10ms + - platform: template + id: test_climate_current_humidity + name: Test Climate Current Humidity + lambda: "return 60.0f;" + update_interval: 10ms + +button: + - platform: template + id: publish_all + name: Publish All + on_press: + - climate.template.publish: + id: test_climate + current_temperature: 20.0 + current_humidity: 60.0 + target_temperature: 23.0 + mode: HEAT + action: HEATING + fan_mode: HIGH + swing_mode: VERTICAL + preset: ECO diff --git a/tests/integration/fixtures/template_climate_sensor_push.yaml b/tests/integration/fixtures/template_climate_sensor_push.yaml new file mode 100644 index 0000000000..1fc004335d --- /dev/null +++ b/tests/integration/fixtures/template_climate_sensor_push.yaml @@ -0,0 +1,49 @@ +esphome: + name: tmpl-clim-sensor-push +host: +api: +logger: + +# No lambda/update_interval: these sensors only ever report a value when a button below +# publishes one (standing in for e.g. a BLE scan callback in a real config). +sensor: + - platform: template + id: room_temperature + name: Room Temperature + - platform: template + id: room_humidity + name: Room Humidity + +climate: + - platform: template + id: test_climate + name: Test Sensor Push Climate + optimistic: true + sensor: room_temperature + humidity_sensor: room_humidity + supported_modes: + - "OFF" + - HEAT + +button: + - platform: template + id: publish_temperature + name: Publish Temperature + on_press: + - sensor.template.publish: + id: room_temperature + state: 24.0 + - platform: template + id: publish_temperature_same + name: Publish Temperature Same Value + on_press: + - sensor.template.publish: + id: room_temperature + state: 24.0 + - platform: template + id: publish_humidity + name: Publish Humidity + on_press: + - sensor.template.publish: + id: room_humidity + state: 65.0 diff --git a/tests/integration/fixtures/template_climate_set_actions.yaml b/tests/integration/fixtures/template_climate_set_actions.yaml new file mode 100644 index 0000000000..b247367f64 --- /dev/null +++ b/tests/integration/fixtures/template_climate_set_actions.yaml @@ -0,0 +1,89 @@ +esphome: + name: tmpl-clim-set-act +host: +api: +logger: + +# Every settable field forwards its requested value to a set_*_action. supports_two_point and +# supports_target_humidity are not declared here: they are derived from the low/high and humidity +# set actions being present. +climate: + - platform: template + id: test_climate + name: Test Set Actions + optimistic: false + restore_mode: NO_RESTORE + supported_modes: + - "OFF" + - HEAT + - COOL + supported_fan_modes: + - AUTO + - LOW + supported_swing_modes: + - "OFF" + - VERTICAL + supported_presets: + - NONE + - ECO + custom_fan_modes: + - turbo + custom_presets: + - eco_plus + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + set_mode_action: + - logger.log: + format: "set_mode_action %d" + args: ["(int) x"] + set_target_temperature_low_action: + - logger.log: + format: "set_target_temperature_low_action %.1f" + args: ["x"] + set_target_temperature_high_action: + - logger.log: + format: "set_target_temperature_high_action %.1f" + args: ["x"] + set_target_humidity_action: + - logger.log: + format: "set_target_humidity_action %.0f" + args: ["x"] + set_fan_mode_action: + - logger.log: + format: "set_fan_mode_action %d" + args: ["(int) x"] + set_custom_fan_mode_action: + - logger.log: + format: "set_custom_fan_mode_action %s" + args: ["x.c_str()"] + set_swing_mode_action: + - logger.log: + format: "set_swing_mode_action %d" + args: ["(int) x"] + set_preset_action: + - logger.log: + format: "set_preset_action %d" + args: ["(int) x"] + set_custom_preset_action: + - logger.log: + format: "set_custom_preset_action %s" + args: ["x.c_str()"] + +button: + - platform: template + id: report_device_state + name: Report Device State + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT + + - platform: template + id: report_unsupported_mode + name: Report Unsupported Mode + on_press: + - climate.template.publish: + id: test_climate + mode: DRY diff --git a/tests/integration/fixtures/template_climate_two_point_temperature.yaml b/tests/integration/fixtures/template_climate_two_point_temperature.yaml new file mode 100644 index 0000000000..ec10785ee8 --- /dev/null +++ b/tests/integration/fixtures/template_climate_two_point_temperature.yaml @@ -0,0 +1,52 @@ +esphome: + name: tmpl-clim-two-point +host: +api: +logger: + +climate: + - platform: template + id: test_climate + name: Test Two-Point Heatpump + optimistic: true + sensor: test_climate_current_temperature + supports_two_point_target_temperature: true + supports_target_humidity: true + supported_modes: + - "OFF" + - HEAT_COOL + - HEAT + - COOL + visual: + min_temperature: 16.0 + max_temperature: 30.0 + temperature_step: 0.5 + on_control: + - lambda: |- + if (x.get_mode().has_value()) + ESP_LOGD("test", "on_control mode=%d", (int) *x.get_mode()); + if (x.get_target_temperature_low().has_value()) + ESP_LOGD("test", "on_control target_temperature_low=%.1f", *x.get_target_temperature_low()); + if (x.get_target_temperature_high().has_value()) + ESP_LOGD("test", "on_control target_temperature_high=%.1f", *x.get_target_temperature_high()); + if (x.get_target_humidity().has_value()) + ESP_LOGD("test", "on_control target_humidity=%.1f", *x.get_target_humidity()); + +sensor: + - platform: template + id: test_climate_current_temperature + name: Test Climate Current Temperature + lambda: "return 21.0f;" + update_interval: 10ms + +button: + - platform: template + id: simulate_device_report + name: Simulate Device Report + on_press: + - climate.template.publish: + id: test_climate + mode: HEAT_COOL + target_temperature_low: 18.0 + target_temperature_high: 24.0 + target_humidity: 50.0 diff --git a/tests/integration/test_template_climate_basic.py b/tests/integration/test_template_climate_basic.py new file mode 100644 index 0000000000..431fd4e3e8 --- /dev/null +++ b/tests/integration/test_template_climate_basic.py @@ -0,0 +1,146 @@ +"""Integration test for template climate: sensor-pushed measured values, on_control + publish +for the settable ones. + +current_temperature/current_humidity are pushed by a referenced sensor/humidity_sensor (no +polling); action is set once at boot via climate.template.publish, since it has no sensor +equivalent. mode/target_temperature/fan_mode/swing_mode/preset are plain internal state: +on_control fires exactly once per command (never before the first one), and +climate.template.publish simulates the device reporting its own state independent of any prior +command -- that report is authoritative, overriding whatever was optimistically applied earlier. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateAction, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-basic" + + +@pytest.mark.asyncio +async def test_template_climate_basic( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Sensor-pushed measured values, on_control + publish for settable ones.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + # Advertised capabilities come straight from the supported_*/custom_* config lists. + assert ClimateMode.OFF in test_climate.supported_modes + assert ClimateMode.HEAT in test_climate.supported_modes + assert ClimateMode.COOL in test_climate.supported_modes + + assert ClimateFanMode.AUTO in test_climate.supported_fan_modes + assert ClimateFanMode.LOW in test_climate.supported_fan_modes + assert ClimateFanMode.HIGH in test_climate.supported_fan_modes + + assert ClimateSwingMode.OFF in test_climate.supported_swing_modes + assert ClimateSwingMode.VERTICAL in test_climate.supported_swing_modes + + assert ClimatePreset.NONE in test_climate.supported_presets + assert ClimatePreset.ECO in test_climate.supported_presets + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.current_temperature == pytest.approx(22.5, abs=0.1) + assert initial.current_humidity == pytest.approx(55.0, abs=0.1) + assert initial.action == ClimateAction.IDLE + assert initial.mode == ClimateMode.OFF + # Nothing was commanded yet: on_control must not have fired. + assert not log_lines + + # Commands apply optimistically and on_control fires with the same values. + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT + + client.climate_command(test_climate.key, target_temperature=22.5) + state = await wait_for_climate_state() + assert state.target_temperature == pytest.approx(22.5, abs=0.1) + + client.climate_command(test_climate.key, fan_mode=ClimateFanMode.HIGH) + state = await wait_for_climate_state() + assert state.fan_mode == ClimateFanMode.HIGH + + client.climate_command(test_climate.key, swing_mode=ClimateSwingMode.VERTICAL) + state = await wait_for_climate_state() + assert state.swing_mode == ClimateSwingMode.VERTICAL + + client.climate_command(test_climate.key, preset=ClimatePreset.ECO) + state = await wait_for_climate_state() + assert state.preset == ClimatePreset.ECO + + await asyncio.sleep(0.2) + assert any( + "on_control mode=3" in line for line in log_lines + ) # CLIMATE_MODE_HEAT + assert any("on_control target_temperature=22.5" in line for line in log_lines) + assert any("on_control fan_mode=" in line for line in log_lines) + assert any("on_control swing_mode=" in line for line in log_lines) + assert any("on_control preset=" in line for line in log_lines) + # Exactly one on_control log line per command, none extra (e.g. from a stray republish). + assert len(log_lines) == 5 + + # measured values are untouched by any of the above (no set action exists for them). + assert state.current_temperature == pytest.approx(22.5, abs=0.1) + assert state.current_humidity == pytest.approx(55.0, abs=0.1) + assert state.action == ClimateAction.IDLE + + # The device's report is authoritative and overrides everything commanded above. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.OFF + assert state.fan_mode == ClimateFanMode.AUTO + assert state.swing_mode == ClimateSwingMode.OFF + assert state.preset == ClimatePreset.NONE diff --git a/tests/integration/test_template_climate_custom_modes.py b/tests/integration/test_template_climate_custom_modes.py new file mode 100644 index 0000000000..4817fe1ddf --- /dev/null +++ b/tests/integration/test_template_climate_custom_modes.py @@ -0,0 +1,98 @@ +"""Integration test for template climate: custom fan modes and presets. + +Same on_control (forward) + climate.template.publish (device report, authoritative) pattern as +the enum-based mode/preset fields, but for the custom string variants. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-custom" + + +@pytest.mark.asyncio +async def test_template_climate_custom_modes( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Custom fan mode/preset: traits, on_control forwarding, and publish precedence.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + assert set(test_climate.supported_custom_fan_modes) == { + "turbo", + "silent", + "eco", + } + assert set(test_climate.supported_custom_presets) == { + "eco_plus", + "power_save", + "max", + } + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.custom_fan_mode == "" + assert initial.custom_preset == "" + + client.climate_command(test_climate.key, custom_fan_mode="turbo") + state = await wait_for_climate_state() + assert state.custom_fan_mode == "turbo" + + client.climate_command(test_climate.key, custom_preset="power_save") + state = await wait_for_climate_state() + assert state.custom_preset == "power_save" + + await asyncio.sleep(0.2) + assert any("on_control custom_fan_mode=turbo" in line for line in log_lines) + assert any("on_control custom_preset=power_save" in line for line in log_lines) + + # The device's report is authoritative and overrides what was commanded above. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.custom_fan_mode == "eco" + assert state.custom_preset == "max" diff --git a/tests/integration/test_template_climate_nonoptimistic.py b/tests/integration/test_template_climate_nonoptimistic.py new file mode 100644 index 0000000000..e922ec31b9 --- /dev/null +++ b/tests/integration/test_template_climate_nonoptimistic.py @@ -0,0 +1,107 @@ +"""Integration test for template climate: optimistic: false. + +A command still fires on_control (so a real device-backed config can forward it out), but must +NOT change the entity's own state -- only an explicit climate.template.publish call (standing in +for the device confirming the command actually took effect) does that. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-nonopt" + + +@pytest.mark.asyncio +async def test_template_climate_nonoptimistic( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Nonoptimistic: a command doesn't change state until explicitly published.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + state_updates: list[aioesphomeapi.ClimateState] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + confirm_button = require_entity( + entities, "simulate_device_confirmation", ButtonInfo + ) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + assert initial.mode == ClimateMode.OFF + + # Send every settable field in one command. on_control must fire with all of them, but + # nothing may be applied to the entity's own state -- no ClimateState update at all. + client.climate_command( + test_climate.key, + mode=ClimateMode.HEAT, + target_temperature=22.5, + fan_mode=ClimateFanMode.HIGH, + swing_mode=ClimateSwingMode.VERTICAL, + preset=ClimatePreset.AWAY, + ) + await asyncio.sleep(0.3) + assert any( + "on_control mode=3" in line for line in log_lines + ) # CLIMATE_MODE_HEAT + assert any("on_control target_temperature=22.5" in line for line in log_lines) + assert any("on_control fan_mode=" in line for line in log_lines) + assert any("on_control swing_mode=" in line for line in log_lines) + assert any("on_control preset=" in line for line in log_lines) + assert not state_updates, ( + "optimistic: false must not publish a state until climate.template.publish reports it" + ) + + # The device confirms the command actually took effect. + client.button_command(confirm_button.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.mode == ClimateMode.HEAT + assert state.target_temperature == pytest.approx(22.5, abs=0.1) + assert state.fan_mode == ClimateFanMode.HIGH + assert state.swing_mode == ClimateSwingMode.VERTICAL + assert state.preset == ClimatePreset.AWAY diff --git a/tests/integration/test_template_climate_on_control_ordering.py b/tests/integration/test_template_climate_on_control_ordering.py new file mode 100644 index 0000000000..8d212b3ccb --- /dev/null +++ b/tests/integration/test_template_climate_on_control_ordering.py @@ -0,0 +1,83 @@ +"""Integration test: on_control fires before control()/on_state, with the full ClimateCall. + +on_control's lambda argument exposes get_mode()/etc. on the *requested* ClimateCall, while the +entity's own .mode field still reflects the state *before* control() applies the change -- +proving the firing order is on_control, then control(), then on_state. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ClimateInfo, ClimateMode +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-oc-order" + + +@pytest.mark.asyncio +async def test_template_climate_on_control_ordering( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """on_control sees the requested value while the entity's own state is still the old one.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line or "on_state " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT + + await asyncio.sleep(0.2) + + # on_control saw the new requested mode (3 == CLIMATE_MODE_HEAT) while the entity's own + # state was still the old one (0 == CLIMATE_MODE_OFF) -- proving it fired before control(). + assert any( + "on_control requested_mode=3 current_mode_before_apply=0" in line + for line in log_lines + ) + # on_state fired afterward, reporting the now-applied mode. + assert any("on_state mode=3" in line for line in log_lines) + + control_index = next( + i for i, line in enumerate(log_lines) if "on_control " in line + ) + state_index = next(i for i, line in enumerate(log_lines) if "on_state " in line) + assert control_index < state_index, "on_control must fire before on_state" diff --git a/tests/integration/test_template_climate_publish_all_fields.py b/tests/integration/test_template_climate_publish_all_fields.py new file mode 100644 index 0000000000..9c4262b311 --- /dev/null +++ b/tests/integration/test_template_climate_publish_all_fields.py @@ -0,0 +1,96 @@ +"""Integration test for template climate: climate.template.publish covering every field at once. + +A single climate.template.publish call resolves into exactly one ClimateState update, and never +triggers on_control (which would misrepresent a device state report as a fresh command). This also +exercises that a sensor/humidity_sensor whose reading matches what's about to be published doesn't +sneak in an extra state update of its own (the sensor callback only re-publishes on an actual +change). +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateAction, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-publish-all" + + +@pytest.mark.asyncio +async def test_template_climate_publish_all_fields( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """One climate.template.publish call setting every field resolves to one state update.""" + clear_host_prefs(DEVICE_NAME) + + state_updates: list[aioesphomeapi.ClimateState] = [] + on_control_count = 0 + + def on_log_line(line: str) -> None: + nonlocal on_control_count + if "on_control fired" in line: + on_control_count += 1 + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + + publish_button = require_entity(entities, "publish_all", ButtonInfo) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + client.button_command(publish_button.key) + try: + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + except TimeoutError: + pytest.fail("Timeout waiting for the published climate state") + + assert state.current_temperature == pytest.approx(20.0, abs=0.1) + assert state.current_humidity == pytest.approx(60.0, abs=0.1) + assert state.target_temperature == pytest.approx(23.0, abs=0.1) + assert state.mode == ClimateMode.HEAT + assert state.action == ClimateAction.HEATING + assert state.fan_mode == ClimateFanMode.HIGH + assert state.swing_mode == ClimateSwingMode.VERTICAL + assert state.preset == ClimatePreset.ECO + + # Give any stray extra update (there shouldn't be one) a moment to arrive. + await asyncio.sleep(0.2) + assert len(state_updates) == 1, ( + f"Expected exactly one ClimateState update, got {len(state_updates)}" + ) + assert on_control_count == 0, ( + "climate.template.publish must not trigger on_control" + ) diff --git a/tests/integration/test_template_climate_sensor_push.py b/tests/integration/test_template_climate_sensor_push.py new file mode 100644 index 0000000000..1db4da81ed --- /dev/null +++ b/tests/integration/test_template_climate_sensor_push.py @@ -0,0 +1,88 @@ +"""Integration test for template climate: current_temperature/current_humidity live sensor push. + +A *later* change to a backing sensor's value -- not just its initial reading at boot -- propagates +into a new climate state via add_on_state_callback. Re-publishing the same sensor value again must +not cause a redundant climate state update. +""" + +from __future__ import annotations + +import asyncio +import math + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-sensor-push" + + +@pytest.mark.asyncio +async def test_template_climate_sensor_push( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """A later change to the backing sensor pushes a new climate state; an unchanged republish does not.""" + clear_host_prefs(DEVICE_NAME) + + state_updates: list[aioesphomeapi.ClimateState] = [] + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + + def on_state(state: aioesphomeapi.EntityState) -> None: + if isinstance(state, aioesphomeapi.ClimateState): + state_updates.append(state) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + publish_temp = require_entity(entities, "publish_temperature", ButtonInfo) + publish_temp_same = require_entity( + entities, "publish_temperature_same", ButtonInfo + ) + publish_humidity = require_entity(entities, "publish_humidity", ButtonInfo) + + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + # Neither backing sensor has published anything yet. + assert math.isnan(initial.current_temperature) + assert math.isnan(initial.current_humidity) + + # A later sensor reading -- not the initial one -- pushes a new climate state. + client.button_command(publish_temp.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.current_temperature == pytest.approx(24.0, abs=0.1) + + client.button_command(publish_humidity.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.current_humidity == pytest.approx(65.0, abs=0.1) + + # Re-publishing the same temperature must not cause a redundant climate state update. + updates_before = len(state_updates) + client.button_command(publish_temp_same.key) + await asyncio.sleep(0.3) + assert len(state_updates) == updates_before, ( + "Re-publishing an unchanged sensor reading must not republish the climate state" + ) diff --git a/tests/integration/test_template_climate_set_actions.py b/tests/integration/test_template_climate_set_actions.py new file mode 100644 index 0000000000..0b1eb80874 --- /dev/null +++ b/tests/integration/test_template_climate_set_actions.py @@ -0,0 +1,114 @@ +"""Integration test: each settable field forwards its value to the matching set_*_action. + +With optimistic: false the entity state stays put until climate.template.publish reports the +device's actual state back, so the actions are the only thing that reacts to a command. +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ( + ButtonInfo, + ClimateFanMode, + ClimateInfo, + ClimateMode, + ClimatePreset, + ClimateSwingMode, +) +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-set-act" + + +@pytest.mark.asyncio +async def test_template_climate_set_actions( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Every set_*_action fires with the requested value; state waits for a publish.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "_action " in line or "Unsupported" in line: + log_lines.append(line) + + def logged(fragment: str) -> bool: + return any(fragment in line for line in log_lines) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + test_climate = climate_infos[0] + + report_button = require_entity(entities, "report_device_state", ButtonInfo) + unsupported_button = require_entity( + entities, "report_unsupported_mode", ButtonInfo + ) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + # Both traits are derived from the low/high and humidity set actions, not declared. + assert test_climate.supports_two_point_target_temperature + assert test_climate.supports_target_humidity + + client.climate_command(test_climate.key, mode=ClimateMode.HEAT) + client.climate_command( + test_climate.key, target_temperature_low=18.0, target_temperature_high=24.0 + ) + client.climate_command(test_climate.key, target_humidity=55) + client.climate_command(test_climate.key, fan_mode=ClimateFanMode.LOW) + client.climate_command(test_climate.key, custom_fan_mode="turbo") + client.climate_command(test_climate.key, swing_mode=ClimateSwingMode.VERTICAL) + client.climate_command(test_climate.key, preset=ClimatePreset.ECO) + client.climate_command(test_climate.key, custom_preset="eco_plus") + + for _ in range(50): + await asyncio.sleep(0.1) + if logged("set_custom_preset_action eco_plus"): + break + + assert logged("set_mode_action 3") # CLIMATE_MODE_HEAT + assert logged("set_target_temperature_low_action 18.0") + assert logged("set_target_temperature_high_action 24.0") + assert logged("set_target_humidity_action 55") + assert logged("set_fan_mode_action 3") # CLIMATE_FAN_LOW + assert logged("set_custom_fan_mode_action turbo") + assert logged("set_swing_mode_action 2") # CLIMATE_SWING_VERTICAL + assert logged("set_preset_action 5") # CLIMATE_PRESET_ECO + assert logged("set_custom_preset_action eco_plus") + + # optimistic: false, so none of the commands above touched the entity's own state -- + # a device report is what actually moves it. + client.button_command(report_button.key) + state = await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState) + ) + assert state.mode == ClimateMode.HEAT + + # A publish naming a mode outside supported_modes warns instead of publishing it. + client.button_command(unsupported_button.key) + for _ in range(50): + await asyncio.sleep(0.1) + if logged("Unsupported mode"): + break + assert logged("Unsupported mode") diff --git a/tests/integration/test_template_climate_two_point_temperature.py b/tests/integration/test_template_climate_two_point_temperature.py new file mode 100644 index 0000000000..9270b59ffc --- /dev/null +++ b/tests/integration/test_template_climate_two_point_temperature.py @@ -0,0 +1,118 @@ +"""Integration tests for template climate: two-point target temperature + humidity. + +Covers the supports_two_point_target_temperature/supports_target_humidity boolean flags plus +on_control (forwarding commands out) and climate.template.publish (the device reporting its own +authoritative state, independent of any prior command -- e.g. a device that owns its own setpoint, +changed via a physical remote). +""" + +from __future__ import annotations + +import asyncio + +import aioesphomeapi +from aioesphomeapi import ButtonInfo, ClimateInfo, ClimateMode +import pytest + +from .host_prefs import clear_host_prefs +from .state_utils import InitialStateHelper, require_entity, wait_for_state +from .types import APIClientConnectedFactory, RunCompiledFunction + +DEVICE_NAME = "tmpl-clim-two-point" + + +@pytest.mark.asyncio +async def test_template_climate_two_point_temperature( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Two-point target temperature + humidity: booleans, on_control, and publish precedence.""" + clear_host_prefs(DEVICE_NAME) + + log_lines: list[str] = [] + + def on_log_line(line: str) -> None: + if "on_control " in line: + log_lines.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + + async def wait_for_climate_state( + timeout: float = 5.0, + ) -> aioesphomeapi.ClimateState: + return await wait_for_state( + client, lambda s: isinstance(s, aioesphomeapi.ClimateState), timeout + ) + + entities, _ = await client.list_entities_services() + initial_state_helper = InitialStateHelper(entities) + climate_infos = [e for e in entities if isinstance(e, ClimateInfo)] + assert len(climate_infos) == 1, "Expected exactly 1 climate entity" + + test_climate = climate_infos[0] + assert test_climate.name == "Test Two-Point Heatpump" + assert test_climate.supports_two_point_target_temperature + assert test_climate.supports_target_humidity + + report_button = require_entity(entities, "simulate_device_report", ButtonInfo) + + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda state: None) + ) + + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + initial = initial_state_helper.initial_states.get(test_climate.key) + assert initial is not None, "No initial climate state received" + assert isinstance(initial, aioesphomeapi.ClimateState) + # Nothing has been published yet: settable fields have no sensor to seed them from, so + # the entity starts at ESPHome's plain defaults. current_temperature is pushed by the + # referenced sensor, which has already settled by the time we get here. + assert initial.mode == ClimateMode.OFF + assert initial.current_temperature == pytest.approx(21.0, abs=0.1) + + # The device reports its actual state for the first time. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.mode == ClimateMode.HEAT_COOL + assert state.target_temperature_low == pytest.approx(18.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(24.0, abs=0.1) + assert state.target_humidity == pytest.approx(50.0, abs=0.1) + + # Commands apply optimistically (settable fields are plain internal state), and on_control + # fires with the same values so a real config could forward them to the device. + client.climate_command( + test_climate.key, target_temperature_low=19.0, target_temperature_high=25.0 + ) + state = await wait_for_climate_state() + assert state.target_temperature_low == pytest.approx(19.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(25.0, abs=0.1) + await asyncio.sleep(0.2) + assert any( + "on_control target_temperature_low=19.0" in line for line in log_lines + ) + assert any( + "on_control target_temperature_high=25.0" in line for line in log_lines + ) + + client.climate_command(test_climate.key, target_humidity=45.0) + state = await wait_for_climate_state() + assert state.target_humidity == pytest.approx(45.0, abs=0.1) + await asyncio.sleep(0.2) + assert any("on_control target_humidity=45.0" in line for line in log_lines) + + # The device's next report is authoritative and overrides whatever was optimistically + # applied above -- this is the whole point of climate.template.publish: a device that owns + # its own state (e.g. changed by a physical remote) always wins. + client.button_command(report_button.key) + state = await wait_for_climate_state() + assert state.target_temperature_low == pytest.approx(18.0, abs=0.1) + assert state.target_temperature_high == pytest.approx(24.0, abs=0.1) + assert state.target_humidity == pytest.approx(50.0, abs=0.1) From 95ab3fb4f29aed85f762f2699484fea9e756b4f6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 6 Sep 2026 23:59:40 +0200 Subject: [PATCH 120/147] [ota] Offer encryption with the api key so enabling it works over OTA (#18979) --- THREAT_MODEL.md | 55 ++- esphome/__main__.py | 14 +- esphome/components/api/__init__.py | 4 +- esphome/components/api/api_connection.cpp | 6 +- .../components/api/api_frame_helper_noise.cpp | 2 +- esphome/components/api/api_server.cpp | 37 +- esphome/components/api/api_server.h | 12 +- esphome/components/esphome/ota/__init__.py | 130 +++--- .../components/esphome/ota/ota_esphome.cpp | 83 ++-- esphome/components/esphome/ota/ota_esphome.h | 13 +- .../esphome/ota/ota_esphome_noise.cpp | 91 +++-- esphome/components/noise/__init__.py | 36 +- esphome/components/noise/noise.cpp | 9 + esphome/components/noise/noise.h | 16 +- esphome/components/noise/noise_handshake.cpp | 5 +- esphome/components/noise/noise_handshake.h | 6 +- esphome/core/defines.h | 3 + esphome/espota2.py | 122 +++++- esphome/wizard.py | 18 +- .../noise/test_encryption_key.py | 14 +- tests/component_tests/ota/test_esphome_ota.py | 242 +++++++++--- .../ota/test_esphome_ota_api_key_offer.yaml | 11 + ...st_esphome_ota_api_key_offer_password.yaml | 12 + .../test_esphome_ota_encryption_required.yaml | 12 + .../ota/test_esphome_ota_own_key.yaml | 11 + .../ota/test_esphome_ota_plain.yaml | 9 + .../ota/test_esphome_ota_runtime_api_key.yaml | 10 + .../components/noise/test_noise_handshake.cpp | 18 +- .../noise/test_noise_primitives.cpp | 13 +- tests/components/ota/api_key_offer.yaml | 12 + tests/components/ota/api_runtime_key.yaml | 10 + .../ota/test-api_key_offer.esp32-idf.yaml | 2 + .../ota/test-api_key_offer.esp8266-ard.yaml | 2 + .../ota/test-api_runtime_key.esp32-idf.yaml | 2 + .../ota/test-api_runtime_key.esp8266-ard.yaml | 2 + tests/integration/conftest.py | 7 + tests/integration/const.py | 7 + .../host_ota_api_key_offer_with_password.yaml | 12 + .../host_ota_provisioned_api_key.yaml | 10 + .../test_api_zero_psk_provisioning.py | 51 ++- tests/integration/test_host_ota.py | 373 ++++++++++++------ tests/unit_tests/test_espota2_noise.py | 136 ++++++- tests/unit_tests/test_main.py | 114 +++++- tests/unit_tests/test_wizard.py | 31 +- 44 files changed, 1342 insertions(+), 443 deletions(-) create mode 100644 tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_encryption_required.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_own_key.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_plain.yaml create mode 100644 tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml create mode 100644 tests/components/ota/api_key_offer.yaml create mode 100644 tests/components/ota/api_runtime_key.yaml create mode 100644 tests/components/ota/test-api_key_offer.esp32-idf.yaml create mode 100644 tests/components/ota/test-api_key_offer.esp8266-ard.yaml create mode 100644 tests/components/ota/test-api_runtime_key.esp32-idf.yaml create mode 100644 tests/components/ota/test-api_runtime_key.esp8266-ard.yaml create mode 100644 tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml create mode 100644 tests/integration/fixtures/host_ota_provisioned_api_key.yaml diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index b4f557e55b..11656ff0b7 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -125,30 +125,47 @@ design is optimal or that it will not change. ## OTA update encryption The `esphome` OTA platform optionally encrypts updates with the same Noise -`NNpsk0` pattern the native API uses; one key protects the device. With an -`encryption:` block configured the guarantees are: the firmware image is -confidential in transit, the uploader is authenticated by the pre-shared key, -and the plaintext negotiation preceding the handshake is bound into the -handshake prologue, so stripping or tampering with it fails the first MAC. -Both ends fail closed with no override: a device built with a key refuses +`NNpsk0` pattern the native API uses; one key protects the device. A device +whose `api:` block has an encryption key, static in the YAML or provisioned at +runtime, compiles in the transport and offers it on every OTA connection once +it holds a key, so an uploader presenting that key gets the guarantees below +even without an `ota: encryption:` block; only that block makes the device +require encryption. The guarantees are: the firmware image is confidential in +transit, the uploader is authenticated by the pre-shared key, and the plaintext +negotiation preceding the handshake is bound into the handshake prologue, so +stripping or tampering with it fails the first MAC. With `ota: encryption:` +configured both ends fail closed with no override: the device refuses plaintext uploads, and the CLI refuses to send plaintext when a key is -configured. +configured. Without that block the CLI tries a static api key when the device +offers and, until 2027.3.0, falls back to plaintext with a warning when the +offer is missing or the handshake fails; a runtime provisioned key never +reaches the CLI, so those uploads stay plaintext. -Defeating any of that without the key is in scope: a keyed device accepting a -plaintext or downgraded upload, getting past the MAC, or recovering image -contents from captured traffic. +Defeating any of that without the key is in scope: a device that requires +encryption accepting a plaintext or downgraded upload, getting past the MAC, +or recovering image contents from captured traffic. The following are **not** vulnerabilities, by design: -- Plaintext OTA on a device with no `encryption:` block. That is the - documented default, authenticated (if at all) by the OTA password. -- The enablement window: turning encryption on takes one last upload of the - encryption-enabled firmware over the existing plaintext channel, with the - pre-existing plaintext exposure. -- The web OTA `/update` endpoint alongside encryption. The `web_server` - component keeps it always reachable, and `captive_portal:` auto-loads it - for the fallback AP window; validation warns about both combinations, and - the operator keeps the recovery path. +- Plaintext OTA on a device with no `ota: encryption:` block, including one + that offers encryption because it has an api key. That is the documented + default, authenticated (if at all) by the OTA password. An uploader that + takes the offer skips the password; the key authenticates it. With a + runtime provisioned key and no `provisioning:` window, whoever provisions + the key gains that upload path too; validation warns about the pair. +- The CLI plaintext fallback until 2027.3.0: without `ota: encryption:` an + active attacker who strips the offer or breaks the handshake can make a + keyed CLI upload plaintext, with the pre-existing plaintext exposure. A + device that requires encryption still refuses that upload. +- The enablement window: firmware built with a static api key already offers + encryption, so turning on `ota: encryption:` is itself an encrypted upload. + Older firmware needs one last plaintext upload of an offering build, with + the pre-existing plaintext exposure. +- The web OTA `/update` endpoint alongside encryption. With the `web_server` + or `prometheus` component the shared listener is always up, so the endpoint + stays reachable and validation warns about that combination; + `captive_portal:` alone brings the listener up only for the fallback AP + window, which is the intended recovery path, so that is not warned about. - CLI retry behavior on transport or MAC failures; every attempt renegotiates a fresh handshake with fresh ephemerals, so retrying does not weaken authentication. diff --git a/esphome/__main__.py b/esphome/__main__.py index b3d58ad13b..30e97f55eb 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -1335,12 +1335,14 @@ def _upload_via_native_api( break from esphome import espota2 + from esphome.components.noise import static_encryption_key remote_port = int(ota_conf[CONF_PORT]) password = ota_conf.get(CONF_PASSWORD) # Fail closed: an encryption block whose key did not resolve must never # fall back to a plaintext upload noise_psk = None + plaintext_fallback = False if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: noise_psk = encryption_conf.get(CONF_KEY) if not noise_psk: @@ -1351,6 +1353,10 @@ def _upload_via_native_api( # Ensure the key is a string, as required by the underlying OTA implementation. # It arrives here as a SensitiveStr which aioesphomeapi rejects. noise_psk = str(noise_psk) + elif api_key := static_encryption_key(config.get(CONF_API) or {}): + # Remove before 2027.3.0: the api key is tried, falling back to plaintext + noise_psk = str(api_key) + plaintext_fallback = True def check_partition_access(option_string: str) -> None: if not ota_conf.get("allow_partition_access"): @@ -1382,7 +1388,13 @@ def _upload_via_native_api( _validate_bootloader_binary(binary) return espota2.run_ota( - network_devices, remote_port, password, binary, ota_type, noise_psk + network_devices, + remote_port, + password, + binary, + ota_type, + noise_psk, + plaintext_fallback=plaintext_fallback, ) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 3568318dad..6202e127bf 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -14,6 +14,7 @@ from esphome.components.noise import ( # noqa: F401 ENCRYPTION_SCHEMA, decode_encryption_key, encryption_schema, + new_psk_progmem, validate_encryption_key, ) from esphome.config_helpers import filter_source_files_from_defines, get_logger_level @@ -589,8 +590,7 @@ async def to_code(config: ConfigType) -> None: if (encryption_config := config.get(CONF_ENCRYPTION, None)) is not None: if key := encryption_config.get(CONF_KEY): - decoded = decode_encryption_key(key) - cg.add(var.set_noise_psk(list(decoded))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], key))) cg.add_define("USE_API_NOISE_PSK_FROM_YAML") else: # No key provided, but encryption desired diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index 9c609aa047..da4b7d7702 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -2161,7 +2161,10 @@ void APIConnection::on_homeassistant_action_response(const HomeassistantActionRe bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptionSetKeyRequest &msg) { NoiseEncryptionSetKeyResponse resp; resp.success = false; - +#ifdef USE_API_NOISE_PSK_FROM_YAML + // A yaml key cannot be changed at runtime, so no decode or save path is built + ESP_LOGW(TAG, "Key set in YAML"); +#else #ifdef USE_PROVISIONING // Refuse to set a key once the provisioning window has closed (defense in depth; // such connections are already rejected at hello). @@ -2196,6 +2199,7 @@ bool APIConnection::send_noise_encryption_set_key_response_(const NoiseEncryptio } #endif } +#endif // USE_API_NOISE_PSK_FROM_YAML return this->send_message(resp); } diff --git a/esphome/components/api/api_frame_helper_noise.cpp b/esphome/components/api/api_frame_helper_noise.cpp index 138dbdddba..29b2858aee 100644 --- a/esphome/components/api/api_frame_helper_noise.cpp +++ b/esphome/components/api/api_frame_helper_noise.cpp @@ -548,7 +548,7 @@ APIError APINoiseFrameHelper::write_frame_(const uint8_t *data, uint16_t len) { * @return 0 on success, -1 on error (check errno) */ APIError APINoiseFrameHelper::init_handshake_() { - int err = this->handshake_.init(this->ctx_.get_psk(), prologue_.data(), prologue_.size()); + int err = this->handshake_.init(this->ctx_, prologue_.data(), prologue_.size()); APIError aerr = handle_noise_error_(err, LOG_STR("noise_handshake_init"), APIError::HANDSHAKESTATE_SETUP_FAILED); if (aerr != APIError::OK) return aerr; diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 43d35363d3..78ebe5c38e 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -41,13 +41,13 @@ void APIServer::setup() { ControllerRegistry::register_controller(this); #ifdef USE_API_NOISE + // Always reserve the slot: flash preferences are positional on esp8266, so + // a yaml key build must keep the layout of a runtime key build uint32_t hash = 88491486UL; - this->noise_pref_ = global_preferences->make_preference(hash, true); - #ifndef USE_API_NOISE_PSK_FROM_YAML - // Only load saved PSK if not set from YAML - if (this->load_and_apply_noise_psk_()) { + // A cleared record loads fine but holds no key + if (this->load_and_apply_noise_psk_() && this->noise_ctx_.has_psk()) { ESP_LOGD(TAG, "Loaded saved Noise PSK"); } #endif @@ -550,6 +550,7 @@ const std::vector &APIServer::get_sta #endif #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg, bool make_active) { if (!this->noise_pref_.save(&new_psk)) { @@ -583,22 +584,19 @@ bool APIServer::update_noise_psk_(const SavedNoisePsk &new_psk, const LogString } bool APIServer::load_and_apply_noise_psk_() { - SavedNoisePsk saved{}; - if (!this->noise_pref_.load(&saved)) + // Load into a temp so a failed read cannot disturb the key in use + SavedNoisePsk loaded{}; + if (!this->noise_pref_.load(&loaded)) return false; - this->set_noise_psk(saved.psk); + this->saved_psk_ = loaded; + // An unprovisioned device stores the reserved all-zeros key, which is no key + const bool has_key = !noise::NoiseContext::is_all_zeros(this->saved_psk_.psk); + this->noise_ctx_.set_psk(has_key ? this->saved_psk_.psk.data() : nullptr); return true; } bool APIServer::save_noise_psk(noise::psk_t psk, bool make_active) { -#ifdef USE_API_NOISE_PSK_FROM_YAML - // When PSK is set from YAML, this function should never be called - // but if it is, reject the change - ESP_LOGW(TAG, "Key set in YAML"); - return false; -#else - auto &old_psk = this->noise_ctx_.get_psk(); - if (std::equal(old_psk.begin(), old_psk.end(), psk.begin())) { + if (this->saved_psk_.psk == psk) { ESP_LOGW(TAG, "New PSK matches old"); return true; } @@ -614,15 +612,8 @@ bool APIServer::save_noise_psk(noise::psk_t psk, bool make_active) { } #endif return result; -#endif } bool APIServer::clear_noise_psk(bool make_active) { -#ifdef USE_API_NOISE_PSK_FROM_YAML - // When PSK is set from YAML, this function should never be called - // but if it is, reject the change - ESP_LOGW(TAG, "Key set in YAML"); - return false; -#else SavedNoisePsk empty_psk{}; bool result = this->update_noise_psk_(empty_psk, LOG_STR("Noise PSK cleared"), LOG_STR("Failed to clear Noise PSK"), make_active); @@ -634,8 +625,8 @@ bool APIServer::clear_noise_psk(bool make_active) { } #endif return result; -#endif } +#endif // USE_API_NOISE_PSK_FROM_YAML #endif #ifdef USE_HOMEASSISTANT_TIME diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 072a583901..618ea4eb11 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -76,9 +76,14 @@ class APIServer final : public Component, APIBuffer &get_shared_buffer_ref() { return shared_write_buffer_; } #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML + // Runtime key changes exist for the provisioning path only (not lambdas); + // with a yaml key they compile out bool save_noise_psk(noise::psk_t psk, bool make_active = true); bool clear_noise_psk(bool make_active = true); - void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#endif + /// psk points at 32 bytes that live in flash for the life of the program + void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } noise::NoiseContext &get_noise_ctx() { return this->noise_ctx_; } #endif // USE_API_NOISE @@ -275,10 +280,12 @@ class APIServer final : public Component, #endif #ifdef USE_API_NOISE +#ifndef USE_API_NOISE_PSK_FROM_YAML bool update_noise_psk_(const SavedNoisePsk &new_psk, const LogString *save_log_msg, const LogString *fail_log_msg, bool make_active); // Load saved PSK from preferences and apply it. Returns true on success. bool load_and_apply_noise_psk_(); +#endif // USE_API_NOISE_PSK_FROM_YAML #endif // USE_API_NOISE #ifdef USE_API_HOMEASSISTANT_STATES // Helper methods to reduce code duplication @@ -358,6 +365,9 @@ class APIServer final : public Component, #ifdef USE_API_NOISE noise::NoiseContext noise_ctx_; +#ifndef USE_API_NOISE_PSK_FROM_YAML + SavedNoisePsk saved_psk_{}; // backs noise_ctx_ for a runtime provisioned key +#endif ESPPreferenceObject noise_pref_; #endif // USE_API_NOISE }; diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index 1fec9e5c9b..f5eb878260 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -2,12 +2,12 @@ import logging import esphome.codegen as cg from esphome.components.noise import ( - decode_encryption_key, encryption_schema, - is_reserved_key, + new_psk_progmem, + static_encryption_key, ) from esphome.components.ota import BASE_OTA_SCHEMA, OTAComponent, ota_to_code -from esphome.config_helpers import merge_config +from esphome.config_helpers import filter_source_files_from_defines, merge_config import esphome.config_validation as cv from esphome.const import ( CONF_API, @@ -31,7 +31,6 @@ import esphome.final_validate as fv from esphome.types import ConfigType CONF_ALLOW_PARTITION_ACCESS = "allow_partition_access" -CONF_CAPTIVE_PORTAL = "captive_portal" _LOGGER = logging.getLogger(__name__) @@ -41,11 +40,10 @@ DEPENDENCIES = ["network"] def AUTO_LOAD(config: ConfigType) -> list[str]: - """Auto-load noise only when encryption is configured.""" + """Auto-load noise only when encryption is configured; the api key offer + inherits it from the api component.""" base = ["sha256", "socket"] - # A falsy config is a tooling probe for the maximal set (None from - # dependency resolution, {} from the components-graph platform probe); - # a validated config always carries defaults, never empty + # A falsy config is a tooling probe for the maximal set if not config or CONF_ENCRYPTION in config: return base + ["noise"] return base @@ -132,12 +130,56 @@ def ota_esphome_final_validate(config: ConfigType) -> None: _validate_no_password_with_encryption(ota_conf) if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: _resolve_encryption_key(encryption_conf, api_conf) - if any( - conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf - ) and any( - CONF_ENCRYPTION in conf for conf in merged_ota_esphome_configs_by_port.values() + elif CONF_PASSWORD in ota_conf and static_encryption_key(api_conf) is not None: + _LOGGER.warning( + "'%s' %s wastes significant flash and RAM (about 3.5 KB and 60 " + "bytes plus the password on the heap): the device already offers " + "encryption with the '%s' %s %s, which authenticates any uploader " + "that takes it, and a password only matters for uploaders without " + "encryption support; remove '%s' and add '%s' under '%s' so " + "uploads use the key and encryption is required", + CONF_OTA, + CONF_PASSWORD, + CONF_API, + CONF_ENCRYPTION, + CONF_KEY, + CONF_PASSWORD, + CONF_ENCRYPTION, + CONF_OTA, + ) + elif ( + CONF_PASSWORD in ota_conf + and CONF_ENCRYPTION in api_conf + and not api_conf[CONF_ENCRYPTION].get(CONF_KEY) + ): + # The CLI still needs the password; whoever provisions the key skips it + _LOGGER.warning( + "The '%s' %s %s provisioned at runtime also authenticates OTA " + "uploads once provisioned; '%s' %s then only guards plaintext " + "uploads. Whoever provisions the key can upload firmware " + "without the password, so add a 'provisioning:' block to limit " + "when that is possible", + CONF_API, + CONF_ENCRYPTION, + CONF_KEY, + CONF_OTA, + CONF_PASSWORD, + ) + # web_server and prometheus keep the shared listener up; the captive + # portal's copy only exists on the fallback AP and is the recovery path + if ( + (CONF_WEB_SERVER in full_conf or "prometheus" in full_conf) + and any(conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf) + and any( + CONF_ENCRYPTION in conf + for conf in merged_ota_esphome_configs_by_port.values() + ) ): - _warn_web_server_ota(full_conf) + _LOGGER.warning( + "OTA encryption does not cover the %s OTA platform; its " + "plaintext /update endpoint accepts the same image", + CONF_WEB_SERVER, + ) full_conf[CONF_OTA] = new_ota_conf fv.full_config.set(full_conf) @@ -152,33 +194,11 @@ def ota_esphome_final_validate(config: ConfigType) -> None: ) -def _warn_web_server_ota(full_conf: ConfigType) -> None: - """The web_server ota platform accepts the same image over plaintext HTTP - with basic auth, bypassing the encryption; warn rather than fail so the - operator keeps the recovery path.""" - if CONF_CAPTIVE_PORTAL in full_conf and CONF_WEB_SERVER not in full_conf: - # The captive_portal auto-load: the endpoint only exists while the - # fallback AP is active - _LOGGER.warning( - "OTA encryption does not cover the %s OTA platform (auto-loaded " - "by captive_portal); the plaintext /update endpoint stays " - "reachable while the fallback AP is active", - CONF_WEB_SERVER, - ) - else: - _LOGGER.warning( - "OTA encryption does not cover the %s OTA platform; its " - "plaintext /update endpoint accepts the same image", - CONF_WEB_SERVER, - ) - - def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) -> None: """Resolve the one encryption key per device into the ota block. An explicit ota key must match the api key, a bare block inherits it, - a runtime provisioned api key cannot be inherited, and the all-zeros - provisioning sentinel is rejected (the device treats it as no key). + a runtime provisioned api key cannot be inherited. """ api_key = api_conf.get(CONF_ENCRYPTION, {}).get(CONF_KEY) if ota_key := encryption_conf.get(CONF_KEY): @@ -201,11 +221,6 @@ def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) - ) else: encryption_conf[CONF_KEY] = api_key - if is_reserved_key(encryption_conf[CONF_KEY]): - raise cv.Invalid( - f"The all-zeros {CONF_KEY} is reserved and provides no protection; " - f"generate a real key with: openssl rand -base64 32" - ) # Also called on merged same-port configs in final validate, where schemas @@ -267,15 +282,9 @@ CONFIG_SCHEMA = cv.All( FINAL_VALIDATE_SCHEMA = ota_esphome_final_validate -def FILTER_SOURCE_FILES() -> list[str]: - """Filter out the noise transport when no ota entry configures encryption.""" - for ota_conf in CORE.config.get(CONF_OTA, []): - if ( - ota_conf.get(CONF_PLATFORM) == CONF_ESPHOME - and ota_conf.get(CONF_ENCRYPTION) is not None - ): - return [] - return ["ota_esphome_noise.cpp"] +FILTER_SOURCE_FILES = filter_source_files_from_defines( + {"ota_esphome_noise.cpp": "USE_OTA_ENCRYPTION"} +) @coroutine_with_priority(CoroPriority.OTA_UPDATES) @@ -296,11 +305,24 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_ALLOW_PARTITION_ACCESS): cg.add_define("USE_OTA_PARTITIONS") - if (encryption_conf := config.get(CONF_ENCRYPTION)) is not None: - # A missing key was resolved from the api component in final validate. - key = encryption_conf[CONF_KEY] + # One key per device: an api encryption block supplies it (static or + # runtime) and offers; the ota block only adds the requirement + api_conf = CORE.config.get(CONF_API) or {} + encryption_conf = config.get(CONF_ENCRYPTION) + own_key = None + if encryption_conf is not None and static_encryption_key(api_conf) is None: + own_key = encryption_conf[CONF_KEY] + if own_key is not None: cg.add_define("USE_OTA_ENCRYPTION") - cg.add(var.set_noise_psk(list(decode_encryption_key(key)))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], own_key))) + elif CONF_ENCRYPTION in api_conf: + cg.add_define("USE_OTA_ENCRYPTION") + cg.add_define("USE_OTA_ENCRYPTION_FROM_API") + if static_encryption_key(api_conf) is None: + # The key arrives at runtime, so the offer has to look for it + cg.add_define("USE_OTA_ENCRYPTION_PROVISIONED") + if encryption_conf is not None: + cg.add_define("USE_OTA_ENCRYPTION_REQUIRED") # Build flag so lwip_fast_select.c (a .c file that can't include defines.h) sees it. cg.add_build_flag("-DUSE_OTA_PLATFORM_ESPHOME") diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 396a47bc52..1005ed214b 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -1,4 +1,7 @@ #include "ota_esphome.h" +#ifdef USE_OTA_ENCRYPTION_FROM_API +#include "esphome/components/api/api_server.h" +#endif #ifdef USE_OTA #ifdef USE_OTA_PASSWORD #include "esphome/components/sha256/sha256.h" @@ -26,6 +29,16 @@ namespace esphome { static const char *const TAG = "esphome.ota"; + +#ifdef USE_OTA_ENCRYPTION +const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { +#ifdef USE_OTA_ENCRYPTION_FROM_API + return api::global_api_server->get_noise_ctx(); +#else + return this->noise_ctx_; +#endif +} +#endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer @@ -97,18 +110,30 @@ void ESPHomeOTAComponent::dump_config() { ESP_LOGCONFIG(TAG, "Over-The-Air updates:\n" " Address: %s:%u\n" - " Version: %d", - network::get_use_address_to(addr_buf), this->port_, USE_OTA_VERSION); + " Version: %d" +#ifdef USE_OTA_ENCRYPTION + "\n Encryption: %s" +#endif + , + network::get_use_address_to(addr_buf), this->port_, USE_OTA_VERSION +#ifdef USE_OTA_ENCRYPTION_REQUIRED + , + LOG_STR_LITERAL("required") +#elif defined(USE_OTA_ENCRYPTION_PROVISIONED) + // A runtime provisioned key may not exist yet + , + this->noise_context_().has_psk() ? LOG_STR_LITERAL("offered, plaintext accepted") + : LOG_STR_LITERAL("offered once the api key is provisioned") +#elif defined(USE_OTA_ENCRYPTION) + , + LOG_STR_LITERAL("offered, plaintext accepted") +#endif + ); #ifdef USE_OTA_PASSWORD if (!this->password_.empty()) { ESP_LOGCONFIG(TAG, " Password configured"); } #endif -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ctx_.has_psk()) { - ESP_LOGCONFIG(TAG, " Encryption configured"); - } -#endif #ifdef USE_OTA_PARTITIONS ESP_LOGCONFIG(TAG, " Partition access allowed\n" @@ -154,10 +179,22 @@ static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_NOISE = 0x08; +// Noise needs the extended protocol: the prologue binds the 2-byte feature ack +static constexpr uint8_t CLIENT_NOISE_FEATURES = + CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_NOISE = 0x04; +inline bool ESPHomeOTAComponent::extended_proto_() const { +#ifdef USE_OTA_ENCRYPTION_REQUIRED + // FEATURE_READ already refused every client without the extended protocol + return true; +#else + return (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL) != 0; +#endif +} + void ESPHomeOTAComponent::handle_handshake_() { /// Handle the OTA handshake and authentication. /// @@ -241,12 +278,9 @@ void ESPHomeOTAComponent::handle_handshake_() { this->ota_features_ = this->handshake_buf_[0]; ESP_LOGV(TAG, "Features: 0x%02X", this->ota_features_); -#ifdef USE_OTA_ENCRYPTION - // Fail closed: with a PSK configured the client must negotiate encryption - // (which requires the extended protocol); refuse plaintext uploads. - static constexpr uint8_t NOISE_REQUIRED_FEATURES = - CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; - if (this->noise_ctx_.has_psk() && (this->ota_features_ & NOISE_REQUIRED_FEATURES) != NOISE_REQUIRED_FEATURES) { +#ifdef USE_OTA_ENCRYPTION_REQUIRED + // `ota: encryption:` requires the client to negotiate encryption + if ((this->ota_features_ & CLIENT_NOISE_FEATURES) != CLIENT_NOISE_FEATURES) { ESP_LOGW(TAG, "Client does not support encryption"); this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_ENCRYPTION_REQUIRED); return; @@ -261,18 +295,21 @@ void ESPHomeOTAComponent::handle_handshake_() { // Compose the feature-ack response. When the client negotiates the extended protocol we emit // a 2-byte response (marker + server feature flags); otherwise we emit the single-byte // legacy response. - this->extended_proto_ = (this->ota_features_ & CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL) != 0; - if (this->extended_proto_) { + if (this->extended_proto_()) { static_assert(HANDSHAKE_BUF_SIZE >= 2, "handshake_buf_ must hold the 2-byte extended-protocol feature ack"); this->handshake_buf_[0] = ota::OTA_RESPONSE_FEATURE_FLAGS; this->handshake_buf_[1] = (supports_compression ? SERVER_FEATURE_SUPPORTS_COMPRESSION : 0); #ifdef USE_OTA_PARTITIONS this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS; #endif -#ifdef USE_OTA_ENCRYPTION - if (this->noise_ctx_.has_psk()) { +#ifdef USE_OTA_ENCRYPTION_PROVISIONED + // A runtime provisioned key may not exist yet + if (this->noise_context_().has_psk()) { this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; } +#elif defined(USE_OTA_ENCRYPTION) + // A yaml key always exists: validation rejects the all-zeros key + this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; #endif } else { this->handshake_buf_[0] = @@ -284,15 +321,15 @@ void ESPHomeOTAComponent::handle_handshake_() { case OTAState::FEATURE_ACK: { static constexpr size_t STANDARD_PROTO_ACK_SIZE = 1; static constexpr size_t EXTENDED_PROTO_ACK_SIZE = 2; - const size_t ack_size = this->extended_proto_ ? EXTENDED_PROTO_ACK_SIZE : STANDARD_PROTO_ACK_SIZE; + const size_t ack_size = this->extended_proto_() ? EXTENDED_PROTO_ACK_SIZE : STANDARD_PROTO_ACK_SIZE; if (!this->try_write_(ack_size, LOG_STR("ack feature"))) { return; } #ifdef USE_OTA_ENCRYPTION - // With a PSK configured the rest of the session runs inside the noise - // transport; the client sends the first handshake frame next, so there - // is nothing to do until data arrives. - if (this->noise_ctx_.has_psk()) { + // Latch the offer actually sent: a key activating between the two + // states must not start a session the client never expects + if ((this->handshake_buf_[1] & SERVER_FEATURE_SUPPORTS_NOISE) != 0 && + (this->ota_features_ & CLIENT_NOISE_FEATURES) == CLIENT_NOISE_FEATURES) { // handshake_buf_ still holds the feature ack composed above; a // would-block re-entry lands here without rebuilding it if (!this->noise_start_session_(this->handshake_buf_[1])) { @@ -412,7 +449,7 @@ void ESPHomeOTAComponent::handle_data_() { // Acknowledge auth OK - 1 byte this->data_write_byte_(ota::OTA_RESPONSE_AUTH_OK); - if (this->extended_proto_) { + if (this->extended_proto_()) { // Read ota type, 1 byte if (!this->data_readall_(buf, 1)) { this->log_read_error_(LOG_STR("OTA type")); diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index fd164b8138..c6f710b3fc 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -44,8 +44,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { } #endif // USE_OTA_PASSWORD -#ifdef USE_OTA_ENCRYPTION - void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#if defined(USE_OTA_ENCRYPTION) && !defined(USE_OTA_ENCRYPTION_FROM_API) + /// psk points at 32 bytes that live in flash for the life of the program + void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } #endif /// Manually set the port OTA should listen on @@ -85,9 +86,12 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { bool writing{false}; // a produced handshake frame is still being flushed uint8_t frame_buf[noise::FRAME_HEADER_SIZE + 1 + noise::MAX_HANDSHAKE_SIZE]; }; + // The api server's live context when the api has encryption, else our own + const noise::NoiseContext &noise_context_() const; bool noise_start_session_(uint8_t server_feature_flags); bool handle_noise_handshake_(); bool noise_try_read_frame_(); + size_t noise_frame_payload_len_(const uint8_t *header, size_t min_len, size_t max_len); bool noise_try_write_frame_(); void noise_send_reject_(const LogString *reason); ssize_t noise_decrypt_(uint8_t *buf, size_t len); @@ -144,7 +148,9 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { std::unique_ptr auth_buf_; #endif // USE_OTA_PASSWORD #ifdef USE_OTA_ENCRYPTION +#ifndef USE_OTA_ENCRYPTION_FROM_API noise::NoiseContext noise_ctx_; +#endif std::unique_ptr noise_; #endif // USE_OTA_ENCRYPTION @@ -166,6 +172,8 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { "OTA_BUFFER_SIZE must fit a full encrypted data frame"); #endif static constexpr uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45}; + // Derived from the feature byte; storing it would pad the trailing bytes + bool extended_proto_() const; #ifdef USE_OTA_PARTITIONS uint32_t running_app_offset_{0}; size_t running_app_size_{0}; @@ -179,7 +187,6 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { uint8_t auth_buf_pos_{0}; uint8_t auth_type_{0}; // Store auth type to know which hasher to use #endif // USE_OTA_PASSWORD - bool extended_proto_{false}; }; } // namespace esphome diff --git a/esphome/components/esphome/ota/ota_esphome_noise.cpp b/esphome/components/esphome/ota/ota_esphome_noise.cpp index 7f8331cf96..7401413d6d 100644 --- a/esphome/components/esphome/ota/ota_esphome_noise.cpp +++ b/esphome/components/esphome/ota/ota_esphome_noise.cpp @@ -3,6 +3,7 @@ #ifdef USE_OTA_ENCRYPTION #include "esphome/components/noise/noise.h" #include "esphome/components/ota/ota_backend.h" +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include @@ -40,24 +41,17 @@ ESPHomeOTAComponent::NoiseSession::~NoiseSession() { * "NoiseOTAInit" | magic(5) | OK,version | client_features | FEATURE_FLAGS,server_flags */ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { + // A provisioned key cleared between the offer and here is not guarded: the + // session runs on the zero key load_psk fills in and fails the client's MAC. + // Default-init: the frame buffer is written before it is read // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) - this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession()); - if (this->noise_ == nullptr) { - ESP_LOGW(TAG, "Session allocation failed"); - this->cleanup_connection_(); - return false; - } - + this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession); static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1; static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags uint8_t prologue[OTA_NOISE_PROLOGUE_INIT_LEN + sizeof(MAGIC_BYTES) + PROLOGUE_ACK_LEN + PROLOGUE_CLIENT_FEATURES_LEN + PROLOGUE_FEATURE_ACK_LEN]; -#ifdef USE_ESP8266 - memcpy_P(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); -#else - std::memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); -#endif + progmem_memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); uint8_t *p = prologue + OTA_NOISE_PROLOGUE_INIT_LEN; // Magic bytes, already validated in MAGIC_READ std::memcpy(p, MAGIC_BYTES, sizeof(MAGIC_BYTES)); @@ -71,9 +65,13 @@ bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { *p++ = ota::OTA_RESPONSE_FEATURE_FLAGS; *p++ = server_feature_flags; - int err = this->noise_->handshake.init(this->noise_ctx_.get_psk(), prologue, sizeof(prologue)); + // The caller only starts a session when the context holds a key + int err = this->noise_ == nullptr ? NOISE_ERROR_NO_MEMORY + : this->noise_->handshake.init(this->noise_context_(), prologue, sizeof(prologue)); if (err != 0) { - ESP_LOGW(TAG, "Handshake init: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + // Raw noise codes throughout: the name table would cost flash in builds + // where only the OTA uses noise + ESP_LOGW(TAG, "Session init: %d", err); this->cleanup_connection_(); return false; } @@ -105,14 +103,16 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { s.frame_pos = 0; s.frame_len = 0; if (s.frame_buf[noise::FRAME_HEADER_SIZE] != noise::HANDSHAKE_STATUS_OK) { - ESP_LOGW(TAG, "Bad handshake error byte: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); + ESP_LOGW(TAG, "Client rejected the handshake: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); this->cleanup_connection_(); return false; } int err = s.handshake.read_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, payload_len - 1); if (err != 0) { - ESP_LOGW(TAG, "Handshake read: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); - this->noise_send_reject_(noise::reject_reason_for(err)); + // A MAC failure here almost always means the uploader has a different key + const LogString *reason = noise::reject_reason_for(err); + ESP_LOGW(TAG, "Handshake read: %s (%d)", LOG_STR_ARG(reason), err); + this->noise_send_reject_(reason); this->cleanup_connection_(); return false; } @@ -123,7 +123,7 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { int err = s.handshake.write_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, noise::MAX_HANDSHAKE_SIZE, msg_len); if (err != 0) { - ESP_LOGW(TAG, "Handshake write: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Handshake write: %d", err); this->cleanup_connection_(); return false; } @@ -138,7 +138,7 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { case noise::NoiseResponderHandshake::Action::ACTION_SPLIT: { int err = s.handshake.split(s.send_cipher, s.recv_cipher); if (err != 0) { - ESP_LOGW(TAG, "Handshake split: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Handshake split: %d", err); this->cleanup_connection_(); return false; } @@ -154,33 +154,41 @@ bool ESPHomeOTAComponent::handle_noise_handshake_() { } } +/// Payload length from a frame header, or 0 (logged) when the indicator or +/// the length is out of range. Callers pass min_len >= 1 so 0 is never valid. +size_t ESPHomeOTAComponent::noise_frame_payload_len_(const uint8_t *header, size_t min_len, size_t max_len) { + const size_t payload_len = encode_uint16(header[1], header[2]); + if (header[0] != noise::FRAME_INDICATOR || payload_len < min_len || payload_len > max_len) { + ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], payload_len); + return 0; + } + return payload_len; +} + /// Non-blocking read of one handshake frame into the session buffer. bool ESPHomeOTAComponent::noise_try_read_frame_() { NoiseSession &s = *this->noise_; - while (s.frame_pos < noise::FRAME_HEADER_SIZE) { - ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, noise::FRAME_HEADER_SIZE - s.frame_pos); - if (!this->handle_read_error_(read, LOG_STR("read noise header"))) { - return false; + while (true) { + // The header first, then the body once the header says how long it is + const uint16_t want = s.frame_len == 0 ? noise::FRAME_HEADER_SIZE : s.frame_len; + if (s.frame_pos < want) { + ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, want - s.frame_pos); + if (!this->handle_read_error_(read, LOG_STR("read noise"))) { + return false; + } + s.frame_pos += read; + continue; } - s.frame_pos += read; - } - if (s.frame_len == 0) { - const uint16_t payload_len = encode_uint16(s.frame_buf[1], s.frame_buf[2]); - if (s.frame_buf[0] != noise::FRAME_INDICATOR || payload_len < 1 || payload_len > 1 + noise::MAX_HANDSHAKE_SIZE) { - ESP_LOGW(TAG, "Bad handshake frame: 0x%02X, %u bytes", s.frame_buf[0], payload_len); + if (s.frame_len != 0) { + return true; + } + const size_t payload_len = this->noise_frame_payload_len_(s.frame_buf, 1, 1 + noise::MAX_HANDSHAKE_SIZE); + if (payload_len == 0) { this->cleanup_connection_(); return false; } s.frame_len = noise::FRAME_HEADER_SIZE + payload_len; } - while (s.frame_pos < s.frame_len) { - ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, s.frame_len - s.frame_pos); - if (!this->handle_read_error_(read, LOG_STR("read noise frame"))) { - return false; - } - s.frame_pos += read; - } - return true; } /// Non-blocking write of the pending session-buffer frame. @@ -214,7 +222,7 @@ ssize_t ESPHomeOTAComponent::noise_decrypt_(uint8_t *buf, size_t len) { noise_buffer_set_inout(mbuf, buf, len, len); int err = noise_cipherstate_decrypt(this->noise_->recv_cipher, &mbuf); if (err != 0) { - ESP_LOGW(TAG, "Decrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Decrypt: %d", err); return -1; } return mbuf.size; @@ -229,9 +237,8 @@ ssize_t ESPHomeOTAComponent::noise_read_frame_blocking_(uint8_t *buf, size_t min if (!this->readall_(header, sizeof(header))) { return -1; } - const size_t ciphertext_len = encode_uint16(header[1], header[2]); - if (header[0] != noise::FRAME_INDICATOR || ciphertext_len < min_ciphertext || ciphertext_len > max_ciphertext) { - ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], ciphertext_len); + const size_t ciphertext_len = this->noise_frame_payload_len_(header, min_ciphertext, max_ciphertext); + if (ciphertext_len == 0) { return -1; } if (!this->readall_(buf, ciphertext_len)) { @@ -267,7 +274,7 @@ bool ESPHomeOTAComponent::noise_write_byte_(uint8_t byte) { noise_buffer_set_inout(mbuf, frame + noise::FRAME_HEADER_SIZE, 1, 1 + noise::MAC_SIZE); int err = noise_cipherstate_encrypt(this->noise_->send_cipher, &mbuf); if (err != 0) { - ESP_LOGW(TAG, "Encrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + ESP_LOGW(TAG, "Encrypt: %d", err); return false; } noise::write_frame_header(frame, mbuf.size); diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 0f9328a482..a1d9444fc0 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -4,7 +4,9 @@ from typing import Any import esphome.codegen as cg import esphome.config_validation as cv -from esphome.const import CONF_KEY +from esphome.const import CONF_ENCRYPTION, CONF_KEY +from esphome.core import ID +from esphome.cpp_generator import MockObj from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] @@ -23,6 +25,14 @@ def validate_encryption_key(value: Any) -> str: if len(decoded) != 32: raise cv.Invalid("Encryption key must be base64 and 32 bytes long") + if not any(decoded): + # The device treats the all-zeros key as no key at all (it is the + # provisioning sentinel), so it must never reach a build + raise cv.Invalid( + f"The all-zeros {CONF_KEY} is reserved and provides no protection; " + f"omit the {CONF_KEY} to provision it at runtime, or generate a real " + "key with: openssl rand -base64 32" + ) # Return original data for roundtrip conversion return value @@ -45,15 +55,6 @@ def decode_encryption_key(value: str) -> bytes: return decoded -def is_reserved_key(value: str) -> bool: - """Whether the key is the reserved all-zeros provisioning sentinel. - - The device treats it as no key configured, so consumers that require a - real key must reject it. - """ - return not any(decode_encryption_key(value)) - - ENCRYPTION_SCHEMA = cv.Schema( { cv.Optional(CONF_KEY): cv.sensitive(validate_encryption_key), @@ -61,6 +62,21 @@ ENCRYPTION_SCHEMA = cv.Schema( ) +def static_encryption_key(conf: ConfigType) -> str | None: + """The build time key of a component config; None without one or when + the key is provisioned at runtime.""" + return (conf.get(CONF_ENCRYPTION) or {}).get(CONF_KEY) or None + + +def new_psk_progmem(parent_id: ID, key: str) -> MockObj: + """Emit the decoded key as a PROGMEM array; the component keeps a pointer + so the key never occupies RAM.""" + return cg.progmem_array( + ID(f"{parent_id.id}_psk", is_declaration=True, type=cg.uint8), + list(decode_encryption_key(key)), + ) + + def encryption_schema(config: ConfigType | None) -> ConfigType: # A bare `encryption:` block is valid; a missing key means the consumer # falls back to its keyless behavior (api provisioning, ota inheriting diff --git a/esphome/components/noise/noise.cpp b/esphome/components/noise/noise.cpp index 95fab322db..4806706167 100644 --- a/esphome/components/noise/noise.cpp +++ b/esphome/components/noise/noise.cpp @@ -1,5 +1,6 @@ #include "noise.h" #ifdef USE_NOISE +#include "esphome/core/hal.h" #include "esphome/core/log.h" #include @@ -15,6 +16,14 @@ namespace esphome::noise { static const char *const TAG = "noise"; +void NoiseContext::load_psk(psk_t &out) const { + if (this->psk_ == nullptr) { + out.fill(0); + return; + } + progmem_memcpy(out.data(), this->psk_, out.size()); +} + const LogString *noise_err_to_logstr(int err) { if (err == NOISE_ERROR_NO_MEMORY) return LOG_STR("NO_MEMORY"); diff --git a/esphome/components/noise/noise.h b/esphome/components/noise/noise.h index f9da8d35b8..1033d5423c 100644 --- a/esphome/components/noise/noise.h +++ b/esphome/components/noise/noise.h @@ -23,16 +23,16 @@ class NoiseContext { } return acc == 0; } - void set_psk(psk_t psk) { - this->psk_ = psk; - this->has_psk_ = !is_all_zeros(psk); - } - const psk_t &get_psk() const { return this->psk_; } - bool has_psk() const { return this->has_psk_; } + /// psk points at 32 bytes that outlive the context (PROGMEM or caller owned + /// RAM); nullptr means no key. Runtime callers map the all-zeros key to + /// nullptr themselves; validation keeps it out of yaml. + void set_psk(const uint8_t *psk) { this->psk_ = psk; } + /// Copy the key out (flash-aware on ESP8266); all zeros when none is set. + void load_psk(psk_t &out) const; + bool has_psk() const { return this->psk_ != nullptr; } protected: - psk_t psk_{}; - bool has_psk_{false}; + const uint8_t *psk_{nullptr}; }; /// Convert a noise error code to a readable error diff --git a/esphome/components/noise/noise_handshake.cpp b/esphome/components/noise/noise_handshake.cpp index 6d426de012..cc7fa603c4 100644 --- a/esphome/components/noise/noise_handshake.cpp +++ b/esphome/components/noise/noise_handshake.cpp @@ -20,7 +20,7 @@ NoiseResponderHandshake::~NoiseResponderHandshake() { } } -int NoiseResponderHandshake::init(const psk_t &psk, const uint8_t *prologue, size_t prologue_len) { +int NoiseResponderHandshake::init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len) { if (this->handshake_ != nullptr) { noise_handshakestate_free(this->handshake_); this->handshake_ = nullptr; @@ -44,6 +44,9 @@ int NoiseResponderHandshake::init(const psk_t &psk, const uint8_t *prologue, siz HANDSHAKE_STEP_LOG("noise_handshakestate_new_by_id", err); return err; } + // noise-c keeps its own copy, so the key only passes through the stack here + psk_t psk; + ctx.load_psk(psk); err = noise_handshakestate_set_pre_shared_key(this->handshake_, psk.data(), psk.size()); if (err != 0) { HANDSHAKE_STEP_LOG("noise_handshakestate_set_pre_shared_key", err); diff --git a/esphome/components/noise/noise_handshake.h b/esphome/components/noise/noise_handshake.h index 30596f35c2..bf1aa8cb7f 100644 --- a/esphome/components/noise/noise_handshake.h +++ b/esphome/components/noise/noise_handshake.h @@ -36,9 +36,9 @@ class NoiseResponderHandshake { NoiseResponderHandshake(const NoiseResponderHandshake &) = delete; NoiseResponderHandshake &operator=(const NoiseResponderHandshake &) = delete; - /// Create and start the handshake with the given PSK and prologue. A - /// repeated call frees the previous handshake state and starts over. - [[nodiscard]] int init(const psk_t &psk, const uint8_t *prologue, size_t prologue_len); + /// Create and start the handshake with the context's PSK and the prologue. + /// A repeated call frees the previous handshake state and starts over. + [[nodiscard]] int init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len); /// ACTION_FAILED is the catch-all: returned before init(), after split() /// has released the state, and when noise-c reports a failed handshake. [[nodiscard]] Action action() const; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 526adf74f0..9dd1e0ced6 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -244,6 +244,9 @@ #define USE_RUNTIME_STATS #define USE_OTA #define USE_OTA_ENCRYPTION +#define USE_OTA_ENCRYPTION_FROM_API +#define USE_OTA_ENCRYPTION_PROVISIONED +#define USE_OTA_ENCRYPTION_REQUIRED #define USE_OTA_PASSWORD #define USE_OTA_VERSION 2 #define USE_TIME_TIMEZONE diff --git a/esphome/espota2.py b/esphome/espota2.py index ac4cbeeb7c..ce403c398d 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -202,6 +202,49 @@ class OTANetworkError(OTAError): """Network-level OTA failure (timeout, reset, closed connection); retrying may succeed.""" +# Remove before 2027.3.0 +class OTAEncryptionFallback(OTAError): + """The encrypted attempt failed and the caller may retry in plaintext.""" + + +# Remove before 2027.3.0 +PLAINTEXT_FALLBACK_NOTICE = ( + "A device with an api encryption key offers encryption after this " + "install; add 'encryption:' under 'ota: platform: esphome' to require it. " + "This plaintext fallback is removed in 2027.3.0." +) + + +# Remove before 2027.3.0 +class _EncryptionAttempt: + """The key an upload tries and whether it may fall back to plaintext; + a rejected handshake falls back at once, a transport fault only on repeat.""" + + def __init__(self, noise_psk: str | None, plaintext_fallback: bool) -> None: + self.noise_psk = noise_psk + self.plaintext_fallback = plaintext_fallback + self.handshake_faults = 0 + + def handshake_fault_falls_back(self) -> bool: + self.handshake_faults += 1 + return self.plaintext_fallback and self.handshake_faults >= 2 + + def downgrade(self, reason: str) -> None: + _LOGGER.warning( + "%s. Retrying in plaintext; a device that requires encryption " + "refuses it. %s", + reason, + PLAINTEXT_FALLBACK_NOTICE, + ) + self.noise_psk = None + self.plaintext_fallback = False + + +# Remove before 2027.3.0: only the fallback decision needs this distinction +class OTAHandshakeNetworkError(OTANetworkError): + """A transport failure inside the noise handshake; retrying encrypted may succeed.""" + + def _committed_error(err: OTANetworkError) -> OTAError: """Wrap a network failure that happened once the device had the full image. @@ -464,6 +507,7 @@ def perform_ota( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> None: # Validate up front; an out-of-range value would only surface as a # ValueError deep inside send_check, bypassing OTAError handling @@ -528,19 +572,28 @@ def perform_ota( else: features = 0 - if noise_psk: - # Fail closed: never fall back to a plaintext upload when an - # encryption key is configured, an active attacker could otherwise - # strip the feature flag and capture the image (it contains the wifi - # credentials and the api encryption key). - if not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + if noise_psk and not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + if plaintext_fallback: + # Remove before 2027.3.0: older firmware that cannot encrypt still + # gets its update on this connection + _LOGGER.warning( + "The device did not offer OTA encryption; continuing in plaintext. %s", + PLAINTEXT_FALLBACK_NOTICE, + ) + noise_psk = None + else: + # Fail closed: an attacker could otherwise strip the offer and + # capture the image (wifi credentials, api key) raise OTAError( "An OTA encryption key is configured but the device did not " "offer encryption; refusing to send the image in plaintext. " - "If the running firmware predates OTA encryption, first update " - "it without the 'ota: encryption:' block (over a trusted " - "network or via USB), then restore the block and upload again." + "The running firmware predates ESPHome 2026.9.0 or has no " + "'api: encryption: key'. With an api key, install once " + "without the 'ota: encryption:' block (that build offers " + "encryption), then restore it; otherwise flash by serial or " + "the web_server OTA platform." ) + if noise_psk: # The prologue binds every negotiation byte both sides saw, so any # tampering with the plaintext preamble breaks the handshake. prologue = ( @@ -549,8 +602,18 @@ def perform_ota( + bytes([RESPONSE_OK, version, features_to_send]) + bytes([RESPONSE_FEATURE_FLAGS, features]) ) + # Built outside the try: a local failure must never downgrade the upload sock = NoiseSocketWrapper(sock, noise_psk, prologue) - sock.do_handshake() + try: + sock.do_handshake() + except OTANetworkError as err: + # A transport fault: retry encrypted before considering plaintext + raise OTAHandshakeNetworkError(str(err)) from err + except OTAError as err: + # Remove before 2027.3.0 + if plaintext_fallback: + raise OTAEncryptionFallback(str(err)) from err + raise _LOGGER.info("Encrypted connection established") if ota_type != OTA_TYPE_UPDATE_APP: @@ -757,6 +820,7 @@ def run_ota_impl_( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> tuple[int, str | None]: from esphome.core import CORE @@ -795,7 +859,9 @@ def run_ota_impl_( total_attempts = len(res) + EXTRA_UPLOAD_ATTEMPTS last_error = "" reached_device = False - for attempt in range(total_attempts): + attempt = 0 + encryption = _EncryptionAttempt(noise_psk, plaintext_fallback) + while attempt < total_attempts: af, socktype, _, _, sa = res[attempt % len(res)] if reached_device or attempt >= len(res): _LOGGER.info( @@ -815,17 +881,40 @@ def run_ota_impl_( sock.close() _LOGGER.warning("Connecting to %s port %s failed: %s", sa[0], sa[1], err) last_error = f"connecting to {sa[0]} failed: {err}" + attempt += 1 continue _LOGGER.info("Connected to %s", sa[0]) reached_device = True with contextlib.closing(sock), Path(filename).open("rb") as file_handle: try: - perform_ota(sock, password, file_handle, filename, ota_type, noise_psk) + perform_ota( + sock, + password, + file_handle, + filename, + ota_type, + encryption.noise_psk, + encryption.plaintext_fallback, + ) + except OTAEncryptionFallback as err: + # Same address and attempt budget: not a network retry + last_error = str(err) + encryption.downgrade(last_error) + continue + except OTAHandshakeNetworkError as err: + last_error = str(err) + if encryption.handshake_fault_falls_back(): + encryption.downgrade(last_error) + continue + _LOGGER.warning("%s", last_error) + attempt += 1 + continue except OTANetworkError as err: # Transient network failure; retry last_error = str(err) _LOGGER.warning("%s", last_error) + attempt += 1 continue except OTAError as err: # Device-reported error (wrong password, wrong flash size, ...); @@ -847,10 +936,17 @@ def run_ota( filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, noise_psk: str | None = None, + plaintext_fallback: bool = False, ) -> tuple[int, str | None]: try: return run_ota_impl_( - remote_host, remote_port, password, filename, ota_type, noise_psk + remote_host, + remote_port, + password, + filename, + ota_type, + noise_psk, + plaintext_fallback, ) except OTAError as err: _LOGGER.error(err) diff --git a/esphome/wizard.py b/esphome/wizard.py index f7706928e9..897d5f60a1 100644 --- a/esphome/wizard.py +++ b/esphome/wizard.py @@ -148,11 +148,13 @@ def wizard_file(**kwargs: Unpack[WizardFileKwargs]) -> str: if "api_encryption_key" in kwargs: config += f' encryption:\n key: "{kwargs["api_encryption_key"]}"\n' - # Configure OTA + # The api key also secures OTA; a password only serves older uploaders config += "\nota:\n" config += " - platform: esphome\n" if "ota_password" in kwargs: config += f' password: "{kwargs["ota_password"]}"' + elif "api_encryption_key" in kwargs: + config += " encryption:" # Configuring wifi config += "\n\nwifi:\n" @@ -529,20 +531,9 @@ def wizard(path: Path) -> int: safe_print() safe_print("You'll need this key when adding the device to Home Assistant.") sleep(1) - - safe_print() - safe_print( - f"Do you want to set a {color(AnsiFore.GREEN, 'password')} for OTA updates? " - "This can be insecure if you do not trust the WiFi network." - ) - safe_print() - sleep(0.25) - safe_print("Press ENTER for no password") - ota_password = safe_input(color(AnsiFore.BOLD_WHITE, "(password): ")) else: ssid, psk = "", "" api_encryption_key = None - ota_password = "" kwargs = { "path": path, @@ -553,10 +544,9 @@ def wizard(path: Path) -> int: "psk": psk, "type": "basic", } + # The api key also secures OTA updates, so the wizard sets no OTA password if api_encryption_key: kwargs["api_encryption_key"] = api_encryption_key - if ota_password: - kwargs["ota_password"] = ota_password if not wizard_write(**kwargs): return 1 diff --git a/tests/component_tests/noise/test_encryption_key.py b/tests/component_tests/noise/test_encryption_key.py index 10f1eb3d4c..2b79bd5464 100644 --- a/tests/component_tests/noise/test_encryption_key.py +++ b/tests/component_tests/noise/test_encryption_key.py @@ -5,11 +5,7 @@ from __future__ import annotations import pytest from esphome import config_validation as cv -from esphome.components.noise import ( - decode_encryption_key, - is_reserved_key, - validate_encryption_key, -) +from esphome.components.noise import decode_encryption_key, validate_encryption_key KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @@ -41,6 +37,8 @@ def test_decode_encryption_key_rejects_short_decode() -> None: decode_encryption_key("AAECAw==") -def test_is_reserved_key() -> None: - assert is_reserved_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") - assert not is_reserved_key(KEY) +def test_validate_encryption_key_rejects_all_zeros() -> None: + """The all-zeros key is the provisioning sentinel the device treats as no + key, so it never reaches a build.""" + with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): + validate_encryption_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") diff --git a/tests/component_tests/ota/test_esphome_ota.py b/tests/component_tests/ota/test_esphome_ota.py index 873f162555..d3092294dc 100644 --- a/tests/component_tests/ota/test_esphome_ota.py +++ b/tests/component_tests/ota/test_esphome_ota.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable import logging from typing import Any @@ -14,6 +15,7 @@ from esphome.components.esphome.ota import ( _validate_no_password_with_encryption, ota_esphome_final_validate, ) +from esphome.components.noise import static_encryption_key from esphome.const import ( CONF_API, CONF_ENCRYPTION, @@ -115,7 +117,6 @@ def test_non_esphome_ota_unaffected() -> None: API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" OTHER_KEY = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" -ZEROS_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" def test_encryption_key_inherited_from_api() -> None: @@ -197,36 +198,6 @@ def test_encryption_without_any_key_rejected() -> None: fv.full_config.reset(token) -def test_encryption_explicit_all_zeros_key_rejected() -> None: - """The all-zeros key is the provisioning sentinel; the device would treat - it as no PSK and accept plaintext, so it must fail validation.""" - full_conf = { - CONF_OTA: [ - _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}) - ], - } - token = fv.full_config.set(full_conf) - try: - with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): - ota_esphome_final_validate({}) - finally: - fv.full_config.reset(token) - - -def test_encryption_inherited_all_zeros_key_rejected() -> None: - """An all-zeros api key must not silently disable ota encryption either.""" - full_conf = { - CONF_API: {CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}, - CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], - } - token = fv.full_config.set(full_conf) - try: - with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): - ota_esphome_final_validate({}) - finally: - fv.full_config.reset(token) - - def test_encryption_key_mismatch_between_merged_configs_rejected() -> None: """Same-port configs with different encryption keys raise.""" full_conf = { @@ -295,13 +266,14 @@ def test_encryption_explicit_key_with_runtime_provisioned_api_accepted() -> None fv.full_config.reset(token) +@pytest.mark.parametrize("component", ["web_server", "prometheus"]) def test_encryption_with_web_server_ota_warns( - caplog: pytest.LogCaptureFixture, + caplog: pytest.LogCaptureFixture, component: str ) -> None: - """With the web_server component the plaintext /update endpoint is always - on; the combination validates with a warning.""" + """web_server and prometheus keep the shared listener up, so the + plaintext /update endpoint is always on and the combination warns.""" full_conf = { - "web_server": {}, + component: {}, CONF_OTA: [ _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, @@ -316,12 +288,12 @@ def test_encryption_with_web_server_ota_warns( fv.full_config.reset(token) -def test_encryption_with_captive_portal_web_server_ota_warns( +def test_encryption_with_captive_portal_does_not_warn( caplog: pytest.LogCaptureFixture, ) -> None: """captive_portal auto-loads the web_server ota platform without the - web_server component; encryption stays usable and only warns, so the - fallback AP recovery path is not lost.""" + web_server component; its endpoint only exists while the fallback AP is + active and is the intended recovery path, so there is no warning.""" full_conf = { "captive_portal": {}, CONF_OTA: [ @@ -333,7 +305,10 @@ def test_encryption_with_captive_portal_web_server_ota_warns( try: with caplog.at_level(logging.WARNING): ota_esphome_final_validate({}) - assert any("captive_portal" in record.message for record in caplog.records) + assert not any( + "OTA encryption does not cover" in record.message + for record in caplog.records + ) esphome_conf = next( conf for conf in fv.full_config.get()[CONF_OTA] @@ -344,6 +319,100 @@ def test_encryption_with_captive_portal_web_server_ota_warns( fv.full_config.reset(token) +def test_password_with_api_key_warns(caplog: pytest.LogCaptureFixture) -> None: + """A static api key makes the device offer encryption and the CLI take + it, so the password is dead weight; the config validates with a warning.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: API_KEY}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert any("wastes significant flash" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_password_with_runtime_api_key_warns_differently( + caplog: pytest.LogCaptureFixture, +) -> None: + """The CLI still needs the password, but the provisioned key also + authenticates uploads; the warning says so without the flash advice.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + messages = [r.message for r in caplog.records] + assert any("provisioned at runtime also authenticates" in m for m in messages) + assert not any("wastes significant flash" in m for m in messages) + finally: + fv.full_config.reset(token) + + +def test_password_without_api_key_no_warning( + caplog: pytest.LogCaptureFixture, +) -> None: + """Without an api key there is no offer, so nothing to warn about.""" + full_conf = { + CONF_API: {}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_PASSWORD: "pw"})], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any("authenticates" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_web_server_component_without_ota_platform_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + """The web_server component alone has no /update endpoint.""" + full_conf = { + "web_server": {}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any( + "OTA encryption does not cover" in r.message for r in caplog.records + ) + finally: + fv.full_config.reset(token) + + +def test_web_server_ota_platform_alone_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + """Only the web_server component starts the shared listener, so the ota + platform on its own never exposes /update.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any("plaintext /update" in r.message for r in caplog.records) + finally: + fv.full_config.reset(token) + + def test_web_server_ota_without_encryption_unaffected() -> None: """web_server ota stays valid alongside an unencrypted esphome entry.""" full_conf = { @@ -370,20 +439,87 @@ def test_auto_load_pulls_noise_only_for_encryption() -> None: assert "noise" in AUTO_LOAD({}) -def test_filter_source_files_excludes_noise_without_encryption() -> None: - """The noise transport source compiles only for encrypted builds.""" - old_config = CORE.config - try: - CORE.config = {CONF_OTA: [_make_ota_config(port=3232)]} - assert FILTER_SOURCE_FILES() == ["ota_esphome_noise.cpp"] - CORE.config = { - CONF_OTA: [ - _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) - ] - } - assert FILTER_SOURCE_FILES() == [] - finally: - CORE.config = old_config +def test_static_encryption_key() -> None: + """Only a build-time key counts; a runtime provisioned one does not.""" + assert static_encryption_key({}) is None + assert static_encryption_key({CONF_ENCRYPTION: {}}) is None + assert static_encryption_key({CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) == API_KEY + + +@pytest.mark.parametrize( + ("yaml_name", "defines_present", "defines_absent"), + [ + # An api key alone compiles the transport in without requiring it; + # the device uses the api server's key, not a copy + ( + "api_key_offer", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API"}, + {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # A password still guards plaintext uploads on an offering device + ( + "api_key_offer_password", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_PASSWORD"}, + {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # The ota encryption block is what makes the device refuse plaintext + ( + "encryption_required", + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_REQUIRED", + "USE_OTA_ENCRYPTION_FROM_API", + }, + {"USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # Without api encryption the ota key is the device's own + ( + "own_key", + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_REQUIRED"}, + {"USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_ENCRYPTION_PROVISIONED"}, + ), + # A key provisioned at runtime lives in the api server; the device + # offers with it once provisioned and never requires it + ( + "runtime_api_key", + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_FROM_API", + "USE_OTA_ENCRYPTION_PROVISIONED", + }, + {"USE_OTA_ENCRYPTION_REQUIRED"}, + ), + # No api encryption at all keeps the noise glue out of the build + ( + "plain", + set(), + { + "USE_OTA_ENCRYPTION", + "USE_OTA_ENCRYPTION_REQUIRED", + "USE_OTA_ENCRYPTION_FROM_API", + "USE_OTA_ENCRYPTION_PROVISIONED", + }, + ), + ], +) +def test_encryption_offer_codegen( + generate_main: Callable[[str], str], + yaml_name: str, + defines_present: set[str], + defines_absent: set[str], +) -> None: + main_cpp = generate_main( + f"tests/component_tests/ota/test_esphome_ota_{yaml_name}.yaml" + ) + defines = {define.name for define in CORE.defines} + assert defines_present <= defines + assert not (defines_absent & defines) + encrypted = "USE_OTA_ENCRYPTION" in defines_present + own_key = encrypted and "USE_OTA_ENCRYPTION_FROM_API" not in defines_present + assert ("esphome_esphomeotacomponent_id->set_noise_psk(" in main_cpp) is own_key + assert ("set_auth_password(" in main_cpp) is ("USE_OTA_PASSWORD" in defines_present) + # The noise transport source compiles only when the define is set + assert FILTER_SOURCE_FILES() == ([] if encrypted else ["ota_esphome_noise.cpp"]) def test_password_with_encryption_rejected() -> None: diff --git a/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml b/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml new file mode 100644 index 0000000000..ca26eb9f46 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_api_key_offer.yaml @@ -0,0 +1,11 @@ +esphome: + name: ota-offer + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome diff --git a/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml b/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml new file mode 100644 index 0000000000..1e23975690 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_api_key_offer_password.yaml @@ -0,0 +1,12 @@ +esphome: + name: ota-offer-password + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + password: "superlongpasswordthatnoonewillknow" diff --git a/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml b/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml new file mode 100644 index 0000000000..36690038d8 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_encryption_required.yaml @@ -0,0 +1,12 @@ +esphome: + name: ota-encryption-required + +host: + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + encryption: diff --git a/tests/component_tests/ota/test_esphome_ota_own_key.yaml b/tests/component_tests/ota/test_esphome_ota_own_key.yaml new file mode 100644 index 0000000000..b6d1e4200d --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_own_key.yaml @@ -0,0 +1,11 @@ +esphome: + name: ota-own-key + +host: + +api: + +ota: + - platform: esphome + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" diff --git a/tests/component_tests/ota/test_esphome_ota_plain.yaml b/tests/component_tests/ota/test_esphome_ota_plain.yaml new file mode 100644 index 0000000000..c5ca7afcf0 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_plain.yaml @@ -0,0 +1,9 @@ +esphome: + name: ota-plain + +host: + +api: + +ota: + - platform: esphome diff --git a/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml b/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml new file mode 100644 index 0000000000..8825335141 --- /dev/null +++ b/tests/component_tests/ota/test_esphome_ota_runtime_api_key.yaml @@ -0,0 +1,10 @@ +esphome: + name: ota-runtime-key + +host: + +api: + encryption: + +ota: + - platform: esphome diff --git a/tests/components/noise/test_noise_handshake.cpp b/tests/components/noise/test_noise_handshake.cpp index d879a26c43..f2081f2965 100644 --- a/tests/components/noise/test_noise_handshake.cpp +++ b/tests/components/noise/test_noise_handshake.cpp @@ -68,6 +68,14 @@ class Initiator { static const uint8_t PROLOGUE[] = {'t', 'e', 's', 't', 'p', 'r', 'o', 'l', 'o', 'g', 'u', 'e'}; +// The context only points at the key and init() copies it before returning, +// so a temporary context over a temporary key is safe within one call +static NoiseContext ctx_for(const psk_t &psk) { + NoiseContext ctx; + ctx.set_psk(psk.data()); + return ctx; +} + static psk_t make_psk(uint8_t seed) { psk_t psk; for (size_t i = 0; i < psk.size(); i++) { @@ -102,7 +110,7 @@ TEST(NoiseResponderHandshakeTest, MessageMethodsErrorBeforeInit) { TEST(NoiseResponderHandshakeTest, FullHandshakeAndTransportRoundTrip) { const psk_t psk = make_psk(7); NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(psk, PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); EXPECT_EQ(responder.action(), Action::ACTION_READ); Initiator initiator(psk, PROLOGUE, sizeof(PROLOGUE)); @@ -155,8 +163,8 @@ TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { // proves the restart took effect; the old state surviving would fail the // MAC here. NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(make_psk(7), PROLOGUE, sizeof(PROLOGUE)), 0); - ASSERT_EQ(responder.init(make_psk(9), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(7)), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(9)), PROLOGUE, sizeof(PROLOGUE)), 0); EXPECT_EQ(responder.action(), Action::ACTION_READ); Initiator initiator(make_psk(9), PROLOGUE, sizeof(PROLOGUE)); @@ -168,7 +176,7 @@ TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { TEST(NoiseResponderHandshakeTest, WrongPskFailsWithMacFailure) { NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(make_psk(7), PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(make_psk(7)), PROLOGUE, sizeof(PROLOGUE)), 0); Initiator initiator(make_psk(200), PROLOGUE, sizeof(PROLOGUE)); uint8_t msg[MAX_HANDSHAKE_SIZE]; @@ -185,7 +193,7 @@ TEST(NoiseResponderHandshakeTest, MismatchedPrologueFailsWithMacFailure) { // tampered preamble must fail even with the right key. const psk_t psk = make_psk(7); NoiseResponderHandshake responder; - ASSERT_EQ(responder.init(psk, PROLOGUE, sizeof(PROLOGUE)), 0); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); static const uint8_t TAMPERED[] = {'x'}; Initiator initiator(psk, TAMPERED, sizeof(TAMPERED)); diff --git a/tests/components/noise/test_noise_primitives.cpp b/tests/components/noise/test_noise_primitives.cpp index 018be9f717..8687c4b963 100644 --- a/tests/components/noise/test_noise_primitives.cpp +++ b/tests/components/noise/test_noise_primitives.cpp @@ -17,12 +17,17 @@ TEST(NoiseContextTest, AllZerosPskIsReserved) { EXPECT_FALSE(NoiseContext::is_all_zeros(psk)); NoiseContext ctx; + psk_t loaded; EXPECT_FALSE(ctx.has_psk()); - ctx.set_psk(zeros); - EXPECT_FALSE(ctx.has_psk()); - ctx.set_psk(psk); + ctx.load_psk(loaded); + EXPECT_EQ(loaded, zeros); + ctx.set_psk(psk.data()); EXPECT_TRUE(ctx.has_psk()); - EXPECT_EQ(ctx.get_psk(), psk); + ctx.load_psk(loaded); + EXPECT_EQ(loaded, psk); + // Callers map the reserved key to nullptr; the context just stores what it is given + ctx.set_psk(nullptr); + EXPECT_FALSE(ctx.has_psk()); } TEST(WireFormatTest, FrameHeaderIsIndicatorPlusBigEndianLength) { diff --git a/tests/components/ota/api_key_offer.yaml b/tests/components/ota/api_key_offer.yaml new file mode 100644 index 0000000000..8d1814bf7e --- /dev/null +++ b/tests/components/ota/api_key_offer.yaml @@ -0,0 +1,12 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + port: 3290 + password: "superlongpasswordthatnoonewillknow" diff --git a/tests/components/ota/api_runtime_key.yaml b/tests/components/ota/api_runtime_key.yaml new file mode 100644 index 0000000000..8976c92f96 --- /dev/null +++ b/tests/components/ota/api_runtime_key.yaml @@ -0,0 +1,10 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + +ota: + - platform: esphome + port: 3291 diff --git a/tests/components/ota/test-api_key_offer.esp32-idf.yaml b/tests/components/ota/test-api_key_offer.esp32-idf.yaml new file mode 100644 index 0000000000..ecda625521 --- /dev/null +++ b/tests/components/ota/test-api_key_offer.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_key_offer.yaml diff --git a/tests/components/ota/test-api_key_offer.esp8266-ard.yaml b/tests/components/ota/test-api_key_offer.esp8266-ard.yaml new file mode 100644 index 0000000000..ecda625521 --- /dev/null +++ b/tests/components/ota/test-api_key_offer.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_key_offer.yaml diff --git a/tests/components/ota/test-api_runtime_key.esp32-idf.yaml b/tests/components/ota/test-api_runtime_key.esp32-idf.yaml new file mode 100644 index 0000000000..4709a9e45c --- /dev/null +++ b/tests/components/ota/test-api_runtime_key.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_runtime_key.yaml diff --git a/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml b/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml new file mode 100644 index 0000000000..4709a9e45c --- /dev/null +++ b/tests/components/ota/test-api_runtime_key.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include api_runtime_key.yaml diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6777e6cabc..15c5860879 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -162,6 +162,13 @@ def integration_test_dir() -> Generator[Path]: yield Path(tmpdir) +@pytest.fixture +def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Host preferences persist per device name; give the test its own so a + provisioned key never leaks into another run.""" + monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) + + @pytest.fixture def reserved_tcp_port() -> Generator[tuple[int, socket.socket]]: """Reserve an unused TCP port by holding the socket open.""" diff --git a/tests/integration/const.py b/tests/integration/const.py index 6876bbd443..e35d4673af 100644 --- a/tests/integration/const.py +++ b/tests/integration/const.py @@ -9,6 +9,13 @@ API_CONNECTION_TIMEOUT = 30.0 # seconds PORT_WAIT_TIMEOUT = 30.0 # seconds PORT_POLL_INTERVAL = 0.1 # seconds +# The well-known all-zeros provisioning PSK, a key to provision over it, and +# the time the device takes to activate a newly saved key (100 ms timer plus +# margin) +ZERO_PSK = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" +PROVISIONING_PSK = b"bm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm5ubm4=" +KEY_ACTIVATION_DELAY = 0.5 # seconds + # Process shutdown timeouts SIGINT_TIMEOUT = 5.0 # seconds SIGTERM_TIMEOUT = 2.0 # seconds diff --git a/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml b/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml new file mode 100644 index 0000000000..1dedcc9ee1 --- /dev/null +++ b/tests/integration/fixtures/host_ota_api_key_offer_with_password.yaml @@ -0,0 +1,12 @@ +esphome: + name: host-ota-test +host: +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +ota: + - platform: esphome + port: __OTA_PORT__ + password: "hunter2" +logger: + level: DEBUG diff --git a/tests/integration/fixtures/host_ota_provisioned_api_key.yaml b/tests/integration/fixtures/host_ota_provisioned_api_key.yaml new file mode 100644 index 0000000000..aa0a9a66c9 --- /dev/null +++ b/tests/integration/fixtures/host_ota_provisioned_api_key.yaml @@ -0,0 +1,10 @@ +esphome: + name: host-ota-test +host: +api: + encryption: +ota: + - platform: esphome + port: __OTA_PORT__ +logger: + level: DEBUG diff --git a/tests/integration/test_api_zero_psk_provisioning.py b/tests/integration/test_api_zero_psk_provisioning.py index bcea2a2471..f315335d1b 100644 --- a/tests/integration/test_api_zero_psk_provisioning.py +++ b/tests/integration/test_api_zero_psk_provisioning.py @@ -10,34 +10,40 @@ from __future__ import annotations import asyncio import base64 +import socket from aioesphomeapi import InvalidEncryptionKeyAPIError, RequiresEncryptionAPIError import pytest -from .types import APIClientConnectedFactory, RunCompiledFunction +from .conftest import run_binary_and_wait_for_port +from .const import KEY_ACTIVATION_DELAY, LOCALHOST, PROVISIONING_PSK, ZERO_PSK +from .types import ( + APIClientConnectedFactory, + CompileFunction, + ConfigWriter, + RunCompiledFunction, +) -# The well-known provisioning PSK: base64 of 32 zero bytes -ZERO_PSK = base64.b64encode(bytes(32)).decode() -# A real key to provision -NEW_KEY = base64.b64encode(b"n" * 32) -# Time for the device to activate a newly saved key (100ms timer plus margin) -KEY_ACTIVATION_DELAY = 0.5 - - -@pytest.fixture(autouse=True) -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: - """Keep host preferences per-test so every run starts unprovisioned.""" - monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) +pytestmark = pytest.mark.usefixtures("isolated_preferences") +NEW_KEY = PROVISIONING_PSK @pytest.mark.asyncio async def test_api_zero_psk_provisioning( yaml_config: str, - run_compiled: RunCompiledFunction, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], api_client_connected: APIClientConnectedFactory, ) -> None: - """Exercise the reject paths, then provision a key over the zero-PSK channel.""" - async with run_compiled(yaml_config): + """Exercise the reject paths, provision a key over the zero-PSK channel, + and check the key comes back from preferences on the next boot.""" + port, port_socket = reserved_tcp_port + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + port_socket.close() + + async with run_binary_and_wait_for_port(binary_path, LOCALHOST, port): # --- Pre-provisioning reject paths (device state is unchanged) --- # A wrong (non-zero) PSK fails against the zero provisioning PSK @@ -97,6 +103,19 @@ async def test_api_zero_psk_provisioning( async with api_client_connected(timeout=5) as client: await client.device_info() + # The key is loaded from preferences on the next boot + lines: list[str] = [] + async with run_binary_and_wait_for_port( + binary_path, LOCALHOST, port, line_callback=lines.append + ): + async with api_client_connected(noise_psk=NEW_KEY.decode()) as client: + device_info = await client.device_info() + assert device_info.api_encryption_provisionable is False + with pytest.raises(InvalidEncryptionKeyAPIError): + async with api_client_connected(noise_psk=ZERO_PSK, timeout=5) as client: + await client.device_info() + assert any("Loaded saved Noise PSK" in line for line in lines) + @pytest.mark.asyncio async def test_api_zero_psk_provisioning_plaintext( diff --git a/tests/integration/test_host_ota.py b/tests/integration/test_host_ota.py index 4e74814534..f8c122c6e1 100644 --- a/tests/integration/test_host_ota.py +++ b/tests/integration/test_host_ota.py @@ -8,9 +8,12 @@ instance covers the FD_CLOEXEC path. from __future__ import annotations import asyncio +import base64 from collections.abc import Generator from contextlib import contextmanager +from dataclasses import dataclass import functools +from pathlib import Path import socket import pytest @@ -18,10 +21,18 @@ import pytest from esphome import espota2 from .conftest import run_binary, wait_and_connect_api_client -from .const import LOCALHOST, PORT_POLL_INTERVAL, PORT_WAIT_TIMEOUT -from .types import CompileFunction, ConfigWriter +from .const import ( + KEY_ACTIVATION_DELAY, + LOCALHOST, + PORT_POLL_INTERVAL, + PORT_WAIT_TIMEOUT, + PROVISIONING_PSK, + ZERO_PSK, +) +from .types import APIClientConnectedFactory, CompileFunction, ConfigWriter DEVICE_NAME = "host-ota-test" +API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @contextmanager @@ -35,6 +46,14 @@ def _reserve_port() -> Generator[tuple[int, socket.socket]]: s.close() +async def _wait_for_line(lines: list[str], needle: str, timeout: float = 5.0) -> None: + """The config dump prints after every setup, a little after the api port + opens, so wait for it rather than assert on the lines seen so far.""" + async with asyncio.timeout(timeout): + while not any(needle in line for line in lines): + await asyncio.sleep(PORT_POLL_INTERVAL) + + async def _wait_for_port(host: str, port: int, timeout: float) -> None: """Poll until a TCP port accepts connections, or raise TimeoutError.""" loop = asyncio.get_running_loop() @@ -51,6 +70,102 @@ async def _wait_for_port(host: str, port: int, timeout: float) -> None: raise TimeoutError(f"Port {port} on {host} did not open within {timeout}s") +async def _build( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> tuple[int, int, Path]: + """Reserve an OTA port, compile the fixture with it, and release both + ports right before the binary is started.""" + api_port, api_socket = reserved_tcp_port + with _reserve_port() as (ota_port, ota_socket): + config_path = await write_yaml_config( + yaml_config.replace("__OTA_PORT__", str(ota_port)) + ) + binary_path = await compile_esphome(config_path) + api_socket.close() + ota_socket.close() + return api_port, ota_port, binary_path + + +async def _run_ota( + ota_port: int, + password: str | None, + binary_path: Path, + noise_psk: str | None, + plaintext_fallback: bool = False, +) -> int: + """espota2 is blocking; run it in the executor and return its exit code.""" + rc, _ = await asyncio.get_running_loop().run_in_executor( + None, + functools.partial( + espota2.run_ota, + LOCALHOST, + ota_port, + password, + binary_path, + noise_psk=noise_psk, + plaintext_fallback=plaintext_fallback, + ), + ) + return rc + + +@dataclass +class _Device: + """A running host binary and the checks every successful OTA repeats: + a safe reboot, the api port back up, and the pid preserved by execv.""" + + api_port: int + ota_port: int + binary_path: Path + proc: asyncio.subprocess.Process | None = None + reboots: int = 0 + + def __post_init__(self) -> None: + self._rebooted = asyncio.Event() + + def on_log(self, line: str) -> None: + if "Rebooting safely" in line: + self.reboots += 1 + self._rebooted.set() + + async def wait_reboot(self, count: int, timeout: float = 10.0) -> None: + async with asyncio.timeout(timeout): + while self.reboots < count: + self._rebooted.clear() + await self._rebooted.wait() + + async def ota( + self, + password: str | None, + noise_psk: str | None, + msg: str, + plaintext_fallback: bool = False, + ) -> None: + """Upload, then expect the re-exec with the pid preserved.""" + pid_before = self.proc.pid + expected_reboots = self.reboots + 1 + rc = await _run_ota( + self.ota_port, password, self.binary_path, noise_psk, plaintext_fallback + ) + assert rc == 0, msg + await self.wait_reboot(expected_reboots) + await _wait_for_port(LOCALHOST, self.api_port, PORT_WAIT_TIMEOUT) + assert self.proc.returncode is None, "process exited instead of execing" + assert self.proc.pid == pid_before + + async def refused_ota( + self, password: str | None, noise_psk: str | None, msg: str + ) -> None: + """Upload must fail and the device must keep running.""" + rc = await _run_ota(self.ota_port, password, self.binary_path, noise_psk) + assert rc == 1, msg + await asyncio.sleep(0.5) + assert self.proc.returncode is None, "process died on rejected OTA" + + @pytest.mark.asyncio async def test_host_ota_self_update( yaml_config: str, @@ -59,57 +174,34 @@ async def test_host_ota_self_update( reserved_tcp_port: tuple[int, socket.socket], ) -> None: """Self-OTA: upload the running binary back to itself, expect re-exec.""" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - api_socket.close() - ota_socket.close() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + staged = asyncio.Event() - loop = asyncio.get_running_loop() - ota_staged = loop.create_future() - rebooted = loop.create_future() + def on_log(line: str) -> None: + if "OTA staged at" in line: + staged.set() + dev.on_log(line) - def on_log(line: str) -> None: - if not ota_staged.done() and "OTA staged at" in line: - ota_staged.set_result(True) - if not rebooted.done() and "Rebooting safely" in line: - rebooted.set_result(True) + async with run_binary(dev.binary_path, line_callback=on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + async with wait_and_connect_api_client(port=dev.api_port) as client: + info_before = await client.device_info() + assert info_before.name == DEVICE_NAME - async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid - async with wait_and_connect_api_client(port=api_port) as client: - info_before = await client.device_info() - assert info_before.name == DEVICE_NAME + await dev.ota(None, None, "espota2 reported failure") + assert staged.is_set() - # espota2 is blocking; run in executor. - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path - ) - assert rc == 0, "espota2 reported failure" + async with wait_and_connect_api_client(port=dev.api_port) as client: + info_after = await client.device_info() + assert info_after.name == info_before.name - await asyncio.wait_for(ota_staged, timeout=10.0) - await asyncio.wait_for(rebooted, timeout=10.0) - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - - # execv preserves pid; mismatch means external respawn. - assert proc.returncode is None, "process exited instead of execing" - assert proc.pid == pid_before - - async with wait_and_connect_api_client(port=api_port) as client: - info_after = await client.device_info() - assert info_after.name == DEVICE_NAME - assert info_after.name == info_before.name - - # Second OTA: catches FD_CLOEXEC regressions (EADDRINUSE on rebind). - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path - ) - assert rc == 0, "second OTA failed -- listener leaked across execv" - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - assert proc.pid == pid_before + # Second OTA: catches FD_CLOEXEC regressions (EADDRINUSE on rebind). + await dev.ota(None, None, "second OTA failed -- listener leaked across execv") @pytest.mark.asyncio @@ -121,51 +213,110 @@ async def test_host_ota_encrypted( ) -> None: """Encrypted self-OTA succeeds; a plaintext upload to the same device fails.""" pytest.importorskip("aioesphomeapi.noise") - noise_psk = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) - api_socket.close() - ota_socket.close() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await dev.refused_ota( + None, None, "plaintext upload to an encrypted device must fail" + ) + await dev.ota(None, API_KEY, "encrypted OTA reported failure") - loop = asyncio.get_running_loop() - rebooted = loop.create_future() - def on_log(line: str) -> None: - if not rebooted.done() and "Rebooting safely" in line: - rebooted.set_result(True) +@pytest.mark.asyncio +async def test_host_ota_api_key_offer_with_password( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + caplog: pytest.LogCaptureFixture, +) -> None: + """With only an api key the device offers encryption without requiring + it: the password still guards plaintext uploads, the key alone + authenticates an encrypted one, and until 2027.3.0 a failed encrypted + attempt falls back to plaintext.""" + pytest.importorskip("aioesphomeapi.noise") + wrong_key = base64.b64encode(b"w" * 32).decode() + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _wait_for_line(lines, "Encryption: offered") - async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid + await dev.refused_ota( + None, None, "plaintext upload without the password must fail" + ) + await dev.ota( + "hunter2", None, "plaintext upload with the password must succeed" + ) + await dev.ota(None, API_KEY, "encrypted upload with the api key must succeed") - # A plaintext upload must be refused with the device unharmed - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path + # Remove before 2027.3.0: a wrong key falls back to plaintext, which + # the password still guards + with caplog.at_level("WARNING", logger="esphome.espota2"): + await dev.ota( + "hunter2", + wrong_key, + "the plaintext retry with the password must succeed", + plaintext_fallback=True, ) - assert rc == 1, "plaintext upload to an encrypted device must fail" - await asyncio.sleep(0.5) - assert proc.returncode is None, "process died on rejected plaintext OTA" + assert any("Retrying in plaintext" in r.message for r in caplog.records) + await dev.ota( + None, + API_KEY, + "the right api key encrypts without touching the fallback", + plaintext_fallback=True, + ) - # The encrypted upload goes through and the device re-execs - rc, _ = await loop.run_in_executor( - None, - functools.partial( - espota2.run_ota, - LOCALHOST, - ota_port, - None, - binary_path, - noise_psk=noise_psk, - ), - ) - assert rc == 0, "encrypted OTA reported failure" - await asyncio.wait_for(rebooted, timeout=10.0) - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - assert proc.returncode is None, "process exited instead of execing" - assert proc.pid == pid_before + +@pytest.mark.asyncio +@pytest.mark.usefixtures("isolated_preferences") +async def test_host_ota_provisioned_api_key( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + api_client_connected: APIClientConnectedFactory, +) -> None: + """A key provisioned over the api feeds the OTA offer: plaintext works + while unprovisioned, the provisioned key encrypts, the key loaded from + preferences on the next boot keeps encrypting, and plaintext stays + accepted because only the ota block requires encryption.""" + pytest.importorskip("aioesphomeapi.noise") + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _wait_for_line(lines, "once the api key is provisioned") + + await dev.ota( + None, None, "plaintext upload to an unprovisioned device must succeed" + ) + + async with api_client_connected( + port=dev.api_port, noise_psk=ZERO_PSK + ) as client: + assert await client.noise_encryption_set_key(PROVISIONING_PSK) is True + await asyncio.sleep(KEY_ACTIVATION_DELAY) + + key = PROVISIONING_PSK.decode() + await dev.ota( + None, key, "encrypted upload with the provisioned key must succeed" + ) + await dev.ota(None, key, "the key loaded at boot must feed the OTA offer") + await dev.ota(None, None, "plaintext must stay accepted on an offering device") @pytest.mark.asyncio @@ -177,33 +328,25 @@ async def test_host_ota_rejects_garbage( integration_test_dir, ) -> None: """Bogus payload is rejected and the device keeps running.""" - api_port, api_socket = reserved_tcp_port - with _reserve_port() as (ota_port, ota_socket): - yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) - config_path = await write_yaml_config(yaml_config) - binary_path = await compile_esphome(config_path) + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + # 192 bytes that are neither ELF nor Mach-O. + bogus_path = integration_test_dir / "bogus.bin" + bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8) - # 192 bytes that are neither ELF nor Mach-O. - bogus_path = integration_test_dir / "bogus.bin" - bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8) + async with run_binary(dev.binary_path) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + pid_before = proc.pid + rc = await _run_ota(dev.ota_port, None, bogus_path, None) + assert rc == 1 + await asyncio.sleep(0.5) + assert proc.returncode is None, "process died on rejected OTA" + assert proc.pid == pid_before - api_socket.close() - ota_socket.close() - - async with run_binary(binary_path) as (proc, _lines): - await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) - pid_before = proc.pid - - loop = asyncio.get_running_loop() - rc, _ = await loop.run_in_executor( - None, espota2.run_ota, LOCALHOST, ota_port, None, bogus_path - ) - assert rc == 1 - - await asyncio.sleep(0.5) - assert proc.returncode is None, "process died on rejected OTA" - assert proc.pid == pid_before - - async with wait_and_connect_api_client(port=api_port) as client: - info = await client.device_info() - assert info.name == DEVICE_NAME + async with wait_and_connect_api_client(port=dev.api_port) as client: + info = await client.device_info() + assert info.name == DEVICE_NAME diff --git a/tests/unit_tests/test_espota2_noise.py b/tests/unit_tests/test_espota2_noise.py index 5b43d05530..439220f09c 100644 --- a/tests/unit_tests/test_espota2_noise.py +++ b/tests/unit_tests/test_espota2_noise.py @@ -10,12 +10,15 @@ when the installed aioesphomeapi predates the noise module. from __future__ import annotations import base64 +from collections.abc import Callable import hashlib import io +import logging from pathlib import Path import socket import sys import threading +from typing import Any from unittest.mock import Mock, patch import pytest @@ -65,8 +68,12 @@ class FakeEncryptedDevice(threading.Thread): offer_noise: bool = True, require_noise: bool = True, prologue_features_override: int | None = None, + connections: int = 1, + drop_handshakes: int = 0, ) -> None: super().__init__(daemon=True) + self.connections = connections + self.drop_handshakes = drop_handshakes # hang up mid-handshake this many times self.psk = psk self.version = version self.offer_noise = offer_noise @@ -81,10 +88,11 @@ class FakeEncryptedDevice(threading.Thread): def run(self) -> None: try: - sock, _ = self.listener.accept() - sock.settimeout(10) - with sock: - self._serve(sock) + for _ in range(self.connections): + sock, _ = self.listener.accept() + sock.settimeout(10) + with sock: + self._serve(sock) except Exception as err: # noqa: BLE001 - surfaced via join_and_check self.error = err finally: @@ -109,8 +117,23 @@ class FakeEncryptedDevice(threading.Thread): return server_flags = espota2.SERVER_FEATURE_SUPPORTS_NOISE if self.offer_noise else 0 sock.sendall(bytes([espota2.RESPONSE_FEATURE_FLAGS, server_flags])) - if not (self.offer_noise and noise_negotiated): - return # the client fails closed; nothing further arrives + if not (noise_negotiated and self.offer_noise): + # A device that does not require encryption continues in + # plaintext whatever the client asked for, like older firmware + try: + self._transfer( + lambda byte: sock.sendall(bytes([byte])), + lambda length: _recv_exact(sock, length), + lambda remaining: _recv_exact( + sock, min(remaining, espota2.UPLOAD_BLOCK_SIZE) + ), + ) + except ConnectionError: + # A keyed client without fallback fails closed and hangs up + if noise_negotiated and not self.offer_noise: + return + raise + return from cryptography.exceptions import InvalidTag from noise.connection import NoiseConnection @@ -134,6 +157,9 @@ class FakeEncryptedDevice(threading.Thread): msg1 = _recv_frame(sock) assert msg1[0] == 0x00 + if self.drop_handshakes > 0: + self.drop_handshakes -= 1 + return # a transport fault: the socket closes with no reply try: proto.read_message(msg1[1:]) except InvalidTag: @@ -149,6 +175,20 @@ class FakeEncryptedDevice(threading.Thread): assert len(plaintext) == length, "control units must be one per frame" return plaintext + def recv_data(_remaining: int) -> bytes: + plaintext = proto.decrypt(_recv_frame(sock)) + assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT + return plaintext + + self._transfer(send_byte, recv_unit, recv_data) + + def _transfer( + self, + send_byte: Callable[[int], None], + recv_unit: Callable[[int], bytes], + recv_data: Callable[[int], bytes], + ) -> None: + """The post-handshake exchange, identical over both transports.""" send_byte(espota2.RESPONSE_AUTH_OK) recv_unit(1) # ota type size = int.from_bytes(recv_unit(4), "big") @@ -159,9 +199,7 @@ class FakeEncryptedDevice(threading.Thread): received = b"" acked = 0 while len(received) < size: - plaintext = proto.decrypt(_recv_frame(sock)) - assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT - received += plaintext + received += recv_data(size - len(received)) if self.version >= espota2.OTA_VERSION_2_0: while acked + espota2.UPLOAD_BLOCK_SIZE <= len(received) or ( len(received) == size and acked < size @@ -176,7 +214,10 @@ class FakeEncryptedDevice(threading.Thread): def _upload( - device: FakeEncryptedDevice, firmware: bytes, noise_psk: str | None + device: FakeEncryptedDevice, + firmware: bytes, + noise_psk: str | None, + plaintext_fallback: bool = False, ) -> None: device.start() sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -184,12 +225,35 @@ def _upload( sock.connect(("127.0.0.1", device.port)) try: espota2.perform_ota( - sock, None, io.BytesIO(firmware), Path("firmware.bin"), noise_psk=noise_psk + sock, + None, + io.BytesIO(firmware), + Path("firmware.bin"), + noise_psk=noise_psk, + plaintext_fallback=plaintext_fallback, ) finally: sock.close() +def _run_ota( + device: FakeEncryptedDevice, firmware: bytes, tmp_path: Path, noise_psk: str +) -> int: + """Drive the retry loop, which is where the plaintext fallback reconnects.""" + path = tmp_path / "firmware.bin" + path.write_bytes(firmware) + device.start() + rc, _ = espota2.run_ota( + "127.0.0.1", + device.port, + None, + path, + noise_psk=noise_psk, + plaintext_fallback=True, + ) + return rc + + def test_encrypted_upload_success() -> None: """A full encrypted v2 upload spanning several 8192-byte blocks.""" pytest.importorskip("aioesphomeapi.noise") @@ -240,6 +304,56 @@ def test_client_fails_closed_when_device_lacks_encryption() -> None: device.join_and_check() +# Remove before 2027.3.0 +def test_fallback_when_device_does_not_offer(caplog: pytest.LogCaptureFixture) -> None: + """The api key is tried opportunistically; an older device that cannot + encrypt still gets its update, with a warning.""" + firmware = b"firmware" + device = FakeEncryptedDevice(offer_noise=False, require_noise=False) + with patch("time.sleep"), caplog.at_level(logging.WARNING): + _upload(device, firmware, PSK, plaintext_fallback=True) + device.join_and_check() + assert device.received == firmware + assert any("fallback is removed in 2027.3.0" in r.message for r in caplog.records) + + +# Remove before 2027.3.0 +@pytest.mark.parametrize( + ("device_kwargs", "expected_rc", "fell_back"), + [ + # A wrong key against an offering device reconnects in plaintext + ({"psk": OTHER_PSK, "require_noise": False, "connections": 2}, 0, True), + # The plaintext retry is refused by a device that requires encryption + ({"psk": OTHER_PSK, "require_noise": True, "connections": 2}, 1, True), + # A dropped connection inside the handshake is retried encrypted + ({"require_noise": False, "connections": 2, "drop_handshakes": 1}, 0, False), + # A second transport fault inside the handshake falls back + ({"require_noise": False, "connections": 3, "drop_handshakes": 2}, 0, True), + ], + ids=["wrong_key", "wrong_key_required", "one_fault", "two_faults"], +) +def test_fallback_through_the_retry_loop( + caplog: pytest.LogCaptureFixture, + tmp_path: Path, + device_kwargs: dict[str, Any], + expected_rc: int, + fell_back: bool, +) -> None: + pytest.importorskip("aioesphomeapi.noise") + firmware = b"firmware" + device = FakeEncryptedDevice(**device_kwargs) + with patch("time.sleep"), caplog.at_level(logging.WARNING): + rc = _run_ota(device, firmware, tmp_path, PSK) + device.join_and_check() + assert rc == expected_rc + assert (device.received == firmware) is (expected_rc == 0) + assert ( + any("Retrying in plaintext" in r.message for r in caplog.records) is fell_back + ) + if expected_rc == 1: + assert any("requires an encrypted OTA" in r.message for r in caplog.records) + + def test_plaintext_client_gets_encryption_required_error() -> None: """A client without a key gets the device's 0x94 error message.""" device = FakeEncryptedDevice() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 5372a7203d..8fb9b7376e 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -2108,7 +2108,13 @@ def test_upload_program_ota_success( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, "secret", expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + "secret", + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -2140,10 +2146,77 @@ def test_upload_program_ota_encryption_key( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, key + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + key, + plaintext_fallback=False, ) +def test_upload_program_ota_api_key_opportunistic( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """Without an ota encryption block the api key is tried with a plaintext + fallback (removed in 2027.3.0).""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + key = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + config = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: key}}, + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}], + } + exit_code, _ = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + expected_firmware = ( + tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" + ) + mock_run_ota.assert_called_once_with( + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + key, + plaintext_fallback=True, + ) + + +@pytest.mark.parametrize( + "api_conf", + [{}, {CONF_ENCRYPTION: {}}], + ids=["no_encryption", "runtime_key"], +) +def test_upload_program_ota_no_usable_api_key_stays_plaintext( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, + api_conf: dict[str, Any], +) -> None: + """A missing or runtime provisioned api key gives the uploader nothing + to try.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_API: api_conf, + CONF_OTA: [{CONF_PLATFORM: CONF_ESPHOME, CONF_PORT: 3232}], + } + exit_code, _ = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + assert mock_run_ota.call_args.args[5] is None + assert mock_run_ota.call_args.kwargs == {"plaintext_fallback": False} + + def test_upload_program_ota_encryption_without_key_fails_closed( mock_run_ota: Mock, mock_get_port_type: Mock, @@ -2194,7 +2267,13 @@ def test_upload_program_ota_with_file_arg( assert exit_code == 0 assert host == "192.168.1.100" mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, Path("custom.bin"), OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + Path("custom.bin"), + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -2250,6 +2329,7 @@ def test_upload_program_ota_partition_table_with_file_arg( partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, None, + plaintext_fallback=False, ) @@ -2312,6 +2392,7 @@ def test_upload_program_ota_partition_table_mqttip( partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, None, + plaintext_fallback=False, ) @@ -2500,6 +2581,7 @@ def test_upload_program_ota_bootloader_with_file_arg( bootloader_file, OTA_TYPE_UPDATE_BOOTLOADER, None, + plaintext_fallback=False, ) @@ -2988,7 +3070,13 @@ def test_upload_program_ota_with_mqtt_resolution( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) @@ -3038,7 +3126,13 @@ def test_upload_program_ota_with_mqtt_empty_broker( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.50"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.50"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) # Verify warning was logged assert "MQTT IP discovery failed" in caplog.text @@ -5211,6 +5305,7 @@ def test_upload_program_ota_static_ip_with_mqttip( expected_firmware, OTA_TYPE_UPDATE_APP, None, + plaintext_fallback=False, ) @@ -5261,6 +5356,7 @@ def test_upload_program_ota_multiple_mqttip_resolves_once( expected_firmware, OTA_TYPE_UPDATE_APP, None, + plaintext_fallback=False, ) @@ -5438,7 +5534,13 @@ def test_upload_program_ota_mqtt_timeout_fallback( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None + ["192.168.1.100"], + 3232, + None, + expected_firmware, + OTA_TYPE_UPDATE_APP, + None, + plaintext_fallback=False, ) diff --git a/tests/unit_tests/test_wizard.py b/tests/unit_tests/test_wizard.py index 244e4eb5a1..f57ae71ae6 100644 --- a/tests/unit_tests/test_wizard.py +++ b/tests/unit_tests/test_wizard.py @@ -37,7 +37,6 @@ def wizard_answers() -> list[str]: "nodemcuv2", # board "SSID", # ssid "psk", # wifi password - "", # ota password (empty for no password) ] @@ -101,6 +100,25 @@ def test_config_file_should_include_ota(default_config: dict[str, Any]): assert "ota:" in config +def test_config_file_should_use_encryption_when_api_key_set( + default_config: dict[str, Any], +): + """ + With an API encryption key and no OTA password the OTA block reuses the key + """ + # Given + default_config["api_encryption_key"] = ( + "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + ) + + # When + config = wz.wizard_file(**default_config) + + # Then + assert "ota:\n - platform: esphome\n encryption:" in config + assert "password" not in config.split("ota:")[1].split("wifi:")[0] + + def test_config_file_should_include_ota_when_password_set( default_config: dict[str, Any], ): @@ -630,15 +648,15 @@ def test_wizard_write_protects_existing_config( assert config_file.read_text() == original_content -def test_wizard_accepts_ota_password( +def test_wizard_uses_the_api_key_for_ota( tmp_path: Path, monkeypatch: MonkeyPatch, wizard_answers: list[str] ): """ - The wizard should pass ota_password to wizard_write when the user provides one + The wizard generates an api key and does not ask for an OTA password; + the key secures OTA updates """ # Given - wizard_answers[5] = "my_ota_password" # Set OTA password config_file = tmp_path / "test.yaml" input_mock = MagicMock(side_effect=wizard_answers) monkeypatch.setattr("builtins.input", input_mock) @@ -653,8 +671,9 @@ def test_wizard_accepts_ota_password( # Then assert retval == 0 call_kwargs = wizard_write_mock.call_args.kwargs - assert "ota_password" in call_kwargs - assert call_kwargs["ota_password"] == "my_ota_password" + assert "api_encryption_key" in call_kwargs + assert "ota_password" not in call_kwargs + assert input_mock.call_count == len(wizard_answers) def test_wizard_accepts_rpipico_board(tmp_path: Path, monkeypatch: MonkeyPatch): From 96b1a03ea493a7281158907c6dd98184a48c05f2 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:05:16 +0000 Subject: [PATCH 121/147] Bump bundled esphome-device-builder to 1.14.4 (#19006) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e875851bfb..da76ab7b6a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.3 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.4 RUN \ platformio settings set enable_telemetry No \ From c1aa41f276e4bc2b05f4b45031229623d93ab84a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:12:52 +0200 Subject: [PATCH 122/147] [noise] Bump noise-c to 0.1.24 and libsodium to 1.10021.6 (#18989) --- esphome/components/noise/__init__.py | 4 +-- platformio.ini | 6 ++-- tests/script/test_platformio_install_deps.py | 34 ++++++++++---------- tests/unit_tests/test_platformio_prefetch.py | 4 +-- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index a1d9444fc0..4de706120e 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -88,12 +88,12 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.21") + cg.add_library("esphome/noise-c", "0.1.24") # noise-c depends on libsodium, but declaring it here too lets the # library manager see the full set up front instead of discovering # libsodium only after noise-c has downloaded, so the two can download # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.4") + cg.add_library("esphome/libsodium", "1.10021.6") # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/platformio.ini b/platformio.ini index fcf7caa7c7..779a05e7de 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.21 ; noise (api, ota) + esphome/noise-c@0.1.24 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -244,7 +244,7 @@ lib_deps = ${common:idf-component-libs.lib_deps} ESP32Async/ESPAsyncWebServer@3.9.6 ; web_server_base droscy/esp_wireguard@0.4.5 ; wireguard - esphome/noise-c@0.1.21 ; noise (api, ota) + esphome/noise-c@0.1.24 ; noise (api, ota) ESP32Async/AsyncTCP@3.4.5 ; async_tcp DNSServer ; captive_portal heman/AsyncMqttClient-esphome@2.0.0 ; mqtt @@ -641,7 +641,7 @@ build_unflags = extends = common platform = platformio/native lib_deps = - esphome/noise-c@0.1.21 ; used by noise (api, ota) + esphome/noise-c@0.1.24 ; used by noise (api, ota) lvgl/lvgl@9.5.0 ; lvgl build_flags = ${common.build_flags} diff --git a/tests/script/test_platformio_install_deps.py b/tests/script/test_platformio_install_deps.py index a263d7937f..4f7f5a4a4c 100644 --- a/tests/script/test_platformio_install_deps.py +++ b/tests/script/test_platformio_install_deps.py @@ -35,8 +35,8 @@ def _load_script(): def test_spec_key_collapses_destinations() -> None: """Two specs delivering one package share a directory and one key.""" mod = _load_script() - assert mod.spec_key("esphome/noise-c @ 0.1.21") == "noise-c" - assert mod.spec_key("esphome/noise-c@0.1.21") == "noise-c" + assert mod.spec_key("esphome/noise-c @ 0.1.24") == "noise-c" + assert mod.spec_key("esphome/noise-c@0.1.24") == "noise-c" assert mod.spec_key("ESP32Async/AsyncTCP @ ^3.4.10") == mod.spec_key( "esp32async/asynctcp @ 3.5.0" ) @@ -54,23 +54,23 @@ def test_parse_specs_and_cli_args(tmp_path: Path) -> None: "[env:a]\n" "platform = fake/platform@1\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.21\n" + " esphome/noise-c @ 0.1.24\n" " ${common.lib_deps}\n" " internal_lib\n" "[env:b]\n" "lib_deps =\n" - " esphome/noise-c @ 0.1.21\n" + " esphome/noise-c @ 0.1.24\n" ) mod = _load_script() args = Namespace(libraries=True, platforms=True, tools=False) libs, platforms, tools = mod.parse_specs(str(ini), args) # exact-string duplicates collapse; distinct version pins survive - assert libs == ["esphome/noise-c @ 0.1.21"] + assert libs == ["esphome/noise-c @ 0.1.24"] assert platforms == ["fake/platform@1"] assert tools == [] assert mod.build_cli_args(libs, platforms, tools) == [ "-l", - "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.24", "-p", "fake/platform@1", ] @@ -162,13 +162,13 @@ def test_parallel_install_behavior(tmp_path: Path) -> None: mod.parallel_install( cls, [ - "esphome/noise-c @ 0.1.21", - "esphome/noise-c @ 0.1.21", + "esphome/noise-c @ 0.1.24", + "esphome/noise-c @ 0.1.24", "esphome/already @ 1.0", "https://x/framework.tar.xz", ], ) - assert cls.calls == ["esphome/noise-c @ 0.1.21"] + assert cls.calls == ["esphome/noise-c @ 0.1.24"] assert cls.lock_events == ["lock", "unlock"] @@ -205,7 +205,7 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, {"name": "SPI"}, ], @@ -213,12 +213,12 @@ def test_parallel_install_runs_dependency_waves(tmp_path: Path) -> None: {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21", "esphome/wg @ 1.0"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24", "esphome/wg @ 1.0"]) assert len(cls.calls) == 3 # the shared dep installs exactly once assert {mod.spec_key(c) for c in cls.calls} == {"noise-c", "wg", "libsodium"} # Wave-1 strings carry no compatibility; the dependency wave does compats = dict(cls.compat_calls) - assert compats["esphome/noise-c @ 0.1.21"] is None + assert compats["esphome/noise-c @ 0.1.24"] is None dep_compat = next(v for k, v in cls.compat_calls if "libsodium" in k) assert dep_compat is not None # mirrors pio's install_dependency @@ -229,11 +229,11 @@ def test_dependency_wave_excludes_url_specs(tmp_path: Path) -> None: mod = _load_script() cls = _reset_fake(str(tmp_path)) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"name": "vendored", "version": "https://github.com/x/y.git"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) assert {mod.spec_key(c) for c in cls.calls} == {"noise-c"} @@ -348,13 +348,13 @@ def test_warm_store_still_walks_dependencies(tmp_path: Path) -> None: """Already-installed top-level packages still feed the dependency wave; a warm store can be missing a transitive dep.""" mod = _load_script() - cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.21"}) + cls = _reset_fake(str(tmp_path), installed={"esphome/noise-c @ 0.1.24"}) cls.deps = { - "esphome/noise-c @ 0.1.21": [ + "esphome/noise-c @ 0.1.24": [ {"owner": "esphome", "name": "libsodium", "version": "^1.0"}, ], } - mod.parallel_install(cls, ["esphome/noise-c @ 0.1.21"]) + mod.parallel_install(cls, ["esphome/noise-c @ 0.1.24"]) assert [mod.spec_key(c) for c in cls.calls] == ["libsodium"] diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 379ef52ebd..fb79885736 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1576,7 +1576,7 @@ def test_preinstall_runs_dependency_waves(tmp_path: Path) -> None: {"name": "SPI"}, ] m.dependency_to_spec.side_effect = lambda dep: _FakeSpec(name=dep["name"]) - pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c", "libsodium"] # dep deduped, SPI left out # The dep wave carries its compatibility so _install searches qualified dep_call = m._install.call_args_list[-1] @@ -1596,7 +1596,7 @@ def test_preinstall_dependency_wave_skips_seen_names(tmp_path: Path) -> None: m._install.side_effect = lambda spec, skip_dependencies, compatibility=None: ( installed.append(getattr(spec, "name", str(spec))) ) - pf._preinstall(m, [("noise-c@0.1.21", _FakeSpec(name="noise-c"))]) + pf._preinstall(m, [("noise-c@0.1.24", _FakeSpec(name="noise-c"))]) assert installed == ["noise-c"] From 5d2ddc658c3db2431fb71dfc78dc2df885f1cf78 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:26:44 +0200 Subject: [PATCH 123/147] [mdns] Guard LEAmDNS main loop calls against lwIP re-entrancy on ESP8266 (#18990) --- esphome/components/mdns/__init__.py | 2 + esphome/components/mdns/mdns_esp8266.cpp | 51 +++++++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index f039bb69f0..c8020104b3 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -192,6 +192,8 @@ async def to_code(config: ConfigType) -> None: if CORE.using_arduino: if CORE.is_esp8266: cg.add_library("ESP8266mDNS", None) + # No MDNS global in the build; mdns_esp8266.cpp owns a guarded MDNSResponder + cg.add_build_flag("-DNO_GLOBAL_MDNS") elif CORE.is_rp2: cg.add_library("LEAmDNS", None) diff --git a/esphome/components/mdns/mdns_esp8266.cpp b/esphome/components/mdns/mdns_esp8266.cpp index 1f0b3c9519..0e600d3bac 100644 --- a/esphome/components/mdns/mdns_esp8266.cpp +++ b/esphome/components/mdns/mdns_esp8266.cpp @@ -13,8 +13,47 @@ namespace esphome::mdns { +// Main-loop calls into LEAmDNS that send (update() and close(); begin(), addService() and +// the scheduled restart never reach a send) can yield inside UdpContext::sendTimeout(); a +// packet arriving then re-enters LEAmDNS from lwIP on the same UdpContext and both sides +// free the same tx pbufs (#18760). Received packets stay queued during such a call and are +// processed from the main loop afterwards. +class GuardedMDNSResponder : public ::esp8266::MDNSImplementation::MDNSResponder { + public: + void update_guarded() { this->run_guarded_(&GuardedMDNSResponder::update); } + void close_guarded() { this->run_guarded_(&GuardedMDNSResponder::close); } + + private: + void run_guarded_(bool (GuardedMDNSResponder::*fn)()) { + UdpContext *ctx = this->m_pUDPContext; + if (ctx == nullptr) { + (this->*fn)(); + return; + } + // Set every time: a restart replaces the context together with its stock handler. Only + // begin() and the scheduled netif callback restart, never update() or close(), so the + // context cannot change underneath this call. + ctx->onRx([this]() { + if (!this->in_loop_call_) { + this->_callProcess(); + } + }); + this->in_loop_call_ = true; + (this->*fn)(); + // close() releases the context; a yield in here queues further packets for this loop too + while (this->m_pUDPContext != nullptr && this->m_pUDPContext->next()) { + this->_parseMessage(); + } + this->in_loop_call_ = false; + } + + volatile bool in_loop_call_{false}; +}; + +static GuardedMDNSResponder mdns_responder; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + static void register_esp8266(MDNSComponent *, StaticVector &services) { - MDNS.begin(App.get_name().c_str()); + mdns_responder.begin(App.get_name().c_str()); for (const auto &service : services) { // Strip the leading underscore from the proto and service_type. While it is @@ -30,10 +69,10 @@ static void register_esp8266(MDNSComponent *, StaticVectoris_roaming() || (!wifi->is_connected() && !wifi->is_ap_active())) return; #endif - MDNS.update(); + mdns_responder.update_guarded(); }); this->set_timeout(MDNS_POLL_STOP_ID, MDNS_POLL_WINDOW_MS, [this]() { this->cancel_interval(MDNS_POLL_ID); }); } @@ -81,7 +120,7 @@ void MDNSComponent::on_ip_state(const network::IPAddresses &ips, const network:: #endif void MDNSComponent::on_shutdown() { - MDNS.close(); + mdns_responder.close_guarded(); delay(10); } From e0e85db822309dd8fe17043552405c8c8b8374af Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 7 Sep 2026 00:28:31 +0200 Subject: [PATCH 124/147] [core] Show the other downloader's progress while a prefetch job waits on its lock (#18983) --- esphome/framework_helpers.py | 61 +++++++++++++- esphome/platformio/prefetch.py | 87 ++++++++++---------- esphome/platformio/registry.py | 67 ++++++++++----- tests/unit_tests/conftest.py | 39 ++++++++- tests/unit_tests/test_framework_helpers.py | 17 ++++ tests/unit_tests/test_platformio_prefetch.py | 87 ++++++++++++++++++-- tests/unit_tests/test_platformio_registry.py | 73 ++++++++++++++-- 7 files changed, 348 insertions(+), 83 deletions(-) diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 82bc0d3727..fc2a18a6ec 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -23,6 +23,7 @@ from esphome.net_retry import ( ) if TYPE_CHECKING: + from filelock import FileLock import requests PathType = str | os.PathLike @@ -909,6 +910,61 @@ def _part_path(dest: Path) -> Path: return dest.with_name(dest.name + ".part") +def downloaded_bytes(dest: Path, size: int | None = None) -> int: + """Bytes of ``dest`` on disk (its ``.part`` while streaming), capped at ``size``.""" + done = 0 + for candidate in (_part_path(dest), dest): + try: + done = candidate.stat().st_size + break + except FileNotFoundError: + continue + return done if size is None else min(done, size) + + +# Short lock-acquire slices so a waiting worker still observes Ctrl-C +_DOWNLOAD_LOCK_POLL = 1 + +# Waiting on another process's download; past this the caller leaves the +# file to its holder (the later sequential install waits on the same lock) +DOWNLOAD_LOCK_TIMEOUT = 60 + + +class DownloadLockUnavailable(OSError): + """The lock file cannot be used at all (a lock-less filesystem).""" + + +def wait_for_download_lock( + lock: "FileLock", + tracker: Callable[[int], None], + on_disk: Callable[[], int], + name: str, +) -> None: + """Acquire ``lock``, reporting ``on_disk()`` to ``tracker`` each poll so the + bar follows the holder's download. Raises filelock's ``Timeout`` once + ``DOWNLOAD_LOCK_TIMEOUT`` seconds pass.""" + from filelock import Timeout + + deadline = time.monotonic() + DOWNLOAD_LOCK_TIMEOUT + waiting = False + while True: + try: + lock.acquire(timeout=_DOWNLOAD_LOCK_POLL) + return + except Timeout: + pass + except OSError as err: + # Distinct from an OSError out of on_disk(), which must not + # read as "locks unsupported" + raise DownloadLockUnavailable(*err.args) from err + if not waiting: + waiting = True + _LOGGER.info("Waiting for another process downloading %s", name) + tracker(on_disk()) # raises when the batch is cancelled + if time.monotonic() >= deadline: + raise Timeout(lock.lock_file) + + def discard_partial_download(dest: Path) -> None: """Remove ``dest`` and the resume sidecars of an abandoned download.""" part = _part_path(dest) @@ -1319,10 +1375,7 @@ def download_from_mirrors( ) # Tick with the bytes already on disk so a combined bar holds # steady during the backoff instead of rewinding to zero - done = 0 - if progress is not None: - part = _part_path(path_target) - done = part.stat().st_size if part.is_file() else 0 + done = downloaded_bytes(path_target) if progress is not None else 0 _cancellable_sleep(delay, progress, done) # 3. Report every attempted URL if all mirrors failed. failures spans diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 5097239065..17a06cb9c1 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -33,11 +33,14 @@ import time from typing import Any, NamedTuple from esphome.framework_helpers import ( + DownloadLockUnavailable, content_length, discard_partial_download, + downloaded_bytes, failure_reason, resume_fetch_job, run_batch_downloads, + wait_for_download_lock, warn_prefetch_failures, ) from esphome.helpers import get_bool_env, get_usable_cpu_count, rmtree @@ -61,16 +64,10 @@ _RESOLVE_WORKERS = 8 # A hung child must not block the build; downloads resume on the next run _PREFETCH_TIMEOUT = 20 * 60 -# Waiting on another process's URL download; past this, leave it to pio -_DOWNLOAD_LOCK_TIMEOUT = 60 - # Child exit for a handled, already-warned failure; 1 would collide with # the interpreter's own import-failure exit _EXIT_HANDLED = 3 -# Short lock-acquire slices so a waiting worker still observes Ctrl-C -_URI_LOCK_POLL = 1 - # Resolution errored (vs a clean skip); suppresses the warm sentinel _RESOLVE_FAILED = object() @@ -462,51 +459,54 @@ def _uri_jobs( def _serialized_fetch_job( - dl_path: Path, lock_path: str, body: Any, unlocked_ok: bool = True + dl_path: Path, + lock_path: str, + body: Any, + size: int, + stream_dest: Path | None = None, + unlocked_ok: bool = True, ) -> Any: - """Wrap ``body`` so the shared destination is single-writer. - - Interleaved writers truncate each other's ``.part`` bytes (see - registry.py). The bounded poll observes Ctrl-C via the tracker; a - blown deadline is a clean skip (the holder's copy is what the build - needs). On a lock-less filesystem a sha256-verified body runs - unlocked with one warning; a checksum-less one - (``unlocked_ok=False``) is a counted failure instead. + """Wrap ``body`` so the shared destination is single-writer (interleaved + writers truncate each other's ``.part``, see registry.py). A blown deadline + is a clean skip. On a lock-less filesystem a sha256-verified body runs + unlocked with one warning; a checksum-less one (``unlocked_ok=False``) fails. """ + def on_disk() -> int: + # A URL job's holder streams beside the staging path until it + # promotes; after that only dl_path is left + done = downloaded_bytes(dl_path, size) + if not done and stream_dest is not None: + done = downloaded_bytes(stream_dest, size) + return done + def run(tracker: Any) -> None: from filelock import FileLock, Timeout # fallback_to_soft would leave a stale marker on lock-less # filesystems that blocks every later build (see git.py) lock = FileLock(lock_path, fallback_to_soft=False) - deadline = time.monotonic() + _DOWNLOAD_LOCK_TIMEOUT - while True: - try: - lock.acquire(timeout=_URI_LOCK_POLL) - break - except Timeout: - tracker(0) # raises when the batch is cancelled - if time.monotonic() >= deadline: - # Another process is fetching this same file; its copy - # is what the build needs (a large framework archive - # can hold the lock far longer than this deadline) - _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) - return - except OSError as err: - if not unlocked_ok: - # A body with no checksum to catch interleaved corruption - raise - lock = None - _LOGGER.warning( - "Could not lock %s (%s); downloading unlocked", - dl_path.name, - err, - ) - break + try: + wait_for_download_lock(lock, tracker, on_disk, dl_path.name) + except Timeout: + # The holder's copy is what the build needs (a large + # framework archive can outlast this deadline) + _LOGGER.debug("Leaving %s to its current downloader", dl_path.name) + return + except DownloadLockUnavailable as err: + if not unlocked_ok: + # A body with no checksum to catch interleaved corruption + raise + lock = None + _LOGGER.warning( + "Could not lock %s (%s); downloading unlocked", + dl_path.name, + err, + ) try: if dl_path.is_file(): - return # another process finished it while we waited + tracker(size) # another process finished it while we waited + return body(tracker) finally: if lock is not None: @@ -540,6 +540,7 @@ def _registry_fetch_job( dl_path, f"{dl_path}.esphome.lock", resume_fetch_job(url, dl_path, sha256=checksum, size=size), + size, ) def run(tracker: Any) -> None: @@ -571,9 +572,9 @@ def _uri_fetch_job(manager: Any, url: str, dl_path: Path, size: int) -> Any: tmp.replace(dl_path) def run(tracker: Any) -> None: - _serialized_fetch_job(dl_path, f"{tmp}.lock", promote, unlocked_ok=False)( - tracker - ) + _serialized_fetch_job( + dl_path, f"{tmp}.lock", promote, size, tmp, unlocked_ok=False + )(tracker) if dl_path.is_file(): # Won or lost, the race is over; staging files left behind # are dead weight PlatformIO's cache never prunes diff --git a/esphome/platformio/registry.py b/esphome/platformio/registry.py index 9538a28ff4..75df82da0e 100644 --- a/esphome/platformio/registry.py +++ b/esphome/platformio/registry.py @@ -17,8 +17,10 @@ from esphome.framework_helpers import ( archive_extract_all, download_from_mirrors, download_with_resume, + downloaded_bytes, rmdir, run_batch_downloads, + wait_for_download_lock, ) from esphome.net_retry import fetch_with_retry, http_request @@ -164,11 +166,17 @@ class _PendingArchive(NamedTuple): name: str version: str dest: Path + archive: Path url: str sha256: str size: int +def _archive_path(downloads_dir: Path, name: str, version: str) -> Path: + """The one archive path the prefetch and the sequential install share.""" + return downloads_dir / f"{name}-{version}" + + def _already_installed(dest: Path) -> bool: """Whether ``dest`` holds a completed install (extraction marker).""" return (dest / ".esphome_extracted").is_file() @@ -187,18 +195,18 @@ def prefetch_packages( lock as ``install_package``: the archive's ``.part`` file is shared, and two concurrent writers would truncate each other's bytes. """ - from filelock import FileLock + from filelock import FileLock, Timeout pending: list[_PendingArchive] = [] - seen: set[str] = set() + seen: set[Path] = set() for name, version, dest, mirrors in packages: if mirrors or (dest / ".esphome_extracted").is_file(): continue - archive_name = f"{name}-{version}" - if archive_name in seen: + archive = _archive_path(downloads_dir, name, version) + if archive in seen: # A duplicate entry would race itself between two workers continue - seen.add(archive_name) + seen.add(archive) try: url, sha256, size = registry_download(name, version) except EsphomeError as err: @@ -207,10 +215,9 @@ def prefetch_packages( continue if not size: continue - archive = downloads_dir / archive_name if archive.is_file() and archive.stat().st_size == size: continue - pending.append(_PendingArchive(name, version, dest, url, sha256, size)) + pending.append(_PendingArchive(name, version, dest, archive, url, sha256, size)) if len(pending) < 2: return downloads_dir.mkdir(parents=True, exist_ok=True) @@ -222,20 +229,36 @@ def prefetch_packages( def _fetch(entry: _PendingArchive, tracker: Callable[[int], None]) -> None: entry.dest.parent.mkdir(parents=True, exist_ok=True) - with FileLock(f"{entry.dest}.lock", fallback_to_soft=False): - # Marker re-check: a concurrent build may have installed (and - # deleted the archive of) this package while we waited; - # re-downloading would orphan a fresh copy in downloads_dir - # no branch: the thread tracer misses the skip edge; both - # arms of _already_installed are pinned directly - if not _already_installed(entry.dest): # pragma: no branch - download_with_resume( - entry.url, - downloads_dir / f"{entry.name}-{entry.version}", - sha256=entry.sha256, - size=entry.size, - progress=tracker, - ) + + def on_disk() -> int: + if done := downloaded_bytes(entry.archive, entry.size): + return done + # The holder deletes the archive once it has installed it + return entry.size if _already_installed(entry.dest) else 0 + + lock = FileLock(f"{entry.dest}.lock", fallback_to_soft=False) + try: + wait_for_download_lock(lock, tracker, on_disk, entry.name) + except Timeout: + # install_package waits on this same lock and verifies the + # holder's copy + _LOGGER.debug("Leaving %s to its current downloader", entry.name) + return + try: + if _already_installed(entry.dest): + # A concurrent build installed it while we waited; a + # re-download would orphan a fresh copy in downloads_dir + tracker(entry.size) + return + download_with_resume( + entry.url, + entry.archive, + sha256=entry.sha256, + size=entry.size, + progress=tracker, + ) + finally: + lock.release() failures = run_batch_downloads( "Downloading packages", @@ -288,7 +311,7 @@ def install_package( rmdir(dest, msg=f"Clean up incomplete {name} install") # Persistent location so an interrupted download resumes across runs. downloads_dir.mkdir(parents=True, exist_ok=True) - archive = downloads_dir / f"{name}-{version}" + archive = _archive_path(downloads_dir, name, version) _LOGGER.info("Downloading %s %s ...", name, version) if mirrors: _LOGGER.warning( diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index 9de8f715ef..ad9c0bb11f 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -9,7 +9,7 @@ not be part of a unit test suite. """ -from collections.abc import Generator +from collections.abc import Callable, Generator import os from pathlib import Path import sys @@ -137,3 +137,40 @@ def mock_get_component() -> Generator[Mock, None, None]: """Mock get_component for config module.""" with patch("esphome.config.get_component") as mock: yield mock + + +@pytest.fixture +def held_lock() -> Callable[..., Callable[..., None]]: + """Factory for a ``FileLock.acquire`` fake held by another downloader. + + Each poll writes the next chunk to ``part`` (or runs it, for a callable) + and raises ``Timeout``; when the chunks run out the part is removed, + ``land()`` runs, and the acquire succeeds (also for any later job, so + ``land`` must be idempotent). + """ + from filelock import Timeout + + def make( + part: Path, + chunks: list[bytes | Callable[[], None]], + land: Callable[[], None], + ) -> Callable[..., None]: + polls = iter(chunks) + + def acquire(*args, **kwargs) -> None: + try: + chunk = next(polls) + except StopIteration: + part.unlink(missing_ok=True) + land() + return + if callable(chunk): + chunk() + else: + part.parent.mkdir(parents=True, exist_ok=True) + part.write_bytes(chunk) + raise Timeout("held") + + return acquire + + return make diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index fcc5572f51..22b34c9df5 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -2353,3 +2353,20 @@ def test_discard_partial_download_logs_undeletable( ): framework_helpers.discard_partial_download(dest) assert "Could not remove" in caplog.text + + +def test_downloaded_bytes_reports_what_is_on_disk(tmp_path: Path) -> None: + """Part file first, then the landed file, both capped at size; else 0.""" + dest = tmp_path / "archive" + assert framework_helpers.downloaded_bytes(dest, 4) == 0 + part = tmp_path / "archive.part" + part.write_bytes(b"ab") + assert framework_helpers.downloaded_bytes(dest, 4) == 2 + part.write_bytes(b"abcdef") + assert framework_helpers.downloaded_bytes(dest, 4) == 4 + part.unlink() + dest.write_bytes(b"abc") + assert framework_helpers.downloaded_bytes(dest, 4) == 3 + assert framework_helpers.downloaded_bytes(dest) == 3 + dest.write_bytes(b"abcdef") + assert framework_helpers.downloaded_bytes(dest, 4) == 4 diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index fb79885736..77490fd861 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -454,23 +454,96 @@ def test_uri_fetch_job_waits_out_a_briefly_held_lock(tmp_path: Path) -> None: assert dl_path.read_bytes() == b"data" -def test_lock_deadline_leaves_download_to_the_holder(tmp_path: Path) -> None: - """A lock held past the deadline means another process is fetching the - same file; skipping cleanly beats a misleading failure warning. The - tracker is still polled so a parked worker observes cancellation.""" +@pytest.mark.parametrize("staged", [b"", b"ab"]) +def test_lock_deadline_leaves_download_to_the_holder( + tmp_path: Path, staged: bytes +) -> None: + """A lock held past the deadline is another process's download; skip + cleanly, polling the tracker with what the holder has staged so far.""" dl_path = tmp_path / "archive" + (tmp_path / "archive.prefetch.part").write_bytes(staged) ticks: list[int] = [] with ( patch("esphome.framework_helpers.download_with_resume") as mock_download, patch("filelock.FileLock.acquire", side_effect=Timeout("held")), - patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), ): pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) mock_download.assert_not_called() - assert ticks == [0] + assert ticks == [len(staged)] assert not dl_path.exists() +@pytest.mark.parametrize( + ("job", "part_name", "chunks", "expected"), + [ + ( + lambda dl_path: pf._registry_fetch_job( + MagicMock(), "https://x/a.tar.gz", dl_path, "ab" * 32, 4 + ), + "archive.part", + [b"a", b"abc"], + [1, 3, 4], + ), + ( + lambda dl_path: pf._uri_fetch_job( + MagicMock(), "https://x/a.zip", dl_path, 4 + ), + "archive.prefetch.part", + [b"ab"], + [2, 4], + ), + ], + ids=["registry", "uri"], +) +def test_lock_wait_reports_the_holders_progress( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + held_lock, + job, + part_name: str, + chunks: list[bytes], + expected: list[int], +) -> None: + """A waiting job reports the holder's part file (the staging one for a + URL job), then the full size once the holder lands the archive.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + acquire = held_lock( + tmp_path / part_name, chunks, lambda: dl_path.write_bytes(b"abcd") + ) + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + caplog.at_level(logging.INFO), + ): + job(dl_path)(ticks.append) + mock_download.assert_not_called() + assert ticks == expected + assert caplog.text.count("Waiting for another process downloading archive") == 1 + + +def test_uri_lock_wait_prefers_the_landed_archive(tmp_path: Path, held_lock) -> None: + """Between the holder's promotion rename and its release the staging + part is gone; the landed cache file is credited instead of 0.""" + dl_path = tmp_path / "archive" + ticks: list[int] = [] + acquire = held_lock( + tmp_path / "archive.prefetch.part", + [b"ab", lambda: dl_path.write_bytes(b"abcd")], + lambda: None, + ) + with ( + patch("esphome.framework_helpers.download_with_resume") as mock_download, + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + ): + pf._uri_fetch_job(MagicMock(), "https://x/a.zip", dl_path, 4)(ticks.append) + mock_download.assert_not_called() + assert ticks == [2, 4, 4] + + def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: """A registry job that lost the download race to another process must not stamp a nonexistent archive into pio's usage.db.""" @@ -479,7 +552,7 @@ def test_registry_lock_deadline_skips_registration(tmp_path: Path) -> None: with ( patch("esphome.framework_helpers.download_with_resume") as mock_download, patch("filelock.FileLock.acquire", side_effect=Timeout("held")), - patch.object(pf, "_DOWNLOAD_LOCK_TIMEOUT", 0), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), ): pf._registry_fetch_job(manager, "https://x/a.tar.gz", dl_path, "ab" * 32, 4)( lambda done: None diff --git a/tests/unit_tests/test_platformio_registry.py b/tests/unit_tests/test_platformio_registry.py index 6ba8691c4e..9d5f6c4ce5 100644 --- a/tests/unit_tests/test_platformio_registry.py +++ b/tests/unit_tests/test_platformio_registry.py @@ -8,6 +8,7 @@ import os from pathlib import Path from unittest.mock import MagicMock, patch +from filelock import Timeout import pytest from esphome.core import EsphomeError @@ -540,16 +541,13 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: dest = tmp_path / "a" dest.mkdir() - from contextlib import contextmanager - - @contextmanager - def marker_appears_under_lock(path, **kwargs): + def marker_appears_under_lock(*args, **kwargs): # Simulates the concurrent build finishing while we waited (dest / ".esphome_extracted").touch() - yield with ( - patch("filelock.FileLock", side_effect=marker_appears_under_lock), + patch("filelock.FileLock.acquire", side_effect=marker_appears_under_lock), + patch("filelock.FileLock.release"), patch.object(registry, "download_with_resume") as mock_download, patch.object( registry, "registry_download", side_effect=_resolve_for({"a": 10}) @@ -559,6 +557,69 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: mock_download.assert_not_called() +def test_prefetch_packages_waits_with_the_holders_progress( + tmp_path: Path, held_lock +) -> None: + """A worker parked on another build's lock reports that build's part + file, then the full size once the marker appears.""" + dest = tmp_path / "a" + dest.mkdir() + ticks: list[int] = [] + part = tmp_path / "dl" / "a-1.0.part" + + def installed_and_pruned() -> None: + # install_package touches the marker, then unlinks the archive + (dest / ".esphome_extracted").touch() + part.unlink() + + acquire = held_lock( + part, + [lambda: None, b"abc", installed_and_pruned], + (dest / ".esphome_extracted").touch, + ) + + def fake_batch(header, jobs): + for _name, _size, fetch in jobs: + fetch(ticks.append) + return [] + + with ( + patch("filelock.FileLock.acquire", side_effect=acquire), + patch("filelock.FileLock.release"), + patch.object(registry, "run_batch_downloads", side_effect=fake_batch), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 5}) + ), + ): + registry.prefetch_packages( + [("a", "1.0", dest, []), ("b", "2.0", tmp_path / "b", [])], + tmp_path / "dl", + ) + assert ticks == [0, 3, 10, 10] + mock_download.assert_called_once() + + +def test_prefetch_packages_leaves_a_long_held_lock_to_its_holder( + tmp_path: Path, +) -> None: + """Past the deadline the worker skips; install_package waits on the same + lock later and verifies whatever the holder produced.""" + with ( + patch("filelock.FileLock.acquire", side_effect=Timeout("held")), + patch("esphome.framework_helpers.DOWNLOAD_LOCK_TIMEOUT", 0), + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10, "b": 5}) + ), + ): + registry.prefetch_packages( + [("a", "1.0", tmp_path / "a", []), ("b", "2.0", tmp_path / "b", [])], + tmp_path / "dl", + ) + mock_download.assert_not_called() + + def test_already_installed_probe(tmp_path: Path) -> None: """Both arms of the marker probe the prefetch worker keys on.""" dest = tmp_path / "pkg" From 8434dc5474e433a61a250800b489674ec5116d84 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:29:20 +1200 Subject: [PATCH 125/147] [esp32_hosted] Add ESP-NOW-over-hosted shim for the ESP32-P4 (#17712) --- esphome/components/esp32_hosted/__init__.py | 36 ++ .../esp32_hosted/esp_now_hosted.cpp | 467 ++++++++++++++++++ .../esp32_hosted/esp_now_hosted_rpc.h | 128 +++++ esphome/components/espnow/__init__.py | 20 + esphome/core/defines.h | 1 + script/ci-custom.py | 17 +- .../test-espnow.esp32-p4-idf.yaml | 5 + tests/unit_tests/components/test_espnow.py | 48 ++ 8 files changed, 721 insertions(+), 1 deletion(-) create mode 100644 esphome/components/esp32_hosted/esp_now_hosted.cpp create mode 100644 esphome/components/esp32_hosted/esp_now_hosted_rpc.h create mode 100644 tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml create mode 100644 tests/unit_tests/components/test_espnow.py diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index ab9455250c..21626e432b 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -37,6 +37,25 @@ CONF_HANDSHAKE_PIN = "handshake_pin" CONF_SDIO_FREQUENCY = "sdio_frequency" CONF_SPI_MODE = "spi_mode" +# ESP-NOW-over-hosted shim (esp_now_hosted.cpp). esp-hosted proxies esp_wifi.h +# but not esp_now.h (espressif/esp-hosted-mcu#19), and esp_wifi_remote injects +# the esp_now.h header on the ESP32-P4 host with no implementation, leaving the +# esp_now_* symbols undefined at link. On a P4 host, esp_now_hosted.cpp DEFINES +# those symbols and forwards each call to the co-processor over esp-hosted's +# CustomRpc "peer data transfer" channel, so ESPHome's `espnow` component links +# and runs unchanged (proven on a Tab5, 2026-07-20). The .cpp is guarded to +# CONFIG_IDF_TARGET_ESP32P4 so it compiles to nothing on hosts with a native +# ESP-NOW stack. CustomRpc needs these two host-side Kconfig options. Host +# registers 3 handlers (RESP, RECV, SEND); the coprocessor registers 1 (REQ); +# we ask for 8 to leave room for other CustomRpc extensions alongside. +# +# The coprocessor must run the matching custom firmware (a parallel effort in +# esphome/esp-hosted-firmware). esp_now_hosted_rpc.h here is the canonical copy +# of the wire contract and MUST stay byte-identical to the copy that coprocessor +# firmware uses — the packed structs are the on-wire layout, so any divergence +# silently corrupts every ESP-NOW frame. +_MAX_CUSTOM_MSG_HANDLERS = 8 + # Shared fields for both transport modes BASE_SCHEMA = cv.Schema( { @@ -262,6 +281,23 @@ async def to_code(config: ConfigType) -> None: else: _configure_spi(config) + # ESP-NOW-over-hosted shim: only the radio-less ESP32-P4 host needs it (see + # the note by _MAX_CUSTOM_MSG_HANDLERS). Enabled for every P4 host, not + # gated on the `espnow` component being present: the shim is tiny and the + # esp_now_* symbols/CustomRpc calls it defines require these Kconfig options + # to link whenever esp_now_hosted.cpp compiles (which is on any P4 host), so + # coupling the two keeps the build consistent. When `espnow` is absent the + # symbols are simply unused and never register a callback at runtime. + if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4: + add_define("USE_ESP_NOW_HOSTED") + # esp-hosted's CustomRpc ("peer data transfer") path — off by default. + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_ENABLE_PEER_DATA_TRANSFER", True + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_MAX_CUSTOM_MSG_HANDLERS", _MAX_CUSTOM_MSG_HANDLERS + ) + # Place the transport mempool in PSRAM. Required on memory-tight host # configurations (e.g. P4 with a large LVGL UI) where the internal-RAM # mempool allocation fails at boot with `sdio_mempool_create` assert. diff --git a/esphome/components/esp32_hosted/esp_now_hosted.cpp b/esphome/components/esp32_hosted/esp_now_hosted.cpp new file mode 100644 index 0000000000..ad29b208fe --- /dev/null +++ b/esphome/components/esp32_hosted/esp_now_hosted.cpp @@ -0,0 +1,467 @@ +/* + * esp_now_hosted — host-side shim implementing over esp-hosted + * CustomRpc, so ESPHome's `espnow` component can run on a radio-less host + * (e.g. the ESP32-P4) whose radio lives on an esp-hosted co-processor. + * + * A radio-less host has no native ESP-NOW. esp_wifi_remote INJECTS the full + * esp_now.h header (types + declarations) but ships NO implementation, so every + * esp_now_* symbol is an undefined reference at link time. This translation + * unit provides those definitions; each forwards to the co-processor over + * CustomRpc (see esphome/esp-hosted-firmware for the matching coprocessor + * handlers). No esp-hosted or esp_wifi_remote source is patched, and there is no + * duplicate-symbol clash because nothing else defines these symbols here. + * + * See esp_now_hosted_rpc.h for the wire protocol. + */ + +#include "sdkconfig.h" + +// Only build the shim on the radio-less host. On chips with a native ESP-NOW +// stack (S3, C6, …) the real symbols exist and this file must stay empty to +// avoid duplicate definitions. +#if defined(CONFIG_IDF_TARGET_ESP32P4) + +#include + +#include "freertos/FreeRTOS.h" +#include "freertos/semphr.h" + +#include "esp_idf_version.h" +#include "esp_log.h" +#include "esp_timer.h" + +#include // injected declarations we are now DEFINING +#include // wifi_pkt_rx_ctrl_t, wifi_tx_info_t + +// esp_hosted_misc.h (host) ships WITHOUT an extern "C" guard, so including it +// from C++ would give its declarations C++ linkage and the real C symbols in +// libesp_hosted would go unresolved at link. Wrap it. (Verified vs +// esp_hosted 2.12.9.) +extern "C" { +#include "esp_hosted_misc.h" // esp_hosted_{send_custom_data,register_custom_callback} +} + +#include "esp_now_hosted_rpc.h" + +namespace { + +const char *const TAG = "esp_now_hosted"; + +// One outstanding request at a time. ESPHome drives esp_now_* from the main +// loop; the matching response and the async RECV/SEND events all arrive on the +// single esp-hosted RPC RX thread. Serializing requests keeps the shared +// response slot race-free; a sequence number stops a late/stale response from +// being mistaken for ours. +SemaphoreHandle_t g_req_mutex = nullptr; +SemaphoreHandle_t g_resp_sem = nullptr; // given when the matching RESP lands +bool g_setup_done = false; // set only after setup fully succeeds +uint8_t g_seq = 0; +volatile uint8_t g_expect_seq = 0; +volatile int32_t g_resp_status = 0; +uint8_t g_resp_ret[16]; +volatile uint16_t g_resp_ret_len = 0; + +// Written from the main loop (register/unregister/deinit), read from the +// esp-hosted RX thread (on_recv/on_send). volatile for the same reason the +// g_resp_* globals are: force the RX thread to observe an updated pointer +// (e.g. a nulling by esp_now_deinit) rather than a cached one. +volatile esp_now_recv_cb_t g_recv_cb = nullptr; +volatile esp_now_send_cb_t g_send_cb = nullptr; + +// Local mirror of the co-processor's peer table. ESPHome's espnow component +// calls esp_now_is_peer_exist() on the main loop for every received frame +// (twice) and every send; forwarding each as a blocking RPC round-trip stalls +// the loop. The shim is the only path that mutates the co-processor peer table +// (add/del/deinit all go through here), so this mirror is authoritative and +// esp_now_is_peer_exist() can answer from it with no round-trip. +// +// esp_now_* are public C symbols: any component or user lambda may call them, +// and although ESPHome's espnow touches peers only from the main loop today +// (its RX/TX callbacks merely enqueue), the shim cannot rely on that. A short +// spinlock keeps the mirror consistent from any task/core, matching native +// esp_now_*'s own internal thread-safety. The critical sections are a bounded +// (<=20-entry) scan, so they stay tiny. ESP_NOW_MAX_TOTAL_PEER_NUM is 20. +constexpr size_t ESP_NOW_HOSTED_MAX_PEERS = 20; +uint8_t g_peer_cache[ESP_NOW_HOSTED_MAX_PEERS][6]; +size_t g_peer_count = 0; +portMUX_TYPE g_peer_lock = portMUX_INITIALIZER_UNLOCKED; + +// Caller must hold g_peer_lock. +int peer_cache_find_locked(const uint8_t *mac) { + for (size_t i = 0; i < g_peer_count; i++) { + if (memcmp(g_peer_cache[i], mac, 6) == 0) + return static_cast(i); + } + return -1; +} + +bool peer_cache_contains(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + const bool found = peer_cache_find_locked(mac) >= 0; + portEXIT_CRITICAL(&g_peer_lock); + return found; +} + +void peer_cache_add(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + if (peer_cache_find_locked(mac) < 0 && g_peer_count < ESP_NOW_HOSTED_MAX_PEERS) + memcpy(g_peer_cache[g_peer_count++], mac, 6); + portEXIT_CRITICAL(&g_peer_lock); +} + +void peer_cache_remove(const uint8_t *mac) { + portENTER_CRITICAL(&g_peer_lock); + const int idx = peer_cache_find_locked(mac); + if (idx >= 0) { + g_peer_count--; + if (static_cast(idx) != g_peer_count) // move the last entry into the gap + memcpy(g_peer_cache[idx], g_peer_cache[g_peer_count], 6); + } + portEXIT_CRITICAL(&g_peer_lock); +} + +void peer_cache_clear() { + portENTER_CRITICAL(&g_peer_lock); + g_peer_count = 0; + portEXIT_CRITICAL(&g_peer_lock); +} + +// ── CustomRpc event handlers (run on the esp-hosted RPC RX thread) ────────── +// Keep them short and non-blocking. In particular they MUST NOT call back into +// any esp_now_* shim function: that would try to take g_req_mutex / wait on the +// RX thread that delivers the response, and deadlock. + +void on_resp(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + if (len < sizeof(esp_now_hosted_resp_t)) { + ESP_LOGW(TAG, "RESP too short: %u bytes", static_cast(len)); + return; + } + const auto *r = reinterpret_cast(data); + if (r->seq != g_expect_seq) { // late response from a timed-out request (expected) + ESP_LOGV(TAG, "dropping stale RESP seq %u (want %u)", r->seq, g_expect_seq); + return; + } + g_resp_status = r->status; + uint16_t rl = r->ret_len; + if (rl > sizeof(g_resp_ret)) { + // Larger than any real opcode return — a likely wire-format drift signal. + ESP_LOGW(TAG, "RESP ret_len %u exceeds buffer, clamping (wire drift?)", rl); + rl = sizeof(g_resp_ret); + } + if (len >= sizeof(esp_now_hosted_resp_t) + rl) { + memcpy(g_resp_ret, r->ret, rl); + } else { + // Truncated frame: fail closed. Never hand the caller stale bytes left in + // g_resp_ret by a previous response, and don't let request() report a + // zeroed payload as success — override the status to an error. + ESP_LOGW(TAG, "RESP truncated: claims %u ret bytes, frame too short", rl); + rl = 0; + g_resp_status = ESP_ERR_INVALID_RESPONSE; + } + g_resp_ret_len = rl; + xSemaphoreGive(g_resp_sem); +} + +void on_recv(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + // Read the volatile pointer once: esp_now_unregister_recv_cb()/deinit() (via + // the espnow component's disable()) can null it on the main loop between the + // guard and the call, which would otherwise turn the call into a null-deref. + const esp_now_recv_cb_t cb = g_recv_cb; + if (cb == nullptr) + return; + if (len < sizeof(esp_now_hosted_recv_evt_t)) { + ESP_LOGW(TAG, "RECV too short: %u bytes", static_cast(len)); + return; + } + const auto *e = reinterpret_cast(data); + if (len < sizeof(esp_now_hosted_recv_evt_t) + e->data_len) { + ESP_LOGW(TAG, "RECV data_len %u exceeds frame", e->data_len); + return; + } + + // ESPHome dereferences info->rx_ctrl->{rssi,timestamp}; give it a real one. + wifi_pkt_rx_ctrl_t rx_ctrl; + memset(&rx_ctrl, 0, sizeof(rx_ctrl)); + rx_ctrl.rssi = e->rssi; + rx_ctrl.channel = e->channel; + rx_ctrl.timestamp = static_cast(esp_timer_get_time()); + + esp_now_recv_info_t info; + info.src_addr = const_cast(e->src_addr); + info.des_addr = const_cast(e->des_addr); + info.rx_ctrl = &rx_ctrl; + cb(&info, e->data, static_cast(e->data_len)); +} + +void on_send(uint32_t /*msg_id*/, const uint8_t *data, size_t len, void * /*ctx*/) { + // Read the volatile pointer once (see on_recv): disable()/deinit() can null it + // on the main loop concurrently with this RX-thread callback. + const esp_now_send_cb_t cb = g_send_cb; + if (cb == nullptr) + return; + if (len < sizeof(esp_now_hosted_send_evt_t)) { + ESP_LOGW(TAG, "SEND evt too short: %u bytes", static_cast(len)); + return; + } + const auto *e = reinterpret_cast(data); +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 5, 0) + // IDF >= 5.5: esp_now_send_cb_t takes esp_now_send_info_t (== wifi_tx_info_t), + // whose des_addr is a POINTER (not an inline array). Point it at the event's + // MAC (valid for this callback) — do NOT memcpy into it (that writes NULL and + // faults). ESPHome reads only info->des_addr. + esp_now_send_info_t si; + memset(&si, 0, sizeof(si)); + si.des_addr = const_cast(e->des_addr); + cb(&si, static_cast(e->status)); +#else + cb(e->des_addr, static_cast(e->status)); +#endif +} + +esp_err_t ensure_setup() { + // Gate on g_setup_done, not on g_req_mutex: a failure part-way through (a + // semaphore that did not allocate, a callback that did not register) must not + // leave a later call thinking setup completed. Semaphore creation is guarded + // so a retry after a partial failure does not leak the earlier handles. + if (g_setup_done) + return ESP_OK; + if (g_req_mutex == nullptr) + g_req_mutex = xSemaphoreCreateMutex(); + if (g_resp_sem == nullptr) + g_resp_sem = xSemaphoreCreateBinary(); + if (g_req_mutex == nullptr || g_resp_sem == nullptr) + return ESP_ERR_NO_MEM; + esp_err_t err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_RESP, on_resp, nullptr)) != ESP_OK) + return err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_RECV, on_recv, nullptr)) != ESP_OK) + return err; + if ((err = esp_hosted_register_custom_callback(ESP_NOW_HOSTED_MSG_SEND, on_send, nullptr)) != ESP_OK) + return err; + g_setup_done = true; + return ESP_OK; +} + +// Send one request envelope. With wait=true (default) block until the matching +// response (or timeout); with wait=false return as soon as the frame is handed +// to the transport (fire-and-forget, used by esp_now_send). +// +// `tail` is an optional second chunk written straight after `payload`. Callers +// with a fixed header plus a bulk body (esp_now_send) pass the two separately +// so they never need a build buffer of their own: both chunks are laid into the +// request buffer here, under g_req_mutex, which keeps concurrent callers from +// racing and saves a full copy of the body on every transmit. +esp_err_t request(uint8_t opcode, const void *payload, uint16_t plen, void *ret, uint16_t ret_cap, uint16_t *ret_len, + bool wait = true, const void *tail = nullptr, uint16_t tail_len = 0) { + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + if (plen > ESP_NOW_HOSTED_MAX_PAYLOAD || tail_len > ESP_NOW_HOSTED_MAX_PAYLOAD - plen) + return ESP_ERR_INVALID_SIZE; + const uint16_t total_len = static_cast(plen + tail_len); + + if (xSemaphoreTake(g_req_mutex, portMAX_DELAY) != pdTRUE) + return ESP_FAIL; + + static uint8_t buf[sizeof(esp_now_hosted_req_t) + ESP_NOW_HOSTED_MAX_PAYLOAD]; // guarded by g_req_mutex + auto *req = reinterpret_cast(buf); + req->opcode = opcode; + req->seq = ++g_seq; + req->payload_len = total_len; + if (plen != 0) + memcpy(req->payload, payload, plen); + if (tail_len != 0) + memcpy(req->payload + plen, tail, tail_len); + g_expect_seq = req->seq; + + xSemaphoreTake(g_resp_sem, 0); // drain any stale signal before sending + err = esp_hosted_send_custom_data(ESP_NOW_HOSTED_MSG_REQ, buf, sizeof(esp_now_hosted_req_t) + total_len); + if (err != ESP_OK) { + xSemaphoreGive(g_req_mutex); + return err; + } + if (!wait) { + // Fire-and-forget (esp_now_send): the co-processor enqueues the frame and + // reports the real TX result later via the async SEND event, exactly like + // native esp_now_send. Returning here keeps the main loop off the ~100 ms+ + // RPC round-trip. The matching RESP is ignored (seq won't match the next + // waited request, so on_resp drops it). + xSemaphoreGive(g_req_mutex); + return ESP_OK; + } + if (xSemaphoreTake(g_resp_sem, pdMS_TO_TICKS(ESP_NOW_HOSTED_TIMEOUT_MS)) != pdTRUE) { + ESP_LOGW(TAG, "opcode %u timed out", opcode); + xSemaphoreGive(g_req_mutex); + return ESP_ERR_TIMEOUT; + } + + const int32_t status = g_resp_status; + if (ret != nullptr && ret_cap != 0) { + uint16_t n = g_resp_ret_len < ret_cap ? g_resp_ret_len : ret_cap; + memcpy(ret, const_cast(g_resp_ret), n); + if (ret_len != nullptr) + *ret_len = n; + } + xSemaphoreGive(g_req_mutex); + return static_cast(status); +} + +} // namespace + +// ── The surface, defined for the radio-less host ──────────────── +extern "C" { + +esp_err_t esp_now_init(void) { return request(ESP_NOW_HOSTED_OP_INIT, nullptr, 0, nullptr, 0, nullptr); } + +esp_err_t esp_now_deinit(void) { + g_recv_cb = nullptr; + g_send_cb = nullptr; + peer_cache_clear(); // the co-processor drops all peers on deinit + return request(ESP_NOW_HOSTED_OP_DEINIT, nullptr, 0, nullptr, 0, nullptr); +} + +esp_err_t esp_now_get_version(uint32_t *version) { + uint32_t v = 0; + uint16_t rl = 0; + esp_err_t err = request(ESP_NOW_HOSTED_OP_GET_VERSION, nullptr, 0, &v, sizeof(v), &rl); + if (version != nullptr) + *version = v; + return err; +} + +esp_err_t esp_now_register_recv_cb(esp_now_recv_cb_t cb) { + // Only arm the callback once the CustomRpc handlers are actually registered, + // so a failed setup leaves g_recv_cb null rather than falsely "registered". + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + g_recv_cb = cb; + return ESP_OK; +} +esp_err_t esp_now_unregister_recv_cb(void) { + g_recv_cb = nullptr; + return ESP_OK; +} +esp_err_t esp_now_register_send_cb(esp_now_send_cb_t cb) { + esp_err_t err = ensure_setup(); + if (err != ESP_OK) + return err; + g_send_cb = cb; + return ESP_OK; +} +esp_err_t esp_now_unregister_send_cb(void) { + g_send_cb = nullptr; + return ESP_OK; +} + +static esp_err_t add_or_mod_peer(uint8_t opcode, const esp_now_peer_info_t *peer, bool wait) { + if (peer == nullptr) + return ESP_ERR_ESPNOW_ARG; + esp_now_hosted_peer_t p; + memset(&p, 0, sizeof(p)); + memcpy(p.peer_addr, peer->peer_addr, 6); + memcpy(p.lmk, peer->lmk, 16); + p.channel = peer->channel; + p.ifidx = static_cast(peer->ifidx); + p.encrypt = peer->encrypt ? 1 : 0; + return request(opcode, &p, sizeof(p), nullptr, 0, nullptr, wait); +} +esp_err_t esp_now_add_peer(const esp_now_peer_info_t *peer) { + // Fire-and-forget (wait=false): adding a peer is a blocking RPC round-trip, + // and ESPHome's espnow calls it on the main loop when a device joins the mesh + // — under co-processor load that stalls the UI (peer-churn stutter). Issue it + // without waiting and mirror it locally. Safe against a following + // esp_now_send to the same peer: both ride the same in-order CustomRpc + // channel (mutex-serialized on the host) and the co-processor processes REQs + // FIFO, so ADD_PEER is applied before the SEND. Trade-off: a co-processor-side + // failure (e.g. peer table full) is no longer reported synchronously — the + // same limitation as esp_now_send — but ESPHome only adds peers it validated. + esp_err_t err = add_or_mod_peer(ESP_NOW_HOSTED_OP_ADD_PEER, peer, /*wait=*/false); + if (err == ESP_OK) + peer_cache_add(peer->peer_addr); // keep the local mirror in sync + return err; +} +esp_err_t esp_now_mod_peer(const esp_now_peer_info_t *peer) { + // mod_peer changes a peer's parameters, not its existence, so the cache is + // unaffected. Kept synchronous — it is not on any hot path (espnow never + // calls it), so the extra round-trip does not matter and the status is useful. + return add_or_mod_peer(ESP_NOW_HOSTED_OP_MOD_PEER, peer, /*wait=*/true); +} + +esp_err_t esp_now_del_peer(const uint8_t *peer_addr) { + if (peer_addr == nullptr) + return ESP_ERR_ESPNOW_ARG; + // Fire-and-forget for the same reason as add_peer (peer churn on the main + // loop). Removal is order-independent, so this is strictly safe. + esp_err_t err = request(ESP_NOW_HOSTED_OP_DEL_PEER, peer_addr, 6, nullptr, 0, nullptr, /*wait=*/false); + if (err == ESP_OK) + peer_cache_remove(peer_addr); // keep the local mirror in sync + return err; +} + +bool esp_now_is_peer_exist(const uint8_t *peer_addr) { + if (peer_addr == nullptr) + return false; + // Answered from the local mirror — no RPC round-trip. ESPHome's espnow calls + // this on the main loop for every received frame and every send, so a + // blocking round-trip here would stall rendering under mesh traffic. + return peer_cache_contains(peer_addr); +} + +esp_err_t esp_now_send(const uint8_t *peer_addr, const uint8_t *data, size_t len) { + if (len > ESP_NOW_HOSTED_MAX_FRAME) + return ESP_ERR_ESPNOW_ARG; + if (data == nullptr && len != 0) // native esp_now_send treats this as an arg error + return ESP_ERR_ESPNOW_ARG; + // Only the small fixed header is built here; the caller's frame goes over as + // the request tail, so request() lays both into its own buffer under + // g_req_mutex. esp_now_send is a public C symbol and may be called from any + // task, and a shared build buffer here would let two callers corrupt each + // other's frame. Passing the body through also drops a full-frame copy per + // transmit, on the path this shim exists to keep quick. + uint8_t hdr[sizeof(esp_now_hosted_send_req_t)]; + auto *s = reinterpret_cast(hdr); + s->has_addr = peer_addr != nullptr ? 1 : 0; + if (peer_addr != nullptr) + memcpy(s->peer_addr, peer_addr, 6); + else + memset(s->peer_addr, 0, 6); + s->data_len = static_cast(len); + // Fire-and-forget (wait=false): native esp_now_send returns once the frame is + // queued, with the real TX result delivered later through the send callback. + // The co-processor mirrors that — it acks enqueue immediately and reports the + // outcome via the async SEND event (on_send -> on_send_report). Waiting for + // the RPC RESP here would block the main loop for the full round-trip on + // every transmit. + return request(ESP_NOW_HOSTED_OP_SEND, hdr, sizeof(hdr), nullptr, 0, nullptr, /*wait=*/false, data, + static_cast(len)); +} + +esp_err_t esp_now_set_pmk(const uint8_t *pmk) { + if (pmk == nullptr) + return ESP_ERR_ESPNOW_ARG; + return request(ESP_NOW_HOSTED_OP_SET_PMK, pmk, 16, nullptr, 0, nullptr); +} + +// Remainder of the surface. Not used by ESPHome's espnow component +// today; provided so the whole header links and future callers get a defined +// (if unimplemented) symbol rather than a link error. Wire them through +// CustomRpc if a use case appears. +esp_err_t esp_now_get_peer(const uint8_t * /*peer_addr*/, esp_now_peer_info_t * /*peer*/) { + return ESP_ERR_NOT_SUPPORTED; +} +esp_err_t esp_now_fetch_peer(bool /*from_head*/, esp_now_peer_info_t * /*peer*/) { return ESP_ERR_NOT_SUPPORTED; } +esp_err_t esp_now_get_peer_num(esp_now_peer_num_t * /*num*/) { return ESP_ERR_NOT_SUPPORTED; } +esp_err_t esp_now_set_wake_window(uint16_t /*window*/) { + return ESP_ERR_NOT_SUPPORTED; // power-save wake window is not forwarded; don't claim success +} +esp_err_t esp_now_set_peer_rate_config(const uint8_t * /*peer_addr*/, esp_now_rate_config_t * /*cfg*/) { + return ESP_ERR_NOT_SUPPORTED; +} +esp_err_t esp_wifi_config_espnow_rate(wifi_interface_t /*ifx*/, wifi_phy_rate_t /*rate*/) { + return ESP_ERR_NOT_SUPPORTED; +} + +} // extern "C" + +#endif // CONFIG_IDF_TARGET_ESP32P4 diff --git a/esphome/components/esp32_hosted/esp_now_hosted_rpc.h b/esphome/components/esp32_hosted/esp_now_hosted_rpc.h new file mode 100644 index 0000000000..bf68c759ee --- /dev/null +++ b/esphome/components/esp32_hosted/esp_now_hosted_rpc.h @@ -0,0 +1,128 @@ +/* + * esp_now_hosted — ESP-NOW-over-CustomRpc wire protocol. + * + * Shared, byte-for-byte-identical contract between: + * - the host shim (esphome/components/esp32_hosted/esp_now_hosted.cpp) + * - the coprocessor firmware (esphome/esp-hosted-firmware) + * + * It rides esp-hosted's CustomRpc channel (RPC ID 388, "peer data transfer", + * available since esp-hosted v2.8.1), teaching the radio-less host <-> radio + * co-processor link to carry esp_now.h, which esp-hosted itself does not proxy + * (Espressif issue espressif/esp-hosted-mcu#19). + * + * KEEP THE TWO COPIES IN SYNC. The canonical copy lives here; the coprocessor + * firmware uses a verbatim copy. Both sides are little-endian, so these packed + * structs are wire-compatible with no byte-swapping. + */ + +#ifndef ESP_NOW_HOSTED_RPC_H +#define ESP_NOW_HOSTED_RPC_H + +#ifdef __cplusplus +#include +#else +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* ── CustomRpc message IDs (any uint32_t except 0xFFFFFFFF) ────────────────── + * One REQ handler slot on the device; three event handler slots on the host. + * The bytes spell "now" + index, a private range unlikely to clash with other + * CustomRpc users (e.g. the stock peer_data_transfer example's 1..6). */ +#define ESP_NOW_HOSTED_MSG_REQ 0x6E6F7701u /* host -> device : request envelope */ +#define ESP_NOW_HOSTED_MSG_RESP 0x6E6F7702u /* device -> host : reply to a REQ */ +#define ESP_NOW_HOSTED_MSG_RECV 0x6E6F7703u /* device -> host : async RX frame */ +#define ESP_NOW_HOSTED_MSG_SEND 0x6E6F7704u /* device -> host : async TX status */ + +/* ── Request opcodes ────────────────────────────────────────────────────── */ +enum { + ESP_NOW_HOSTED_OP_INIT = 1, /* esp_now_init + register device recv/send cbs */ + ESP_NOW_HOSTED_OP_DEINIT = 2, /* unregister cbs + esp_now_deinit */ + ESP_NOW_HOSTED_OP_ADD_PEER = 3, /* payload: esp_now_hosted_peer_t */ + ESP_NOW_HOSTED_OP_DEL_PEER = 4, /* payload: 6-byte peer MAC */ + ESP_NOW_HOSTED_OP_IS_PEER_EXIST = 5, /* payload: 6-byte MAC; ret: 1 byte bool */ + ESP_NOW_HOSTED_OP_SEND = 6, /* payload: esp_now_hosted_send_req_t */ + ESP_NOW_HOSTED_OP_GET_VERSION = 7, /* ret: uint32 version */ + ESP_NOW_HOSTED_OP_SET_PMK = 8, /* payload: 16-byte PMK */ + ESP_NOW_HOSTED_OP_MOD_PEER = 9, /* payload: esp_now_hosted_peer_t */ +}; + +/* Largest ESP-NOW payload we forward. ESP-NOW v2 (IDF >= 5.4) is 1470 B; well + * under esp-hosted's 8166 B CustomRpc cap, so the shim never truncates. */ +#define ESP_NOW_HOSTED_MAX_FRAME 1470u +/* Envelope slack for the largest opcode payload (a SEND req wrapping a frame). */ +#define ESP_NOW_HOSTED_MAX_PAYLOAD (ESP_NOW_HOSTED_MAX_FRAME + 16u) +/* Host request/response round-trip timeout over the transport. Generous: + * normal RTT is sub-millisecond, but Wi-Fi/BLE contention on the co-processor + * can stall the RX thread. */ +#define ESP_NOW_HOSTED_TIMEOUT_MS 2000 + +/* ── Envelopes ──────────────────────────────────────────────────────────── */ + +/* These payloads are shared verbatim with the C co-processor firmware, so they + * use C's `typedef struct {...} name;` idiom rather than C++ `using` aliases, + * which would not compile there. Silence clang-tidy's modernize-use-using for + * the shared struct block. */ +// NOLINTBEGIN(modernize-use-using) +typedef struct { + uint8_t opcode; /* one of ESP_NOW_HOSTED_OP_* */ + uint8_t seq; /* wraps 0..255; echoed in the response for matching */ + uint16_t payload_len; /* bytes of opcode-specific payload that follow */ + uint8_t payload[]; /* flexible */ +} __attribute__((packed)) esp_now_hosted_req_t; + +typedef struct { + uint8_t opcode; /* echoes the request opcode */ + uint8_t seq; /* echoes the request seq */ + int32_t status; /* esp_err_t from the native call on the co-processor */ + uint16_t ret_len; /* bytes of return payload that follow */ + uint8_t ret[]; /* flexible (e.g. version u32, is_peer_exist bool) */ +} __attribute__((packed)) esp_now_hosted_resp_t; + +/* ── Opcode payloads ────────────────────────────────────────────────────── */ + +/* esp_now_peer_info_t minus the host-only `priv` pointer, which is meaningless + * across the transport and never set by ESPHome's espnow component. */ +typedef struct { + uint8_t peer_addr[6]; + uint8_t lmk[16]; + uint8_t channel; /* 0 = current channel */ + uint8_t ifidx; /* wifi_interface_t (0=STA, 1=AP) */ + uint8_t encrypt; /* bool */ +} __attribute__((packed)) esp_now_hosted_peer_t; + +typedef struct { + uint8_t has_addr; /* 0 => peer_addr is NULL (broadcast to all peers) */ + uint8_t peer_addr[6]; + uint16_t data_len; + uint8_t data[]; /* flexible, up to ESP_NOW_HOSTED_MAX_FRAME */ +} __attribute__((packed)) esp_now_hosted_send_req_t; + +/* ── Async events (device -> host) ──────────────────────────────────────── */ + +/* Reconstructed on the host into an esp_now_recv_info_t + a minimal + * wifi_pkt_rx_ctrl_t. ESPHome's espnow reads info->src_addr, info->des_addr, + * info->rx_ctrl->rssi and info->rx_ctrl->timestamp. */ +typedef struct { + uint8_t src_addr[6]; + uint8_t des_addr[6]; + int8_t rssi; + uint8_t channel; + uint16_t data_len; + uint8_t data[]; /* flexible */ +} __attribute__((packed)) esp_now_hosted_recv_evt_t; + +typedef struct { + uint8_t des_addr[6]; + uint8_t status; /* esp_now_send_status_t (0 = success) */ +} __attribute__((packed)) esp_now_hosted_send_evt_t; +// NOLINTEND(modernize-use-using) + +#ifdef __cplusplus +} +#endif + +#endif /* ESP_NOW_HOSTED_RPC_H */ diff --git a/esphome/components/espnow/__init__.py b/esphome/components/espnow/__init__.py index 5541a6ee97..14d099ec06 100644 --- a/esphome/components/espnow/__init__.py +++ b/esphome/components/espnow/__init__.py @@ -3,6 +3,7 @@ from typing import Any from esphome import automation, core import esphome.codegen as cg from esphome.components import wifi +from esphome.components.esp32 import VARIANT_ESP32P4, get_esp32_variant from esphome.components.udp import CONF_ON_RECEIVE import esphome.config_validation as cv from esphome.const import ( @@ -17,6 +18,7 @@ from esphome.const import ( ) from esphome.core import CORE, HexInt from esphome.cpp_generator import MockObj, TemplateArgsType +import esphome.final_validate as fv from esphome.types import ConfigType CODEOWNERS = ["@jesserockz"] @@ -132,6 +134,24 @@ CONFIG_SCHEMA = cv.All( ) +def _validate_variant(config: ConfigType) -> ConfigType: + # ESP-NOW rides the Wi-Fi PHY. Radio-less esp32 variants have no native + # ESP-NOW; only the ESP32-P4 has a path, via the esp32_hosted shim that + # supplies the esp_now_* symbols. Fail here with a clear message instead of + # letting the build reach an "undefined reference to esp_now_*" link error. + variant = get_esp32_variant() + if wifi.variant_has_wifi(variant): + return config + if variant != VARIANT_ESP32P4: + raise cv.Invalid(f"ESP-NOW is not supported on {variant} (no Wi-Fi radio)") + if "esp32_hosted" not in fv.full_config.get(): + raise cv.Invalid(f"ESP-NOW on {variant} requires the esp32_hosted component") + return config + + +FINAL_VALIDATE_SCHEMA = _validate_variant + + async def _trigger_to_code(config: ConfigType) -> MockObj: if address := config.get(CONF_ADDRESS): address = address.parts diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 9dd1e0ced6..eaece6d5ff 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -71,6 +71,7 @@ #define USE_ESP32_HOSTED #define USE_ESP32_HOSTED_HTTP_UPDATE #define USE_ESP32_IMPROV_STATE_CALLBACK +#define USE_ESP_NOW_HOSTED #define USE_EVENT #define USE_FAN #define USE_GPIO_BINARY_SENSOR_INTERRUPT diff --git a/script/ci-custom.py b/script/ci-custom.py index f481fda860..e2b7cd8d37 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -294,6 +294,9 @@ def highlight(s): "esphome/components/socket/headers.h", "esphome/core/defines.h", "esphome/components/http_request/httplib.h", + # Shared C wire header (byte-identical with the co-processor firmware); + # these are protocol constants and constexpr is C++-only. + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", ], ) def lint_no_defines(fname, match): @@ -816,6 +819,10 @@ def lint_relative_py_import(fname: Path, line, col, content): "esphome/components/host/helpers.cpp", "esphome/components/zephyr/helpers.cpp", "esphome/components/http_request/httplib.h", + # Global extern "C" esp_now_* linker symbols + shared C wire header; + # neither can live in a C++ namespace. + "esphome/components/esp32_hosted/esp_now_hosted.cpp", + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", ], ) def lint_namespace(fname: Path, content: str) -> str | None: @@ -841,7 +848,15 @@ def lint_esphome_h(fname, line, col, content): ) -@lint_content_check(include=["*.h"], exclude=["esphome/core/entity_types.h"]) +@lint_content_check( + include=["*.h"], + exclude=[ + "esphome/core/entity_types.h", + # Shared C wire header; uses a classic #ifndef guard for portability + # across the co-processor firmware repo it stays byte-identical with. + "esphome/components/esp32_hosted/esp_now_hosted_rpc.h", + ], +) def lint_pragma_once(fname, content): if "#pragma once" not in content: return ( diff --git a/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml b/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml new file mode 100644 index 0000000000..fab0a64ab8 --- /dev/null +++ b/tests/components/esp32_hosted/test-espnow.esp32-p4-idf.yaml @@ -0,0 +1,5 @@ +# Exercises the ESP-NOW-over-hosted shim: on the ESP32-P4 host, esp32_hosted +# supplies the esp_now_* symbols that the espnow component links against. +packages: + esp32_hosted: !include common.yaml + espnow: !include ../espnow/common.yaml diff --git a/tests/unit_tests/components/test_espnow.py b/tests/unit_tests/components/test_espnow.py new file mode 100644 index 0000000000..21305c2b33 --- /dev/null +++ b/tests/unit_tests/components/test_espnow.py @@ -0,0 +1,48 @@ +"""Tests for the espnow component's final validation.""" + +import pytest + +from esphome.components.esp32.const import ( + VARIANT_ESP32C3, + VARIANT_ESP32H2, + VARIANT_ESP32P4, +) +from esphome.components.espnow import _validate_variant +import esphome.config_validation as cv +import esphome.final_validate as fv +from esphome.types import ConfigType + + +def _run( + monkeypatch, variant: str, full_config: dict, config: ConfigType +) -> ConfigType: + monkeypatch.setattr("esphome.components.espnow.get_esp32_variant", lambda: variant) + token = fv.full_config.set(full_config) + try: + return _validate_variant(config) + finally: + fv.full_config.reset(token) + + +def test_variant_with_native_wifi_passes(monkeypatch) -> None: + """A variant with a native Wi-Fi PHY needs no shim; config passes through.""" + config = {"id": "espnow"} + assert _run(monkeypatch, VARIANT_ESP32C3, {}, config) is config + + +def test_radioless_non_p4_variant_rejected(monkeypatch) -> None: + """Radio-less variants without any ESP-NOW path are rejected outright.""" + with pytest.raises(cv.Invalid, match="not supported"): + _run(monkeypatch, VARIANT_ESP32H2, {}, {}) + + +def test_p4_without_esp32_hosted_rejected(monkeypatch) -> None: + """The P4 needs the esp32_hosted shim to supply the esp_now_* symbols.""" + with pytest.raises(cv.Invalid, match="esp32_hosted"): + _run(monkeypatch, VARIANT_ESP32P4, {}, {}) + + +def test_p4_with_esp32_hosted_passes(monkeypatch) -> None: + """The P4 with esp32_hosted present validates; config passes through.""" + config = {"id": "espnow"} + assert _run(monkeypatch, VARIANT_ESP32P4, {"esp32_hosted": {}}, config) is config From 9ba4477ada0f207b7426fe83b37fde69c9ed9947 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:46:22 +1200 Subject: [PATCH 126/147] Bump version to 2026.9.0b2 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 7b2d21027a..060de51d3a 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.9.0b1 +PROJECT_NUMBER = 2026.9.0b2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 378da14197..287804ace3 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.9.0b1" +__version__ = "2026.9.0b2" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From 5e37872da24f4626dc7ea8bdd61f4c5534f6c0ee Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:32:18 +1200 Subject: [PATCH 127/147] [ci] Sync pre-commit revs and prek version from requirements files (#19026) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 13 +- .../workflows/sync-dependency-versions.yml | 94 ++++++++ .pre-commit-config.yaml | 5 +- AGENTS.md | 2 +- requirements_dev.txt | 4 +- requirements_test.txt | 9 +- script/sync_dependency_versions.py | 164 +++++++++++++ tests/script/test_sync_dependency_versions.py | 219 ++++++++++++++++++ 8 files changed, 498 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/sync-dependency-versions.yml create mode 100755 script/sync_dependency_versions.py create mode 100644 tests/script/test_sync_dependency_versions.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7c93b3b86..173d2c227a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,11 +244,20 @@ jobs: steps: - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Read prek version from requirements_test.txt + id: prek + # requirements_test.txt is the only place the version is pinned, so a + # Dependabot bump there is picked up here without a second edit. + run: | + if ! version=$(sed -nE 's/^prek==([^[:space:]#]+).*/\1/p' requirements_test.txt) || [ -z "$version" ]; then + echo "::error::No prek== pin found in requirements_test.txt." + exit 1 + fi + echo "version=$version" >> "$GITHUB_OUTPUT" - name: Run prek uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0 with: - # Keep in sync with requirements_test.txt. - prek-version: "0.4.11" + prek-version: ${{ steps.prek.outputs.version }} # This job only runs on pull requests, so nothing ever populates # the cache on dev. Every run would miss and then write a per-pull # request copy, which is what the old seed-cache job existed to diff --git a/.github/workflows/sync-dependency-versions.yml b/.github/workflows/sync-dependency-versions.yml new file mode 100644 index 0000000000..5599691ed1 --- /dev/null +++ b/.github/workflows/sync-dependency-versions.yml @@ -0,0 +1,94 @@ +# Keeps pre-commit hook revs in sync with the requirements files. +# +# Dependabot only bumps the pins in requirements*.txt. Some of those tools +# are pinned again as hook revs in .pre-commit-config.yaml. This workflow +# runs script/sync_dependency_versions.py against the pull request branch +# and pushes a commit with the revs updated. + +name: Sync dependency versions + +on: + # pull_request_target rather than pull_request so the App secret is + # available on Dependabot pull requests (pull_request runs opened by + # Dependabot only see Dependabot secrets). The job below only touches + # branches in this repository and only ever executes the script from the + # base branch checkout, so fork code never runs with the token. + pull_request_target: + types: [opened, synchronize, reopened] + paths: + - requirements_dev.txt + - requirements_test.txt + - .pre-commit-config.yaml + - script/sync_dependency_versions.py + +# The push to the pull request branch uses the App token minted below, so +# the workflow's GITHUB_TOKEN does not need any scopes. +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + sync: + name: Sync pinned versions + runs-on: ubuntu-latest + # Same-repository branches only: a push to a fork is not possible with + # this token, and it keeps untrusted heads out of a privileged job. + if: >- + github.repository == 'esphome/esphome' + && github.event.pull_request.head.repo.full_name == github.repository + steps: + - name: Generate a token + id: generate-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.ESPHOME_GITHUB_APP_CLIENT_ID }} + private-key: ${{ secrets.ESPHOME_GITHUB_APP_PRIVATE_KEY }} + # A push made with the workflow's own GITHUB_TOKEN would not start + # CI on the new commit; a push with the App token does. + permission-contents: write # git push of the sync commit to the pull request branch + + - name: Check out base branch + # Provides the script that runs below. Deliberately the base branch + # so the pull request cannot change what executes here. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.base.sha }} + persist-credentials: false + + - name: Check out pull request branch + # No allow-unsafe-pr-checkout here on purpose: checkout v7 only + # refuses heads that live in a different repository, and the job + # condition above already limits runs to same-repository branches. + # Leaving it off keeps that refusal as a backstop for fork heads. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.ref }} + path: pull-request + token: ${{ steps.generate-token.outputs.token }} + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install yamlrocks + # The script edits YAML through yamlrocks. Take the pin from the + # base branch requirements so this workflow has no copy of its own. + run: pip install "$(grep -E '^yamlrocks==' requirements_test.txt | cut -d'#' -f1)" + + - name: Sync pinned versions + run: python script/sync_dependency_versions.py --root pull-request + + - name: Push changes + working-directory: pull-request + run: | + if git diff --quiet; then + echo "All pinned versions already match the requirements files." + exit 0 + fi + git config user.name "esphome[bot]" + git config user.email "115708604+esphome[bot]@users.noreply.github.com" + git commit -am "Sync pinned tool versions with requirements files" + git push diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0ea799aa4d..1af0e19273 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,6 @@ --- # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks - ci: autoupdate_commit_msg: 'pre-commit: autoupdate' autoupdate_schedule: off # Disabled until ruff versions are synced between deps and pre-commit @@ -11,7 +10,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.16.3 + rev: v0.16.5 hooks: # Run the linter. - id: ruff @@ -42,7 +41,7 @@ repos: - id: pyupgrade args: [--py312-plus] - repo: https://github.com/adrienverge/yamllint.git - rev: v1.37.1 + rev: v1.38.0 hooks: - id: yamllint exclude: ^(\.clang-format|\.clang-tidy)$ diff --git a/AGENTS.md b/AGENTS.md index 15b92c4deb..98bdd58ec5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -840,7 +840,7 @@ file does, and it is the authority when they disagree. The most useful starting cv.rename_key( CONF_OLD_KEY, CONF_NEW_KEY, removed_in="2026.6.0", component="my_component" ), - cv.Schema({ ... }), + cv.Schema({...}), ) ``` For other deprecations, warn manually during validation: diff --git a/requirements_dev.txt b/requirements_dev.txt index f2cf855d6b..ee94a2401a 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,4 @@ # Useful stuff when working in a development environment -clang-format==13.0.1 # also change in .pre-commit-config.yaml and Dockerfile when updating +clang-format==13.0.1 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py clang-tidy==22.1.8 -yamllint==1.38.0 # also change in .pre-commit-config.yaml when updating +yamllint==1.38.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py diff --git a/requirements_test.txt b/requirements_test.txt index 897445a4cb..ef70a5ac0c 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,8 +1,9 @@ pylint==4.0.8 -flake8==7.3.0 # also change in .pre-commit-config.yaml when updating -ruff==0.16.5 # also change in .pre-commit-config.yaml when updating -pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating -prek==0.5.1 # also change in .github/workflows/ci.yml when updating +flake8==7.3.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +ruff==0.16.5 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +prek==0.5.1 # .github/workflows/ci.yml reads this pin +yamlrocks==0.6.1 # used by script/sync_dependency_versions.py # Unit tests pytest==9.1.1 diff --git a/script/sync_dependency_versions.py b/script/sync_dependency_versions.py new file mode 100755 index 0000000000..a97a58b3b0 --- /dev/null +++ b/script/sync_dependency_versions.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Keep pre-commit hook revs in sync with the requirements files. + +Dependabot only bumps the ``package==version`` pins in ``requirements*.txt``. +Some of those tools are pinned a second time as hook ``rev`` values in +``.pre-commit-config.yaml``. This script treats the requirements files as +the source of truth and rewrites the revs to match, editing the config +through yamlrocks so comments and layout survive. + +Run without arguments to apply the changes in place, or with ``--check`` to +only report drift (exit status 1 when anything is out of sync). +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from pathlib import Path +import re +import sys +from typing import Any + +import yamlrocks + +REPO_ROOT = Path(__file__).resolve().parent.parent +PRECOMMIT_CONFIG = ".pre-commit-config.yaml" + + +class SyncError(Exception): + """A pin could not be located in a requirements file or the config.""" + + +@dataclass(frozen=True) +class SyncTarget: + """A requirements pin and the pre-commit repo whose rev mirrors it.""" + + package: str + requirements_file: str + repo: str + + +SYNC_TARGETS: tuple[SyncTarget, ...] = ( + SyncTarget( + "ruff", "requirements_test.txt", "https://github.com/astral-sh/ruff-pre-commit" + ), + SyncTarget("flake8", "requirements_test.txt", "https://github.com/PyCQA/flake8"), + SyncTarget( + "pyupgrade", "requirements_test.txt", "https://github.com/asottile/pyupgrade" + ), + SyncTarget( + "clang-format", + "requirements_dev.txt", + "https://github.com/pre-commit/mirrors-clang-format", + ), + SyncTarget( + "yamllint", + "requirements_dev.txt", + "https://github.com/adrienverge/yamllint.git", + ), +) + + +def read_requirement_version(requirements: str, package: str) -> str | None: + """Return the ``==`` pin for ``package`` or None when it is not pinned.""" + pattern = re.compile( + rf"^{re.escape(package)}==(?P[^\s#]+)", + re.MULTILINE | re.IGNORECASE, + ) + match = pattern.search(requirements) + return match.group("version") if match else None + + +def find_repo_entry(doc: Any, repo: str) -> Any: + """Return the single ``- repo:`` block for ``repo`` in a pre-commit doc.""" + try: + entries = [entry for entry in doc["repos"] if entry["repo"] == repo] + except KeyError as err: + raise SyncError(f"malformed pre-commit config, missing key {err}") from None + if len(entries) != 1: + raise SyncError( + f"expected exactly one block for repo {repo}, found {len(entries)}" + ) + return entries[0] + + +def current_rev(entry: Any, repo: str) -> tuple[str, str]: + """Split the block's rev into its tag prefix (``v`` or empty) and version.""" + if "rev" not in entry: + raise SyncError(f"repo {repo} has no rev") + rev = entry["rev"] + if not isinstance(rev, str): + # A rev such as ``1.0`` parses as a number and cannot be compared or + # rewritten safely; quote it in the config instead. + raise SyncError(f"rev of repo {repo} is not a string: {rev!r}") + prefix = "v" if rev.startswith("v") else "" + return prefix, rev.removeprefix("v") + + +def sync(root: Path, *, write: bool) -> list[str]: + """Bring every hook rev in line with its requirements pin. + + Returns one description per rev that was (or, when ``write`` is False, + would be) changed. Raises SyncError when a pin cannot be found, which + means SYNC_TARGETS has gone stale and needs updating by hand. + """ + config_path = root / PRECOMMIT_CONFIG + doc = yamlrocks.loads(config_path.read_bytes(), option=yamlrocks.OPT_ROUND_TRIP) + requirements: dict[str, str] = {} + changes: list[str] = [] + for target in SYNC_TARGETS: + if target.requirements_file not in requirements: + requirements[target.requirements_file] = ( + root / target.requirements_file + ).read_text() + version = read_requirement_version( + requirements[target.requirements_file], target.package + ) + if version is None: + raise SyncError( + f"{target.requirements_file}: no '{target.package}==' pin found" + ) + + entry = find_repo_entry(doc, target.repo) + prefix, current = current_rev(entry, target.repo) + if current == version: + continue + changes.append(f"{target.package}: {current} -> {version}") + entry["rev"] = f"{prefix}{version}" + + if changes and write: + config_path.write_bytes(doc.to_yaml()) + return changes + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--check", + action="store_true", + help="report drift without modifying any file; exit 1 if out of sync", + ) + parser.add_argument( + "--root", + type=Path, + default=REPO_ROOT, + help="repository checkout to operate on (default: this checkout)", + ) + args = parser.parse_args(argv) + + try: + changes = sync(args.root, write=not args.check) + except SyncError as err: + print(f"error: {err}", file=sys.stderr) + return 1 + + for change in changes: + print(change) + if args.check and changes: + return 1 + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/tests/script/test_sync_dependency_versions.py b/tests/script/test_sync_dependency_versions.py new file mode 100644 index 0000000000..787c8112d9 --- /dev/null +++ b/tests/script/test_sync_dependency_versions.py @@ -0,0 +1,219 @@ +"""Unit tests for script/sync_dependency_versions.py.""" + +from pathlib import Path +import subprocess +import sys + +import pytest +import yamlrocks + +sys.path.insert(0, str((Path(__file__).parent / ".." / ".." / "script").resolve())) + +import sync_dependency_versions as sync_mod # noqa: E402 + +PRECOMMIT = """\ +# See https://pre-commit.com for more information +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.1.0 + hooks: + - id: ruff + - repo: https://github.com/PyCQA/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + - repo: https://github.com/asottile/pyupgrade + rev: v3.0.0 + hooks: + - id: pyupgrade + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v13.0.1 + hooks: + - id: clang-format + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.0.0 + hooks: + - id: yamllint + - repo: local + hooks: + - id: pylint +""" + +REQ_TEST = """\ +pylint==4.0.8 +flake8==7.1.0 +ruff==0.2.0 # comment +pyupgrade==3.0.0 +""" + +REQ_DEV = """\ +clang-format==13.0.1 +yamllint==1.0.0 +""" + +RUFF_REPO = "https://github.com/astral-sh/ruff-pre-commit" +DUPLICATE_RUFF_BLOCK = f" - repo: {RUFF_REPO}\n rev: v0.3.0\n hooks: []\n" + +EXPECTED_DRIFT = ["ruff: 0.1.0 -> 0.2.0", "flake8: 7.0.0 -> 7.1.0"] +EXPECTED_PRECOMMIT = PRECOMMIT.replace("rev: v0.1.0", "rev: v0.2.0").replace( + "rev: 7.0.0", "rev: 7.1.0" +) + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + """A fake checkout where ruff (v-prefixed) and flake8 (bare) have drifted.""" + (tmp_path / ".pre-commit-config.yaml").write_text(PRECOMMIT) + (tmp_path / "requirements_test.txt").write_text(REQ_TEST) + (tmp_path / "requirements_dev.txt").write_text(REQ_DEV) + return tmp_path + + +def _load(text: str) -> object: + return yamlrocks.loads(text.encode(), option=yamlrocks.OPT_ROUND_TRIP) + + +@pytest.mark.parametrize( + ("requirements", "expected"), + [ + ("prek==0.5.1 # comment\n", "0.5.1"), + ("Prek==0.5.1\n", "0.5.1"), + ("other==1.0\nprek==0.5.1\n", "0.5.1"), + ("prek>=0.5.1\n", None), + ("prek-extra==0.5.1\n", None), + ("", None), + ], +) +def test_read_requirement_version(requirements: str, expected: str | None) -> None: + assert sync_mod.read_requirement_version(requirements, "prek") == expected + + +def test_find_repo_entry() -> None: + entry = sync_mod.find_repo_entry(_load(PRECOMMIT), RUFF_REPO) + assert entry["rev"] == "v0.1.0" + + +@pytest.mark.parametrize( + ("text", "message"), + [ + ("hooks: []\n", "missing key 'repos'"), + ("repos:\n - rev: 1.0.0\n", "missing key 'repo'"), + (PRECOMMIT + DUPLICATE_RUFF_BLOCK, "found 2"), + ("repos:\n - repo: other\n rev: 1.0.0\n", "found 0"), + ], +) +def test_find_repo_entry_errors(text: str, message: str) -> None: + with pytest.raises(sync_mod.SyncError, match=message): + sync_mod.find_repo_entry(_load(text), RUFF_REPO) + + +@pytest.mark.parametrize( + ("rev", "expected"), + [("v0.1.0", ("v", "0.1.0")), ("7.0.0", ("", "7.0.0")), ("'1.0'", ("", "1.0"))], +) +def test_current_rev(rev: str, expected: tuple[str, str]) -> None: + doc = _load(f"repos:\n - repo: {RUFF_REPO}\n rev: {rev}\n") + assert sync_mod.current_rev(doc["repos"][0], RUFF_REPO) == expected + + +@pytest.mark.parametrize( + ("block", "message"), + [(" hooks: []\n", "has no rev"), (" rev: 1.0\n", "not a string: 1.0")], +) +def test_current_rev_errors(block: str, message: str) -> None: + doc = _load(f"repos:\n - repo: {RUFF_REPO}\n{block}") + with pytest.raises(sync_mod.SyncError, match=message): + sync_mod.current_rev(doc["repos"][0], RUFF_REPO) + + +def test_sync_reports_without_writing(root: Path) -> None: + assert sync_mod.sync(root, write=False) == EXPECTED_DRIFT + assert (root / ".pre-commit-config.yaml").read_text() == PRECOMMIT + + +def test_sync_writes_keeps_layout_and_is_idempotent(root: Path) -> None: + assert sync_mod.sync(root, write=True) == EXPECTED_DRIFT + assert (root / ".pre-commit-config.yaml").read_text() == EXPECTED_PRECOMMIT + assert sync_mod.sync(root, write=True) == [] + + +def test_sync_does_not_touch_a_config_that_matches(root: Path) -> None: + (root / ".pre-commit-config.yaml").write_text(EXPECTED_PRECOMMIT) + before = (root / ".pre-commit-config.yaml").stat().st_mtime_ns + assert sync_mod.sync(root, write=True) == [] + assert (root / ".pre-commit-config.yaml").stat().st_mtime_ns == before + + +def test_sync_missing_requirement_pin(root: Path) -> None: + (root / "requirements_dev.txt").write_text("") + with pytest.raises(sync_mod.SyncError, match="no 'clang-format==' pin"): + sync_mod.sync(root, write=True) + + +def test_sync_propagates_config_errors(root: Path) -> None: + (root / ".pre-commit-config.yaml").write_text(PRECOMMIT + DUPLICATE_RUFF_BLOCK) + with pytest.raises(sync_mod.SyncError, match="found 2"): + sync_mod.sync(root, write=True) + + +def test_main_check_reports_drift( + root: Path, capsys: pytest.CaptureFixture[str] +) -> None: + assert sync_mod.main(["--check", "--root", str(root)]) == 1 + assert capsys.readouterr().out.splitlines() == EXPECTED_DRIFT + assert (root / ".pre-commit-config.yaml").read_text() == PRECOMMIT + + +def test_main_writes_then_check_is_clean( + root: Path, capsys: pytest.CaptureFixture[str] +) -> None: + assert sync_mod.main(["--root", str(root)]) == 0 + assert capsys.readouterr().out.splitlines() == EXPECTED_DRIFT + assert sync_mod.main(["--check", "--root", str(root)]) == 0 + assert capsys.readouterr().out == "" + + +def test_main_reports_sync_error( + root: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (root / "requirements_dev.txt").write_text("") + assert sync_mod.main(["--root", str(root)]) == 1 + assert ( + "error: requirements_dev.txt: no 'clang-format==' pin" + in capsys.readouterr().err + ) + + +def test_main_defaults_to_repo_root(monkeypatch: pytest.MonkeyPatch) -> None: + seen: dict[str, object] = {} + + def fake_sync(root: Path, *, write: bool) -> list[str]: + seen["root"] = root + seen["write"] = write + return [] + + monkeypatch.setattr(sync_mod, "sync", fake_sync) + assert sync_mod.main([]) == 0 + assert seen == {"root": sync_mod.REPO_ROOT, "write": True} + + +def test_repository_is_in_sync() -> None: + """The real checkout must match; a failure here means a rev has drifted. + + Also proves every SYNC_TARGETS entry still resolves in the real files. + """ + assert sync_mod.sync(sync_mod.REPO_ROOT, write=False) == [] + + +def test_cli_entry_point(root: Path) -> None: + """Run the script the way the workflow does, as a subprocess.""" + script = Path(sync_mod.__file__) + result = subprocess.run( + [sys.executable, str(script), "--check", "--root", str(root)], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 1 + assert result.stdout.splitlines() == EXPECTED_DRIFT From 390742cf9ba113ea89bc88a05582c4c409a1bd63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Sep 2026 23:47:46 +0000 Subject: [PATCH 128/147] Bump ruff from 0.16.5 to 0.16.6 (#19022) Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Signed-off-by: dependabot[bot] --- .pre-commit-config.yaml | 2 +- esphome/api_client.py | 4 +--- esphome/components/debug/sensor.py | 6 +----- esphome/components/debug/text_sensor.py | 6 +----- esphome/components/esp32/const.py | 11 ++--------- esphome/components/nextion/display.py | 7 +------ esphome/happy_eyeballs.py | 5 +---- requirements_test.txt | 2 +- 8 files changed, 9 insertions(+), 34 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1af0e19273..95e6f0f73e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.16.5 + rev: v0.16.6 hooks: # Run the linter. - id: ruff diff --git a/esphome/api_client.py b/esphome/api_client.py index fb41075de8..2b93b4790f 100644 --- a/esphome/api_client.py +++ b/esphome/api_client.py @@ -23,9 +23,7 @@ from esphome.util import safe_print if TYPE_CHECKING: from collections.abc import Callable - from aioesphomeapi.api_pb2 import ( - SubscribeLogsResponse, # pylint: disable=no-name-in-module - ) + from aioesphomeapi.api_pb2 import SubscribeLogsResponse # pylint: disable=no-name-in-module _LOGGER = logging.getLogger(__name__) diff --git a/esphome/components/debug/sensor.py b/esphome/components/debug/sensor.py index e53cb0d1e4..80d1daa81f 100644 --- a/esphome/components/debug/sensor.py +++ b/esphome/components/debug/sensor.py @@ -23,11 +23,7 @@ from esphome.const import ( ) from esphome.types import ConfigType -from . import ( # noqa: F401 pylint: disable=unused-import - CONF_DEBUG_ID, - FILTER_SOURCE_FILES, - DebugComponent, -) +from . import CONF_DEBUG_ID, FILTER_SOURCE_FILES, DebugComponent # noqa: F401 pylint: disable=unused-import DEPENDENCIES = ["debug"] diff --git a/esphome/components/debug/text_sensor.py b/esphome/components/debug/text_sensor.py index 9d4fcc1b42..2e02af67cb 100644 --- a/esphome/components/debug/text_sensor.py +++ b/esphome/components/debug/text_sensor.py @@ -9,11 +9,7 @@ from esphome.const import ( ) from esphome.types import ConfigType -from . import ( # noqa: F401 pylint: disable=unused-import - CONF_DEBUG_ID, - FILTER_SOURCE_FILES, - DebugComponent, -) +from . import CONF_DEBUG_ID, FILTER_SOURCE_FILES, DebugComponent # noqa: F401 pylint: disable=unused-import DEPENDENCIES = ["debug"] diff --git a/esphome/components/esp32/const.py b/esphome/components/esp32/const.py index e7d8a66e7a..a0c9809c50 100644 --- a/esphome/components/esp32/const.py +++ b/esphome/components/esp32/const.py @@ -3,18 +3,11 @@ import esphome.codegen as cg # Re-exported for the many esp32-side users; defined in esphome.const # and esphome.espidf so the upload/logs fast path can use them without # importing this package. -from esphome.const import ( # noqa: F401 # pylint: disable=unused-import - KEY_ESP32, - KEY_FLASH_SIZE, - KEY_IDF_VERSION, - KEY_VARIANT, -) +from esphome.const import KEY_ESP32, KEY_FLASH_SIZE, KEY_IDF_VERSION, KEY_VARIANT # noqa: F401 # pylint: disable=unused-import # Back compat for external components only; in-tree callers import it # from esphome.espidf directly. -from esphome.espidf import ( # noqa: F401 # pylint: disable=unused-import - variant_to_idf_target, -) +from esphome.espidf import variant_to_idf_target # noqa: F401 # pylint: disable=unused-import KEY_BOARD = "board" KEY_SDKCONFIG_OPTIONS = "sdkconfig_options" diff --git a/esphome/components/nextion/display.py b/esphome/components/nextion/display.py index 3f5ba94b40..a5894bdaf7 100644 --- a/esphome/components/nextion/display.py +++ b/esphome/components/nextion/display.py @@ -14,12 +14,7 @@ from esphome.const import ( ) from esphome.core import CORE, TimePeriod -from . import ( # noqa: F401 pylint: disable=unused-import - FILTER_SOURCE_FILES, - Nextion, - nextion_ns, - nextion_ref, -) +from . import FILTER_SOURCE_FILES, Nextion, nextion_ns, nextion_ref # noqa: F401 pylint: disable=unused-import from .base_component import ( CONF_AUTO_WAKE_ON_TOUCH, CONF_COMMAND_SPACING, diff --git a/esphome/happy_eyeballs.py b/esphome/happy_eyeballs.py index 35092e7daa..8b0d020862 100644 --- a/esphome/happy_eyeballs.py +++ b/esphome/happy_eyeballs.py @@ -69,10 +69,7 @@ def _make_create_connection() -> Callable[..., socket.socket]: from aiohappyeyeballs import start_connection from urllib3.exceptions import LocationParseError - from urllib3.util.connection import ( # noqa: PLC2701 - _set_socket_options, - allowed_gai_family, - ) + from urllib3.util.connection import _set_socket_options, allowed_gai_family # noqa: PLC2701 from urllib3.util.timeout import _DEFAULT_TIMEOUT # noqa: PLC2701 from esphome import async_thread diff --git a/requirements_test.txt b/requirements_test.txt index ef70a5ac0c..9fd82b7509 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.0.8 flake8==7.3.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py -ruff==0.16.5 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +ruff==0.16.6 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py prek==0.5.1 # .github/workflows/ci.yml reads this pin yamlrocks==0.6.1 # used by script/sync_dependency_versions.py From 89a56298c231a138080a8604deea8ebb5a630369 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:00:28 +0000 Subject: [PATCH 129/147] Bump prek from 0.5.1 to 0.5.2 (#19021) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 9fd82b7509..cd0427f33e 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,7 @@ pylint==4.0.8 flake8==7.3.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py ruff==0.16.6 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py -prek==0.5.1 # .github/workflows/ci.yml reads this pin +prek==0.5.2 # .github/workflows/ci.yml reads this pin yamlrocks==0.6.1 # used by script/sync_dependency_versions.py # Unit tests From 50ca38119873fc717db2ec00ef9b614d5539921b Mon Sep 17 00:00:00 2001 From: mipa87 <62723159+mipa87@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:10:55 +0200 Subject: [PATCH 130/147] [i2s_audio] Keep a start request that arrives while the speaker task stops (#19027) Co-authored-by: Claude Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- .../components/i2s_audio/speaker/i2s_audio_speaker.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 5e271e671e..1c2eb12904 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -91,7 +91,14 @@ void I2SAudioSpeakerBase::loop() { this->speaker_task_handle_ = nullptr; this->stop_i2s_driver_(); - xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::ALL_BITS); + // ALL_BITS includes COMMAND_START. Take the bits from the clear itself, not from the snapshot at + // the top of loop(): the audio source's task can raise a start at any point above, including + // during stop_i2s_driver_(), and nothing would ever re-issue it. + const EventBits_t bits_before_clear = xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::ALL_BITS); + if (bits_before_clear & SpeakerEventGroupBits::COMMAND_START) { + ESP_LOGD(TAG, "Start requested while stopping; keeping the request"); + xEventGroupSetBits(this->event_group_, SpeakerEventGroupBits::COMMAND_START); + } this->status_clear_error(); this->on_task_stopped(); From 56c3361b9adafd3f1f433987d04f27f549a4d965 Mon Sep 17 00:00:00 2001 From: mipa87 <62723159+mipa87@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:13:03 +0200 Subject: [PATCH 131/147] [audio] Do not treat MP3_STREAM_INFO_CHANGED as a fatal decoder error (#19028) Co-authored-by: Claude --- esphome/components/audio/audio_decoder.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index fe9ad9c9ad..051395606c 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -313,9 +313,10 @@ FileDecoderState AudioDecoder::decode_mp3_() { this->output_transfer_buffer_->increase_buffer_length( this->audio_stream_info_.value().frames_to_bytes(samples_decoded)); } - } else if (result == micro_mp3::MP3_STREAM_INFO_READY) { - // First successful header parse: capture stream info and resize the output buffer to fit one full frame. - // microMP3 always outputs 16-bit PCM. + } else if (result == micro_mp3::MP3_STREAM_INFO_READY || result == micro_mp3::MP3_STREAM_INFO_CHANGED) { + // Header parsed: capture stream info and resize the output buffer to fit one full frame. + // microMP3 always outputs 16-bit PCM. MP3_STREAM_INFO_CHANGED is handled identically: despite its + // negative value it is documented as recoverable, so it must not reach the catch-all below. this->audio_stream_info_ = audio::AudioStreamInfo(16, this->mp3_decoder_->get_channels(), this->mp3_decoder_->get_sample_rate()); this->free_buffer_required_ = From 62eafc477d9ab731b1ce5b8d493d5b69c7e5b468 Mon Sep 17 00:00:00 2001 From: Ryan Ronnander <61520+ryan-ronnander@users.noreply.github.com> Date: Mon, 7 Sep 2026 21:09:02 -0400 Subject: [PATCH 132/147] [mqtt] Restore brightness flag in light discovery (#18950) --- esphome/components/mqtt/mqtt_light.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/esphome/components/mqtt/mqtt_light.cpp b/esphome/components/mqtt/mqtt_light.cpp index aa47bdf996..a8b52a3839 100644 --- a/esphome/components/mqtt/mqtt_light.cpp +++ b/esphome/components/mqtt/mqtt_light.cpp @@ -67,6 +67,9 @@ void MQTTJSONLightComponent::send_discovery(JsonObject root, mqtt::SendDiscovery if (traits.supports_color_mode(ColorMode::RGB_COLD_WARM_WHITE)) color_modes.add(ESPHOME_F("rgbww")); + if (traits.supports_color_capability(ColorCapability::BRIGHTNESS)) + root[ESPHOME_F("brightness")] = true; + if (traits.supports_color_mode(ColorMode::COLOR_TEMPERATURE) || traits.supports_color_mode(ColorMode::COLD_WARM_WHITE)) { root[MQTT_MIN_MIREDS] = traits.get_min_mireds(); From 639ce609bf70332146f25d04cdf5ac6a15bb4ee2 Mon Sep 17 00:00:00 2001 From: AndreKR Date: Tue, 8 Sep 2026 03:13:51 +0200 Subject: [PATCH 133/147] [logger] Fix garbled stack traces (#17939) --- esphome/components/logger/logger_esp32.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esphome/components/logger/logger_esp32.cpp b/esphome/components/logger/logger_esp32.cpp index 05fc959ceb..c3d777299d 100644 --- a/esphome/components/logger/logger_esp32.cpp +++ b/esphome/components/logger/logger_esp32.cpp @@ -5,6 +5,7 @@ #include #include +#include #ifdef USE_LOGGER_UART_SELECTION_USB_SERIAL_JTAG #include @@ -76,7 +77,11 @@ void init_uart(uart_port_t uart_num, uint32_t baud_rate, int tx_buffer_size) { uart_config.parity = UART_PARITY_DISABLE; uart_config.stop_bits = UART_STOP_BITS_1; uart_config.flow_ctrl = UART_HW_FLOWCTRL_DISABLE; +#if SOC_UART_SUPPORT_XTAL_CLK + uart_config.source_clk = UART_SCLK_XTAL; +#else uart_config.source_clk = UART_SCLK_DEFAULT; +#endif uart_param_config(uart_num, &uart_config); // The logger only writes to UART, never reads, so use the minimum RX buffer. // ESP-IDF requires rx_buffer_size > UART_HW_FIFO_LEN (128 bytes). From e6aa575f2e960f406cc8edaccb9719dc11f2a92b Mon Sep 17 00:00:00 2001 From: Samuel Sieb Date: Mon, 7 Sep 2026 18:27:49 -0700 Subject: [PATCH 134/147] [dallas_temp] filter 85 temp from sensor reset (#17877) Co-authored-by: Samuel Sieb --- esphome/components/dallas_temp/dallas_temp.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/esphome/components/dallas_temp/dallas_temp.cpp b/esphome/components/dallas_temp/dallas_temp.cpp index ab4a8c458f..c418362ced 100644 --- a/esphome/components/dallas_temp/dallas_temp.cpp +++ b/esphome/components/dallas_temp/dallas_temp.cpp @@ -6,6 +6,7 @@ namespace esphome::dallas_temp { static const char *const TAG = "dallas.temp.sensor"; static const uint8_t DALLAS_MODEL_DS18S20 = 0x10; +static const uint8_t DALLAS_MODEL_DS18B20 = 0x28; static const uint8_t DALLAS_COMMAND_START_CONVERSION = 0x44; static const uint8_t DALLAS_COMMAND_READ_SCRATCH_PAD = 0xBE; static const uint8_t DALLAS_COMMAND_WRITE_SCRATCH_PAD = 0x4E; @@ -154,7 +155,14 @@ float DallasTemperatureSensor::get_temp_c_() { default: break; } - + // undocumented test for powerup measurement of 85 + // https://github.com/cpetrich/counterfeit_DS18B20#solution-to-the-85-c-problem + if ((this->address_ & 0xff) == DALLAS_MODEL_DS18B20) { + if ((temp == 85 * 16) && (this->scratch_pad_[6] == 0xc)) { + ESP_LOGD(TAG, "dropping reading caused by sensor reset"); + return NAN; + } + } return temp / 16.0f; } From d34ffaf3928ef4a5fdaccc94f30b3ab59f6b75c4 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Mon, 7 Sep 2026 18:57:50 -0700 Subject: [PATCH 135/147] [ble_client] Report Established from nodes that never read services (#17920) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/ble_client/automation.h | 34 +++++++++++++++------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/esphome/components/ble_client/automation.h b/esphome/components/ble_client/automation.h index 94eeb83b3e..93aae23b6a 100644 --- a/esphome/components/ble_client/automation.h +++ b/esphome/components/ble_client/automation.h @@ -22,6 +22,23 @@ class Automation { static const char *const TAG; }; +// Base for nodes that never read the parent's services. +// The parent releases its services only once every node reports Established, so a node that never +// reports it keeps that memory allocated for the life of the connection. +class BLEClientServicelessNode : public BLEClientNode { + public: + // Final so that Established is always reported on SEARCH_CMPL, before the derived node sees the event. + void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) final { + if (event == ESP_GATTC_SEARCH_CMPL_EVT) + this->node_state = espbt::ClientState::ESTABLISHED; + this->on_gattc_event(event, gattc_if, param); + } + + protected: + // Derived nodes handle GATT events here rather than by overriding the handler above. + virtual void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) {} +}; + // implement on_connect automation. class BLEClientConnectTrigger final : public Trigger<>, public BLEClientNode { public: @@ -61,7 +78,7 @@ class BLEClientDisconnectTrigger final : public Trigger<>, public BLEClientNode } }; -class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientNode { +class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientServicelessNode { public: explicit BLEClientPasskeyRequestTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -71,7 +88,7 @@ class BLEClientPasskeyRequestTrigger final : public Trigger<>, public BLEClientN } }; -class BLEClientPasskeyNotificationTrigger final : public Trigger, public BLEClientNode { +class BLEClientPasskeyNotificationTrigger final : public Trigger, public BLEClientServicelessNode { public: explicit BLEClientPasskeyNotificationTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -82,7 +99,7 @@ class BLEClientPasskeyNotificationTrigger final : public Trigger, publ } }; -class BLEClientNumericComparisonRequestTrigger final : public Trigger, public BLEClientNode { +class BLEClientNumericComparisonRequestTrigger final : public Trigger, public BLEClientServicelessNode { public: explicit BLEClientNumericComparisonRequestTrigger(BLEClient *parent) { parent->register_ble_node(this); } void loop() override {} @@ -315,19 +332,17 @@ template class BLEClientRemoveBondAction final : public Action class BLEClientConnectAction final : public Action, public BLEClientNode { +template class BLEClientConnectAction final : public Action, public BLEClientServicelessNode { public: BLEClientConnectAction(BLEClient *ble_client) { ble_client->register_ble_node(this); ble_client_ = ble_client; } - void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, - esp_ble_gattc_cb_param_t *param) override { + void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) override { if (this->num_running_ == 0) return; switch (event) { case ESP_GATTC_SEARCH_CMPL_EVT: - this->node_state = espbt::ClientState::ESTABLISHED; this->parent()->run_later([this]() { this->play_next_tuple_(this->var_); }); break; // if the connection is closed, terminate the automation chain. @@ -364,14 +379,13 @@ template class BLEClientConnectAction final : public Action var_{}; }; -template class BLEClientDisconnectAction final : public Action, public BLEClientNode { +template class BLEClientDisconnectAction final : public Action, public BLEClientServicelessNode { public: BLEClientDisconnectAction(BLEClient *ble_client) { ble_client->register_ble_node(this); ble_client_ = ble_client; } - void gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, - esp_ble_gattc_cb_param_t *param) override { + void on_gattc_event(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_if, esp_ble_gattc_cb_param_t *param) override { if (this->num_running_ == 0) return; switch (event) { From 574762f07861b7008225bb7de89164ed697836da Mon Sep 17 00:00:00 2001 From: Davide D M Date: Tue, 8 Sep 2026 03:59:05 +0200 Subject: [PATCH 136/147] [debug] Check reboot source pref on ESP_RST_WDT and guard against empty source (#17537) --- esphome/components/debug/debug_esp32.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/esphome/components/debug/debug_esp32.cpp b/esphome/components/debug/debug_esp32.cpp index 969cd840cf..8e1a67224e 100644 --- a/esphome/components/debug/debug_esp32.cpp +++ b/esphome/components/debug/debug_esp32.cpp @@ -66,11 +66,15 @@ const char *DebugComponent::get_reset_reason_(std::spanmake_preference(REBOOT_MAX_LEN, fnv1_hash_extend(fnv1_hash(REBOOT_KEY), App.get_name().c_str())); char reboot_source[REBOOT_MAX_LEN]{}; - if (pref.load(&reboot_source)) { + if (pref.load(&reboot_source) && reboot_source[0] != '\0') { reboot_source[REBOOT_MAX_LEN - 1] = '\0'; snprintf(buf, size, "Reboot request from %s", reboot_source); } else { From 94e5c3839d8372e95a320cd1796ca500de75be91 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 16:17:22 +1200 Subject: [PATCH 137/147] [udp] Use cv.invalid for relocated packet_transport options (#19032) --- esphome/components/udp/__init__.py | 16 +++------ tests/unit_tests/components/udp/__init__.py | 0 tests/unit_tests/components/udp/test_init.py | 37 ++++++++++++++++++++ 3 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 tests/unit_tests/components/udp/__init__.py create mode 100644 tests/unit_tests/components/udp/test_init.py diff --git a/esphome/components/udp/__init__.py b/esphome/components/udp/__init__.py index a782d875b9..d96a731e9c 100644 --- a/esphome/components/udp/__init__.py +++ b/esphome/components/udp/__init__.py @@ -1,5 +1,4 @@ -from collections.abc import Callable -from typing import Any, NoReturn +from typing import Any from esphome import automation from esphome.automation import Trigger @@ -48,17 +47,10 @@ UDP_SCHEMA = cv.Schema( ) -def is_relocated(option: str) -> Callable[[Any], NoReturn]: - def validator(value: Any) -> NoReturn: - raise cv.Invalid( - f"The '{option}' option should now be configured in the 'packet_transport' component" - ) - - return validator - - RELOCATED = { - cv.Optional(x): is_relocated(x) + cv.Optional(x): cv.invalid( + f"The '{x}' option should now be configured in the 'packet_transport' component" + ) for x in ( CONF_PROVIDERS, CONF_ENCRYPTION, diff --git a/tests/unit_tests/components/udp/__init__.py b/tests/unit_tests/components/udp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/components/udp/test_init.py b/tests/unit_tests/components/udp/test_init.py new file mode 100644 index 0000000000..5afc92e9c6 --- /dev/null +++ b/tests/unit_tests/components/udp/test_init.py @@ -0,0 +1,37 @@ +"""Tests for the udp component configuration schema.""" + +from __future__ import annotations + +import pytest + +from esphome.components import udp +from esphome.components.packet_transport import ( + CONF_BINARY_SENSORS, + CONF_ENCRYPTION, + CONF_PING_PONG_ENABLE, + CONF_PROVIDERS, + CONF_ROLLING_CODE_ENABLE, + CONF_SENSORS, +) +import esphome.config_validation as cv + + +@pytest.mark.parametrize( + "option", + [ + CONF_PROVIDERS, + CONF_ENCRYPTION, + CONF_PING_PONG_ENABLE, + CONF_ROLLING_CODE_ENABLE, + CONF_SENSORS, + CONF_BINARY_SENSORS, + ], +) +def test_relocated_option_rejected(option: str) -> None: + """Options that moved to packet_transport raise a pointing error.""" + with pytest.raises(cv.Invalid) as exc_info: + udp.CONFIG_SCHEMA({option: True}) + assert ( + f"The '{option}' option should now be configured in the 'packet_transport' component" + in str(exc_info.value) + ) From 5722ccba372857c175e75de217b2e455ca57f88d Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Tue, 8 Sep 2026 00:36:04 -0400 Subject: [PATCH 138/147] [tuya] Build without a network component (#18948) --- esphome/components/tuya/tuya.cpp | 17 +++++++++-- .../tuya/test-no-network.bk72xx-ard.yaml | 29 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/components/tuya/test-no-network.bk72xx-ard.yaml diff --git a/esphome/components/tuya/tuya.cpp b/esphome/components/tuya/tuya.cpp index 82fb96d787..f9b4fe2453 100644 --- a/esphome/components/tuya/tuya.cpp +++ b/esphome/components/tuya/tuya.cpp @@ -1,10 +1,13 @@ #include "tuya.h" -#include "esphome/components/network/util.h" #include "esphome/core/gpio.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/util.h" +#ifdef USE_NETWORK +#include "esphome/components/network/util.h" +#endif + #ifdef USE_WIFI #include "esphome/components/wifi/wifi_component.h" #endif @@ -22,6 +25,14 @@ static const int MAX_RETRIES = 5; // Max bytes to log for datapoint values (larger values are truncated) static constexpr size_t MAX_DATAPOINT_LOG_BYTES = 16; +static bool network_is_connected() { +#ifdef USE_NETWORK + return network::is_connected(); +#else + return false; +#endif +} + void Tuya::setup() { this->set_interval("heartbeat", 15000, [this] { this->send_empty_command_(TuyaCommandType::HEARTBEAT); }); if (this->status_pin_ != nullptr) { @@ -554,14 +565,14 @@ void Tuya::send_empty_command_(TuyaCommandType command) { } void Tuya::set_status_pin_() { - bool is_network_ready = network::is_connected() && remote_is_connected(); + bool is_network_ready = network_is_connected() && remote_is_connected(); this->status_pin_->digital_write(is_network_ready); } uint8_t Tuya::get_wifi_status_code_() { uint8_t status = 0x02; - if (network::is_connected()) { + if (network_is_connected()) { status = 0x03; // Protocol version 3 also supports specifying when connected to "the cloud" diff --git a/tests/components/tuya/test-no-network.bk72xx-ard.yaml b/tests/components/tuya/test-no-network.bk72xx-ard.yaml new file mode 100644 index 0000000000..64207e94e3 --- /dev/null +++ b/tests/components/tuya/test-no-network.bk72xx-ard.yaml @@ -0,0 +1,29 @@ +# Tuya without any network component (no wifi/ethernet/api), as used on +# serial-only or BLE-only Tuya MCU boards. Regression test for +# https://github.com/esphome/esphome/issues/18942 +substitutions: + status_pin: P6 + +packages: + uart: !include ../../test_build_components/common/uart/bk72xx-ard.yaml + +tuya: + status_pin: ${status_pin} + +binary_sensor: + - platform: tuya + id: tuya_presence + sensor_datapoint: 101 + +sensor: + - platform: tuya + id: tuya_light_intensity + sensor_datapoint: 103 + +number: + - platform: tuya + id: tuya_far_detection + number_datapoint: 109 + min_value: 0 + max_value: 600 + step: 1 From 53075e41391a706a52d69885f70057cc9616c675 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 06:42:30 +0200 Subject: [PATCH 139/147] [core] Skip PlatformIO's private-package authorization probe (#18823) --- esphome/platformio/library.py | 6 ++- esphome/platformio/prefetch.py | 2 + esphome/platformio/runner.py | 14 ++++++- tests/unit_tests/test_platformio_library.py | 19 ++++++++++ tests/unit_tests/test_platformio_prefetch.py | 14 +++++++ tests/unit_tests/test_platformio_runner.py | 40 ++++++++++++++++++++ 6 files changed, 93 insertions(+), 2 deletions(-) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 3ff60f8aaa..fb6779b807 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -616,11 +616,15 @@ def _make_registry_client() -> Any: elsewhere, not by the PlatformIO registry. """ from platformio.package.manager._registry import PackageManagerRegistryMixin + from platformio.registry.client import RegistryClient class _Registry(PackageManagerRegistryMixin): def __init__(self) -> None: - self._registry_client = None self.pkg_type = "library" + self._registry_client = RegistryClient() + # The probe sleeps ~500 ms per lookup (see runner.patch_registry_private_packages); + # instance-level so the ESPHome process never patches PlatformIO's class + self._registry_client.allowed_private_packages = lambda: False @staticmethod def is_system_compatible(value: Any, custom_system: Any = None) -> bool: diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 17a06cb9c1..e648192b73 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -951,8 +951,10 @@ def main(argv: list[str]) -> int: """Subprocess entry point: ``prefetch ``.""" from esphome.core import CORE from esphome.log import setup_log + from esphome.platformio.runner import patch_registry_private_packages signal.signal(signal.SIGTERM, _sigterm) + patch_registry_private_packages() raw_level = os.environ.get("ESPHOME_PREFETCH_LOG_LEVEL") try: level = int(raw_level) if raw_level is not None else logging.INFO diff --git a/esphome/platformio/runner.py b/esphome/platformio/runner.py index 9bb2205a90..b9fbdec38d 100644 --- a/esphome/platformio/runner.py +++ b/esphome/platformio/runner.py @@ -2,7 +2,8 @@ Invoked via ``python -m esphome.platformio.runner`` instead of ``python -m platformio`` so that the patches (incremental rebuild -preservation, download retries) apply inside the subprocess. Running +preservation, download retries, skipping the private-package probe) apply +inside the subprocess. Running PlatformIO in a subprocess keeps its ``sys.path`` mutations and other global state from leaking into the ESPHome process. """ @@ -105,6 +106,16 @@ def patch_file_downloader() -> None: FileDownloader.__init__ = patched_init +def patch_registry_private_packages() -> None: + """Skip PlatformIO's private-package probe; it sleeps ~500 ms per lookup. + + ESPHome never uses private packages, so the answer is always False. + """ + from platformio.registry.client import RegistryClient + + RegistryClient.allowed_private_packages = staticmethod(lambda: False) # type: ignore[method-assign] + + _IGNORE_LIB_WARNINGS = "(?:Hash|Update)" # Regex patterns matched against each line of PlatformIO output. Lines that # match are dropped by RedirectText before they reach the parent process. @@ -152,6 +163,7 @@ FILTER_PLATFORMIO_LINES = [ def main() -> int: patch_structhash() patch_file_downloader() + patch_registry_private_packages() # Wrap stdout/stderr with RedirectText before PlatformIO runs: # diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 3bae39b3c1..512c883c37 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -7,6 +7,7 @@ exercised in their own test modules).""" import json import logging from pathlib import Path +from unittest.mock import Mock import pytest @@ -228,6 +229,24 @@ def test_resolve_registry_version_raises_without_pkg_file(monkeypatch): _resolve_registry_version("owner", "pkg", set()) +def test_make_registry_client_skips_private_package_probe(monkeypatch): + """Our client answers the probe locally without patching PlatformIO's class.""" + from platformio.account.client import AccountClient + from platformio.registry.client import RegistryClient + + pio_probe = RegistryClient.__dict__["allowed_private_packages"] + monkeypatch.setattr( + AccountClient, + "get_account_info", + Mock(side_effect=AssertionError("account probe must not run")), + ) + + client = lib._make_registry_client().get_registry_client_instance() + + assert client.allowed_private_packages() is False + assert RegistryClient.__dict__["allowed_private_packages"] is pio_probe + + def _patch_registry_resolve(monkeypatch: pytest.MonkeyPatch) -> None: """Stub the registry lookup so tests never touch the network.""" monkeypatch.setattr( diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 77490fd861..14c52dda8d 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1225,6 +1225,20 @@ def test_main_runs_prefetch(tmp_path: Path) -> None: mock_prefetch.assert_called_once_with(tmp_path, "testenv") +def test_main_skips_private_package_probe_before_prefetch(tmp_path: Path) -> None: + """The registry probe patch is applied before any package manager runs.""" + order: list[str] = [] + with ( + patch.object(pf, "_prefetch", side_effect=lambda *_: order.append("prefetch")), + patch( + "esphome.platformio.runner.patch_registry_private_packages", + side_effect=lambda: order.append("patch"), + ), + ): + assert pf.main([str(tmp_path), "testenv"]) == 0 + assert order == ["patch", "prefetch"] + + def test_main_bad_argv_is_a_distinct_exit( caplog: pytest.LogCaptureFixture, ) -> None: diff --git a/tests/unit_tests/test_platformio_runner.py b/tests/unit_tests/test_platformio_runner.py index f375aa457a..007455f45a 100644 --- a/tests/unit_tests/test_platformio_runner.py +++ b/tests/unit_tests/test_platformio_runner.py @@ -6,7 +6,9 @@ from collections.abc import Callable import io import sys from types import ModuleType +from unittest.mock import Mock +from platformio.registry.client import RegistryClient import pytest from esphome.platformio import runner @@ -30,6 +32,7 @@ def _prepare_main( monkeypatch.setattr(sys, "stderr", stream) monkeypatch.setattr(runner, "patch_structhash", lambda: None) monkeypatch.setattr(runner, "patch_file_downloader", lambda: None) + monkeypatch.setattr(runner, "patch_registry_private_packages", lambda: None) platformio = ModuleType("platformio") platformio_main = ModuleType("platformio.__main__") @@ -91,3 +94,40 @@ def test_main_still_filters_a_drained_partial_line( assert runner.main() == 0 assert buf.getvalue() == b"" + + +def test_main_applies_registry_private_packages_patch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The probe is patched before PlatformIO runs.""" + order: list[str] = [] + _prepare_main(monkeypatch, lambda: order.append("pio") or 0) + monkeypatch.setattr( + runner, "patch_registry_private_packages", lambda: order.append("patch") + ) + + assert runner.main() == 0 + assert order == ["patch", "pio"] + + +# Snapshot PlatformIO's own probe at import, before any test can patch it +_PIO_PROBE = RegistryClient.__dict__["allowed_private_packages"] + + +def test_patch_registry_private_packages_skips_account_probe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Answers False without touching the account client.""" + from platformio.account.client import AccountClient + + monkeypatch.setattr(RegistryClient, "allowed_private_packages", _PIO_PROBE) + monkeypatch.setattr( + AccountClient, + "get_account_info", + Mock(side_effect=AssertionError("account probe must not run")), + ) + + runner.patch_registry_private_packages() + + assert RegistryClient.allowed_private_packages() is False + assert RegistryClient().allowed_private_packages() is False From a23f7bb5693a3ef0ec23e0bcc49f6e30561ae986 Mon Sep 17 00:00:00 2001 From: Gytis Date: Tue, 8 Sep 2026 08:31:30 +0200 Subject: [PATCH 140/147] [lvgl] Add missing label dependency to qrcode, keyboard and tabview (#18387) --- esphome/components/lvgl/widgets/keyboard.py | 3 +- esphome/components/lvgl/widgets/qrcode.py | 3 +- esphome/components/lvgl/widgets/tabview.py | 3 +- .../lvgl/config/keyboard_no_label.yaml | 32 +++++++++++++++++ .../lvgl/config/qrcode_no_label.yaml | 34 ++++++++++++++++++ .../lvgl/config/tabview_no_label.yaml | 35 +++++++++++++++++++ .../lvgl/test_widget_label_dependency.py | 32 +++++++++++++++++ 7 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 tests/component_tests/lvgl/config/keyboard_no_label.yaml create mode 100644 tests/component_tests/lvgl/config/qrcode_no_label.yaml create mode 100644 tests/component_tests/lvgl/config/tabview_no_label.yaml create mode 100644 tests/component_tests/lvgl/test_widget_label_dependency.py diff --git a/esphome/components/lvgl/widgets/keyboard.py b/esphome/components/lvgl/widgets/keyboard.py index bcd2d2ae59..65516513a6 100644 --- a/esphome/components/lvgl/widgets/keyboard.py +++ b/esphome/components/lvgl/widgets/keyboard.py @@ -15,6 +15,7 @@ from ..defines import ( from ..types import LvCompound, LvType from . import Widget, WidgetType, get_widgets from .buttonmatrix import CONF_BUTTONMATRIX +from .label import CONF_LABEL from .textarea import CONF_TEXTAREA, lv_textarea_t CONF_KEYBOARD = "keyboard" @@ -49,7 +50,7 @@ class KeyboardType(WidgetType): ) def get_uses(self): - return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX + return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX, CONF_LABEL async def to_code(self, w: Widget, config: dict): add_lv_use("KEY_LISTENER") diff --git a/esphome/components/lvgl/widgets/qrcode.py b/esphome/components/lvgl/widgets/qrcode.py index df76ab6bb0..59af9168aa 100644 --- a/esphome/components/lvgl/widgets/qrcode.py +++ b/esphome/components/lvgl/widgets/qrcode.py @@ -10,6 +10,7 @@ from ..types import lv_obj_t from . import Widget, WidgetType from .canvas import CONF_CANVAS from .img import CONF_IMAGE +from .label import CONF_LABEL CONF_QRCODE = "qrcode" CONF_DARK_COLOR = "dark_color" @@ -41,7 +42,7 @@ class QrCodeType(WidgetType): ) def get_uses(self): - return CONF_CANVAS, CONF_IMAGE + return CONF_CANVAS, CONF_IMAGE, CONF_LABEL async def to_code(self, w: Widget, config): await w.set_property( diff --git a/esphome/components/lvgl/widgets/tabview.py b/esphome/components/lvgl/widgets/tabview.py index ee252ecf0b..77c88c48ff 100644 --- a/esphome/components/lvgl/widgets/tabview.py +++ b/esphome/components/lvgl/widgets/tabview.py @@ -28,6 +28,7 @@ from ..types import LV_EVENT, LvType, ObjUpdateAction, lv_obj_t, lv_obj_t_ptr from . import Widget, WidgetType, add_widgets, get_widgets, set_obj_properties from .button import button_spec from .buttonmatrix import CONF_BUTTONMATRIX, buttonmatrix_spec +from .label import CONF_LABEL from .obj import obj_spec CONF_TABVIEW = "tabview" @@ -74,7 +75,7 @@ class TabviewType(WidgetType): ) def get_uses(self): - return CONF_BUTTONMATRIX, TYPE_FLEX, CONF_BUTTON + return CONF_BUTTONMATRIX, TYPE_FLEX, CONF_BUTTON, CONF_LABEL async def to_code(self, w: Widget, config: dict): await w.set_property( diff --git a/tests/component_tests/lvgl/config/keyboard_no_label.yaml b/tests/component_tests/lvgl/config/keyboard_no_label.yaml new file mode 100644 index 0000000000..7a45a537d3 --- /dev/null +++ b/tests/component_tests/lvgl/config/keyboard_no_label.yaml @@ -0,0 +1,32 @@ +esphome: + name: test-keyboard-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - keyboard: + id: keyboard_widget diff --git a/tests/component_tests/lvgl/config/qrcode_no_label.yaml b/tests/component_tests/lvgl/config/qrcode_no_label.yaml new file mode 100644 index 0000000000..8bb1aafdd6 --- /dev/null +++ b/tests/component_tests/lvgl/config/qrcode_no_label.yaml @@ -0,0 +1,34 @@ +esphome: + name: test-qrcode-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - qrcode: + id: qr_widget + size: 100 + text: "esphome.io" diff --git a/tests/component_tests/lvgl/config/tabview_no_label.yaml b/tests/component_tests/lvgl/config/tabview_no_label.yaml new file mode 100644 index 0000000000..a3c16ab347 --- /dev/null +++ b/tests/component_tests/lvgl/config/tabview_no_label.yaml @@ -0,0 +1,35 @@ +esphome: + name: test-tabview-no-label + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - tabview: + id: tabview_widget + tabs: + - name: "Tab 1" + id: tab_1 diff --git a/tests/component_tests/lvgl/test_widget_label_dependency.py b/tests/component_tests/lvgl/test_widget_label_dependency.py new file mode 100644 index 0000000000..9d3e24c8c5 --- /dev/null +++ b/tests/component_tests/lvgl/test_widget_label_dependency.py @@ -0,0 +1,32 @@ +"""Widgets whose LVGL C implementation creates or references labels +internally (tab titles, key legends, the QR canvas fallback) must declare +the label dependency in ``get_uses()``. Otherwise a config that contains +no ``label`` widget of its own compiles LVGL without ``LV_USE_LABEL`` and +fails at C compile time with undefined ``lv_label_*`` symbols. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.components.lvgl import defines as df + + +@pytest.mark.parametrize( + "yaml_file", + [ + "qrcode_no_label.yaml", + "keyboard_no_label.yaml", + "tabview_no_label.yaml", + ], +) +def test_label_less_config_enables_lv_use_label( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + yaml_file: str, +) -> None: + generate_main(component_config_path(yaml_file)) + assert "LV_USE_LABEL" in df.get_defines() From 10a9baff746613b52df73ff07895e177cf1a0f81 Mon Sep 17 00:00:00 2001 From: Bryan Li Date: Mon, 7 Sep 2026 23:36:42 -0700 Subject: [PATCH 141/147] [rf_bridge] Fix bucket sniffing with Portisch firmware (#17683) Co-authored-by: Bryan Li Co-authored-by: Claude Fable 5 --- esphome/components/rf_bridge/rf_bridge.cpp | 109 +++++++++++++++++---- esphome/components/rf_bridge/rf_bridge.h | 13 +++ 2 files changed, 101 insertions(+), 21 deletions(-) diff --git a/esphome/components/rf_bridge/rf_bridge.cpp b/esphome/components/rf_bridge/rf_bridge.cpp index 549cce72df..a4a4da5d8c 100644 --- a/esphome/components/rf_bridge/rf_bridge.cpp +++ b/esphome/components/rf_bridge/rf_bridge.cpp @@ -18,6 +18,16 @@ void RFBridgeComponent::ack_() { } bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { + if (this->bucket_frame_candidate_ && byte == RF_CODE_START) { + // A queued next frame proves the trailing 0x55 really was the bucket + // frame's terminator: Portisch builds pulse entries from alternating + // signal edges, so the two level bits inside one pulse byte are always + // opposite — 0xAA (two high-level nibbles) cannot occur in pulse data. + // Finalize before this byte starts the new frame, so back-to-back + // deliveries are split even when loop() never observed a quiet gap + // between them. + this->finish_bucket_frame_(); + } size_t at = this->rx_buffer_.size(); this->rx_buffer_.push_back(byte); const uint8_t *raw = &this->rx_buffer_[0]; @@ -84,26 +94,21 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { break; } case RF_CODE_RFIN_BUCKET: { - if (byte != RF_CODE_STOP) { - return true; + if (at == 2) { + // The count byte: Portisch sends at most 7 buckets + sync, so 0 or + // >8 cannot be a genuine capture — reject before it can occupy the + // buffer for a full frame timeout. + return byte != 0 && byte <= B1_MAX_BUCKET_COUNT; } - - uint8_t buckets = raw[2] << 1; - std::string str; - char next_byte[3]; // 2 hex chars + null - - for (uint32_t i = 0; i <= at; i++) { - buf_append_printf(next_byte, sizeof(next_byte), 0, "%02X", raw[i]); - str += next_byte; - if ((i > 3) && buckets) { - buckets--; - } - if ((i < 3) || (buckets % 2) || (i == at - 1)) { - str += " "; - } - } - ESP_LOGI(TAG, "Received RFBridge Bucket: %s", str.c_str()); - break; + // 0x55 is legal DATA inside a B1 frame: bucket durations are sent + // with only their HIGH byte masked to 7 bits, so a duration such as + // 0x0155 puts a raw 0x55 low byte inside the table — the first 0x55 + // must therefore not end the capture. The header declares the table + // length (raw[2] pairs), so a 0x55 there is always data; one at or + // past the first pulse index is a terminator CANDIDATE, confirmed + // once the UART goes quiet (finish_bucket_frame_ in loop()). + this->bucket_frame_candidate_ = byte == RF_CODE_STOP && at >= 3 + static_cast(raw[2]) * 2; + return true; } default: ESP_LOGW(TAG, "Unknown action: 0x%02X", action); @@ -119,6 +124,47 @@ bool RFBridgeComponent::parse_bridge_byte_(uint8_t byte) { return false; } +void RFBridgeComponent::finish_bucket_frame_() { + if (this->rx_buffer_.size() < 4) { + // The candidate flag requires a header + non-empty bucket table, so + // this cannot happen while flag and buffer stay consistent; guard the + // raw[2] / size-1 reads against any future divergence anyway. + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; + return; + } + const uint8_t *raw = this->rx_buffer_.data(); + const size_t at = this->rx_buffer_.size() - 1; + + uint8_t buckets = raw[2] << 1; + std::string str; + char next_byte[3]; // 2 hex chars + null + + for (uint32_t i = 0; i <= at; i++) { + buf_append_printf(next_byte, sizeof(next_byte), 0, "%02X", raw[i]); + str += next_byte; + if ((i > 3) && buckets) { + buckets--; + } + if ((i < 3) || (buckets % 2) || (i == at - 1)) { + str += " "; + } + } + ESP_LOGI(TAG, "Received RFBridge Bucket: %s", str.c_str()); + + // Deliberately NOT ACKed: Portisch's B1 command handler leaves its + // last_sniffing_command at the previous mode (RF_CODE_RFIN), and its + // host-ACK handler re-arms sniffing from that stale value — so ACKing a + // bucket delivery silently reverts the radio to standard sniffing and + // ends bucket capture. Its delivery path is fire-and-forget and never + // waits for a host ACK. Stock Itead firmware never sends B1 frames, so + // suppressing this ACK cannot change stock-firmware behavior. + // https://github.com/esphome/esphome/issues/17682 + + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; +} + void RFBridgeComponent::write_byte_str_(const std::string &codes) { uint8_t code; int size = codes.length(); @@ -130,12 +176,31 @@ void RFBridgeComponent::write_byte_str_(const std::string &codes) { void RFBridgeComponent::loop() { const uint32_t now = App.get_loop_component_start_time(); - if (now - this->last_bridge_byte_ > 50) { + size_t avail = this->available(); + if (avail == 0 && this->bucket_frame_candidate_ && now - this->last_bridge_byte_ > BUCKET_CANDIDATE_QUIET_MS) { + // The trailing 0x55 was followed by UART quiet, so it really was the + // frame terminator and not an interior data byte. + this->finish_bucket_frame_(); + this->last_bridge_byte_ = now; + } + const bool receiving_bucket = this->rx_buffer_.size() >= 2 && this->rx_buffer_[1] == RF_CODE_RFIN_BUCKET; + if (receiving_bucket) { + // Never declare an in-progress bucket frame dead while its continuation + // bytes are already queued: a stalled loop() otherwise discards a live + // frame that the UART buffer proves is still arriving. + if (avail == 0 && now - this->last_bridge_byte_ > BUCKET_FRAME_TIMEOUT_MS) { + ESP_LOGD(TAG, "Discarding incomplete RFBridge Bucket frame (%u bytes)", + static_cast(this->rx_buffer_.size())); + this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; + this->last_bridge_byte_ = now; + } + } else if (now - this->last_bridge_byte_ > 50) { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; this->last_bridge_byte_ = now; } - size_t avail = this->available(); while (avail > 0) { uint8_t buf[64]; size_t to_read = std::min(avail, sizeof(buf)); @@ -146,12 +211,14 @@ void RFBridgeComponent::loop() { for (size_t i = 0; i < to_read; i++) { if (this->rx_buffer_.size() > MAX_RX_BUFFER_SIZE) { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; } if (this->parse_bridge_byte_(buf[i])) { ESP_LOGVV(TAG, "Parsed: 0x%02X", buf[i]); this->last_bridge_byte_ = now; } else { this->rx_buffer_.clear(); + this->bucket_frame_candidate_ = false; } } } diff --git a/esphome/components/rf_bridge/rf_bridge.h b/esphome/components/rf_bridge/rf_bridge.h index 5ad75650ab..cbb1880ec5 100644 --- a/esphome/components/rf_bridge/rf_bridge.h +++ b/esphome/components/rf_bridge/rf_bridge.h @@ -30,6 +30,17 @@ static const uint8_t RF_CODE_BEEP = 0xC0; static const uint8_t RF_CODE_STOP = 0x55; static const uint8_t RF_DEBOUNCE = 200; static const size_t MAX_RX_BUFFER_SIZE = 512; +// ~10 byte times at 19200 baud: long enough to prove the UART went quiet +// after a possible bucket-frame terminator, short enough to finish well +// before the next radio capture can be delivered. +static const uint32_t BUCKET_CANDIDATE_QUIET_MS = 5; +// Portisch drains a B1 frame's header, bucket table, and pulse data as +// separate UART writes, so an in-progress bucket frame tolerates a longer +// inter-region gap than the generic 50 ms inter-byte timeout. +static const uint32_t BUCKET_FRAME_TIMEOUT_MS = 250; +// Portisch's uart_put_RF_buckets sends at most 7 buckets plus the sync +// bucket, so a B1 count byte above 8 (or 0) is malformed for any protocol. +static const uint8_t B1_MAX_BUCKET_COUNT = 8; struct RFBridgeData { uint16_t sync; @@ -67,10 +78,12 @@ class RFBridgeComponent final : public uart::UARTDevice, public Component { void ack_(); void decode_(); bool parse_bridge_byte_(uint8_t byte); + void finish_bucket_frame_(); void write_byte_str_(const std::string &codes); std::vector rx_buffer_; uint32_t last_bridge_byte_{0}; + bool bucket_frame_candidate_{false}; CallbackManager data_callback_; CallbackManager advanced_data_callback_; From 28588310e74f93140e020fcaf6f95584412f671a Mon Sep 17 00:00:00 2001 From: raykholo Date: Tue, 8 Sep 2026 02:57:33 -0400 Subject: [PATCH 142/147] [anova] Re-assert temperature unit on every poll cycle (#17141) --- esphome/components/anova/anova.cpp | 107 ++++++++++++++--------------- esphome/components/anova/anova.h | 13 +++- 2 files changed, 62 insertions(+), 58 deletions(-) diff --git a/esphome/components/anova/anova.cpp b/esphome/components/anova/anova.cpp index 6e382872e2..b0769bb622 100644 --- a/esphome/components/anova/anova.cpp +++ b/esphome/components/anova/anova.cpp @@ -13,7 +13,7 @@ void Anova::dump_config() { LOG_CLIMATE("", "Anova BLE Cooker", this); } void Anova::setup() { this->codec_ = make_unique(); - this->current_request_ = 0; + this->poll_step_ = PollStep::IDLE; } void Anova::loop() { @@ -22,6 +22,15 @@ void Anova::loop() { this->disable_loop(); } +void Anova::write_request_(AnovaPacket *pkt) { + auto status = + esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, + pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); + if (status) { + ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); + } +} + void Anova::control(const ClimateCall &call) { auto mode_val = call.get_mode(); if (mode_val.has_value()) { @@ -38,22 +47,11 @@ void Anova::control(const ClimateCall &call) { ESP_LOGW(TAG, "Unsupported mode: %d", mode); return; } - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } + this->write_request_(pkt); } auto target_temp = call.get_target_temperature(); if (target_temp.has_value()) { - auto *pkt = this->codec_->get_set_target_temp_request(*target_temp); - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } + this->write_request_(this->codec_->get_set_target_temp_request(*target_temp)); } } @@ -62,6 +60,7 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ case ESP_GATTC_DISCONNECT_EVT: { this->current_temperature = NAN; this->target_temperature = NAN; + this->poll_step_ = PollStep::IDLE; this->publish_state(); break; } @@ -83,8 +82,8 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ } case ESP_GATTC_REG_FOR_NOTIFY_EVT: { this->node_state = espbt::ClientState::ESTABLISHED; - this->current_request_ = 0; - this->update(); + this->poll_step_ = PollStep::IDLE; + this->update(); // begin the first poll cycle immediately break; } case ESP_GATTC_NOTIFY_EVT: { @@ -101,33 +100,30 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ this->mode = this->codec_->running_ ? climate::CLIMATE_MODE_HEAT : climate::CLIMATE_MODE_OFF; } if (this->codec_->has_unit()) { - this->fahrenheit_ = (this->codec_->unit_ == 'f'); - ESP_LOGD(TAG, "Anova units is %s", this->fahrenheit_ ? "fahrenheit" : "celsius"); - this->current_request_++; + ESP_LOGD(TAG, "Anova units is %s", (this->codec_->unit_ == 'f') ? "fahrenheit" : "celsius"); } this->publish_state(); - if (this->current_request_ > 1) { - AnovaPacket *pkt = nullptr; - switch (this->current_request_++) { - case 2: - pkt = this->codec_->get_read_target_temp_request(); - break; - case 3: - pkt = this->codec_->get_read_current_temp_request(); - break; - default: - this->current_request_ = 1; - break; - } - if (pkt != nullptr) { - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } - } + // Advance the poll cycle to its next request based on the reply we got. + switch (this->poll_step_) { + case PollStep::SET_UNIT: + this->poll_step_ = PollStep::STATUS; + this->write_request_(this->codec_->get_read_device_status_request()); + break; + case PollStep::STATUS: + this->poll_step_ = PollStep::TARGET; + this->write_request_(this->codec_->get_read_target_temp_request()); + break; + case PollStep::TARGET: + this->poll_step_ = PollStep::CURRENT; + this->write_request_(this->codec_->get_read_current_temp_request()); + break; + case PollStep::CURRENT: + this->poll_step_ = PollStep::IDLE; // full cycle complete + break; + default: + // A reply to an ad-hoc control() write, outside a managed cycle. + break; } break; } @@ -136,27 +132,26 @@ void Anova::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t gattc_ } } -void Anova::set_unit_of_measurement(const char *unit) { this->fahrenheit_ = !strncmp(unit, "f", 1); } +void Anova::set_unit_of_measurement(const char *unit) { this->want_fahrenheit_ = !strncmp(unit, "f", 1); } void Anova::update() { if (this->node_state != espbt::ClientState::ESTABLISHED) return; - - if (this->current_request_ < 2) { - AnovaPacket *pkt; - if (this->current_request_ == 0) { - pkt = this->codec_->get_set_unit_request(this->fahrenheit_ ? 'f' : 'c'); - } else { - pkt = this->codec_->get_read_device_status_request(); - } - auto status = - esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->char_handle_, - pkt->length, pkt->data, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) { - ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); - } - this->current_request_++; + if (this->poll_step_ != PollStep::IDLE) { + // The previous cycle never finished within a full polling interval -- a + // reply was missed or a write failed. Restart the cycle rather than stall; + // the polling interval itself acts as the timeout. A late reply from the + // abandoned cycle is harmless: state decoding happens on every notify + // regardless of step, and each notify sends at most one follow-up request. + ESP_LOGW(TAG, "[%s] Poll cycle incomplete (step %u); restarting cycle", this->parent_->address_str(), + static_cast(this->poll_step_)); } + // Re-assert the configured unit at the start of every poll cycle, then fall + // through the status/temperature reads via the notification handler. Always + // command the configured unit (want_fahrenheit_) -- never the last value the + // device reported, or a drift to 'c' would lock itself in. + this->poll_step_ = PollStep::SET_UNIT; + this->write_request_(this->codec_->get_set_unit_request(this->want_fahrenheit_ ? 'f' : 'c')); } } // namespace esphome::anova diff --git a/esphome/components/anova/anova.h b/esphome/components/anova/anova.h index 49b1100c37..a0fa03df01 100644 --- a/esphome/components/anova/anova.h +++ b/esphome/components/anova/anova.h @@ -37,11 +37,20 @@ class Anova final : public climate::Climate, public esphome::ble_client::BLEClie void set_unit_of_measurement(const char *unit); protected: + // A poll cycle re-asserts the configured unit, then reads device state. + // Re-asserting every cycle prevents the cooker from silently reverting to + // its default (Celsius); previously the unit was only set once on + // connection, so a drift persisted (and corrupted the F/C interpretation of + // subsequent readings) until the BLE link was re-established. + enum class PollStep : uint8_t { SET_UNIT, STATUS, TARGET, CURRENT, IDLE }; + + void write_request_(AnovaPacket *pkt); + std::unique_ptr codec_; void control(const climate::ClimateCall &call) override; uint16_t char_handle_; - uint8_t current_request_; - bool fahrenheit_; + bool want_fahrenheit_{true}; // configured target unit; never overwritten by device replies + PollStep poll_step_{PollStep::IDLE}; }; } // namespace esphome::anova From 1700a40b7cc0cb4a033ba7f2d7c27c1ba65ea0f7 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:59:55 +1200 Subject: [PATCH 143/147] [core] Restore the shared git hooks after post-checkout runs script/setup (#19036) --- script/git-hooks/post-checkout | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/script/git-hooks/post-checkout b/script/git-hooks/post-checkout index 8f4085ae6e..853c2b0352 100755 --- a/script/git-hooks/post-checkout +++ b/script/git-hooks/post-checkout @@ -14,7 +14,31 @@ top=$(git rev-parse --show-toplevel 2>/dev/null) || exit 0 [ -x "$top/venv/bin/python" ] && exit 0 [ -x "$top/script/setup" ] || exit 0 +# Every worktree shares the hooks directory of the checkout it was created +# from, and the script/setup run below is the one from whichever branch was just +# checked out. Older branches install their own pre-commit hook without checking +# for a worktree: that moves the shared hook aside as pre-commit.legacy and +# replaces it with one tied to this worktree's virtual environment, so commits +# break in every checkout. To rule that out, the hooks directory is copied +# before script/setup runs and put back exactly as it was afterwards, including +# removing any file script/setup added. +hooks=$(git rev-parse --path-format=absolute --git-path hooks 2>/dev/null) || exit 0 +snap=$(mktemp -d "$hooks/.post-checkout.XXXXXX") || exit 0 +cp -p "$hooks"/* "$snap"/ 2>/dev/null + # Clear VIRTUAL_ENV so a checkout made from a shell with an environment already # activated still gets its own, rather than having the active one repointed at # this working tree. -exec env -u VIRTUAL_ENV "$top/script/setup" +env -u VIRTUAL_ENV "$top/script/setup" +status=$? + +for f in "$hooks"/*; do + [ -e "$snap/${f##*/}" ] || rm -f "$f" +done +# Files are moved rather than copied so a hook that is still running, such as +# this one, is swapped out atomically instead of being rewritten in place. +for f in "$snap"/*; do + cmp -s "$f" "$hooks/${f##*/}" 2>/dev/null || mv -f "$f" "$hooks/${f##*/}" +done +rm -rf "$snap" +exit $status From 227ca90aad10d3084e3e8a45a13c46aea65a6c3f Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:00:08 +1200 Subject: [PATCH 144/147] [core] Restore the shared git hooks after post-checkout runs script/setup (#19036) From f191d5e0c384ca785e562e652cc03cdaf21a99d4 Mon Sep 17 00:00:00 2001 From: John <34163498+CircuitSetup@users.noreply.github.com> Date: Tue, 8 Sep 2026 04:02:49 -0400 Subject: [PATCH 145/147] [atm90e32] Verify offset calibration writes (#18701) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/atm90e32/atm90e32.cpp | 360 ++++++++++-------- esphome/components/atm90e32/atm90e32.h | 71 ++-- tests/components/atm90e32/__init__.py | 5 + .../offset_register_verification_test.cpp | 62 +++ 4 files changed, 322 insertions(+), 176 deletions(-) create mode 100644 tests/components/atm90e32/__init__.py create mode 100644 tests/components/atm90e32/offset_register_verification_test.cpp diff --git a/esphome/components/atm90e32/atm90e32.cpp b/esphome/components/atm90e32/atm90e32.cpp index d948b3741d..23701e7834 100644 --- a/esphome/components/atm90e32/atm90e32.cpp +++ b/esphome/components/atm90e32/atm90e32.cpp @@ -9,6 +9,10 @@ namespace esphome::atm90e32 { static const char *const TAG = "atm90e32"; +static const LogString *offset_calibration_name(bool power_offsets) { + return power_offsets ? LOG_STR("Power offset") : LOG_STR("Offset"); +} + static uint32_t pref_hash(const char *prefix, const char *name_space) { auto hash = fnv1_hash(prefix); return fnv1_hash_extend(hash, name_space); @@ -203,13 +207,12 @@ void ATM90E32Component::setup() { // Initialize flash storage for power offset calibrations uint32_t po_hash = pref_hash("_power_offset_calibration_", cs); - this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); + this->power_offset_pref_ = global_preferences->make_preference(po_hash, true); bool migrated_power_offset = false; if (has_distinct_legacy_namespace) { uint32_t legacy_po_hash = pref_hash("_power_offset_calibration_", legacy_cs); - auto legacy_power_offset_pref = - global_preferences->make_preference(legacy_po_hash, true); - PowerOffsetCalibration power_offset_data[3]{}; + auto legacy_power_offset_pref = global_preferences->make_preference(legacy_po_hash, true); + OffsetCalibration power_offset_data[3]{}; int migration_status = migrate_legacy_pref_if_needed(this->power_offset_pref_, legacy_power_offset_pref, &power_offset_data); migrated_power_offset = migration_status > 0; @@ -224,20 +227,20 @@ void ATM90E32Component::setup() { global_preferences->sync(); } - this->restore_offset_calibrations_(); - this->restore_power_offset_calibrations_(); + this->restore_offset_calibrations_(OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); + this->restore_offset_calibrations_(OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); } else { ESP_LOGI(TAG, "[CALIBRATION][%s] Power & Voltage/Current offset calibration is disabled. Using config file values.", cs); for (uint8_t phase = 0; phase < 3; ++phase) { this->write16_(this->voltage_offset_registers[phase], - static_cast(this->offset_phase_[phase].voltage_offset_)); + static_cast(this->offset_phase_[phase].first_offset)); this->write16_(this->current_offset_registers[phase], - static_cast(this->offset_phase_[phase].current_offset_)); + static_cast(this->offset_phase_[phase].second_offset)); this->write16_(this->power_offset_registers[phase], - static_cast(this->power_offset_phase_[phase].active_power_offset)); + static_cast(this->power_offset_phase_[phase].first_offset)); this->write16_(this->reactive_power_offset_registers[phase], - static_cast(this->power_offset_phase_[phase].reactive_power_offset)); + static_cast(this->power_offset_phase_[phase].second_offset)); } } @@ -317,8 +320,8 @@ void ATM90E32Component::log_calibration_status_() { cs); for (uint8_t phase = 0; phase < 3; ++phase) { ESP_LOGW(TAG, "[CALIBRATION][%s] | %c | %6d | %6d | %6d | %6d |", cs, 'A' + phase, - this->config_offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].voltage_offset_, - this->config_offset_phase_[phase].current_offset_, this->offset_phase_[phase].current_offset_); + this->config_offset_phase_[phase].first_offset, this->offset_phase_[phase].first_offset, + this->config_offset_phase_[phase].second_offset, this->offset_phase_[phase].second_offset); } ESP_LOGW(TAG, "[CALIBRATION][%s] ===============================================================================", cs); @@ -335,10 +338,8 @@ void ATM90E32Component::log_calibration_status_() { cs); for (uint8_t phase = 0; phase < 3; ++phase) { ESP_LOGW(TAG, "[CALIBRATION][%s] | %c | %6d | %6d | %6d | %6d |", cs, 'A' + phase, - this->config_power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].active_power_offset, - this->config_power_offset_phase_[phase].reactive_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->config_power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].first_offset, + this->config_power_offset_phase_[phase].second_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGW(TAG, "[CALIBRATION][%s] ===============================================================================", cs); @@ -372,7 +373,7 @@ void ATM90E32Component::log_calibration_status_() { ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].current_offset_); + this->offset_phase_[phase].first_offset, this->offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] ==============================================================\\n", cs); } @@ -385,8 +386,7 @@ void ATM90E32Component::log_calibration_status_() { ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); } @@ -756,36 +756,68 @@ void ATM90E32Component::save_gain_calibration_to_memory_() { } } -void ATM90E32Component::save_offset_calibration_to_memory_() { +void ATM90E32Component::finish_offset_calibration_(const OffsetCalibration (&previous)[3], bool previous_restored, + bool previous_using_saved, OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; const char *cs = this->get_calibration_id_(); - bool success = this->offset_pref_.save(&this->offset_phase_); - global_preferences->sync(); - if (success) { - this->using_saved_calibrations_ = true; - this->restored_offset_calibration_ = true; - for (bool &phase : this->offset_calibration_mismatch_) - phase = false; - ESP_LOGI(TAG, "[CALIBRATION][%s] Offset calibration saved to memory.", cs); - } else { - this->using_saved_calibrations_ = false; - ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save offset calibration to memory!", cs); - } -} + const LogString *name = offset_calibration_name(power_offsets); + OffsetCalibration(*offsets)[3] = power_offsets ? &this->power_offset_phase_ : &this->offset_phase_; + ESPPreferenceObject *preference = power_offsets ? &this->power_offset_pref_ : &this->offset_pref_; + bool *has_stored = + power_offsets ? &this->has_stored_power_offset_calibration_ : &this->has_stored_offset_calibration_; + bool *restored = power_offsets ? &this->restored_power_offset_calibration_ : &this->restored_offset_calibration_; + bool *mismatches = power_offsets ? this->power_offset_calibration_mismatch_ : this->offset_calibration_mismatch_; -void ATM90E32Component::save_power_offset_calibration_to_memory_() { - const char *cs = this->get_calibration_id_(); - bool success = this->power_offset_pref_.save(&this->power_offset_phase_); - global_preferences->sync(); - if (success) { - this->using_saved_calibrations_ = true; - this->restored_power_offset_calibration_ = true; - for (bool &phase : this->power_offset_calibration_mismatch_) - phase = false; - ESP_LOGI(TAG, "[CALIBRATION][%s] Power offset calibration saved to memory.", cs); - } else { - this->using_saved_calibrations_ = false; - ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save power offset calibration to memory!", cs); + const bool writes_verified = this->verify_offset_writes_(type); + bool saved = false; + bool synced = false; + if (writes_verified) { + saved = preference->save(offsets); + synced = global_preferences->sync(); } + + if (writes_verified && saved && synced) { + this->using_saved_calibrations_ = true; + *has_stored = true; + *restored = true; + for (uint8_t phase = 0; phase < 3; phase++) + mismatches[phase] = false; + ESP_LOGI(TAG, "[CALIBRATION][%s] %s calibration saved to memory. %s calibration completed and verified.", cs, + LOG_STR_ARG(name), LOG_STR_ARG(name)); + return; + } + + if (writes_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to save %s calibration to memory!", cs, LOG_STR_ARG(name)); + } + + for (uint8_t phase = 0; phase < 3; phase++) { + this->write_offsets_to_registers_(phase, previous[phase].first_offset, previous[phase].second_offset, type); + } + const bool rollback_verified = this->verify_offset_writes_(type); + + bool rollback_persisted = false; + if (writes_verified) { + OffsetCalibration rollback[3]{}; + prepare_offset_rollback(previous, previous_restored, rollback); + const bool rollback_saved = preference->save(&rollback); + const bool rollback_synced = global_preferences->sync(); + rollback_persisted = rollback_saved && rollback_synced; + if (!rollback_saved || !rollback_synced) { + ESP_LOGE(TAG, "[CALIBRATION][%s] Failed to persist restored %s calibration values!", cs, LOG_STR_ARG(name)); + } + } + + *restored = previous_restored; + if (rollback_persisted) + *has_stored = previous_restored; + this->using_saved_calibrations_ = previous_using_saved; + if (!rollback_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration failed; rollback readback verification failed.", cs, + LOG_STR_ARG(name)); + return; + } + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration failed; previous values restored.", cs, LOG_STR_ARG(name)); } void ATM90E32Component::run_offset_calibrations() { @@ -803,11 +835,16 @@ void ATM90E32Component::run_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_voltage | offset_current |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ------------------------------------------------------------------", cs); + OffsetCalibration previous_offsets[3] = {this->offset_phase_[0], this->offset_phase_[1], this->offset_phase_[2]}; + const bool previous_restored = this->restored_offset_calibration_; + const bool previous_using_saved = this->using_saved_calibrations_; + for (uint8_t phase = 0; phase < 3; phase++) { int16_t voltage_offset = calibrate_offset(phase, true); int16_t current_offset = calibrate_offset(phase, false); - this->write_offsets_to_registers_(phase, voltage_offset, current_offset); + this->write_offsets_to_registers_(phase, voltage_offset, current_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, voltage_offset, current_offset); @@ -815,7 +852,8 @@ void ATM90E32Component::run_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] ==================================================================\n", cs); - this->save_offset_calibration_to_memory_(); + this->finish_offset_calibration_(previous_offsets, previous_restored, previous_using_saved, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); } void ATM90E32Component::run_power_offset_calibrations() { @@ -834,18 +872,25 @@ void ATM90E32Component::run_power_offset_calibrations() { ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_active_power | offset_reactive_power |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); + OffsetCalibration previous_offsets[3] = {this->power_offset_phase_[0], this->power_offset_phase_[1], + this->power_offset_phase_[2]}; + const bool previous_restored = this->restored_power_offset_calibration_; + const bool previous_using_saved = this->using_saved_calibrations_; + for (uint8_t phase = 0; phase < 3; ++phase) { int16_t active_offset = calibrate_power_offset(phase, false); int16_t reactive_offset = calibrate_power_offset(phase, true); - this->write_power_offsets_to_registers_(phase, active_offset, reactive_offset); + this->write_offsets_to_registers_(phase, active_offset, reactive_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, active_offset, reactive_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); - this->save_power_offset_calibration_to_memory_(); + this->finish_offset_calibration_(previous_offsets, previous_restored, previous_using_saved, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); } void ATM90E32Component::write_gains_to_registers_() { @@ -859,35 +904,26 @@ void ATM90E32Component::write_gains_to_registers_() { this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); } -void ATM90E32Component::write_offsets_to_registers_(uint8_t phase, int16_t voltage_offset, int16_t current_offset) { - // Save to runtime - this->offset_phase_[phase].voltage_offset_ = voltage_offset; - this->phase_[phase].voltage_offset_ = voltage_offset; +void ATM90E32Component::write_offsets_to_registers_(uint8_t phase, int16_t first_offset, int16_t second_offset, + OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; + OffsetCalibration &offsets = power_offsets ? this->power_offset_phase_[phase] : this->offset_phase_[phase]; + offsets.first_offset = first_offset; + offsets.second_offset = second_offset; + if (power_offsets) { + this->phase_[phase].active_power_offset_ = first_offset; + this->phase_[phase].reactive_power_offset_ = second_offset; + } else { + this->phase_[phase].voltage_offset_ = first_offset; + this->phase_[phase].current_offset_ = second_offset; + } - // Save to flash-storable struct - this->offset_phase_[phase].current_offset_ = current_offset; - this->phase_[phase].current_offset_ = current_offset; - - // Write to registers + const uint16_t *first_registers = power_offsets ? this->power_offset_registers : this->voltage_offset_registers; + const uint16_t *second_registers = + power_offsets ? this->reactive_power_offset_registers : this->current_offset_registers; this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x55AA); - this->write16_(voltage_offset_registers[phase], static_cast(voltage_offset)); - this->write16_(current_offset_registers[phase], static_cast(current_offset)); - this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); -} - -void ATM90E32Component::write_power_offsets_to_registers_(uint8_t phase, int16_t p_offset, int16_t q_offset) { - // Save to runtime - this->phase_[phase].active_power_offset_ = p_offset; - this->phase_[phase].reactive_power_offset_ = q_offset; - - // Save to flash-storable struct - this->power_offset_phase_[phase].active_power_offset = p_offset; - this->power_offset_phase_[phase].reactive_power_offset = q_offset; - - // Write to registers - this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x55AA); - this->write16_(this->power_offset_registers[phase], static_cast(p_offset)); - this->write16_(this->reactive_power_offset_registers[phase], static_cast(q_offset)); + this->write16_(first_registers[phase], static_cast(first_offset)); + this->write16_(second_registers[phase], static_cast(second_offset)); this->write16_(ATM90E32_REGISTER_CFGREGACCEN, 0x0000); } @@ -947,89 +983,78 @@ void ATM90E32Component::restore_gain_calibrations_() { ESP_LOGW(TAG, "[CALIBRATION][%s] No stored gain calibrations found. Using config file values.", cs); } -void ATM90E32Component::restore_offset_calibrations_() { +void ATM90E32Component::restore_offset_calibrations_(OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; const char *cs = this->get_calibration_id_(); + const LogString *name = power_offsets ? LOG_STR("power offset") : LOG_STR("offset"); + OffsetCalibration(*offsets)[3] = power_offsets ? &this->power_offset_phase_ : &this->offset_phase_; + OffsetCalibration(*config_offsets)[3] = + power_offsets ? &this->config_power_offset_phase_ : &this->config_offset_phase_; + ESPPreferenceObject *preference = power_offsets ? &this->power_offset_pref_ : &this->offset_pref_; + bool *has_stored = + power_offsets ? &this->has_stored_power_offset_calibration_ : &this->has_stored_offset_calibration_; + bool *restored = power_offsets ? &this->restored_power_offset_calibration_ : &this->restored_offset_calibration_; + bool *mismatches = power_offsets ? this->power_offset_calibration_mismatch_ : this->offset_calibration_mismatch_; + const bool *has_first = power_offsets ? this->has_config_active_power_offset_ : this->has_config_voltage_offset_; + const bool *has_second = power_offsets ? this->has_config_reactive_power_offset_ : this->has_config_current_offset_; + for (uint8_t i = 0; i < 3; ++i) - this->config_offset_phase_[i] = this->offset_phase_[i]; - - bool have_data = this->offset_pref_.load(&this->offset_phase_); + (*config_offsets)[i] = (*offsets)[i]; + const bool have_data = preference->load(offsets); bool all_zero = true; if (have_data) { - for (auto &phase : this->offset_phase_) { - if (phase.voltage_offset_ != 0 || phase.current_offset_ != 0) { + for (const auto &phase : *offsets) { + if (phase.first_offset != 0 || phase.second_offset != 0) { all_zero = false; break; } } } - if (have_data && !all_zero) { - this->restored_offset_calibration_ = true; - for (uint8_t phase = 0; phase < 3; phase++) { - auto &offset = this->offset_phase_[phase]; - bool mismatch = false; - if (this->has_config_voltage_offset_[phase] && - offset.voltage_offset_ != this->config_offset_phase_[phase].voltage_offset_) - mismatch = true; - if (this->has_config_current_offset_[phase] && - offset.current_offset_ != this->config_offset_phase_[phase].current_offset_) - mismatch = true; - if (mismatch) - this->offset_calibration_mismatch_[phase] = true; + *has_stored = have_data && !all_zero; + *restored = false; + for (uint8_t phase = 0; phase < 3; phase++) { + mismatches[phase] = false; + if (*has_stored) { + mismatches[phase] = + (has_first[phase] && (*offsets)[phase].first_offset != (*config_offsets)[phase].first_offset) || + (has_second[phase] && (*offsets)[phase].second_offset != (*config_offsets)[phase].second_offset); } - } else { + } + + if (!*has_stored) { for (uint8_t phase = 0; phase < 3; phase++) - this->offset_phase_[phase] = this->config_offset_phase_[phase]; - ESP_LOGW(TAG, "[CALIBRATION][%s] No stored offset calibrations found. Using default values.", cs); + (*offsets)[phase] = (*config_offsets)[phase]; + ESP_LOGW(TAG, "[CALIBRATION][%s] No stored %s calibrations found. Using default values.", cs, LOG_STR_ARG(name)); } for (uint8_t phase = 0; phase < 3; phase++) { - write_offsets_to_registers_(phase, this->offset_phase_[phase].voltage_offset_, - this->offset_phase_[phase].current_offset_); + this->write_offsets_to_registers_(phase, (*offsets)[phase].first_offset, (*offsets)[phase].second_offset, type); } -} - -void ATM90E32Component::restore_power_offset_calibrations_() { - const char *cs = this->get_calibration_id_(); - for (uint8_t i = 0; i < 3; ++i) - this->config_power_offset_phase_[i] = this->power_offset_phase_[i]; - - bool have_data = this->power_offset_pref_.load(&this->power_offset_phase_); - - bool all_zero = true; - if (have_data) { - for (auto &phase : this->power_offset_phase_) { - if (phase.active_power_offset != 0 || phase.reactive_power_offset != 0) { - all_zero = false; - break; - } - } + const bool initial_values_verified = this->verify_offset_writes_(type); + if (initial_values_verified) { + const auto state = resolve_offset_restore_state(*has_stored, true, false); + *restored = state.restored; + ESP_LOGI(TAG, "[CALIBRATION][%s] %s calibration values verified.", cs, LOG_STR_ARG(name)); + return; } - if (have_data && !all_zero) { - this->restored_power_offset_calibration_ = true; - for (uint8_t phase = 0; phase < 3; ++phase) { - auto &offset = this->power_offset_phase_[phase]; - bool mismatch = false; - if (this->has_config_active_power_offset_[phase] && - offset.active_power_offset != this->config_power_offset_phase_[phase].active_power_offset) - mismatch = true; - if (this->has_config_reactive_power_offset_[phase] && - offset.reactive_power_offset != this->config_power_offset_phase_[phase].reactive_power_offset) - mismatch = true; - if (mismatch) - this->power_offset_calibration_mismatch_[phase] = true; - } + this->using_saved_calibrations_ = false; + for (uint8_t phase = 0; phase < 3; phase++) + mismatches[phase] = false; + for (uint8_t phase = 0; phase < 3; phase++) { + (*offsets)[phase] = (*config_offsets)[phase]; + this->write_offsets_to_registers_(phase, (*offsets)[phase].first_offset, (*offsets)[phase].second_offset, type); + } + const auto state = resolve_offset_restore_state(*has_stored, false, this->verify_offset_writes_(type)); + *restored = state.restored; + if (state.values_verified) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration restore failed verification; config values verified.", cs, + LOG_STR_ARG(name)); } else { - for (uint8_t phase = 0; phase < 3; ++phase) - this->power_offset_phase_[phase] = this->config_power_offset_phase_[phase]; - ESP_LOGW(TAG, "[CALIBRATION][%s] No stored power offsets found. Using default values.", cs); - } - - for (uint8_t phase = 0; phase < 3; ++phase) { - write_power_offsets_to_registers_(phase, this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + ESP_LOGE(TAG, "[CALIBRATION][%s] %s calibration restore and config fallback both failed verification.", cs, + LOG_STR_ARG(name)); } } @@ -1084,14 +1109,14 @@ void ATM90E32Component::clear_gain_calibrations() { void ATM90E32Component::clear_offset_calibrations() { const char *cs = this->get_calibration_id_(); - if (!this->restored_offset_calibration_) { + if (!this->has_stored_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored offset calibrations to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_voltage | offset_current |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] --------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->offset_phase_[phase].voltage_offset_, this->offset_phase_[phase].current_offset_); + this->offset_phase_[phase].first_offset, this->offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] ==============================================================\n", cs); return; @@ -1104,10 +1129,11 @@ void ATM90E32Component::clear_offset_calibrations() { for (uint8_t phase = 0; phase < 3; phase++) { int16_t voltage_offset = - this->has_config_voltage_offset_[phase] ? this->config_offset_phase_[phase].voltage_offset_ : 0; + this->has_config_voltage_offset_[phase] ? this->config_offset_phase_[phase].first_offset : 0; int16_t current_offset = - this->has_config_current_offset_[phase] ? this->config_offset_phase_[phase].current_offset_ : 0; - this->write_offsets_to_registers_(phase, voltage_offset, current_offset); + this->has_config_current_offset_[phase] ? this->config_offset_phase_[phase].second_offset : 0; + this->write_offsets_to_registers_(phase, voltage_offset, current_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, voltage_offset, current_offset); } @@ -1117,6 +1143,7 @@ void ATM90E32Component::clear_offset_calibrations() { this->offset_pref_.save(&zero_offsets); // Clear stored values in flash global_preferences->sync(); + this->has_stored_offset_calibration_ = false; this->restored_offset_calibration_ = false; for (bool &phase : this->offset_calibration_mismatch_) phase = false; @@ -1126,15 +1153,14 @@ void ATM90E32Component::clear_offset_calibrations() { void ATM90E32Component::clear_power_offset_calibrations() { const char *cs = this->get_calibration_id_(); - if (!this->restored_power_offset_calibration_) { + if (!this->has_stored_power_offset_calibration_) { ESP_LOGI(TAG, "[CALIBRATION][%s] No stored power offsets to clear. Current values:", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] | Phase | offset_active_power | offset_reactive_power |", cs); ESP_LOGI(TAG, "[CALIBRATION][%s] ---------------------------------------------------------------------", cs); for (uint8_t phase = 0; phase < 3; phase++) { ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, - this->power_offset_phase_[phase].active_power_offset, - this->power_offset_phase_[phase].reactive_power_offset); + this->power_offset_phase_[phase].first_offset, this->power_offset_phase_[phase].second_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); return; @@ -1147,20 +1173,21 @@ void ATM90E32Component::clear_power_offset_calibrations() { for (uint8_t phase = 0; phase < 3; phase++) { int16_t active_offset = - this->has_config_active_power_offset_[phase] ? this->config_power_offset_phase_[phase].active_power_offset : 0; - int16_t reactive_offset = this->has_config_reactive_power_offset_[phase] - ? this->config_power_offset_phase_[phase].reactive_power_offset - : 0; - this->write_power_offsets_to_registers_(phase, active_offset, reactive_offset); + this->has_config_active_power_offset_[phase] ? this->config_power_offset_phase_[phase].first_offset : 0; + int16_t reactive_offset = + this->has_config_reactive_power_offset_[phase] ? this->config_power_offset_phase_[phase].second_offset : 0; + this->write_offsets_to_registers_(phase, active_offset, reactive_offset, + OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER); ESP_LOGI(TAG, "[CALIBRATION][%s] | %c | %6d | %6d |", cs, 'A' + phase, active_offset, reactive_offset); } ESP_LOGI(TAG, "[CALIBRATION][%s] =====================================================================\n", cs); - PowerOffsetCalibration zero_power_offsets[3]{{0, 0}, {0, 0}, {0, 0}}; + OffsetCalibration zero_power_offsets[3]{{0, 0}, {0, 0}, {0, 0}}; this->power_offset_pref_.save(&zero_power_offsets); global_preferences->sync(); + this->has_stored_power_offset_calibration_ = false; this->restored_power_offset_calibration_ = false; for (bool &phase : this->power_offset_calibration_mismatch_) phase = false; @@ -1215,6 +1242,31 @@ bool ATM90E32Component::verify_gain_writes_() { return success; // Return true if all writes were successful, false otherwise } +bool ATM90E32Component::verify_offset_writes_(OffsetCalibrationType type) { + const bool power_offsets = type == OffsetCalibrationType::OFFSET_CALIBRATION_TYPE_POWER; + const char *cs = this->get_calibration_id_(); + const LogString *name = offset_calibration_name(power_offsets); + const LogString *first_name = power_offsets ? LOG_STR("active") : LOG_STR("voltage"); + const LogString *second_name = power_offsets ? LOG_STR("reactive") : LOG_STR("current"); + const OffsetCalibration *offsets = power_offsets ? this->power_offset_phase_ : this->offset_phase_; + const uint16_t *first_registers = power_offsets ? this->power_offset_registers : this->voltage_offset_registers; + const uint16_t *second_registers = + power_offsets ? this->reactive_power_offset_registers : this->current_offset_registers; + bool success = true; + for (uint8_t phase = 0; phase < 3; phase++) { + const uint16_t first = this->read16_(first_registers[phase]); + const uint16_t second = this->read16_(second_registers[phase]); + if (!offset_register_value_matches(first, offsets[phase].first_offset) || + !offset_register_value_matches(second, offsets[phase].second_offset)) { + ESP_LOGE(TAG, "[CALIBRATION][%s] %s readback failed for Phase %s: %s %d/%d, %s %d/%d.", cs, LOG_STR_ARG(name), + phase_labels[phase], LOG_STR_ARG(first_name), static_cast(first), offsets[phase].first_offset, + LOG_STR_ARG(second_name), static_cast(second), offsets[phase].second_offset); + success = false; + } + } + return success; +} + #ifdef USE_TEXT_SENSOR void ATM90E32Component::check_phase_status() { uint16_t state0 = this->read16_(ATM90E32_REGISTER_EMMSTATE0); diff --git a/esphome/components/atm90e32/atm90e32.h b/esphome/components/atm90e32/atm90e32.h index c636e5065a..fe7d903962 100644 --- a/esphome/components/atm90e32/atm90e32.h +++ b/esphome/components/atm90e32/atm90e32.h @@ -13,6 +13,40 @@ namespace esphome::atm90e32 { +inline bool offset_register_value_matches(uint16_t actual, int16_t expected) { + return actual == static_cast(expected); +} + +struct OffsetCalibration { + int16_t first_offset{0}; + int16_t second_offset{0}; +}; + +static_assert(sizeof(OffsetCalibration[3]) == 12, "Offset calibration preference layout must remain compatible"); + +enum class OffsetCalibrationType : uint8_t { + OFFSET_CALIBRATION_TYPE_VOLTAGE_CURRENT, + OFFSET_CALIBRATION_TYPE_POWER, +}; + +struct OffsetRestoreState { + bool restored; + bool values_verified; +}; + +inline OffsetRestoreState resolve_offset_restore_state(bool has_stored_values, bool initial_values_verified, + bool fallback_values_verified) { + if (initial_values_verified) + return {has_stored_values, true}; + return {false, fallback_values_verified}; +} + +inline void prepare_offset_rollback(const OffsetCalibration (&previous)[3], bool had_stored_values, + OffsetCalibration (&rollback)[3]) { + for (uint8_t phase = 0; phase < 3; phase++) + rollback[phase] = had_stored_values ? previous[phase] : OffsetCalibration{}; +} + class ATM90E32Component final : public PollingComponent, public spi::SPIDevice { @@ -71,19 +105,19 @@ class ATM90E32Component final : public PollingComponent, this->has_config_current_gain_[phase] = true; } void set_voltage_offset(uint8_t phase, int16_t offset) { - this->offset_phase_[phase].voltage_offset_ = offset; + this->offset_phase_[phase].first_offset = offset; this->has_config_voltage_offset_[phase] = true; } void set_current_offset(uint8_t phase, int16_t offset) { - this->offset_phase_[phase].current_offset_ = offset; + this->offset_phase_[phase].second_offset = offset; this->has_config_current_offset_[phase] = true; } void set_active_power_offset(uint8_t phase, int16_t offset) { - this->power_offset_phase_[phase].active_power_offset = offset; + this->power_offset_phase_[phase].first_offset = offset; this->has_config_active_power_offset_[phase] = true; } void set_reactive_power_offset(uint8_t phase, int16_t offset) { - this->power_offset_phase_[phase].reactive_power_offset = offset; + this->power_offset_phase_[phase].second_offset = offset; this->has_config_reactive_power_offset_[phase] = true; } void set_freq_sensor(sensor::Sensor *freq_sensor) { freq_sensor_ = freq_sensor; } @@ -171,16 +205,16 @@ class ATM90E32Component final : public PollingComponent, float get_chip_temperature_(); bool get_publish_interval_flag_() { return publish_interval_flag_; }; void set_publish_interval_flag_(bool flag) { publish_interval_flag_ = flag; }; - void restore_offset_calibrations_(); - void restore_power_offset_calibrations_(); + void restore_offset_calibrations_(OffsetCalibrationType type); void restore_gain_calibrations_(); - void save_offset_calibration_to_memory_(); void save_gain_calibration_to_memory_(); - void save_power_offset_calibration_to_memory_(); - void write_offsets_to_registers_(uint8_t phase, int16_t voltage_offset, int16_t current_offset); - void write_power_offsets_to_registers_(uint8_t phase, int16_t p_offset, int16_t q_offset); + void finish_offset_calibration_(const OffsetCalibration (&previous)[3], bool previous_restored, + bool previous_using_saved, OffsetCalibrationType type); + void write_offsets_to_registers_(uint8_t phase, int16_t first_offset, int16_t second_offset, + OffsetCalibrationType type); void write_gains_to_registers_(); bool verify_gain_writes_(); + bool verify_offset_writes_(OffsetCalibrationType type); bool validate_spi_read_(uint16_t expected, const char *context = nullptr); void log_calibration_status_(); const char *get_calibration_id_(); @@ -219,19 +253,10 @@ class ATM90E32Component final : public PollingComponent, uint32_t cumulative_reverse_active_energy_{0}; } phase_[3]; - struct OffsetCalibration { - int16_t voltage_offset_{0}; - int16_t current_offset_{0}; - } offset_phase_[3]; - + OffsetCalibration offset_phase_[3]; OffsetCalibration config_offset_phase_[3]; - - struct PowerOffsetCalibration { - int16_t active_power_offset{0}; - int16_t reactive_power_offset{0}; - } power_offset_phase_[3]; - - PowerOffsetCalibration config_power_offset_phase_[3]; + OffsetCalibration power_offset_phase_[3]; + OffsetCalibration config_power_offset_phase_[3]; struct GainCalibration { uint16_t voltage_gain{1}; @@ -265,6 +290,8 @@ class ATM90E32Component final : public PollingComponent, bool enable_offset_calibration_{false}; bool enable_gain_calibration_{false}; const char *instance_id_{nullptr}; + bool has_stored_offset_calibration_{false}; + bool has_stored_power_offset_calibration_{false}; bool restored_offset_calibration_{false}; bool restored_power_offset_calibration_{false}; bool restored_gain_calibration_{false}; diff --git a/tests/components/atm90e32/__init__.py b/tests/components/atm90e32/__init__.py new file mode 100644 index 0000000000..37d6797e2d --- /dev/null +++ b/tests/components/atm90e32/__init__.py @@ -0,0 +1,5 @@ +from tests.testing_helpers import ComponentManifestOverride + + +def override_manifest(manifest: ComponentManifestOverride) -> None: + manifest.dependencies = manifest.dependencies + ["sensor", "spi"] diff --git a/tests/components/atm90e32/offset_register_verification_test.cpp b/tests/components/atm90e32/offset_register_verification_test.cpp new file mode 100644 index 0000000000..3bb3eb76ea --- /dev/null +++ b/tests/components/atm90e32/offset_register_verification_test.cpp @@ -0,0 +1,62 @@ +#include + +#include "esphome/components/atm90e32/atm90e32.h" + +namespace esphome::atm90e32::testing { + +TEST(ATM90E32OffsetRegisterVerification, AcceptsExactSignedReadback) { + EXPECT_TRUE(offset_register_value_matches(0x007B, 123)); + EXPECT_TRUE(offset_register_value_matches(0xFF85, -123)); +} + +TEST(ATM90E32OffsetRegisterVerification, RejectsMismatchedReadback) { + EXPECT_FALSE(offset_register_value_matches(0x007C, 123)); + EXPECT_FALSE(offset_register_value_matches(0xFF84, -123)); +} + +TEST(ATM90E32OffsetRestoreState, ReportsVerifiedStoredValuesAsRestored) { + const auto state = resolve_offset_restore_state(true, true, false); + + EXPECT_TRUE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsVerifiedConfigFallbackAsNotRestored) { + const auto state = resolve_offset_restore_state(true, false, true); + + EXPECT_FALSE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsFailedConfigFallbackAsUnverified) { + const auto state = resolve_offset_restore_state(true, false, false); + + EXPECT_FALSE(state.restored); + EXPECT_FALSE(state.values_verified); +} + +TEST(ATM90E32OffsetRestoreState, ReportsConfigWithoutStoredValuesAsNotRestored) { + const auto state = resolve_offset_restore_state(false, true, false); + + EXPECT_FALSE(state.restored); + EXPECT_TRUE(state.values_verified); +} + +TEST(ATM90E32OffsetPersistence, RollsBackStoredValuesOrZeroSentinel) { + const OffsetCalibration previous[3]{{1, -1}, {2, -2}, {3, -3}}; + OffsetCalibration rollback[3]{}; + + prepare_offset_rollback(previous, true, rollback); + for (uint8_t phase = 0; phase < 3; phase++) { + EXPECT_EQ(rollback[phase].first_offset, previous[phase].first_offset); + EXPECT_EQ(rollback[phase].second_offset, previous[phase].second_offset); + } + + prepare_offset_rollback(previous, false, rollback); + for (const auto &phase : rollback) { + EXPECT_EQ(phase.first_offset, 0); + EXPECT_EQ(phase.second_offset, 0); + } +} + +} // namespace esphome::atm90e32::testing From f91486305ff20743d086651517d652360ab58ff7 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:21:48 +0200 Subject: [PATCH 146/147] Bump bundled esphome-device-builder to 1.14.5 (#19040) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index da76ab7b6a..ac84ee4689 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.14.4 +RUN uv pip install --no-cache-dir esphome-device-builder==1.14.5 RUN \ platformio settings set enable_telemetry No \ From 1ce0bed3f672d3a4699dad0cbfd8617c3b8950e5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 8 Sep 2026 18:10:45 +0200 Subject: [PATCH 147/147] [core] Share compiled binaries across modbus integration tests (#18945) --- script/helpers.py | 17 + tests/integration/README.md | 7 + tests/integration/conftest.py | 416 ++++++++++++++---- .../fixtures/sensor_filters_batch_window.yaml | 58 --- .../uart_mock_modbus_client_read_write.yaml | 111 ----- .../fixtures/uart_mock_modbus_custom_pdu.yaml | 88 ---- ...t_mock_modbus_deprecated_write_buffer.yaml | 106 ----- .../uart_mock_modbus_lambda_invert.yaml | 95 ---- .../uart_mock_modbus_lambda_write.yaml | 97 ---- .../fixtures/uart_mock_modbus_loopback.yaml | 233 ++++++++++ ...roller.yaml => uart_mock_modbus_mesh.yaml} | 121 ++++- .../uart_mock_modbus_register_offset.yaml | 138 ------ .../fixtures/uart_mock_modbus_server.yaml | 124 ------ ...ock_modbus_server_controller_multiple.yaml | 116 ----- ... => uart_mock_modbus_server_injected.yaml} | 55 ++- tests/integration/host_prefs.py | 14 +- .../test_api_zero_psk_provisioning.py | 1 - .../test_host_preferences_suspend_resume.py | 11 +- tests/integration/test_light_initial_state.py | 8 - tests/integration/test_uart_mock_modbus.py | 23 +- tests/script/test_helpers.py | 28 ++ 21 files changed, 805 insertions(+), 1062 deletions(-) delete mode 100644 tests/integration/fixtures/sensor_filters_batch_window.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_client_read_write.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_custom_pdu.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_lambda_invert.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml create mode 100644 tests/integration/fixtures/uart_mock_modbus_loopback.yaml rename tests/integration/fixtures/{uart_mock_modbus_server_controller.yaml => uart_mock_modbus_mesh.yaml} (58%) delete mode 100644 tests/integration/fixtures/uart_mock_modbus_register_offset.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_server.yaml delete mode 100644 tests/integration/fixtures/uart_mock_modbus_server_controller_multiple.yaml rename tests/integration/fixtures/{uart_mock_modbus_server_read_write.yaml => uart_mock_modbus_server_injected.yaml} (52%) diff --git a/script/helpers.py b/script/helpers.py index bf22e15808..a8a237118f 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -1104,6 +1104,10 @@ def get_components_per_integration_fixture() -> dict[str, set[str]]: _TEST_FUNC_RE = re.compile(r"async def (test_\w+)") +# Any usage form (decorator, pytestmark assignment or list element); only +# test_*.py files are scanned, so the marker docs elsewhere cannot false-hit +_SHARED_YAML_USE_RE = re.compile(r"\bmark\.shared_yaml") +_SHARED_YAML_ARG_RE = re.compile(r"\(\s*[\"'](\w+)[\"']\s*\)") @cache @@ -1123,6 +1127,19 @@ def get_fixture_to_test_files() -> dict[str, frozenset[str]]: for func in _TEST_FUNC_RE.findall(content): base_name = func.replace("test_", "").partition("[")[0] result.setdefault(base_name, set()).add(rel_path) + # Shared fixtures are named by marker, not by a test function; each + # decorator must carry a string literal or its fixture would silently + # map to no tests + for use in _SHARED_YAML_USE_RE.finditer(content): + arg = _SHARED_YAML_ARG_RE.match(content, use.end()) + if arg is None: + line = content.count("\n", 0, use.start()) + 1 + raise ValueError( + f"{rel_path}:{line}: shared_yaml marker must take a " + "single-line string literal so CI test selection can map " + "its fixture" + ) + result.setdefault(arg.group(1), set()).add(rel_path) return {k: frozenset(v) for k, v in result.items()} diff --git a/tests/integration/README.md b/tests/integration/README.md index 790d9a3a11..bee20409e8 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -21,6 +21,13 @@ The `yaml_config` fixture automatically loads YAML configurations based on the t - The fixture file must exist or the test will fail with a clear error message - The fixture automatically injects a dynamic port number into the API configuration +Tests marked `@pytest.mark.shared_yaml("name")` load `fixtures/name.yaml` instead +of the test-named file and compile it in a shared, hash-keyed build directory, so +the whole group pays one full compile and each test only a relink. The marker +argument must be a single-line string literal (CI test selection maps fixtures to +test files by scanning for it), and marked tests must hand the `yaml_config` +content to `run_compiled` unmodified. + ### Key Fixtures - `run_compiled` - Combines write, compile, and run operations into a single context manager diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 15c5860879..78e0b1a36c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,17 +4,22 @@ from __future__ import annotations import asyncio from collections.abc import AsyncGenerator, Callable, Generator -from contextlib import AbstractAsyncContextManager, asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress import fcntl +from functools import cache +import hashlib import logging import os from pathlib import Path import platform +import re +import shutil import signal import socket import subprocess import sys import tempfile +import time from typing import TextIO from aioesphomeapi import APIClient, APIConnectionError, LogParser, ReconnectLogic @@ -23,7 +28,13 @@ import pytest_asyncio import esphome.config from esphome.core import CORE -from esphome.helpers import get_usable_cpu_count +from esphome.helpers import ( + get_usable_cpu_count, + read_file, + rmtree, + write_file, + write_file_if_changed, +) from esphome.platformio.toolchain import get_idedata from .const import ( @@ -56,6 +67,21 @@ import pty # not available on Windows pytest.register_assert_rewrite("tests.integration.entity_utils") +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line( + "markers", + "shared_yaml(name): load fixtures/.yaml and compile it in a shared, " + "hash-keyed incremental build directory", + ) + + +FIXTURES_DIR = Path(__file__).parent / "fixtures" +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +# CI caches parts of this path; keep in sync with ci.yml integration-tests. +INTEGRATION_TESTS_ROOT = Path.home() / ".esphome-integration-tests" + + def _get_platformio_env(cache_dir: Path) -> dict[str, str]: """Get environment variables for PlatformIO with shared cache.""" env = os.environ.copy() @@ -78,7 +104,7 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]: ) # Compile with THIS tree's esphome sources, not wherever the venv's editable # install points (which may be a different git worktree or checkout). - repo_root = str(Path(__file__).resolve().parent.parent.parent) + repo_root = str(REPO_ROOT) existing = env.get("PYTHONPATH") env["PYTHONPATH"] = f"{repo_root}{os.pathsep}{existing}" if existing else repo_root return env @@ -88,8 +114,7 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]: def shared_platformio_cache() -> Generator[Path]: """Initialize a shared PlatformIO cache for all integration tests.""" # Use a dedicated directory for integration tests to avoid conflicts. - # CI caches parts of this path; keep in sync with ci.yml integration-tests. - test_cache_dir = Path.home() / ".esphome-integration-tests" + test_cache_dir = INTEGRATION_TESTS_ROOT cache_dir = test_cache_dir / "platformio" # Use a lock file in the home directory to ensure only one process initializes the cache @@ -112,7 +137,9 @@ def shared_platformio_cache() -> Generator[Path]: init_dir = Path(tmpdir) fixture_path = Path(__file__).parent / "fixtures" / "cache_init.yaml" config_path = init_dir / "cache_init.yaml" - config_path.write_text(fixture_path.read_text()) + config_path.write_text( + fixture_path.read_text(encoding="utf-8"), encoding="utf-8" + ) # Run compilation to populate the cache # We must succeed here to avoid race conditions where multiple @@ -162,13 +189,6 @@ def integration_test_dir() -> Generator[Path]: yield Path(tmpdir) -@pytest.fixture -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - """Host preferences persist per device name; give the test its own so a - provisioned key never leaks into another run.""" - monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) - - @pytest.fixture def reserved_tcp_port() -> Generator[tuple[int, socket.socket]]: """Reserve an unused TCP port by holding the socket open.""" @@ -188,21 +208,29 @@ def unused_tcp_port(reserved_tcp_port: tuple[int, socket.socket]) -> int: return reserved_tcp_port[0] +@pytest.fixture(autouse=True) +def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """Give every test its own host prefs dir; prefs are keyed only by device + name, which tests sharing a fixture also share.""" + prefdir = tmp_path / "prefs" + monkeypatch.setenv("ESPHOME_PREFDIR", str(prefdir)) + return prefdir + + @pytest_asyncio.fixture async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> str: """Load YAML configuration based on test name.""" - # Get the test function name - test_name: str = request.node.name - # Extract the base test name (remove test_ prefix and any parametrization) - base_name = test_name.replace("test_", "").partition("[")[0] + shared_name = _shared_yaml_name(request) + # Base test name: test_ prefix and any parametrization stripped + base_name = shared_name or request.node.name.replace("test_", "").partition("[")[0] # Load the fixture file - fixture_path = Path(__file__).parent / "fixtures" / f"{base_name}.yaml" + fixture_path = FIXTURES_DIR / f"{base_name}.yaml" if not fixture_path.exists(): raise FileNotFoundError(f"Fixture file not found: {fixture_path}") loop = asyncio.get_running_loop() - content = await loop.run_in_executor(None, fixture_path.read_text) + content = await loop.run_in_executor(None, read_file, fixture_path) # Replace the port in the config if it contains api section if "api:" in content: @@ -226,11 +254,13 @@ async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> s # Replace external component path placeholder if present if "EXTERNAL_COMPONENT_PATH" in content: - external_components_path = str( - Path(__file__).parent / "fixtures" / "external_components" - ) + external_components_path = str(FIXTURES_DIR / "external_components") content = content.replace("EXTERNAL_COMPONENT_PATH", external_components_path) + if shared_name is not None: + # _compile verifies the marked test compiles this content unmodified + request.node._shared_yaml_content = content + return content @@ -240,24 +270,218 @@ async def write_yaml_config( ) -> AsyncGenerator[ConfigWriter]: """Write YAML configuration to a file.""" # Get the test name for default filename - test_name = request.node.name - base_name = test_name.replace("test_", "").split("[")[0] + base_name = request.node.name.replace("test_", "").partition("[")[0] async def _write_config(content: str, filename: str | None = None) -> Path: if filename is None: filename = f"{base_name}.yaml" config_path = integration_test_dir / filename loop = asyncio.get_running_loop() - await loop.run_in_executor(None, config_path.write_text, content) + await loop.run_in_executor(None, write_file, config_path, content) return config_path yield _write_config +# Deliberately not CI-cached (ci.yml caches only platformio/ subpaths); stale +# dirs for a fixture are pruned when its content hash changes. +SHARED_BUILDS_ROOT = INTEGRATION_TESTS_ROOT / "builds" + +# In the dir name (not just the hash) so pruning stays inside this checkout +_REPO_KEY = hashlib.sha256(str(REPO_ROOT).encode()).hexdigest()[:8] + +# Give a contended shared build lock time for a full cold compile ahead of us +_SHARED_LOCK_TIMEOUT_S = 900 +_SHARED_LOCK_POLL_S = 0.1 +_SHARED_LOCK_REPORT_S = 30 + +# Reclaims dirs orphaned by fixture renames or deleted checkouts +_STALE_BUILD_MAX_AGE_S = 30 * 24 * 3600 + +# ELF path per shared build dir; constant once compiled, so resolve it only once +_shared_elf_paths: dict[Path, Path] = {} + +# Dirs this process already swept; pruning is session-scoped work +_pruned_dirs: set[Path] = set() + + +def _shared_yaml_name(request: pytest.FixtureRequest) -> str | None: + """Name passed to the shared_yaml marker, or None when unmarked.""" + marker = request.node.get_closest_marker("shared_yaml") + if marker is None: + return None + # Exactly one \w+ positional arg: the name doubles as a build dir + # component, and CI test selection (script/helpers.py) parses the same shape + if ( + len(marker.args) != 1 + or marker.kwargs + or not re.fullmatch(r"\w+", str(marker.args[0])) + ): + raise ValueError( + "shared_yaml marker requires exactly one \\w+ fixture name literal" + ) + return marker.args[0] + + +def _shared_build_prefix(name: str) -> str: + return f"{name}-{_REPO_KEY}-" + + +@cache +def _shared_build_dir(name: str) -> Path: + """Dir keyed by checkout and fixture source, before per-test injections.""" + key = hashlib.sha256((FIXTURES_DIR / f"{name}.yaml").read_bytes()).hexdigest()[:16] + return SHARED_BUILDS_ROOT / (_shared_build_prefix(name) + key) + + +def _read_stamp(stamp: Path, shared_dir: Path) -> Path | None: + """ELF path recorded by the last completed compile, or None.""" + try: + text = stamp.read_text(encoding="utf-8").strip() + except FileNotFoundError: + return None + except OSError as err: + print(f"Cannot read {stamp}: {err}") + return None + if not text: + print(f"Ignoring empty stamp {stamp}") + return None + built = Path(text) + # Never trust a stamp pointing outside its own build dir as an unlink target + if shared_dir.resolve() in built.resolve().parents: + return built + print(f"Ignoring stamp {stamp} pointing outside {shared_dir}") + return None + + +def _unused_since(stale: Path, cutoff: float) -> bool: + """Whether a build dir looks untouched since cutoff; unknown counts as used.""" + # Newest of the .built stamp (rewritten by every completed compile) and the + # dir itself (freshened by a worker claiming the dir before locking) + newest: float | None = None + for probe in (stale / ".built", stale): + try: + mtime = probe.stat().st_mtime + except FileNotFoundError: + continue + except NotADirectoryError: + return True # a stray file where a dir should be; reclaimable + except OSError as err: + print(f"Cannot age-probe {stale}: {err}") + return False # unknown never authorizes deletion + newest = mtime if newest is None else max(newest, mtime) + return newest is not None and newest < cutoff + + +def _prune_stale_builds(name: str, keep: Path) -> None: + """Remove outdated build dirs (blocking, run in executor): this checkout's + other dirs for the fixture, plus anything untouched for 30 days. Tolerates + other workers pruning the same dirs concurrently.""" + cutoff = time.time() - _STALE_BUILD_MAX_AGE_S + prefix = _shared_build_prefix(name) + for stale in SHARED_BUILDS_ROOT.iterdir(): + if stale == keep: + continue + same_fixture = stale.name.startswith(prefix) + if not same_fixture and not _unused_since(stale, cutoff): + continue + # Creating .lock bumps the dir mtime, so remember whether the re-probe + # under the lock can trust it + lock_preexisting = (stale / ".lock").exists() + try: + lock_file = (stale / ".lock").open("w") + except FileNotFoundError: + continue # pruned by another worker meanwhile + except NotADirectoryError: + print(f"Removing stray file {stale}") + stale.unlink(missing_ok=True) + continue + except OSError as err: + print(f"Cannot prune {stale}: {err}") + continue + with lock_file: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + continue # still in use by another run + # Re-probe under the lock: a worker freshens its dir before + # locking, so a just-claimed dir no longer looks unused. A dir + # whose .lock we just created cannot be held by anyone, and our + # own open bumped its mtime, so its pre-open probe stands + if ( + lock_preexisting + and not same_fixture + and not _unused_since(stale, cutoff) + ): + continue + # rmtree tolerates races; a leftover partial tree only costs a + # rebuild, since the ELF is deleted before every compile + try: + rmtree(stale) + except OSError as err: + print(f"Failed to prune {stale}: {err}") + + +async def _run_esphome_compile( + config_path: Path, cwd: Path, env: dict[str, str] +) -> None: + """Run `esphome compile`, retrying up to 3 times on a segfault.""" + max_retries = 3 + for attempt in range(max_retries): + # Compile using subprocess, inheriting stdout/stderr to show progress + proc = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "esphome", + "compile", + str(config_path), + cwd=cwd, + stdout=None, # Inherit stdout + stderr=None, # Inherit stderr + stdin=asyncio.subprocess.DEVNULL, + # Start in a new process group to isolate signal handling + start_new_session=True, + env=env, + close_fds=False, + ) + await proc.wait() + + if proc.returncode == 0: + break + if proc.returncode == -11 and attempt < max_retries - 1: + # Segfault (-11 = SIGSEGV), retry + print( + f"Compilation segfaulted (attempt {attempt + 1}/{max_retries}), retrying..." + ) + await asyncio.sleep(1) # Brief pause before retry + continue + raise RuntimeError( + f"Failed to compile {config_path}, return code: {proc.returncode}. " + f"Run with 'pytest -s' to see compilation output." + ) + + +def _resolve_compiled_binary(config_path: Path) -> Path: + """Load the config to learn the compiled ELF path (blocking, run in executor).""" + CORE.reset() # Reset CORE state between test runs + CORE.config_path = config_path + config = esphome.config.read_config( + {"command": "compile", "config": str(config_path)} + ) + if config is None: + raise RuntimeError(f"Failed to read config from {config_path}") + idedata = get_idedata(config) + binary_path = Path(idedata.firmware_elf_path) + if not binary_path.exists(): + raise RuntimeError(f"Compiled binary not found at {binary_path}") + return binary_path + + @pytest_asyncio.fixture async def compile_esphome( integration_test_dir: Path, shared_platformio_cache: Path, + request: pytest.FixtureRequest, ) -> AsyncGenerator[CompileFunction]: """Compile an ESPHome configuration and return the binary path.""" @@ -265,66 +489,96 @@ async def compile_esphome( # Use the shared PlatformIO cache for faster compilation # This avoids re-downloading dependencies for each test env = _get_platformio_env(shared_platformio_cache) - - # Retry compilation up to 3 times if we get a segfault - max_retries = 3 - for attempt in range(max_retries): - # Compile using subprocess, inheriting stdout/stderr to show progress - proc = await asyncio.create_subprocess_exec( - sys.executable, - "-m", - "esphome", - "compile", - str(config_path), - cwd=integration_test_dir, - stdout=None, # Inherit stdout - stderr=None, # Inherit stderr - stdin=asyncio.subprocess.DEVNULL, - # Start in a new process group to isolate signal handling - start_new_session=True, - env=env, - close_fds=False, - ) - await proc.wait() - - if proc.returncode == 0: - # Success! - break - if proc.returncode == -11 and attempt < max_retries - 1: - # Segfault (-11 = SIGSEGV), retry - print( - f"Compilation segfaulted (attempt {attempt + 1}/{max_retries}), retrying..." - ) - await asyncio.sleep(1) # Brief pause before retry - continue - # Other error or final retry - raise RuntimeError( - f"Failed to compile {config_path}, return code: {proc.returncode}. " - f"Run with 'pytest -s' to see compilation output." - ) - - # Load the config to get idedata (blocking call, must use executor) loop = asyncio.get_running_loop() - def _read_config_and_get_binary(): - CORE.reset() # Reset CORE state between test runs - CORE.config_path = config_path - config = esphome.config.read_config( - {"command": "compile", "config": str(config_path)} + name = _shared_yaml_name(request) + if name is None: + await _run_esphome_compile(config_path, integration_test_dir, env) + return await loop.run_in_executor( + None, _resolve_compiled_binary, config_path ) - if config is None: - raise RuntimeError(f"Failed to read config from {config_path}") - # Get the compiled binary path - idedata = get_idedata(config) - return Path(idedata.firmware_elf_path) - - binary_path = await loop.run_in_executor(None, _read_config_and_get_binary) - - if not binary_path.exists(): - raise RuntimeError(f"Compiled binary not found at {binary_path}") - - return binary_path + # Shared fixture: build in a hash-keyed dir so tests sharing a config + # pay one full compile and later only a main.cpp (port) rebuild + relink + shared_dir = _shared_build_dir(name) + shared_dir.mkdir(parents=True, exist_ok=True) + # Freshen the dir before locking so a concurrent age sweep, which + # re-probes under the lock, never reaps a dir a worker just claimed; + # if a peer reaped it already, the guarded lock open recreates it + with suppress(FileNotFoundError): + os.utime(shared_dir) + if shared_dir not in _pruned_dirs: + _pruned_dirs.add(shared_dir) + await loop.run_in_executor(None, _prune_stale_builds, name, shared_dir) + shared_config = shared_dir / f"{name}.yaml" + private_binary = integration_test_dir / f"{name}.elf" + content = await loop.run_in_executor(None, read_file, config_path) + if content != getattr(request.node, "_shared_yaml_content", None): + # The dir is keyed by the fixture source; a mutated config would be + # cached under a hash that does not describe it + raise RuntimeError( + "shared_yaml tests must compile the yaml_config content unmodified" + ) + # flock serializes concurrent xdist workers; closing the fd releases it. + # Hand-rolled rather than filelock.FileLock: non-blocking retries keep + # the wait cancellable, while a blocking acquire in an executor thread + # would survive test cancellation holding the fd + try: + lock_file = (shared_dir / ".lock").open("w") + except FileNotFoundError: + # A peer run pruning divergent hashes reaped the dir between our + # mkdir and this open; recreate it and pay a full rebuild + shared_dir.mkdir(parents=True, exist_ok=True) + lock_file = (shared_dir / ".lock").open("w") + with lock_file: + start = time.monotonic() + last_report = start + while True: + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + break + except BlockingIOError: + now = time.monotonic() + if now - start > _SHARED_LOCK_TIMEOUT_S: + raise RuntimeError( + f"Timed out waiting for the {shared_dir} lock" + ) from None + if now - last_report >= _SHARED_LOCK_REPORT_S: + last_report = now + print( + f"Waited {now - start:.0f}s for another worker's " + f"build of {shared_dir.name}" + ) + await asyncio.sleep(_SHARED_LOCK_POLL_S) + # .built carries the ELF path of the last completed compile, so + # later workers skip the config re-read in _resolve_compiled_binary + stamp = shared_dir / ".built" + if (built := _shared_elf_paths.get(shared_dir)) is None: + built = await loop.run_in_executor(None, _read_stamp, stamp, shared_dir) + # Delete the ELF before compiling: whatever exists afterwards is + # this compile's output, so no staleness check is ever needed. + # With no usable stamp, sweep any leftover at the known layout + if built is not None: + built.unlink(missing_ok=True) + else: + # Layout-agnostic: ESPHOME_BUILD_PATH can move the build tree + for leftover in shared_dir.rglob("program"): + if leftover.is_file(): + leftover.unlink() + await loop.run_in_executor( + None, write_file_if_changed, shared_config, content + ) + await _run_esphome_compile(shared_config, shared_dir, env) + if built is None or not built.exists(): + built = await loop.run_in_executor( + None, _resolve_compiled_binary, shared_config + ) + _shared_elf_paths[shared_dir] = built + await loop.run_in_executor(None, write_file, stamp, str(built)) + # Copy out before unlocking: another worker may relink firmware.elf + # while this test is still running its private copy + await loop.run_in_executor(None, shutil.copy2, built, private_binary) + return private_binary yield _compile diff --git a/tests/integration/fixtures/sensor_filters_batch_window.yaml b/tests/integration/fixtures/sensor_filters_batch_window.yaml deleted file mode 100644 index 58a254c215..0000000000 --- a/tests/integration/fixtures/sensor_filters_batch_window.yaml +++ /dev/null @@ -1,58 +0,0 @@ -esphome: - name: test-batch-window-filters - -host: -api: - batch_delay: 0ms # Disable batching to receive all state updates -logger: - level: DEBUG - -# Template sensor that we'll use to publish values -sensor: - - platform: template - name: "Source Sensor" - id: source_sensor - accuracy_decimals: 2 - - # Batch window filters (window_size == send_every) - use streaming filters - - platform: copy - source_id: source_sensor - name: "Min Sensor" - id: min_sensor - filters: - - min: - window_size: 5 - send_every: 5 - send_first_at: 1 - - - platform: copy - source_id: source_sensor - name: "Max Sensor" - id: max_sensor - filters: - - max: - window_size: 5 - send_every: 5 - send_first_at: 1 - - - platform: copy - source_id: source_sensor - name: "Moving Avg Sensor" - id: moving_avg_sensor - filters: - - sliding_window_moving_average: - window_size: 5 - send_every: 5 - send_first_at: 1 - -# Button to trigger publishing test values -button: - - platform: template - name: "Publish Values Button" - id: publish_button - on_press: - - lambda: |- - // Publish 10 values: 1.0, 2.0, ..., 10.0 - for (int i = 1; i <= 10; i++) { - id(source_sensor).publish_state(float(i)); - } diff --git a/tests/integration/fixtures/uart_mock_modbus_client_read_write.yaml b/tests/integration/fixtures/uart_mock_modbus_client_read_write.yaml deleted file mode 100644 index 1f89889c95..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_client_read_write.yaml +++ /dev/null @@ -1,111 +0,0 @@ -esphome: - name: uart-mock-modbus-cli-rw - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -# Two virtual buses looped back to each other: the client's transmissions reach the server and the -# server's replies reach the client. auto_start so forwarding is active before the button fires. -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_client - data: !lambda return data; - - id: virtual_uart_client - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: stored_1 - type: uint16_t - initial_value: "0" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_client - id: virtual_modbus_client - role: client - turnaround_time: 10ms - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - registers: - # Writable + readable register: the read publishes what it returns, so the test can confirm the - # write half of the 0x17 ran before the read half (Modbus 6.17). - - address: 0x01 - value_type: U_WORD - read_lambda: |- - id(srv_read_1).publish_state(id(stored_1)); - return id(stored_1); - write_lambda: |- - id(stored_1) = x; - id(srv_write_1).publish_state(x); - return true; - # Read-only register, returned together with 0x01 by the 2-register read half. - - address: 0x02 - value_type: U_WORD - read_lambda: return 0x00AA; - -sensor: - # Server-side observations. - - platform: template - name: "srv_write_1" - id: srv_write_1 - - platform: template - name: "srv_read_1" - id: srv_read_1 - # Client-side read-back: the values the client's on_response received. - - platform: template - name: "client_read_0" - id: client_read_0 - - platform: template - name: "client_read_1" - id: client_read_1 - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - on_press: - # FC 0x17: write reg 0x0001 = 0x1234, then read regs 0x0001..0x0002 back in the same transaction. - - modbus_client.read_write_multiple_registers: - address: 0x01 - read_address: 0x0001 - read_count: 2 - write_address: 0x0001 - values: [0x1234] - on_response: - then: - - lambda: |- - // values is the read-back block: reg 0x0001 (must be the just-written 0x1234) and reg 0x0002. - if (values.size() >= 2) { - id(client_read_0).publish_state(values[0]); - id(client_read_1).publish_state(values[1]); - } diff --git a/tests/integration/fixtures/uart_mock_modbus_custom_pdu.yaml b/tests/integration/fixtures/uart_mock_modbus_custom_pdu.yaml deleted file mode 100644 index 188abf90f1..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_custom_pdu.yaml +++ /dev/null @@ -1,88 +0,0 @@ -esphome: - name: uart-mock-modbus-custom-pdu - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x01 - value_type: U_WORD - read_lambda: return 259; - -sensor: - # Plain read to confirm the controller <-> server link is up. - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "plain_read" - address: 0x01 - register_type: holding - value_type: U_WORD - # Custom PDU: read holding register 0x0001, count 1. The PDU is - # {function code, address hi, address lo, count hi, count lo}; the device - # address and CRC are added by the hub. The lambda parses the response payload - # (the register value, big-endian). - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "custom_read" - custom_pdu: [0x03, 0x00, 0x01, 0x00, 0x01] - lambda: |- - if (data.size() < 2) return {}; - return (float) ((data[0] << 8) | data[1]); - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml b/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml deleted file mode 100644 index f378e3de43..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_deprecated_write_buffer.yaml +++ /dev/null @@ -1,106 +0,0 @@ -esphome: - name: uart-mock-modbus-dep-buffer - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: reg10 - type: uint16_t - initial_value: "0" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x10 - value_type: U_WORD - read_lambda: return id(reg10); - write_lambda: |- - id(reg10) = x; - return true; - -# A number whose write_lambda uses the DEPRECATED buffer parameter (fills `payload` with a legacy raw -# frame as words: device address + function code + data) instead of the new item->write_* API. The write -# must still land with its legacy semantics, and the one-time deprecation warning must fire only once per -# entity no matter how many writes happen. -number: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "buf_number" - id: buf_number - address: 0x10 - register_type: holding - value_type: U_WORD - min_value: 0 - max_value: 1000 - step: 1 - write_lambda: |- - // Legacy raw frame as words: [addr 0x01 | fc 0x06], register 0x0010, value. - payload.push_back(0x0106); - payload.push_back(0x0010); - payload.push_back((uint16_t) x); - return {}; - -# Reports the server-side register so the test can observe that the deprecated buffer write landed. -sensor: - - platform: template - name: "written_value" - id: written_value - update_interval: 0.5s - lambda: "return id(reg10);" - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # The test drives the writes via number_command; the mock is autostart. diff --git a/tests/integration/fixtures/uart_mock_modbus_lambda_invert.yaml b/tests/integration/fixtures/uart_mock_modbus_lambda_invert.yaml deleted file mode 100644 index 41afce70d6..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_lambda_invert.yaml +++ /dev/null @@ -1,95 +0,0 @@ -esphome: - name: uart-mock-modbus-lambda-invert - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: reg40 - type: uint16_t - initial_value: "5" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x40 - value_type: U_WORD - read_lambda: return id(reg40); - write_lambda: id(reg40) = x; return true; - -# An active-low holding switch: the write_lambda inverts the wire value, but the entity must still -# report the REQUESTED state. assumed_state keeps the register unpolled, so the published state comes -# only from write_state() - turning ON writes 0x0000 yet the switch shows ON. -switch: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "invert_switch" - register_type: holding - address: 0x40 - assumed_state: true - write_lambda: |- - return !x; - -sensor: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_40" - address: 0x40 - register_type: holding - value_type: U_WORD - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml b/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml deleted file mode 100644 index 86e17ea0d7..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_lambda_write.yaml +++ /dev/null @@ -1,97 +0,0 @@ -esphome: - name: uart-mock-modbus-lambda-write - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: reg30 - type: uint16_t - initial_value: "0" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x30 - value_type: U_WORD - read_lambda: return id(reg30); - write_lambda: id(reg30) = x; return true; - -# A COIL-type switch (assumed_state, write-only) whose write_lambda ignores its own coil type and instead -# drives a HOLDING-REGISTER write on the mock server through the entity itself: `item` IS the command, so -# item->write_single_register() sends a register write from a coil entity (cross-type). Returning nothing -# (an empty optional) tells the write path the lambda already dispatched the frame - no default coil write. -switch: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "cross_switch" - register_type: coil - address: 0x00 - assumed_state: true - write_lambda: |- - item->write_single_register(0x30, x ? 1234 : 0); - return {}; - -sensor: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_30" - address: 0x30 - register_type: holding - value_type: U_WORD - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_loopback.yaml b/tests/integration/fixtures/uart_mock_modbus_loopback.yaml new file mode 100644 index 0000000000..7212bfb2b2 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_loopback.yaml @@ -0,0 +1,233 @@ +esphome: + name: uart-mock-modbus-loopback + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +# Shared loopback fixture (see the shared_yaml markers in the test file); +# register spaces are disjoint so each test only observes its own entities. +uart_mock: + - id: virtual_uart_server + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + +globals: + - id: reg10 + type: uint16_t + initial_value: "100" + - id: reg11 + type: uint16_t + initial_value: "200" + - id: reg12 + type: uint16_t + initial_value: "300" + - id: reg13 + type: uint16_t + initial_value: "0xABCD" + - id: reg30 + type: uint16_t + initial_value: "0" + - id: reg40 + type: uint16_t + initial_value: "5" + - id: reg50 + type: uint16_t + initial_value: "0" + +modbus: + - uart_id: virtual_uart_server + id: virtual_modbus_server + role: server + - uart_id: virtual_uart_controller + id: virtual_modbus_controller + role: client + turnaround_time: 10ms + +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_controller + id: modbus_controller_1 + update_interval: 1s + +modbus_server: + - address: 1 + modbus_id: virtual_modbus_server + registers: + - address: 0x01 + value_type: U_WORD + read_lambda: return 259; + - address: 0x10 + value_type: U_WORD + read_lambda: return id(reg10); + write_lambda: id(reg10) = x; return true; + - address: 0x11 + value_type: U_WORD + read_lambda: return id(reg11); + write_lambda: id(reg11) = x; return true; + - address: 0x12 + value_type: U_WORD + read_lambda: return id(reg12); + write_lambda: id(reg12) = x; return true; + - address: 0x13 + value_type: U_WORD + read_lambda: return id(reg13); + - address: 0x30 + value_type: U_WORD + read_lambda: return id(reg30); + write_lambda: id(reg30) = x; return true; + - address: 0x40 + value_type: U_WORD + read_lambda: return id(reg40); + write_lambda: id(reg40) = x; return true; + - address: 0x50 + value_type: U_WORD + read_lambda: return id(reg50); + write_lambda: id(reg50) = x; return true; + +# Byte-based offset: 2 bytes -> register 0x11 (the old code folded it in as a +# register count, hitting 0x12). assumed_state keeps the switch write-only. +switch: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "offset_switch" + register_type: holding + address: 0x10 + offset: 2 + assumed_state: true + # Reading switch, byte offset 6 -> register 0x13; the pre-fix resolution (0x16) + # would draw ILLEGAL_DATA_ADDRESS and never publish. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "read_offset_switch" + register_type: holding + address: 0x10 + offset: 6 + bitmask: 0x1 + # Coil switch whose write_lambda dispatches a holding-register write via `item`; + # returning an empty optional suppresses the default coil write. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "cross_switch" + register_type: coil + address: 0x00 + assumed_state: true + write_lambda: |- + item->write_single_register(0x30, x ? 1234 : 0); + return {}; + # Active-low: the write_lambda inverts the wire value but the entity must still + # report the requested state (assumed_state keeps the register unpolled). + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "invert_switch" + register_type: holding + address: 0x40 + assumed_state: true + write_lambda: |- + return !x; + +# Uses the deprecated buffer parameter (legacy raw frame as words); the write +# must land and the deprecation warning must fire only once per entity. +number: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "buf_number" + id: buf_number + address: 0x50 + register_type: holding + value_type: U_WORD + min_value: 0 + max_value: 1000 + step: 1 + write_lambda: |- + // Legacy raw frame as words: [addr 0x01 | fc 0x06], register 0x0050, value. + payload.push_back(0x0106); + payload.push_back(0x0050); + payload.push_back((uint16_t) x); + return {}; + +sensor: + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "plain_read" + address: 0x01 + register_type: holding + value_type: U_WORD + # Custom PDU: read holding register 0x0001; device address and CRC are added + # by the hub. The lambda parses the big-endian register value. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "custom_read" + custom_pdu: [0x03, 0x00, 0x01, 0x00, 0x01] + lambda: |- + if (data.size() < 2) return {}; + return (float) ((data[0] << 8) | data[1]); + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_10" + address: 0x10 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_11" + address: 0x11 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_12" + address: 0x12 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_30" + address: 0x30 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_40" + address: 0x40 + register_type: holding + value_type: U_WORD + # Reports the server-side register so the test can observe that the deprecated buffer write landed. + - platform: template + name: "written_value" + id: written_value + update_interval: 0.5s + lambda: "return id(reg50);" + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + # Nothing to start (mock is autostart); tests drive entities directly diff --git a/tests/integration/fixtures/uart_mock_modbus_server_controller.yaml b/tests/integration/fixtures/uart_mock_modbus_mesh.yaml similarity index 58% rename from tests/integration/fixtures/uart_mock_modbus_server_controller.yaml rename to tests/integration/fixtures/uart_mock_modbus_mesh.yaml index 4a5d280a2f..69edd614d7 100644 --- a/tests/integration/fixtures/uart_mock_modbus_server_controller.yaml +++ b/tests/integration/fixtures/uart_mock_modbus_mesh.yaml @@ -1,5 +1,5 @@ esphome: - name: uart-mock-modbus-server-contro + name: uart-mock-modbus-mesh host: api: @@ -17,13 +17,14 @@ uart: baud_rate: 115200 port: /dev/null +# Shared 3-bus mesh (see the shared_yaml markers): addr 1 = typed read-only +# registers, addr 5 = the read/write 0x17 target, addr 2/3 on the second +# server hub. auto_start everywhere: the controller polls at boot, so the +# forwarding must already be live or early requests generate warnings. +# Every test presses Start Scenario, so all merged actions fire in every test. uart_mock: - id: virtual_uart_server baud_rate: 9600 - # auto_start must be true for loopback fixtures: the modbus controller - # polls on its update_interval immediately at boot, so the uart_mock - # forwarding must already be active or early requests are lost and - # generate modbus warnings. auto_start: true debug: on_tx: @@ -31,35 +32,68 @@ uart_mock: - uart_mock.inject_rx: id: virtual_uart_controller data: !lambda return data; - - id: virtual_uart_controller + - uart_mock.inject_rx: + id: virtual_uart_server_2 + data: !lambda return data; + - id: virtual_uart_server_2 baud_rate: 9600 - auto_start: true # See comment on virtual_uart_server above + auto_start: true debug: on_tx: - then: - uart_mock.inject_rx: id: virtual_uart_server data: !lambda return data; + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + - uart_mock.inject_rx: + id: virtual_uart_server_2 + data: !lambda return data; + +globals: + - id: stored_1 + type: uint16_t + initial_value: "0" modbus: - uart_id: virtual_uart_server id: virtual_modbus_server role: server + - uart_id: virtual_uart_server_2 + id: virtual_modbus_server_2 + role: server - uart_id: virtual_uart_controller - id: virtual_modbus_controller + id: virtual_modbus_client role: client turnaround_time: 10ms modbus_controller: - address: 1 - modbus_id: virtual_modbus_controller + modbus_id: virtual_modbus_client id: modbus_controller_1 update_interval: 1s + - address: 2 + modbus_id: virtual_modbus_client + id: modbus_controller_2 + update_interval: 1s + - address: 3 + modbus_id: virtual_modbus_client + id: modbus_controller_3 + update_interval: 1s modbus_server: - address: 1 modbus_id: virtual_modbus_server - id: modbus_server_1 registers: - address: 0x01 value_type: U_WORD @@ -103,6 +137,34 @@ modbus_server: - address: 0x28 value_type: FP32_R read_lambda: return 3.14; + - address: 5 + modbus_id: virtual_modbus_server + registers: + # Writable + readable register: srv_write_1 plus the client's read-back + # confirm the write half of the 0x17 ran before the read half (Modbus 6.17). + - address: 0x01 + value_type: U_WORD + read_lambda: return id(stored_1); + write_lambda: |- + id(stored_1) = x; + id(srv_write_1).publish_state(x); + return true; + # Read-only register, returned together with 0x01 by the 2-register read half. + - address: 0x02 + value_type: U_WORD + read_lambda: return 0x00AA; + - address: 2 + modbus_id: virtual_modbus_server_2 + registers: + - address: 0x01 + value_type: U_WORD + read_lambda: return 919; + - address: 3 + modbus_id: virtual_modbus_server_2 + registers: + - address: 0x01 + value_type: U_WORD + read_lambda: return 929; sensor: - platform: modbus_controller @@ -195,9 +257,46 @@ sensor: address: 0x28 register_type: holding value_type: FP32_R + - platform: modbus_controller + modbus_controller_id: modbus_controller_2 + name: "multi_reg_a" + address: 0x01 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_3 + name: "multi_reg_b" + address: 0x01 + register_type: holding + value_type: U_WORD + # client_read_write observations, server- and client-side. + - platform: template + name: "srv_write_1" + id: srv_write_1 + - platform: template + name: "client_read_0" + id: client_read_0 + - platform: template + name: "client_read_1" + id: client_read_1 button: - platform: template name: "Start Scenario" id: start_scenario_btn - # This test does not have anything to start (mock is autostart) + on_press: + # FC 0x17: write reg 0x0001 = 0x1234, then read regs 0x0001..0x0002 back in the same transaction. + - modbus_client.read_write_multiple_registers: + address: 5 + read_address: 0x0001 + read_count: 2 + write_address: 0x0001 + values: [0x1234] + on_response: + then: + - lambda: |- + // values is the read-back block: reg 0x0001 (must be the just-written 0x1234) and reg 0x0002. + if (values.size() >= 2) { + id(client_read_0).publish_state(values[0]); + id(client_read_1).publish_state(values[1]); + } diff --git a/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml b/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml deleted file mode 100644 index 21c451aa99..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_register_offset.yaml +++ /dev/null @@ -1,138 +0,0 @@ -esphome: - name: uart-mock-modbus-reg-offset - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - -globals: - - id: reg10 - type: uint16_t - initial_value: "100" - - id: reg11 - type: uint16_t - initial_value: "200" - - id: reg12 - type: uint16_t - initial_value: "300" - - id: reg13 - type: uint16_t - initial_value: "0xABCD" - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_controller - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_controller - id: modbus_controller_1 - update_interval: 1s - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - id: modbus_server_1 - registers: - - address: 0x10 - value_type: U_WORD - read_lambda: return id(reg10); - write_lambda: id(reg10) = x; return true; - - address: 0x11 - value_type: U_WORD - read_lambda: return id(reg11); - write_lambda: id(reg11) = x; return true; - - address: 0x12 - value_type: U_WORD - read_lambda: return id(reg12); - write_lambda: id(reg12) = x; return true; - - address: 0x13 - value_type: U_WORD - read_lambda: return id(reg13); - write_lambda: id(reg13) = x; return true; - -# A holding-register switch at 0x10 with a 2-BYTE offset. offset is byte-based, so the write must target -# register 0x10 + 2/2 = 0x11. The old (pre-fix) behavior folded offset into the address as a register -# count, hitting 0x12 instead. assumed_state keeps the switch write-only so it does not read any register. -switch: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "offset_switch" - register_type: holding - address: 0x10 - offset: 2 - assumed_state: true - # A holding-register switch that READS its state. Byte offset 6 -> register 0x10 + 6/2 = 0x13. Post-fix - # the switch itself resolves to 0x13 (the even byte offset folds into the address as whole registers) and - # joins the 0x10..0x13 range, so no separate 0x13 sensor is needed. Pre-fix the whole byte offset folds - # into the address (0x16), where the server answers ILLEGAL_DATA_ADDRESS and the switch never publishes. - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "read_offset_switch" - register_type: holding - address: 0x10 - offset: 6 - bitmask: 0x1 - -sensor: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_10" - address: 0x10 - register_type: holding - value_type: U_WORD - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_11" - address: 0x11 - register_type: holding - value_type: U_WORD - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_12" - address: 0x12 - register_type: holding - value_type: U_WORD - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_server.yaml b/tests/integration/fixtures/uart_mock_modbus_server.yaml deleted file mode 100644 index cc5a59e242..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_server.yaml +++ /dev/null @@ -1,124 +0,0 @@ -esphome: - name: uart-mock-modbus-server-test - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_dev - baud_rate: 9600 - rx_full_threshold: 120 - rx_timeout: 2 - auto_start: false - debug: - injections: - - delay: 100ms - inject_rx: [0x01, 0x03, 0x00, 0x03, 0x00, 0x01, 0x74, 0x0A] # Read holding register 3 on device 1 (basic_read) - - delay: 100ms - # Read holding register 7 on device 2 - # Reply from device 2 - # Read holding register 5 on device 1 (read_after_peer_response) - inject_rx: - [ - 0x02, - 0x03, - 0x00, - 0x07, - 0x00, - 0x01, - 0x35, - 0xF8, - 0x02, - 0x03, - 0x02, - 0x00, - 0xF0, - 0xFC, - 0x00, - 0x01, - 0x03, - 0x00, - 0x05, - 0x00, - 0x01, - 0x94, - 0x0B, - ] - - delay: 100ms - inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8] # Read holding register 7 on device 2, with no response - - delay: 100ms - # Read holding register 7 on device 2, with no response - # Read holding register A on device 1 (read_after_peer_timeout) - inject_rx: - [ - 0x02, - 0x03, - 0x00, - 0x07, - 0x00, - 0x01, - 0x35, - 0xF8, - 0x01, - 0x03, - 0x00, - 0x0A, - 0x00, - 0x01, - 0xA4, - 0x08, - ] - -modbus: - uart_id: virtual_uart_dev - role: server - -modbus_server: - - address: 1 - registers: - - address: 0x03 - value_type: U_WORD - read_lambda: |- - id(basic_read).publish_state(1); - return 1; - - address: 0x05 - value_type: U_WORD - read_lambda: |- - id(read_after_peer_response).publish_state(1); - return 1; - - address: 0x0A - value_type: U_WORD - read_lambda: |- - id(read_after_peer_timeout).publish_state(1); - return 1; - -sensor: - - platform: template - name: "basic_read" - id: basic_read - - platform: template - name: "read_after_peer_response" - id: read_after_peer_response - - platform: template - name: "read_after_peer_timeout" - id: read_after_peer_timeout - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - on_press: - - lambda: "id(virtual_uart_dev).start_scenario();" diff --git a/tests/integration/fixtures/uart_mock_modbus_server_controller_multiple.yaml b/tests/integration/fixtures/uart_mock_modbus_server_controller_multiple.yaml deleted file mode 100644 index 18423be6d5..0000000000 --- a/tests/integration/fixtures/uart_mock_modbus_server_controller_multiple.yaml +++ /dev/null @@ -1,116 +0,0 @@ -esphome: - name: uart-mock-modbus-server-mult - -host: -api: -logger: - level: VERBOSE - -external_components: - - source: - type: local - path: EXTERNAL_COMPONENT_PATH - -# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] -# The actual UART bus used is the uart_mock component below -uart: - baud_rate: 115200 - port: /dev/null - -uart_mock: - - id: virtual_uart_server - baud_rate: 9600 - # auto_start must be true for loopback fixtures: the modbus controller - # polls on its update_interval immediately at boot, so the uart_mock - # forwarding must already be active or early requests are lost and - # generate modbus warnings. - auto_start: true - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - uart_mock.inject_rx: - id: virtual_uart_server_2 - data: !lambda return data; - - id: virtual_uart_server_2 - baud_rate: 9600 - auto_start: true # See comment on virtual_uart_server above - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - - uart_mock.inject_rx: - id: virtual_uart_controller - data: !lambda return data; - - id: virtual_uart_controller - baud_rate: 9600 - auto_start: true # See comment on virtual_uart_server above - debug: - on_tx: - - then: - - uart_mock.inject_rx: - id: virtual_uart_server - data: !lambda return data; - - uart_mock.inject_rx: - id: virtual_uart_server_2 - data: !lambda return data; - -modbus: - - uart_id: virtual_uart_server - id: virtual_modbus_server - role: server - - uart_id: virtual_uart_server_2 - id: virtual_modbus_server_2 - role: server - - uart_id: virtual_uart_controller - id: virtual_modbus_client - role: client - turnaround_time: 10ms - -modbus_controller: - - address: 1 - modbus_id: virtual_modbus_client - update_interval: 1s - id: modbus_controller_1 - - address: 2 - modbus_id: virtual_modbus_client - update_interval: 1s - id: modbus_controller_2 - -modbus_server: - - address: 1 - modbus_id: virtual_modbus_server - registers: - - address: 0x01 - value_type: U_WORD - read_lambda: return 919; - - address: 2 - modbus_id: virtual_modbus_server_2 - registers: - - address: 0x01 - value_type: U_WORD - read_lambda: return 929; - -sensor: - - platform: modbus_controller - modbus_controller_id: modbus_controller_1 - name: "reg_u_word" - address: 0x01 - register_type: holding - value_type: U_WORD - - platform: modbus_controller - modbus_controller_id: modbus_controller_2 - name: "reg_u_word_2" - address: 0x01 - register_type: holding - value_type: U_WORD - -button: - - platform: template - name: "Start Scenario" - id: start_scenario_btn - # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_server_read_write.yaml b/tests/integration/fixtures/uart_mock_modbus_server_injected.yaml similarity index 52% rename from tests/integration/fixtures/uart_mock_modbus_server_read_write.yaml rename to tests/integration/fixtures/uart_mock_modbus_server_injected.yaml index e998861c2d..2cd1c610f1 100644 --- a/tests/integration/fixtures/uart_mock_modbus_server_read_write.yaml +++ b/tests/integration/fixtures/uart_mock_modbus_server_injected.yaml @@ -1,5 +1,5 @@ esphome: - name: uart-mock-modbus-srv-rw + name: uart-mock-modbus-srv-injected host: api: @@ -17,6 +17,8 @@ uart: baud_rate: 115200 port: /dev/null +# Shared server-role fixture (see the shared_yaml markers in the test file); +# the injections concatenate and each test waits only on its own sensors. uart_mock: - id: virtual_uart_dev baud_rate: 9600 @@ -25,18 +27,31 @@ uart_mock: auto_start: false debug: injections: - # FC 0x17 Read/Write Multiple Registers on device 1: - # write reg 0x0001 = 0x1234 (qty 1), then read regs 0x0001..0x0002 (qty 2). - # Per Modbus 6.17 the write is performed before the read, so reg 0x0001 must - # read back the just-written 0x1234 in the same request. + - delay: 100ms + inject_rx: [0x01, 0x03, 0x00, 0x03, 0x00, 0x01, 0x74, 0x0A] # Read holding register 3 on device 1 (basic_read) + - delay: 100ms + # Read holding register 7 on device 2, its reply, then read holding + # register 5 on device 1 (read_after_peer_response) + inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8, + 0x02, 0x03, 0x02, 0x00, 0xF0, 0xFC, + 0x00, 0x01, 0x03, 0x00, 0x05, 0x00, 0x01, 0x94, 0x0B] + - delay: 100ms + inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8] # Read holding register 7 on device 2, with no response + - delay: 100ms + # Read holding register 7 on device 2 with no response, then read + # holding register A on device 1 (read_after_peer_timeout) + inject_rx: [0x02, 0x03, 0x00, 0x07, 0x00, 0x01, 0x35, 0xF8, + 0x01, 0x03, 0x00, 0x0A, 0x00, 0x01, 0xA4, 0x08] + # FC 0x17 on device 1: write reg 0x0001 = 0x1234 then read 0x0001..0x0002; + # per Modbus 6.17 the write runs first, so 0x0001 must read back 0x1234. - delay: 100ms inject_rx: [0x01, 0x17, 0x00, 0x01, 0x00, 0x02, 0x00, 0x01, 0x00, 0x01, 0x02, 0x12, 0x34, 0x49, 0xD8] - # FC 0x17: write reg 0x0003 = 0x5678 (qty 1), then read reg 0x0003 (qty 1) - + # FC 0x17: write reg 0x0006 = 0x5678 (qty 1), then read reg 0x0006 (qty 1) - # a write and read targeting a different register block. - delay: 100ms inject_rx: - [0x01, 0x17, 0x00, 0x03, 0x00, 0x01, 0x00, 0x03, 0x00, 0x01, 0x02, 0x56, 0x78, 0x9B, 0x10] + [0x01, 0x17, 0x00, 0x06, 0x00, 0x01, 0x00, 0x06, 0x00, 0x01, 0x02, 0x56, 0x78, 0x8B, 0x55] globals: - id: stored_1 @@ -70,8 +85,18 @@ modbus_server: read_lambda: |- id(rw_read_2).publish_state(0x00AA); return 0x00AA; - # Second writable + readable register, targeted by the second request. - address: 0x03 + value_type: U_WORD + read_lambda: |- + id(basic_read).publish_state(1); + return 1; + - address: 0x05 + value_type: U_WORD + read_lambda: |- + id(read_after_peer_response).publish_state(1); + return 1; + # Second writable + readable register, targeted by the second FC 0x17 request. + - address: 0x06 value_type: U_WORD read_lambda: |- id(rw_read_3).publish_state(id(stored_3)); @@ -80,8 +105,22 @@ modbus_server: id(stored_3) = x; id(rw_write_3).publish_state(x); return true; + - address: 0x0A + value_type: U_WORD + read_lambda: |- + id(read_after_peer_timeout).publish_state(1); + return 1; sensor: + - platform: template + name: "basic_read" + id: basic_read + - platform: template + name: "read_after_peer_response" + id: read_after_peer_response + - platform: template + name: "read_after_peer_timeout" + id: read_after_peer_timeout - platform: template name: "rw_write_1" id: rw_write_1 diff --git a/tests/integration/host_prefs.py b/tests/integration/host_prefs.py index c7f21d8a01..5f526dce5f 100644 --- a/tests/integration/host_prefs.py +++ b/tests/integration/host_prefs.py @@ -1,7 +1,7 @@ """Helpers for manipulating the host platform's preferences file. ESPHome's host platform stores preferences in -``~/.esphome/prefs/.prefs`` using a simple binary layout that +``$ESPHOME_PREFDIR/.prefs`` using a simple binary layout that mirrors ``HostPreferences::sync()``: ``[uint32_t key][uint8_t len][uint8_t data[len]]`` per entry. @@ -11,13 +11,21 @@ boot (e.g. forcing safe mode) or to clear stale state between runs. from __future__ import annotations +import os from pathlib import Path import struct def host_prefs_path(device_name: str) -> Path: - """Return the on-disk prefs file path for a host-platform device.""" - return Path.home() / ".esphome" / "prefs" / f"{device_name}.prefs" + """Return the on-disk prefs file path for a host-platform device. + + Requires ESPHOME_PREFDIR, which the autouse isolated_preferences fixture + sets; refusing the ~/.esphome/prefs fallback keeps tests off real user + data if the fixture is ever bypassed.""" + prefdir = os.environ.get("ESPHOME_PREFDIR") + if not prefdir: + raise RuntimeError("ESPHOME_PREFDIR is not set; refusing the real prefs dir") + return Path(prefdir) / f"{device_name}.prefs" def clear_host_prefs(device_name: str) -> None: diff --git a/tests/integration/test_api_zero_psk_provisioning.py b/tests/integration/test_api_zero_psk_provisioning.py index f315335d1b..d103167a00 100644 --- a/tests/integration/test_api_zero_psk_provisioning.py +++ b/tests/integration/test_api_zero_psk_provisioning.py @@ -24,7 +24,6 @@ from .types import ( RunCompiledFunction, ) -pytestmark = pytest.mark.usefixtures("isolated_preferences") NEW_KEY = PROVISIONING_PSK diff --git a/tests/integration/test_host_preferences_suspend_resume.py b/tests/integration/test_host_preferences_suspend_resume.py index ab08d5c440..5f08d5519e 100644 --- a/tests/integration/test_host_preferences_suspend_resume.py +++ b/tests/integration/test_host_preferences_suspend_resume.py @@ -41,15 +41,6 @@ async def _poll_until_exists(path: Path) -> None: await asyncio.sleep(0.05) -@pytest.fixture(autouse=True) -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> Path: - """Keep host preferences per-test so this test never touches the real - ~/.esphome/prefs and never races other tests over ESPHOME_PREFDIR.""" - prefdir = tmp_path / "prefs" - monkeypatch.setenv("ESPHOME_PREFDIR", str(prefdir)) - return prefdir / f"{DEVICE_NAME}.prefs" - - @pytest.mark.asyncio async def test_host_preferences_suspend_resume( yaml_config: str, @@ -58,7 +49,7 @@ async def test_host_preferences_suspend_resume( isolated_preferences: Path, ) -> None: """Test that a running syncer flushes, a suspended one doesn't, and resume restores flushing.""" - pref_file = isolated_preferences + pref_file = isolated_preferences / f"{DEVICE_NAME}.prefs" loop = asyncio.get_running_loop() saved_in_memory = loop.create_future() diff --git a/tests/integration/test_light_initial_state.py b/tests/integration/test_light_initial_state.py index 657e273fe7..12ebf7c4a1 100644 --- a/tests/integration/test_light_initial_state.py +++ b/tests/integration/test_light_initial_state.py @@ -11,14 +11,6 @@ from .state_utils import InitialStateHelper, require_entity from .types import APIClientConnectedFactory, RunCompiledFunction -@pytest.fixture(autouse=True) -def isolated_preferences(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None: - """Keep host preferences per-test so RESTORE_AND_ON never loads a stale value left - behind by a previous run (host preferences otherwise persist to ~/.esphome/prefs, - keyed only by device name).""" - monkeypatch.setenv("ESPHOME_PREFDIR", str(tmp_path / "prefs")) - - @pytest.mark.asyncio async def test_light_initial_state( yaml_config: str, diff --git a/tests/integration/test_uart_mock_modbus.py b/tests/integration/test_uart_mock_modbus.py index 864275f5ed..232e1fb654 100644 --- a/tests/integration/test_uart_mock_modbus.py +++ b/tests/integration/test_uart_mock_modbus.py @@ -173,6 +173,7 @@ async def test_uart_mock_modbus_no_threshold( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_server_injected") @pytest.mark.asyncio async def test_uart_mock_modbus_server( yaml_config: str, @@ -203,6 +204,7 @@ async def test_uart_mock_modbus_server( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_server_injected") @pytest.mark.asyncio async def test_uart_mock_modbus_server_read_write( yaml_config: str, @@ -231,8 +233,8 @@ async def test_uart_mock_modbus_server_read_write( "rw_write_1": 4660, # 0x1234 written to reg 0x0001 "rw_read_1": 4660, # reg 0x0001 reads back the just-written value "rw_read_2": 170, # 0x00AA read from reg 0x0002 in the same request - "rw_write_3": 22136, # 0x5678 written to reg 0x0003 - "rw_read_3": 22136, # reg 0x0003 reads back the just-written value + "rw_write_3": 22136, # 0x5678 written to reg 0x0006 + "rw_read_3": 22136, # reg 0x0006 reads back the just-written value } ) @@ -241,7 +243,8 @@ async def test_uart_mock_modbus_server_read_write( api_client_connected() as client, ): await tracker.setup_and_start_scenario(client) - await tracker.await_all(futures) + # The FC 0x17 injections fire last, behind four earlier 100ms delays + await tracker.await_all(futures, timeout=4.0) _assert_no_modbus_errors(error_log_lines, warning_log_lines) @@ -296,6 +299,7 @@ async def test_uart_mock_modbus_server_read_write_invalid( ) +@pytest.mark.shared_yaml("uart_mock_modbus_mesh") @pytest.mark.asyncio async def test_uart_mock_modbus_server_controller( yaml_config: str, @@ -485,6 +489,7 @@ async def test_uart_mock_modbus_server_controller_bits( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_mesh") @pytest.mark.asyncio async def test_uart_mock_modbus_server_controller_multiple( yaml_config: str, @@ -495,7 +500,7 @@ async def test_uart_mock_modbus_server_controller_multiple( line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback() - expected_values = {"reg_u_word": 919, "reg_u_word_2": 929} + expected_values = {"multi_reg_a": 919, "multi_reg_b": 929} tracker = SensorTracker(list(expected_values.keys())) futures = tracker.expect_all(expected_values) @@ -706,6 +711,7 @@ async def test_uart_mock_modbus_shared_address( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_custom_pdu( yaml_config: str, @@ -932,6 +938,7 @@ async def test_uart_mock_modbus_broadcast_write( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_mesh") @pytest.mark.asyncio async def test_uart_mock_modbus_client_read_write( yaml_config: str, @@ -947,9 +954,7 @@ async def test_uart_mock_modbus_client_read_write( """ line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback() - tracker = SensorTracker( - ["srv_write_1", "srv_read_1", "client_read_0", "client_read_1"] - ) + tracker = SensorTracker(["srv_write_1", "client_read_0", "client_read_1"]) futures = tracker.expect_all( { "srv_write_1": 4660, # server wrote 0x1234 to reg 0x0001 @@ -967,6 +972,7 @@ async def test_uart_mock_modbus_client_read_write( _assert_no_modbus_errors(error_log_lines, warning_log_lines) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_register_offset( yaml_config: str, @@ -1022,6 +1028,7 @@ async def test_uart_mock_modbus_register_offset( ) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_lambda_write( yaml_config: str, @@ -1058,6 +1065,7 @@ async def test_uart_mock_modbus_lambda_write( await tracker.await_change(wrote_30, "reg_30", timeout=4.0) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_lambda_invert( yaml_config: str, @@ -1113,6 +1121,7 @@ async def test_uart_mock_modbus_lambda_invert( ) +@pytest.mark.shared_yaml("uart_mock_modbus_loopback") @pytest.mark.asyncio async def test_uart_mock_modbus_deprecated_write_buffer( yaml_config: str, diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 7d4059da2f..8f82a121c6 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -2122,6 +2122,34 @@ def test_get_cpp_changed_components_independent_of_cwd( ) == ["time"] +def test_fixture_map_includes_shared_yaml_markers() -> None: + """Fixtures named only by shared_yaml markers must map to their test file.""" + helpers.get_fixture_to_test_files.cache_clear() + mapping = helpers.get_fixture_to_test_files() + for fixture in ( + "uart_mock_modbus_loopback", + "uart_mock_modbus_mesh", + "uart_mock_modbus_server_injected", + ): + assert mapping[fixture] == frozenset( + {"tests/integration/test_uart_mock_modbus.py"} + ) + + +def test_no_orphan_integration_fixtures() -> None: + """Every fixture must reach CI test selection; an orphan selects nothing.""" + helpers.get_fixture_to_test_files.cache_clear() + mapping = helpers.get_fixture_to_test_files() + fixtures_dir = (Path(__file__).parent.parent / "integration" / "fixtures").resolve() + fixtures = list(fixtures_dir.glob("*.yaml")) + assert fixtures, f"no fixtures found under {fixtures_dir}" + # cache_init is covered via INTEGRATION_TESTS_TRIGGER_FILES instead + orphans = [ + f.stem for f in fixtures if f.stem != "cache_init" and f.stem not in mapping + ] + assert not orphans, f"fixtures invisible to CI test selection: {orphans}" + + def test_lpt_partition_balances_skewed_weights() -> None: """Heavy items spread across groups instead of clustering.""" items = [f"i{n}" for n in range(6)]