[xiaomi_ble] Drop the trailing underscore from the virtual publish_result and check optionals in the tests

Format the address only when the result is logged, keep the header flags
local, and add XiaomiEncryptedDevice::dump_bindkey_() for dump_config.

AGENTS.md

@@ -209,7 +209,8 @@ file does, and it is the authority when they disagree. The most useful starting
         registered with `automation.ApplyBytes` (see Actions below), which keeps a constant payload in flash;
         `TEMPLATABLE_BYTES` is for a `play()` that does more with the bytes.
     *   **Fixed sets of names** (status words, model names): a `PROGMEM_STRING_TABLE` (`esphome/core/progmem.h`),
-        logged with `get_log_str` and `LOG_STR_ARG`, copied out with `ESPHOME_strncpy_P` before `publish_state`.
+        logged with `get_log_str` and `LOG_STR_ARG`, published on a text sensor with
+        `publish_state(Table::get_progmem_str(idx, fallback))`, which copies it out of flash on ESP8266.
         A `dump_config` helper that picks a name returns `const LogString *` from `LOG_STR(...)`.
     *   **CRCs and bit tricks** come from `esphome/core/helpers.h` (`crc8`, `crc16`, `xor_bytes`, `sum16_bytes`,
         `reverse_bits`, `encode_uint16`, `encode_value_le`, `sign_extend<BITS>`, `bcd_to_uint8`, `assign_bits`); pick

CODEOWNERS

@@ -436,6 +436,7 @@ esphome/components/modbus_controller/sensor/* @martgras
 esphome/components/modbus_controller/switch/* @martgras
 esphome/components/modbus_controller/text_sensor/* @martgras
 esphome/components/modbus_server/* @exciton
+esphome/components/modem/* @oarcher
 esphome/components/mopeka_ble/* @Fabian-Schmidt @spbrogan
 esphome/components/mopeka_pro_check/* @spbrogan
 esphome/components/mopeka_std_check/* @Fabian-Schmidt
@@ -696,6 +697,7 @@ esphome/components/uart/packet_transport/* @clydebarrow
 esphome/components/uart_mux/* @kbx81
 esphome/components/uart_tcp/* @Bascht74
 esphome/components/udp/* @clydebarrow
+esphome/components/udp/packet_interface/* @clydebarrow
 esphome/components/ufire_ec/* @pvizeli
 esphome/components/ufire_ise/* @pvizeli
 esphome/components/ufm01/* @ljungqvist

docker/Dockerfile

@@ -84,3 +84,5 @@ FROM base-${BUILD_TYPE} AS final
 # Copy esphome and install
 COPY . /esphome
 RUN uv pip install --no-cache-dir -e /esphome
+# The tree has no .git; lets SBOM manifests describe it as the release it was built from
+ENV ESPHOME_IS_DOCKER_IMAGE=true

esphome/codegen.py

@@ -49,6 +49,7 @@ from esphome.cpp_generator import (  # noqa: F401
     static_const_array,
     static_function,
     templatable,
+    unique_global_id,
     variable,
     with_local_variable,
 )

esphome/components/alarm_control_panel/__init__.py

@@ -91,17 +91,19 @@ def alarm_control_panel_schema(
 
 
 ALARM_CONTROL_PANEL_ACTION_SCHEMA = maybe_simple_id(
+    {
+        cv.GenerateID(): cv.use_id(AlarmControlPanel),
+    }
+)
+
+ALARM_CONTROL_PANEL_CODE_ACTION_SCHEMA = maybe_simple_id(
     {
         cv.GenerateID(): cv.use_id(AlarmControlPanel),
         cv.Optional(CONF_CODE): cv.templatable(cv.string),
     }
 )
 
-ALARM_CONTROL_PANEL_CONDITION_SCHEMA = maybe_simple_id(
-    {
-        cv.GenerateID(): cv.use_id(AlarmControlPanel),
-    }
-)
+ALARM_CONTROL_PANEL_CONDITION_SCHEMA = ALARM_CONTROL_PANEL_ACTION_SCHEMA
 
 
 _CALLBACK_AUTOMATIONS = (
@@ -194,7 +196,7 @@ for _name, _arm in (
 ):
     automation.register_apply_action(
         _name,
-        ALARM_CONTROL_PANEL_ACTION_SCHEMA,
+        ALARM_CONTROL_PANEL_CODE_ACTION_SCHEMA,
         automation.ApplyCall(_arm),
         automation.ApplyField(CONF_CODE, "set_code", cg.std_string),
         call="make_call",

esphome/components/as7341/as7341.cpp

@@ -110,12 +110,12 @@ bool AS7341Component::read_channels(uint16_t *data) {
   this->set_smux_low_channels(true);
   this->enable_spectral_measurement(true);
   this->wait_for_data();
-  bool low_success = this->read_bytes_16(AS7341_CH0_DATA_L, data, 6);
+  bool low_success = this->read_bytes_16_le(AS7341_CH0_DATA_L, data, 6);
 
   this->set_smux_low_channels(false);
   this->enable_spectral_measurement(true);
   this->wait_for_data();
-  bool high_sucess = this->read_bytes_16(AS7341_CH0_DATA_L, &data[6], 6);
+  bool high_sucess = this->read_bytes_16_le(AS7341_CH0_DATA_L, &data[6], 6);
 
   return low_success && high_sucess;
 }

esphome/components/beken_spi_led_strip/led_strip.cpp

@@ -293,14 +293,9 @@ void BekenSPILEDStripLightOutput::set_led_params(uint8_t bit0, uint8_t bit1, uin
 }
 
 void BekenSPILEDStripLightOutput::write_state(light::LightState *state) {
-  // protect from refreshing too often
   uint32_t now = micros();
-  if (this->max_refresh_rate_.has_value() && *this->max_refresh_rate_ != 0 &&
-      (now - this->last_refresh_) < *this->max_refresh_rate_) {
-    // try again next loop iteration, so that this change won't get lost
-    this->schedule_show();
+  if (this->throttle_show_(now, this->last_refresh_, this->max_refresh_rate_))
     return;
-  }
   this->last_refresh_ = now;
   this->mark_shown_();
 
@@ -369,17 +364,6 @@ void BekenSPILEDStripLightOutput::write_state(light::LightState *state) {
   this->status_clear_warning();
 }
 
-light::ESPColorView BekenSPILEDStripLightOutput::get_view_internal(int32_t index) const {
-  const light::ChannelColors &colors = this->channel_colors_;
-  uint8_t *led = this->buf_ + (index * colors.bytes_per_led());
-  return {led + colors.r,
-          led + colors.g,
-          led + colors.b,
-          colors.has_white() ? led + colors.w : nullptr,
-          &this->effect_data_[index],
-          &this->correction_};
-}
-
 void BekenSPILEDStripLightOutput::dump_config() {
   ESP_LOGCONFIG(TAG,
                 "Beken SPI LED Strip:\n"
@@ -390,7 +374,7 @@ void BekenSPILEDStripLightOutput::dump_config() {
                 "  Channel colors: %s\n"
                 "  Max refresh rate: %" PRIu32 "\n"
                 "  Number of LEDs: %u",
-                this->channel_colors_.to_string(channel_colors), this->max_refresh_rate_.value_or(0), this->num_leds_);
+                this->channel_colors_.to_string(channel_colors), this->max_refresh_rate_, this->num_leds_);
 }
 
 float BekenSPILEDStripLightOutput::get_setup_priority() const { return setup_priority::HARDWARE; }

esphome/components/beken_spi_led_strip/led_strip.h

@@ -18,15 +18,7 @@ class BekenSPILEDStripLightOutput final : public light::AddressableLight {
   float get_setup_priority() const override;
 
   int32_t size() const override { return this->num_leds_; }
-  light::LightTraits get_traits() override {
-    auto traits = light::LightTraits();
-    if (this->channel_colors_.has_white()) {
-      traits.set_supported_color_modes({light::ColorMode::RGB_WHITE, light::ColorMode::WHITE});
-    } else {
-      traits.set_supported_color_modes({light::ColorMode::RGB});
-    }
-    return traits;
-  }
+  light::LightTraits get_traits() override { return this->channel_colors_.traits(); }
 
   void set_pin(uint8_t pin) { this->pin_ = pin; }
   void set_num_leds(uint16_t num_leds) { this->num_leds_ = num_leds; }
@@ -38,14 +30,16 @@ class BekenSPILEDStripLightOutput final : public light::AddressableLight {
   void set_led_params(uint8_t bit0, uint8_t bit1, uint32_t spi_frequency);
 
   void clear_effect_data() override {
-    for (int i = 0; i < this->size(); i++)
-      this->effect_data_[i] = 0;
+    if (this->effect_data_ != nullptr)
+      memset(this->effect_data_, 0, this->num_leds_);
   }
 
   void dump_config() override;
 
  protected:
-  light::ESPColorView get_view_internal(int32_t index) const override;
+  light::ESPColorView get_view_internal(int32_t index) const override {
+    return this->channel_colors_.view(this->buf_, this->effect_data_, index, &this->correction_);
+  }
 
   size_t get_buffer_size_() const { return this->num_leds_ * this->channel_colors_.bytes_per_led(); }
 
@@ -62,7 +56,7 @@ class BekenSPILEDStripLightOutput final : public light::AddressableLight {
   light::ChannelColors channel_colors_{0, 1, 2, light::ChannelColors::NO_WHITE};
 
   uint32_t last_refresh_{0};
-  optional<uint32_t> max_refresh_rate_{};
+  uint32_t max_refresh_rate_{0};
 };
 
 }  // namespace esphome::beken_spi_led_strip

esphome/components/climate_ir/climate_ir.cpp

@@ -24,7 +24,9 @@ climate::ClimateTraits ClimateIR::traits() {
   return traits;
 }
 
-void ClimateIR::setup() {
+void ClimateIR::setup() { this->setup_(this->restore_state_()); }
+
+void ClimateIR::setup_(optional<climate::ClimateDeviceRestoreState> restore) {
   if (this->sensor_) {
     this->sensor_->add_on_state_callback([this](float state) {
       this->current_temperature = state;
@@ -43,7 +45,6 @@ void ClimateIR::setup() {
   }
 
   // restore set points
-  auto restore = this->restore_state_();
   if (restore.has_value()) {
     restore->apply(this);
   } else {

esphome/components/climate_ir/climate_ir.h

@@ -60,6 +60,10 @@ class ClimateIR : public Component,
   /// Transmit via IR the state of this climate controller.
   virtual void transmit_state() = 0;
 
+  /// setup() with the restored state already loaded, for subclasses that read it first. Load it only once:
+  /// every restore_state_() call takes a new preference slot on some platforms.
+  void setup_(optional<climate::ClimateDeviceRestoreState> restore);
+
   // Dummy implement on_receive so implementation is optional for inheritors
   bool on_receive(remote_base::RemoteReceiveData data) override { return false; };
 

esphome/components/const/__init__.py

@@ -19,6 +19,7 @@ CONF_BITRATE = "bitrate"
 CONF_BLUE_CHANNEL = "blue_channel"
 CONF_BORDER_ROUTER = "border_router"
 CONF_BREATH_VOC_EQUIVALENT = "breath_voc_equivalent"
+CONF_BUTTONS = "buttons"
 CONF_BYTE_ORDER = "byte_order"
 CONF_CHANNEL_COLORS = "channel_colors"
 CONF_CLEAR_CHANNEL = "clear_channel"
@@ -53,6 +54,7 @@ CONF_LIBRETINY = "libretiny"
 CONF_LOOP = "loop"
 CONF_MANUFACTURER = "manufacturer"
 CONF_MAX_PACKET_SIZE = "max_packet_size"
+CONF_MODEM = "modem"
 CONF_MODULATION = "modulation"
 CONF_NOX_INDEX = "nox_index"
 CONF_ON_PACKET = "on_packet"

esphome/components/demo/__init__.py

@@ -18,6 +18,7 @@ from esphome.components import (
     text_sensor,
     valve,
 )
+from esphome.components.const import CONF_BUTTONS
 import esphome.config_validation as cv
 from esphome.const import (
     CONF_ACCURACY_DECIMALS,
@@ -167,7 +168,6 @@ VALVE_TYPES = {
 
 
 CONF_ALARM_CONTROL_PANELS = "alarm_control_panels"
-CONF_BUTTONS = "buttons"
 CONF_CLIMATES = "climates"
 CONF_COVERS = "covers"
 CONF_DATETIMES = "datetimes"

esphome/components/ecocomfort2/ecocomfort2_fan.cpp

@@ -1,5 +1,5 @@
 #include "esphome/core/defines.h"
-#ifdef USE_ESP32
+#if defined(USE_ESP32) && defined(USE_FAN)
 #include "ecocomfort2_fan.h"
 #include "esphome/core/log.h"
 
@@ -138,4 +138,4 @@ void Ecocomfort2Fan::on_status() {
 
 }  // namespace esphome::ecocomfort2
 
-#endif  // USE_ESP32
+#endif  // USE_ESP32 && USE_FAN

esphome/components/ecocomfort2/ecocomfort2_fan.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "esphome/core/defines.h"
-#ifdef USE_ESP32
+#if defined(USE_ESP32) && defined(USE_FAN)
 
 #include "esphome/core/component.h"
 #include "esphome/components/fan/fan.h"
@@ -23,4 +23,4 @@ class Ecocomfort2Fan : public fan::Fan, public Ecocomfort2Client, public Compone
 
 }  // namespace esphome::ecocomfort2
 
-#endif  // USE_ESP32
+#endif  // USE_ESP32 && USE_FAN

esphome/components/esp32/__init__.py

@@ -56,6 +56,7 @@ from esphome.const import (
 from esphome.core import CORE, EsphomeError, HexInt
 from esphome.core.config import BOARD_MAX_LENGTH
 from esphome.coroutine import CoroPriority, coroutine_with_priority
+from esphome.espidf import sbom
 from esphome.espidf.component import IDFComponent, generate_idf_components
 import esphome.final_validate as fv
 from esphome.helpers import copy_file_if_changed, rmtree, write_file_if_changed
@@ -3993,6 +3994,7 @@ def copy_files():
         CORE.relative_build_path("version.txt"),
         __version__,
     )
+    sbom.write_manifests()
 
     # Remote extra build files are fetched into the shared download cache in
     # one parallel batch (conditional requests skip unchanged files), then

esphome/components/esp32_ble/ble.cpp

@@ -667,6 +667,7 @@ void ESP32BLE::gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_pa
     }
 
     // Ignore these GAP events as they are not relevant for our use case
+    case ESP_GAP_BLE_KEY_EVT:
     case ESP_GAP_BLE_SET_PKT_LENGTH_COMPLETE_EVT:
     case ESP_GAP_BLE_PHY_UPDATE_COMPLETE_EVT:       // BLE 5.0 PHY update complete
     case ESP_GAP_BLE_CHANNEL_SELECT_ALGORITHM_EVT:  // BLE 5.0 channel selection algorithm

esphome/components/esp32_hosted/__init__.py

@@ -289,7 +289,9 @@ async def to_code(config: ConfigType) -> None:
         )
     if config[CONF_USE_PSRAM]:
         esp32.add_idf_sdkconfig_option("CONFIG_EH_HOST_PORT_DMA_PREFER_SPIRAM", True)
-        esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_DFLT_TASK_FROM_SPIRAM", True)
+        # Task stacks in PSRAM corrupt the heap on esp_hosted 3.0.10
+        # (esp-hosted-mcu#254); revisit with the next esp_hosted release
+        esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_DFLT_TASK_FROM_SPIRAM", False)
 
     # ESP-NOW-over-hosted shim: only the radio-less ESP32-P4 host needs it (see
     # the note by _MAX_CUSTOM_MSG_HANDLERS). Enabled for every P4 host, not

esphome/components/esp32_rmt_led_strip/led_strip.cpp

@@ -157,14 +157,9 @@ void ESP32RMTLEDStripLightOutput::set_led_params(uint32_t bit0_high, uint32_t bi
 }
 
 void ESP32RMTLEDStripLightOutput::write_state(light::LightState *state) {
-  // protect from refreshing too often
   uint32_t now = micros();
-  auto rate = this->max_refresh_rate_.value_or(0);
-  if (rate != 0 && (now - this->last_refresh_) < rate) {
-    // try again next loop iteration, so that this change won't get lost
-    this->schedule_show();
+  if (this->throttle_show_(now, this->last_refresh_, this->max_refresh_rate_))
     return;
-  }
   this->last_refresh_ = now;
   this->mark_shown_();
 
@@ -217,17 +212,6 @@ void ESP32RMTLEDStripLightOutput::write_state(light::LightState *state) {
   }
 }
 
-light::ESPColorView ESP32RMTLEDStripLightOutput::get_view_internal(int32_t index) const {
-  const light::ChannelColors &colors = this->channel_colors_;
-  uint8_t *led = this->buf_ + (index * colors.bytes_per_led());
-  return {led + colors.r,
-          led + colors.g,
-          led + colors.b,
-          colors.has_white() ? led + colors.w : nullptr,
-          &this->effect_data_[index],
-          &this->correction_};
-}
-
 void ESP32RMTLEDStripLightOutput::dump_config() {
   ESP_LOGCONFIG(TAG,
                 "ESP32 RMT LED Strip:\n"
@@ -239,7 +223,7 @@ void ESP32RMTLEDStripLightOutput::dump_config() {
                 "  Channel colors: %s\n"
                 "  Max refresh rate: %" PRIu32 "\n"
                 "  Number of LEDs: %u",
-                this->channel_colors_.to_string(channel_colors), this->max_refresh_rate_.value_or(0), this->num_leds_);
+                this->channel_colors_.to_string(channel_colors), this->max_refresh_rate_, this->num_leds_);
 }
 
 float ESP32RMTLEDStripLightOutput::get_setup_priority() const { return setup_priority::HARDWARE; }

esphome/components/esp32_rmt_led_strip/led_strip.h

@@ -29,15 +29,7 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight {
   float get_setup_priority() const override;
 
   int32_t size() const override { return this->num_leds_; }
-  light::LightTraits get_traits() override {
-    auto traits = light::LightTraits();
-    if (this->channel_colors_.has_white()) {
-      traits.set_supported_color_modes({light::ColorMode::RGB_WHITE, light::ColorMode::WHITE});
-    } else {
-      traits.set_supported_color_modes({light::ColorMode::RGB});
-    }
-    return traits;
-  }
+  light::LightTraits get_traits() override { return this->channel_colors_.traits(); }
 
   void set_pin(uint8_t pin) { this->pin_ = pin; }
   void set_inverted(bool inverted) { this->invert_out_ = inverted; }
@@ -55,14 +47,16 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight {
   void set_rmt_symbols(uint32_t rmt_symbols) { this->rmt_symbols_ = rmt_symbols; }
 
   void clear_effect_data() override {
-    for (int i = 0; i < this->size(); i++)
-      this->effect_data_[i] = 0;
+    if (this->effect_data_ != nullptr)
+      memset(this->effect_data_, 0, this->num_leds_);
   }
 
   void dump_config() override;
 
  protected:
-  light::ESPColorView get_view_internal(int32_t index) const override;
+  light::ESPColorView get_view_internal(int32_t index) const override {
+    return this->channel_colors_.view(this->buf_, this->effect_data_, index, &this->correction_);
+  }
 
   size_t get_buffer_size_() const { return this->num_leds_ * this->channel_colors_.bytes_per_led(); }
 
@@ -86,7 +80,7 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight {
   light::ChannelColors channel_colors_{0, 1, 2, light::ChannelColors::NO_WHITE};
 
   uint32_t last_refresh_{0};
-  optional<uint32_t> max_refresh_rate_{};
+  uint32_t max_refresh_rate_{0};
 };
 
 }  // namespace esphome::esp32_rmt_led_strip

esphome/components/esphome/ota/__init__.py

@@ -30,6 +30,13 @@ from esphome.const import (
     CONF_SAFE_MODE,
     CONF_VERSION,
     CONF_WEB_SERVER,
+    PLATFORM_BK72XX,
+    PLATFORM_ESP32,
+    PLATFORM_ESP8266,
+    PLATFORM_HOST,
+    PLATFORM_LN882X,
+    PLATFORM_RP2,
+    PLATFORM_RTL87XX,
 )
 from esphome.core import CORE, coroutine_with_priority
 from esphome.coroutine import CoroPriority
@@ -276,6 +283,31 @@ def _validate_no_password_with_encryption(config: ConfigType) -> ConfigType:
     )
 
 
+# Platforms with an OTA flash backend (ota_backend_factory.h)
+_SUPPORTED_PLATFORMS = [
+    PLATFORM_ESP8266,
+    PLATFORM_ESP32,
+    PLATFORM_RP2,
+    PLATFORM_BK72XX,
+    PLATFORM_LN882X,
+    PLATFORM_RTL87XX,
+    PLATFORM_HOST,
+]
+
+
+def _validate_platform(config: ConfigType) -> ConfigType:
+    if CORE.target_platform in _SUPPORTED_PLATFORMS:
+        return config
+    hint = (
+        f"; use '{CONF_PLATFORM}: zephyr_mcumgr' for OTA updates on nRF52"
+        if CORE.is_nrf52
+        else ""
+    )
+    raise cv.Invalid(
+        f"'{CONF_PLATFORM}: {DOMAIN}' is not supported on {CORE.target_platform}{hint}"
+    )
+
+
 def _consume_ota_sockets(config: ConfigType) -> ConfigType:
     """Register socket needs for OTA component."""
     from esphome.components import socket
@@ -287,6 +319,7 @@ def _consume_ota_sockets(config: ConfigType) -> ConfigType:
 
 
 CONFIG_SCHEMA = cv.All(
+    _validate_platform,
     cv.Schema(
         {
             cv.GenerateID(): cv.declare_id(ESPHomeOTAComponent),

esphome/components/external_components/__init__.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass, field
 import logging
 from pathlib import Path
 from typing import Any
@@ -24,6 +25,33 @@ _LOGGER = logging.getLogger(__name__)
 DOMAIN = "external_components"
 
 
+@dataclass
+class ExternalSource:
+    """Origin of the components found in one source's components directory."""
+
+    components_dir: Path
+    url: str | None = None
+    ref: str | None = None
+    # Root of the git checkout holding components_dir
+    repo_dir: Path | None = None
+
+
+@dataclass
+class ExternalComponentsData:
+    sources: list[ExternalSource] = field(default_factory=list)
+
+
+def _get_data() -> ExternalComponentsData:
+    if DOMAIN not in CORE.data:
+        CORE.data[DOMAIN] = ExternalComponentsData()
+    return CORE.data[DOMAIN]
+
+
+def get_sources() -> list[ExternalSource]:
+    """Return the sources processed for this configuration."""
+    return _get_data().sources
+
+
 CONFIG_SCHEMA = cv.ensure_list(
     {
         cv.Required(CONF_SOURCE): cv.SOURCE_SCHEMA,
@@ -67,6 +95,15 @@ def _process_git_config(config: dict[str, Any], refresh: TimePeriodSeconds) -> P
             "Could not find components folder for source. Please check the source contains a 'components' or 'esphome/components' folder"
         )
 
+    get_sources().append(
+        ExternalSource(
+            components_dir=components_dir,
+            # Ends up in SBOMs that are shared outside
+            url=git.strip_url_credentials(config[CONF_URL]),
+            ref=config.get(CONF_REF),
+            repo_dir=repo_dir,
+        )
+    )
     return components_dir
 
 
@@ -106,6 +143,7 @@ def _process_single_config(config: dict[str, Any]) -> None:
             )
     elif conf[CONF_TYPE] == TYPE_LOCAL:
         components_dir = Path(CORE.relative_config_path(conf[CONF_PATH]))
+        get_sources().append(ExternalSource(components_dir=components_dir))
     else:
         raise NotImplementedError
 

esphome/components/ezo_pmp/ezo_pmp.cpp

@@ -48,12 +48,16 @@ void EzoPMP::update() {
   }
 
   if (this->is_first_read_) {
+#ifdef USE_TEXT_SENSOR
     this->queue_command_(EZO_PMP_COMMAND_READ_CALIBRATION_STATUS, 0, 0, (bool) this->calibration_status_);
+#endif
+#ifdef USE_SENSOR
     this->queue_command_(EZO_PMP_COMMAND_READ_MAX_FLOW_RATE, 0, 0, (bool) this->max_flow_rate_);
     this->queue_command_(EZO_PMP_COMMAND_READ_SINGLE_REPORT, 0, 0, (bool) this->current_volume_dosed_);
     this->queue_command_(EZO_PMP_COMMAND_READ_TOTAL_VOLUME_DOSED, 0, 0, (bool) this->total_volume_dosed_);
     this->queue_command_(EZO_PMP_COMMAND_READ_ABSOLUTE_TOTAL_VOLUME_DOSED, 0, 0,
                          (bool) this->absolute_total_volume_dosed_);
+#endif
     this->queue_command_(EZO_PMP_COMMAND_READ_PAUSE_STATUS, 0, 0, true);
     this->is_first_read_ = false;
   }
@@ -61,6 +65,7 @@ void EzoPMP::update() {
   if (!this->is_waiting_ && this->peek_next_command_() == EZO_PMP_COMMAND_NONE) {
     this->queue_command_(EZO_PMP_COMMAND_READ_DOSING, 0, 0, true);
 
+#ifdef USE_SENSOR
     if (this->is_dosing_flag_) {
       this->queue_command_(EZO_PMP_COMMAND_READ_SINGLE_REPORT, 0, 0, (bool) this->current_volume_dosed_);
       this->queue_command_(EZO_PMP_COMMAND_READ_TOTAL_VOLUME_DOSED, 0, 0, (bool) this->total_volume_dosed_);
@@ -69,6 +74,7 @@ void EzoPMP::update() {
     }
 
     this->queue_command_(EZO_PMP_COMMAND_READ_PUMP_VOLTAGE, 0, 0, (bool) this->pump_voltage_);
+#endif
   } else {
     ESP_LOGV(TAG, "Not Scheduling new Command during update()");
   }
@@ -192,51 +198,68 @@ void EzoPMP::read_command_result_() {
       if (parsed_third_parameter.has_value())
         this->is_dosing_flag_ = parsed_third_parameter.value_or(0) == 1;
 
+#ifdef USE_BINARY_SENSOR
       if (this->is_dosing_)
         this->is_dosing_->publish_state(this->is_dosing_flag_);
+#endif
 
+#ifdef USE_SENSOR
       if (parsed_second_parameter.has_value() && this->last_volume_requested_) {
         this->last_volume_requested_->publish_state(parsed_second_parameter.value_or(0));
       }
+#endif
 
+#ifdef USE_TEXT_SENSOR
       if (!this->is_dosing_flag_ && !this->is_paused_flag_) {
         // If pump is not paused and not dispensing
         if (this->dosing_mode_ && this->dosing_mode_->state != DOSING_MODE_NONE)
           this->dosing_mode_->publish_state(DOSING_MODE_NONE);
       }
+#endif
 
       break;
 
     case EZO_PMP_COMMAND_READ_SINGLE_REPORT:  // Single Report (page 53)
+#ifdef USE_SENSOR
       if (parsed_first_parameter.has_value() && (bool) this->current_volume_dosed_) {
         this->current_volume_dosed_->publish_state(parsed_first_parameter.value_or(0));
       }
+#endif
       break;
 
     case EZO_PMP_COMMAND_READ_MAX_FLOW_RATE:  // Constant Flow Rate (page 57)
+#ifdef USE_SENSOR
       if (parsed_second_parameter.has_value() && this->max_flow_rate_)
         this->max_flow_rate_->publish_state(parsed_second_parameter.value_or(0));
+#endif
       break;
 
     case EZO_PMP_COMMAND_READ_PAUSE_STATUS:  // Pause (page 61)
       if (parsed_second_parameter.has_value())
         this->is_paused_flag_ = parsed_second_parameter.value_or(0) == 1;
 
+#ifdef USE_BINARY_SENSOR
       if (this->is_paused_)
         this->is_paused_->publish_state(this->is_paused_flag_);
+#endif
       break;
 
     case EZO_PMP_COMMAND_READ_TOTAL_VOLUME_DOSED:  // Total Volume Dispensed (page 64)
+#ifdef USE_SENSOR
       if (parsed_second_parameter.has_value() && this->total_volume_dosed_)
         this->total_volume_dosed_->publish_state(parsed_second_parameter.value_or(0));
+#endif
       break;
 
     case EZO_PMP_COMMAND_READ_ABSOLUTE_TOTAL_VOLUME_DOSED:  // Total Volume Dispensed (page 64)
+#ifdef USE_SENSOR
       if (parsed_second_parameter.has_value() && this->absolute_total_volume_dosed_)
         this->absolute_total_volume_dosed_->publish_state(parsed_second_parameter.value_or(0));
+#endif
       break;
 
     case EZO_PMP_COMMAND_READ_CALIBRATION_STATUS:  // Calibration (page 65)
+#ifdef USE_TEXT_SENSOR
       if (parsed_second_parameter.has_value() && this->calibration_status_) {
         if (parsed_second_parameter.value_or(0) == 1) {
           this->calibration_status_->publish_state("Fixed Volume");
@@ -248,41 +271,56 @@ void EzoPMP::read_command_result_() {
           this->calibration_status_->publish_state("Uncalibrated");
         }
       }
+#endif
       break;
 
     case EZO_PMP_COMMAND_READ_PUMP_VOLTAGE:  // Pump Voltage (page 67)
+#ifdef USE_SENSOR
       if (parsed_second_parameter.has_value() && this->pump_voltage_)
         this->pump_voltage_->publish_state(parsed_second_parameter.value_or(0));
+#endif
       break;
 
       // Non-Read Commands
 
     case EZO_PMP_COMMAND_DOSE_VOLUME:  // Volume Dispensing (page 55)
+#ifdef USE_TEXT_SENSOR
       if (this->dosing_mode_ && this->dosing_mode_->state != DOSING_MODE_VOLUME)
         this->dosing_mode_->publish_state(DOSING_MODE_VOLUME);
+#endif
       break;
 
     case EZO_PMP_COMMAND_DOSE_VOLUME_OVER_TIME:  // Dose over time (page 56)
+#ifdef USE_TEXT_SENSOR
       if (this->dosing_mode_ && this->dosing_mode_->state != DOSING_MODE_VOLUME_OVER_TIME)
         this->dosing_mode_->publish_state(DOSING_MODE_VOLUME_OVER_TIME);
+#endif
       break;
 
     case EZO_PMP_COMMAND_DOSE_WITH_CONSTANT_FLOW_RATE:  // Constant Flow Rate (page 57)
+#ifdef USE_TEXT_SENSOR
       if (this->dosing_mode_ && this->dosing_mode_->state != DOSING_MODE_CONSTANT_FLOW_RATE)
         this->dosing_mode_->publish_state(DOSING_MODE_CONSTANT_FLOW_RATE);
+#endif
       break;
 
     case EZO_PMP_COMMAND_DOSE_CONTINUOUSLY:  // Continuous Dispensing (page 54)
+#ifdef USE_TEXT_SENSOR
       if (this->dosing_mode_ && this->dosing_mode_->state != DOSING_MODE_CONTINUOUS)
         this->dosing_mode_->publish_state(DOSING_MODE_CONTINUOUS);
+#endif
       break;
 
     case EZO_PMP_COMMAND_STOP_DOSING:  // Stop (page 62)
       this->is_paused_flag_ = false;
+#ifdef USE_BINARY_SENSOR
       if (this->is_paused_)
         this->is_paused_->publish_state(this->is_paused_flag_);
+#endif
+#ifdef USE_TEXT_SENSOR
       if (this->dosing_mode_ && this->dosing_mode_->state != DOSING_MODE_NONE)
         this->dosing_mode_->publish_state(DOSING_MODE_NONE);
+#endif
       break;
 
     case EZO_PMP_COMMAND_EXEC_ARBITRARY_COMMAND_ADDRESS:
@@ -489,25 +527,33 @@ void EzoPMP::find() { this->queue_command_(EZO_PMP_COMMAND_FIND, 0, 0, true); }
 void EzoPMP::dose_continuously() {
   this->queue_command_(EZO_PMP_COMMAND_DOSE_CONTINUOUSLY, 0, 0, true);
   this->queue_command_(EZO_PMP_COMMAND_READ_DOSING, 0, 0, true);
+#ifdef USE_SENSOR
   this->queue_command_(EZO_PMP_COMMAND_READ_SINGLE_REPORT, 0, 0, (bool) this->current_volume_dosed_);
+#endif
 }
 
 void EzoPMP::dose_volume(double volume) {
   this->queue_command_(EZO_PMP_COMMAND_DOSE_VOLUME, volume, 0, true);
   this->queue_command_(EZO_PMP_COMMAND_READ_DOSING, 0, 0, true);
+#ifdef USE_SENSOR
   this->queue_command_(EZO_PMP_COMMAND_READ_SINGLE_REPORT, 0, 0, (bool) this->current_volume_dosed_);
+#endif
 }
 
 void EzoPMP::dose_volume_over_time(double volume, int duration) {
   this->queue_command_(EZO_PMP_COMMAND_DOSE_VOLUME_OVER_TIME, volume, duration, true);
   this->queue_command_(EZO_PMP_COMMAND_READ_DOSING, 0, 0, true);
+#ifdef USE_SENSOR
   this->queue_command_(EZO_PMP_COMMAND_READ_SINGLE_REPORT, 0, 0, (bool) this->current_volume_dosed_);
+#endif
 }
 
 void EzoPMP::dose_with_constant_flow_rate(double volume, int duration) {
   this->queue_command_(EZO_PMP_COMMAND_DOSE_WITH_CONSTANT_FLOW_RATE, volume, duration, true);
   this->queue_command_(EZO_PMP_COMMAND_READ_DOSING, 0, 0, true);
+#ifdef USE_SENSOR
   this->queue_command_(EZO_PMP_COMMAND_READ_SINGLE_REPORT, 0, 0, (bool) this->current_volume_dosed_);
+#endif
 }
 
 void EzoPMP::set_calibration_volume(double volume) {

esphome/components/gpio_expander/__init__.py

@@ -22,3 +22,9 @@ def validate_interrupt_pin(value: ConfigType) -> ConfigType:
             f"Remove the '{CONF_INTERRUPT_PIN}' to fall back to polling."
         )
     return value
+
+
+# Schema of a CachedGpioExpanderComponent; to_code sets the pin with cg.add_pin().
+INTERRUPT_EXPANDER_SCHEMA = cv.COMPONENT_SCHEMA.extend(
+    {cv.Optional(CONF_INTERRUPT_PIN): validate_interrupt_pin}
+)

esphome/components/gpio_expander/cached_gpio_component.cpp

@@ -0,0 +1,29 @@
+#include "cached_gpio_component.h"
+
+namespace esphome::gpio_expander {
+
+// Not a member of the template: GCC 10 (ESP8266) ignores section attributes on template instantiations.
+static void IRAM_ATTR interrupt_pin_isr(Component *arg) { arg->enable_loop_soon_any_context(); }
+
+bool InterruptExpanderComponent::attach_interrupt_pin_() {
+  // Polling mode enables the loop in pin_mode() for the first input; interrupt mode from the ISR.
+  this->disable_loop();
+  if (this->interrupt_pin_ == nullptr)
+    return false;
+  this->interrupt_pin_->setup();
+  this->interrupt_pin_->attach_interrupt(&interrupt_pin_isr, static_cast<Component *>(this),
+                                         gpio::INTERRUPT_FALLING_EDGE);
+  // An INT line already low at attach gives no falling edge, so read the inputs now
+  if (!this->interrupt_pin_->digital_read())
+    this->enable_loop();
+  return true;
+}
+
+// Input changes that straddle the read leave INT asserted without a new falling edge, so keep looping
+// until the line is released or the expander would keep stale state forever.
+void InterruptExpanderComponent::disable_loop_when_released_() {
+  if (this->interrupt_pin_->digital_read())
+    this->disable_loop();
+}
+
+}  // namespace esphome::gpio_expander

esphome/components/gpio_expander/cached_gpio_component.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "esphome/components/gpio_expander/cached_gpio.h"
+#include "esphome/core/component.h"
+#include "esphome/core/hal.h"
+
+namespace esphome::gpio_expander {
+
+/// Component side of a cached GPIO expander with an optional active-low INT line. Not a template, so the
+/// ISR and the shared code are emitted once for every expander.
+class InterruptExpanderComponent : public Component {
+ public:
+  void set_interrupt_pin(InternalGPIOPin *pin) { this->interrupt_pin_ = pin; }
+  float get_setup_priority() const override { return setup_priority::IO; }
+
+ protected:
+  /// Attach the falling-edge ISR when there is an interrupt pin, then disable the loop until an input
+  /// needs it. Returns whether the interrupt pin is in use.
+  bool attach_interrupt_pin_();
+  /// The loop runs again after the INT line falls; stop it once the line is released (HIGH).
+  /// Only called with an interrupt pin.
+  void disable_loop_when_released_();
+  /// Call from pin_mode() for an input pin: without an interrupt pin the loop polls instead.
+  void enable_polling_loop_() {
+    if (this->interrupt_pin_ == nullptr)
+      this->enable_loop();
+  }
+
+  InternalGPIOPin *interrupt_pin_{nullptr};
+};
+
+/// Base for expanders that cache reads per loop. With an interrupt pin the cache stays valid until INT fires.
+template<typename T, uint16_t N>
+class CachedGpioExpanderComponent : public InterruptExpanderComponent, public CachedGpioExpander<T, N> {
+ public:
+  void loop() override {
+    this->reset_pin_cache_();
+    if (this->interrupt_pin_ != nullptr)
+      this->disable_loop_when_released_();
+  }
+
+ protected:
+  /// Call at the end of setup().
+  void setup_interrupt_pin_() {
+    if (this->attach_interrupt_pin_())
+      this->set_invalidate_on_read_(false);
+  }
+};
+
+}  // namespace esphome::gpio_expander

esphome/components/hitachi_ac344/hitachi_ac344.cpp

@@ -1,6 +1,8 @@
 #include "hitachi_ac344.h"
 #include "esphome/core/helpers.h"
 
+#include <cstring>
+
 namespace esphome::hitachi_ac344 {
 
 ESPHOME_LOG_TAG(TAG, "climate.hitachi_ac344");
@@ -101,21 +103,7 @@ void HitachiClimate::set_fan_(uint8_t speed) {
   remote_state_[29] = 0x01;
 }
 
-void HitachiClimate::set_swing_v_toggle_(bool on) {
-  uint8_t button = get_button_();  // Get the current button value.
-  if (on) {
-    button = HITACHI_AC344_BUTTON_SWINGV;              // Set the button to SwingV.
-  } else if (button == HITACHI_AC344_BUTTON_SWINGV) {  // Asked to unset it
-    // It was set previous, so use Power as a default
-    button = HITACHI_AC344_BUTTON_POWER;
-  }
-  set_button_(button);
-}
-
-bool HitachiClimate::get_swing_v_toggle_() { return get_button_() == HITACHI_AC344_BUTTON_SWINGV; }
-
 void HitachiClimate::set_swing_v_(bool on) {
-  set_swing_v_toggle_(on);  // Set the button value.
   assign_bits(this->remote_state_[HITACHI_AC344_SWINGV_BYTE], 1 << HITACHI_AC344_SWINGV_OFFSET, on);
 }
 
@@ -129,18 +117,59 @@ void HitachiClimate::set_swing_h_(uint8_t position) {
     return;
   }
   set_bits(&remote_state_[HITACHI_AC344_SWINGH_BYTE], HITACHI_AC344_SWINGH_OFFSET, HITACHI_AC344_SWINGH_SIZE, position);
-  set_button_(HITACHI_AC344_BUTTON_SWINGH);
 }
 
 uint8_t HitachiClimate::get_swing_h_() {
   return GETBITS8(remote_state_[HITACHI_AC344_SWINGH_BYTE], HITACHI_AC344_SWINGH_OFFSET, HITACHI_AC344_SWINGH_SIZE);
 }
 
-uint8_t HitachiClimate::get_button_() { return remote_state_[HITACHI_AC344_BUTTON_BYTE]; }
-
 void HitachiClimate::set_button_(uint8_t button) { remote_state_[HITACHI_AC344_BUTTON_BYTE] = button; }
 
-void HitachiClimate::transmit_state() {
+static bool swing_vertical(climate::ClimateSwingMode mode) {
+  return mode == climate::CLIMATE_SWING_VERTICAL || mode == climate::CLIMATE_SWING_BOTH;
+}
+
+bool HitachiClimate::get_mildew_proof() const {
+  return GETBIT8(this->remote_state_[HITACHI_AC344_MILDEWPROOF_BYTE], HITACHI_AC344_MILDEWPROOF_OFFSET);
+}
+
+void HitachiClimate::set_mildew_proof(bool on) {
+  // A repeated write resends the frame, so a missed burst can be retried
+  assign_bits(this->remote_state_[HITACHI_AC344_MILDEWPROOF_BYTE], 1 << HITACHI_AC344_MILDEWPROOF_OFFSET, on);
+  this->transmit_state_(HITACHI_AC344_BUTTON_MILDEWPROOF);
+  this->publish_mildew_proof_();
+}
+
+void HitachiClimate::publish_mildew_proof_() {
+#ifdef USE_SWITCH
+  if (this->mildew_proof_switch_ != nullptr)
+    this->mildew_proof_switch_->publish_state(this->get_mildew_proof());
+#endif
+}
+
+void HitachiClimate::setup() {
+  // Seed the frame before the base publishes the restored state; an on_state automation may transmit from that
+  // callback. Nothing is sent at boot, the restored settings ride along with the next frame.
+  auto restore = this->restore_state_();
+  if (restore.has_value()) {
+    set_swing_v_(swing_vertical(restore->swing_mode));
+  }
+#ifdef USE_SWITCH
+  // The restored value is the switch state, which an inverted switch flips relative to the frame bit
+  if (this->mildew_proof_switch_ != nullptr) {
+    if (auto initial = this->mildew_proof_switch_->get_initial_state_with_restore_mode()) {
+      assign_bits(this->remote_state_[HITACHI_AC344_MILDEWPROOF_BYTE], 1 << HITACHI_AC344_MILDEWPROOF_OFFSET,
+                  *initial != this->mildew_proof_switch_->is_inverted());
+    }
+  }
+#endif
+  this->setup_(restore);
+  this->publish_mildew_proof_();
+}
+
+void HitachiClimate::transmit_state() { this->transmit_state_(HITACHI_AC344_BUTTON_POWER); }
+
+void HitachiClimate::transmit_state_(uint8_t button) {
   switch (this->mode) {
     case climate::CLIMATE_MODE_COOL:
       set_mode_(HITACHI_AC344_MODE_COOL);
@@ -167,6 +196,9 @@ void HitachiClimate::transmit_state() {
   set_temp_(static_cast<uint8_t>(this->target_temperature));
 
   switch (this->fan_mode.value_or(climate::CLIMATE_FAN_ON)) {
+    case climate::CLIMATE_FAN_QUIET:
+      set_fan_(HITACHI_AC344_FAN_MIN);
+      break;
     case climate::CLIMATE_FAN_LOW:
       set_fan_(HITACHI_AC344_FAN_LOW);
       break;
@@ -182,27 +214,18 @@ void HitachiClimate::transmit_state() {
       set_fan_(HITACHI_AC344_FAN_AUTO);
   }
 
-  switch (this->swing_mode) {
-    case climate::CLIMATE_SWING_BOTH:
-      set_swing_v_(true);
-      set_swing_h_(HITACHI_AC344_SWINGH_AUTO);
-      break;
-    case climate::CLIMATE_SWING_VERTICAL:
-      set_swing_v_(true);
-      set_swing_h_(HITACHI_AC344_SWINGH_MIDDLE);
-      break;
-    case climate::CLIMATE_SWING_HORIZONTAL:
-      set_swing_v_(false);
-      set_swing_h_(HITACHI_AC344_SWINGH_AUTO);
-      break;
-    case climate::CLIMATE_SWING_OFF:
-      set_swing_v_(false);
-      set_swing_h_(HITACHI_AC344_SWINGH_MIDDLE);
-      break;
-  }
+  const bool swing_v = swing_vertical(this->swing_mode);
+  const bool swing_h =
+      this->swing_mode == climate::CLIMATE_SWING_HORIZONTAL || this->swing_mode == climate::CLIMATE_SWING_BOTH;
+  set_swing_h_(swing_h ? HITACHI_AC344_SWINGH_AUTO : HITACHI_AC344_SWINGH_MIDDLE);
+  // The unit ignores the swing bit and toggles on the swing button, so only a vertical swing change presses it;
+  // everything else sends the given button
+  set_button_(swing_v != get_swing_v_() ? HITACHI_AC344_BUTTON_SWINGV : button);
+  set_swing_v_(swing_v);
 
-  // TODO: find change value to set button, now always set to power button
-  set_button_(HITACHI_AC344_BUTTON_POWER);
+  // The remote sends this when powering off with mildew proof on
+  if (this->mode == climate::CLIMATE_MODE_OFF && this->get_mildew_proof())
+    remote_state_[9] = 0xA5;
 
   invert_byte_pairs(remote_state_ + 3, HITACHI_AC344_STATE_LENGTH - 3);
 
@@ -275,6 +298,8 @@ bool HitachiClimate::parse_fan_(const uint8_t remote_state[]) {
   ESP_LOGV(TAG, "Fan: %02X %02X", remote_state[HITACHI_AC344_FAN_BYTE], fan_mode);
   switch (fan_mode) {
     case HITACHI_AC344_FAN_MIN:
+      this->fan_mode = climate::CLIMATE_FAN_QUIET;
+      break;
     case HITACHI_AC344_FAN_LOW:
       this->fan_mode = climate::CLIMATE_FAN_LOW;
       break;
@@ -298,11 +323,13 @@ bool HitachiClimate::parse_swing_(const uint8_t remote_state[]) {
   uint8_t swing_modeh =
       GETBITS8(remote_state[HITACHI_AC344_SWINGH_BYTE], HITACHI_AC344_SWINGH_OFFSET, HITACHI_AC344_SWINGH_SIZE);
   ESP_LOGV(TAG, "SwingH: %02X %02X", remote_state[HITACHI_AC344_SWINGH_BYTE], swing_modeh);
+  bool swing_v = GETBIT8(remote_state[HITACHI_AC344_SWINGV_BYTE], HITACHI_AC344_SWINGV_OFFSET);
+  ESP_LOGV(TAG, "SwingV: %02X %s", remote_state[HITACHI_AC344_SWINGV_BYTE], ONOFF(swing_v));
 
-  if ((swing_modeh & 0x3) == 0x3) {
-    this->swing_mode = climate::CLIMATE_SWING_OFF;
+  if (swing_modeh != HITACHI_AC344_SWINGH_AUTO) {
+    this->swing_mode = swing_v ? climate::CLIMATE_SWING_VERTICAL : climate::CLIMATE_SWING_OFF;
   } else {
-    this->swing_mode = climate::CLIMATE_SWING_HORIZONTAL;
+    this->swing_mode = swing_v ? climate::CLIMATE_SWING_BOTH : climate::CLIMATE_SWING_HORIZONTAL;
   }
 
   return true;
@@ -345,9 +372,10 @@ bool HitachiClimate::on_receive(remote_base::RemoteReceiveData data) {
   this->parse_fan_(recv_state);
   // parse swingv
   this->parse_swing_(recv_state);
+  // Copy first so an on_state automation transmits from the received frame
+  memcpy(remote_state_, recv_state, HITACHI_AC344_STATE_LENGTH);
   this->publish_state();
-  for (uint8_t i = 0; i < HITACHI_AC344_STATE_LENGTH; i++)
-    remote_state_[i] = recv_state[i];
+  this->publish_mildew_proof_();
 
   return true;
 }

esphome/components/hitachi_ac344/hitachi_ac344.h

@@ -2,6 +2,9 @@
 
 #include "esphome/core/log.h"
 #include "esphome/components/climate_ir/climate_ir.h"
+#ifdef USE_SWITCH
+#include "esphome/components/switch/switch.h"
+#endif
 
 namespace esphome::hitachi_ac344 {
 
@@ -79,9 +82,18 @@ class HitachiClimate final : public climate_ir::ClimateIR {
  public:
   HitachiClimate()
       : climate_ir::ClimateIR(HITACHI_AC344_TEMP_MIN, HITACHI_AC344_TEMP_MAX, 1.0F, true, true,
-                              {climate::CLIMATE_FAN_AUTO, climate::CLIMATE_FAN_LOW, climate::CLIMATE_FAN_MEDIUM,
-                               climate::CLIMATE_FAN_HIGH},
-                              {climate::CLIMATE_SWING_OFF, climate::CLIMATE_SWING_HORIZONTAL}) {}
+                              {climate::CLIMATE_FAN_AUTO, climate::CLIMATE_FAN_QUIET, climate::CLIMATE_FAN_LOW,
+                               climate::CLIMATE_FAN_MEDIUM, climate::CLIMATE_FAN_HIGH},
+                              {climate::CLIMATE_SWING_OFF, climate::CLIMATE_SWING_VERTICAL,
+                               climate::CLIMATE_SWING_HORIZONTAL, climate::CLIMATE_SWING_BOTH}) {}
+  bool get_mildew_proof() const;
+  /// Mildew proof keeps the fan running after power off to dry the unit
+  void set_mildew_proof(bool on);
+#ifdef USE_SWITCH
+  void set_mildew_proof_switch(switch_::Switch *sw) { this->mildew_proof_switch_ = sw; }
+#endif
+
+  void setup() override;
 
  protected:
   uint8_t remote_state_[HITACHI_AC344_STATE_LENGTH]{0x01, 0x10, 0x00, 0x40, 0x00, 0xFF, 0x00, 0xCC, 0x00, 0x00, 0x00,
@@ -89,8 +101,14 @@ class HitachiClimate final : public climate_ir::ClimateIR {
                                                     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00,
                                                     0x80, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
   uint8_t previous_temp_{27};
+#ifdef USE_SWITCH
+  switch_::Switch *mildew_proof_switch_{nullptr};
+#endif
   // Transmit via IR the state of this climate controller.
   void transmit_state() override;
+  // The frame with the button byte the remote sends for this change
+  void transmit_state_(uint8_t button);
+  void publish_mildew_proof_();
   bool get_power_();
   void set_power_(bool on);
   uint8_t get_mode_();
@@ -98,13 +116,10 @@ class HitachiClimate final : public climate_ir::ClimateIR {
   void set_temp_(uint8_t celsius, bool set_previous = true);
   uint8_t get_fan_();
   void set_fan_(uint8_t speed);
-  void set_swing_v_toggle_(bool on);
-  bool get_swing_v_toggle_();
   void set_swing_v_(bool on);
   bool get_swing_v_();
   void set_swing_h_(uint8_t position);
   uint8_t get_swing_h_();
-  uint8_t get_button_();
   void set_button_(uint8_t button);
   // Handle received IR Buffer
   bool on_receive(remote_base::RemoteReceiveData data) override;

esphome/components/hitachi_ac344/switch/__init__.py

@@ -0,0 +1,30 @@
+import esphome.codegen as cg
+from esphome.components import switch
+import esphome.config_validation as cv
+from esphome.const import ENTITY_CATEGORY_CONFIG
+from esphome.types import ConfigType
+
+from ..climate import HitachiClimate
+
+CONF_HITACHI_AC344_ID = "hitachi_ac344_id"
+CONF_MILDEW_PROOF = "mildew_proof"
+
+CONFIG_SCHEMA = cv.Schema(
+    {
+        cv.GenerateID(CONF_HITACHI_AC344_ID): cv.use_id(HitachiClimate),
+        cv.Required(CONF_MILDEW_PROOF): switch.switch_schema(
+            switch.ApplySwitch,
+            icon="mdi:shield-check-outline",
+            entity_category=ENTITY_CATEGORY_CONFIG,
+            default_restore_mode="RESTORE_DEFAULT_OFF",
+        ),
+    }
+)
+
+
+async def to_code(config: ConfigType) -> None:
+    hub = await cg.get_variable(config[CONF_HITACHI_AC344_ID])
+    switches = switch.sub_switches(config, parent=hub)
+    await switches(
+        CONF_MILDEW_PROOF, hub.set_mildew_proof_switch, apply="set_mildew_proof(state)"
+    )

esphome/components/http_request/__init__.py

@@ -56,6 +56,7 @@ CONF_HTTP_REQUEST_ID = "http_request_id"
 CONF_USERAGENT = "useragent"
 CONF_FOLLOW_REDIRECTS = "follow_redirects"
 CONF_REDIRECT_LIMIT = "redirect_limit"
+CONF_MAX_CONCURRENT_REQUESTS = "max_concurrent_requests"
 CONF_BUFFER_SIZE_RX = "buffer_size_rx"
 CONF_BUFFER_SIZE_TX = "buffer_size_tx"
 CONF_TLS_BUFFER_SIZE_RX = "tls_buffer_size_rx"
@@ -144,6 +145,10 @@ CONFIG_SCHEMA = cv.All(
             ): cv.string,
             cv.Optional(CONF_FOLLOW_REDIRECTS, True): cv.boolean,
             cv.Optional(CONF_REDIRECT_LIMIT, 3): cv.int_,
+            # Each request holds a client, and a TLS client about 40 kB
+            cv.Optional(CONF_MAX_CONCURRENT_REQUESTS, default=1): cv.int_range(
+                min=1, max=4
+            ),
             cv.Optional(
                 CONF_TIMEOUT, default="4.5s"
             ): cv.positive_time_period_milliseconds,
@@ -193,6 +198,8 @@ async def to_code(config: ConfigType) -> None:
     cg.add(var.set_useragent(config[CONF_USERAGENT]))
     cg.add(var.set_follow_redirects(config[CONF_FOLLOW_REDIRECTS]))
     cg.add(var.set_redirect_limit(config[CONF_REDIRECT_LIMIT]))
+    if (max_concurrent := config[CONF_MAX_CONCURRENT_REQUESTS]) != 1:
+        cg.add(var.set_max_concurrent_requests(max_concurrent))
 
     if CORE.is_esp8266 and not config[CONF_ESP8266_DISABLE_SSL_SUPPORT]:
         cg.add_define("USE_HTTP_REQUEST_ESP8266_HTTPS")
@@ -312,19 +319,19 @@ HTTP_REQUEST_SEND_ACTION_SCHEMA = HTTP_REQUEST_ACTION_SCHEMA.extend(
     "http_request.get",
     HttpRequestSendAction,
     HTTP_REQUEST_GET_ACTION_SCHEMA,
-    synchronous=True,
+    synchronous=False,
 )
 @automation.register_action(
     "http_request.post",
     HttpRequestSendAction,
     HTTP_REQUEST_POST_ACTION_SCHEMA,
-    synchronous=True,
+    synchronous=False,
 )
 @automation.register_action(
     "http_request.send",
     HttpRequestSendAction,
     HTTP_REQUEST_SEND_ACTION_SCHEMA,
-    synchronous=True,
+    synchronous=False,
 )
 async def http_request_action_to_code(
     config: ConfigType,

esphome/components/http_request/http_request.cpp

@@ -1,5 +1,7 @@
 #include "http_request.h"
 
+#include "esphome/core/application.h"
+#include "esphome/core/hal.h"
 #include "esphome/core/log.h"
 
 #include <cinttypes>
@@ -14,8 +16,10 @@ void HttpRequestComponent::dump_config() {
                 "  Timeout: %" PRIu32 "ms\n"
                 "  User-Agent: %s\n"
                 "  Follow redirects: %s\n"
-                "  Redirect limit: %d",
-                this->timeout_, this->useragent_, YESNO(this->follow_redirects_), this->redirect_limit_);
+                "  Redirect limit: %d\n"
+                "  Max concurrent requests: %u",
+                this->timeout_, this->useragent_, YESNO(this->follow_redirects_), this->redirect_limit_,
+                this->max_concurrent_);
   if (this->watchdog_timeout_ > 0) {
     ESP_LOGCONFIG(TAG, "  Watchdog Timeout: %" PRIu32 "ms", this->watchdog_timeout_);
   }
@@ -32,4 +36,209 @@ std::string HttpContainer::get_response_header(const std::string &header_name) {
   return "";
 }
 
+void PendingRequest::free_buffer() {
+  if (this->buf != nullptr)
+    RAMAllocator<uint8_t>().deallocate(this->buf, this->buf_size);
+  this->buf = nullptr;
+}
+
+void HttpRequestComponent::setup() {
+  // An on_boot automation may already have queued a request
+  if (this->requests_ == nullptr)
+    this->disable_loop();
+}
+
+void HttpRequestComponent::loop() {
+  if (!this->process_())
+    this->disable_loop();
+}
+
+void HttpRequestComponent::on_safe_shutdown() {
+  // Called before any on_shutdown(). Drop the requests already made, so that they do not hold up the reboot;
+  // only the ones that on_shutdown automations make are sent.
+  this->shutting_down_ = true;
+  this->busy_++;
+  unsigned dropped = 0;
+  for (PendingRequest *request = this->requests_; request != nullptr; request = request->next) {
+    if (request->finished || request->cancelled)
+      continue;
+    dropped++;
+    request->cancelled = true;
+    // As script.stop would, so that a script that was waiting for it can run again from on_shutdown
+    request->handler(request, RequestEvent::REQUEST_EVENT_ABANDON);
+  }
+  this->busy_--;
+  if (dropped != 0) {
+    ESP_LOGW(TAG, "Shutting down; %u requests dropped", dropped);
+  }
+  if (this->busy_ == 0)
+    this->remove_dead_();
+}
+
+void HttpRequestComponent::on_shutdown() {
+  // Shutdown runs no loop(), so finish the requests here, blocking as they did before. A forced reboot calls
+  // only this, so it also finishes the requests made before it.
+  this->shutting_down_ = true;
+  this->drain_();
+}
+
+void HttpRequestComponent::submit(PendingRequest *request) {
+  this->busy_++;
+  uint8_t running = 0;
+  uint8_t waiting = 0;
+  PendingRequest **link = &this->requests_;
+  for (; *link != nullptr; link = &(*link)->next) {
+    const PendingRequest *other = *link;
+    if (other->finished || other->cancelled)
+      continue;
+    if (other->container == nullptr) {
+      waiting++;
+    } else {
+      running++;
+    }
+  }
+  *link = request;
+  this->enable_loop();
+  if (waiting >= MAX_QUEUED) {
+    ESP_LOGW(TAG, "Too many requests waiting; request dropped");
+    request->failed = true;
+    this->finish_(request);
+  } else if (waiting == 0 && running < this->max_concurrent_) {
+    // Start at once; one that cannot start is reported at once
+    this->begin_request_(request);
+    if (request->failed)
+      this->finish_(request);
+  }
+  this->busy_--;
+  if (this->busy_ != 0)
+    return;
+  if (this->shutting_down_) {
+    this->drain_();
+  } else {
+    this->remove_dead_();
+  }
+}
+
+void HttpRequestComponent::cancel(void *owner) {
+  for (PendingRequest *request = this->requests_; request != nullptr; request = request->next) {
+    if (request->owner == owner)
+      request->cancelled = true;
+  }
+  if (this->busy_ == 0)
+    this->remove_dead_();
+}
+
+bool HttpRequestComponent::process_() {
+  this->busy_++;
+  // The first max_concurrent_ live requests run; a request finished here frees its slot on the next pass
+  uint8_t slots = this->max_concurrent_;
+  for (PendingRequest *request = this->requests_; request != nullptr && slots != 0; request = request->next) {
+    if (request->finished || request->cancelled)
+      continue;
+    slots--;
+    if (request->container == nullptr)
+      this->begin_request_(request);
+    if (!request->cancelled && (request->failed || this->advance_(request)))
+      this->finish_(request);
+  }
+  this->busy_--;
+  if (this->busy_ == 0)
+    this->remove_dead_();
+  return this->requests_ != nullptr;
+}
+
+void HttpRequestComponent::drain_() {
+  // A call further up holds request pointers, and runs the new requests itself
+  if (this->busy_ != 0)
+    return;
+  while (this->process_()) {
+    App.feed_wdt();
+    delay(1);
+  }
+}
+
+void HttpRequestComponent::begin_request_(PendingRequest *request) {
+  request->start_time = millis();
+  request->handler(request, RequestEvent::REQUEST_EVENT_START);
+  // Stopped from the request's own lambdas: it is removed without being polled
+  if (!request->cancelled && request->container == nullptr)
+    request->failed = true;
+}
+
+bool HttpRequestComponent::advance_(PendingRequest *request) {
+  if (!request->reading) {
+    auto progress = request->container->poll();
+    if (progress == HttpRequestProgress::HTTP_REQUEST_PROGRESS_PENDING)
+      return false;
+    if (progress == HttpRequestProgress::HTTP_REQUEST_PROGRESS_FAILED) {
+      request->failed = true;
+      return true;
+    }
+    if (!request->capture)
+      return true;
+    request->buf = RAMAllocator<uint8_t>().allocate(request->buf_size);
+    if (request->buf == nullptr) {
+      // As before, on_response still fires, with an empty body
+      ESP_LOGW(TAG, "Could not allocate %zu bytes for the response body", request->buf_size);
+      return true;
+    }
+    request->reading = true;
+    // millis(), not the loop time: poll() may have blocked earlier in this pass
+    request->last_data_time = millis();
+  }
+  // NOTE: HttpContainer::read() has non-BSD socket semantics - see http_request.h
+  // millis(), not the loop time: the budget is shorter than one pass of the loop
+  const uint32_t start = millis();
+  while (request->read_index < request->buf_size) {
+    int read_or_error =
+        request->container->read_available(request->buf + request->read_index, request->buf_size - request->read_index);
+    const uint32_t now = millis();
+    // As before, a body that is cut off still fires on_response with what was read
+    if (read_or_error < 0) {
+      ESP_LOGW(TAG, "Response body read failed after %zu bytes", request->read_index);
+      return true;
+    }
+    if (read_or_error == 0) {
+      if (request->container->is_read_complete())
+        return true;
+      if (now - request->last_data_time < request->timeout)
+        return false;
+      ESP_LOGW(TAG, "Response body timed out after %zu bytes", request->read_index);
+      return true;
+    }
+    request->read_index += read_or_error;
+    request->last_data_time = now;
+    if (now - start >= READ_BUDGET_MS)
+      return false;
+  }
+  // As before, on_response still fires, with the part that fitted
+  if (!request->container->is_read_complete()) {
+    ESP_LOGW(TAG, "Response body is larger than max_response_buffer_size; kept the first %zu bytes", request->buf_size);
+  }
+  return true;
+}
+
+void HttpRequestComponent::finish_(PendingRequest *request) {
+  request->finished = true;
+  // The body was read over several passes, so time it from the start, as a blocking read would
+  if (request->reading && request->container != nullptr)
+    request->container->duration_ms = millis() - request->start_time;
+  request->handler(request, RequestEvent::REQUEST_EVENT_FINISH);
+}
+
+void HttpRequestComponent::remove_dead_() {
+  PendingRequest **link = &this->requests_;
+  while (PendingRequest *request = *link) {
+    if (!request->finished && !request->cancelled) {
+      link = &request->next;
+      continue;
+    }
+    *link = request->next;
+    if (request->container != nullptr)
+      request->container->end();
+    request->free_buffer();
+    request->handler(request, RequestEvent::REQUEST_EVENT_DESTROY);
+  }
+}
+
 }  // namespace esphome::http_request

esphome/components/http_request/http_request.h

@@ -12,6 +12,7 @@
 #include "esphome/core/component.h"
 #include "esphome/core/defines.h"
 #include "esphome/core/alloc_helpers.h"
+#include "esphome/core/hal.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
 
@@ -186,6 +187,13 @@ enum class HttpReadLoopResult : uint8_t {
   TIMEOUT,   ///< Timeout waiting for data, caller should exit loop
 };
 
+/// Progress of a request started with HttpRequestComponent::begin()
+enum class HttpRequestProgress : uint8_t {
+  HTTP_REQUEST_PROGRESS_PENDING,  ///< Still connecting, sending, or waiting for the response headers
+  HTTP_REQUEST_PROGRESS_DONE,     ///< Response headers received; status_code is set and the body can be read
+  HTTP_REQUEST_PROGRESS_FAILED,   ///< No response; the container must still be ended
+};
+
 /// Process a read result with timeout tracking and delay handling
 /// @param bytes_read_or_error Return value from read() - positive for bytes read, negative for error
 /// @param last_data_time Time of last successful read, updated when data received
@@ -249,8 +257,15 @@ class HttpContainer : public Parented<HttpRequestComponent> {
    *   - http_read_fully(): for simple "read N bytes" operations
    */
   virtual int read(uint8_t *buf, size_t max_len) = 0;
+  /// Like read(), but returns 0 at once when no data has arrived yet, where read() may wait for it.
+  virtual int read_available(uint8_t *buf, size_t max_len) { return this->read(buf, max_len); }
   virtual void end() = 0;
 
+  /// Advance a request started with HttpRequestComponent::begin(); call from the main loop until it is no
+  /// longer pending. Containers from start() are already done. Most calls return quickly, but on ESP-IDF the
+  /// first call for each host waits for the name lookup and the TCP connection, up to the whole timeout.
+  virtual HttpRequestProgress poll() { return HttpRequestProgress::HTTP_REQUEST_PROGRESS_DONE; }
+
   void set_secure(bool secure) { this->secure_ = secure; }
   void set_watchdog_timeout(uint32_t watchdog_timeout) { this->watchdog_timeout_ = watchdog_timeout; }
   uint32_t get_watchdog_timeout() const { return this->watchdog_timeout_; }
@@ -332,10 +347,56 @@ class HttpRequestResponseTrigger final : public Trigger<std::shared_ptr<HttpCont
   }
 };
 
+/// What HttpRequestComponent asks of the action that owns a request
+enum class RequestEvent : uint8_t {
+  REQUEST_EVENT_START,    ///< Build the request and call begin()
+  REQUEST_EVENT_FINISH,   ///< Fire the triggers and continue the action chain
+  REQUEST_EVENT_DESTROY,  ///< Free the request
+  REQUEST_EVENT_ABANDON,  ///< Dropped at shutdown: stop the action, so that it can run again
+};
+
+/// A request from an HttpRequestSendAction, run from HttpRequestComponent::loop().
+/// The action derives from it to add the trigger arguments.
+struct PendingRequest {
+  using Handler = void (*)(PendingRequest *request, RequestEvent event);
+  PendingRequest(Handler handler, void *owner) : handler(handler), owner(owner) {}
+  PendingRequest(const PendingRequest &) = delete;
+  PendingRequest &operator=(const PendingRequest &) = delete;
+  void free_buffer();
+
+  PendingRequest *next{nullptr};
+  Handler handler;
+  // The action, so that a stop of it can find its requests
+  void *owner;
+  // nullptr until the request has started
+  std::shared_ptr<HttpContainer> container;
+  uint8_t *buf{nullptr};
+  size_t buf_size{0};
+  size_t read_index{0};
+  uint32_t last_data_time{0};
+  uint32_t timeout{0};
+  uint32_t start_time{0};
+  bool capture{false};
+  bool reading{false};
+  bool failed{false};
+  bool finished{false};
+  bool cancelled{false};
+};
+
 class HttpRequestComponent : public Component {
  public:
   void dump_config() override;
   float get_setup_priority() const override { return setup_priority::AFTER_WIFI; }
+  void setup() override;
+  void loop() override;
+  void on_safe_shutdown() override;
+  void on_shutdown() override;
+
+  /// For HttpRequestSendAction: queue a request and take ownership of it. Requests start in order, up to
+  /// max_concurrent_requests at a time; one that can start at once is started here.
+  void submit(PendingRequest *request);
+  /// For HttpRequestSendAction: drop the requests of a stopped action without firing their triggers.
+  void cancel(void *owner);
 
   void set_useragent(const char *useragent) { this->useragent_ = useragent; }
   void set_timeout(uint32_t timeout) { this->timeout_ = timeout; }
@@ -349,6 +410,7 @@ class HttpRequestComponent : public Component {
   }
   void set_follow_redirects(bool follow_redirects) { this->follow_redirects_ = follow_redirects; }
   void set_redirect_limit(uint16_t limit) { this->redirect_limit_ = limit; }
+  void set_max_concurrent_requests(uint8_t max_concurrent) { this->max_concurrent_ = max_concurrent; }
 
   std::shared_ptr<HttpContainer> get(const char *url) { return this->start(url, "GET", "", std::vector<Header>{}); }
   std::shared_ptr<HttpContainer> get(const char *url, const std::vector<Header> &request_headers) {
@@ -491,16 +553,44 @@ class HttpRequestComponent : public Component {
     return this->start(url.c_str(), method.c_str(), body, request_headers, lower_case_collect_headers);
   }
 
+  /// Start a request without waiting for the response, then poll() the container until it is not pending.
+  /// Returns nullptr when the request cannot be started. Platforms without a non-blocking client complete
+  /// the request here.
+  virtual std::shared_ptr<HttpContainer> begin(const char *url, const char *method, std::string &&body,
+                                               const std::vector<Header> &request_headers,
+                                               const std::vector<std::string> &lower_case_collect_headers,
+                                               uint32_t timeout_ms) {
+    return this->perform(url, method, body, request_headers, lower_case_collect_headers, timeout_ms);
+  }
+
  protected:
   virtual std::shared_ptr<HttpContainer> perform(const char *url, const char *method, const std::string &body,
                                                  const std::vector<Header> &request_headers,
                                                  const std::vector<std::string> &lower_case_collect_headers,
                                                  uint32_t timeout_ms) = 0;
+  /// Longest time one loop() pass spends reading a response body
+  static constexpr uint32_t READ_BUDGET_MS = 10;
+  /// Requests that wait for a free slot; more are dropped with on_error
+  static constexpr uint8_t MAX_QUEUED = 4;
+
+  bool process_();
+  void drain_();
+  void begin_request_(PendingRequest *request);
+  bool advance_(PendingRequest *request);
+  void finish_(PendingRequest *request);
+  void remove_dead_();
+
   const char *useragent_{nullptr};
   bool follow_redirects_{};
   uint16_t redirect_limit_{};
   uint32_t timeout_{4500};
   uint32_t watchdog_timeout_{0};
+  // Oldest first; the first max_concurrent_ live requests run
+  PendingRequest *requests_{nullptr};
+  // Depth of calls that hold request pointers; finished and cancelled requests are freed when it returns to 0
+  uint8_t busy_{0};
+  uint8_t max_concurrent_{1};
+  bool shutting_down_{false};
 };
 
 template<typename... Ts> class HttpRequestSendAction final : public Action<Ts...> {
@@ -542,7 +632,42 @@ template<typename... Ts> class HttpRequestSendAction final : public Action<Ts...
     this->max_response_buffer_size_ = max_response_buffer_size;
   }
 
-  void play(const Ts &...x) override {
+  void play_complex(const Ts &...x) override {
+    this->num_running_++;
+    this->parent_->submit(new Request(this, x...));  // NOLINT(cppcoreguidelines-owning-memory)
+  }
+
+  void play(const Ts &...x) override { /* see play_complex */ }
+
+ protected:
+  struct Request final : PendingRequest {
+    Request(HttpRequestSendAction *action, const Ts &...x)
+        : PendingRequest(&HttpRequestSendAction::handle_event, action), args(x...) {}
+    std::tuple<stored_trigger_arg_t<Ts>...> args;
+  };
+
+  static void handle_event(PendingRequest *request, RequestEvent event) {
+    auto *self = static_cast<Request *>(request);
+    auto *action = static_cast<HttpRequestSendAction *>(request->owner);
+    switch (event) {
+      case RequestEvent::REQUEST_EVENT_START:
+        std::apply([action, self](auto &...x) { action->start_request_(*self, x...); }, self->args);
+        break;
+      case RequestEvent::REQUEST_EVENT_FINISH:
+        action->finish_(*self);
+        break;
+      case RequestEvent::REQUEST_EVENT_DESTROY:
+        delete self;  // NOLINT(cppcoreguidelines-owning-memory)
+        break;
+      case RequestEvent::REQUEST_EVENT_ABANDON:
+        action->stop_complex();
+        break;
+    }
+  }
+
+  void stop() override { this->parent_->cancel(this); }
+
+  void start_request_(Request &request, const Ts &...x) {
     std::string body;
     if (this->body_.has_value()) {
       body = this->body_.value(x...);
@@ -559,62 +684,47 @@ template<typename... Ts> class HttpRequestSendAction final : public Action<Ts...
       request_headers.push_back({header.name, header.value(x...)});
     }
 
-    const uint32_t timeout = this->timeout_.has_value() ? this->timeout_.value(x...) : this->parent_->get_timeout();
-    auto container = this->parent_->start(this->url_.value(x...).c_str(), this->method_.value(x...), body,
-                                          request_headers, this->lower_case_collect_headers_, timeout);
-
-    auto captured_args = std::make_tuple(x...);
-
-    if (container == nullptr) {
-      std::apply([this](Ts... captured_args_inner) { this->error_trigger_.trigger(captured_args_inner...); },
-                 captured_args);
-      return;
-    }
-
+    request.timeout = this->timeout_.has_value() ? this->timeout_.value(x...) : this->parent_->get_timeout();
 #ifdef USE_HTTP_REQUEST_RESPONSE
-    if (this->capture_response_.value(x...)) {
-      size_t max_length = this->max_response_buffer_size_;
-      std::string response_body;
-      RAMAllocator<uint8_t> allocator;
-      uint8_t *buf = allocator.allocate(max_length);
-      if (buf != nullptr) {
-        // NOTE: HttpContainer::read() has non-BSD socket semantics - see top of this file
-        // Use http_read_loop_result() helper instead of checking return values directly
-        size_t read_index = 0;
-        uint32_t last_data_time = millis();
-        const uint32_t read_timeout = timeout;
-        while (container->get_bytes_read() < max_length) {
-          int read_or_error = container->read(buf + read_index, std::min<size_t>(max_length - read_index, 512));
-          App.feed_wdt();
-          yield();
-          auto result =
-              http_read_loop_result(read_or_error, last_data_time, read_timeout, container->is_read_complete());
-          if (result == HttpReadLoopResult::RETRY)
-            continue;
-          if (result != HttpReadLoopResult::DATA)
-            break;  // COMPLETE, ERROR, or TIMEOUT
-          read_index += read_or_error;
-        }
-        response_body.reserve(read_index);
-        response_body.assign((char *) buf, read_index);
-        allocator.deallocate(buf, max_length);
-      }
-      std::apply(
-          [this, &container, &response_body](Ts... captured_args_inner) {
-            this->success_trigger_with_response_.trigger(container, response_body, captured_args_inner...);
-          },
-          captured_args);
-    } else
+    request.capture = this->capture_response_.value(x...);
 #endif
-    {
-      std::apply([this, &container](
-                     Ts... captured_args_inner) { this->success_trigger_.trigger(container, captured_args_inner...); },
-                 captured_args);
+    request.buf_size = this->max_response_buffer_size_;
+    request.container = this->parent_->begin(this->url_.value(x...).c_str(), this->method_.value(x...), std::move(body),
+                                             request_headers, this->lower_case_collect_headers_, request.timeout);
+  }
+
+  void finish_(Request &request) {
+    auto &container = request.container;
+    if (request.failed) {
+      std::apply([this](auto &...args) { this->error_trigger_.trigger(args...); }, request.args);
     }
-    container->end();
+#ifdef USE_HTTP_REQUEST_RESPONSE
+    else if (request.capture) {
+      std::string response_body;
+      if (request.buf != nullptr)
+        response_body.assign(reinterpret_cast<char *>(request.buf), request.read_index);
+      request.free_buffer();
+      std::apply(
+          [this, &container, &response_body](auto &...args) {
+            this->success_trigger_with_response_.trigger(container, response_body, args...);
+          },
+          request.args);
+    }
+#endif
+    else {
+      std::apply([this, &container](auto &...args) { this->success_trigger_.trigger(container, args...); },
+                 request.args);
+    }
+    // End it before the chain can start another request, so that one client is held at a time
+    if (container != nullptr) {
+      container->end();
+      container.reset();
+    }
+    // A stop from the triggers ended this run; a run started after it continues the chain instead
+    if (!request.cancelled)
+      std::apply([this](auto &...args) { this->play_next_(args...); }, request.args);
   }
 
- protected:
   void encode_json_(Ts... x, JsonObject root) {
     for (const auto &item : this->json_) {
       auto val = item.second;

esphome/components/http_request/http_request_idf.cpp

@@ -2,8 +2,13 @@
 
 #ifdef USE_ESP32
 
+#include <algorithm>
+#include <cerrno>
 #include <cstring>
 
+#include <fcntl.h>
+#include <sys/select.h>
+
 #include "esphome/components/network/util.h"
 #include "esphome/components/watchdog/watchdog.h"
 
@@ -20,6 +25,10 @@
 namespace esphome::http_request {
 
 ESPHOME_LOG_TAG(TAG, "http_request");
+
+// Request body sent per esp_http_client_write() call, and the time one poll() spends sending it
+static constexpr size_t SEND_CHUNK_SIZE = 1024;
+static constexpr uint32_t SEND_BUDGET_MS = 10;
 static constexpr uint32_t ERROR_DURATION_MS = 1000;
 
 void HttpRequestIDF::dump_config() {
@@ -51,10 +60,9 @@ esp_err_t HttpRequestIDF::http_event_handler(esp_http_client_event_t *evt) {
   return ESP_OK;
 }
 
-std::shared_ptr<HttpContainer> HttpRequestIDF::perform(const char *url, const char *method, const std::string &body,
-                                                       const std::vector<Header> &request_headers,
-                                                       const std::vector<std::string> &lower_case_collect_headers,
-                                                       uint32_t timeout_ms) {
+std::shared_ptr<HttpContainerIDF> HttpRequestIDF::create_container_(
+    const char *url, const char *method, const std::vector<Header> &request_headers,
+    const std::vector<std::string> &lower_case_collect_headers, uint32_t timeout_ms, [[maybe_unused]] bool async) {
   if (!network::is_connected()) {
     this->status_momentary_error(ERROR_DURATION_MS);
     ESP_LOGE(TAG, "HTTP Request failed; Not connected to network");
@@ -106,11 +114,12 @@ std::shared_ptr<HttpContainer> HttpRequestIDF::perform(const char *url, const ch
   config.buffer_size = this->buffer_size_rx_;
   config.buffer_size_tx = this->buffer_size_tx_;
 
-  const uint32_t start = millis();
-  const uint32_t watchdog_timeout = this->get_watchdog_timeout(timeout_ms);
-  watchdog::WatchdogManager wdm(watchdog_timeout);
-
   config.event_handler = http_event_handler;
+#ifdef USE_HTTP_REQUEST_IDF_SOCKET_POLL
+  // Only TLS has a non-blocking connect worth having: the handshake. A plain connection waits for
+  // the TCP connection either way.
+  config.is_async = async && secure;
+#endif
 
   esp_http_client_handle_t client = esp_http_client_init(&config);
   if (client == nullptr) {
@@ -121,7 +130,9 @@ std::shared_ptr<HttpContainer> HttpRequestIDF::perform(const char *url, const ch
 
   std::shared_ptr<HttpContainerIDF> container = std::make_shared<HttpContainerIDF>(client);
   container->set_parent(this);
-  container->set_watchdog_timeout(watchdog_timeout);
+  container->set_watchdog_timeout(this->get_watchdog_timeout(timeout_ms));
+  container->timeout_ms_ = timeout_ms;
+  container->async_ = config.is_async;
 
   container->set_secure(secure);
 
@@ -132,101 +143,229 @@ std::shared_ptr<HttpContainer> HttpRequestIDF::perform(const char *url, const ch
     esp_http_client_set_header(client, header.name.c_str(), header.value.c_str());
   }
 
-  const int body_len = body.length();
+  return container;
+}
 
-  esp_err_t err = esp_http_client_open(client, body_len);
-  if (err != ESP_OK) {
-    this->status_momentary_error(ERROR_DURATION_MS);
-    ESP_LOGE(TAG, "HTTP Request failed: %s", esp_err_to_name(err));
-    esp_http_client_cleanup(client);
+std::shared_ptr<HttpContainer> HttpRequestIDF::perform(const char *url, const char *method, const std::string &body,
+                                                       const std::vector<Header> &request_headers,
+                                                       const std::vector<std::string> &lower_case_collect_headers,
+                                                       uint32_t timeout_ms) {
+  std::shared_ptr<HttpContainerIDF> container = this->start_container_(url, method, std::string(body), request_headers,
+                                                                       lower_case_collect_headers, timeout_ms, false);
+  if (container == nullptr) {
     return nullptr;
   }
+  // The same stages as begin(), run here until the response headers are in
+  watchdog::WatchdogManager wdm(container->get_watchdog_timeout());
+  HttpRequestProgress progress;
+  while ((progress = container->poll()) == HttpRequestProgress::HTTP_REQUEST_PROGRESS_PENDING) {
+    container->feed_wdt();
+    delay(1);
+  }
+  if (progress == HttpRequestProgress::HTTP_REQUEST_PROGRESS_FAILED) {
+    container->end();
+    return nullptr;
+  }
+  return container;
+}
 
-  if (body_len > 0) {
-    int write_left = body_len;
-    int write_index = 0;
-    const char *buf = body.c_str();
-    while (write_left > 0) {
-      int written = esp_http_client_write(client, buf + write_index, write_left);
-      if (written <= 0) {
-        err = ESP_FAIL;
+std::shared_ptr<HttpContainer> HttpRequestIDF::begin(const char *url, const char *method, std::string &&body,
+                                                     const std::vector<Header> &request_headers,
+                                                     const std::vector<std::string> &lower_case_collect_headers,
+                                                     uint32_t timeout_ms) {
+  return this->start_container_(url, method, std::move(body), request_headers, lower_case_collect_headers, timeout_ms,
+                                true);
+}
+
+std::shared_ptr<HttpContainerIDF> HttpRequestIDF::start_container_(
+    const char *url, const char *method, std::string &&body, const std::vector<Header> &request_headers,
+    const std::vector<std::string> &lower_case_collect_headers, uint32_t timeout_ms, bool async) {
+  std::shared_ptr<HttpContainerIDF> container =
+      this->create_container_(url, method, request_headers, lower_case_collect_headers, timeout_ms, async);
+  if (container == nullptr) {
+    return nullptr;
+  }
+  container->body_ = std::move(body);
+  container->start_time_ = millis();
+  container->progress_time_ = container->start_time_;
+  container->redirects_left_ = this->follow_redirects_ ? this->redirect_limit_ : 0;
+  container->stage_ = HttpContainerIDF::Stage::STAGE_CONNECT;
+  return container;
+}
+
+HttpRequestIDF *HttpContainerIDF::idf_parent_() { return static_cast<HttpRequestIDF *>(this->parent_); }
+
+HttpRequestProgress HttpContainerIDF::fail_(esp_err_t err) {
+  ESP_LOGE(TAG, "HTTP Request failed: %s", esp_err_to_name(err));
+  this->parent_->status_momentary_error(ERROR_DURATION_MS);
+  this->stage_ = Stage::STAGE_FAILED;
+  return HttpRequestProgress::HTTP_REQUEST_PROGRESS_FAILED;
+}
+
+HttpRequestProgress HttpContainerIDF::wait_() {
+  // millis(), not the loop time: progress_time_ is set after calls that block within this pass
+  if (millis() - this->progress_time_ >= this->timeout_ms_) {
+    return this->fail_(ESP_ERR_TIMEOUT);
+  }
+  return HttpRequestProgress::HTTP_REQUEST_PROGRESS_PENDING;
+}
+
+bool HttpContainerIDF::socket_ready_(bool write) {
+#ifdef USE_HTTP_REQUEST_IDF_SOCKET_POLL
+  const int fd = esp_http_client_get_socket(this->client_);
+  if (fd < 0) {
+    return true;
+  }
+  fd_set fds;
+  FD_ZERO(&fds);
+  FD_SET(fd, &fds);
+  timeval no_wait{};
+  // An error also counts as ready, so that the call that follows reports it
+  // Qualified: esphome::select is a namespace
+  return ::select(fd + 1, write ? nullptr : &fds, write ? &fds : nullptr, nullptr, &no_wait) != 0;
+#else
+  return true;
+#endif
+}
+
+// Each stage either finishes, or returns PENDING after a call that did not wait. The waits left are
+// the name lookup and TCP connection in the first esp_http_client_open() of each hop: a non-blocking
+// esp-tls connect that times out once never completes, as select() clears the sets it keeps.
+HttpRequestProgress HttpContainerIDF::poll() {
+  while (true) {
+    switch (this->stage_) {
+      case Stage::STAGE_DONE:
+        return HttpRequestProgress::HTTP_REQUEST_PROGRESS_DONE;
+
+      case Stage::STAGE_FAILED:
+        return HttpRequestProgress::HTTP_REQUEST_PROGRESS_FAILED;
+
+      case Stage::STAGE_CONNECT: {
+        // Only the first call of a hop waits
+        watchdog::WatchdogManager wdm(this->connecting_ ? 0 : this->get_watchdog_timeout());
+        esp_http_client_set_timeout_ms(this->client_, this->connecting_ ? 1 : this->timeout_ms_);
+        this->feed_wdt();
+        esp_err_t err = esp_http_client_open(this->client_, this->body_.size());
+        this->feed_wdt();
+        if (err == ESP_ERR_HTTP_EAGAIN) {
+          this->connecting_ = true;
+          return this->wait_();
+        }
+        this->connecting_ = false;
+        if (err != ESP_OK) {
+          return this->fail_(err);
+        }
+#ifdef USE_HTTP_REQUEST_IDF_SOCKET_POLL
+        if (this->async_) {
+          // Back to a blocking socket, so that mbedtls waits for the rest of a record instead of
+          // failing; reads only start once the socket is readable
+          const int fd = esp_http_client_get_socket(this->client_);
+          const int flags = fd < 0 ? -1 : fcntl(fd, F_GETFL, 0);
+          if (flags < 0 || fcntl(fd, F_SETFL, flags & ~O_NONBLOCK) < 0) {
+            ESP_LOGE(TAG, "Could not make the socket blocking: fd %d, errno %d", fd, errno);
+            return this->fail_(ESP_FAIL);
+          }
+        }
+#endif
+        esp_http_client_set_timeout_ms(this->client_, this->timeout_ms_);
+        this->progress_time_ = millis();
+        this->stage_ = this->body_sent_ < this->body_.size() ? Stage::STAGE_SEND_BODY : Stage::STAGE_HEADERS;
         break;
       }
-      write_left -= written;
-      write_index += written;
-      container->feed_wdt();
-    }
-  }
 
-  if (err != ESP_OK) {
-    this->status_momentary_error(ERROR_DURATION_MS);
-    ESP_LOGE(TAG, "HTTP Request failed: %s", esp_err_to_name(err));
-    esp_http_client_cleanup(client);
-    return nullptr;
-  }
-
-  container->feed_wdt();
-  // esp_http_client_fetch_headers() returns 0 for chunked transfer encoding (no Content-Length header).
-  // The read() method handles content_length == 0 specially to support chunked responses.
-  container->content_length = esp_http_client_fetch_headers(client);
-  container->set_chunked(esp_http_client_is_chunked_response(client));
-  container->feed_wdt();
-  container->status_code = esp_http_client_get_status_code(client);
-  container->feed_wdt();
-  container->duration_ms = millis() - start;
-  if (is_success(container->status_code)) {
-    return container;
-  }
-
-  if (this->follow_redirects_) {
-    auto num_redirects = this->redirect_limit_;
-    while (is_redirect(container->status_code) && num_redirects > 0) {
-      err = esp_http_client_set_redirection(client);
-      if (err != ESP_OK) {
-        ESP_LOGE(TAG, "esp_http_client_set_redirection failed: %s", esp_err_to_name(err));
-        this->status_momentary_error(ERROR_DURATION_MS);
-        esp_http_client_cleanup(client);
-        return nullptr;
+      case Stage::STAGE_SEND_BODY: {
+        // esp_http_client_write() returns only once all it was given is sent, so give it a piece at a time
+        watchdog::WatchdogManager wdm(this->get_watchdog_timeout());
+        const uint32_t start = millis();
+        while (this->body_sent_ < this->body_.size()) {
+          if (!this->socket_ready_(true)) {
+            return this->wait_();
+          }
+          const size_t len = std::min(this->body_.size() - this->body_sent_, SEND_CHUNK_SIZE);
+          int written = esp_http_client_write(this->client_, this->body_.data() + this->body_sent_, len);
+          this->feed_wdt();
+          if (written < 0) {
+            return this->fail_(ESP_FAIL);
+          }
+          if (written == 0) {
+            return this->wait_();
+          }
+          this->body_sent_ += written;
+          this->progress_time_ = millis();
+          if (this->progress_time_ - start >= SEND_BUDGET_MS && this->body_sent_ < this->body_.size()) {
+            return HttpRequestProgress::HTTP_REQUEST_PROGRESS_PENDING;
+          }
+        }
+        this->body_ = std::string();
+        this->stage_ = Stage::STAGE_HEADERS;
+        break;
       }
+
+      case Stage::STAGE_HEADERS: {
+        if (!this->socket_ready_(false)) {
+          if (millis() - this->progress_time_ < this->timeout_ms_) {
+            return HttpRequestProgress::HTTP_REQUEST_PROGRESS_PENDING;
+          }
+          // As perform(), where esp_http_client_fetch_headers() times out: no response is status -1
+          return this->response_failed_(-1);
+        }
+        // The response has begun to arrive, so this waits only for the rest of the headers, or over TLS
+        // the rest of the record that holds them
+        watchdog::WatchdogManager wdm(this->get_watchdog_timeout());
+        this->feed_wdt();
+        // esp_http_client_fetch_headers() returns 0 for chunked transfer encoding (no Content-Length header)
+        const int64_t length = esp_http_client_fetch_headers(this->client_);
+        this->feed_wdt();
+        this->duration_ms = millis() - this->start_time_;
+        if (length < 0) {
+          return this->response_failed_(-1);
+        }
+        this->content_length = length;
+        this->set_chunked(esp_http_client_is_chunked_response(this->client_));
+        this->status_code = esp_http_client_get_status_code(this->client_);
+        if (is_success(this->status_code)) {
+          this->stage_ = Stage::STAGE_DONE;
+          break;
+        }
+        if (this->redirects_left_ > 0 && is_redirect(this->status_code)) {
+          esp_err_t err = esp_http_client_set_redirection(this->client_);
+          if (err != ESP_OK) {
+            ESP_LOGE(TAG, "esp_http_client_set_redirection failed: %s", esp_err_to_name(err));
+            return this->fail_(err);
+          }
 #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE
-      char redirect_url[256]{};
-      if (esp_http_client_get_url(client, redirect_url, sizeof(redirect_url) - 1) == ESP_OK) {
-        ESP_LOGV(TAG, "redirecting to url: %s", redirect_url);
-      }
+          char redirect_url[256]{};
+          if (esp_http_client_get_url(this->client_, redirect_url, sizeof(redirect_url) - 1) == ESP_OK) {
+            ESP_LOGV(TAG, "redirecting to url: %s", redirect_url);
+          }
 #endif
-      err = esp_http_client_open(client, 0);
-      if (err != ESP_OK) {
-        ESP_LOGE(TAG, "esp_http_client_open failed: %s", esp_err_to_name(err));
-        this->status_momentary_error(ERROR_DURATION_MS);
-        esp_http_client_cleanup(client);
-        return nullptr;
+          this->redirects_left_--;
+          // Collect only the final response's headers; redirects do not resend the body
+          this->response_headers_.clear();
+          this->body_ = std::string();
+          this->body_sent_ = 0;
+          this->progress_time_ = millis();
+          this->stage_ = Stage::STAGE_CONNECT;
+          break;
+        }
+        if (is_redirect(this->status_code) && this->idf_parent_()->follow_redirects_) {
+          ESP_LOGW(TAG, "Reach redirect limit count=%d", this->idf_parent_()->redirect_limit_);
+        }
+        return this->response_failed_(this->status_code);
       }
-
-      container->feed_wdt();
-      // IDF is the only backend reusing the container across redirect hops;
-      // drop the previous hop's headers (Arduino/host collect only the final response)
-      container->response_headers_.clear();
-      container->content_length = esp_http_client_fetch_headers(client);
-      container->set_chunked(esp_http_client_is_chunked_response(client));
-      container->feed_wdt();
-      container->status_code = esp_http_client_get_status_code(client);
-      container->feed_wdt();
-      container->duration_ms = millis() - start;
-      if (is_success(container->status_code)) {
-        return container;
-      }
-
-      num_redirects--;
-    }
-
-    if (num_redirects == 0) {
-      ESP_LOGW(TAG, "Reach redirect limit count=%d", this->redirect_limit_);
     }
   }
+}
 
-  ESP_LOGE(TAG, "HTTP Request failed; URL: %s; Code: %d", url, container->status_code);
-  this->status_momentary_error(ERROR_DURATION_MS);
-  return container;
+HttpRequestProgress HttpContainerIDF::response_failed_(int status_code) {
+  this->status_code = status_code;
+  this->duration_ms = millis() - this->start_time_;
+  char url[256]{};
+  esp_http_client_get_url(this->client_, url, sizeof(url) - 1);
+  ESP_LOGE(TAG, "HTTP Request failed; URL: %s; Code: %d", url, this->status_code);
+  this->parent_->status_momentary_error(ERROR_DURATION_MS);
+  this->stage_ = Stage::STAGE_DONE;
+  return HttpRequestProgress::HTTP_REQUEST_PROGRESS_DONE;
 }
 
 bool HttpContainerIDF::is_read_complete() const {
@@ -265,8 +404,21 @@ bool HttpContainerIDF::is_read_complete() const {
 // after its internal transport timeout (configured via timeout_ms) expires.
 // This is passed through as a negative return value, which callers treat as an error.
 int HttpContainerIDF::read(uint8_t *buf, size_t max_len) {
-  const uint32_t start = millis();
   watchdog::WatchdogManager wdm(this->get_watchdog_timeout());
+  esp_http_client_set_timeout_ms(this->client_, this->timeout_ms_);
+  return this->read_(buf, max_len);
+}
+
+int HttpContainerIDF::read_available(uint8_t *buf, size_t max_len) {
+  // A zero timeout only polls; TLS still waits for the rest of a record that has begun to arrive
+  watchdog::WatchdogManager wdm(this->get_watchdog_timeout());
+  esp_http_client_set_timeout_ms(this->client_, 0);
+  int read_len_or_error = this->read_(buf, max_len);
+  return read_len_or_error == -ESP_ERR_HTTP_EAGAIN ? 0 : read_len_or_error;
+}
+
+int HttpContainerIDF::read_(uint8_t *buf, size_t max_len) {
+  const uint32_t start = millis();
 
   // Check if we've already read all expected content (non-chunked and no-body only).
   // Use the base class check here, NOT the override: esp_http_client_is_complete_data_received()

esphome/components/http_request/http_request_idf.h

@@ -5,30 +5,71 @@
 #ifdef USE_ESP32
 
 #include <esp_event.h>
+#include <esp_idf_version.h>
 #include <esp_http_client.h>
 #include <esp_netif.h>
 #include <esp_tls.h>
 
+// The socket is needed to wait for the response without blocking; esp_http_client_get_socket() is new in 5.5.3
+#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 5, 3)
+#define USE_HTTP_REQUEST_IDF_SOCKET_POLL
+#endif
+
 namespace esphome::http_request {
 
+class HttpRequestIDF;
+
 class HttpContainerIDF : public HttpContainer {
  public:
   HttpContainerIDF(esp_http_client_handle_t client) : client_(client) {}
   int read(uint8_t *buf, size_t max_len) override;
+  int read_available(uint8_t *buf, size_t max_len) override;
   void end() override;
   bool is_read_complete() const override;
+  HttpRequestProgress poll() override;
 
   /// @brief Feeds the watchdog timer if the executing task has one attached
   void feed_wdt();
 
  protected:
   friend class HttpRequestIDF;
+  enum class Stage : uint8_t {
+    STAGE_CONNECT,
+    STAGE_SEND_BODY,
+    STAGE_HEADERS,
+    STAGE_DONE,
+    STAGE_FAILED,
+  };
+
+  int read_(uint8_t *buf, size_t max_len);
+  /// True when the socket can be read or written without waiting; always true when it cannot be checked
+  bool socket_ready_(bool write);
+  HttpRequestProgress wait_();
+  HttpRequestProgress fail_(esp_err_t err);
+  /// Finish with an unsuccessful status, which is reported through on_response like a success
+  HttpRequestProgress response_failed_(int status_code);
+  HttpRequestIDF *idf_parent_();
+
   esp_http_client_handle_t client_;
   // Owned copy (not a reference): must outlive perform() for the response-header event handler
   std::vector<std::string> collect_headers_;
+  // Request body for begin(), released once sent
+  std::string body_;
+  size_t body_sent_{0};
+  uint32_t timeout_ms_{0};
+  // Time of the last step forward, for the request timeout of begin()
+  uint32_t progress_time_{0};
+  uint32_t start_time_{0};
+  uint16_t redirects_left_{0};
+  Stage stage_{Stage::STAGE_DONE};
+  // An esp_http_client_open() of a non-blocking client returned ESP_ERR_HTTP_EAGAIN
+  bool connecting_{false};
+  bool async_{false};
 };
 
 class HttpRequestIDF final : public HttpRequestComponent {
+  friend class HttpContainerIDF;
+
  public:
   // User provided, not "= default": `new(p) HttpRequestIDF()` would zero-fill .bss that is already zero.
   HttpRequestIDF() {}
@@ -40,7 +81,21 @@ class HttpRequestIDF final : public HttpRequestComponent {
   void set_verify_ssl(bool verify_ssl) { this->verify_ssl_ = verify_ssl; }
   void set_ca_certificate(const char *ca_certificate) { this->ca_certificate_ = ca_certificate; }
 
+  std::shared_ptr<HttpContainer> begin(const char *url, const char *method, std::string &&body,
+                                       const std::vector<Header> &request_headers,
+                                       const std::vector<std::string> &lower_case_collect_headers,
+                                       uint32_t timeout_ms) override;
+
  protected:
+  /// A container that poll() takes from the connect through to the response headers
+  std::shared_ptr<HttpContainerIDF> start_container_(const char *url, const char *method, std::string &&body,
+                                                     const std::vector<Header> &request_headers,
+                                                     const std::vector<std::string> &lower_case_collect_headers,
+                                                     uint32_t timeout_ms, bool async);
+  std::shared_ptr<HttpContainerIDF> create_container_(const char *url, const char *method,
+                                                      const std::vector<Header> &request_headers,
+                                                      const std::vector<std::string> &lower_case_collect_headers,
+                                                      uint32_t timeout_ms, bool async);
   std::shared_ptr<HttpContainer> perform(const char *url, const char *method, const std::string &body,
                                          const std::vector<Header> &request_headers,
                                          const std::vector<std::string> &lower_case_collect_headers,

esphome/components/i2c/i2c.h

@@ -322,8 +322,9 @@ class I2CDevice {
   bool update_register(uint8_t a_register, uint8_t mask, uint8_t value);
 
  protected:
+  I2CBus *bus_{nullptr};  ///< pointer to I2CBus instance
+  // Last so a derived class can place its byte-sized members in the padding after it
   uint8_t address_{0x00};  ///< store the address of the device on the bus
-  I2CBus *bus_{nullptr};   ///< pointer to I2CBus instance
 };
 
 }  // namespace esphome::i2c

esphome/components/i2s_clockless_led_strip/i2s_clockless_led_strip.cpp

@@ -149,17 +149,12 @@ bool I2SClocklessLedStrip::allocate_buffers_() {
   RAMAllocator<uint8_t> allocator(RAMAllocator<uint8_t>::ALLOC_INTERNAL);
   this->i2s_data_ = allocator.allocate(i2s_data_bytes);
   if (this->i2s_data_ != nullptr) {
-    this->color_data_ = allocator.allocate(this->color_data_bytes_);
+    // The effect data follows the color data
+    this->color_data_ = allocator.allocate(this->color_data_bytes_ + this->num_leds_);
     if (this->color_data_ != nullptr) {
-      this->effect_data_ = allocator.allocate(this->num_leds_);
-      if (this->effect_data_ != nullptr) {
-        memset(this->i2s_data_, 0, i2s_data_bytes);
-        memset(this->color_data_, 0, this->color_data_bytes_);
-        memset(this->effect_data_, 0, this->num_leds_);
-        return true;
-      }
-      allocator.deallocate(this->color_data_, this->color_data_bytes_);
-      this->color_data_ = nullptr;
+      memset(this->i2s_data_, 0, i2s_data_bytes);
+      memset(this->color_data_, 0, this->color_data_bytes_ + this->num_leds_);
+      return true;
     }
     allocator.deallocate(this->i2s_data_, i2s_data_bytes);
     this->i2s_data_ = nullptr;
@@ -167,16 +162,6 @@ bool I2SClocklessLedStrip::allocate_buffers_() {
   return false;
 }
 
-light::LightTraits I2SClocklessLedStrip::get_traits() {
-  auto traits = light::LightTraits();
-  if (this->channel_colors_.has_white()) {
-    traits.set_supported_color_modes({light::ColorMode::RGB_WHITE, light::ColorMode::WHITE});
-  } else {
-    traits.set_supported_color_modes({light::ColorMode::RGB});
-  }
-  return traits;
-}
-
 void I2SClocklessLedStrip::write_state(light::LightState *state) {
   if (this->is_failed())
     return;
@@ -236,19 +221,6 @@ bool IRAM_ATTR HOT I2SClocklessLedStrip::i2s_on_sent_callback(i2s_chan_handle_t
   return false;
 }
 
-void I2SClocklessLedStrip::clear_effect_data() { memset(this->effect_data_, 0, this->num_leds_); }
-
-light::ESPColorView I2SClocklessLedStrip::get_view_internal(int32_t index) const {
-  const light::ChannelColors &colors = this->channel_colors_;
-  uint8_t *led = this->color_data_ + (index * colors.bytes_per_led());
-  return {led + colors.r,
-          led + colors.g,
-          led + colors.b,
-          colors.has_white() ? led + colors.w : nullptr,
-          &this->effect_data_[index],
-          &this->correction_};
-}
-
 }  // namespace esphome::i2s_clockless_led_strip
 
 #endif

esphome/components/i2s_clockless_led_strip/i2s_clockless_led_strip.h

@@ -8,6 +8,7 @@
 
 #include <atomic>
 #include <cstdint>
+#include <cstring>
 
 #include "esphome/core/component.h"
 #include "esphome/core/log.h"
@@ -27,20 +28,27 @@ class I2SClocklessLedStrip final : public light::AddressableLight {
   float get_setup_priority() const override;
   void setup() override;
 
-  light::LightTraits get_traits() override;
+  light::LightTraits get_traits() override { return this->channel_colors_.traits(); }
 
   int32_t size() const override { return this->num_leds_; }
-  void clear_effect_data() override;
+  void clear_effect_data() override {
+    if (uint8_t *effect = this->effect_data_())
+      memset(effect, 0, this->num_leds_);
+  }
   void write_state(light::LightState *state) override;
 
  protected:
   static bool i2s_on_sent_callback(i2s_chan_handle_t handle, i2s_event_data_t *event, void *user_ctx);
 
   bool allocate_buffers_();
-  light::ESPColorView get_view_internal(int32_t index) const override;
+  uint8_t *effect_data_() const {
+    return this->color_data_ == nullptr ? nullptr : this->color_data_ + this->color_data_bytes_;
+  }
+  light::ESPColorView get_view_internal(int32_t index) const override {
+    return this->channel_colors_.view(this->color_data_, this->effect_data_(), index, &this->correction_);
+  }
 
   const size_t color_data_bytes_;
-  uint8_t *effect_data_{nullptr};
   uint8_t *color_data_{nullptr};
   uint8_t *i2s_data_{nullptr};
   size_t i2s_data_sent_{0};

esphome/components/libretiny_pwm/output.py

@@ -1,6 +1,7 @@
 from esphome import automation, pins
 import esphome.codegen as cg
-from esphome.components import output
+from esphome.components import libretiny, output
+from esphome.components.libretiny.const import FAMILY_LN882H
 import esphome.config_validation as cv
 from esphome.const import CONF_FREQUENCY, CONF_ID, CONF_PIN
 from esphome.types import ConfigType
@@ -12,13 +13,16 @@ LibreTinyPWM = libretinypwm_ns.class_("LibreTinyPWM", output.FloatOutput, cg.Com
 
 validate_frequency = cv.All(cv.frequency, cv.float_range(min=0, min_included=False))
 
-CONFIG_SCHEMA = output.FLOAT_OUTPUT_SCHEMA.extend(
-    {
-        cv.Required(CONF_ID): cv.declare_id(LibreTinyPWM),
-        cv.Required(CONF_PIN): pins.internal_gpio_output_pin_schema,
-        cv.Optional(CONF_FREQUENCY, default="1kHz"): validate_frequency,
-    }
-).extend(cv.COMPONENT_SCHEMA)
+CONFIG_SCHEMA = cv.All(
+    libretiny.only_on_family(unsupported=FAMILY_LN882H),
+    output.FLOAT_OUTPUT_SCHEMA.extend(
+        {
+            cv.Required(CONF_ID): cv.declare_id(LibreTinyPWM),
+            cv.Required(CONF_PIN): pins.internal_gpio_output_pin_schema,
+            cv.Optional(CONF_FREQUENCY, default="1kHz"): validate_frequency,
+        }
+    ).extend(cv.COMPONENT_SCHEMA),
+)
 
 
 async def to_code(config: ConfigType) -> None:

esphome/components/light/addressable_light.h

@@ -134,6 +134,13 @@ class AddressableLight : public LightOutput, public Component {
 #endif
   }
   virtual ESPColorView get_view_internal(int32_t index) const = 0;
+  // Queues the frame for the next loop and returns true until interval_us has passed since last_us
+  bool throttle_show_(uint32_t now_us, uint32_t last_us, uint32_t interval_us) {
+    if (now_us - last_us >= interval_us)
+      return false;
+    this->schedule_show();
+    return true;
+  }
 
 #ifdef USE_LIGHT_POWER_ESTIMATION
   friend class AddressableLightState;

esphome/components/light/addressable_light_effect.h

@@ -1,8 +1,10 @@
 #pragma once
 
+#include <type_traits>
 #include <utility>
 
 #include "esphome/core/component.h"
+#include "esphome/core/hal.h"
 #include "esphome/core/helpers.h"
 #include "esphome/components/light/light_state.h"
 #include "esphome/components/light/addressable_light.h"
@@ -101,17 +103,28 @@ class AddressableRainbowLightEffect : public AddressableLightEffect {
   uint16_t width_{50};
 };
 
+/// One wipe color. Codegen keeps the colors in a flash table, so it is read with progmem_memcpy.
 struct AddressableColorWipeEffectColor {
   uint8_t r, g, b, w;
-  bool random;
-  size_t num_leds;
+  uint8_t random_slot;  ///< index into the effect's random colors, or NO_RANDOM_SLOT
   bool gradient;
+  uint32_t num_leds;
 };
+static_assert(std::is_trivially_copyable_v<AddressableColorWipeEffectColor>,
+              "AddressableColorWipeEffectColor is copied out of flash");
+static_assert(sizeof(AddressableColorWipeEffectColor) % sizeof(uint32_t) == 0,
+              "AddressableColorWipeEffectColor must be whole words for ESP8266 flash reads");
+static constexpr uint8_t NO_RANDOM_SLOT = 0xFF;
 
 class AddressableColorWipeEffect : public AddressableLightEffect {
  public:
   explicit AddressableColorWipeEffect(const char *name) : AddressableLightEffect(name) {}
-  void set_colors(const std::initializer_list<AddressableColorWipeEffectColor> &colors) { this->colors_ = colors; }
+  /// Codegen only: point at the generated flash table of colors, which must outlive the effect.
+  /// Random colors keep their current value in a RAM array, one slot per random entry.
+  void set_colors(const AddressableColorWipeEffectColor *colors, size_t count, Color *random_colors) {
+    this->colors_ = {colors, count};
+    this->random_colors_ = random_colors;
+  }
   void set_add_led_interval(uint32_t add_led_interval) { this->add_led_interval_ = add_led_interval; }
   void set_reverse(bool reverse) { this->reverse_ = reverse; }
   void apply(AddressableLight &it, const Color &current_color) override {
@@ -124,12 +137,11 @@ class AddressableColorWipeEffect : public AddressableLightEffect {
     } else {
       it.shift_right(1);
     }
-    const AddressableColorWipeEffectColor &color = this->colors_[this->at_color_];
-    Color esp_color = Color(color.r, color.g, color.b, color.w);
+    const AddressableColorWipeEffectColor color = this->entry_(this->at_color_);
+    Color esp_color = this->color_of_(color);
     if (color.gradient) {
       size_t next_color_index = (this->at_color_ + 1) % this->colors_.size();
-      const AddressableColorWipeEffectColor &next_color = this->colors_[next_color_index];
-      const Color next_esp_color = Color(next_color.r, next_color.g, next_color.b, next_color.w);
+      const Color next_esp_color = this->color_of_(this->entry_(next_color_index));
       uint8_t gradient = 255 * ((float) this->leds_added_ / color.num_leds);
       esp_color = esp_color.gradient(next_esp_color, gradient);
     }
@@ -141,19 +153,32 @@ class AddressableColorWipeEffect : public AddressableLightEffect {
     if (++this->leds_added_ >= color.num_leds) {
       this->leds_added_ = 0;
       this->at_color_ = (this->at_color_ + 1) % this->colors_.size();
-      AddressableColorWipeEffectColor &new_color = this->colors_[this->at_color_];
-      if (new_color.random) {
+      const AddressableColorWipeEffectColor new_color = this->entry_(this->at_color_);
+      if (new_color.random_slot != NO_RANDOM_SLOT) {
         Color c = Color::random_color();
-        new_color.r = c.r;
-        new_color.g = c.g;
-        new_color.b = c.b;
+        Color &slot = this->random_colors_[new_color.random_slot];
+        slot.r = c.r;
+        slot.g = c.g;
+        slot.b = c.b;
       }
     }
     it.schedule_show();
   }
 
  protected:
-  FixedVector<AddressableColorWipeEffectColor> colors_;
+  AddressableColorWipeEffectColor entry_(size_t index) const {
+    AddressableColorWipeEffectColor entry;
+    progmem_memcpy(&entry, &this->colors_[index], sizeof(entry));
+    return entry;
+  }
+  Color color_of_(const AddressableColorWipeEffectColor &entry) const {
+    if (entry.random_slot != NO_RANDOM_SLOT)
+      return this->random_colors_[entry.random_slot];
+    return Color(entry.r, entry.g, entry.b, entry.w);
+  }
+
+  ConstVector<AddressableColorWipeEffectColor> colors_;
+  Color *random_colors_{nullptr};
   size_t at_color_{0};
   uint32_t last_add_{0};
   uint32_t add_led_interval_{};

esphome/components/light/base_light_effects.h

@@ -1,8 +1,10 @@
 #pragma once
 
+#include <type_traits>
 #include <utility>
 
 #include "esphome/core/automation.h"
+#include "esphome/core/hal.h"
 #include "esphome/core/helpers.h"
 #include "light_effect.h"
 
@@ -154,11 +156,25 @@ class AutomationLightEffect : public LightEffect {
   Trigger<> trig_;
 };
 
+/// One strobe step. Codegen keeps the steps in a flash table, so it is read with progmem_memcpy.
 struct StrobeLightEffectColor {
-  LightColorValues color;
+  float state;
+  float brightness;
+  float color_brightness;
+  float red;
+  float green;
+  float blue;
+  float white;
+  float color_temperature;
+  float cold_white;
+  float warm_white;
   uint32_t duration;
   uint32_t transition_length;
+  ColorMode color_mode;
 };
+static_assert(std::is_trivially_copyable_v<StrobeLightEffectColor>, "StrobeLightEffectColor is copied out of flash");
+static_assert(sizeof(StrobeLightEffectColor) % sizeof(uint32_t) == 0,
+              "StrobeLightEffectColor must be whole words for ESP8266 flash reads");
 
 class StrobeLightEffect : public LightEffect {
  public:
@@ -166,19 +182,22 @@ class StrobeLightEffect : public LightEffect {
   void start() override {
     // Place the cycle at the end of the last color, so the first apply() switches straight to the first color
     this->at_color_ = this->colors_.size() - 1;
-    this->last_switch_ = millis() - this->colors_.back().duration;
+    this->last_switch_ = millis() - this->duration_(this->at_color_);
   }
   void apply() override {
     const uint32_t now = millis();
-    if (now - this->last_switch_ < this->colors_[this->at_color_].duration)
+    if (now - this->last_switch_ < this->duration_(this->at_color_))
       return;
 
     // Switch to next color
     this->at_color_ = (this->at_color_ + 1) % this->colors_.size();
-    auto color = this->colors_[this->at_color_].color;
+    StrobeLightEffectColor step;
+    progmem_memcpy(&step, &this->colors_[this->at_color_], sizeof(step));
+    LightColorValues color(step.color_mode, step.state, step.brightness, step.color_brightness, step.red, step.green,
+                           step.blue, step.white, step.color_temperature, step.cold_white, step.warm_white);
 
     auto call = this->state_->turn_on();
-    call.from_light_color_values(this->colors_[this->at_color_].color);
+    call.from_light_color_values(color);
 
     if (!color.is_on()) {
       // Don't turn the light off, otherwise the light effect will be stopped
@@ -187,15 +206,22 @@ class StrobeLightEffect : public LightEffect {
     }
     call.set_publish(false);
     call.set_save(false);
-    call.set_transition_length_if_supported(this->colors_[this->at_color_].transition_length);
+    call.set_transition_length_if_supported(step.transition_length);
     call.perform();
     this->last_switch_ = now;
   }
 
-  void set_colors(const std::initializer_list<StrobeLightEffectColor> &colors) { this->colors_ = colors; }
+  /// Codegen only: point at the generated flash table of steps, which must outlive the effect.
+  void set_colors(const StrobeLightEffectColor *colors, size_t count) { this->colors_ = {colors, count}; }
 
  protected:
-  FixedVector<StrobeLightEffectColor> colors_;
+  uint32_t duration_(size_t index) const {
+    uint32_t duration;
+    progmem_memcpy(&duration, &this->colors_[index].duration, sizeof(duration));
+    return duration;
+  }
+
+  ConstVector<StrobeLightEffectColor> colors_;
   uint32_t last_switch_{0};
   size_t at_color_{0};
 };

esphome/components/light/channel_colors.h

@@ -2,6 +2,9 @@
 
 #include <cstdint>
 
+#include "esp_color_view.h"
+#include "light_traits.h"
+
 namespace esphome::light {
 
 /// Which byte of an addressable LED's data carries each colour.
@@ -36,6 +39,30 @@ struct ChannelColors {
     buf[this->bytes_per_led()] = '\0';
     return buf;
   }
+
+  /// Colour modes of a strip with this channel order.
+  LightTraits traits() const {
+    LightTraits traits;
+    if (this->has_white()) {
+      traits.set_supported_color_modes({ColorMode::RGB_WHITE, ColorMode::WHITE});
+    } else {
+      traits.set_supported_color_modes({ColorMode::RGB});
+    }
+    return traits;
+  }
+
+  /// View of LED `index` in `buf`; a scratch pixel while `buf` or `effect_data` is unallocated (failed setup()).
+  ESPColorView view(uint8_t *buf, uint8_t *effect_data, int32_t index, const ESPColorCorrection *correction) const {
+    static uint8_t scratch[4];
+    if (buf == nullptr || effect_data == nullptr) {
+      buf = scratch;
+      effect_data = nullptr;
+      index = 0;
+    }
+    uint8_t *led = buf + index * this->bytes_per_led();
+    return {led + this->r,       led + this->g, led + this->b, this->has_white() ? led + this->w : nullptr,
+            effect_data + index, correction};
+  }
 };
 
 }  // namespace esphome::light

esphome/components/light/effects.py

@@ -29,6 +29,7 @@ from esphome.const import (
     CONF_WHITE,
     CONF_WIDTH,
 )
+from esphome.core import CORE
 from esphome.cpp_generator import MockObjClass
 from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor
 from esphome.types import ConfigType
@@ -52,7 +53,6 @@ from .types import (
     ColorMode,
     FlickerLightEffect,
     LambdaLightEffect,
-    LightColorValues,
     LightStateRef,
     PulseLightEffect,
     RandomLightEffect,
@@ -351,28 +351,26 @@ async def strobe_effect_to_code(config, effect_id):
     colors = [
         cg.StructInitializer(
             StrobeLightEffectColor,
-            (
-                "color",
-                LightColorValues(
-                    color.get(CONF_COLOR_MODE, ColorMode.UNKNOWN),
-                    color[CONF_STATE],
-                    color[CONF_BRIGHTNESS],
-                    color[CONF_COLOR_BRIGHTNESS],
-                    color[CONF_RED],
-                    color[CONF_GREEN],
-                    color[CONF_BLUE],
-                    color[CONF_WHITE],
-                    color.get(CONF_COLOR_TEMPERATURE, 0.0),
-                    color[CONF_COLD_WHITE],
-                    color[CONF_WARM_WHITE],
-                ),
-            ),
+            ("state", color[CONF_STATE]),
+            ("brightness", color[CONF_BRIGHTNESS]),
+            ("color_brightness", color[CONF_COLOR_BRIGHTNESS]),
+            ("red", color[CONF_RED]),
+            ("green", color[CONF_GREEN]),
+            ("blue", color[CONF_BLUE]),
+            ("white", color[CONF_WHITE]),
+            ("color_temperature", color.get(CONF_COLOR_TEMPERATURE, 0.0)),
+            ("cold_white", color[CONF_COLD_WHITE]),
+            ("warm_white", color[CONF_WARM_WHITE]),
             ("duration", color[CONF_DURATION]),
             ("transition_length", color[CONF_TRANSITION_LENGTH]),
+            ("color_mode", color.get(CONF_COLOR_MODE, ColorMode.UNKNOWN)),
         )
         for color in config.get(CONF_COLORS, [])
     ]
-    cg.add(var.set_colors(colors))
+    table = cg.shared_progmem_array(
+        "light_strobe_colors", StrobeLightEffectColor, cg.ArrayInitializer(*colors)
+    )
+    cg.add(var.set_colors(table, len(colors)))
     return var
 
 
@@ -431,6 +429,16 @@ async def addressable_rainbow_effect_to_code(config, effect_id):
     return var
 
 
+# Matches NO_RANDOM_SLOT in addressable_light_effect.h
+NO_RANDOM_SLOT = 0xFF
+
+
+def _validate_random_slots(colors: list[ConfigType]) -> list[ConfigType]:
+    if sum(1 for color in colors if color[CONF_RANDOM]) > NO_RANDOM_SLOT:
+        raise cv.Invalid(f"At most {NO_RANDOM_SLOT} colors can be random")
+    return colors
+
+
 ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA = cv.Schema(
     {
         cv.Optional(CONF_COLOR_BRIGHTNESS): cv.percentage,
@@ -459,7 +467,11 @@ ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA = _with_color_option(
     {
         cv.Optional(
             CONF_COLORS, default=[{CONF_NUM_LEDS: 1, CONF_RANDOM: True}]
-        ): cv.ensure_list(ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA),
+        ): cv.All(
+            cv.ensure_list(ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA),
+            cv.Length(min=1),
+            _validate_random_slots,
+        ),
         cv.Optional(
             CONF_ADD_LED_INTERVAL, default="0.1s"
         ): cv.positive_time_period_milliseconds,
@@ -471,21 +483,47 @@ async def addressable_color_wipe_effect_to_code(config, effect_id):
     cg.add(var.set_add_led_interval(config[CONF_ADD_LED_INTERVAL]))
     cg.add(var.set_reverse(config[CONF_REVERSE]))
     colors = []
+    random_colors = []
     for color in config.get(CONF_COLORS, []):
         color_brightness = color[CONF_COLOR_BRIGHTNESS]
+        rgbw = (
+            int(round(color[CONF_RED] * color_brightness * 255)),
+            int(round(color[CONF_GREEN] * color_brightness * 255)),
+            int(round(color[CONF_BLUE] * color_brightness * 255)),
+            int(round(color[CONF_WHITE] * 255)),
+        )
+        random_slot = cg.RawExpression("light::NO_RANDOM_SLOT")
+        if color[CONF_RANDOM]:
+            random_slot = len(random_colors)
+            random_colors.append(rgbw)
         colors.append(
             cg.StructInitializer(
                 AddressableColorWipeEffectColor,
-                ("r", int(round(color[CONF_RED] * color_brightness * 255))),
-                ("g", int(round(color[CONF_GREEN] * color_brightness * 255))),
-                ("b", int(round(color[CONF_BLUE] * color_brightness * 255))),
-                ("w", int(round(color[CONF_WHITE] * 255))),
-                ("random", color[CONF_RANDOM]),
-                ("num_leds", color[CONF_NUM_LEDS]),
+                ("r", rgbw[0]),
+                ("g", rgbw[1]),
+                ("b", rgbw[2]),
+                ("w", rgbw[3]),
+                ("random_slot", random_slot),
                 ("gradient", color[CONF_GRADIENT]),
+                ("num_leds", color[CONF_NUM_LEDS]),
             )
         )
-    cg.add(var.set_colors(colors))
+    table = cg.shared_progmem_array(
+        "light_color_wipe_colors",
+        AddressableColorWipeEffectColor,
+        cg.ArrayInitializer(*colors),
+    )
+    random_storage = cg.nullptr
+    if random_colors:
+        # One RAM slot per random entry; it starts at the configured color, as before
+        id_ = cg.unique_global_id(f"{effect_id.id}_random_colors", Color)
+        values = cg.ArrayInitializer(
+            *(Color(r, g, b, w) for r, g, b, w in random_colors)
+        )
+        cg.add_global(cg.RawStatement(f"static {Color} {id_}[] = {values};"))
+        random_storage = cg.MockObj(id_, ".")
+        CORE.register_variable(id_, random_storage)
+    cg.add(var.set_colors(table, len(colors), random_storage))
     return var
 
 

esphome/components/ln882x_spi_led_strip/led_strip.cpp

@@ -141,32 +141,13 @@ void LN882xSPILEDStripLightOutput::dump_config() {
 
 float LN882xSPILEDStripLightOutput::get_setup_priority() const { return setup_priority::HARDWARE; }
 
-light::LightTraits LN882xSPILEDStripLightOutput::get_traits() {
-  auto traits = light::LightTraits();
-  if (this->channel_colors_.has_white()) {
-    traits.set_supported_color_modes({light::ColorMode::RGB_WHITE, light::ColorMode::WHITE});
-  } else {
-    traits.set_supported_color_modes({light::ColorMode::RGB});
-  }
-  return traits;
-}
-
-void LN882xSPILEDStripLightOutput::clear_effect_data() {
-  if (this->effect_data_ != nullptr)
-    memset(this->effect_data_, 0, this->num_leds_);
-}
-
 void LN882xSPILEDStripLightOutput::write_state(light::LightState *state) {
   if (this->is_failed())
     return;
 
-  // protect from refreshing too often
   const uint32_t now = micros();
-  if (this->max_refresh_rate_ != 0 && (now - this->last_refresh_) < this->max_refresh_rate_) {
-    // try again next loop iteration, so that this change won't get lost
-    this->schedule_show();
+  if (this->throttle_show_(now, this->last_refresh_, this->max_refresh_rate_))
     return;
-  }
 
   if (spi_dma_busy()) {
     // Busy sending the last frame, try again later.
@@ -191,20 +172,6 @@ void LN882xSPILEDStripLightOutput::write_state(light::LightState *state) {
   spi_dma_transmit(this->dma_data_, this->dma_data_bytes_);
 }
 
-light::ESPColorView LN882xSPILEDStripLightOutput::get_view_internal(int32_t index) const {
-  const light::ChannelColors &colors = this->channel_colors_;
-  // After a failed setup() effects and partitions still write pixels, so they get a scratch pixel.
-  static uint8_t scratch[5];
-  const bool ready = this->color_data_ != nullptr;
-  uint8_t *led = ready ? this->color_data_ + (index * colors.bytes_per_led()) : scratch;
-  return {led + colors.r,
-          led + colors.g,
-          led + colors.b,
-          colors.has_white() ? led + colors.w : nullptr,
-          ready ? &this->effect_data_[index] : scratch + 4,
-          &this->correction_};
-}
-
 }  // namespace esphome::ln882x_spi_led_strip
 
 #endif  // USE_LN882X

esphome/components/ln882x_spi_led_strip/led_strip.h

@@ -23,16 +23,21 @@ class LN882xSPILEDStripLightOutput final : public light::AddressableLight {
   void dump_config() override;
   float get_setup_priority() const override;
 
-  light::LightTraits get_traits() override;
+  light::LightTraits get_traits() override { return this->channel_colors_.traits(); }
   int32_t size() const override { return this->num_leds_; }
-  void clear_effect_data() override;
+  void clear_effect_data() override {
+    if (this->effect_data_ != nullptr)
+      memset(this->effect_data_, 0, this->num_leds_);
+  }
   void write_state(light::LightState *state) override;
 
   /// Set a maximum refresh rate in µs as some lights do not like being updated too often.
   void set_max_refresh_rate(uint32_t interval_us) { this->max_refresh_rate_ = interval_us; }
 
  protected:
-  light::ESPColorView get_view_internal(int32_t index) const override;
+  light::ESPColorView get_view_internal(int32_t index) const override {
+    return this->channel_colors_.view(this->color_data_, this->effect_data_, index, &this->correction_);
+  }
 
   const size_t color_data_bytes_;
   const size_t dma_data_bytes_;

esphome/components/lvgl/defines.py

@@ -729,7 +729,6 @@ CONF_DISP_BG_COLOR = "disp_bg_color"
 CONF_DISP_BG_IMAGE = "disp_bg_image"
 CONF_DISP_BG_OPA = "disp_bg_opa"
 CONF_BODY = "body"
-CONF_BUTTONS = "buttons"
 CONF_CHANGE_RATE = "change_rate"
 CONF_CLICKABLE = "clickable"
 CONF_CLOSE_BUTTON = "close_button"

esphome/components/lvgl/lvgl_esphome.cpp

@@ -887,7 +887,7 @@ void LvglComponent::setup() {
   lv_timer_set_period(this->refr_timer_, this->refr_timer_period_);
 #if LV_USE_LOG
   lv_log_register_print_cb([](lv_log_level_t level, const char *buf) {
-    auto next = strchr(buf, ')');
+    auto *next = strchr(buf, ')');
     if (next != nullptr)
       buf = next + 1;
     while (isspace(*buf))

esphome/components/lvgl/widgets/__init__.py

@@ -210,10 +210,15 @@ class WidgetType:
 
     async def update_to_code(self, w: "Widget", config: dict) -> None:
         """
-        Update a widget. Defaults to calling to_code, but can be overridden
+        Update a widget. Defaults to calling to_code when the widget declares
+        update options; a widget with an empty modify_schema has nothing
+        widget-specific to update because the shared update path already
+        applies the common properties. Can be overridden.
         :param w: The widget to update
         :param config: The configuration for the update
         """
+        if not self.modify_schema.schema:
+            return
         await self.to_code(w, config)
 
     async def obj_creator(self, parent: MockObj, config: dict):

esphome/components/lvgl/widgets/buttonmatrix.py

@@ -1,6 +1,6 @@
 from esphome import automation
 import esphome.codegen as cg
-from esphome.components.const import CONF_ROWS
+from esphome.components.const import CONF_BUTTONS, CONF_ROWS
 from esphome.components.key_provider import KeyProvider
 import esphome.config_validation as cv
 from esphome.const import CONF_ID, CONF_ITEMS, CONF_TEXT, CONF_WIDTH
@@ -10,7 +10,6 @@ from esphome.types import Expression
 from ..automation import action_to_code
 from ..defines import (
     BUTTONMATRIX_CTRLS,
-    CONF_BUTTONS,
     CONF_CONTROL,
     CONF_KEY_CODE,
     CONF_MAIN,

esphome/components/lvgl/widgets/msgbox.py

@@ -1,4 +1,5 @@
 from esphome import codegen as cg, config_validation as cv
+from esphome.components.const import CONF_BUTTONS
 from esphome.const import CONF_BUTTON, CONF_HIDDEN, CONF_ID, CONF_TEXT
 from esphome.core import ID
 from esphome.cpp_generator import MockObjClass
@@ -7,7 +8,6 @@ from esphome.cpp_types import nullptr
 from ..defines import (
     CONF_BODY,
     CONF_BUTTON_STYLE,
-    CONF_BUTTONS,
     CONF_CLOSE_BUTTON,
     CONF_HEADER_BUTTONS,
     CONF_MAIN,

esphome/components/mcp23016/__init__.py

@@ -20,10 +20,9 @@ CONFIG_SCHEMA = (
     cv.Schema(
         {
             cv.Required(CONF_ID): cv.declare_id(MCP23016),
-            cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin,
         }
     )
-    .extend(cv.COMPONENT_SCHEMA)
+    .extend(gpio_expander.INTERRUPT_EXPANDER_SCHEMA)
     .extend(i2c.i2c_device_schema(0x20))
 )
 

esphome/components/mcp23016/mcp23016.cpp

@@ -25,25 +25,9 @@ void MCP23016::setup() {
   // all pins input
   this->write_reg_(MCP23016_IODIR1, 0xFFFF);
 
-  if (this->interrupt_pin_ != nullptr) {
-    this->interrupt_pin_->setup();
-    this->interrupt_pin_->attach_interrupt(&MCP23016::gpio_intr, this, gpio::INTERRUPT_FALLING_EDGE);
-    this->set_invalidate_on_read_(false);
-  }
-  this->disable_loop();
+  this->setup_interrupt_pin_();
 }
 
-void IRAM_ATTR MCP23016::gpio_intr(MCP23016 *arg) { arg->enable_loop_soon_any_context(); }
-void MCP23016::loop() {
-  // Invalidate cache at the start of each loop
-  this->reset_pin_cache_();
-  // Only disable the loop once INT has actually gone HIGH. Input transitions that straddle the
-  // I2C read leave INT asserted without re-firing a falling edge, which would strand us with
-  // stale state forever; keep looping until the line is released so we self-heal.
-  if (this->interrupt_pin_ != nullptr && this->interrupt_pin_->digital_read()) {
-    this->disable_loop();
-  }
-}
 bool MCP23016::digital_read_hw(uint8_t pin) { return this->read_reg_(MCP23016_GP1, &this->input_mask_); }
 
 bool MCP23016::digital_read_cache(uint8_t pin) { return this->input_mask_ & (1 << pin); }
@@ -51,14 +35,11 @@ void MCP23016::digital_write_hw(uint8_t pin, bool value) { this->update_reg_(pin
 void MCP23016::pin_mode(uint8_t pin, gpio::Flags flags) {
   if (flags == gpio::FLAG_INPUT) {
     this->update_reg_(pin, true, MCP23016_IODIR1);
-    if (this->interrupt_pin_ == nullptr) {
-      this->enable_loop();
-    }
+    this->enable_polling_loop_();
   } else if (flags == gpio::FLAG_OUTPUT) {
     this->update_reg_(pin, false, MCP23016_IODIR1);
   }
 }
-float MCP23016::get_setup_priority() const { return setup_priority::IO; }
 bool MCP23016::read_reg_(uint8_t reg, uint16_t *value) {
   if (this->is_failed())
     return false;

esphome/components/mcp23016/mcp23016.h

@@ -3,7 +3,7 @@
 #include "esphome/core/component.h"
 #include "esphome/core/hal.h"
 #include "esphome/components/i2c/i2c.h"
-#include "esphome/components/gpio_expander/cached_gpio.h"
+#include "esphome/components/gpio_expander/cached_gpio_component.h"
 #include "esphome/components/gpio_expander/expander_gpio_pin.h"
 
 namespace esphome::mcp23016 {
@@ -25,20 +25,14 @@ enum MCP23016GPIORegisters {
   MCP23016_IOCON1 = 0x0B,
 };
 
-class MCP23016 final : public Component, public i2c::I2CDevice, public gpio_expander::CachedGpioExpander<uint16_t, 16> {
+class MCP23016 final : public gpio_expander::CachedGpioExpanderComponent<uint16_t, 16>, public i2c::I2CDevice {
  public:
   MCP23016() = default;
 
   void setup() override;
-  void loop() override;
   void pin_mode(uint8_t pin, gpio::Flags flags);
 
-  float get_setup_priority() const override;
-
-  void set_interrupt_pin(InternalGPIOPin *pin) { this->interrupt_pin_ = pin; }
-
  protected:
-  static void gpio_intr(MCP23016 *arg);
   // Virtual methods from CachedGpioExpander
   bool digital_read_hw(uint8_t pin) override;
   bool digital_read_cache(uint8_t pin) override;
@@ -54,7 +48,6 @@ class MCP23016 final : public Component, public i2c::I2CDevice, public gpio_expa
   uint16_t olat_{0x0000};
   // Cache for input values (16-bit combined for both banks)
   uint16_t input_mask_{0x0000};
-  InternalGPIOPin *interrupt_pin_{nullptr};
 };
 
 using MCP23016GPIOPin = gpio_expander::ExpanderGPIOPin<MCP23016>;

esphome/components/mcp23x08_base/mcp23x08_base.cpp

@@ -32,15 +32,11 @@ void MCP23X08Base::pin_mode(uint8_t pin, gpio::Flags flags) {
   } else if (flags == gpio::FLAG_OUTPUT) {
     this->update_reg(pin, false, iodir);
   }
-  // When interrupt_pin is configured, auto-enable CHANGE interrupt for input pins
-  // so the chip's INT output fires on any input state change
-  if (this->interrupt_pin_ != nullptr && (flags & gpio::FLAG_INPUT)) {
-    this->pin_interrupt_mode(pin, mcp23xxx_base::MCP23XXX_CHANGE);
-  }
-  // Enable polling loop for input pins (not needed for interrupt-driven mode
-  // where the ISR handles re-enabling loop)
-  if (this->interrupt_pin_ == nullptr && (flags & gpio::FLAG_INPUT)) {
-    this->enable_loop();
+  if (flags & gpio::FLAG_INPUT) {
+    // With an interrupt pin, INT fires on any change of an input
+    if (this->interrupt_pin_ != nullptr)
+      this->pin_interrupt_mode(pin, mcp23xxx_base::MCP23XXX_CHANGE);
+    this->enable_polling_loop_();
   }
 }
 

esphome/components/mcp23x17_base/mcp23x17_base.cpp

@@ -43,15 +43,11 @@ void MCP23X17Base::pin_mode(uint8_t pin, gpio::Flags flags) {
   } else if (flags == gpio::FLAG_OUTPUT) {
     this->update_reg(pin, false, iodir);
   }
-  // When interrupt_pin is configured, auto-enable CHANGE interrupt for input pins
-  // so the chip's INT output fires on any input state change
-  if (this->interrupt_pin_ != nullptr && (flags & gpio::FLAG_INPUT)) {
-    this->pin_interrupt_mode(pin, mcp23xxx_base::MCP23XXX_CHANGE);
-  }
-  // Enable polling loop for input pins (not needed for interrupt-driven mode
-  // where the ISR handles re-enabling loop)
-  if (this->interrupt_pin_ == nullptr && (flags & gpio::FLAG_INPUT)) {
-    this->enable_loop();
+  if (flags & gpio::FLAG_INPUT) {
+    // With an interrupt pin, INT fires on any change of an input
+    if (this->interrupt_pin_ != nullptr)
+      this->pin_interrupt_mode(pin, mcp23xxx_base::MCP23XXX_CHANGE);
+    this->enable_polling_loop_();
   }
 }
 

esphome/components/mcp23xxx_base/__init__.py

@@ -32,9 +32,8 @@ MCP23XXX_INTERRUPT_MODES = MCP23XXXInterruptMode.options(
 MCP23XXX_CONFIG_SCHEMA = cv.Schema(
     {
         cv.Optional(CONF_OPEN_DRAIN_INTERRUPT, default=False): cv.boolean,
-        cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin,
     }
-).extend(cv.COMPONENT_SCHEMA)
+).extend(gpio_expander.INTERRUPT_EXPANDER_SCHEMA)
 
 
 @coroutine

esphome/components/mcp23xxx_base/mcp23xxx_base.cpp

@@ -4,8 +4,6 @@
 
 namespace esphome::mcp23xxx_base {
 
-void IRAM_ATTR mcp23xxx_gpio_intr(Component *arg) { arg->enable_loop_soon_any_context(); }
-
 template<uint8_t N> void MCP23XXXGPIOPin<N>::setup() {
   this->pin_mode(flags_);
   // When interrupt_pin is configured, pin_mode() already auto-enables CHANGE

esphome/components/mcp23xxx_base/mcp23xxx_base.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "esphome/components/gpio_expander/cached_gpio.h"
+#include "esphome/components/gpio_expander/cached_gpio_component.h"
 #include "esphome/core/component.h"
 #include "esphome/core/hal.h"
 
@@ -8,47 +8,15 @@ namespace esphome::mcp23xxx_base {
 
 enum MCP23XXXInterruptMode : uint8_t { MCP23XXX_NO_INTERRUPT = 0, MCP23XXX_CHANGE, MCP23XXX_RISING, MCP23XXX_FALLING };
 
-// Not a template member: GCC 10 (ESP8266) ignores section attributes on template instantiations, leaving the
-// ISR in flash.
-void mcp23xxx_gpio_intr(Component *arg);
-
-template<uint8_t N> class MCP23XXXBase : public Component, public gpio_expander::CachedGpioExpander<uint8_t, N> {
+template<uint8_t N> class MCP23XXXBase : public gpio_expander::CachedGpioExpanderComponent<uint8_t, N> {
  public:
   virtual void pin_mode(uint8_t pin, gpio::Flags flags);
   virtual void pin_interrupt_mode(uint8_t pin, MCP23XXXInterruptMode interrupt_mode);
 
   void set_open_drain_ints(const bool value) { this->open_drain_ints_ = value; }
-  void set_interrupt_pin(InternalGPIOPin *pin) { this->interrupt_pin_ = pin; }
   InternalGPIOPin *get_interrupt_pin() const { return this->interrupt_pin_; }
-  float get_setup_priority() const override { return setup_priority::IO; }
-
-  void loop() override {
-    this->reset_pin_cache_();
-    // Only disable the loop once INT has actually gone HIGH. Input transitions that straddle the
-    // I2C read leave INT asserted without re-firing a falling edge, which would strand us with
-    // stale state forever; keep looping until the line is released so we self-heal.
-    if (this->interrupt_pin_ != nullptr && this->interrupt_pin_->digital_read()) {
-      this->disable_loop();
-    }
-  }
 
  protected:
-  // No need to clear latched interrupts before attaching the ISR — if INT is
-  // already low the ISR fires immediately, loop runs, cache invalidates, and
-  // the GPIO read clears the latch. One harmless extra read at most.
-  void setup_interrupt_pin_() {
-    if (this->interrupt_pin_ != nullptr) {
-      this->interrupt_pin_->setup();
-      this->interrupt_pin_->attach_interrupt(&mcp23xxx_gpio_intr, static_cast<Component *>(this),
-                                             gpio::INTERRUPT_FALLING_EDGE);
-      this->set_invalidate_on_read_(false);
-    }
-    // Disable loop until an input pin is configured via pin_mode()
-    // For interrupt-driven mode, loop is re-enabled by the ISR
-    // For polling mode, loop is re-enabled when pin_mode() registers an input pin
-    this->disable_loop();
-  }
-
   // read a given register
   virtual bool read_reg(uint8_t reg, uint8_t *value) = 0;
   // write a value to a given register
@@ -57,7 +25,6 @@ template<uint8_t N> class MCP23XXXBase : public Component, public gpio_expander:
   virtual void update_reg(uint8_t pin, bool pin_value, uint8_t reg_a) = 0;
 
   bool open_drain_ints_;
-  InternalGPIOPin *interrupt_pin_{nullptr};
 };
 
 template<uint8_t N> class MCP23XXXGPIOPin final : public GPIOPin {

esphome/components/midea/air_conditioner.cpp

@@ -173,9 +173,8 @@ void AirConditioner::do_follow_me(float temperature, bool use_fahrenheit, bool b
     return;
   }
 
-  // Round and convert temperature to long, then clamp and convert it to uint8_t
-  uint8_t temp_uint8 =
-      static_cast<uint8_t>(esphome::clamp<long>(std::lroundf(temperature), 0L, static_cast<long>(UINT8_MAX)));
+  // Clamp to the uint8_t range before rounding so lroundf never sees an out-of-range value
+  uint8_t temp_uint8 = static_cast<uint8_t>(std::lroundf(esphome::clamp(temperature, 0.0f, 255.0f)));
 
   char temp_symbol = use_fahrenheit ? 'F' : 'C';
   ESP_LOGD(Constants::TAG, "Follow me action called with temperature: %.5f °%c, rounded to: %u °%c", temperature,

esphome/components/modem/__init__.py

@@ -0,0 +1,232 @@
+import voluptuous as vol
+
+from esphome import automation, pins
+import esphome.codegen as cg
+from esphome.components.const import CONF_RTS_PIN
+from esphome.components.esp32 import add_idf_component, add_idf_sdkconfig_option
+from esphome.components.network import add_use_address, require_ipv4
+import esphome.config_validation as cv
+from esphome.const import (
+    CONF_BAUD_RATE,
+    CONF_COMMAND,
+    CONF_ENABLE_ON_BOOT,
+    CONF_ID,
+    CONF_MODEL,
+    CONF_ON_CONNECT,
+    CONF_ON_DISCONNECT,
+    CONF_REBOOT_TIMEOUT,
+    CONF_RX_BUFFER_SIZE,
+    CONF_RX_PIN,
+    CONF_TX_BUFFER_SIZE,
+    CONF_TX_PIN,
+    CONF_USE_ADDRESS,
+)
+from esphome.core import CORE, ID, coroutine_with_priority
+from esphome.coroutine import CoroPriority
+from esphome.cpp_generator import MockObj
+from esphome.types import ConfigType, TemplateArgsType
+
+# Static so `api:` finds network before its dependency check; a config-dependent
+# AUTO_LOAD resolves after that check and fails a cellular-only node
+AUTO_LOAD = ["network"]
+CODEOWNERS = ["@oarcher"]
+CONFLICTS_WITH = ["captive_portal", "ethernet"]
+DEPENDENCIES = ["esp32"]
+DOMAIN = "modem"
+
+CONF_PIN_CODE = "pin_code"
+CONF_APN = "apn"
+CONF_CTS_PIN = "cts_pin"
+CONF_INIT_AT = "init_at"
+CONF_ON_SYNC = "on_sync"
+CONF_DTE_BUFFER_SIZE = "dte_buffer_size"
+CONF_MODEM_ID = "modem_id"
+
+esp_modem_ns = cg.global_ns.namespace("esp_modem")
+
+# Model name to the esp_modem factory that drives it
+MODEM_MODELS = {
+    "BG96": esp_modem_ns.create_BG96_dce,
+    "SIM800": esp_modem_ns.create_SIM800_dce,
+    "SIM7000": esp_modem_ns.create_SIM7000_dce,
+    "SIM7070": esp_modem_ns.create_SIM7070_dce,
+    "SIM7080": esp_modem_ns.create_SIM7000_dce,
+    "SIM7600": esp_modem_ns.create_SIM7600_dce,
+    "SIM7670": esp_modem_ns.create_SIM7600_dce,
+    "GENERIC": esp_modem_ns.create_generic_dce,
+}
+modem_ns = cg.esphome_ns.namespace("modem")
+ModemComponent = modem_ns.class_("ModemComponent", cg.Component)
+
+ModemState = modem_ns.enum("ModemState", is_class=True)
+
+
+def _on_state(conf_key: str, state: str) -> automation.CallbackAutomation:
+    return automation.CallbackAutomation(
+        conf_key,
+        "add_on_state_callback",
+        params=[(ModemState, "state")],
+        forward=[],
+        when=f"state == modem::ModemState::{state}",
+    )
+
+
+def _default_use_address(config: ConfigType) -> ConfigType:
+    # The host tools resolve the device through this, like the other network components
+    config.setdefault(CONF_USE_ADDRESS, f"{CORE.name}.local")
+    return config
+
+
+_CALLBACK_AUTOMATIONS = (
+    _on_state(CONF_ON_CONNECT, "MODEM_STATE_CONNECTED"),
+    _on_state(CONF_ON_DISCONNECT, "MODEM_STATE_DISCONNECTING"),
+    _on_state(CONF_ON_SYNC, "MODEM_STATE_SYNCED"),
+)
+
+# AT_COMMAND_SIZE - 1 in modem_component.h; the buffer keeps one byte for the terminator
+MAX_AT_COMMAND_LENGTH = 127
+
+
+# A SIM PIN is 4 to 8 digits; anything else would spend one of the SIM's attempts. \Z, not $, so a
+# trailing newline is refused too
+_validate_pin_code = cv.All(
+    cv.string_strict,
+    vol.Match(r"^[0-9]{4,8}\Z", msg="pin_code must be 4 to 8 digits"),
+)
+# The IDF UART driver needs more than its 128 byte FIFO; the setters take uint16_t
+_UART_BUFFER_SIZE = cv.int_range(min=129, max=65535)
+
+
+CONFIG_SCHEMA = cv.All(
+    cv.Schema(
+        {
+            cv.GenerateID(): cv.declare_id(ModemComponent),
+            cv.Required(CONF_TX_PIN): pins.internal_gpio_output_pin_schema,
+            cv.Required(CONF_RX_PIN): pins.internal_gpio_input_pin_schema,
+            cv.Optional(CONF_RTS_PIN): pins.internal_gpio_output_pin_schema,
+            cv.Optional(CONF_CTS_PIN): pins.internal_gpio_input_pin_schema,
+            cv.Optional(CONF_BAUD_RATE): cv.positive_int,
+            cv.Required(CONF_MODEL): cv.one_of(*MODEM_MODELS, upper=True),
+            cv.Required(CONF_APN): cv.string,
+            cv.Optional(CONF_PIN_CODE): cv.sensitive(_validate_pin_code),
+            cv.Optional(CONF_USE_ADDRESS): cv.string_strict,
+            # The command count is stored in a byte
+            cv.Optional(CONF_INIT_AT): cv.All(
+                cv.ensure_list(
+                    cv.All(cv.string, cv.ByteLength(max=MAX_AT_COMMAND_LENGTH))
+                ),
+                cv.Length(max=255),
+            ),
+            cv.Optional(CONF_ENABLE_ON_BOOT, default=True): cv.boolean,
+            cv.Optional(CONF_TX_BUFFER_SIZE, default=1024): _UART_BUFFER_SIZE,
+            cv.Optional(CONF_RX_BUFFER_SIZE, default=1024): _UART_BUFFER_SIZE,
+            # CMUX keeps 128 bytes of the DTE buffer back, so a smaller buffer could not be read
+            cv.Optional(CONF_DTE_BUFFER_SIZE, default=1024): cv.int_range(
+                min=256, max=65535
+            ),
+            cv.Optional(
+                CONF_REBOOT_TIMEOUT, default="10min"
+            ): cv.positive_time_period_milliseconds,
+            **{
+                cv.Optional(a.conf_key): automation.validate_automation({})
+                for a in _CALLBACK_AUTOMATIONS
+            },
+        }
+    ).extend(cv.COMPONENT_SCHEMA),
+    cv.has_none_or_all_keys(CONF_RTS_PIN, CONF_CTS_PIN),
+    _default_use_address,
+    require_ipv4(DOMAIN),
+)
+
+
+MODEM_ID_SCHEMA = automation.maybe_simple_id(
+    {cv.GenerateID(): cv.use_id(ModemComponent)}
+)
+SEND_AT_SCHEMA = cv.maybe_simple_value(
+    {
+        cv.GenerateID(): cv.use_id(ModemComponent),
+        cv.Required(CONF_COMMAND): cv.templatable(
+            cv.All(cv.string, cv.ByteLength(max=MAX_AT_COMMAND_LENGTH))
+        ),
+    },
+    key=CONF_COMMAND,
+)
+
+for _name in ("enable", "disable"):
+    automation.register_apply_action(
+        f"modem.{_name}", MODEM_ID_SCHEMA, automation.ApplyCall(f"{_name}()")
+    )
+_send_at_builder = automation.apply_action_builder(
+    "modem.send_at",
+    SEND_AT_SCHEMA,
+    # A constant renders as a literal and takes the const char * overload; a lambda returns std::string
+    automation.ApplyField(CONF_COMMAND, "send_at", cg.std_string),
+)
+
+
+@automation.register_action(
+    "modem.send_at", automation.ApplyAction, SEND_AT_SCHEMA, synchronous=True
+)
+async def send_at_to_code(
+    config: ConfigType,
+    action_id: ID,
+    template_arg: cg.TemplateArguments,
+    args: TemplateArgsType,
+) -> MockObj:
+    cg.add_define("USE_MODEM_SEND_AT")
+    return await _send_at_builder(config, action_id, template_arg, args)
+
+
+automation.register_apply_condition(
+    "modem.connected", MODEM_ID_SCHEMA, "is_connected()"
+)
+automation.register_apply_condition("modem.enabled", MODEM_ID_SCHEMA, "is_enabled()")
+
+
+@coroutine_with_priority(CoroPriority.COMMUNICATION)
+async def to_code(config: ConfigType) -> None:
+    add_idf_component(name="espressif/esp_modem", ref="2.1.0")
+
+    add_idf_sdkconfig_option("CONFIG_LWIP_PPP_SUPPORT", True)
+    # LCP echo probes notice a modem that went away without a word, which ends the PPP session
+    add_idf_sdkconfig_option("CONFIG_LWIP_ENABLE_LCP_ECHO", True)
+    add_idf_sdkconfig_option("CONFIG_ESP_MODEM_CMUX_DELAY_AFTER_DLCI_SETUP", 500)
+    add_idf_sdkconfig_option(
+        "CONFIG_ESP_MODEM_CMUX_USE_SHORT_PAYLOADS_ONLY", True
+    )  # Slower but more reliable
+
+    cg.add_define("USE_MODEM")
+
+    model = config[CONF_MODEL]
+    factory = MODEM_MODELS[model]
+    tx = await cg.gpio_pin_expression(config[CONF_TX_PIN])
+    rx = await cg.gpio_pin_expression(config[CONF_RX_PIN])
+    var = cg.new_Pvariable(config[CONF_ID], tx, rx, model, factory, config[CONF_APN])
+
+    cg.add(var.set_reboot_timeout(config[CONF_REBOOT_TIMEOUT]))
+    add_use_address(var, config[CONF_USE_ADDRESS])
+    if (pin_code := config.get(CONF_PIN_CODE)) is not None:
+        cg.add_define("USE_MODEM_PIN_CODE")
+        cg.add(var.set_pin_code(pin_code))
+
+    if config[CONF_ENABLE_ON_BOOT]:
+        cg.add(var.enable())
+
+    if init_at := config.get(CONF_INIT_AT):
+        # The commands are literals in flash; the component keeps a pointer to the array
+        arr = cg.shared_progmem_array(
+            "modem_init_at", cg.const_char_ptr, cg.ArrayInitializer(*init_at)
+        )
+        cg.add(var.set_init_at_commands(arr, len(init_at)))
+
+    await cg.add_pin(var.set_rts_pin, config.get(CONF_RTS_PIN))
+    await cg.add_pin(var.set_cts_pin, config.get(CONF_CTS_PIN))
+
+    cg.add_optional(var.set_baud_rate, config.get(CONF_BAUD_RATE))
+    cg.add(var.set_rx_buffer_size(config[CONF_RX_BUFFER_SIZE]))
+    cg.add(var.set_tx_buffer_size(config[CONF_TX_BUFFER_SIZE]))
+    cg.add(var.set_dte_buffer_size(config[CONF_DTE_BUFFER_SIZE]))
+
+    await automation.build_callback_automations(var, config, _CALLBACK_AUTOMATIONS)
+
+    await cg.register_component(var, config)

esphome/components/modem/modem_component.h

@@ -0,0 +1,282 @@
+#pragma once
+#ifdef USE_ESP32
+
+#include "esphome/core/component.h"
+#include "esphome/core/gpio.h"
+#ifdef USE_MODEM_URC
+#include "esphome/core/line_reader.h"
+#endif
+#include "esphome/core/preferences.h"
+#include "esphome/core/static_task.h"
+#include "esphome/components/network/ip_address.h"
+#ifdef USE_MODEM_URC
+#include "esphome/components/text_sensor/text_sensor.h"
+#endif
+
+#include <cxx_include/esp_modem_api.hpp>
+#include <esp_modem_config.h>
+
+#include <atomic>
+#include <memory>
+#include <string>
+#include <utility>
+
+namespace esphome::modem {
+
+enum class ModemState : uint8_t {
+  MODEM_STATE_DISABLED,
+  MODEM_STATE_ENABLING,
+  MODEM_STATE_SYNCED,
+  MODEM_STATE_CONNECTING,
+  MODEM_STATE_WAIT_IP,
+  MODEM_STATE_CONNECTED,
+  MODEM_STATE_DISCONNECTING,
+};
+
+// CONNECTING runs its long commands once; the later passes only poll the attachment
+enum class ConnectStep : uint8_t {
+  CONNECT_STEP_START,
+  CONNECT_STEP_RADIO_ON,
+  CONNECT_STEP_ATTACH_REQUESTED,
+};
+
+enum class SimResult : uint8_t {
+  SIM_RESULT_READY,
+  SIM_RESULT_RETRY,   // status not readable yet, or a PIN step is in progress
+  SIM_RESULT_LOCKED,  // nothing more to try: PUK, another lock, no pin_code or the PIN is not sent again
+  SIM_RESULT_HALT,    // stop this attempt; a reboot may still open the SIM, so the reboot timer stays
+};
+
+// The step that failed most recently; CONNECTED clears it
+enum class ModemFault : uint8_t {
+  MODEM_FAULT_NONE,
+  MODEM_FAULT_NO_RESPONSE,
+  MODEM_FAULT_BAUD,
+  MODEM_FAULT_INIT_AT,
+  MODEM_FAULT_SIM_NOT_READY,
+  MODEM_FAULT_SIM_LOCKED,
+  MODEM_FAULT_PIN_NOT_SENT,
+  MODEM_FAULT_PIN_RECORD,
+  MODEM_FAULT_SETUP,
+  MODEM_FAULT_NOT_ATTACHED,
+  MODEM_FAULT_PPP,
+  MODEM_FAULT_TEARDOWN,
+};
+
+#ifdef USE_MODEM_PIN_CODE
+enum class PinStep : uint8_t {
+  PIN_STEP_IDLE,
+  PIN_STEP_SAVING,  // the send is counted and waits for the record to reach flash
+  PIN_STEP_SENT,
+};
+
+// Written by loop() once it tried to save pin_record_; the task leaves the record alone while PENDING
+enum class PinSave : uint8_t {
+  PIN_SAVE_DONE,
+  PIN_SAVE_PENDING,
+  PIN_SAVE_FAILED,
+};
+#endif
+
+// Every esp_modem factory shares this signature, so codegen picks the one for the model
+using DceFactory = std::unique_ptr<esp_modem::DCE> (*)(const esp_modem::dce_config *, std::shared_ptr<esp_modem::DTE>,
+                                                       esp_netif_t *);
+
+class ModemComponent : public Component {
+ public:
+  ModemComponent(InternalGPIOPin *tx_pin, InternalGPIOPin *rx_pin, const char *model, DceFactory factory,
+                 const char *apn);
+  void setup() override;
+  void loop() override;
+  void dump_config() override;
+  float get_setup_priority() const override { return setup_priority::WIFI + 1; }  // Just before Wi-Fi
+
+  void set_reboot_timeout(uint32_t timeout) { this->timeout_ = timeout; }
+  void set_use_address(const char *use_address) { this->use_address_ = use_address; }
+  void set_rts_pin(InternalGPIOPin *rts_pin) { this->rts_pin_ = rts_pin; }
+  void set_cts_pin(InternalGPIOPin *cts_pin) { this->cts_pin_ = cts_pin; }
+  void set_baud_rate(int baud_rate) { this->baud_rate_ = baud_rate; }
+#ifdef USE_MODEM_PIN_CODE
+  void set_pin_code(const char *pin_code) { this->pin_code_ = pin_code; }
+#endif
+  void set_tx_buffer_size(uint16_t tx_buffer_size) { this->tx_buffer_size_ = tx_buffer_size; }
+  void set_rx_buffer_size(uint16_t rx_buffer_size) { this->rx_buffer_size_ = rx_buffer_size; }
+  void set_dte_buffer_size(uint16_t dte_buffer_size) { this->dte_buffer_size_ = dte_buffer_size; }
+  void set_init_at_commands(const char *const *cmds, uint8_t count) {
+    this->init_at_commands_ = cmds;
+    this->init_at_count_ = count;
+  }
+#ifdef USE_MODEM_URC
+  void set_urc_text_sensor(text_sensor::TextSensor *urc_text_sensor) { this->urc_text_sensor_ = urc_text_sensor; }
+#endif
+  template<typename F> void add_on_state_callback(F &&callback) {
+    this->on_state_callback_.add(std::forward<F>(callback));
+  }
+
+  bool is_connected() const { return this->component_state_.load() == ModemState::MODEM_STATE_CONNECTED; }
+  bool is_disabled() const {
+    return this->component_state_.load() == ModemState::MODEM_STATE_DISABLED &&
+           this->target_state_.load() == ModemState::MODEM_STATE_DISABLED;
+  }
+  bool is_enabled() const { return !this->is_disabled(); }
+  void enable();
+  void disable();
+#ifdef USE_MODEM_SEND_AT
+  /// Queue an AT command for the modem task, which sends it at its next pass and logs the result and response.
+  /// Returns false when a command is still queued or the command is too long.
+  bool send_at(const char *cmd, uint32_t timeout = COMMAND_TIMEOUT_MS);
+  bool send_at(const std::string &cmd, uint32_t timeout = COMMAND_TIMEOUT_MS);
+#endif
+
+  network::IPAddresses get_ip_addresses();
+  const char *get_use_address() const { return this->use_address_; }
+
+ protected:
+  static constexpr uint32_t COMMAND_TIMEOUT_MS = 1000;
+  static constexpr uint32_t RETRY_DELAY_MS = 1000;
+  static constexpr size_t URC_LINE_SIZE = 128;
+  static constexpr size_t AT_COMMAND_SIZE = 128;
+
+  // esp_modem's calls block until the modem answers or their own timeouts run out, up to a minute for a
+  // mode change, so the state machine runs on its own task and leaves the main loop alone
+  static void task_entry(void *arg);
+  void run_task_();
+  void wake_task_();
+  ModemState compute_next_state_();
+  ModemState handle_state_enabling_();
+  ModemState handle_state_synced_();
+  ModemState handle_state_connecting_();
+  ModemState handle_state_wait_ip_();
+  ModemState handle_state_disconnecting_();
+  /// Pause the task before the next pass, then go to state
+  uint32_t next_probe_delay_();
+  ModemState retry_(ModemState state, uint32_t delay_ms = RETRY_DELAY_MS);
+  void transition_to_(ModemState next_state);
+  // What the task left for loop() to do
+  enum Pending : uint32_t {
+    PENDING_STATUS = 1 << 0,
+    PENDING_SAVE_BAUD = 1 << 1,
+#ifdef USE_MODEM_PIN_CODE
+    PENDING_SAVE_PIN = 1 << 2,
+#endif
+    PENDING_CONNECT_TIMEOUT = 1 << 3,  // arm or cancel the reboot timer for the current state
+    PENDING_ON_SYNC = 1 << 4,          // loop() wakes the task once on_sync ran
+    PENDING_ON_CONNECT = 1 << 5,
+    PENDING_ON_DISCONNECT = 1 << 6,
+  };
+
+  // Main loop only, so arming and cancelling cannot interleave
+  void arm_connect_timeout_();
+  void cancel_connect_timeout_();
+  void fail_setup_(const LogString *reason);
+  void set_fault_(ModemFault fault);
+  void set_teardown_fault_();
+  void clear_config_fault_(ModemFault fault);
+  void hand_over_(uint32_t pending);
+  /// Send an AT command from the task and log the result; blocks for up to the timeout
+  esp_modem::command_result at_(const char *cmd, uint32_t timeout = COMMAND_TIMEOUT_MS);
+  void publish_status_();
+  void not_responding_(int baud);
+  void dump_connect_params_();
+  SimResult prepare_sim_();
+#ifdef USE_MODEM_PIN_CODE
+  SimResult send_pin_();
+  void save_pin_record_();
+#endif
+
+  InternalGPIOPin *tx_pin_;
+  InternalGPIOPin *rx_pin_;
+  InternalGPIOPin *rts_pin_{nullptr};
+  InternalGPIOPin *cts_pin_{nullptr};
+  const char *model_;
+  DceFactory dce_factory_;
+  const char *apn_;
+  const char *const *init_at_commands_{nullptr};  // static array from code generation
+#ifdef USE_MODEM_URC
+  text_sensor::TextSensor *urc_text_sensor_{nullptr};
+  const char *urc_line_{nullptr};  // the line urc_pending_ hands to loop(), inside urc_reader_
+#endif
+  std::unique_ptr<esp_modem::DCE> dce_;
+  std::shared_ptr<esp_modem::DTE> dte_;
+  esp_netif_t *ppp_netif_{nullptr};
+  ESPPreferenceObject pref_;
+  LazyCallbackManager<void(ModemState)> on_state_callback_;
+  int baud_rate_{0};  // 0 = use the esp_modem default
+  int current_baud_rate_{0};
+  int restored_baud_{0};
+  StaticTask task_;
+  uint32_t timeout_{0};
+  uint32_t step_delay_ms_{0};
+  uint32_t state_started_ms_{0};  // when the IP wait or the PIN unlock wait began
+  std::atomic<uint32_t> pending_{0};
+#ifdef USE_MODEM_PIN_CODE
+  const char *pin_code_{nullptr};
+  ESPPreferenceObject pin_pref_;
+#endif
+  // Kept in flash so reboots cannot keep spending the SIM's attempts on a PIN it never confirmed
+  struct PinRecord {
+    uint32_t hash;  // pin_code the record is about
+    uint8_t sends;  // AT+CPIN= sent without the SIM reporting READY since
+  };
+#ifdef USE_MODEM_PIN_CODE
+  PinRecord pin_record_{};  // the task writes it, loop() saves it while pin_save_state_ is PENDING
+#endif
+#ifdef USE_MODEM_SEND_AT
+  uint32_t at_timeout_{COMMAND_TIMEOUT_MS};
+#endif
+  uint16_t tx_buffer_size_{1024};
+  uint16_t rx_buffer_size_{1024};
+  uint16_t dte_buffer_size_{1024};
+  // The task owns component_state_; the main loop sets target_state_ and reads both
+  std::atomic<ModemState> component_state_{ModemState::MODEM_STATE_DISABLED};
+  std::atomic<ModemState> target_state_{ModemState::MODEM_STATE_DISABLED};
+  // Written by the task only, read by loop()
+  std::atomic<ModemFault> fault_{ModemFault::MODEM_FAULT_NONE};
+  // A failed baud_rate or init_at step, shown once fault_ is clear until the step passes again
+  std::atomic<ModemFault> config_fault_{ModemFault::MODEM_FAULT_NONE};
+  // std::atomic<uint8_t> flags: GCC on Xtensa calls out of line for std::atomic<bool>
+  // Written by the IP event task, read by the modem task
+  std::atomic<uint8_t> got_ip_{0};
+  std::atomic<uint8_t> ppp_error_{0};  // last NETIF_PPP_STATUS error code, 0 = none
+  // Doublings of the silent-modem probe delay; enable() and an answer reset it
+  std::atomic<uint8_t> probe_backoff_{0};
+  // Set by loop() once on_sync ran; the task waits for it, not for any notification
+  std::atomic<uint8_t> sync_acked_{0};
+#ifdef USE_MODEM_SEND_AT
+  // Set by send_at() on the main loop once at_command_ is written, cleared by the task after sending it
+  std::atomic<uint8_t> at_pending_{0};
+#endif
+#ifdef USE_MODEM_URC
+  // Set by the UART task with urc_line_; until loop() has published it, new lines are dropped
+  std::atomic<uint8_t> urc_pending_{0};
+#endif
+#ifdef USE_MODEM_PIN_CODE
+  std::atomic<PinSave> pin_save_state_{PinSave::PIN_SAVE_DONE};
+  PinStep pin_step_{PinStep::PIN_STEP_IDLE};
+  bool pin_record_unsaved_{false};  // loop() only; the last PIN record save failed
+#endif
+  ConnectStep connect_step_{ConnectStep::CONNECT_STEP_START};
+  uint8_t init_at_count_{0};
+  uint8_t uart_port_{UART_NUM_MAX};
+  uint8_t attach_polls_{0};
+  bool connect_timeout_armed_{false};
+  bool modem_synced_{false};  // a modem that never answered has nothing to tear down
+  bool resync_{false};        // the attachment read failed, so CONNECTING syncs again first
+#ifdef USE_MODEM_SEND_AT
+  char at_command_[AT_COMMAND_SIZE];
+#endif
+#ifdef USE_MODEM_URC
+  // Fed on the UART task
+  LineReader<URC_LINE_SIZE> urc_reader_;
+#endif
+
+ private:
+  // Points at a string literal from code generation, never at dynamic storage
+  const char *use_address_{""};
+};
+
+extern ModemComponent *global_modem_component;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
+
+}  // namespace esphome::modem
+
+#endif

esphome/components/modem/text_sensor.py

@@ -0,0 +1,27 @@
+import esphome.codegen as cg
+from esphome.components import text_sensor
+from esphome.components.esp32 import add_idf_sdkconfig_option
+import esphome.config_validation as cv
+from esphome.types import ConfigType
+
+from . import CONF_MODEM_ID, ModemComponent
+
+DEPENDENCIES = ["modem"]
+
+CONF_URC = "urc"
+
+CONFIG_SCHEMA = cv.Schema(
+    {
+        cv.GenerateID(CONF_MODEM_ID): cv.use_id(ModemComponent),
+        cv.Optional(CONF_URC): text_sensor.text_sensor_schema(),
+    }
+)
+
+
+async def to_code(config: ConfigType) -> None:
+    hub = await cg.get_variable(config[CONF_MODEM_ID])
+    texts = text_sensor.sub_text_sensors(config)
+    if await texts(CONF_URC, hub.set_urc_text_sensor) is not None:
+        cg.add_define("USE_MODEM_URC")
+        cg.add_define("USE_LINE_READER")
+        add_idf_sdkconfig_option("CONFIG_ESP_MODEM_URC_HANDLER", True)

esphome/components/mqtt/__init__.py

@@ -1,3 +1,5 @@
+from collections.abc import Iterator
+
 from esphome import automation
 import esphome.codegen as cg
 from esphome.components import logger, network, socket
@@ -10,6 +12,7 @@ from esphome.components.esp32 import (
     request_tls,
     require_certificate_bundle,
 )
+from esphome.config import iter_ids
 from esphome.config_helpers import (
     filter_source_files_from_defines,
     filter_source_files_from_platform,
@@ -26,6 +29,8 @@ from esphome.const import (
     CONF_CLIENT_ID,
     CONF_COMMAND_RETAIN,
     CONF_COMMAND_TOPIC,
+    CONF_DEVICE_ID,
+    CONF_DEVICES,
     CONF_DISCOVER_IP,
     CONF_DISCOVERY,
     CONF_DISCOVERY_OBJECT_ID_GENERATOR,
@@ -33,11 +38,17 @@ from esphome.const import (
     CONF_DISCOVERY_RETAIN,
     CONF_DISCOVERY_UNIQUE_ID_GENERATOR,
     CONF_ENABLE_ON_BOOT,
+    CONF_ESPHOME,
     CONF_ID,
+    CONF_INTERNAL,
     CONF_KEEPALIVE,
     CONF_LEVEL,
     CONF_LOG_TOPIC,
+    CONF_MODE_COMMAND_TOPIC,
     CONF_MQTT,
+    CONF_MQTT_ID,
+    CONF_NAME,
+    CONF_NAME_ADD_MAC_SUFFIX,
     CONF_ON_CONNECT,
     CONF_ON_DISCONNECT,
     CONF_ON_JSON_MESSAGE,
@@ -46,6 +57,7 @@ from esphome.const import (
     CONF_PAYLOAD,
     CONF_PAYLOAD_AVAILABLE,
     CONF_PAYLOAD_NOT_AVAILABLE,
+    CONF_PLATFORM,
     CONF_PORT,
     CONF_PUBLISH_NAN_AS_NONE,
     CONF_QOS,
@@ -55,6 +67,9 @@ from esphome.const import (
     CONF_SKIP_CERT_CN_CHECK,
     CONF_STATE_TOPIC,
     CONF_SUBSCRIBE_QOS,
+    CONF_TARGET_TEMPERATURE_COMMAND_TOPIC,
+    CONF_TARGET_TEMPERATURE_HIGH_COMMAND_TOPIC,
+    CONF_TARGET_TEMPERATURE_LOW_COMMAND_TOPIC,
     CONF_TOPIC,
     CONF_TOPIC_PREFIX,
     CONF_USE_ABBREVIATIONS,
@@ -69,6 +84,14 @@ from esphome.const import (
     PlatformFramework,
 )
 from esphome.core import CORE, CoroPriority, coroutine_with_priority
+from esphome.cpp_generator import MockObjClass
+import esphome.final_validate as fv
+from esphome.helpers import sanitize, snake_case
+from esphome.mqtt_topics import (
+    DEFAULT_TOPIC_SUFFIXES,
+    per_board_suffix,
+    prefix_gets_mac_suffix,
+)
 from esphome.types import ConfigType
 
 DEPENDENCIES = ["network"]
@@ -94,10 +117,12 @@ DEFAULT_WS_PATH = "/mqtt"
 
 # Max lengths for stack-based topic building.
 # These values are used in cv.Length() validators below to ensure the C++ code
-# in mqtt_component.cpp can safely use fixed-size stack buffers without overflow.
-# If you change these, update the corresponding constants in mqtt_component.cpp.
+# can safely use fixed-size stack buffers without overflow. If you change these,
+# update the corresponding constants in mqtt_client.h and mqtt_component.h.
 TOPIC_PREFIX_MAX_LEN = 64  # Default is device name, typically short
 DISCOVERY_PREFIX_MAX_LEN = 64  # Default is "homeassistant" (13 chars)
+# Size of the stack buffer topics are built into (static_assert in mqtt_client.cpp)
+MQTT_DEFAULT_TOPIC_MAX_LEN = 248
 
 
 def validate_message_just_topic(value):
@@ -127,34 +152,91 @@ MQTT_MESSAGE_SCHEMA = cv.Any(
 )
 
 mqtt_ns = cg.esphome_ns.namespace("mqtt")
+
+
+def _topic_source(topic: str) -> cg.MockObj:
+    # On ESP8266 a topic that fits the buffer stays in flash and is copied out when needed
+    if CORE.is_esp8266 and len(topic.encode()) < MQTT_DEFAULT_TOPIC_MAX_LEN:
+        return mqtt_ns.progmem_topic(cg.FlashStringLiteral(topic))
+    return mqtt_ns.static_topic(topic)
+
+
 MQTTMessage = mqtt_ns.struct("MQTTMessage")
 MQTTClientDisconnectReason = mqtt_ns.enum("MQTTClientDisconnectReason")
 MQTTClientComponent = mqtt_ns.class_("MQTTClientComponent", cg.Component)
 MQTTPublishJsonAction = mqtt_ns.class_("MQTTPublishJsonAction", automation.Action)
 MQTTComponent = mqtt_ns.class_("MQTTComponent", cg.Component)
 
-MQTTAlarmControlPanelComponent = mqtt_ns.class_(
-    "MQTTAlarmControlPanelComponent", MQTTComponent
+# A topic the firmware builds from a suffix, unless the config key replaces the whole topic
+_Topic = tuple[str, str]
+# The firmware subscribes to exactly one of these alternatives, chosen by entity traits
+_Subscription = tuple[tuple[_Topic, ...], ...]
+# The subscriptions each entity MQTT class always makes, keyed by class: component type,
+# then the subscriptions
+_COMMAND_TOPICS: dict[str, tuple[str, tuple[_Subscription, ...]]] = {}
+
+
+def _single_topic(suffix: str, custom_key: str) -> _Subscription:
+    return (((suffix, custom_key),),)
+
+
+_COMMAND = _single_topic("command", CONF_COMMAND_TOPIC)
+
+
+def _entity_class(
+    name: str,
+    component_type: str | None = None,
+    subscriptions: tuple[_Subscription, ...] = (_COMMAND,),
+) -> MockObjClass:
+    """Declare an entity MQTT class; a component_type means it subscribes to command topics."""
+    cls = mqtt_ns.class_(name, MQTTComponent)
+    if component_type is not None:
+        _COMMAND_TOPICS[str(cls)] = (component_type, subscriptions)
+    return cls
+
+
+MQTTAlarmControlPanelComponent = _entity_class(
+    "MQTTAlarmControlPanelComponent", "alarm_control_panel"
 )
-MQTTBinarySensorComponent = mqtt_ns.class_("MQTTBinarySensorComponent", MQTTComponent)
-MQTTClimateComponent = mqtt_ns.class_("MQTTClimateComponent", MQTTComponent)
-MQTTCoverComponent = mqtt_ns.class_("MQTTCoverComponent", MQTTComponent)
-MQTTFanComponent = mqtt_ns.class_("MQTTFanComponent", MQTTComponent)
-MQTTJSONLightComponent = mqtt_ns.class_("MQTTJSONLightComponent", MQTTComponent)
-MQTTSensorComponent = mqtt_ns.class_("MQTTSensorComponent", MQTTComponent)
-MQTTSwitchComponent = mqtt_ns.class_("MQTTSwitchComponent", MQTTComponent)
-MQTTTextSensor = mqtt_ns.class_("MQTTTextSensor", MQTTComponent)
-MQTTNumberComponent = mqtt_ns.class_("MQTTNumberComponent", MQTTComponent)
-MQTTDateComponent = mqtt_ns.class_("MQTTDateComponent", MQTTComponent)
-MQTTTimeComponent = mqtt_ns.class_("MQTTTimeComponent", MQTTComponent)
-MQTTDateTimeComponent = mqtt_ns.class_("MQTTDateTimeComponent", MQTTComponent)
-MQTTTextComponent = mqtt_ns.class_("MQTTTextComponent", MQTTComponent)
-MQTTSelectComponent = mqtt_ns.class_("MQTTSelectComponent", MQTTComponent)
-MQTTButtonComponent = mqtt_ns.class_("MQTTButtonComponent", MQTTComponent)
-MQTTLockComponent = mqtt_ns.class_("MQTTLockComponent", MQTTComponent)
-MQTTEventComponent = mqtt_ns.class_("MQTTEventComponent", MQTTComponent)
-MQTTUpdateComponent = mqtt_ns.class_("MQTTUpdateComponent", MQTTComponent)
-MQTTValveComponent = mqtt_ns.class_("MQTTValveComponent", MQTTComponent)
+MQTTBinarySensorComponent = _entity_class("MQTTBinarySensorComponent")
+MQTTClimateComponent = _entity_class(
+    "MQTTClimateComponent",
+    "climate",
+    (
+        _single_topic("mode/command", CONF_MODE_COMMAND_TOPIC),
+        # One target temperature topic, or the _low and _high pair on two-point devices
+        (
+            (("target_temperature/command", CONF_TARGET_TEMPERATURE_COMMAND_TOPIC),),
+            (
+                (
+                    "target_temperature_low/command",
+                    CONF_TARGET_TEMPERATURE_LOW_COMMAND_TOPIC,
+                ),
+                (
+                    "target_temperature_high/command",
+                    CONF_TARGET_TEMPERATURE_HIGH_COMMAND_TOPIC,
+                ),
+            ),
+        ),
+    ),
+)
+MQTTCoverComponent = _entity_class("MQTTCoverComponent", "cover")
+MQTTFanComponent = _entity_class("MQTTFanComponent", "fan")
+MQTTJSONLightComponent = _entity_class("MQTTJSONLightComponent", "light")
+MQTTSensorComponent = _entity_class("MQTTSensorComponent")
+MQTTSwitchComponent = _entity_class("MQTTSwitchComponent", "switch")
+MQTTTextSensor = _entity_class("MQTTTextSensor")
+MQTTNumberComponent = _entity_class("MQTTNumberComponent", "number")
+MQTTDateComponent = _entity_class("MQTTDateComponent", "date")
+MQTTTimeComponent = _entity_class("MQTTTimeComponent", "time")
+MQTTDateTimeComponent = _entity_class("MQTTDateTimeComponent", "datetime")
+MQTTTextComponent = _entity_class("MQTTTextComponent", "text")
+MQTTSelectComponent = _entity_class("MQTTSelectComponent", "select")
+MQTTButtonComponent = _entity_class("MQTTButtonComponent", "button")
+MQTTLockComponent = _entity_class("MQTTLockComponent", "lock")
+MQTTEventComponent = _entity_class("MQTTEventComponent")
+MQTTUpdateComponent = _entity_class("MQTTUpdateComponent", "update")
+MQTTValveComponent = _entity_class("MQTTValveComponent", "valve")
 
 MQTTDiscoveryUniqueIdGenerator = mqtt_ns.enum("MQTTDiscoveryUniqueIdGenerator")
 MQTT_DISCOVERY_UNIQUE_ID_GENERATOR_OPTIONS = {
@@ -189,6 +271,31 @@ def _validate_transport(value: str) -> str:
     return value
 
 
+# (config key, extra fields when defaulted from the topic prefix, (setter, disabler))
+_MESSAGES = (
+    (
+        CONF_BIRTH_MESSAGE,
+        {CONF_PAYLOAD: "online"},
+        lambda var: (var.set_birth_message, var.disable_birth_message),
+    ),
+    (
+        CONF_WILL_MESSAGE,
+        {CONF_PAYLOAD: "offline"},
+        lambda var: (var.set_last_will, var.disable_last_will),
+    ),
+    (
+        CONF_SHUTDOWN_MESSAGE,
+        {CONF_PAYLOAD: "offline"},
+        lambda var: (var.set_shutdown_message, var.disable_shutdown_message),
+    ),
+    (
+        CONF_LOG_TOPIC,
+        {},
+        lambda var: (var.set_log_message_template, var.disable_log_message),
+    ),
+)
+
+
 def validate_config(value):
     if (
         value.get(CONF_TRANSPORT) not in (MQTT_TRANSPORT_WS, MQTT_TRANSPORT_WSS)
@@ -203,45 +310,19 @@ def validate_config(value):
     topic_prefix = value[CONF_TOPIC_PREFIX]
     # If the topic prefix is not null and these messages are not configured, then set them to the default
     # If the topic prefix is null and these messages are not configured, then set them to null
-    if CONF_BIRTH_MESSAGE not in value:
-        if topic_prefix != "":
-            out[CONF_BIRTH_MESSAGE] = {
-                CONF_TOPIC: f"{topic_prefix}/status",
-                CONF_PAYLOAD: "online",
+    for key, extra, _ in _MESSAGES:
+        if key in value:
+            continue
+        out[key] = (
+            {}
+            if topic_prefix == ""
+            else {
+                CONF_TOPIC: f"{topic_prefix}{DEFAULT_TOPIC_SUFFIXES[key]}",
+                **extra,
                 CONF_QOS: 0,
                 CONF_RETAIN: True,
             }
-        else:
-            out[CONF_BIRTH_MESSAGE] = {}
-    if CONF_WILL_MESSAGE not in value:
-        if topic_prefix != "":
-            out[CONF_WILL_MESSAGE] = {
-                CONF_TOPIC: f"{topic_prefix}/status",
-                CONF_PAYLOAD: "offline",
-                CONF_QOS: 0,
-                CONF_RETAIN: True,
-            }
-        else:
-            out[CONF_WILL_MESSAGE] = {}
-    if CONF_SHUTDOWN_MESSAGE not in value:
-        if topic_prefix != "":
-            out[CONF_SHUTDOWN_MESSAGE] = {
-                CONF_TOPIC: f"{topic_prefix}/status",
-                CONF_PAYLOAD: "offline",
-                CONF_QOS: 0,
-                CONF_RETAIN: True,
-            }
-        else:
-            out[CONF_SHUTDOWN_MESSAGE] = {}
-    if CONF_LOG_TOPIC not in value:
-        if topic_prefix != "":
-            out[CONF_LOG_TOPIC] = {
-                CONF_TOPIC: f"{topic_prefix}/debug",
-                CONF_QOS: 0,
-                CONF_RETAIN: True,
-            }
-        else:
-            out[CONF_LOG_TOPIC] = {}
+        )
     return out
 
 
@@ -398,13 +479,146 @@ CONFIG_SCHEMA = cv.All(
     network.require_ipv4(CONF_MQTT),
 )
 
+# AsyncMqttClient, the ESP8266 and LibreTiny backend, drops any inbound message whose
+# topic is longer than this many bytes (its default maxTopicLength)
+ASYNC_MQTT_CLIENT_MAX_TOPIC_LEN = 128
+# Keep in sync with OBJECT_ID_MAX_LEN in esphome/core/entity_base.h
+OBJECT_ID_MAX_LEN = 128
+# name_add_mac_suffix appends "-" plus 6 hex digits to the name at runtime
+MAC_SUFFIX = "-xxxxxx"
 
-def exp_mqtt_message(config):
+
+def _entity_object_id(
+    config: ConfigType, mac_suffix: bool, devices: dict[str, str]
+) -> str:
+    """Build the object id the way EntityBase::write_object_id_to() does."""
+    # The name falls back the way EntityBase::configure_entity_() does
+    name = config.get(CONF_NAME) or ""
+    if not name:
+        if (device := config.get(CONF_DEVICE_ID)) is not None:
+            name = devices[device.id]
+        elif mac_suffix:
+            # Only the friendly name gains the MAC suffix, and an empty one stays empty
+            name = f"{CORE.friendly_name}{MAC_SUFFIX}" if CORE.friendly_name else ""
+        else:
+            name = CORE.friendly_name or CORE.name
+    # The firmware sanitizes each UTF-8 byte to one character and caps the result
+    raw = name.encode()[: OBJECT_ID_MAX_LEN - 1]
+    return sanitize(snake_case(raw.decode("ascii", "replace")))
+
+
+def _entity_topic(
+    config: ConfigType, default_topic: str, suffix: str, custom_key: str
+) -> str | None:
+    """Return the topic an entity subscribes to, or None when only known at runtime."""
+    custom = config.get(custom_key)
+    # An empty specialized topic falls back to the default (MQTT_COMPONENT_CUSTOM_TOPIC_)
+    # while an empty command_topic stays empty
+    if custom == "" and custom_key != CONF_COMMAND_TOPIC:
+        custom = None
+    if custom is not None:
+        # A lambda topic is only known at runtime
+        return custom if isinstance(custom, str) else None
+    return f"{default_topic}/{suffix}" if default_topic else None
+
+
+def _entity_subscriptions(
+    config: ConfigType, topic_prefix: str, mac_suffix: bool, devices: dict[str, str]
+) -> Iterator[tuple[tuple[str, ...], ...]]:
+    """Yield the subscriptions an entity's MQTT component always makes, as known topics."""
+    component_type, subscriptions = _COMMAND_TOPICS[str(config[CONF_MQTT_ID].type)]
+    # Mirrors MQTTComponent::compute_is_internal_(): internal components never subscribe
+    if CONF_STATE_TOPIC in config:
+        internal = config[CONF_STATE_TOPIC] == ""
+    elif CONF_COMMAND_TOPIC in config:
+        internal = config[CONF_COMMAND_TOPIC] == ""
+    else:
+        internal = not topic_prefix or config.get(CONF_INTERNAL, False)
+    if internal:
+        return
+    default_topic = ""
+    if topic_prefix:
+        object_id = _entity_object_id(config, mac_suffix, devices)
+        default_topic = f"{topic_prefix}/{component_type}/{object_id}"
+    for subscription in subscriptions:
+        yield tuple(
+            tuple(
+                topic
+                for suffix, custom_key in alternative
+                if (topic := _entity_topic(config, default_topic, suffix, custom_key))
+                is not None
+            )
+            for alternative in subscription
+        )
+
+
+def _subscriptions(config: ConfigType) -> Iterator[tuple[tuple[str, ...], ...]]:
+    """Yield every subscription known at build time, as alternatives of topics."""
+    full_config = fv.full_config.get()
+    esphome_config = full_config[CONF_ESPHOME]
+    mac_suffix = esphome_config.get(CONF_NAME_ADD_MAC_SUFFIX, False)
+    devices = {
+        device[CONF_ID].id: device[CONF_NAME]
+        for device in esphome_config.get(CONF_DEVICES, [])
+    }
+    topic_prefix = config[CONF_TOPIC_PREFIX]
+    if prefix_gets_mac_suffix(topic_prefix, CORE.name, mac_suffix):
+        topic_prefix += MAC_SUFFIX
+    # Entities can sit at any depth (a platform can declare its own switches), so find them by mqtt_id
+    for id_, path in iter_ids(full_config):
+        if path[-1] != CONF_MQTT_ID or str(id_.type) not in _COMMAND_TOPICS:
+            continue
+        entity = full_config
+        for key in path[:-1]:
+            entity = entity[key]
+        yield from _entity_subscriptions(entity, topic_prefix, mac_suffix, devices)
+    for key in (CONF_ON_MESSAGE, CONF_ON_JSON_MESSAGE):
+        for trigger in config.get(key, []):
+            yield ((trigger[CONF_TOPIC],),)
+    for domain in ("sensor", "text_sensor"):
+        for entity in full_config.get(domain) or []:
+            if entity[CONF_PLATFORM] == "mqtt_subscribe":
+                yield ((entity[CONF_TOPIC],),)
+
+
+def _too_long(topic: str) -> bool:
+    # A wildcard filter matches topics of any length, so only literals are checked
+    if "+" in topic or "#" in topic:
+        return False
+    return len(topic.encode()) > ASYNC_MQTT_CLIENT_MAX_TOPIC_LEN
+
+
+def _validate_topic_lengths(config: ConfigType) -> ConfigType:
+    """Reject topics that the ESP8266 and LibreTiny MQTT client would silently drop."""
+    if not (CORE.is_esp8266 or CORE.is_libretiny):
+        return config
+    for subscription in _subscriptions(config):
+        # The config cannot tell which alternative the firmware picks, so only a
+        # subscription that is too long whichever way is rejected
+        too_long = [
+            next((topic for topic in alternative if _too_long(topic)), None)
+            for alternative in subscription
+        ]
+        if all(too_long):
+            topic = too_long[0]
+            raise cv.Invalid(
+                f"MQTT topic '{topic}' is {len(topic.encode())} bytes; the MQTT client "
+                f"on this platform drops topics longer than {ASYNC_MQTT_CLIENT_MAX_TOPIC_LEN}"
+            )
+    return config
+
+
+FINAL_VALIDATE_SCHEMA = _validate_topic_lengths
+
+
+def exp_mqtt_message(
+    config: ConfigType | None, topic: cg.MockObj | None = None
+) -> cg.Expression:
     if config is None:
         return cg.optional(cg.TemplateArguments(MQTTMessage))
     return cg.StructInitializer(
         MQTTMessage,
-        ("topic", config[CONF_TOPIC]),
+        ("topic", config[CONF_TOPIC] if topic is None else topic),
         ("payload", config.get(CONF_PAYLOAD, "")),
         ("qos", config[CONF_QOS]),
         ("retain", config[CONF_RETAIN]),
@@ -488,37 +702,33 @@ async def to_code(config):
             )
         )
 
-    cg.add(var.set_topic_prefix(config[CONF_TOPIC_PREFIX], CORE.name))
+    topic_prefix = config[CONF_TOPIC_PREFIX]
+    # The MAC suffix is only known at runtime, so the prefix and the topics defaulted from it
+    # are built from App.get_name(); topics written out by hand stay literal
+    mac_suffix = CORE.config[CONF_ESPHOME].get(CONF_NAME_ADD_MAC_SUFFIX, False)
+    mac_prefix = prefix_gets_mac_suffix(topic_prefix, CORE.name, mac_suffix)
+    cg.add(var.set_topic_prefix(cg.App.get_name() if mac_prefix else topic_prefix))
 
     if config[CONF_USE_ABBREVIATIONS]:
         cg.add_define("USE_MQTT_ABBREVIATIONS")
 
-    birth_message = config[CONF_BIRTH_MESSAGE]
-    if not birth_message:
-        cg.add(var.disable_birth_message())
-    else:
-        cg.add(var.set_birth_message(exp_mqtt_message(birth_message)))
-    will_message = config[CONF_WILL_MESSAGE]
-    if not will_message:
-        cg.add(var.disable_last_will())
-    else:
-        cg.add(var.set_last_will(exp_mqtt_message(will_message)))
-    shutdown_message = config[CONF_SHUTDOWN_MESSAGE]
-    if not shutdown_message:
-        cg.add(var.disable_shutdown_message())
-    else:
-        cg.add(var.set_shutdown_message(exp_mqtt_message(shutdown_message)))
+    for key, _, methods in _MESSAGES:
+        set_message, disable_message = methods(var)
+        if not (message := config[key]):
+            cg.add(disable_message())
+            continue
+        topic = (
+            cg.App.get_name() + suffix
+            if (suffix := per_board_suffix(config, key, CORE.name, mac_suffix))
+            else None
+        )
+        cg.add(set_message(exp_mqtt_message(message, topic)))
 
-    log_topic = config[CONF_LOG_TOPIC]
-    if not log_topic:
-        cg.add(var.disable_log_message())
-    else:
-        cg.add(var.set_log_message_template(exp_mqtt_message(log_topic)))
+    if log_topic := config[CONF_LOG_TOPIC]:
         # Request a log listener slot only when log topic is enabled
         logger.request_log_listener()
-
-        if CONF_LEVEL in log_topic:
-            cg.add(var.set_log_level(logger.LOG_LEVELS[log_topic[CONF_LEVEL]]))
+        if (level := log_topic.get(CONF_LEVEL)) is not None:
+            cg.add(var.set_log_level(logger.LOG_LEVELS[level]))
 
     cg.add(var.set_keep_alive(config[CONF_KEEPALIVE]))
 
@@ -566,9 +776,7 @@ async def to_code(config):
                 ((CONF_PAYLOAD, cg.std_string, automation.string_ref_literal),),
             ),
         )
-        cg.add(
-            var.subscribe(cg.progmem_string(conf[CONF_TOPIC]), callback, conf[CONF_QOS])
-        )
+        cg.add(var.subscribe(_topic_source(conf[CONF_TOPIC]), callback, conf[CONF_QOS]))
 
     for conf in config.get(CONF_ON_JSON_MESSAGE, []):
         callback = await automation.build_trigger_callback(
@@ -579,7 +787,7 @@ async def to_code(config):
         )
         cg.add(
             var.subscribe_json(
-                cg.progmem_string(conf[CONF_TOPIC]), callback, conf[CONF_QOS]
+                _topic_source(conf[CONF_TOPIC]), callback, conf[CONF_QOS]
             )
         )
 

esphome/components/mqtt/mqtt_alarm_control_panel.cpp

@@ -50,7 +50,7 @@ static bool apply_command(AlarmControlPanelCall &call, const char *state) {
 void MQTTAlarmControlPanelComponent::setup() {
   this->alarm_control_panel_->add_on_state_callback(
       [this](AlarmControlPanelState /*state*/) { this->publish_state(); });
-  this->subscribe(this->get_command_topic_(), [this](const std::string &topic, const std::string &payload) {
+  this->subscribe(this->command_topic_source_(), [this](const std::string &topic, const std::string &payload) {
     auto call = this->alarm_control_panel_->make_call();
     if (!payload.empty() && payload[0] == '{') {
       // JSON payload: {"state": "DISARM", "code": "1234"}

esphome/components/mqtt/mqtt_button.cpp

@@ -15,7 +15,7 @@ using namespace esphome::button;
 MQTTButtonComponent::MQTTButtonComponent(button::Button *button) : button_(button) {}
 
 void MQTTButtonComponent::setup() {
-  this->subscribe(this->get_command_topic_(), [this](const std::string &topic, const std::string &payload) {
+  this->subscribe(this->command_topic_source_(), [this](const std::string &topic, const std::string &payload) {
     if (payload == "PRESS") {
       this->button_->press();
     } else {

esphome/components/mqtt/mqtt_client.cpp

@@ -98,12 +98,16 @@ void MQTTClientComponent::setup() {
 
   if (this->is_discovery_ip_enabled()) {
     this->subscribe(
-        "esphome/discover", [this](const std::string &topic, const std::string &payload) { this->send_device_info_(); },
-        2);
+        static_topic("esphome/discover"),
+        [this](const std::string &topic, const std::string &payload) { this->send_device_info_(); }, 2);
 
-    // Format topic on stack - subscribe() copies it
-    char ping_topic[ESPHOME_TOPIC_BUFFER_SIZE];
-    format_esphome_topic(ping_topic, "ping");
+    static_assert(sizeof("esphome/ping/") + ESPHOME_DEVICE_NAME_MAX_LEN <= MQTT_DEFAULT_TOPIC_MAX_LEN);
+    MQTTTopicSource ping_topic{[](const void *, std::span<char, MQTT_DEFAULT_TOPIC_MAX_LEN> buf) {
+                                 size_t len = buf_append_printf(buf.data(), buf.size(), 0, "esphome/ping/%s",
+                                                                App.get_name().c_str());
+                                 return StringRef(buf.data(), len);
+                               },
+                               nullptr};
     this->subscribe(
         ping_topic, [this](const std::string &topic, const std::string &payload) { this->send_device_info_(); }, 2);
   }
@@ -479,6 +483,12 @@ bool MQTTClientComponent::subscribe_(const char *topic, uint8_t qos) {
   }
   return ret != 0;
 }
+// Out of line so the topic buffer is not part of loop()'s stack frame
+__attribute__((noinline)) bool MQTTClientComponent::subscribe_topic_(const MQTTTopicSource &topic, uint8_t qos) {
+  char buf[MQTT_DEFAULT_TOPIC_MAX_LEN];
+  return this->subscribe_(topic.build(buf).c_str(), qos);
+}
+
 void MQTTClientComponent::resubscribe_subscription_(MQTTSubscription *sub) {
   if (sub->subscribed)
     return;
@@ -487,7 +497,7 @@ void MQTTClientComponent::resubscribe_subscription_(MQTTSubscription *sub) {
   bool do_resub = sub->resubscribe_timeout == 0 || now - sub->resubscribe_timeout > 1000;
 
   if (do_resub) {
-    sub->subscribed = this->subscribe_(sub->topic.c_str(), sub->qos);
+    sub->subscribed = this->subscribe_topic_(sub->topic, sub->qos);
     sub->resubscribe_timeout = now;
   }
 }
@@ -510,20 +520,57 @@ void MQTTClientComponent::handle_crash_log_request_(const std::string &topic, co
 }
 #endif
 
-void MQTTClientComponent::add_subscription_(std::string &&topic, mqtt_subscription_callback_t callback, bool boxed,
+static StringRef static_topic_builder(const void *context, std::span<char, MQTT_DEFAULT_TOPIC_MAX_LEN>) {
+  return StringRef(static_cast<const char *>(context));
+}
+// Same as static_topic_builder, but a distinct function so its address marks the topics to free
+static StringRef copied_topic_builder(const void *context, std::span<char, MQTT_DEFAULT_TOPIC_MAX_LEN>) {
+  return StringRef(static_cast<const char *>(context));
+}
+
+MQTTTopicSource static_topic(const char *topic) { return {static_topic_builder, topic}; }
+
+static_assert(MQTT_DEFAULT_TOPIC_MAX_LEN == 248, "update MQTT_DEFAULT_TOPIC_MAX_LEN in mqtt/__init__.py");
+
+#ifdef USE_ESP8266
+// mqtt/__init__.py picks this source only for topics that fit
+static StringRef progmem_topic_builder(const void *context, std::span<char, MQTT_DEFAULT_TOPIC_MAX_LEN> buf) {
+  size_t len = buf_append_str_p(buf.data(), buf.size(), 0, static_cast<const char *>(context));
+  return StringRef(buf.data(), len);
+}
+
+MQTTTopicSource progmem_topic(ProgmemStr topic) {
+  return {progmem_topic_builder, reinterpret_cast<const char *>(topic)};
+}
+#endif
+
+MQTTTopicSource MQTTClientComponent::copy_topic(StringRef topic) {
+  char *copy = new char[topic.size() + 1];  // NOLINT(cppcoreguidelines-owning-memory)
+  memcpy(copy, topic.c_str(), topic.size());
+  copy[topic.size()] = '\0';
+  return {copied_topic_builder, copy};
+}
+
+void MQTTClientComponent::add_subscription_(MQTTTopicSource topic, mqtt_subscription_callback_t callback, bool boxed,
                                             uint8_t qos) {
   auto &subscription = this->subscriptions_.emplace_back(MQTTSubscription{
-      .topic = std::move(topic),
+      .topic = topic,
+      .callback = callback,
+      .resubscribe_timeout = 0,
       .qos = qos,
       .subscribed = false,
       .boxed = boxed,
       .removed = false,
-      .callback = callback,
-      .resubscribe_timeout = 0,
   });
   this->resubscribe_subscription_(&subscription);
 }
 
+// Out of line so the topic buffer is not part of unsubscribe()'s stack frame
+__attribute__((noinline)) static bool topic_equals(const MQTTTopicSource &source, const std::string &topic) {
+  char buf[MQTT_DEFAULT_TOPIC_MAX_LEN];
+  return source.build(buf) == topic;
+}
+
 void MQTTClientComponent::unsubscribe(const std::string &topic) {
   bool ret = this->mqtt_backend_.unsubscribe(topic.c_str());
   yield();
@@ -536,7 +583,7 @@ void MQTTClientComponent::unsubscribe(const std::string &topic) {
   }
 
   for (auto &subscription : this->subscriptions_) {
-    if (subscription.topic == topic)
+    if (topic_equals(subscription.topic, topic))
       subscription.removed = true;
   }
   // A callback may be running; the outermost dispatch erases them once it returns.
@@ -553,6 +600,8 @@ void MQTTClientComponent::erase_removed_subscriptions_() {
     if (it->removed) {
       if (it->boxed)
         it->callback.free_boxed();
+      if (it->topic.builder == copied_topic_builder)
+        delete[] static_cast<const char *>(it->topic.context);  // NOLINT(cppcoreguidelines-owning-memory)
       it = this->subscriptions_.erase(it);
     } else {
       ++it;
@@ -629,6 +678,12 @@ void MQTTClientComponent::disable() {
   this->on_shutdown();
 }
 
+// Out of line so the topic buffer is off the stack while the matched callback runs
+__attribute__((noinline)) static bool topic_matches(const char *message, const MQTTTopicSource &topic) {
+  char buf[MQTT_DEFAULT_TOPIC_MAX_LEN];
+  return topic_match(message, topic.build(buf).c_str());
+}
+
 void MQTTClientComponent::on_message(const std::string &topic, const std::string &payload) {
 #ifdef USE_ESP8266
   // IMPORTANT: This defer is REQUIRED to prevent stack overflow crashes on ESP8266.
@@ -662,7 +717,7 @@ void MQTTClientComponent::on_message(const std::string &topic, const std::string
     const size_t count = this->subscriptions_.size();
     for (size_t i = 0; i < count; i++) {
       const auto &subscription = this->subscriptions_[i];
-      if (!subscription.removed && topic_match(topic.c_str(), subscription.topic.c_str()))
+      if (!subscription.removed && topic_matches(topic.c_str(), subscription.topic))
         subscription.callback.call(topic, payload);
     }
     this->dispatching_ = was_dispatching;
@@ -682,14 +737,7 @@ void MQTTClientComponent::register_mqtt_component(MQTTComponent *component) { th
 void MQTTClientComponent::set_keep_alive(uint16_t keep_alive_s) { this->mqtt_backend_.set_keep_alive(keep_alive_s); }
 void MQTTClientComponent::set_log_message_template(MQTTMessage &&message) { this->log_message_ = std::move(message); }
 const MQTTDiscoveryInfo &MQTTClientComponent::get_discovery_info() const { return this->discovery_info_; }
-void MQTTClientComponent::set_topic_prefix(const std::string &topic_prefix, const std::string &check_topic_prefix) {
-  if (App.is_name_add_mac_suffix_enabled() && (topic_prefix == check_topic_prefix)) {
-    char buf[ESPHOME_DEVICE_NAME_MAX_LEN + 1];
-    this->topic_prefix_ = str_sanitize_to(buf, App.get_name().c_str());
-  } else {
-    this->topic_prefix_ = topic_prefix;
-  }
-}
+void MQTTClientComponent::set_topic_prefix(std::string topic_prefix) { this->topic_prefix_ = std::move(topic_prefix); }
 const std::string &MQTTClientComponent::get_topic_prefix() const { return this->topic_prefix_; }
 void MQTTClientComponent::disable_birth_message() {
   this->birth_message_.topic = "";

esphome/components/mqtt/mqtt_client.h

@@ -8,8 +8,11 @@
 #include "esphome/components/network/ip_address.h"
 #include "esphome/core/automation.h"
 #include "esphome/core/component.h"
+#include "esphome/core/entity_base.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
+#include "esphome/core/progmem.h"
+#include "esphome/core/string_ref.h"
 #ifdef USE_LOGGER
 #include "esphome/components/logger/logger.h"
 #endif
@@ -22,6 +25,7 @@
 #endif
 #include "lwip/ip_addr.h"
 
+#include <span>
 #include <vector>
 
 // The crash log replay needs the logger, which mqtt does not depend on
@@ -31,6 +35,18 @@
 
 namespace esphome::mqtt {
 
+// Max lengths for stack-based topic building.
+// These limits are enforced at Python config validation time in mqtt/__init__.py
+// using cv.Length() validators for topic_prefix.
+// This ensures the stack buffers are always large enough.
+static constexpr size_t MQTT_COMPONENT_TYPE_MAX_LEN = 20;
+static constexpr size_t MQTT_SUFFIX_MAX_LEN = 32;
+static constexpr size_t MQTT_TOPIC_PREFIX_MAX_LEN = 64;  // Validated in Python: cv.Length(max=64)
+// Stack buffer size - safe because all inputs are length-validated at config time
+// Format: prefix + "/" + type + "/" + object_id + "/" + suffix + null
+static constexpr size_t MQTT_DEFAULT_TOPIC_MAX_LEN =
+    MQTT_TOPIC_PREFIX_MAX_LEN + 1 + MQTT_COMPONENT_TYPE_MAX_LEN + 1 + OBJECT_ID_MAX_LEN + 1 + MQTT_SUFFIX_MAX_LEN + 1;
+
 /** Callback for MQTT events.
  */
 using mqtt_on_connect_callback_t = std::function<MQTTBackend::on_connect_callback_t>;
@@ -46,15 +62,33 @@ using mqtt_json_callback_t ESPDEPRECATED(
     "subscribe_json() takes any callable; pass the lambda directly. Removed in 2027.5.0",
     "2026.11.0") = std::function<void(const std::string &, JsonObject)>;
 
+/// Returns a subscription's topic, written into buf or referring to storage that outlives the subscription.
+/// Never a null c_str(); an empty topic is "".
+using mqtt_topic_builder_t = StringRef (*)(const void *context, std::span<char, MQTT_DEFAULT_TOPIC_MAX_LEN> buf);
+
+/// Where a subscription's topic comes from. The topic is rebuilt whenever it is needed, so nothing is stored.
+struct MQTTTopicSource {
+  mqtt_topic_builder_t builder;
+  const void *context;
+  StringRef build(std::span<char, MQTT_DEFAULT_TOPIC_MAX_LEN> buf) const { return this->builder(this->context, buf); }
+};
+
+/// A topic string that outlives the subscription, such as a literal; it is referenced, not copied.
+MQTTTopicSource static_topic(const char *topic);
+#ifdef USE_ESP8266
+/// A topic in flash, copied into the buffer when needed, so at most MQTT_DEFAULT_TOPIC_MAX_LEN - 1 characters.
+MQTTTopicSource progmem_topic(ProgmemStr topic);
+#endif
+
 /// internal struct for MQTT subscriptions.
 struct MQTTSubscription {
-  std::string topic;
+  MQTTTopicSource topic;
+  mqtt_subscription_callback_t callback;
+  uint32_t resubscribe_timeout;
   uint8_t qos;
   bool subscribed;
   bool boxed;    // made by create_boxed(), freed on unsubscribe
   bool removed;  // unsubscribed while a callback ran, erased once dispatch returns
-  mqtt_subscription_callback_t callback;
-  uint32_t resubscribe_timeout;
 };
 
 /// internal struct for MQTT credentials.
@@ -167,7 +201,7 @@ class MQTTClientComponent final : public Component {
    *
    * @param topic_prefix The topic prefix. The last "/" is appended automatically.
    */
-  void set_topic_prefix(const std::string &topic_prefix, const std::string &check_topic_prefix);
+  void set_topic_prefix(std::string topic_prefix);
   /// Get the topic prefix of this device, using default if necessary
   const std::string &get_topic_prefix() const;
 
@@ -183,32 +217,44 @@ class MQTTClientComponent final : public Component {
    * A callback that fits in a pointer (capture at most `this`) is stored inline; a larger one is
    * boxed on the heap and freed by unsubscribe().
    *
+   * @param topic Where the topic comes from; it is rebuilt when needed instead of stored.
+   * @param callback The callback function.
+   * @param qos The QoS of this subscription.
+   */
+  template<typename F> void subscribe(MQTTTopicSource topic, F &&callback, uint8_t qos = 0) {
+    constexpr bool boxed = !mqtt_subscription_callback_t::fits_inline<F>();
+    this->add_subscription_(topic,
+                            boxed ? mqtt_subscription_callback_t::create_boxed(std::forward<F>(callback))
+                                  : mqtt_subscription_callback_t::create(std::forward<F>(callback)),
+                            boxed, qos);
+  }
+
+  /** Subscribe to an MQTT topic and call callback when a message is received.
+   *
+   * The topic is copied to the heap and freed by unsubscribe().
+   *
    * @param topic The topic. Wildcards are currently not supported.
    * @param callback The callback function.
    * @param qos The QoS of this subscription.
    */
-  template<typename F> void subscribe(std::string topic, F &&callback, uint8_t qos = 0) {
-    constexpr bool boxed = !mqtt_subscription_callback_t::fits_inline<F>();
-    this->add_subscription_(std::move(topic),
-                            boxed ? mqtt_subscription_callback_t::create_boxed(std::forward<F>(callback))
-                                  : mqtt_subscription_callback_t::create(std::forward<F>(callback)),
-                            boxed, qos);
+  template<typename F> void subscribe(const std::string &topic, F &&callback, uint8_t qos = 0) {
+    this->subscribe(copy_topic(StringRef(topic)), std::forward<F>(callback), qos);
   }
 
   /** Subscribe to a MQTT topic and automatically parse JSON payload.
    *
    * If an invalid JSON payload is received, the callback will not be called.
    *
-   * @param topic The topic. Wildcards are currently not supported.
+   * @param topic An MQTTTopicSource, or a string that is copied to the heap.
    * @param callback The callback with a parsed JsonObject that will be called when a message with matching topic is
    * received.
    * @param qos The QoS of this subscription.
    */
-  template<typename F> void subscribe_json(std::string topic, F &&callback, uint8_t qos = 0) {
+  template<typename T, typename F> void subscribe_json(T &&topic, F &&callback, uint8_t qos = 0) {
     static_assert(std::is_invocable_v<const std::decay_t<F> &, const std::string &, JsonObject>,
                   "subscribe_json() needs a callback that does not mutate its captures");
     this->subscribe(
-        std::move(topic),
+        std::forward<T>(topic),
         [cb = std::forward<F>(callback)](const std::string &topic, const std::string &payload) {
           call_json(cb, topic, payload);
         },
@@ -301,6 +347,12 @@ class MQTTClientComponent final : public Component {
   void set_wait_for_connection(bool wait_for_connection) { this->wait_for_connection_ = wait_for_connection; }
 
  protected:
+  // For the lambda command topic, which is copied once
+  friend class MQTTComponent;
+
+  /// A heap copy of topic, owned by the one subscription it is passed to and freed when that is erased.
+  static MQTTTopicSource copy_topic(StringRef topic);
+
   void send_device_info_();
 
   /// Reconnect to the MQTT broker if not already connected.
@@ -317,6 +369,7 @@ class MQTTClientComponent final : public Component {
   void recalculate_availability_();
 
   bool subscribe_(const char *topic, uint8_t qos);
+  bool subscribe_topic_(const MQTTTopicSource &topic, uint8_t qos);
   void resubscribe_subscription_(MQTTSubscription *sub);
   template<typename F> static void call_json(const F &callback, const std::string &topic, const std::string &payload) {
     JsonDocument doc = json::parse_json(payload);
@@ -328,7 +381,7 @@ class MQTTClientComponent final : public Component {
 #ifdef USE_MQTT_CRASH_LOG
   void handle_crash_log_request_(const std::string &topic, const std::string &payload);
 #endif
-  void add_subscription_(std::string &&topic, mqtt_subscription_callback_t callback, bool boxed, uint8_t qos);
+  void add_subscription_(MQTTTopicSource topic, mqtt_subscription_callback_t callback, bool boxed, uint8_t qos);
   void resubscribe_subscriptions_();
 
   MQTTCredentials credentials_;

esphome/components/mqtt/mqtt_climate.cpp

@@ -209,7 +209,7 @@ void MQTTClimateComponent::send_discovery(JsonObject root, mqtt::SendDiscoveryCo
 }
 void MQTTClimateComponent::setup() {
   auto traits = this->device_->get_traits();
-  this->subscribe(this->get_mode_command_topic(), [this](const std::string &topic, const std::string &payload) {
+  this->subscribe(this->get_mode_command_topic_source(), [this](const std::string &topic, const std::string &payload) {
     auto call = this->device_->make_call();
     call.set_mode(payload);
     call.perform();
@@ -217,7 +217,7 @@ void MQTTClimateComponent::setup() {
 
   if (traits.has_feature_flags(climate::CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE |
                                climate::CLIMATE_REQUIRES_TWO_POINT_TARGET_TEMPERATURE)) {
-    this->subscribe(this->get_target_temperature_low_command_topic(),
+    this->subscribe(this->get_target_temperature_low_command_topic_source(),
                     [this](const std::string &topic, const std::string &payload) {
                       auto val = parse_number<float>(payload);
                       if (!val.has_value()) {
@@ -228,7 +228,7 @@ void MQTTClimateComponent::setup() {
                       call.set_target_temperature_low(*val);
                       call.perform();
                     });
-    this->subscribe(this->get_target_temperature_high_command_topic(),
+    this->subscribe(this->get_target_temperature_high_command_topic_source(),
                     [this](const std::string &topic, const std::string &payload) {
                       auto val = parse_number<float>(payload);
                       if (!val.has_value()) {
@@ -240,7 +240,7 @@ void MQTTClimateComponent::setup() {
                       call.perform();
                     });
   } else {
-    this->subscribe(this->get_target_temperature_command_topic(),
+    this->subscribe(this->get_target_temperature_command_topic_source(),
                     [this](const std::string &topic, const std::string &payload) {
                       auto val = parse_number<float>(payload);
                       if (!val.has_value()) {
@@ -254,7 +254,7 @@ void MQTTClimateComponent::setup() {
   }
 
   if (traits.has_feature_flags(climate::CLIMATE_SUPPORTS_TARGET_HUMIDITY)) {
-    this->subscribe(this->get_target_humidity_command_topic(),
+    this->subscribe(this->get_target_humidity_command_topic_source(),
                     [this](const std::string &topic, const std::string &payload) {
                       auto val = parse_number<float>(payload);
                       if (!val.has_value()) {
@@ -268,27 +268,30 @@ void MQTTClimateComponent::setup() {
   }
 
   if (traits.get_supports_presets() || !traits.get_supported_custom_presets().empty()) {
-    this->subscribe(this->get_preset_command_topic(), [this](const std::string &topic, const std::string &payload) {
-      auto call = this->device_->make_call();
-      call.set_preset(payload);
-      call.perform();
-    });
+    this->subscribe(this->get_preset_command_topic_source(),
+                    [this](const std::string &topic, const std::string &payload) {
+                      auto call = this->device_->make_call();
+                      call.set_preset(payload);
+                      call.perform();
+                    });
   }
 
   if (traits.get_supports_fan_modes()) {
-    this->subscribe(this->get_fan_mode_command_topic(), [this](const std::string &topic, const std::string &payload) {
-      auto call = this->device_->make_call();
-      call.set_fan_mode(payload);
-      call.perform();
-    });
+    this->subscribe(this->get_fan_mode_command_topic_source(),
+                    [this](const std::string &topic, const std::string &payload) {
+                      auto call = this->device_->make_call();
+                      call.set_fan_mode(payload);
+                      call.perform();
+                    });
   }
 
   if (traits.get_supports_swing_modes()) {
-    this->subscribe(this->get_swing_mode_command_topic(), [this](const std::string &topic, const std::string &payload) {
-      auto call = this->device_->make_call();
-      call.set_swing_mode(payload);
-      call.perform();
-    });
+    this->subscribe(this->get_swing_mode_command_topic_source(),
+                    [this](const std::string &topic, const std::string &payload) {
+                      auto call = this->device_->make_call();
+                      call.set_swing_mode(payload);
+                      call.perform();
+                    });
   }
 
   this->device_->add_on_state_callback([this](Climate & /*unused*/) { this->publish_state_(); });
@@ -318,15 +321,15 @@ bool MQTTClimateComponent::publish_state_() {
   }
   if (traits.has_feature_flags(climate::CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE |
                                climate::CLIMATE_REQUIRES_TWO_POINT_TARGET_TEMPERATURE)) {
-    len = value_accuracy_to_buf(payload, this->device_->target_temperature_low, target_accuracy);
-    if (!this->publish(this->get_target_temperature_low_state_topic_to(topic_buf), payload, len))
+    if (!this->publish_float_(this->get_target_temperature_low_state_topic_to(topic_buf),
+                              this->device_->target_temperature_low, target_accuracy))
       success = false;
-    len = value_accuracy_to_buf(payload, this->device_->target_temperature_high, target_accuracy);
-    if (!this->publish(this->get_target_temperature_high_state_topic_to(topic_buf), payload, len))
+    if (!this->publish_float_(this->get_target_temperature_high_state_topic_to(topic_buf),
+                              this->device_->target_temperature_high, target_accuracy))
       success = false;
   } else {
-    len = value_accuracy_to_buf(payload, this->device_->target_temperature, target_accuracy);
-    if (!this->publish(this->get_target_temperature_state_topic_to(topic_buf), payload, len))
+    if (!this->publish_float_(this->get_target_temperature_state_topic_to(topic_buf), this->device_->target_temperature,
+                              target_accuracy))
       success = false;
   }
 
@@ -337,9 +340,8 @@ bool MQTTClimateComponent::publish_state_() {
       success = false;
   }
   if (traits.has_feature_flags(climate::CLIMATE_SUPPORTS_TARGET_HUMIDITY) &&
-      !std::isnan(this->device_->target_humidity)) {
-    len = value_accuracy_to_buf(payload, this->device_->target_humidity, 0);
-    if (!this->publish(this->get_target_humidity_state_topic_to(topic_buf), payload, len))
+      (!std::isnan(this->device_->target_humidity) || mqtt::global_mqtt_client->is_publish_nan_as_none())) {
+    if (!this->publish_float_(this->get_target_humidity_state_topic_to(topic_buf), this->device_->target_humidity, 0))
       success = false;
   }
 

esphome/components/mqtt/mqtt_component.cpp

@@ -2,6 +2,8 @@
 
 #ifdef USE_MQTT
 
+#include <cmath>
+
 #include "esphome/core/application.h"
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
@@ -124,6 +126,14 @@ StringRef MQTTComponent::get_command_topic_to_(std::span<char, MQTT_DEFAULT_TOPI
   return this->get_default_topic_for_to_(buf, "command", 7);
 }
 
+MQTTTopicSource MQTTComponent::command_topic_source_() const {
+  if (this->custom_command_topic_.is_lambda()) {
+    char buf[MQTT_DEFAULT_TOPIC_MAX_LEN];
+    return MQTTClientComponent::copy_topic(this->get_command_topic_to_(buf));
+  }
+  return this->topic_source_<MQTTComponent, &MQTTComponent::get_command_topic_to_>();
+}
+
 std::string MQTTComponent::get_state_topic_() const {
   char buf[MQTT_DEFAULT_TOPIC_MAX_LEN];
   StringRef ref = this->get_state_topic_to_(buf);
@@ -170,6 +180,14 @@ bool MQTTComponent::publish(const char *topic, ProgmemStr payload) {
 }
 #endif
 
+bool MQTTComponent::publish_float_(StringRef topic, float value, int8_t accuracy) {
+  if (global_mqtt_client->is_publish_nan_as_none() && std::isnan(value))
+    return this->publish(topic, "None", 4);
+  char buf[VALUE_ACCURACY_MAX_LEN];
+  size_t len = value_accuracy_to_buf(buf, value, accuracy);
+  return this->publish(topic, buf, len);
+}
+
 bool MQTTComponent::publish_json(const std::string &topic, const json::json_build_t &f) {
   return this->publish_json(topic.c_str(), f);
 }

esphome/components/mqtt/mqtt_component.h

@@ -21,23 +21,14 @@ struct SendDiscoveryConfig {
   bool command_topic{true};  ///< If the command topic should be included. Default to true.
 };
 
-// Max lengths for stack-based topic building.
-// These limits are enforced at Python config validation time in mqtt/__init__.py
-// using cv.Length() validators for topic_prefix and discovery_prefix.
-// This ensures the stack buffers are always large enough.
-static constexpr size_t MQTT_COMPONENT_TYPE_MAX_LEN = 20;
-static constexpr size_t MQTT_SUFFIX_MAX_LEN = 32;
-static constexpr size_t MQTT_TOPIC_PREFIX_MAX_LEN = 64;  // Validated in Python: cv.Length(max=64)
-// Stack buffer size - safe because all inputs are length-validated at config time
-// Format: prefix + "/" + type + "/" + object_id + "/" + suffix + null
-static constexpr size_t MQTT_DEFAULT_TOPIC_MAX_LEN =
-    MQTT_TOPIC_PREFIX_MAX_LEN + 1 + MQTT_COMPONENT_TYPE_MAX_LEN + 1 + OBJECT_ID_MAX_LEN + 1 + MQTT_SUFFIX_MAX_LEN + 1;
-static constexpr size_t MQTT_DISCOVERY_PREFIX_MAX_LEN = 64;  // Validated in Python: cv.Length(max=64)
+class MQTTComponent;  // Forward declaration
+
+// Validated in Python: cv.Length(max=64), DISCOVERY_PREFIX_MAX_LEN in mqtt/__init__.py
+static constexpr size_t MQTT_DISCOVERY_PREFIX_MAX_LEN = 64;
 // Format: prefix + "/" + type + "/" + name + "/" + object_id + "/config" + null
 static constexpr size_t MQTT_DISCOVERY_TOPIC_MAX_LEN = MQTT_DISCOVERY_PREFIX_MAX_LEN + 1 + MQTT_COMPONENT_TYPE_MAX_LEN +
                                                        1 + ESPHOME_DEVICE_NAME_MAX_LEN + 1 + OBJECT_ID_MAX_LEN + 7 + 1;
 
-class MQTTComponent;  // Forward declaration
 void log_mqtt_component(const char *tag, MQTTComponent *obj, bool state_topic, bool command_topic);
 
 #define LOG_MQTT_COMPONENT(state_topic, command_topic) log_mqtt_component(TAG, this, state_topic, command_topic)
@@ -49,7 +40,9 @@ void log_mqtt_component(const char *tag, MQTTComponent *obj, bool state_topic, b
   static_assert(sizeof(type_str) - 1 <= MQTT_COMPONENT_TYPE_MAX_LEN, \
                 #class_name "::component_type() exceeds MQTT_COMPONENT_TYPE_MAX_LEN");
 
-// Macro to define custom topic getter/setter with compile-time suffix length verification
+// Macro to define custom topic getter/setter with compile-time suffix length verification.
+// Set the topic before setup(): subscriptions rebuild it when needed, so a later change only reaches the broker
+// after a reconnect.
 #define MQTT_COMPONENT_CUSTOM_TOPIC_(name, type) \
   static_assert(sizeof(#name "/" #type) - 1 <= MQTT_SUFFIX_MAX_LEN, \
                 "topic suffix " #name "/" #type " exceeds MQTT_SUFFIX_MAX_LEN"); \
@@ -64,6 +57,10 @@ void log_mqtt_component(const char *tag, MQTTComponent *obj, bool state_topic, b
       return StringRef(this->custom_##name##_##type##_topic_.data(), this->custom_##name##_##type##_topic_.size()); \
     return this->get_default_topic_for_to_(buf, #name "/" #type, sizeof(#name "/" #type) - 1); \
   } \
+  MQTTTopicSource get_##name##_##type##_topic_source() const { \
+    using Self = std::remove_cvref_t<decltype(*this)>; \
+    return this->topic_source_<Self, &Self::get_##name##_##type##_topic_to>(); \
+  } \
   std::string get_##name##_##type##_topic() const { \
     if (this->custom_##name##_##type##_topic_.empty()) \
       return this->get_default_topic_for_(#name "/" #type); \
@@ -128,6 +125,7 @@ class MQTTComponent : public Component {
   template<typename T> void set_custom_state_topic(T &&custom_state_topic) {
     this->custom_state_topic_ = std::forward<T>(custom_state_topic);
   }
+  /// Set a custom command topic before setup(); a later change only reaches the broker after a reconnect.
   template<typename T> void set_custom_command_topic(T &&custom_command_topic) {
     this->custom_command_topic_ = std::forward<T>(custom_command_topic);
   }
@@ -255,30 +253,33 @@ class MQTTComponent : public Component {
 
   /** Subscribe to a MQTT topic.
    *
-   * @param topic The topic. Wildcards are currently not supported.
+   * @param topic The topic: an MQTTTopicSource, or a string that is copied to the heap.
    * @param callback The callback that will be called when a message with matching topic is received.
    * @param qos The MQTT quality of service. Defaults to 0.
    */
-  template<typename F> void subscribe(std::string topic, F &&callback, uint8_t qos = 0) {
-    global_mqtt_client->subscribe(std::move(topic), std::forward<F>(callback), qos);
+  template<typename T, typename F> void subscribe(T &&topic, F &&callback, uint8_t qos = 0) {
+    global_mqtt_client->subscribe(std::forward<T>(topic), std::forward<F>(callback), qos);
   }
 
   /** Subscribe to a MQTT topic and automatically parse JSON payload.
    *
    * If an invalid JSON payload is received, the callback will not be called.
    *
-   * @param topic The topic. Wildcards are currently not supported.
+   * @param topic The topic: an MQTTTopicSource, or a string that is copied to the heap.
    * @param callback The callback with a parsed JsonObject that will be called when a message with matching topic is
    * received.
    * @param qos The MQTT quality of service. Defaults to 0.
    */
-  template<typename F> void subscribe_json(std::string topic, F &&callback, uint8_t qos = 0) {
-    global_mqtt_client->subscribe_json(std::move(topic), std::forward<F>(callback), qos);
+  template<typename T, typename F> void subscribe_json(T &&topic, F &&callback, uint8_t qos = 0) {
+    global_mqtt_client->subscribe_json(std::forward<T>(topic), std::forward<F>(callback), qos);
   }
 
  protected:
   static constexpr uint32_t SEND_DEFER_ID = 0;
 
+  /// Publish a float formatted to accuracy, or "None" for NaN when publish_nan_as_none is set.
+  bool publish_float_(StringRef topic, float value, int8_t accuracy);
+
   /// Helper method to get the discovery topic for this component into a buffer.
   StringRef get_discovery_topic_to_(std::span<char, MQTT_DISCOVERY_TOPIC_MAX_LEN> buf,
                                     const MQTTDiscoveryInfo &discovery_info) const;
@@ -322,6 +323,18 @@ class MQTTComponent : public Component {
   /// @return StringRef pointing to the topic in the buffer.
   StringRef get_command_topic_to_(std::span<char, MQTT_DEFAULT_TOPIC_MAX_LEN> buf) const;
 
+  /// The command topic as a subscription source. A lambda topic is read once and copied, as it may change.
+  MQTTTopicSource command_topic_source_() const;
+
+  /// A topic built by one of this component's `get_*_topic_to` getters, rebuilt when needed.
+  template<typename C, StringRef (C::*Getter)(std::span<char, MQTT_DEFAULT_TOPIC_MAX_LEN>) const>
+  MQTTTopicSource topic_source_() const {
+    return {[](const void *context, std::span<char, MQTT_DEFAULT_TOPIC_MAX_LEN> buf) {
+              return (static_cast<const C *>(context)->*Getter)(buf);
+            },
+            static_cast<const C *>(this)};
+  }
+
   /// Get the MQTT topic that new states will be shared to (allocates std::string).
   std::string get_state_topic_() const;
 

esphome/components/mqtt/mqtt_cover.cpp

@@ -31,38 +31,40 @@ MQTTCoverComponent::MQTTCoverComponent(Cover *cover) : cover_(cover) {}
 void MQTTCoverComponent::setup() {
   auto traits = this->cover_->get_traits();
   this->cover_->add_on_state_callback([this]() { this->publish_state(); });
-  this->subscribe(this->get_command_topic_(), [this](const std::string &topic, const std::string &payload) {
+  this->subscribe(this->command_topic_source_(), [this](const std::string &topic, const std::string &payload) {
     auto call = this->cover_->make_call();
     call.set_command(payload.c_str());
     call.perform();
   });
   if (traits.get_supports_position()) {
-    this->subscribe(this->get_position_command_topic(), [this](const std::string &topic, const std::string &payload) {
-      auto value = parse_number<float>(payload);
-      if (!value.has_value()) {
-        ESP_LOGW(TAG, "Invalid position value: '%s'", payload.c_str());
-        return;
-      }
-      auto call = this->cover_->make_call();
-      call.set_position(*value / 100.0f);
-      call.perform();
-    });
+    this->subscribe(this->get_position_command_topic_source(),
+                    [this](const std::string &topic, const std::string &payload) {
+                      auto value = parse_number<float>(payload);
+                      if (!value.has_value()) {
+                        ESP_LOGW(TAG, "Invalid position value: '%s'", payload.c_str());
+                        return;
+                      }
+                      auto call = this->cover_->make_call();
+                      call.set_position(*value / 100.0f);
+                      call.perform();
+                    });
   }
   if (traits.get_supports_tilt()) {
-    this->subscribe(this->get_tilt_command_topic(), [this](const std::string &topic, const std::string &payload) {
-      if (ESPHOME_strcasecmp_P(payload.c_str(), ESPHOME_PSTR("STOP")) == 0) {
-        this->cover_->make_call().set_command_stop_tilt().perform();
-        return;
-      }
-      auto value = parse_number<float>(payload);
-      if (!value.has_value()) {
-        ESP_LOGW(TAG, "Invalid tilt value: '%s'", payload.c_str());
-        return;
-      }
-      auto call = this->cover_->make_call();
-      call.set_tilt(*value / 100.0f);
-      call.perform();
-    });
+    this->subscribe(this->get_tilt_command_topic_source(),
+                    [this](const std::string &topic, const std::string &payload) {
+                      if (ESPHOME_strcasecmp_P(payload.c_str(), ESPHOME_PSTR("STOP")) == 0) {
+                        this->cover_->make_call().set_command_stop_tilt().perform();
+                        return;
+                      }
+                      auto value = parse_number<float>(payload);
+                      if (!value.has_value()) {
+                        ESP_LOGW(TAG, "Invalid tilt value: '%s'", payload.c_str());
+                        return;
+                      }
+                      auto call = this->cover_->make_call();
+                      call.set_tilt(*value / 100.0f);
+                      call.perform();
+                    });
   }
 }
 

esphome/components/mqtt/mqtt_date.cpp

@@ -17,7 +17,7 @@ using namespace esphome::datetime;
 MQTTDateComponent::MQTTDateComponent(DateEntity *date) : date_(date) {}
 
 void MQTTDateComponent::setup() {
-  this->subscribe_json(this->get_command_topic_(), [this](const std::string &topic, JsonObject root) {
+  this->subscribe_json(this->command_topic_source_(), [this](const std::string &topic, JsonObject root) {
     auto call = this->date_->make_call();
     if (root[ESPHOME_F("year")].is<uint16_t>()) {
       call.set_year(root[ESPHOME_F("year")]);

esphome/components/mqtt/mqtt_datetime.cpp

@@ -17,7 +17,7 @@ using namespace esphome::datetime;
 MQTTDateTimeComponent::MQTTDateTimeComponent(DateTimeEntity *datetime) : datetime_(datetime) {}
 
 void MQTTDateTimeComponent::setup() {
-  this->subscribe_json(this->get_command_topic_(), [this](const std::string &topic, JsonObject root) {
+  this->subscribe_json(this->command_topic_source_(), [this](const std::string &topic, JsonObject root) {
     auto call = this->datetime_->make_call();
     if (root[ESPHOME_F("year")].is<uint16_t>()) {
       call.set_year(root[ESPHOME_F("year")]);

esphome/components/mqtt/mqtt_fan.cpp

@@ -28,7 +28,7 @@ MQTT_COMPONENT_TYPE(MQTTFanComponent, "fan")
 const EntityBase *MQTTFanComponent::get_entity() const { return this->state_; }
 
 void MQTTFanComponent::setup() {
-  this->subscribe(this->get_command_topic_(), [this](const std::string &topic, const std::string &payload) {
+  this->subscribe(this->command_topic_source_(), [this](const std::string &topic, const std::string &payload) {
     auto val = parse_on_off(payload.c_str());
     switch (val) {
       case PARSE_ON:
@@ -52,33 +52,34 @@ void MQTTFanComponent::setup() {
   });
 
   if (this->state_->get_traits().supports_direction()) {
-    this->subscribe(this->get_direction_command_topic(), [this](const std::string &topic, const std::string &payload) {
-      auto val = parse_on_off(payload.c_str(), "forward", "reverse");
-      switch (val) {
-        case PARSE_ON:
-          ESP_LOGD(TAG, "'%s': Setting direction FORWARD", LOG_STR_ARG(this->log_name_()));
-          this->state_->make_call().set_direction(fan::FanDirection::FORWARD).perform();
-          break;
-        case PARSE_OFF:
-          ESP_LOGD(TAG, "'%s': Setting direction REVERSE", LOG_STR_ARG(this->log_name_()));
-          this->state_->make_call().set_direction(fan::FanDirection::REVERSE).perform();
-          break;
-        case PARSE_TOGGLE:
-          this->state_->make_call()
-              .set_direction(this->state_->direction == fan::FanDirection::FORWARD ? fan::FanDirection::REVERSE
-                                                                                   : fan::FanDirection::FORWARD)
-              .perform();
-          break;
-        case PARSE_NONE:
-          ESP_LOGW(TAG, "Unknown direction Payload %s", payload.c_str());
-          this->status_momentary_warning(5000);
-          break;
-      }
-    });
+    this->subscribe(
+        this->get_direction_command_topic_source(), [this](const std::string &topic, const std::string &payload) {
+          auto val = parse_on_off(payload.c_str(), "forward", "reverse");
+          switch (val) {
+            case PARSE_ON:
+              ESP_LOGD(TAG, "'%s': Setting direction FORWARD", LOG_STR_ARG(this->log_name_()));
+              this->state_->make_call().set_direction(fan::FanDirection::FORWARD).perform();
+              break;
+            case PARSE_OFF:
+              ESP_LOGD(TAG, "'%s': Setting direction REVERSE", LOG_STR_ARG(this->log_name_()));
+              this->state_->make_call().set_direction(fan::FanDirection::REVERSE).perform();
+              break;
+            case PARSE_TOGGLE:
+              this->state_->make_call()
+                  .set_direction(this->state_->direction == fan::FanDirection::FORWARD ? fan::FanDirection::REVERSE
+                                                                                       : fan::FanDirection::FORWARD)
+                  .perform();
+              break;
+            case PARSE_NONE:
+              ESP_LOGW(TAG, "Unknown direction Payload %s", payload.c_str());
+              this->status_momentary_warning(5000);
+              break;
+          }
+        });
   }
 
   if (this->state_->get_traits().supports_oscillation()) {
-    this->subscribe(this->get_oscillation_command_topic(),
+    this->subscribe(this->get_oscillation_command_topic_source(),
                     [this](const std::string &topic, const std::string &payload) {
                       auto val = parse_on_off(payload.c_str(), "oscillate_on", "oscillate_off");
                       switch (val) {
@@ -102,7 +103,7 @@ void MQTTFanComponent::setup() {
   }
 
   if (this->state_->get_traits().supports_speed()) {
-    this->subscribe(this->get_speed_level_command_topic(),
+    this->subscribe(this->get_speed_level_command_topic_source(),
                     [this](const std::string &topic, const std::string &payload) {
                       optional<int> speed_level_opt = parse_number<int>(payload);
                       if (speed_level_opt.has_value()) {

esphome/components/mqtt/mqtt_light.cpp

@@ -18,7 +18,7 @@ MQTT_COMPONENT_TYPE(MQTTJSONLightComponent, "light")
 const EntityBase *MQTTJSONLightComponent::get_entity() const { return this->state_; }
 
 void MQTTJSONLightComponent::setup() {
-  this->subscribe_json(this->get_command_topic_(), [this](const std::string &topic, JsonObject root) {
+  this->subscribe_json(this->command_topic_source_(), [this](const std::string &topic, JsonObject root) {
     LightCall call = this->state_->make_call();
     LightJSONSchema::parse_json(*this->state_, call, root);
     call.perform();

esphome/components/mqtt/mqtt_lock.cpp

@@ -16,7 +16,7 @@ using namespace esphome::lock;
 MQTTLockComponent::MQTTLockComponent(lock::Lock *a_lock) : lock_(a_lock) {}
 
 void MQTTLockComponent::setup() {
-  this->subscribe(this->get_command_topic_(), [this](const std::string &topic, const std::string &payload) {
+  this->subscribe(this->command_topic_source_(), [this](const std::string &topic, const std::string &payload) {
     if (ESPHOME_strcasecmp_P(payload.c_str(), ESPHOME_PSTR("LOCK")) == 0) {
       this->lock_->lock();
     } else if (ESPHOME_strcasecmp_P(payload.c_str(), ESPHOME_PSTR("UNLOCK")) == 0) {

esphome/components/mqtt/mqtt_number.cpp

@@ -19,7 +19,7 @@ PROGMEM_STRING_TABLE(NumberMqttModeStrings, "", "box", "slider");
 MQTTNumberComponent::MQTTNumberComponent(Number *number) : number_(number) {}
 
 void MQTTNumberComponent::setup() {
-  this->subscribe(this->get_command_topic_(), [this](const std::string &topic, const std::string &state) {
+  this->subscribe(this->command_topic_source_(), [this](const std::string &topic, const std::string &state) {
     auto val = parse_number<float>(state);
     if (!val.has_value()) {
       ESP_LOGW(TAG, "Can't convert '%s' to number!", state.c_str());

esphome/components/mqtt/mqtt_select.cpp

@@ -15,7 +15,7 @@ using namespace esphome::select;
 MQTTSelectComponent::MQTTSelectComponent(Select *select) : select_(select) {}
 
 void MQTTSelectComponent::setup() {
-  this->subscribe(this->get_command_topic_(), [this](const std::string &topic, const std::string &state) {
+  this->subscribe(this->command_topic_source_(), [this](const std::string &topic, const std::string &state) {
     auto call = this->select_->make_call();
     call.set_option(state);
     call.perform();

esphome/components/mqtt/mqtt_sensor.cpp

@@ -73,12 +73,7 @@ bool MQTTSensorComponent::send_initial_state() {
 }
 bool MQTTSensorComponent::publish_state(float value) {
   char topic_buf[MQTT_DEFAULT_TOPIC_MAX_LEN];
-  if (mqtt::global_mqtt_client->is_publish_nan_as_none() && std::isnan(value))
-    return this->publish(this->get_state_topic_to_(topic_buf), "None", 4);
-  int8_t accuracy = this->sensor_->get_accuracy_decimals();
-  char buf[VALUE_ACCURACY_MAX_LEN];
-  size_t len = value_accuracy_to_buf(buf, value, accuracy);
-  return this->publish(this->get_state_topic_to_(topic_buf), buf, len);
+  return this->publish_float_(this->get_state_topic_to_(topic_buf), value, this->sensor_->get_accuracy_decimals());
 }
 
 }  // namespace esphome::mqtt

esphome/components/mqtt/mqtt_switch.cpp

@@ -15,7 +15,7 @@ using namespace esphome::switch_;
 MQTTSwitchComponent::MQTTSwitchComponent(switch_::Switch *a_switch) : switch_(a_switch) {}
 
 void MQTTSwitchComponent::setup() {
-  this->subscribe(this->get_command_topic_(), [this](const std::string &topic, const std::string &payload) {
+  this->subscribe(this->command_topic_source_(), [this](const std::string &topic, const std::string &payload) {
     switch (parse_on_off(payload.c_str())) {
       case PARSE_ON:
         this->switch_->turn_on();

esphome/components/mqtt/mqtt_text.cpp

@@ -19,7 +19,7 @@ PROGMEM_STRING_TABLE(TextMqttModeStrings, "text", "password");
 MQTTTextComponent::MQTTTextComponent(Text *text) : text_(text) {}
 
 void MQTTTextComponent::setup() {
-  this->subscribe(this->get_command_topic_(), [this](const std::string &topic, const std::string &state) {
+  this->subscribe(this->command_topic_source_(), [this](const std::string &topic, const std::string &state) {
     auto call = this->text_->make_call();
     call.set_value(state);
     call.perform();

esphome/components/mqtt/mqtt_time.cpp

@@ -17,7 +17,7 @@ using namespace esphome::datetime;
 MQTTTimeComponent::MQTTTimeComponent(TimeEntity *time) : time_(time) {}
 
 void MQTTTimeComponent::setup() {
-  this->subscribe_json(this->get_command_topic_(), [this](const std::string &topic, JsonObject root) {
+  this->subscribe_json(this->command_topic_source_(), [this](const std::string &topic, JsonObject root) {
     auto call = this->time_->make_call();
     if (root[ESPHOME_F("hour")].is<uint8_t>()) {
       call.set_hour(root[ESPHOME_F("hour")]);

esphome/components/mqtt/mqtt_update.cpp

@@ -15,7 +15,7 @@ using namespace esphome::update;
 MQTTUpdateComponent::MQTTUpdateComponent(UpdateEntity *update) : update_(update) {}
 
 void MQTTUpdateComponent::setup() {
-  this->subscribe(this->get_command_topic_(), [this](const std::string &topic, const std::string &payload) {
+  this->subscribe(this->command_topic_source_(), [this](const std::string &topic, const std::string &payload) {
     if (payload == "INSTALL") {
       this->update_->perform();
     } else {

esphome/components/mqtt/mqtt_valve.cpp

@@ -31,22 +31,23 @@ MQTTValveComponent::MQTTValveComponent(Valve *valve) : valve_(valve) {}
 void MQTTValveComponent::setup() {
   auto traits = this->valve_->get_traits();
   this->valve_->add_on_state_callback([this]() { this->publish_state(); });
-  this->subscribe(this->get_command_topic_(), [this](const std::string &topic, const std::string &payload) {
+  this->subscribe(this->command_topic_source_(), [this](const std::string &topic, const std::string &payload) {
     auto call = this->valve_->make_call();
     call.set_command(payload.c_str());
     call.perform();
   });
   if (traits.get_supports_position()) {
-    this->subscribe(this->get_position_command_topic(), [this](const std::string &topic, const std::string &payload) {
-      auto value = parse_number<float>(payload);
-      if (!value.has_value()) {
-        ESP_LOGW(TAG, "Invalid position value: '%s'", payload.c_str());
-        return;
-      }
-      auto call = this->valve_->make_call();
-      call.set_position(*value / 100.0f);
-      call.perform();
-    });
+    this->subscribe(this->get_position_command_topic_source(),
+                    [this](const std::string &topic, const std::string &payload) {
+                      auto value = parse_number<float>(payload);
+                      if (!value.has_value()) {
+                        ESP_LOGW(TAG, "Invalid position value: '%s'", payload.c_str());
+                        return;
+                      }
+                      auto call = this->valve_->make_call();
+                      call.set_position(*value / 100.0f);
+                      call.perform();
+                    });
   }
 }
 

esphome/components/online_image/online_image.cpp

@@ -231,30 +231,47 @@ void OnlineImage::loop() {
       this->end_connection_();
       this->download_error_callback_.call();
       return;
-    } else if (this->downloader_->is_read_complete()) {
-      // The response has ended, so data left in the buffer can never be completed
-      size_t unread = this->download_buffer_.unread();
-      if (unread == 0 && this->downloader_->get_bytes_read() > 0) {
-        // A decoder that counts pixels knows the image was cut off; PNG cannot tell, so it passes
-        if (this->is_decode_finished() || !this->decoder_knows_end()) {
-          this->finish_download_();
+    } else {
+      // Nothing new arrived. The decoder works for a limited time per call, so it may still have data to use.
+      if (this->download_buffer_.unread() != 0) {
+        auto consumed = this->feed_data(this->download_buffer_.data(), this->download_buffer_.unread());
+        if (consumed < 0) {
+          ESP_LOGE(TAG, "Error decoding image: %s", esphome::runtime_image::decode_error_to_string(consumed));
+          this->end_connection_();
+          this->download_error_callback_.call();
+          return;
+        }
+        if (consumed > 0) {
+          this->download_buffer_.read(consumed);
+          this->last_data_time_ = App.get_loop_component_start_time();
           return;
         }
-        ESP_LOGE(TAG, "Download ended before the image was complete");
-      } else if (unread == 0) {
-        ESP_LOGE(TAG, "Server sent an empty response");
-      } else {
-        ESP_LOGE(TAG, "Download ended with %zu bytes the decoder could not use", unread);
       }
-      this->end_connection_();
-      this->download_error_callback_.call();
-      return;
-    } else if (App.get_loop_component_start_time() - this->last_data_time_ >= this->parent_->get_timeout()) {
-      // ESP-IDF keeps returning 0 after the server closes the connection early
-      ESP_LOGE(TAG, "No data received for %" PRIu32 " ms", this->parent_->get_timeout());
-      this->end_connection_();
-      this->download_error_callback_.call();
-      return;
+      if (this->downloader_->is_read_complete()) {
+        // The response has ended, so data left in the buffer can never be completed
+        size_t unread = this->download_buffer_.unread();
+        if (unread == 0 && this->downloader_->get_bytes_read() > 0) {
+          // A decoder that counts pixels knows the image was cut off; PNG cannot tell, so it passes
+          if (this->is_decode_finished() || !this->decoder_knows_end()) {
+            this->finish_download_();
+            return;
+          }
+          ESP_LOGE(TAG, "Download ended before the image was complete");
+        } else if (unread == 0) {
+          ESP_LOGE(TAG, "Server sent an empty response");
+        } else {
+          ESP_LOGE(TAG, "Download ended with %zu bytes the decoder could not use", unread);
+        }
+        this->end_connection_();
+        this->download_error_callback_.call();
+        return;
+      } else if (App.get_loop_component_start_time() - this->last_data_time_ >= this->parent_->get_timeout()) {
+        // ESP-IDF keeps returning 0 after the server closes the connection early
+        ESP_LOGE(TAG, "No data received for %" PRIu32 " ms", this->parent_->get_timeout());
+        this->end_connection_();
+        this->download_error_callback_.call();
+        return;
+      }
     }
   } else {
     // Buffer is full, need to decode some data first

esphome/components/ota/ota_backend_esp_idf.cpp

@@ -198,7 +198,7 @@ OTAResponseTypes IDFOTABackend::end() {
   // signature, and only re-points boot at an already-installed app -- so it is
   // intentionally not run through the signature verifier below.
   if (this->ota_type_ == ota::OTA_TYPE_UPDATE_PARTITION_TABLE) {
-    return this->hold_lock_if_ok_(this->update_partition_table());
+    return this->hold_lock_if_ok_(this->update_partition_table_());
   }
   if (!this->is_app_or_bootloader_update_()) {
     return OTA_RESPONSE_ERROR_UNSUPPORTED_OTA_TYPE;

esphome/components/ota/ota_backend_esp_idf.h

@@ -54,7 +54,7 @@ class IDFOTABackend final {
 
   OTAResponseTypes validate_new_partition_table_(uint32_t running_app_offset, size_t running_app_size,
                                                  PartitionTablePlan &plan);
-  OTAResponseTypes update_partition_table();
+  OTAResponseTypes update_partition_table_();
   OTAResponseTypes register_and_validate_partition_table_part_();
   // Defined in ota_bootloader_esp_idf.cpp:
   OTAResponseTypes register_and_validate_bootloader_part_();

esphome/components/ota/ota_partitions_esp_idf.cpp

@@ -40,7 +40,7 @@ static const esp_partition_t *find_app_partition_at(uint32_t address, size_t min
 }
 
 // Validates the staged partition table and picks the post-update boot slot. All non-destructive
-// checks live here; the destructive write is in update_partition_table().
+// checks live here; the destructive write is in update_partition_table_().
 // Side effect: registers the live partition-table region as partition_table_part_ so the caller
 // can write to it; abort() releases it on error.
 OTAResponseTypes IDFOTABackend::validate_new_partition_table_(uint32_t running_app_offset, size_t running_app_size,
@@ -165,7 +165,7 @@ OTAResponseTypes IDFOTABackend::validate_new_partition_table_(uint32_t running_a
   return OTA_RESPONSE_OK;
 }
 
-OTAResponseTypes IDFOTABackend::update_partition_table() {
+OTAResponseTypes IDFOTABackend::update_partition_table_() {
   if (this->buf_written_ == 0 || this->image_size_ != this->buf_written_) {
     ESP_LOGE(TAG, "Not enough data received");
     return OTA_RESPONSE_ERROR_PARTITION_TABLE_VERIFY;
@@ -300,9 +300,9 @@ OTAResponseTypes IDFOTABackend::register_and_validate_partition_table_part_() {
 // Process-scoped cache. Cannot be a backend member: backends are per-connection but the cache
 // must outlive a connection that called esp_partition_unload_all(), after which
 // esp_ota_get_running_partition() no longer returns valid data.
-static bool s_running_app_initialized = false;
-static uint32_t s_running_app_cached_offset = 0;
-static size_t s_running_app_cached_size = 0;
+static bool s_running_app_initialized = false;    // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
+static uint32_t s_running_app_cached_offset = 0;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
+static size_t s_running_app_cached_size = 0;      // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
 
 // Flag-gated rather than size==0 so a failed first call doesn't poison the cache.
 void get_running_app_position(uint32_t &offset, size_t &size) {

esphome/components/packet_interface/__init__.py

@@ -1,3 +1,5 @@
+import importlib
+import pkgutil
 from typing import Any
 
 import esphome.codegen as cg
@@ -7,14 +9,27 @@ from esphome.const import CONF_ID
 from esphome.cpp_generator import MockObj, MockObjClass
 from esphome.types import ConfigType
 
+from . import layers
+from .layers import (
+    PacketInterface,
+    build_stack,
+    check_session,
+    packet_interface_ns,
+    stack_auto_load,
+    stack_schema,
+)
+
 CODEOWNERS = ["@clydebarrow"]
 DOMAIN = "packet_interface"
 IS_PLATFORM_COMPONENT = True
 
 CONF_PACKET_INTERFACE = "packet_interface"
 
-packet_interface_ns = cg.esphome_ns.namespace("packet_interface")
-PacketInterface = packet_interface_ns.class_("PacketInterface", cg.Component)
+# Each module in layers registers a layer type; import them all before building the schemas from the registry
+for module_info in pkgutil.iter_modules(layers.__path__):
+    importlib.import_module(f".layers.{module_info.name}", package=__package__)
+
+LINK_LAYER_SCHEMAS = stack_schema()
 
 __all__ = [
     "CONF_MAX_PACKET_SIZE",
@@ -26,24 +41,47 @@ __all__ = [
 ]
 
 
+def AUTO_LOAD(config: list[ConfigType] | None = None) -> list[str]:
+    """Load only what the configured layers use.
+
+    Without a config, as tooling that lists dependencies calls it, everything a layer could load.
+    """
+    if config is None:
+        return ["binary_sensor", "xxtea"]
+    loads: set[str] = set()
+    for interface in config:
+        loads |= stack_auto_load(interface)
+    return sorted(loads)
+
+
+def _validate_layers(config: ConfigType) -> ConfigType:
+    check_session(config)
+    return config
+
+
 def packet_interface_schema(
     class_: MockObjClass,
     default_max_packet_size: int,
     max_packet_size_limit: int = 4096,
 ) -> cv.Schema:
-    """Schema for a link platform."""
-    return cv.Schema(
-        {
-            cv.GenerateID(): cv.declare_id(class_),
-            cv.Optional(
-                CONF_MAX_PACKET_SIZE, default=default_max_packet_size
-            ): cv.int_range(16, max_packet_size_limit),
-        }
-    ).extend(cv.COMPONENT_SCHEMA)
+    """Schema for a link platform, with the layers that can be stacked on it."""
+    return (
+        cv.Schema(
+            {
+                cv.GenerateID(): cv.declare_id(class_),
+                cv.Optional(
+                    CONF_MAX_PACKET_SIZE, default=default_max_packet_size
+                ): cv.int_range(16, max_packet_size_limit),
+                **LINK_LAYER_SCHEMAS,
+            }
+        )
+        .extend(cv.COMPONENT_SCHEMA)
+        .add_extra(_validate_layers)
+    )
 
 
 async def new_packet_interface(config: ConfigType, *args: Any) -> MockObj:
-    """Create the link layer.
+    """Create the link layer and the layers configured on it, from the bottom up.
 
     The link's constructor takes its maximum packet size, usually as a std::span over a buffer that code generation
     sized from the configuration, so the two cannot differ.
@@ -53,4 +91,5 @@ async def new_packet_interface(config: ConfigType, *args: Any) -> MockObj:
     var = cg.new_Pvariable(config[CONF_ID], *args)
     cg.add(var.set_name(config[CONF_ID].id))
     await cg.register_component(var, config)
+    await build_stack(config, var)
     return var

esphome/components/packet_interface/basic_session.cpp

@@ -0,0 +1,135 @@
+#include "basic_session.h"
+#ifdef USE_PACKET_INTERFACE_BASIC_SESSION
+#include "esphome/core/application.h"
+#include "esphome/core/helpers.h"
+#include "esphome/core/log.h"
+
+namespace esphome::packet_interface {
+
+ESPHOME_LOG_TAG(TAG, "packet_interface.session");
+
+static constexpr uint8_t PACKET_TYPE_DATA = 0x00;
+static constexpr uint8_t PACKET_TYPE_KEEP_ALIVE = 0x01;
+// Type byte and the 32-bit boot id, little endian
+static constexpr size_t KEEP_ALIVE_SIZE = 5;
+
+BasicSession::BasicSession(PacketInterface *lower)
+    : PacketInterface(remaining_packet_size(lower->get_max_packet_size(), HEADER_SIZE)), lower_(lower) {
+  this->stack_on_(lower, CAPABILITY_SESSION);
+  // Data lost by a reliability layer below must be sent again, as after a restart of the peer
+  if (lower->has_capability(CAPABILITY_RELIABILITY)) {
+    lower->add_on_session_state_callback([this](bool connected) {
+      if (!connected && this->connected_)
+        this->set_connected_(false);
+    });
+  }
+}
+
+void BasicSession::setup() {
+  if (this->lower_->get_max_packet_size() < KEEP_ALIVE_SIZE) {
+    this->mark_failed(LOG_STR("max_packet_size too small for the configured layers"));
+    return;
+  }
+  this->boot_id_ = random_uint32();
+#ifdef USE_BINARY_SENSOR
+  if (this->connected_sensor_ != nullptr)
+    this->connected_sensor_->publish_initial_state(false);
+#endif
+  this->lower_->add_packet_interface_listener(
+      [this](const PacketBuffer &data, const PacketMetaData &meta_data) { this->receive_(data, meta_data); });
+  // A reliability layer below queues this until its link is up, so the peer hears of a restart straight away
+  this->send_keep_alive_();
+  this->set_interval(this->keep_alive_ms_, [this] { this->on_interval_(); });
+}
+
+void BasicSession::dump_config() {
+  ESP_LOGCONFIG(TAG,
+                "Basic Session on '%s':\n"
+                "  Max Packet Size: %zu\n"
+                "  Keep Alive: %" PRIu32 "ms\n"
+                "  Timeout: %" PRIu32 "ms",
+                this->name_, this->max_packet_size_, this->keep_alive_ms_, this->timeout_ms_);
+#ifdef USE_BINARY_SENSOR
+  LOG_BINARY_SENSOR("  ", "Connected", this->connected_sensor_);
+#endif
+}
+
+bool BasicSession::send_to_interface(const PacketBuffer &data, const PacketMetaData &meta_data) {
+  if (this->is_failed())
+    return false;
+  uint8_t header = PACKET_TYPE_DATA;
+  PacketBuffer packet(&header, HEADER_SIZE, &data, nullptr, 0);
+  return this->lower_->send_to_interface(packet, meta_data);
+}
+
+void BasicSession::send_keep_alive_() {
+  uint8_t packet[KEEP_ALIVE_SIZE] = {
+      PACKET_TYPE_KEEP_ALIVE, static_cast<uint8_t>(this->boot_id_), static_cast<uint8_t>(this->boot_id_ >> 8),
+      static_cast<uint8_t>(this->boot_id_ >> 16), static_cast<uint8_t>(this->boot_id_ >> 24)};
+  if (!this->lower_->send_to_interface(PacketBuffer(packet, sizeof(packet)), {}))
+    ESP_LOGV(TAG, "%s: Lower layer did not accept keep-alive", this->name_);
+}
+
+void BasicSession::on_interval_() {
+  this->send_keep_alive_();
+  if (this->connected_ && App.get_loop_component_start_time() - this->last_receive_ms_ > this->timeout_ms_) {
+    ESP_LOGW(TAG, "%s: Peer not heard for %" PRIu32 "ms", this->name_, this->timeout_ms_);
+    this->set_connected_(false);
+  }
+}
+
+void BasicSession::set_connected_(bool connected) {
+  this->connected_ = connected;
+  ESP_LOGD(TAG, "%s: Session %s", this->name_, connected ? LOG_STR_LITERAL("connected") : LOG_STR_LITERAL("lost"));
+#ifdef USE_BINARY_SENSOR
+  if (this->connected_sensor_ != nullptr)
+    this->connected_sensor_->publish_state(connected);
+#endif
+  this->session_callback_.call(connected);
+}
+
+void BasicSession::receive_(const PacketBuffer &data, const PacketMetaData &meta_data) {
+  // Lower layers deliver packets in one piece; the payload is passed up without copying
+  if (!data.is_contiguous()) {
+    ESP_LOGW(TAG, "%s: Packet split across buffers is not supported", this->name_);
+    return;
+  }
+  auto packet = data.as_span();
+  if (packet.empty()) {
+    this->note_received_(false);
+    return;
+  }
+  uint8_t type = packet[0];
+  if (type == PACKET_TYPE_KEEP_ALIVE) {
+    if (packet.size() != KEEP_ALIVE_SIZE) {
+      ESP_LOGV(TAG, "%s: Keep-alive of %zu bytes dropped", this->name_, packet.size());
+      this->note_received_(false);
+      return;
+    }
+    uint32_t boot_id = encode_uint32(packet[4], packet[3], packet[2], packet[1]);
+    if (!this->peer_known_ || boot_id != this->peer_boot_id_) {
+      // Data may arrive before the first keep-alive, so only a changed id means a restart
+      if (this->connected_ && this->peer_known_) {
+        ESP_LOGD(TAG, "%s: Peer has restarted", this->name_);
+        this->set_connected_(false);
+      }
+      this->peer_boot_id_ = boot_id;
+      this->peer_known_ = true;
+      // Answer a new peer at once; it answers only if our id is new to it, so this cannot repeat
+      this->send_keep_alive_();
+    }
+  } else if (type != PACKET_TYPE_DATA) {
+    ESP_LOGV(TAG, "%s: Unknown packet type 0x%02X dropped", this->name_, type);
+    this->note_received_(false);
+    return;
+  }
+  this->note_received_(true);
+  this->last_receive_ms_ = App.get_loop_component_start_time();
+  if (!this->connected_)
+    this->set_connected_(true);
+  if (type == PACKET_TYPE_DATA)
+    this->on_receive_from_interface_(PacketBuffer(packet.data() + HEADER_SIZE, packet.size() - HEADER_SIZE), meta_data);
+}
+
+}  // namespace esphome::packet_interface
+#endif

esphome/components/packet_interface/basic_session.h

@@ -0,0 +1,61 @@
+#pragma once
+
+#include "esphome/core/defines.h"
+#ifdef USE_PACKET_INTERFACE_BASIC_SESSION
+#include "packet_interface.h"
+#ifdef USE_BINARY_SENSOR
+#include "esphome/components/binary_sensor/binary_sensor.h"
+#endif
+
+namespace esphome::packet_interface {
+
+/**
+ * A session layer that reports whether the peer can be reached.
+ *
+ * Every packet starts with a type byte. A data packet carries the payload of the layer above. A keep-alive packet
+ * carries a random number chosen at boot, so that a restart of the peer is noticed even if it is quicker than the
+ * timeout. A keep-alive is sent every keep_alive interval, and at once when a new peer is heard. The peer is
+ * considered gone when nothing has been received from it for the timeout; this is checked every keep_alive interval.
+ * A restart of the peer is reported as a disconnect followed by a connect. So is a reset of a reliability layer below,
+ * which discards data: the session is reported lost, and connected again when the peer is next heard.
+ */
+class BasicSession : public PacketInterface {
+ public:
+  /// The packet type byte.
+  static constexpr size_t HEADER_SIZE = 1;
+
+  explicit BasicSession(PacketInterface *lower);
+
+  void setup() override;
+  void dump_config() override;
+  bool send_to_interface(const PacketBuffer &data, const PacketMetaData &meta_data) override;
+
+  void set_keep_alive(uint32_t keep_alive_ms) { this->keep_alive_ms_ = keep_alive_ms; }
+  void set_session_timeout(uint32_t timeout_ms) { this->timeout_ms_ = timeout_ms; }
+#ifdef USE_BINARY_SENSOR
+  void set_connected_sensor(binary_sensor::BinarySensor *sensor) { this->connected_sensor_ = sensor; }
+#endif
+
+  bool is_connected() const { return this->connected_; }
+
+ protected:
+  void receive_(const PacketBuffer &data, const PacketMetaData &meta_data);
+  void on_interval_();
+  void send_keep_alive_();
+  void set_connected_(bool connected);
+
+  PacketInterface *lower_;
+#ifdef USE_BINARY_SENSOR
+  binary_sensor::BinarySensor *connected_sensor_{nullptr};
+#endif
+  uint32_t keep_alive_ms_{10000};
+  uint32_t timeout_ms_{30000};
+  uint32_t last_receive_ms_{0};
+  uint32_t boot_id_{0};
+  uint32_t peer_boot_id_{0};
+  bool peer_known_{false};
+  bool connected_{false};
+};
+
+}  // namespace esphome::packet_interface
+#endif

esphome/components/packet_interface/hdlc_reliability.cpp

@@ -0,0 +1,365 @@
+#include "hdlc_reliability.h"
+#ifdef USE_PACKET_INTERFACE_HDLC
+#include "esphome/core/helpers.h"
+#include "esphome/core/log.h"
+
+namespace esphome::packet_interface {
+
+ESPHOME_LOG_TAG(TAG, "packet_interface.hdlc");
+
+static constexpr uint32_t RETRANSMIT_TIMER_ID = 0;
+static constexpr uint32_t ACK_TIMER_ID = 1;
+static constexpr uint32_t SEND_RETRY_TIMER_ID = 2;
+// A frame or acknowledgement the lower layer refused, for example while a radio was receiving, is sent again after this
+static constexpr uint32_t SEND_RETRY_MS = 50;
+// SABM is repeated at this interval, or the timeout if longer, until the peer answers
+static constexpr uint32_t RESET_INTERVAL_MS = 1000;
+
+// Control byte formats, with the poll/final bit (0x10) ignored
+static constexpr uint8_t POLL_FINAL_BIT = 0x10;
+static constexpr uint8_t SUPERVISORY_FRAME = 0x01;
+static constexpr uint8_t SUPERVISORY_RR = 0x00;
+static constexpr uint8_t SUPERVISORY_REJ = 0x02;
+static constexpr uint8_t UNNUMBERED_SABM = 0x2F;
+static constexpr uint8_t UNNUMBERED_UA = 0x63;
+
+static uint16_t frame_crc(uint8_t control, const uint8_t *payload, size_t length) {
+  uint16_t crc = crc16(&control, 1);
+  return crc16(payload, length, crc);
+}
+
+HdlcReliability::HdlcReliability(PacketInterface *lower)
+    : PacketInterface(remaining_packet_size(lower->get_max_packet_size(), FRAME_OVERHEAD)), lower_(lower) {
+  this->stack_on_(lower, CAPABILITY_RELIABILITY);
+}
+
+void HdlcReliability::setup() {
+  if (this->max_packet_size_ == 0) {
+    this->mark_failed(LOG_STR("max_packet_size too small for the configured layers"));
+    return;
+  }
+  RAMAllocator<uint8_t> allocator;
+  this->tx_slots_ = allocator.allocate(size_t{this->max_queue_length_} * this->max_packet_size_);
+  if (this->tx_slots_ == nullptr) {
+    this->mark_failed(LOG_STR("Not enough memory for the send queue"));
+    return;
+  }
+  this->lower_->add_packet_interface_listener(
+      [this](const PacketBuffer &data, const PacketMetaData &meta_data) { this->receive_(data, meta_data); });
+}
+
+void HdlcReliability::dump_config() {
+  ESP_LOGCONFIG(TAG,
+                "HDLC Reliability on '%s':\n"
+                "  Max Packet Size: %zu\n"
+                "  Timeout: %" PRIu32 "ms\n"
+                "  Ack Delay: %" PRIu32 "ms\n"
+                "  Max Queue Length: %u\n"
+                "  Max Retries: %u",
+                this->name_, this->max_packet_size_, this->timeout_ms_, this->ack_delay_ms_, this->max_queue_length_,
+                this->max_retries_);
+}
+
+void HdlcReliability::loop() {
+  // Disabled first, as sending may deliver a reply to this layer and request the loop again
+  this->disable_loop();
+  if (this->state_ == LinkState::LINK_STATE_RESET_PENDING) {
+    this->state_ = LinkState::LINK_STATE_RESETTING;
+    this->start_timer_();
+    this->send_frame_(UNNUMBERED_SABM, nullptr, 0);
+  }
+  if (this->state_ != LinkState::LINK_STATE_CONNECTED)
+    return;
+  // Queued frames go first, as each carries the acknowledgement
+  if (this->send_pending_() && this->ack_due_ && !this->send_supervisory_(SUPERVISORY_RR))
+    this->retry_send_();
+}
+
+void HdlcReliability::retry_send_() {
+  this->set_timeout(SEND_RETRY_TIMER_ID, SEND_RETRY_MS, [this]() { this->request_loop_(); });
+}
+
+void HdlcReliability::owe_ack_(bool now) {
+  if (now || this->ack_delay_ms_ == 0) {
+    this->ack_due_ = true;
+    this->request_loop_();
+  } else if (!this->ack_pending_) {
+    // Waiting lets one acknowledgement cover several frames, or a reply carry it
+    this->set_timeout(ACK_TIMER_ID, this->ack_delay_ms_, [this]() {
+      this->ack_due_ = true;
+      this->request_loop_();
+    });
+  }
+  this->ack_pending_ = true;
+}
+
+void HdlcReliability::ack_sent_() {
+  this->ack_pending_ = false;
+  this->ack_due_ = false;
+  this->cancel_timeout(ACK_TIMER_ID);
+}
+
+void HdlcReliability::request_loop_() { this->enable_loop(); }
+
+bool HdlcReliability::send_to_interface(const PacketBuffer &data, const PacketMetaData &meta_data) {
+  if (this->tx_slots_ == nullptr)
+    return false;
+  size_t size = data.size();
+  if (size > this->max_packet_size_) {
+    ESP_LOGW(TAG, "%s: Packet of %zu bytes is larger than %zu", this->name_, size, this->max_packet_size_);
+    return false;
+  }
+  uint8_t outstanding = this->outstanding_();
+  if (outstanding >= this->max_queue_length_) {
+    if (this->state_ == LinkState::LINK_STATE_CONNECTED) {
+      ESP_LOGV(TAG, "%s: Send queue full", this->name_);
+    } else if (!this->reset_wait_logged_) {
+      // Once for each reset; the layers above keep trying while the peer is away
+      this->reset_wait_logged_ = true;
+      ESP_LOGD(TAG, "%s: Send queue full, waiting for the peer to answer the link reset", this->name_);
+    }
+    return false;
+  }
+  uint8_t seq = this->vs_;
+  data.copy_to(this->slot_(seq), size);
+  this->tx_lengths_[this->slot_index_(seq)] = size;
+  this->vs_ = (seq + 1) & 7;
+  // While the link is being reset, the frame waits in the queue and is sent when UA arrives
+  if (this->state_ == LinkState::LINK_STATE_CONNECTED) {
+    this->send_pending_();
+    if (outstanding == 0)
+      this->start_timer_();
+  }
+  return true;
+}
+
+bool HdlcReliability::send_frame_(uint8_t control, const uint8_t *payload, size_t length) {
+  uint16_t crc = frame_crc(control, payload, length);
+  uint8_t trailer[2] = {static_cast<uint8_t>(crc), static_cast<uint8_t>(crc >> 8)};
+  PacketBuffer body(payload, length);
+  PacketBuffer frame(&control, 1, length != 0 ? &body : nullptr, trailer, sizeof(trailer));
+  if (this->lower_->send_to_interface(frame, {}))
+    return true;
+  ESP_LOGV(TAG, "%s: Lower layer did not accept frame 0x%02X", this->name_, control);
+  return false;
+}
+
+bool HdlcReliability::send_supervisory_(uint8_t type) {
+  if (!this->send_frame_(SUPERVISORY_FRAME | (type << 2) | (this->vr_ << 5), nullptr, 0))
+    return false;
+  this->ack_sent_();
+  return true;
+}
+
+// Sends the next queued frame not yet sent, and returns true if none are left. One frame per loop pass lets other
+// components run between frames: on a radio, sending blocks, and the peer's frames are received in between. If the
+// lower layer refuses the frame, it is sent again shortly.
+bool HdlcReliability::send_pending_() {
+  if (this->vn_ == this->vs_)
+    return true;
+  uint8_t control = (this->vn_ << 1) | (this->vr_ << 5);
+  if (!this->send_frame_(control, this->slot_(this->vn_), this->tx_lengths_[this->slot_index_(this->vn_)])) {
+    this->retry_send_();
+    return false;
+  }
+  // Every I-frame carries an acknowledgement, so a pending RR is no longer needed
+  this->ack_sent_();
+  this->vn_ = (this->vn_ + 1) & 7;
+  if (this->vn_ == this->vs_)
+    return true;
+  this->request_loop_();
+  return false;
+}
+
+void HdlcReliability::resend_outstanding_() {
+  this->vn_ = this->va_;
+  this->send_pending_();
+}
+
+void HdlcReliability::start_timer_() {
+  uint32_t interval = this->timeout_ms_;
+  if (this->state_ != LinkState::LINK_STATE_CONNECTED && interval < RESET_INTERVAL_MS)
+    interval = RESET_INTERVAL_MS;
+  // Up to a quarter longer at random, so devices whose frames collided on a shared channel do not resend together
+  interval += random_uint32() % (interval / 4 + 1);
+  this->set_timeout(RETRANSMIT_TIMER_ID, interval, [this]() { this->on_timeout_(); });
+}
+
+void HdlcReliability::on_timeout_() {
+  if (this->state_ != LinkState::LINK_STATE_CONNECTED) {
+    // Counted up to max_retries_ only, so the warning is logged once until the link comes up
+    if (this->retries_ < this->max_retries_) {
+      this->retries_++;
+      if (this->retries_ == this->max_retries_) {
+        ESP_LOGW(TAG, "%s: No answer to the link reset after %u tries; check that the peer has a reliability layer",
+                 this->name_, this->max_retries_);
+      }
+    }
+    this->send_frame_(UNNUMBERED_SABM, nullptr, 0);
+    this->start_timer_();
+    return;
+  }
+  if (this->outstanding_() == 0)
+    return;
+  if (this->retries_ == this->max_retries_) {
+    ESP_LOGW(TAG, "%s: No acknowledgement after %u retries, resetting link", this->name_, this->max_retries_);
+    this->start_reset_();
+    return;
+  }
+  this->retries_++;
+  ESP_LOGV(TAG, "%s: Timeout, resending %u frames", this->name_, this->outstanding_());
+  this->resend_outstanding_();
+  this->start_timer_();
+}
+
+// Unacknowledged frames are discarded; frames queued from now on are numbered from 0 and sent once the link is up.
+// Resetting a connected link can lose data both ways, which is reported to the layers above.
+void HdlcReliability::start_reset_() {
+  bool was_connected = this->state_ == LinkState::LINK_STATE_CONNECTED;
+  this->state_ = LinkState::LINK_STATE_RESET_PENDING;
+  this->va_ = this->vs_ = this->vn_ = 0;
+  this->head_ = 0;
+  this->vr_ = 0;
+  this->retries_ = 0;
+  this->rej_sent_ = false;
+  this->ack_sent_();
+  this->cancel_timeout(RETRANSMIT_TIMER_ID);
+  this->request_loop_();
+  if (was_connected)
+    this->session_callback_.call(false);
+}
+
+void HdlcReliability::link_up_() {
+  ESP_LOGD(TAG, "%s: Link connected", this->name_);
+  this->state_ = LinkState::LINK_STATE_CONNECTED;
+  this->reset_wait_logged_ = false;
+  this->peer_frame_seen_ = false;
+  this->retries_ = 0;
+  this->cancel_timeout(RETRANSMIT_TIMER_ID);
+  if (this->outstanding_() != 0) {
+    this->resend_outstanding_();
+    this->start_timer_();
+  }
+}
+
+void HdlcReliability::process_ack_(uint8_t nr) {
+  uint8_t acked = (nr - this->va_) & 7;
+  if (acked > this->outstanding_()) {
+    ESP_LOGV(TAG, "%s: Ignoring invalid N(R) %u", this->name_, nr);
+    return;
+  }
+  if (acked == 0)
+    return;
+  // The peer may have had frames that were refused when last resent
+  if (acked > ((this->vn_ - this->va_) & 7))
+    this->vn_ = nr;
+  this->head_ = (this->head_ + acked) % this->max_queue_length_;
+  this->va_ = nr;
+  this->retries_ = 0;
+  if (this->outstanding_() == 0) {
+    this->cancel_timeout(RETRANSMIT_TIMER_ID);
+  } else {
+    this->start_timer_();
+  }
+}
+
+void HdlcReliability::receive_(const PacketBuffer &data, const PacketMetaData &meta_data) {
+  // Lower layers deliver packets in one piece; the payload is passed up without copying
+  if (!data.is_contiguous()) {
+    ESP_LOGW(TAG, "%s: Packet split across buffers is not supported", this->name_);
+    return;
+  }
+  auto frame = data.as_span();
+  if (frame.size() < FRAME_OVERHEAD) {
+    this->note_received_(false);
+    return;
+  }
+  size_t length = frame.size() - FRAME_OVERHEAD;
+  uint8_t control = frame[0];
+  const uint8_t *payload = frame.data() + 1;
+  uint16_t crc = frame[length + 1] | (frame[length + 2] << 8);
+  if (frame_crc(control, payload, length) != crc) {
+    ESP_LOGV(TAG, "%s: CRC error, frame dropped", this->name_);
+    this->note_received_(false);
+    return;
+  }
+  // The peer's packets may only be larger if it has a larger max_packet_size, which the layers above cannot take
+  if (length > this->max_packet_size_) {
+    if (!this->oversize_warned_) {
+      this->oversize_warned_ = true;
+      ESP_LOGW(TAG,
+               "%s: Frame of %zu bytes is larger than %zu, dropped; check that both devices use the same "
+               "max_packet_size",
+               this->name_, length, this->max_packet_size_);
+    } else {
+      ESP_LOGV(TAG, "%s: Frame of %zu bytes is larger than %zu, dropped", this->name_, length, this->max_packet_size_);
+    }
+    this->note_received_(false);
+    return;
+  }
+  this->note_received_(true);
+
+  uint8_t command = control & ~POLL_FINAL_BIT;
+  if (command == UNNUMBERED_SABM) {
+    // A peer that has not heard our UA repeats its SABM, and cannot have sent a numbered frame meanwhile; answering
+    // again keeps this side connected, and the queued frames, instead of treating the repeat as a restart.
+    if (this->state_ == LinkState::LINK_STATE_CONNECTED && this->peer_frame_seen_) {
+      // The peer has restarted, so frames it has not acknowledged are lost
+      ESP_LOGD(TAG, "%s: Link reset by peer", this->name_);
+      this->start_reset_();
+    }
+    this->vr_ = 0;
+    this->rej_sent_ = false;
+    this->send_frame_(UNNUMBERED_UA, nullptr, 0);
+    // Both sides now start from 0, so the link is up even if our own SABM is unanswered
+    this->link_up_();
+    return;
+  }
+  if (command == UNNUMBERED_UA) {
+    if (this->state_ == LinkState::LINK_STATE_RESETTING)
+      this->link_up_();
+    return;
+  }
+  // Numbered frames are ignored until the link is reset; the peer answers our SABM first
+  if (this->state_ != LinkState::LINK_STATE_CONNECTED)
+    return;
+  this->peer_frame_seen_ = true;
+
+  uint8_t nr = control >> 5;
+  if ((control & 1) == 0) {
+    this->process_ack_(nr);
+    this->receive_info_(control, payload, length, meta_data);
+    return;
+  }
+  if ((control & 3) != SUPERVISORY_FRAME)
+    return;
+  this->process_ack_(nr);
+  if (((control >> 2) & 3) == SUPERVISORY_REJ && this->outstanding_() != 0) {
+    ESP_LOGV(TAG, "%s: Peer rejected from %u", this->name_, nr);
+    this->resend_outstanding_();
+    this->start_timer_();
+  }
+}
+
+void HdlcReliability::receive_info_(uint8_t control, const uint8_t *payload, size_t length,
+                                    const PacketMetaData &meta_data) {
+  uint8_t ns = (control >> 1) & 7;
+  if (ns != this->vr_) {
+    // A gap or a duplicate; ask once for a resend from the expected frame, then just acknowledge at once
+    if (!this->rej_sent_) {
+      this->rej_sent_ = this->send_supervisory_(SUPERVISORY_REJ);
+      if (this->rej_sent_)
+        return;
+    }
+    this->owe_ack_(true);
+    return;
+  }
+  this->vr_ = (this->vr_ + 1) & 7;
+  this->rej_sent_ = false;
+  // The acknowledgement is sent from loop(), so a reply sent from the callback can carry it instead
+  this->owe_ack_(false);
+  this->on_receive_from_interface_(PacketBuffer(payload, length), meta_data);
+}
+
+}  // namespace esphome::packet_interface
+#endif

esphome/components/packet_interface/hdlc_reliability.h

@@ -0,0 +1,99 @@
+#pragma once
+
+#include "esphome/core/defines.h"
+#ifdef USE_PACKET_INTERFACE_HDLC
+#include <array>
+
+#include "packet_interface.h"
+
+namespace esphome::packet_interface {
+
+/**
+ * A reliability layer that delivers packets in order, without loss or duplicates, between two peers.
+ *
+ * It follows HDLC asynchronous balanced mode, with sequence numbers modulo 8 and go-back-N retransmission:
+ * - I-frames carry a payload, the sequence number N(S) and an acknowledgement N(R) of the peer's frames.
+ * - RR frames acknowledge without a payload; REJ frames ask the peer to resend from N(R).
+ * - SABM resets both peers' sequence numbers, and UA answers it.
+ *
+ * Each frame is the HDLC control byte, the payload, then a CRC-16 of both in little endian order.
+ * Unacknowledged frames are resent after the timeout; after max_retries the link is reset.
+ */
+class HdlcReliability : public PacketInterface {
+ public:
+  /// Most frames that can be unacknowledged at once, set by the 3 bit sequence numbers.
+  static constexpr uint8_t MAX_WINDOW = 7;
+  /// Control byte plus CRC-16.
+  static constexpr size_t FRAME_OVERHEAD = 3;
+
+  explicit HdlcReliability(PacketInterface *lower);
+
+  void setup() override;
+  void loop() override;
+  void dump_config() override;
+  bool send_to_interface(const PacketBuffer &data, const PacketMetaData &meta_data) override;
+  /// No room while the layer has no send queue: before setup(), or when setup() failed.
+  size_t get_send_capacity() const override {
+    return this->tx_slots_ == nullptr ? 0 : this->max_queue_length_ - this->outstanding_();
+  }
+
+  void set_ack_timeout(uint32_t timeout_ms) { this->timeout_ms_ = timeout_ms; }
+  void set_ack_delay(uint32_t ack_delay_ms) { this->ack_delay_ms_ = ack_delay_ms; }
+  void set_max_queue_length(uint8_t length) { this->max_queue_length_ = length; }
+  void set_max_retries(uint8_t retries) { this->max_retries_ = retries; }
+
+  bool is_connected() const { return this->state_ == LinkState::LINK_STATE_CONNECTED; }
+
+ protected:
+  enum class LinkState : uint8_t {
+    LINK_STATE_RESET_PENDING,  // SABM to be sent from loop()
+    LINK_STATE_RESETTING,      // SABM sent, waiting for UA
+    LINK_STATE_CONNECTED,
+  };
+
+  void receive_(const PacketBuffer &data, const PacketMetaData &meta_data);
+  void receive_info_(uint8_t control, const uint8_t *payload, size_t length, const PacketMetaData &meta_data);
+  bool send_frame_(uint8_t control, const uint8_t *payload, size_t length);
+  bool send_supervisory_(uint8_t type);
+  bool send_pending_();
+  void resend_outstanding_();
+  void retry_send_();
+  /// An RR is owed: at once, or after the ack delay unless an I-frame carries it first.
+  void owe_ack_(bool now);
+  void ack_sent_();
+  void process_ack_(uint8_t nr);
+  void on_timeout_();
+  void start_reset_();
+  void link_up_();
+  void start_timer_();
+  void request_loop_();
+  uint8_t outstanding_() const { return (this->vs_ - this->va_) & 7; }
+  uint8_t slot_index_(uint8_t seq) const { return (this->head_ + ((seq - this->va_) & 7)) % this->max_queue_length_; }
+  uint8_t *slot_(uint8_t seq) { return this->tx_slots_ + this->slot_index_(seq) * this->max_packet_size_; }
+
+  PacketInterface *lower_;
+  // max_queue_length_ slots of max_packet_size_ bytes, a ring starting at head_ which holds frame va_; allocated in
+  // setup() and never freed
+  uint8_t *tx_slots_{nullptr};
+  std::array<uint16_t, MAX_WINDOW> tx_lengths_{};  // indexed like the slots
+  uint32_t timeout_ms_{200};
+  uint32_t ack_delay_ms_{0};
+  uint8_t max_queue_length_{4};
+  uint8_t max_retries_{5};
+  uint8_t retries_{0};
+  uint8_t head_{0};
+  uint8_t vs_{0};  // sequence number of the next new frame
+  uint8_t vn_{0};  // next frame to send; frames from va_ up to it have been sent
+  uint8_t va_{0};  // oldest unacknowledged frame
+  uint8_t vr_{0};  // next frame expected from the peer
+  LinkState state_{LinkState::LINK_STATE_RESET_PENDING};
+  bool ack_pending_{false};  // an RR is owed
+  bool ack_due_{false};      // and should be sent now
+  bool rej_sent_{false};
+  bool peer_frame_seen_{false};  // a numbered frame has arrived since the link came up
+  bool oversize_warned_{false};
+  bool reset_wait_logged_{false};  // the queue filled while waiting for this reset to be answered
+};
+
+}  // namespace esphome::packet_interface
+#endif

esphome/components/packet_interface/layers/__init__.py

@@ -0,0 +1,155 @@
+"""The kinds of layer that can be stacked on a packet interface, and the registry of their implementations.
+
+Each module in this package implements one layer type and registers it with its kind, for example:
+
+    @RELIABILITY.register("hdlc", schema=HDLC_SCHEMA)
+    async def build_hdlc(config: ConfigType, lower: MockObj) -> MockObj:
+        ...
+
+The packet_interface component imports every module here before it builds its schemas, so a new layer type needs
+only a new module. The kinds, and the order in which they are stacked, are fixed here.
+"""
+
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass
+from typing import Any
+
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.const import CONF_ENCRYPTION, CONF_KEY, CONF_TYPE
+from esphome.cpp_generator import MockObj
+from esphome.types import ConfigType
+
+packet_interface_ns = cg.esphome_ns.namespace("packet_interface")
+PacketInterface = packet_interface_ns.class_("PacketInterface", cg.Component)
+
+CONF_RELIABILITY = "reliability"
+CONF_SESSION = "session"
+
+# Builds a layer above lower and returns it
+LayerBuilder = Callable[[ConfigType, MockObj], Awaitable[Any]]
+
+
+def _never(config: ConfigType) -> bool:
+    return False
+
+
+def _nothing(config: ConfigType) -> list[str]:
+    return []
+
+
+@dataclass(frozen=True)
+class LayerType:
+    """One implementation of a kind of layer."""
+
+    schema: Any
+    build: LayerBuilder
+    # True if this configuration works only when the peer sends packets back, which a session layer makes sure of
+    needs_session: Callable[[ConfigType], bool]
+    # The components this configuration needs loaded
+    auto_load: Callable[[ConfigType], list[str]]
+
+
+class LayerKind:
+    """A kind of layer, such as encryption, with the types that implement it."""
+
+    def __init__(
+        self, key: str, default_type: str, shorthand: str | None = None
+    ) -> None:
+        self.key = key
+        self.default_type = default_type
+        # The option that a plain string value sets, as in `encryption: my key`
+        self.shorthand = shorthand
+        self.types: dict[str, LayerType] = {}
+        # Set once the schema is built, as types registered after that would not be in it
+        self.frozen = False
+
+    def register(
+        self,
+        name: str,
+        *,
+        schema: Any,
+        needs_session: Callable[[ConfigType], bool] = _never,
+        auto_load: Callable[[ConfigType], list[str]] = _nothing,
+    ) -> Callable[[LayerBuilder], LayerBuilder]:
+        """Register a type with its schema."""
+
+        def decorator(build: LayerBuilder) -> LayerBuilder:
+            if self.frozen:
+                raise ValueError(
+                    f"{self.key} layer type '{name}' is registered after the schemas were built; register it "
+                    "before the packet_interface component is imported"
+                )
+            if name in self.types:
+                raise ValueError(
+                    f"{self.key} layer type '{name}' is already registered"
+                )
+            self.types[name] = LayerType(schema, build, needs_session, auto_load)
+            return build
+
+        return decorator
+
+    def _expand(self, value: Any) -> Any:
+        """Allow a layer to be enabled with no options, as in `reliability:`, or with its shorthand."""
+        if value is None:
+            return {}
+        if self.shorthand is not None and isinstance(value, str):
+            return {self.shorthand: value}
+        return value
+
+    def schema(self) -> Callable[[Any], ConfigType]:
+        """The schema for this kind; no type can be registered after this."""
+        self.frozen = True
+        return cv.All(
+            self._expand,
+            cv.typed_schema(
+                {name: layer.schema for name, layer in self.types.items()},
+                default_type=self.default_type,
+            ),
+        )
+
+    def get_type(self, config: ConfigType) -> LayerType:
+        return self.types[config[CONF_TYPE]]
+
+
+ENCRYPTION = LayerKind(CONF_ENCRYPTION, "xxtea", shorthand=CONF_KEY)
+RELIABILITY = LayerKind(CONF_RELIABILITY, "hdlc")
+SESSION = LayerKind(CONF_SESSION, "basic")
+# The kinds stacked above the link, from the bottom up
+STACK_ORDER = (ENCRYPTION, RELIABILITY, SESSION)
+
+
+def stack_schema() -> dict:
+    """Schema entries for the layers of one stack; call only once all layer types are registered."""
+    return {cv.Optional(kind.key): kind.schema() for kind in STACK_ORDER}
+
+
+def check_session(config: ConfigType) -> None:
+    """Layers that need packets back from the peer need a session layer, whose keep-alives send them."""
+    if CONF_SESSION in config:
+        return
+    for kind in STACK_ORDER:
+        if (layer := config.get(kind.key)) is not None and kind.get_type(
+            layer
+        ).needs_session(layer):
+            raise cv.Invalid(
+                f"This '{kind.key}' needs packets from the peer, so it needs a '{CONF_SESSION}' layer, whose "
+                "keep-alives make sure they are sent",
+                path=[kind.key],
+            )
+
+
+def stack_auto_load(config: ConfigType) -> set[str]:
+    """The components that the layers configured in config need loaded."""
+    loads: set[str] = set()
+    for kind in STACK_ORDER:
+        if (layer := config.get(kind.key)) is not None:
+            loads.update(kind.get_type(layer).auto_load(layer))
+    return loads
+
+
+async def build_stack(config: ConfigType, lower: MockObj) -> None:
+    """Create the layers configured in config, from the bottom up, above lower."""
+    for kind in STACK_ORDER:
+        if (layer := config.get(kind.key)) is not None:
+            lower = await kind.get_type(layer).build(layer, lower)

esphome/components/packet_interface/layers/basic_session.py

@@ -0,0 +1,70 @@
+"""Session layer that sends keep-alives and reports whether the peer can be reached."""
+
+import esphome.codegen as cg
+from esphome.components import binary_sensor
+import esphome.config_validation as cv
+from esphome.const import (
+    CONF_BINARY_SENSOR,
+    CONF_ID,
+    CONF_TIMEOUT,
+    DEVICE_CLASS_CONNECTIVITY,
+    ENTITY_CATEGORY_DIAGNOSTIC,
+)
+from esphome.cpp_generator import MockObj
+from esphome.types import ConfigType
+
+from . import SESSION, PacketInterface, packet_interface_ns
+
+CONF_KEEP_ALIVE = "keep_alive"
+
+BasicSession = packet_interface_ns.class_("BasicSession", PacketInterface)
+
+
+def _session_timeout(config: ConfigType) -> ConfigType:
+    """Default the timeout to three keep-alive intervals, and require it to be longer than one."""
+    keep_alive = config[CONF_KEEP_ALIVE].total_milliseconds
+    if (timeout := config.get(CONF_TIMEOUT)) is None:
+        config[CONF_TIMEOUT] = cv.positive_time_period_milliseconds(
+            f"{keep_alive * 3}ms"
+        )
+    elif timeout.total_milliseconds <= keep_alive:
+        raise cv.Invalid("Must be longer than keep_alive", path=[CONF_TIMEOUT])
+    return config
+
+
+BASIC_SESSION_SCHEMA = cv.All(
+    cv.Schema(
+        {
+            cv.GenerateID(): cv.declare_id(BasicSession),
+            cv.Optional(CONF_KEEP_ALIVE, default="10s"): cv.All(
+                cv.positive_time_period_milliseconds,
+                cv.Range(min=cv.TimePeriod(seconds=1), max=cv.TimePeriod(hours=1)),
+            ),
+            cv.Optional(CONF_TIMEOUT): cv.All(
+                cv.positive_time_period_milliseconds,
+                cv.Range(max=cv.TimePeriod(hours=3)),
+            ),
+            cv.Optional(CONF_BINARY_SENSOR): binary_sensor.binary_sensor_schema(
+                device_class=DEVICE_CLASS_CONNECTIVITY,
+                entity_category=ENTITY_CATEGORY_DIAGNOSTIC,
+            ),
+        }
+    ).extend(cv.COMPONENT_SCHEMA),
+    _session_timeout,
+)
+
+
+@SESSION.register(
+    "basic",
+    schema=BASIC_SESSION_SCHEMA,
+    auto_load=lambda config: ["binary_sensor"] if CONF_BINARY_SENSOR in config else [],
+)
+async def build_basic_session(config: ConfigType, lower: MockObj) -> MockObj:
+    cg.add_define("USE_PACKET_INTERFACE_BASIC_SESSION")
+    var = cg.new_Pvariable(config[CONF_ID], lower)
+    cg.add(var.set_keep_alive(config[CONF_KEEP_ALIVE]))
+    cg.add(var.set_session_timeout(config[CONF_TIMEOUT]))
+    await cg.register_component(var, config)
+    binary_sensors = binary_sensor.sub_binary_sensors(config)
+    await binary_sensors(CONF_BINARY_SENSOR, var.set_connected_sensor)
+    return var

esphome/components/packet_interface/layers/hdlc_reliability.py

@@ -0,0 +1,59 @@
+"""Reliability layer using HDLC style acknowledgement and go-back-N retransmission."""
+
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.const import CONF_ID, CONF_TIMEOUT
+from esphome.cpp_generator import MockObj
+from esphome.types import ConfigType
+
+from . import RELIABILITY, PacketInterface, packet_interface_ns
+
+CONF_ACK_DELAY = "ack_delay"
+CONF_MAX_QUEUE_LENGTH = "max_queue_length"
+CONF_MAX_RETRIES = "max_retries"
+
+HdlcReliability = packet_interface_ns.class_("HdlcReliability", PacketInterface)
+
+
+def _ack_delay_within_timeout(config: ConfigType) -> ConfigType:
+    """The peer, configured the same, must hear the acknowledgement before its timeout."""
+    if (
+        config[CONF_ACK_DELAY].total_milliseconds
+        >= config[CONF_TIMEOUT].total_milliseconds
+    ):
+        raise cv.Invalid(
+            f"Must be shorter than '{CONF_TIMEOUT}'", path=[CONF_ACK_DELAY]
+        )
+    return config
+
+
+HDLC_SCHEMA = cv.All(
+    cv.Schema(
+        {
+            cv.GenerateID(): cv.declare_id(HdlcReliability),
+            # Retransmitting more often than the main loop runs would exhaust the retries at once
+            cv.Optional(CONF_TIMEOUT, default="200ms"): cv.All(
+                cv.positive_time_period_milliseconds,
+                cv.Range(min=cv.TimePeriod(milliseconds=50)),
+            ),
+            cv.Optional(
+                CONF_ACK_DELAY, default="0ms"
+            ): cv.positive_time_period_milliseconds,
+            cv.Optional(CONF_MAX_QUEUE_LENGTH, default=4): cv.int_range(1, 7),
+            cv.Optional(CONF_MAX_RETRIES, default=5): cv.int_range(1, 255),
+        }
+    ).extend(cv.COMPONENT_SCHEMA),
+    _ack_delay_within_timeout,
+)
+
+
+@RELIABILITY.register("hdlc", schema=HDLC_SCHEMA)
+async def build_hdlc(config: ConfigType, lower: MockObj) -> MockObj:
+    cg.add_define("USE_PACKET_INTERFACE_HDLC")
+    var = cg.new_Pvariable(config[CONF_ID], lower)
+    cg.add(var.set_ack_timeout(config[CONF_TIMEOUT]))
+    cg.add(var.set_ack_delay(config[CONF_ACK_DELAY]))
+    cg.add(var.set_max_queue_length(config[CONF_MAX_QUEUE_LENGTH]))
+    cg.add(var.set_max_retries(config[CONF_MAX_RETRIES]))
+    await cg.register_component(var, config)
+    return var

esphome/components/packet_interface/layers/xxtea_encryption.py

@@ -0,0 +1,50 @@
+"""Encryption layer using XXTEA, with an optional rolling code."""
+
+import hashlib
+
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.const import CONF_ID, CONF_KEY
+from esphome.cpp_generator import MockObj
+from esphome.types import ConfigType
+
+from . import ENCRYPTION, PacketInterface, packet_interface_ns
+
+CONF_ROLLING_CODE_ENABLE = "rolling_code_enable"
+
+# A captured packet lets a guessed key be checked offline, so the key text is stretched to make each guess costly.
+# Both ends derive the same key from the same text, so the salt is fixed; it keeps tables built for other uses of
+# PBKDF2 from applying here.
+KEY_SALT = b"esphome packet_interface xxtea"
+KEY_ITERATIONS = 600_000
+
+XxteaEncryption = packet_interface_ns.class_("XxteaEncryption", PacketInterface)
+
+XXTEA_SCHEMA = cv.Schema(
+    {
+        cv.GenerateID(): cv.declare_id(XxteaEncryption),
+        cv.Required(CONF_KEY): cv.All(cv.sensitive(cv.string), cv.Length(min=1)),
+        cv.Optional(CONF_ROLLING_CODE_ENABLE, default=False): cv.boolean,
+    }
+).extend(cv.COMPONENT_SCHEMA)
+
+
+def derive_key(text: str) -> list[int]:
+    """The 256-bit key for the key text, as 32-bit words in little endian order."""
+    digest = hashlib.pbkdf2_hmac("sha256", text.encode(), KEY_SALT, KEY_ITERATIONS)
+    return [int.from_bytes(digest[i : i + 4], "little") for i in range(0, 32, 4)]
+
+
+@ENCRYPTION.register(
+    "xxtea",
+    schema=XXTEA_SCHEMA,
+    # The rolling code accepts a packet only once the peer has sent back this device's boot number
+    needs_session=lambda config: config[CONF_ROLLING_CODE_ENABLE],
+    auto_load=lambda config: ["xxtea"],
+)
+async def build_xxtea(config: ConfigType, lower: MockObj) -> MockObj:
+    cg.add_define("USE_PACKET_INTERFACE_XXTEA")
+    var = cg.new_Pvariable(config[CONF_ID], lower, config[CONF_ROLLING_CODE_ENABLE])
+    cg.add(var.set_key(derive_key(config[CONF_KEY])))
+    await cg.register_component(var, config)
+    return var

esphome/components/packet_interface/xxtea_encryption.cpp

@@ -0,0 +1,182 @@
+#include "xxtea_encryption.h"
+#ifdef USE_PACKET_INTERFACE_XXTEA
+#include <algorithm>
+#include <cstring>
+#include "esphome/components/xxtea/xxtea.h"
+#include "esphome/core/helpers.h"
+#include "esphome/core/log.h"
+
+namespace esphome::packet_interface {
+
+ESPHOME_LOG_TAG(TAG, "packet_interface.xxtea");
+
+static constexpr uint32_t PREF_HASH = fnv1a_hash("packet_interface.xxtea");
+// Different for each mode, so devices set up differently drop each other's packets
+static constexpr uint8_t CHECK_BYTES[3] = {0x58, 0x54, 0x45};
+static constexpr uint8_t ROLLING_CHECK_BYTES[3] = {0x58, 0x54, 0x52};
+static constexpr size_t CHECK_SIZE = 4;
+// Boot number, counter, and the echo of the receiver's boot number
+static constexpr size_t ROLLING_SIZE = 12;
+// XXTEA needs at least two words
+static constexpr size_t MIN_LENGTH = 8;
+static constexpr size_t MAX_PADDING = 4;
+
+static void put_uint32(uint8_t *buf, uint32_t value) {
+  buf[0] = value;
+  buf[1] = value >> 8;
+  buf[2] = value >> 16;
+  buf[3] = value >> 24;
+}
+
+static uint32_t get_uint32(const uint8_t *buf) { return encode_uint32(buf[3], buf[2], buf[1], buf[0]); }
+
+static constexpr size_t trailer_size(bool rolling_code) {
+  return rolling_code ? ROLLING_SIZE + CHECK_SIZE : CHECK_SIZE;
+}
+
+XxteaEncryption::XxteaEncryption(PacketInterface *lower, bool rolling_code)
+    // XXTEA works on whole 32-bit words
+    : PacketInterface(remaining_packet_size(lower->get_max_packet_size() & ~size_t{3}, trailer_size(rolling_code))),
+      lower_(lower),
+      trailer_size_(trailer_size(rolling_code)),
+      rolling_code_(rolling_code) {
+  this->stack_on_(lower, CAPABILITY_ENCRYPTION);
+}
+
+void XxteaEncryption::set_key(std::initializer_list<uint32_t> key) {
+  std::copy_n(key.begin(), std::min(key.size(), this->key_.size()), this->key_.begin());
+}
+
+void XxteaEncryption::setup() {
+  if (this->max_packet_size_ == 0) {
+    this->mark_failed(LOG_STR("max_packet_size too small for the configured layers"));
+    return;
+  }
+  size_t words = this->lower_->get_max_packet_size() / 4;
+  RAMAllocator<uint32_t> allocator;
+  this->tx_buffer_ = allocator.allocate(words * 2);
+  if (this->tx_buffer_ == nullptr) {
+    this->mark_failed(LOG_STR("Not enough memory for the packet buffers"));
+    return;
+  }
+  this->rx_buffer_ = this->tx_buffer_ + words;
+  if (this->rolling_code_) {
+    this->pref_ = global_preferences->make_preference<uint32_t>(fnv1a_hash_extend(PREF_HASH, this->name_), true);
+    this->pref_.load(&this->boot_);
+    this->next_boot_();
+  }
+  this->lower_->add_packet_interface_listener(
+      [this](const PacketBuffer &data, const PacketMetaData &meta_data) { this->receive_(data, meta_data); });
+}
+
+void XxteaEncryption::dump_config() {
+  ESP_LOGCONFIG(TAG,
+                "XXTEA Encryption on '%s':\n"
+                "  Max Packet Size: %zu\n"
+                "  Rolling Code: %s",
+                this->name_, this->max_packet_size_, YESNO(this->rolling_code_));
+  if (this->rolling_code_) {
+    ESP_LOGCONFIG(TAG, "  Boot Number: %" PRIu32, this->boot_);
+  }
+}
+
+// A new boot number must be saved before it is used, or a crash could make the peer see it again
+void XxteaEncryption::next_boot_() {
+  this->boot_++;
+  this->counter_ = 0;
+  if (!this->pref_.save(&this->boot_) || !global_preferences->sync()) {
+    ESP_LOGW(TAG, "%s: Could not save the boot number, so a restart could reuse it", this->name_);
+  }
+}
+
+bool XxteaEncryption::send_to_interface(const PacketBuffer &data, const PacketMetaData &meta_data) {
+  if (this->tx_buffer_ == nullptr)
+    return false;
+  size_t size = data.size();
+  if (size > this->max_packet_size_) {
+    ESP_LOGW(TAG, "%s: Packet of %zu bytes is larger than %zu", this->name_, size, this->max_packet_size_);
+    return false;
+  }
+  size_t padding = (4 - (size + this->trailer_size_) % 4) % 4;
+  if (size + padding + this->trailer_size_ < MIN_LENGTH)
+    padding += 4;
+  size_t length = size + padding + this->trailer_size_;
+  auto *buf = reinterpret_cast<uint8_t *>(this->tx_buffer_);
+  data.copy_to(buf, size);
+  memset(buf + size, 0, padding);
+  uint8_t *check = buf + length - CHECK_SIZE;
+  if (this->rolling_code_) {
+    put_uint32(check - ROLLING_SIZE, this->boot_);
+    put_uint32(check - ROLLING_SIZE + 4, this->counter_);
+    put_uint32(check - ROLLING_SIZE + 8, this->echo_boot_);
+    memcpy(check, ROLLING_CHECK_BYTES, sizeof(ROLLING_CHECK_BYTES));
+    if (++this->counter_ == 0)
+      this->next_boot_();
+  } else {
+    memcpy(check, CHECK_BYTES, sizeof(CHECK_BYTES));
+  }
+  check[CHECK_SIZE - 1] = padding;
+  xxtea::encrypt(this->tx_buffer_, length / 4, this->key_.data());
+  return this->lower_->send_to_interface(PacketBuffer(buf, length), meta_data);
+}
+
+bool XxteaEncryption::accept_counter_(uint32_t boot, uint32_t counter) {
+  if (this->peer_seen_ && (boot < this->peer_boot_ || (boot == this->peer_boot_ && counter <= this->peer_counter_)))
+    return false;
+  this->peer_seen_ = true;
+  this->peer_boot_ = boot;
+  this->peer_counter_ = counter;
+  return true;
+}
+
+void XxteaEncryption::receive_(const PacketBuffer &data, const PacketMetaData &meta_data) {
+  size_t length = data.size();
+  if (length % 4 != 0 || length < std::max(MIN_LENGTH, this->trailer_size_) ||
+      length > this->lower_->get_max_packet_size()) {
+    ESP_LOGV(TAG, "%s: Packet of %zu bytes dropped", this->name_, length);
+    this->note_received_(false);
+    return;
+  }
+  auto *buf = reinterpret_cast<uint8_t *>(this->rx_buffer_);
+  data.copy_to(buf, length);
+  xxtea::decrypt(this->rx_buffer_, length / 4, this->key_.data());
+  const uint8_t *check = buf + length - CHECK_SIZE;
+  const uint8_t *expected = this->rolling_code_ ? ROLLING_CHECK_BYTES : CHECK_BYTES;
+  size_t padding = check[CHECK_SIZE - 1];
+  if (memcmp(check, expected, sizeof(CHECK_BYTES)) != 0 || padding > MAX_PADDING ||
+      padding > length - this->trailer_size_) {
+    ESP_LOGV(TAG, "%s: Packet failed authentication", this->name_);
+    this->note_received_(false);
+    return;
+  }
+  this->note_received_(true);
+  if (this->rolling_code_) {
+    uint32_t boot = get_uint32(check - ROLLING_SIZE);
+    uint32_t counter = get_uint32(check - ROLLING_SIZE + 4);
+    uint32_t echo = get_uint32(check - ROLLING_SIZE + 8);
+    // Learn the peer's boot number even from a packet that is then dropped, so both sides can catch up
+    if (boot > this->echo_boot_)
+      this->echo_boot_ = boot;
+    // A packet recorded before this boot cannot carry this boot's number
+    if (echo != this->boot_) {
+      ESP_LOGV(TAG, "%s: Packet for boot %" PRIu32 " dropped", this->name_, echo);
+      return;
+    }
+    if (!this->accept_counter_(boot, counter)) {
+      // A boot number lower than one already heard means the peer lost its saved boot number, or a replay
+      if (boot < this->peer_boot_ && !this->old_boot_warned_) {
+        this->old_boot_warned_ = true;
+        ESP_LOGW(TAG,
+                 "%s: Packets from an older boot %" PRIu32 " of the peer are dropped; if its flash was erased, "
+                 "restart this device",
+                 this->name_, boot);
+      }
+      ESP_LOGV(TAG, "%s: Old packet %" PRIu32 ":%" PRIu32 " dropped", this->name_, boot, counter);
+      return;
+    }
+  }
+  this->on_receive_from_interface_(PacketBuffer(buf, length - this->trailer_size_ - padding), meta_data);
+}
+
+}  // namespace esphome::packet_interface
+#endif

esphome/components/packet_interface/xxtea_encryption.h

@@ -0,0 +1,61 @@
+#pragma once
+
+#include "esphome/core/defines.h"
+#ifdef USE_PACKET_INTERFACE_XXTEA
+#include <array>
+#include <initializer_list>
+
+#include "esphome/core/preferences.h"
+#include "packet_interface.h"
+
+namespace esphome::packet_interface {
+
+/**
+ * An encryption layer that encrypts and authenticates each packet with XXTEA and a 256-bit shared key.
+ *
+ * The packet is padded with zeros and followed by a trailer, then the whole is encrypted as one block. With the
+ * rolling code enabled, the trailer starts with:
+ * - a 32-bit boot number, saved in flash and increased at every boot;
+ * - a 32-bit packet counter, starting at 0 at every boot;
+ * - the highest boot number heard from the peer, echoed back to it.
+ * The trailer always ends with 3 fixed check bytes and the number of padding bytes. XXTEA mixes the whole block, so
+ * a packet that was changed or encrypted with another key fails the check.
+ * With the rolling code, a packet is dropped as a replay unless it echoes this device's current boot number, and its
+ * boot number and counter are higher than those of the last packet accepted from the peer. A boot number is never
+ * reused, so the echo acts as a fresh challenge after every restart.
+ */
+class XxteaEncryption : public PacketInterface {
+ public:
+  XxteaEncryption(PacketInterface *lower, bool rolling_code);
+
+  void setup() override;
+  void dump_config() override;
+  bool send_to_interface(const PacketBuffer &data, const PacketMetaData &meta_data) override;
+
+  void set_key(std::initializer_list<uint32_t> key);
+
+ protected:
+  void receive_(const PacketBuffer &data, const PacketMetaData &meta_data);
+  bool accept_counter_(uint32_t boot, uint32_t counter);
+  void next_boot_();
+
+  PacketInterface *lower_;
+  const size_t trailer_size_;
+  std::array<uint32_t, 8> key_{};
+  // Separate buffers, as a layer above may send a reply while a received packet is being passed up; both are in one
+  // block allocated in setup(), and tx_buffer_ stays null if that fails
+  uint32_t *tx_buffer_{nullptr};
+  uint32_t *rx_buffer_{nullptr};
+  ESPPreferenceObject pref_;
+  uint32_t boot_{0};
+  uint32_t counter_{0};
+  uint32_t peer_boot_{0};
+  uint32_t peer_counter_{0};
+  uint32_t echo_boot_{0};  // highest boot number heard from the peer
+  bool peer_seen_{false};
+  bool old_boot_warned_{false};
+  const bool rolling_code_;
+};
+
+}  // namespace esphome::packet_interface
+#endif

esphome/components/pca6416a/__init__.py

@@ -29,10 +29,9 @@ CONFIG_SCHEMA = (
     cv.Schema(
         {
             cv.Required(CONF_ID): cv.declare_id(PCA6416AComponent),
-            cv.Optional(CONF_INTERRUPT_PIN): gpio_expander.validate_interrupt_pin,
         }
     )
-    .extend(cv.COMPONENT_SCHEMA)
+    .extend(gpio_expander.INTERRUPT_EXPANDER_SCHEMA)
     .extend(i2c.i2c_device_schema(0x21))
 )