[core] Report unknown for switch, climate and water heater with no state (#19458)

Co-authored-by: J. Nick Koston <nick@home-assistant.io>

esphome/components/api/api.proto

@@ -803,6 +803,9 @@ message SwitchStateResponse {
   fixed32 key = 1 [(force) = true];
   bool state = 2;
   uint32 device_id = 3 [(field_ifdef) = "USE_DEVICES"];
+  // If the switch does not have a valid state yet.
+  // Equivalent to `!obj->has_state()` - inverse logic to make state packets smaller
+  bool missing_state = 4;
 }
 message SwitchCommandRequest {
   option (id) = 33;
@@ -1244,6 +1247,9 @@ message ClimateStateResponse {
   float current_humidity = 14;
   float target_humidity = 15;
   uint32 device_id = 16 [(field_ifdef) = "USE_DEVICES"];
+  // If the climate device does not have a valid state yet.
+  // Equivalent to `!obj->has_state()` - inverse logic to make state packets smaller
+  bool missing_state = 17;
 }
 message ClimateCommandRequest {
   option (id) = 48;
@@ -1330,6 +1336,9 @@ message WaterHeaterStateResponse {
   uint32 state = 6;
   float target_temperature_low = 7;
   float target_temperature_high = 8;
+  // If the water heater does not have a valid state yet.
+  // Equivalent to `!obj->has_state()` - inverse logic to make state packets smaller
+  bool missing_state = 9;
 }
 
 // Bitmask for WaterHeaterCommandRequest.has_fields

esphome/components/api/api_connection.cpp

@@ -709,6 +709,7 @@ uint16_t APIConnection::try_send_switch_state(EntityBase *entity, APIConnection
   auto *a_switch = static_cast<switch_::Switch *>(entity);
   SwitchStateResponse resp;
   resp.state = a_switch->state;
+  resp.missing_state = !a_switch->has_state();
   return fill_and_encode_entity_state(a_switch, resp, conn, remaining_size);
 }
 
@@ -754,6 +755,7 @@ uint16_t APIConnection::try_send_climate_state(EntityBase *entity, APIConnection
   auto traits = climate->get_traits();
   resp.mode = static_cast<enums::ClimateMode>(climate->mode);
   resp.action = static_cast<enums::ClimateAction>(climate->action);
+  resp.missing_state = !climate->has_state();
   if (traits.has_feature_flags(climate::CLIMATE_SUPPORTS_CURRENT_TEMPERATURE))
     resp.current_temperature = climate->current_temperature;
   if (traits.has_feature_flags(climate::CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE |
@@ -1447,6 +1449,7 @@ uint16_t APIConnection::try_send_water_heater_state(EntityBase *entity, APIConne
   auto *wh = static_cast<water_heater::WaterHeater *>(entity);
   WaterHeaterStateResponse resp;
   resp.mode = static_cast<enums::WaterHeaterMode>(wh->get_mode());
+  resp.missing_state = !wh->has_state();
   resp.current_temperature = wh->get_current_temperature();
   resp.target_temperature = wh->get_target_temperature();
   resp.target_temperature_low = wh->get_target_temperature_low();

esphome/components/api/api_pb2.cpp

@@ -938,6 +938,7 @@ uint8_t *SwitchStateResponse::encode_msg(const void *self, ProtoWriteBuffer &buf
 #ifdef USE_DEVICES
   pos = ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 3, msg.device_id);
 #endif
+  pos = ProtoEncode::encode_bool(pos PROTO_ENCODE_DEBUG_ARG, 4, msg.missing_state);
   return pos;
 }
 uint32_t SwitchStateResponse::calc_size_msg(const void *self) {
@@ -948,6 +949,7 @@ uint32_t SwitchStateResponse::calc_size_msg(const void *self) {
 #ifdef USE_DEVICES
   size += ProtoSize::calc_uint32(1, msg.device_id);
 #endif
+  size += ProtoSize::calc_bool(1, msg.missing_state);
   return size;
 }
 void SwitchCommandRequest::decode_field(void *self, uint32_t tag, const uint8_t *data, proto_varint_value_t scalar) {
@@ -1630,6 +1632,7 @@ uint8_t *ClimateStateResponse::encode_msg(const void *self, ProtoWriteBuffer &bu
 #ifdef USE_DEVICES
   pos = ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 16, msg.device_id);
 #endif
+  pos = ProtoEncode::encode_bool(pos PROTO_ENCODE_DEBUG_ARG, 17, msg.missing_state);
   return pos;
 }
 uint32_t ClimateStateResponse::calc_size_msg(const void *self) {
@@ -1652,6 +1655,7 @@ uint32_t ClimateStateResponse::calc_size_msg(const void *self) {
 #ifdef USE_DEVICES
   size += ProtoSize::calc_uint32(2, msg.device_id);
 #endif
+  size += ProtoSize::calc_bool(2, msg.missing_state);
   return size;
 }
 void ClimateCommandRequest::decode_field(void *self, uint32_t tag, const uint8_t *data, proto_varint_value_t scalar) {
@@ -1806,6 +1810,7 @@ uint8_t *WaterHeaterStateResponse::encode_msg(const void *self, ProtoWriteBuffer
   if (uint32_t raw = float_to_raw(msg.target_temperature_high); raw != 0) [[likely]] {
     pos = ProtoEncode::write_tag_and_fixed32(pos PROTO_ENCODE_DEBUG_ARG, 69, raw);
   }
+  pos = ProtoEncode::encode_bool(pos PROTO_ENCODE_DEBUG_ARG, 9, msg.missing_state);
   return pos;
 }
 uint32_t WaterHeaterStateResponse::calc_size_msg(const void *self) {
@@ -1821,6 +1826,7 @@ uint32_t WaterHeaterStateResponse::calc_size_msg(const void *self) {
   size += ProtoSize::calc_uint32(1, msg.state);
   size += ProtoSize::calc_float(1, msg.target_temperature_low);
   size += ProtoSize::calc_float(1, msg.target_temperature_high);
+  size += ProtoSize::calc_bool(1, msg.missing_state);
   return size;
 }
 void WaterHeaterCommandRequest::decode_field(void *self, uint32_t tag, const uint8_t *data,

esphome/components/api/api_pb2.h

@@ -1109,11 +1109,12 @@ class ListEntitiesSwitchResponse final : public InfoResponseProtoMessage {
 class SwitchStateResponse final : public StateResponseProtoMessage {
  public:
   static constexpr uint16_t MESSAGE_TYPE = 26;
-  static constexpr uint8_t ESTIMATED_SIZE = 11;
+  static constexpr uint8_t ESTIMATED_SIZE = 13;
 #ifdef HAS_PROTO_MESSAGE_DUMP
   const LogString *message_name() const override { return LOG_STR("switch_state_response"); }
 #endif
   bool state{false};
+  bool missing_state{false};
   static uint8_t *encode_msg(const void *self, ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM);
   uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const {
     return encode_msg(this, buffer PROTO_ENCODE_DEBUG_ARG);
@@ -1690,7 +1691,7 @@ class ListEntitiesClimateResponse final : public InfoResponseProtoMessage {
 class ClimateStateResponse final : public StateResponseProtoMessage {
  public:
   static constexpr uint16_t MESSAGE_TYPE = 47;
-  static constexpr uint8_t ESTIMATED_SIZE = 68;
+  static constexpr uint8_t ESTIMATED_SIZE = 71;
 #ifdef HAS_PROTO_MESSAGE_DUMP
   const LogString *message_name() const override { return LOG_STR("climate_state_response"); }
 #endif
@@ -1707,6 +1708,7 @@ class ClimateStateResponse final : public StateResponseProtoMessage {
   StringRef custom_preset{nullptr, 0};  // null until set, encode only
   float current_humidity{0.0f};
   float target_humidity{0.0f};
+  bool missing_state{false};
   static uint8_t *encode_msg(const void *self, ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM);
   uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const {
     return encode_msg(this, buffer PROTO_ENCODE_DEBUG_ARG);
@@ -1786,7 +1788,7 @@ class ListEntitiesWaterHeaterResponse final : public InfoResponseProtoMessage {
 class WaterHeaterStateResponse final : public StateResponseProtoMessage {
  public:
   static constexpr uint16_t MESSAGE_TYPE = 133;
-  static constexpr uint8_t ESTIMATED_SIZE = 35;
+  static constexpr uint8_t ESTIMATED_SIZE = 37;
 #ifdef HAS_PROTO_MESSAGE_DUMP
   const LogString *message_name() const override { return LOG_STR("water_heater_state_response"); }
 #endif
@@ -1796,6 +1798,7 @@ class WaterHeaterStateResponse final : public StateResponseProtoMessage {
   uint32_t state{0};
   float target_temperature_low{0.0f};
   float target_temperature_high{0.0f};
+  bool missing_state{false};
   static uint8_t *encode_msg(const void *self, ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM);
   uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const {
     return encode_msg(this, buffer PROTO_ENCODE_DEBUG_ARG);

esphome/components/api/api_pb2_dump.cpp

@@ -1342,6 +1342,7 @@ const char *SwitchStateResponse::dump_to(DumpBuffer &out) const {
 #ifdef USE_DEVICES
   dump_field(out, ESPHOME_PSTR("device_id"), this->device_id);
 #endif
+  dump_field(out, ESPHOME_PSTR("missing_state"), this->missing_state);
   return out.c_str();
 }
 const char *SwitchCommandRequest::dump_to(DumpBuffer &out) const {
@@ -1684,6 +1685,7 @@ const char *ClimateStateResponse::dump_to(DumpBuffer &out) const {
 #ifdef USE_DEVICES
   dump_field(out, ESPHOME_PSTR("device_id"), this->device_id);
 #endif
+  dump_field(out, ESPHOME_PSTR("missing_state"), this->missing_state);
   return out.c_str();
 }
 const char *ClimateCommandRequest::dump_to(DumpBuffer &out) const {
@@ -1751,6 +1753,7 @@ const char *WaterHeaterStateResponse::dump_to(DumpBuffer &out) const {
   dump_field(out, ESPHOME_PSTR("state"), this->state);
   dump_field(out, ESPHOME_PSTR("target_temperature_low"), this->target_temperature_low);
   dump_field(out, ESPHOME_PSTR("target_temperature_high"), this->target_temperature_high);
+  dump_field(out, ESPHOME_PSTR("missing_state"), this->missing_state);
   return out.c_str();
 }
 const char *WaterHeaterCommandRequest::dump_to(DumpBuffer &out) const {

esphome/components/climate/climate.cpp

@@ -435,6 +435,7 @@ void Climate::save_state_(const ClimateTraits &traits) {
 }
 
 void Climate::publish_state() {
+  this->set_has_state(true);
   ESP_LOGV(TAG, "'%s' >>", this->name_.c_str());
   auto traits = this->get_traits();
 

esphome/components/switch/switch.cpp

@@ -57,6 +57,7 @@ void Switch::publish_state(bool state) {
   if (!this->publish_dedup_.next(state))
     return;
   this->state = state != this->inverted_;
+  this->set_has_state(true);
 
   if (restore_mode & RESTORE_MODE_PERSISTENT_MASK)
     this->rtc_.save(&this->state);

esphome/components/template/climate/__init__.py

@@ -374,6 +374,7 @@ async def to_code(config: ConfigType) -> None:
             cg.add(var.set_preset(v))
         if (v := initial_state.get(CONF_CUSTOM_PRESET)) is not None:
             cg.add(var.set_custom_preset(v))
+        cg.add(var.set_has_state(True))
 
 
 CLIMATE_TEMPLATE_PUBLISH_ACTION_SCHEMA = cv.All(

esphome/components/template/water_heater/template_water_heater.cpp

@@ -91,7 +91,11 @@ void TemplateWaterHeater::loop() {
     }
   }
 
-  if (changed) {
+  // The first value always publishes, even one equal to the default (mode OFF, say), or the
+  // entity would report unknown forever.
+  const bool has_value =
+      curr_temp.has_value() || target_temp.has_value() || new_mode.has_value() || away.has_value() || is_on.has_value();
+  if (changed || (has_value && !this->has_state())) {
     this->publish_state();
   }
 }

esphome/components/water_heater/water_heater.cpp

@@ -161,6 +161,7 @@ void WaterHeaterCall::validate_() {
 }
 
 void WaterHeater::publish_state() {
+  this->set_has_state(true);
   auto traits = this->get_traits();
   ESP_LOGV(TAG,
            "'%s' >>\n"

tests/integration/fixtures/entity_missing_state.yaml

@@ -0,0 +1,78 @@
+esphome:
+  name: test-missing-state
+
+host:
+api:
+  batch_delay: 0ms  # Disable batching to receive all state updates
+logger:
+
+# None of these publish a state at startup, so each one starts out unknown
+switch:
+  - platform: template
+    name: "Test Switch"
+    id: test_switch
+    optimistic: true
+    restore_mode: DISABLED
+
+climate:
+  - platform: template
+    name: "Test Climate"
+    id: test_climate
+    optimistic: true
+    supported_modes:
+      - "OFF"
+      - HEAT
+
+  # A configured initial state is a state, so this one starts out known
+  - platform: template
+    name: "Initial Climate"
+    id: initial_climate
+    optimistic: true
+    restore_mode: NO_RESTORE
+    supported_modes:
+      - "OFF"
+      - HEAT
+    initial_state:
+      mode: HEAT
+
+globals:
+  - id: lambda_mode_known
+    type: bool
+    initial_value: "false"
+
+water_heater:
+  - platform: template
+    name: "Test Water Heater"
+    id: test_water_heater
+    supported_modes:
+      - "OFF"
+      - ECO
+
+  # Reports OFF, which is also the default, once the MCU-equivalent global says so
+  - platform: template
+    name: "Lambda Water Heater"
+    id: lambda_water_heater
+    supported_modes:
+      - "OFF"
+      - ECO
+    mode: |-
+      if (!id(lambda_mode_known)) {
+        return {};
+      }
+      return water_heater::WATER_HEATER_MODE_OFF;
+
+button:
+  - platform: template
+    name: "Publish States"
+    id: publish_states_button
+    on_press:
+      - switch.turn_on: test_switch
+      - climate.template.publish:
+          id: test_climate
+          mode: HEAT
+      - water_heater.template.publish:
+          id: test_water_heater
+          mode: ECO
+      - globals.set:
+          id: lambda_mode_known
+          value: "true"

tests/integration/test_entity_missing_state.py

@@ -0,0 +1,132 @@
+"""Integration test for missing_state on switch, climate and water heater.
+
+These three entity types always sent a concrete state, so a client could not
+tell a real OFF from an entity whose value has not been read yet. They now
+report missing_state until something publishes, like every other stateful
+entity does.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+from aioesphomeapi import (
+    ClimateInfo,
+    ClimateMode,
+    ClimateState,
+    EntityInfo,
+    EntityState,
+    SwitchInfo,
+    SwitchState,
+    WaterHeaterInfo,
+    WaterHeaterMode,
+    WaterHeaterState,
+)
+import pytest
+
+from .state_utils import InitialStateHelper, require_entity
+from .types import APIClientConnectedFactory, RunCompiledFunction
+
+
+@pytest.mark.asyncio
+async def test_entity_missing_state(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """Test that a switch, climate and water heater start out unknown.
+
+    This verifies that:
+    1. They report missing_state=True while nothing has published
+    2. Publishing clears missing_state and reports the published value
+    3. A first value that happens to equal the default still publishes
+    """
+    loop = asyncio.get_running_loop()
+    futures: dict[int, asyncio.Future[EntityState]] = {}
+
+    def on_state(state: EntityState) -> None:
+        """Resolve the pending future for the entity that changed."""
+        future = futures.get(state.key)
+        if future is not None and not future.done():
+            future.set_result(state)
+
+    async with (
+        run_compiled(yaml_config),
+        api_client_connected() as client,
+    ):
+        entities, _ = await client.list_entities_services()
+
+        switch_info = require_entity(entities, "test_switch", SwitchInfo)
+        climate_info = require_entity(entities, "test_climate", ClimateInfo)
+        initial_climate_info = require_entity(entities, "initial_climate", ClimateInfo)
+        water_heater_info = require_entity(
+            entities, "test_water_heater", WaterHeaterInfo
+        )
+        lambda_water_heater_info = require_entity(
+            entities, "lambda_water_heater", WaterHeaterInfo
+        )
+        publish_button = require_entity(
+            entities, "publish_states", description="Publish States button"
+        )
+        stateful: list[EntityInfo] = [
+            switch_info,
+            climate_info,
+            water_heater_info,
+            lambda_water_heater_info,
+        ]
+
+        initial_state_helper = InitialStateHelper(entities)
+        client.subscribe_states(initial_state_helper.on_state_wrapper(on_state))
+
+        try:
+            await initial_state_helper.wait_for_initial_states()
+        except TimeoutError:
+            pytest.fail("Timeout waiting for initial states")
+
+        # Nothing has published yet, so all of them must report unknown
+        for info in stateful:
+            state = initial_state_helper.initial_states.get(info.key)
+            assert state is not None, f"No initial state received for {info.object_id}"
+            assert state.missing_state is True, (
+                f"Initial state for {info.object_id} should have "
+                f"missing_state=True, got {state}"
+            )
+
+        # A configured initial_state must not start out unknown
+        initial_climate_state = initial_state_helper.initial_states.get(
+            initial_climate_info.key
+        )
+        assert isinstance(initial_climate_state, ClimateState)
+        assert initial_climate_state.missing_state is False
+        assert initial_climate_state.mode is ClimateMode.HEAT
+
+        # Publishing a state on each one clears missing_state
+        futures = {info.key: loop.create_future() for info in stateful}
+        client.button_command(publish_button.key)
+
+        try:
+            await asyncio.wait_for(asyncio.gather(*futures.values()), timeout=5.0)
+        except TimeoutError:
+            pytest.fail("Timeout waiting for published states")
+
+        switch_state = futures[switch_info.key].result()
+        assert isinstance(switch_state, SwitchState)
+        assert switch_state.missing_state is False
+        assert switch_state.state is True
+
+        climate_state = futures[climate_info.key].result()
+        assert isinstance(climate_state, ClimateState)
+        assert climate_state.missing_state is False
+        assert climate_state.mode is ClimateMode.HEAT
+
+        water_heater_state = futures[water_heater_info.key].result()
+        assert isinstance(water_heater_state, WaterHeaterState)
+        assert water_heater_state.missing_state is False
+        assert water_heater_state.mode is WaterHeaterMode.ECO
+
+        # Its first value is OFF, which is also the default the entity starts on,
+        # so it only leaves unknown if the first value publishes regardless
+        lambda_state = futures[lambda_water_heater_info.key].result()
+        assert isinstance(lambda_state, WaterHeaterState)
+        assert lambda_state.missing_state is False
+        assert lambda_state.mode is WaterHeaterMode.OFF