[api] Encode action strings straight from flash

The service name, description and argument strings are progmem fields, so
list entities no longer copies them into a stack scratch buffer, and the
ESP8266 per action string budget goes away.
This commit is contained in:
J. Nick Koston
2026-10-08 20:10:41 -10:00
parent b63b74f236
commit 7b5a1f9de9
10 changed files with 61 additions and 140 deletions
-33
View File
@@ -251,9 +251,6 @@ def _validate_supports_response(value: Any) -> str:
return cv.enum(SUPPORTS_RESPONSE_OPTIONS, lower=True)(value)
# ESP8266 copies every string of an action into a stack buffer sized by codegen; keep it small
ESP8266_ACTION_STRINGS_MAX_TOTAL = 384
VARIABLE_SCHEMA = cv.Schema(
{
cv.Required(CONF_TYPE): cv.one_of(*SERVICE_ARG_NATIVE_TYPES, lower=True),
@@ -484,29 +481,6 @@ def _action_strings(conf: ConfigType, has_metadata: bool) -> list[str | None]:
return strings
def _action_strings_size(strings: list[str | None]) -> int:
"""Bytes needed to copy every string out of flash, each with its terminator."""
return sum(
len(string.encode("utf-8")) + 1 for string in strings if string is not None
)
def _validate_esp8266_action_strings(config: ConfigType) -> ConfigType:
if not CORE.is_esp8266:
return config
actions = config.get(CONF_ACTIONS, [])
has_metadata = _has_action_metadata(actions)
for conf in actions:
size = _action_strings_size(_action_strings(conf, has_metadata))
if size > ESP8266_ACTION_STRINGS_MAX_TOTAL:
raise cv.Invalid(
f"Action '{conf[CONF_ACTION]}' has {size} bytes of name, variable name, "
f"description and example text; ESP8266 allows at most "
f"{ESP8266_ACTION_STRINGS_MAX_TOTAL} bytes per action"
)
return config
def _validate_outgoing_host_ipv6(config: ConfigType) -> ConfigType:
"""An IPv6 host can never be parsed, so never dialed, without IPv6."""
if (
@@ -531,7 +505,6 @@ def _validate_wizard(config: ConfigType) -> ConfigType:
FINAL_VALIDATE_SCHEMA = cv.All(
_validate_esp8266_action_strings,
_validate_outgoing_host_ipv6,
_validate_wizard,
)
@@ -604,7 +577,6 @@ async def to_code(config: ConfigType) -> None:
if config[CONF_HOMEASSISTANT_STATES]:
cg.add_define("USE_API_HOMEASSISTANT_STATES")
scratch_size = 0
if actions:
# Metadata is compiled in for every action once any action declares it, because the
# string table layout is fixed by the define rather than per action
@@ -657,8 +629,6 @@ async def to_code(config: ConfigType) -> None:
func_args.append((native, name))
strings = _action_strings(conf, has_metadata)
table = _add_action_strings(index, strings, interned_strings)
if CORE.is_esp8266:
scratch_size = max(scratch_size, _action_strings_size(strings))
# Template args: supports_response mode, then user service arg types
templ = cg.TemplateArguments(supports_response, *service_template_args)
# Key is hashed here because the name is not readable at runtime on ESP8266
@@ -685,9 +655,6 @@ async def to_code(config: ConfigType) -> None:
cg.add(auto.add_actions([unregister_action]))
# Register all services at once - single allocation, no reallocations
cg.add(var.initialize_user_services(triggers))
if CORE.is_esp8266 and has_user_actions:
# Stack buffer that list-entities copies PROGMEM strings into, sized for the largest action
cg.add_define("API_USER_ACTION_STRINGS_SCRATCH_SIZE", max(scratch_size, 1))
if (wizard_config := config.get(wizard.CONF_WIZARD)) is not None:
await wizard.to_code(wizard_config)
+5 -5
View File
@@ -1144,21 +1144,21 @@ enum SupportsResponseType {
}
message ListEntitiesServicesArgument {
option (ifdef) = "USE_API_USER_DEFINED_ACTIONS";
string name = 1;
string name = 1 [(progmem) = true];
ServiceArgType type = 2;
string description = 3 [(field_ifdef) = "USE_API_USER_DEFINED_ACTION_METADATA"];
string example = 4 [(field_ifdef) = "USE_API_USER_DEFINED_ACTION_METADATA"];
string description = 3 [(field_ifdef) = "USE_API_USER_DEFINED_ACTION_METADATA", (progmem) = true];
string example = 4 [(field_ifdef) = "USE_API_USER_DEFINED_ACTION_METADATA", (progmem) = true];
}
message ListEntitiesServicesResponse {
option (id) = 41;
option (source) = SOURCE_SERVER;
option (ifdef) = "USE_API_USER_DEFINED_ACTIONS";
string name = 1;
string name = 1 [(progmem) = true];
fixed32 key = 2 [(force) = true];
repeated ListEntitiesServicesArgument args = 3 [(fixed_vector) = true];
SupportsResponseType supports_response = 4;
string description = 5 [(field_ifdef) = "USE_API_USER_DEFINED_ACTION_METADATA"];
string description = 5 [(field_ifdef) = "USE_API_USER_DEFINED_ACTION_METADATA", (progmem) = true];
}
message ExecuteServiceArgument {
option (ifdef) = "USE_API_USER_DEFINED_ACTIONS";
+12 -5
View File
@@ -1324,13 +1324,17 @@ void GetTimeResponse::decode_field(void *self, uint32_t tag, const uint8_t *data
uint8_t *ListEntitiesServicesArgument::encode_msg(const void *self,
uint8_t *__restrict__ pos PROTO_ENCODE_DEBUG_PARAM) {
const auto &msg = *static_cast<const ListEntitiesServicesArgument *>(self);
pos = ProtoEncode::encode_string(pos PROTO_ENCODE_DEBUG_ARG, 1, msg.name);
pos = ProtoEncode::encode_progmem_bytes(pos PROTO_ENCODE_DEBUG_ARG, 1,
reinterpret_cast<const uint8_t *>(msg.name.progmem_ptr()), msg.name.size());
pos = ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 2, static_cast<uint32_t>(msg.type));
#ifdef USE_API_USER_DEFINED_ACTION_METADATA
pos = ProtoEncode::encode_string(pos PROTO_ENCODE_DEBUG_ARG, 3, msg.description);
pos = ProtoEncode::encode_progmem_bytes(pos PROTO_ENCODE_DEBUG_ARG, 3,
reinterpret_cast<const uint8_t *>(msg.description.progmem_ptr()),
msg.description.size());
#endif
#ifdef USE_API_USER_DEFINED_ACTION_METADATA
pos = ProtoEncode::encode_string(pos PROTO_ENCODE_DEBUG_ARG, 4, msg.example);
pos = ProtoEncode::encode_progmem_bytes(
pos PROTO_ENCODE_DEBUG_ARG, 4, reinterpret_cast<const uint8_t *>(msg.example.progmem_ptr()), msg.example.size());
#endif
return pos;
}
@@ -1350,14 +1354,17 @@ uint32_t ListEntitiesServicesArgument::calc_size_msg(const void *self) {
uint8_t *ListEntitiesServicesResponse::encode_msg(const void *self,
uint8_t *__restrict__ pos PROTO_ENCODE_DEBUG_PARAM) {
const auto &msg = *static_cast<const ListEntitiesServicesResponse *>(self);
pos = ProtoEncode::encode_string(pos PROTO_ENCODE_DEBUG_ARG, 1, msg.name);
pos = ProtoEncode::encode_progmem_bytes(pos PROTO_ENCODE_DEBUG_ARG, 1,
reinterpret_cast<const uint8_t *>(msg.name.progmem_ptr()), msg.name.size());
pos = ProtoEncode::write_tag_and_fixed32(pos PROTO_ENCODE_DEBUG_ARG, 21, msg.key);
for (auto &it : msg.args) {
pos = ProtoEncode::encode_sub_message(pos PROTO_ENCODE_DEBUG_ARG, 3, it);
}
pos = ProtoEncode::encode_uint32(pos PROTO_ENCODE_DEBUG_ARG, 4, static_cast<uint32_t>(msg.supports_response));
#ifdef USE_API_USER_DEFINED_ACTION_METADATA
pos = ProtoEncode::encode_string(pos PROTO_ENCODE_DEBUG_ARG, 5, msg.description);
pos = ProtoEncode::encode_progmem_bytes(pos PROTO_ENCODE_DEBUG_ARG, 5,
reinterpret_cast<const uint8_t *>(msg.description.progmem_ptr()),
msg.description.size());
#endif
return pos;
}
+5 -5
View File
@@ -1576,13 +1576,13 @@ class GetTimeResponse final : public ProtoDecodableMessage {
#ifdef USE_API_USER_DEFINED_ACTIONS
class ListEntitiesServicesArgument final : public ProtoMessage {
public:
StringRef name{nullptr, 0}; // null until set, encode only
ProgmemStringRef name{};
enums::ServiceArgType type{};
#ifdef USE_API_USER_DEFINED_ACTION_METADATA
StringRef description{nullptr, 0}; // null until set, encode only
ProgmemStringRef description{};
#endif
#ifdef USE_API_USER_DEFINED_ACTION_METADATA
StringRef example{nullptr, 0}; // null until set, encode only
ProgmemStringRef example{};
#endif
static uint8_t *encode_msg(const void *self, uint8_t *pos PROTO_ENCODE_DEBUG_PARAM);
uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const {
@@ -1603,12 +1603,12 @@ class ListEntitiesServicesResponse final : public ProtoMessage {
#ifdef HAS_PROTO_MESSAGE_DUMP
const LogString *message_name() const override { return LOG_STR("list_entities_services_response"); }
#endif
StringRef name{nullptr, 0}; // null until set, encode only
ProgmemStringRef name{};
uint32_t key{0};
FixedVector<ListEntitiesServicesArgument> args{};
enums::SupportsResponseType supports_response{};
#ifdef USE_API_USER_DEFINED_ACTION_METADATA
StringRef description{nullptr, 0}; // null until set, encode only
ProgmemStringRef description{};
#endif
static uint8_t *encode_msg(const void *self, uint8_t *pos PROTO_ENCODE_DEBUG_PARAM);
uint8_t *encode(ProtoWriteBuffer &buffer PROTO_ENCODE_DEBUG_PARAM) const {
+12 -5
View File
@@ -134,6 +134,13 @@ static void dump_progmem_bytes_field(DumpBuffer &out, const char *field_name, co
progmem_memcpy(data_buf, data, len);
dump_bytes_field(out, field_name, data_buf, len, indent);
}
// Helper for string fields in flash: copies the shown characters out with progmem_memcpy first
static void dump_progmem_string_field(DumpBuffer &out, const char *field_name, const ProgmemStringRef &value,
int indent = 2) {
char buf[DUMP_BYTES_MAX];
dump_field(out, field_name, StringRef(buf, value.write_to(buf, sizeof(buf))), indent);
}
#pragma GCC diagnostic pop
template<> const char *proto_enum_to_string<enums::DisconnectReason>(enums::DisconnectReason value) {
@@ -1612,19 +1619,19 @@ const char *GetTimeResponse::dump_to(DumpBuffer &out) const {
#ifdef USE_API_USER_DEFINED_ACTIONS
const char *ListEntitiesServicesArgument::dump_to(DumpBuffer &out) const {
MessageDumpHelper helper(out, ESPHOME_PSTR("ListEntitiesServicesArgument"));
dump_field(out, ESPHOME_PSTR("name"), this->name);
dump_progmem_string_field(out, ESPHOME_PSTR("name"), this->name);
dump_field(out, ESPHOME_PSTR("type"), static_cast<enums::ServiceArgType>(this->type));
#ifdef USE_API_USER_DEFINED_ACTION_METADATA
dump_field(out, ESPHOME_PSTR("description"), this->description);
dump_progmem_string_field(out, ESPHOME_PSTR("description"), this->description);
#endif
#ifdef USE_API_USER_DEFINED_ACTION_METADATA
dump_field(out, ESPHOME_PSTR("example"), this->example);
dump_progmem_string_field(out, ESPHOME_PSTR("example"), this->example);
#endif
return out.c_str();
}
const char *ListEntitiesServicesResponse::dump_to(DumpBuffer &out) const {
MessageDumpHelper helper(out, ESPHOME_PSTR("ListEntitiesServicesResponse"));
dump_field(out, ESPHOME_PSTR("name"), this->name);
dump_progmem_string_field(out, ESPHOME_PSTR("name"), this->name);
dump_field(out, ESPHOME_PSTR("key"), this->key);
for (const auto &it : this->args) {
out.append(4, ' ').append_p(ESPHOME_PSTR("args")).append(": ");
@@ -1633,7 +1640,7 @@ const char *ListEntitiesServicesResponse::dump_to(DumpBuffer &out) const {
}
dump_field(out, ESPHOME_PSTR("supports_response"), static_cast<enums::SupportsResponseType>(this->supports_response));
#ifdef USE_API_USER_DEFINED_ACTION_METADATA
dump_field(out, ESPHOME_PSTR("description"), this->description);
dump_progmem_string_field(out, ESPHOME_PSTR("description"), this->description);
#endif
return out.c_str();
}
+1 -2
View File
@@ -99,8 +99,7 @@ ListEntitiesIterator::ListEntitiesIterator(APIConnection *client) : client_(clie
static constexpr uint8_t SERVICE_YIELD_INTERVAL = 3;
bool ListEntitiesIterator::on_service(UserServiceDescriptor *service) {
UserActionScratch scratch;
auto resp = service->encode_list_service_response(scratch);
auto resp = service->encode_list_service_response();
if (!this->client_->send_message(resp))
return false;
// at_ is this service's index
+9 -20
View File
@@ -1,47 +1,36 @@
#include "user_services.h"
#include "esphome/core/hal.h"
#include "esphome/core/log.h"
#include "esphome/core/progmem.h"
#include "esphome/core/string_ref.h"
namespace esphome::api {
StringRef UserServiceStatic::str_(size_t idx, std::span<char> &scratch) const {
ProgmemStringRef UserServiceStatic::str_(size_t idx) const {
const char *s = progmem_read_ptr(&this->strings_[idx]);
if (s == nullptr)
return {};
#ifdef USE_ESP8266
// Codegen sizes the scratch buffer for the largest service; the bound only guards other callers
if (scratch.empty())
return {};
size_t len = strnlen_P(s, scratch.size() - 1);
progmem_memcpy(scratch.data(), s, len);
scratch[len] = '\0';
StringRef ref(scratch.data(), len);
scratch = scratch.subspan(len + 1);
return ref;
#else
return StringRef(s);
#endif
return ProgmemStringRef(s, ESPHOME_strlen_P(s));
}
ListEntitiesServicesResponse UserServiceStatic::encode_list_service_response_(
std::span<const enums::ServiceArgType> arg_types, std::span<char> scratch) const {
std::span<const enums::ServiceArgType> arg_types) const {
ListEntitiesServicesResponse msg;
msg.name = this->str_(0, scratch);
msg.name = this->str_(0);
msg.key = this->key_;
msg.supports_response = this->supports_response_;
#ifdef USE_API_USER_DEFINED_ACTION_METADATA
msg.description = this->str_(1, scratch);
msg.description = this->str_(1);
#endif
msg.args.init(arg_types.size());
for (size_t i = 0; i < arg_types.size(); i++) {
size_t base = USER_ACTION_HEADER_STRINGS + i * USER_ACTION_ARG_STRINGS;
auto &arg = msg.args.emplace_back();
arg.type = arg_types[i];
arg.name = this->str_(base, scratch);
arg.name = this->str_(base);
#ifdef USE_API_USER_DEFINED_ACTION_METADATA
arg.description = this->str_(base + 1, scratch);
arg.example = this->str_(base + 2, scratch);
arg.description = this->str_(base + 1);
arg.example = this->str_(base + 2);
#endif
}
return msg;
+10 -21
View File
@@ -20,9 +20,8 @@ class APIServer;
class UserServiceDescriptor {
public:
/// Build the list-entities message. On ESP8266 the strings live in PROGMEM and are copied into
/// `scratch`, so the returned message is only valid while `scratch` is; other platforms ignore it.
virtual ListEntitiesServicesResponse encode_list_service_response(std::span<char> scratch) = 0;
/// Build the list-entities message; its strings may point into flash (PROGMEM).
virtual ListEntitiesServicesResponse encode_list_service_response() = 0;
virtual bool execute_service(const ExecuteServiceRequest &req) = 0;
#ifdef USE_API_USER_DEFINED_ACTION_RESPONSES
@@ -37,13 +36,6 @@ template<typename T> T get_execute_arg_value(const ExecuteServiceArgument &arg);
template<typename T> enums::ServiceArgType to_service_arg_type();
// Scratch buffer list-entities hands to encode_list_service_response(); only ESP8266 copies into it
#ifdef USE_ESP8266
using UserActionScratch = std::array<char, API_USER_ACTION_STRINGS_SCRATCH_SIZE>;
#else
using UserActionScratch = std::array<char, 0>;
#endif
// Non-template base for YAML-defined services so the list-entities encoder is compiled once.
// All strings live in one PROGMEM pointer table emitted by codegen (see _action_strings in
// __init__.py), so each service costs a single pointer of RAM. Layout: the action name, then
@@ -63,12 +55,9 @@ class UserServiceStatic : public UserServiceDescriptor {
: strings_(strings), key_(key), supports_response_(supports_response) {}
protected:
ListEntitiesServicesResponse encode_list_service_response_(std::span<const enums::ServiceArgType> arg_types,
std::span<char> scratch) const;
/// Reference table entry `idx`; nullptr gives an empty StringRef.
/// On ESP8266 the bytes are copied out of PROGMEM into `scratch` with a terminator, and the span
/// is advanced past the copy.
StringRef str_(size_t idx, std::span<char> &scratch) const;
ListEntitiesServicesResponse encode_list_service_response_(std::span<const enums::ServiceArgType> arg_types) const;
/// Table entry `idx` as it sits in flash; nullptr gives an empty string.
ProgmemStringRef str_(size_t idx) const;
const char *const *strings_; // PROGMEM pointer table, read with progmem_read_ptr()
uint32_t key_;
@@ -79,9 +68,9 @@ template<typename... Ts> class UserServiceBase : public UserServiceStatic {
public:
using UserServiceStatic::UserServiceStatic;
ListEntitiesServicesResponse encode_list_service_response(std::span<char> scratch) override {
ListEntitiesServicesResponse encode_list_service_response() override {
std::array<enums::ServiceArgType, sizeof...(Ts)> arg_types = {to_service_arg_type<Ts>()...};
return this->encode_list_service_response_(arg_types, scratch);
return this->encode_list_service_response_(arg_types);
}
bool execute_service(const ExecuteServiceRequest &req) override {
@@ -125,9 +114,9 @@ template<typename... Ts> class UserServiceDynamic : public UserServiceDescriptor
this->key_ = fnv1_hash(this->name_.c_str());
}
ListEntitiesServicesResponse encode_list_service_response(std::span<char> /*scratch*/) override {
ListEntitiesServicesResponse encode_list_service_response() override {
ListEntitiesServicesResponse msg;
msg.name = StringRef(this->name_);
msg.name = ProgmemStringRef(StringRef(this->name_));
msg.key = this->key_;
msg.supports_response = enums::SUPPORTS_RESPONSE_NONE; // Dynamic services don't support responses yet
std::array<enums::ServiceArgType, sizeof...(Ts)> arg_types = {to_service_arg_type<Ts>()...};
@@ -135,7 +124,7 @@ template<typename... Ts> class UserServiceDynamic : public UserServiceDescriptor
for (size_t i = 0; i < sizeof...(Ts); i++) {
auto &arg = msg.args.emplace_back();
arg.type = arg_types[i];
arg.name = StringRef(this->arg_names_[i]);
arg.name = ProgmemStringRef(StringRef(this->arg_names_[i]));
}
return msg;
}
-1
View File
@@ -283,7 +283,6 @@
#define USE_API_USER_DEFINED_ACTION_RESPONSES
#define USE_API_USER_DEFINED_ACTION_RESPONSES_JSON
#define API_MAX_SEND_QUEUE 8
#define API_USER_ACTION_STRINGS_SCRATCH_SIZE 64
#define API_WIZARD_DATA_SIZE 1
#define API_WIZARD_INPUT_COUNT 1
#define MAX_API_CONNECTIONS 6
@@ -7,18 +7,14 @@ import pytest
from esphome.components.api import (
_action_strings,
_action_strings_size,
_has_action_metadata,
_validate_esp8266_action_strings,
validate_variable,
)
from esphome.config_validation import Invalid
from esphome.const import PlatformFramework
from esphome.core import CORE
from esphome.cpp_generator import safe_exp
from esphome.helpers import fnv1_hash
from tests.component_tests.helpers import get_define_value
from tests.component_tests.types import SetCoreConfigCallable
CONFIG = "tests/component_tests/api/test_action_metadata.yaml"
CONFIG_ESP8266 = "tests/component_tests/api/test_action_metadata_esp8266.yaml"
@@ -54,18 +50,16 @@ def test_metadata_is_emitted_as_progmem_table(
)
assert f"(api_action0_strings, {safe_exp(fnv1_hash('play_buzzer'))});" in main_cpp
assert "USE_API_USER_DEFINED_ACTION_METADATA" in {d.name for d in CORE.defines}
assert get_define_value("API_USER_ACTION_STRINGS_SCRATCH_SIZE") is None
def test_esp8266_sizes_scratch_buffer_for_largest_action(
def test_esp8266_encodes_action_strings_from_flash(
generate_main: Callable[[str | Path], str],
) -> None:
"""ESP8266 gets a scratch buffer define equal to the byte total of the largest action."""
generate_main(CONFIG_ESP8266)
"""ESP8266 uses the same flash string table, with no scratch buffer to size."""
main_cpp = generate_main(CONFIG_ESP8266)
# play_buzzer: name, description, two variable names, one description, one example,
# each with a terminator
assert get_define_value("API_USER_ACTION_STRINGS_SCRATCH_SIZE") == "117"
assert "api_action0_strings[] PROGMEM" in main_cpp
assert get_define_value("API_USER_ACTION_STRINGS_SCRATCH_SIZE") is None
def test_shorthand_variables_emit_no_metadata(
@@ -101,36 +95,8 @@ def test_variable_rejects_invalid(value: object) -> None:
validate_variable(value)
def _oversized_action_config() -> dict:
return {
"actions": [
{
"action": "big",
"description": "x" * 300,
"variables": {"a": {"type": "string", "example": "y" * 300}},
}
]
}
def test_esp8266_rejects_actions_over_string_budget(
set_core_config: SetCoreConfigCallable,
) -> None:
set_core_config(PlatformFramework.ESP8266_ARDUINO)
with pytest.raises(Invalid, match="ESP8266 allows at most 384 bytes"):
_validate_esp8266_action_strings(_oversized_action_config())
def test_other_platforms_have_no_string_budget(
set_core_config: SetCoreConfigCallable,
) -> None:
set_core_config(PlatformFramework.ESP32_IDF)
config = _oversized_action_config()
assert _validate_esp8266_action_strings(config) is config
def test_empty_metadata_is_unset_and_not_counted() -> None:
"""An empty description or example emits nullptr and takes no scratch space."""
def test_empty_metadata_is_unset() -> None:
"""An empty description or example emits nullptr."""
conf = {
"action": "a",
"description": "",
@@ -138,8 +104,6 @@ def test_empty_metadata_is_unset_and_not_counted() -> None:
}
strings = _action_strings(conf, has_metadata=True)
assert strings == ["a", None, "b", None, "ex"]
# Every emitted string counts its terminator: "a" + "b" + "ex"
assert _action_strings_size(strings) == 2 + 2 + 3
def test_empty_metadata_does_not_enable_the_define() -> None: