Merge remote-tracking branch 'origin/dev' into apply-bytes

AGENTS.md

@@ -205,8 +205,9 @@ file does, and it is the authority when they disagree. The most useful starting
         for a table of structs copy the whole entry out and return the copy, never a pointer into flash.
         On other platforms they are a plain `memcpy` and dereference, so no `#ifdef` is needed. Declare log tags with
         `ESPHOME_LOG_TAG(TAG, "name")`, pass status text as `LOG_STR`, e.g.
-        `this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL))`, and give an action whose payload is a byte array
-        a `TEMPLATABLE_BYTES` field so a static code stays in flash.
+        `this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL))`. An action that only forwards a byte payload is
+        registered with `automation.ApplyBytes` (see Actions below), which keeps a constant payload in flash;
+        `TEMPLATABLE_BYTES` is for a `play()` that does more with the bytes.
     *   **Fixed sets of names** (status words, model names): a `PROGMEM_STRING_TABLE` (`esphome/core/progmem.h`),
         logged with `get_log_str` and `LOG_STR_ARG`, copied out with `ESPHOME_strncpy_P` before `publish_state`.
         A `dump_config` helper that picks a name returns `const LogString *` from `LOG_STR(...)`.
@@ -604,7 +605,9 @@ file does, and it is the authority when they disagree. The most useful starting
         )
         ```
         The `ApplyField`, `ApplyCall` and `register_apply_action` docstrings in `esphome/automation.py` cover
-        the rest; `cover.control` and `cover.template.publish` are in-tree examples. `TEMPLATABLE_VALUE` with
+        the rest; `cover.control` and `cover.template.publish` are in-tree examples. An action whose input is a
+        byte payload passes `automation.ApplyBytes(CONF_DATA, "send({})")` in the same call; a constant payload
+        stays in flash and a lambda's vector is forwarded as data and size. `TEMPLATABLE_VALUE` with
         `cg.templatable` stays for actions whose `play()` has real logic beyond forwarding values.
 
     *   **Conditions:**

esphome/automation.py

@@ -59,10 +59,15 @@ async def templatable_bytes(
     if cg.is_template(value):
         fn = await cg.templatable(value, args, cg.std_vector.template(cg.uint8))
         cg.add(set_template(fn))
-    elif len(value) > 0xFFFF:
-        raise EsphomeError(f"Byte payload is {len(value)} bytes; the maximum is 65535")
     else:
-        cg.add(set_static(progmem_bytes(table_name, value), len(value)))
+        cg.add(set_static(*_static_bytes(table_name, value)))
+
+
+def _static_bytes(table_name: str, value: bytes | list[int]) -> tuple[MockObj, int]:
+    """The shared PROGMEM table for a constant payload and its length."""
+    if (size := len(value)) > 0xFFFF:
+        raise EsphomeError(f"Byte payload is {size} bytes; the maximum is 65535")
+    return progmem_bytes(table_name, value), size
 
 
 def maybe_simple_id(*validators):
@@ -356,6 +361,33 @@ class ApplyField:
         return ApplyCall(target, ((self.conf_key, self.type_, self.const_fn),))
 
 
+@dataclass(frozen=True)
+class ApplyBytes:
+    """A byte payload key forwarded as ``target``, whose ``{}`` stands for ``data, len``.
+
+    A constant payload is a shared PROGMEM table passed as is; on ESP8266 it is first copied
+    out, to the stack up to ``stack`` bytes and to the heap beyond, unless ``progmem_target``
+    names a call that reads flash itself. A lambda's ``std::vector<uint8_t>`` is passed as
+    its data and size. The template may name the parent as ``{parent}``. An absent key
+    emits nothing.
+    """
+
+    conf_key: str | tuple[str, ...]
+    target: str
+    stack: int = 64
+    progmem_target: str | None = None
+
+    def __post_init__(self) -> None:
+        for target in (self.target, self.progmem_target):
+            if target is None:
+                continue
+            fields = [
+                f for _, f, _, _ in string.Formatter().parse(target) if f is not None
+            ]
+            if fields.count("") != 1 or any(f not in ("", "parent") for f in fields):
+                raise ValueError(f"apply bytes target {target!r} needs one {{}}")
+
+
 def _config_lookup(config: ConfigType, key: str | tuple[str, ...]) -> Any:
     if isinstance(key, str):
         return config.get(key)
@@ -488,7 +520,7 @@ def _apply_values(config: ConfigType, members: list[tuple[Any, Any, Any]]) -> li
 def register_apply_action(
     name: str,
     schema: cv.Schema,
-    *fields: ApplyField | ApplyCall,
+    *fields: ApplyField | ApplyCall | ApplyBytes,
     call: str | None = None,
     id_key: str = CONF_ID,
 ) -> None:
@@ -504,7 +536,7 @@ def register_apply_action(
 def apply_action_builder(
     name: str,
     schema: cv.Schema,
-    *fields: ApplyField | ApplyCall,
+    *fields: ApplyField | ApplyCall | ApplyBytes,
     call: str | None = None,
     id_key: str = CONF_ID,
 ) -> Callable[..., Awaitable[MockObj]]:
@@ -517,17 +549,20 @@ def apply_action_builder(
     and ``apply_call.perform()`` is appended.
     """
     # An action stores the value, so a std::string constant stays in flash on ESP8266.
-    statements_spec = [
-        (
-            c.target,
-            _default_std_string(c.members, flash_string),
-        )
-        for c in (f if isinstance(f, ApplyCall) else f.call() for f in fields)
-    ]
     _check_key_in_schema(name, schema, id_key)
-    for _, members in statements_spec:
+    statements_spec: list[ApplyBytes | tuple[str, list[tuple[Any, Any, Any]]]] = []
+    for f in fields:
+        if isinstance(f, ApplyBytes):
+            _check_key_in_schema(name, schema, f.conf_key)
+            statements_spec.append(f)
+            continue
+        c = f if isinstance(f, ApplyCall) else f.call()
+        members = _default_std_string(c.members, flash_string)
         for conf_key, _, _ in members:
             _check_key_in_schema(name, schema, conf_key)
+        statements_spec.append((c.target, members))
+    # Named like the generated function, so it cannot clash with an SDK symbol
+    table_name = f"esphome__{name.replace('.', '_')}__bytes"
 
     async def builder(
         config: ConfigType,
@@ -539,7 +574,15 @@ def apply_action_builder(
         lambda_args = _apply_lambda_args(args)
         receiver = "apply_call." if call else f"{parent}->"
         statements: list[str] = []
-        for target, members in statements_spec:
+        for spec in statements_spec:
+            if isinstance(spec, ApplyBytes):
+                statements.extend(
+                    await _render_bytes(
+                        spec, config, receiver, parent, lambda_args, table_name
+                    )
+                )
+                continue
+            target, members = spec
             values = _apply_values(config, members)
             if not _apply_call_active(members, values):
                 continue
@@ -560,6 +603,56 @@ def apply_action_builder(
     return builder
 
 
+async def _render_bytes(
+    spec: ApplyBytes,
+    config: ConfigType,
+    receiver: str,
+    parent: str,
+    lambda_args: TemplateArgsType,
+    table_name: str,
+) -> list[str]:
+    """The statements for one ``ApplyBytes``: what ``TemplatableBytes::visit`` does, inlined."""
+    value = _config_lookup(config, spec.conf_key)
+    if value is None:
+        return []
+
+    # The temporary carries the generated-name prefix so a trigger argument cannot shadow it
+    def send(target: str, data: str) -> str:
+        return f"{receiver}{target.format(data, parent=parent)};"
+
+    if isinstance(value, Lambda):
+        inner = await cg.process_lambda(
+            value, lambda_args, return_type=cg.std_vector.template(cg.uint8)
+        )
+        return [
+            "{",
+            f"  const std::vector<uint8_t> esphome__bytes = ({call_lambda(inner)});",
+            f"  {send(spec.target, 'esphome__bytes.data(), esphome__bytes.size()')}",
+            "}",
+        ]
+    table, size = _static_bytes(table_name, value)
+    # Qualified so a trigger argument cannot shadow a reused table; an empty payload is nullptr
+    args = f"::{table}, {size}" if size else f"{table}, {size}"
+    if CORE.is_esp8266 and spec.progmem_target is not None:
+        return [send(spec.progmem_target, args)]
+    if not CORE.is_esp8266:
+        return [send(spec.target, args)]
+    # A PROGMEM table must be read with aligned loads on ESP8266, so copy it out first;
+    # the length is known here, so the copy is an exact array, on the heap past ``stack``.
+    # An empty payload still gets a one byte array: the raw lwIP socket rejects a null buffer.
+    if size <= spec.stack:
+        buffer, data = f"uint8_t esphome__bytes[{max(size, 1)}];", "esphome__bytes"
+    else:
+        buffer, data = (
+            f"auto esphome__bytes = std::make_unique<uint8_t[]>({size});",
+            "esphome__bytes.get()",
+        )
+    statements = ["{", f"  {buffer}"]
+    if size:
+        statements.append(f"  esphome::progmem_memcpy({data}, {args});")
+    return [*statements, f"  {send(spec.target, f'{data}, {size}')}", "}"]
+
+
 def _apply_call_active(members: list[tuple[Any, Any, Any]], values: list[Any]) -> bool:
     """An ``ApplyCall`` is emitted unless it has keys and none of them is set."""
     return not members or any(value is not None for value in values)

tests/unit_tests/test_automation.py

@@ -11,6 +11,7 @@ from esphome.automation import (
     CONF_CHOICE_ID,
     CONF_CHOOSE,
     ApplyAction,
+    ApplyBytes,
     ApplyCall,
     ApplyCondition,
     ApplyField,
@@ -1281,6 +1282,145 @@ async def test_apply_string_ref_literal_stays_in_flash_on_esp8266(
     assert f"::{PARENT_OBJ}->set_payload({rendered});" in _apply_definition(mock_cg)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("platform", ["esp32", "esp8266"])
+async def test_apply_bytes_lambda_is_a_vector_in_a_block(
+    registries: tuple[Registry, Registry], mock_cg: MockCodegen, platform: str
+) -> None:
+    """A lambda payload is called once into a scoped vector and passed as data and size."""
+    fields = (ApplyBytes("data", "send({})"),)
+    config = {"data": Lambda("return {0x01, x};")}
+    await _run_apply_action(
+        registries, fields, config, args=[(cg.uint8, "x")], platform=platform
+    )
+    text = _apply_definition(mock_cg)
+    # Parenthesized so a reduced `return a, b;` stays one initializer
+    assert "const std::vector<uint8_t> esphome__bytes = (" in text
+    assert "-> std::vector<uint8_t>" in text
+    assert (
+        f"::{PARENT_OBJ}->send(esphome__bytes.data(), esphome__bytes.size());" in text
+    )
+    assert "progmem_memcpy" not in text
+
+
+@pytest.mark.asyncio
+async def test_apply_bytes_constant_is_a_shared_table(
+    registries: tuple[Registry, Registry], mock_cg: MockCodegen
+) -> None:
+    """A constant payload is the shared PROGMEM table and its length, named after the action."""
+    CORE.config = {}
+    fields = (ApplyBytes("data", "send({})"),)
+    await _run_apply_action(registries, fields, {"data": [0xA1, 0x02]})
+    assert f"::{PARENT_OBJ}->send(::esphome__my_apply__bytes, 2);" in _apply_definition(
+        mock_cg
+    )
+
+    mock_cg.new_pvariable.reset_mock()
+    await _run_apply_action(registries, fields, {"data": b""})
+    assert f"::{PARENT_OBJ}->send(nullptr, 0);" in _apply_definition(mock_cg)
+
+    mock_cg.new_pvariable.reset_mock()
+    await _run_apply_action(registries, fields, {})
+    assert "send(" not in _apply_definition(mock_cg)
+
+    with pytest.raises(EsphomeError, match="maximum is 65535"):
+        await _run_apply_action(registries, fields, {"data": bytes(0x10000)})
+
+
+@pytest.mark.asyncio
+async def test_apply_bytes_constant_is_copied_out_of_flash_on_esp8266(
+    registries: tuple[Registry, Registry], mock_cg: MockCodegen
+) -> None:
+    """On ESP8266 a constant goes through a stack buffer, or straight to a flash-aware target."""
+    CORE.config = {}
+    fields = (ApplyBytes("data", "send({})", stack=128),)
+    await _run_apply_action(
+        registries, fields, {"data": [0xA1, 0x02]}, platform="esp8266"
+    )
+    text = _apply_definition(mock_cg)
+    lines = [
+        "uint8_t esphome__bytes[2];",
+        "esphome::progmem_memcpy(esphome__bytes, ::esphome__my_apply__bytes, 2);",
+        f"::{PARENT_OBJ}->send(esphome__bytes, 2);",
+    ]
+    positions = [text.index(line) for line in lines]
+    assert positions == sorted(positions)
+
+    # Past the stack budget the copy goes to the heap
+    mock_cg.new_pvariable.reset_mock()
+    fields = (ApplyBytes("data", "send({})", stack=1),)
+    await _run_apply_action(
+        registries, fields, {"data": [0xA1, 0x02]}, platform="esp8266"
+    )
+    text = _apply_definition(mock_cg)
+    assert "auto esphome__bytes = std::make_unique<uint8_t[]>(2);" in text
+    assert (
+        "esphome::progmem_memcpy(esphome__bytes.get(), ::esphome__my_apply__bytes, 2);"
+        in text
+    )
+    assert f"::{PARENT_OBJ}->send(esphome__bytes.get(), 2);" in text
+
+    mock_cg.new_pvariable.reset_mock()
+    fields = (ApplyBytes("data", "send({})", progmem_target="send_progmem({})"),)
+    await _run_apply_action(
+        registries, fields, {"data": [0xA1, 0x02]}, platform="esp8266"
+    )
+    text = _apply_definition(mock_cg)
+    assert f"::{PARENT_OBJ}->send_progmem(::esphome__my_apply__bytes, 2);" in text
+    assert "progmem_memcpy" not in text
+
+    # An empty payload never touches flash but still passes a non-null buffer
+    mock_cg.new_pvariable.reset_mock()
+    fields = (ApplyBytes("data", "send({})"),)
+    await _run_apply_action(registries, fields, {"data": []}, platform="esp8266")
+    text = _apply_definition(mock_cg)
+    assert "uint8_t esphome__bytes[1];" in text
+    assert f"::{PARENT_OBJ}->send(esphome__bytes, 0);" in text
+    assert "progmem_memcpy" not in text
+
+
+@pytest.mark.asyncio
+async def test_apply_bytes_beside_other_fields_and_call_object(
+    registries: tuple[Registry, Registry], mock_cg: MockCodegen
+) -> None:
+    CORE.config = {}
+    fields = (
+        ApplyField("channel", "set_channel", cg.uint8),
+        ApplyBytes("data", "write({})"),
+    )
+    await _run_apply_action(
+        registries, fields, {"channel": 3, "data": [1]}, call="make_call"
+    )
+    text = _apply_definition(mock_cg)
+    lines = [
+        f"auto apply_call = ::{PARENT_OBJ}->make_call();",
+        "apply_call.set_channel(3);",
+        "apply_call.write(::esphome__my_apply__bytes, 1);",
+        "apply_call.perform();",
+    ]
+    positions = [text.index(line) for line in lines]
+    assert positions == sorted(positions)
+
+
+def test_apply_bytes_checks_its_shape(registries: tuple[Registry, Registry]) -> None:
+    # Escaped braces are not placeholders
+    assert ApplyBytes("data", "send({{}}, {})").target == "send({{}}, {})"
+    with pytest.raises(ValueError, match="needs one {}"):
+        ApplyBytes("data", "send({{}})")
+    with pytest.raises(ValueError, match="needs one {}"):
+        ApplyBytes("data", "send()")
+    with pytest.raises(ValueError, match="needs one {}"):
+        ApplyBytes("data", "send({})", progmem_target="send_progmem({}, {})")
+    with pytest.raises(ValueError, match="'dat' is not in the schema"):
+        register_apply_action(
+            "my.bad",
+            cv.Schema(
+                {cv.Required(CONF_ID): cv.string, cv.Required("data"): cv.string}
+            ),
+            ApplyBytes("dat", "send({})"),
+        )
+
+
 @pytest.mark.asyncio
 async def test_register_apply_condition_predicate(
     registries: tuple[Registry, Registry], mock_cg: MockCodegen