[rp2040] Tune oversized lwIP defaults for ESPHome (#14843)

MANIFEST.in

@@ -4,4 +4,5 @@ include requirements.txt
 recursive-include esphome *.yaml
 recursive-include esphome *.cpp *.h *.tcc *.c
 recursive-include esphome *.py.script
+recursive-include esphome *.jinja
 recursive-include esphome LICENSE.txt

esphome/components/rp2040/__init__.py

@@ -26,7 +26,7 @@ from esphome.core.config import BOARD_MAX_LENGTH
 from esphome.helpers import copy_file_if_changed, read_file, write_file_if_changed
 
 from . import boards
-from .const import KEY_BOARD, KEY_PIO_FILES, KEY_RP2040, rp2040_ns
+from .const import KEY_BOARD, KEY_LWIP_OPTS, KEY_PIO_FILES, KEY_RP2040, rp2040_ns
 
 # force import gpio to register pin schema
 from .gpio import rp2040_pin_to_code  # noqa
@@ -240,6 +240,160 @@ async def to_code(config):
     cg.add_define("USE_RP2040_WATCHDOG_TIMEOUT", config[CONF_WATCHDOG_TIMEOUT])
     cg.add_define("USE_RP2040_CRASH_HANDLER")
 
+    _configure_lwip()
+
+
+def _configure_lwip() -> None:
+    """Configure lwIP options for RP2040 by generating a custom lwipopts.h.
+
+    Arduino-pico's lwipopts.h has no #ifndef guards, so -D flags cannot override
+    its settings. Instead, we generate a replacement lwipopts.h and place it in an
+    include directory that shadows the framework's version.
+
+    lwIP is compiled from source on RP2040 (not pre-built), so our replacement
+    header fully controls the compiled lwIP behavior.
+
+    RP2040 uses NO_SYS=1 (polling, no RTOS thread), LWIP_SOCKET=0, LWIP_NETCONN=0.
+    DHCP/DNS use raw udp_new() which allocates from MEMP_NUM_UDP_PCB.
+
+    Comparison of arduino-pico defaults vs ESPHome targets (TCP_MSS=1460):
+
+    Setting                   ESP8266  ESP32  arduino-pico  New
+    ────────────────────────────────────────────────────────────────
+    TCP_SND_BUF               2×MSS   4×MSS  8×MSS         4×MSS
+    TCP_WND                   4×MSS   4×MSS  8×MSS         4×MSS
+    MEM_LIBC_MALLOC           1       1      0             0*
+    MEMP_MEM_MALLOC           1       1      0             0**
+    MEM_SIZE                  N/A***  N/A*** 16KB          16KB
+    PBUF_POOL_SIZE            10      16     24            16
+    MEMP_NUM_TCP_SEG          10      16     32            17
+    MEMP_NUM_TCP_PCB          5       16     5             dynamic
+    MEMP_NUM_TCP_PCB_LISTEN   4       16     8****         dynamic
+    MEMP_NUM_UDP_PCB          4       16     7             dynamic
+    TCP_SND_QUEUELEN          ~8      17     32            17
+
+    * MEM_LIBC_MALLOC must stay 0: arduino-pico uses
+      PICO_CYW43_ARCH_THREADSAFE_BACKGROUND which runs lwIP callbacks from
+      a low-priority pendsv IRQ. The pico-sdk explicitly blocks
+      MEM_LIBC_MALLOC=1 because libc malloc uses mutexes (unsafe in IRQ).
+    ** MEMP_MEM_MALLOC must stay 0: the dedicated lwIP heap (MEM_SIZE=16KB)
+      is too small to hold all pools dynamically. The PBUF_POOL alone needs
+      ~24KB (16 × 1524 bytes). Increasing MEM_SIZE would negate BSS savings.
+    *** ESP8266/ESP32 use MEM_LIBC_MALLOC=1 (system heap, no dedicated pool).
+    **** opt.h default; arduino-pico doesn't override MEMP_NUM_TCP_PCB_LISTEN.
+    "dynamic" = auto-calculated from component socket registrations via
+    socket.get_socket_counts() with minimums of 8 TCP / 6 UDP / 2 TCP_LISTEN.
+    """
+    from esphome.components.socket import (
+        MIN_TCP_LISTEN_SOCKETS,
+        MIN_TCP_SOCKETS,
+        MIN_UDP_SOCKETS,
+        get_socket_counts,
+    )
+
+    sc = get_socket_counts()
+    # Apply platform minimums — ensure headroom for ESPHome's needs
+    tcp_sockets = max(MIN_TCP_SOCKETS, sc.tcp)
+    udp_sockets = max(MIN_UDP_SOCKETS, sc.udp)
+    # RP2040 has more RAM (264KB) than most LibreTiny boards, so DHCP/DNS
+    # UDP PCBs (2) are absorbed by the generous minimum of 6.
+    listening_tcp = max(MIN_TCP_LISTEN_SOCKETS, sc.tcp_listen)
+
+    # TCP_SND_BUF: 4×MSS=5,840 matches ESP32. Down from arduino-pico's 8×MSS.
+    # ESPAsyncWebServer allocates malloc(tcp_sndbuf()) per response chunk.
+    tcp_snd_buf = "(4*TCP_MSS)"
+
+    # TCP_WND: receive window. 4×MSS matches ESP32. Down from arduino-pico's 8×MSS.
+    tcp_wnd = "(4*TCP_MSS)"
+
+    # TCP_SND_QUEUELEN: max pbufs queued for send buffer
+    # ESP-IDF formula: (4 * TCP_SND_BUF + (TCP_MSS - 1)) / TCP_MSS
+    # With 4×MSS: (4*5840 + 1459) / 1460 = 17 — match ESP32
+    tcp_snd_queuelen = 17
+    # MEMP_NUM_TCP_SEG: segment pool, must be >= TCP_SND_QUEUELEN (lwIP sanity check)
+    memp_num_tcp_seg = tcp_snd_queuelen
+
+    # PBUF_POOL_SIZE: RP2040 has 264KB RAM, more generous than LibreTiny.
+    # 16 matches ESP32 (vs arduino-pico's 24). With MEMP_MEM_MALLOC=1,
+    # this is a max count (allocated on demand from heap).
+    pbuf_pool_size = 16
+
+    # Build the lwIP override defines for the Jinja2 template.
+    # The template uses #include_next to chain to the framework's original
+    # lwipopts.h, then #undef/#define only the values we need to change.
+    #
+    # Note: MEMP_MEM_MALLOC stays 0 (framework default). While the memp
+    # allocations use the dedicated lwIP heap (IRQ-safe), the 16KB MEM_SIZE
+    # is too small to hold all pools dynamically under stress. The PBUF_POOL
+    # alone needs ~24KB (16 × 1524 bytes). Increasing MEM_SIZE would negate
+    # the BSS savings.
+    #
+    # MEM_LIBC_MALLOC stays 0 (framework default): arduino-pico uses
+    # PICO_CYW43_ARCH_THREADSAFE_BACKGROUND which runs lwIP callbacks from
+    # a low-priority pendsv IRQ where libc malloc (mutex-based) is unsafe.
+    lwip_defines: dict[str, str] = {
+        "TCP_SND_BUF": tcp_snd_buf,
+        "TCP_WND": tcp_wnd,
+        "TCP_SND_QUEUELEN": str(tcp_snd_queuelen),
+        "MEMP_NUM_TCP_SEG": str(memp_num_tcp_seg),
+        "PBUF_POOL_SIZE": str(pbuf_pool_size),
+        "MEMP_NUM_TCP_PCB": str(tcp_sockets),
+        "MEMP_NUM_TCP_PCB_LISTEN": str(listening_tcp),
+        "MEMP_NUM_UDP_PCB": str(udp_sockets),
+    }
+
+    # Store for copy_files() to generate the header
+    CORE.data[KEY_RP2040][KEY_LWIP_OPTS] = lwip_defines
+
+    # Add a pre-build extra script that injects our lwip_override directory
+    # into CCFLAGS so our lwipopts.h shadows the framework's version.
+    # Regular build_flags (-I/-isystem) come after -iwithprefixbefore in GCC's
+    # search order, so we must prepend via an extra_scripts hook.
+    cg.add_platformio_option("extra_scripts", ["pre:inject_lwip_include.py"])
+
+    tcp_min = " (min)" if tcp_sockets > sc.tcp else ""
+    udp_min = " (min)" if udp_sockets > sc.udp else ""
+    listen_min = " (min)" if listening_tcp > sc.tcp_listen else ""
+    _LOGGER.info(
+        "Configuring lwIP: TCP=%d%s [%s], UDP=%d%s [%s], TCP_LISTEN=%d%s [%s]",
+        tcp_sockets,
+        tcp_min,
+        sc.tcp_details,
+        udp_sockets,
+        udp_min,
+        sc.udp_details,
+        listening_tcp,
+        listen_min,
+        sc.tcp_listen_details,
+    )
+
+
+def _generate_lwipopts_h() -> None:
+    """Generate a custom lwipopts.h that shadows the framework's version.
+
+    Uses Jinja2 to render the template with the lwIP defines calculated
+    during code generation. The generated header is placed in lwip_override/
+    in the build directory, and a pre-build script injects this directory
+    into the compiler include path before the framework's own include dir.
+    """
+    from jinja2 import Environment, FileSystemLoader
+
+    lwip_defines = CORE.data[KEY_RP2040].get(KEY_LWIP_OPTS)
+    if not lwip_defines:
+        return
+
+    template_dir = Path(__file__).parent
+    jinja_env = Environment(
+        loader=FileSystemLoader(str(template_dir)),
+        keep_trailing_newline=True,
+    )
+    template = jinja_env.get_template("lwipopts.h.jinja")
+    content = template.render(**lwip_defines)
+
+    lwip_dir = CORE.relative_build_path("lwip_override")
+    lwip_dir.mkdir(parents=True, exist_ok=True)
+    write_file_if_changed(lwip_dir / "lwipopts.h", content)
+
 
 def add_pio_file(component: str, key: str, data: str):
     try:
@@ -289,6 +443,12 @@ def copy_files():
         post_build_file,
         CORE.relative_build_path("post_build.py"),
     )
+    inject_lwip_file = dir / "inject_lwip_include.py.script"
+    copy_file_if_changed(
+        inject_lwip_file,
+        CORE.relative_build_path("inject_lwip_include.py"),
+    )
+    _generate_lwipopts_h()
     if generate_pio_files():
         path = CORE.relative_src_path("esphome.h")
         content = read_file(path).rstrip("\n")

esphome/components/rp2040/const.py

@@ -1,6 +1,7 @@
 import esphome.codegen as cg
 
 KEY_BOARD = "board"
+KEY_LWIP_OPTS = "lwip_opts"
 KEY_RP2040 = "rp2040"
 KEY_PIO_FILES = "pio_files"
 

esphome/components/rp2040/inject_lwip_include.py.script

@@ -0,0 +1,18 @@
+# pylint: disable=E0602
+Import("env")  # noqa
+
+import os
+
+# PlatformIO pre-build script: inject lwip_override include path so our
+# lwipopts.h shadows the framework's version during lwIP compilation.
+#
+# The arduino-pico builder uses -iprefix + -iwithprefixbefore for includes,
+# which takes priority over CPPPATH (-I). We must inject our path into the
+# CCFLAGS BEFORE the -iprefix flag to ensure our lwipopts.h is found first.
+
+lwip_dir = os.path.join(env["PROJECT_DIR"], "lwip_override")
+
+if os.path.isdir(lwip_dir):
+    # Insert -I<lwip_dir> at the beginning of CCFLAGS, before the framework's
+    # -iprefix/-iwithprefixbefore flags which would otherwise take priority.
+    env.Prepend(CCFLAGS=["-I", lwip_dir])

esphome/components/rp2040/lwipopts.h.jinja

@@ -0,0 +1,46 @@
+// ESPHome lwIP configuration override for RP2040.
+// Includes the framework's original lwipopts.h, then overrides specific
+// settings to tune lwIP for ESPHome's IoT use case.
+//
+// This file is found first via -I injection (see inject_lwip_include.py.script).
+// #include_next chains to the framework's original in include/lwipopts.h.
+// Since the original uses #pragma once, it won't be included again later
+// (e.g. via tusb_config.h), avoiding duplicate definition warnings.
+
+// Include the framework's original lwipopts.h first
+#include_next "lwipopts.h"
+
+// --- ESPHome overrides below ---
+// Only #undef and redefine values that differ from the framework defaults.
+
+// TCP send/receive buffers: 4xMSS matches ESP32 (down from 8xMSS)
+#undef TCP_SND_BUF
+#define TCP_SND_BUF                   {{ TCP_SND_BUF }}
+
+#undef TCP_WND
+#define TCP_WND                       {{ TCP_WND }}
+
+// Queued segment limits: derived from 4xMSS buffer size, matching ESP32
+#undef TCP_SND_QUEUELEN
+#define TCP_SND_QUEUELEN              {{ TCP_SND_QUEUELEN }}
+
+#undef MEMP_NUM_TCP_SEG
+#define MEMP_NUM_TCP_SEG              {{ MEMP_NUM_TCP_SEG }}
+
+// Packet buffer pool: 16 matches ESP32 (down from 24)
+#undef PBUF_POOL_SIZE
+#define PBUF_POOL_SIZE                {{ PBUF_POOL_SIZE }}
+
+// PCB pools: sized to actual component needs via socket.get_socket_counts()
+#undef MEMP_NUM_TCP_PCB
+#define MEMP_NUM_TCP_PCB              {{ MEMP_NUM_TCP_PCB }}
+
+#undef MEMP_NUM_TCP_PCB_LISTEN
+#define MEMP_NUM_TCP_PCB_LISTEN       {{ MEMP_NUM_TCP_PCB_LISTEN }}
+
+#undef MEMP_NUM_UDP_PCB
+#define MEMP_NUM_UDP_PCB              {{ MEMP_NUM_UDP_PCB }}
+
+// Listen backlog: match component needs
+#undef TCP_DEFAULT_LISTEN_BACKLOG
+#define TCP_DEFAULT_LISTEN_BACKLOG    {{ MEMP_NUM_TCP_PCB_LISTEN }}

esphome/components/wifi/__init__.py

@@ -289,12 +289,12 @@ def final_validate(config):
 def _consume_wifi_sockets(config: ConfigType) -> ConfigType:
     """Register UDP PCBs used internally by lwIP for DHCP and DNS.
 
-    Only needed on LibreTiny where we directly set MEMP_NUM_UDP_PCB (the raw
+    Needed on LibreTiny and RP2040 where we directly set MEMP_NUM_UDP_PCB (the
-    PCB pool shared by both application sockets and lwIP internals like DHCP/DNS).
+    raw PCB pool shared by both application sockets and lwIP internals like
-    On ESP32, CONFIG_LWIP_MAX_SOCKETS only controls the POSIX socket layer —
+    DHCP/DNS). On ESP32, CONFIG_LWIP_MAX_SOCKETS only controls the POSIX socket
-    DHCP/DNS use raw udp_new() which bypasses it entirely.
+    layer — DHCP/DNS use raw udp_new() which bypasses it entirely.
     """
-    if not (CORE.is_bk72xx or CORE.is_rtl87xx or CORE.is_ln882x):
+    if not (CORE.is_bk72xx or CORE.is_rtl87xx or CORE.is_ln882x or CORE.is_rp2040):
         return config
     from esphome.components import socket
 

script/stress_test_connect.py

@@ -0,0 +1,84 @@
+"""Rapid connect/disconnect stress test for ESPHome native API."""
+
+import asyncio
+import sys
+import time
+
+from aioesphomeapi import APIClient
+
+HOST = "192.168.1.100"
+PORT = 6053
+PASSWORD = ""
+NOISE_PSK = None
+ITERATIONS = 500
+CONCURRENCY = 4  # simultaneous connection attempts
+
+
+async def connect_disconnect(client_id: int, iteration: int) -> tuple[int, bool, str]:
+    """Connect and immediately disconnect."""
+    cli = APIClient(HOST, PORT, PASSWORD, noise_psk=NOISE_PSK)
+    try:
+        await asyncio.wait_for(cli.connect(login=True), timeout=10)
+        await cli.disconnect()
+        return iteration, True, ""
+    except Exception as e:
+        return (
+            iteration,
+            False,
+            f"client{client_id} iter{iteration}: {type(e).__name__}: {e}",
+        )
+    finally:
+        await cli.disconnect(force=True)
+
+
+async def main() -> None:
+    iterations = int(sys.argv[1]) if len(sys.argv) > 1 else ITERATIONS
+    concurrency = int(sys.argv[2]) if len(sys.argv) > 2 else CONCURRENCY
+
+    print(f"Stress testing {HOST}:{PORT}")
+    print(f"Iterations: {iterations}, Concurrency: {concurrency}")
+    print()
+
+    success = 0
+    fail = 0
+    errors: list[str] = []
+    start = time.monotonic()
+
+    sem = asyncio.Semaphore(concurrency)
+
+    async def run(client_id: int, iteration: int) -> tuple[int, bool, str]:
+        async with sem:
+            return await connect_disconnect(client_id, iteration)
+
+    tasks = [asyncio.create_task(run(i % concurrency, i)) for i in range(iterations)]
+
+    for coro in asyncio.as_completed(tasks):
+        iteration, ok, err = await coro
+        if ok:
+            success += 1
+        else:
+            fail += 1
+            errors.append(err)
+        total = success + fail
+        if total % 10 == 0 or not ok:
+            elapsed = time.monotonic() - start
+            rate = total / elapsed if elapsed > 0 else 0
+            print(f"[{total}/{iterations}] ok={success} fail={fail} ({rate:.1f}/s)")
+            if err:
+                print(f"  ERROR: {err}")
+
+    elapsed = time.monotonic() - start
+    print()
+    print(f"Done in {elapsed:.1f}s")
+    print(f"Success: {success}, Failed: {fail}, Rate: {iterations / elapsed:.1f}/s")
+
+    if errors:
+        print("\nLast 10 errors:")
+        for e in errors[-10:]:
+            print(f"  {e}")
+
+    sys.exit(1 if fail > 0 else 0)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())