From 482721d1ce3e31696eb7f9a95c9144a70a49f61f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 21 Feb 2026 11:09:33 -0600 Subject: [PATCH] [libretiny] Tune oversized lwIP defaults for ESPHome The Beken/RTL SDKs ship lwIP defaults tuned for a general-purpose WiFi SoC: TCP_SND_BUF=10*MSS (14.6KB), MEM_SIZE=32KB, 12 TCP + 22 UDP sockets. These waste significant RAM on memory-constrained chips. ESPAsyncWebServer allocates malloc(tcp_sndbuf()) per response chunk, and at 14.6KB this causes OOM on BK7231N. This tunes lwIP to match ESP32/ESP8266 ranges: - TCP_SND_BUF/TCP_WND: 4*MSS (was 10*MSS) - MEM_SIZE: 12KB BK72XX / 16KB RTL,LN (was 32KB) - Socket counts: dynamic from component registrations - All derived pool sizes adjusted accordingly Also splits socket consumer tracking into TCP/UDP, adds get_socket_counts() API, and updates ESP32 to use it. Closes https://github.com/esphome/esphome/issues/14095 --- esphome/components/esp32/__init__.py | 48 ++++---- esphome/components/libretiny/__init__.py | 110 +++++++++++++++++- esphome/components/mdns/__init__.py | 2 +- esphome/components/socket/__init__.py | 64 +++++++++- esphome/components/udp/__init__.py | 2 +- .../socket/test_wake_loop_threadsafe.py | 8 +- 6 files changed, 190 insertions(+), 44 deletions(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 06677006ea..ac73bf449e 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1258,21 +1258,13 @@ def _configure_lwip_max_sockets(conf: dict) -> None: This function runs in to_code() after all components have registered their socket needs. User-provided sdkconfig_options take precedence. """ - from esphome.components.socket import KEY_SOCKET_CONSUMERS + from esphome.components.socket import get_socket_counts # Check if user manually specified CONFIG_LWIP_MAX_SOCKETS user_max_sockets = conf[CONF_SDKCONFIG_OPTIONS].get("CONFIG_LWIP_MAX_SOCKETS") - socket_consumers: dict[str, int] = CORE.data.get(KEY_SOCKET_CONSUMERS, {}) - total_sockets = sum(socket_consumers.values()) - - # Early return if no sockets registered and no user override - if total_sockets == 0 and user_max_sockets is None: - return - - components_list = ", ".join( - f"{name}={count}" for name, count in sorted(socket_consumers.items()) - ) + tcp_sockets, udp_sockets = get_socket_counts() + total_sockets = tcp_sockets + udp_sockets # User specified their own value - respect it but warn if insufficient if user_max_sockets is not None: @@ -1281,22 +1273,21 @@ def _configure_lwip_max_sockets(conf: dict) -> None: user_max_sockets, ) - # Warn if user's value is less than what components need - if total_sockets > 0: - user_sockets_int = 0 - with contextlib.suppress(ValueError, TypeError): - user_sockets_int = int(user_max_sockets) + user_sockets_int = 0 + with contextlib.suppress(ValueError, TypeError): + user_sockets_int = int(user_max_sockets) - if user_sockets_int < total_sockets: - _LOGGER.warning( - "CONFIG_LWIP_MAX_SOCKETS is set to %d but your configuration " - "needs %d sockets (registered: %s). You may experience socket " - "exhaustion errors. Consider increasing to at least %d.", - user_sockets_int, - total_sockets, - components_list, - total_sockets, - ) + if user_sockets_int < total_sockets: + _LOGGER.warning( + "CONFIG_LWIP_MAX_SOCKETS is set to %d but your configuration " + "needs %d sockets (%d TCP + %d UDP). You may experience " + "socket exhaustion errors. Consider increasing to at least %d.", + user_sockets_int, + total_sockets, + tcp_sockets, + udp_sockets, + total_sockets, + ) # User's value already added via sdkconfig_options processing return @@ -1307,9 +1298,10 @@ def _configure_lwip_max_sockets(conf: dict) -> None: log_level = logging.INFO if max_sockets > DEFAULT_MAX_SOCKETS else logging.DEBUG _LOGGER.log( log_level, - "Setting CONFIG_LWIP_MAX_SOCKETS to %d (registered: %s)", + "Setting CONFIG_LWIP_MAX_SOCKETS to %d (%d TCP + %d UDP)", max_sockets, - components_list, + tcp_sockets, + udp_sockets, ) add_idf_sdkconfig_option("CONFIG_LWIP_MAX_SOCKETS", max_sockets) diff --git a/esphome/components/libretiny/__init__.py b/esphome/components/libretiny/__init__.py index 01445da7ee..5804421f9e 100644 --- a/esphome/components/libretiny/__init__.py +++ b/esphome/components/libretiny/__init__.py @@ -275,6 +275,103 @@ BASE_SCHEMA.add_extra(_detect_variant) BASE_SCHEMA.add_extra(_update_core_data) +def _configure_lwip(config: dict) -> None: + """Configure lwIP options for LibreTiny platforms. + + The Beken/RTL SDKs ship with lwIP defaults tuned for a general-purpose + WiFi SoC, wildly oversized for ESPHome's IoT use case. This causes OOM + on memory-constrained chips like BK7231N. + + See https://github.com/esphome/esphome/issues/14095 + + Comparison of SDK defaults vs ESPHome targets: + + TCP_MSS is 1460 on BK/RTL/LN, 1440 on ESP32, 1460 on ESP8266. + + Setting ESP8266 ESP32 BK/RTL SDK New + ───────────────────────────────────────────────────────────── + TCP_SND_BUF 2×MSS 4×MSS 10×MSS 4×MSS + TCP_WND 4×MSS 4×MSS 10×MSS 4×MSS + MEM_SIZE 1.6KB N/A* 32KB 12KB/16KB + MAX_SOCKETS_TCP 5 16 12 dynamic + MAX_SOCKETS_UDP 4 16 22 dynamic + TCP_SND_QUEUELEN ~8 17 20 17 + MEMP_NUM_TCP_SEG 10 16 40 17 + MEMP_NUM_TCP_PCB 5 16 12 =TCP + MEMP_NUM_UDP_PCB 4 16 24/7** =UDP+2 + MEMP_NUM_NETCONN 0 10 38 =sum + MEMP_NUM_NETBUF 0 2 16 4 + MEMP_NUM_TCPIP_MSG_INPKT 4 8 16 8 + + MEM_SIZE: 12KB for BK72XX, 16KB for RTL87XX/LN882H. + * ESP32 uses MEM_LIBC_MALLOC=1 (no dedicated lwIP heap). + ** MEMP_NUM_UDP_PCB: BK base=24 (22+2), RTL/LN platform override=7. + "dynamic" = auto-calculated from component socket registrations via + socket.get_socket_counts() with minimums of 10 TCP / 6 UDP. + """ + from esphome.components.socket import ( + MIN_TCP_SOCKETS, + MIN_UDP_SOCKETS, + get_socket_counts, + ) + + raw_tcp, raw_udp = get_socket_counts() + # Apply platform minimums — ensure headroom for ESPHome's needs + tcp_sockets = max(MIN_TCP_SOCKETS, raw_tcp) + udp_sockets = max(MIN_UDP_SOCKETS, raw_udp) + listening_tcp = 4 + + # TCP_SND_BUF / TCP_WND: 4×MSS matches ESP32 and ESP8266 TCP_WND + # SDK default is 10×MSS=14,600 — ESPAsyncWebServer malloc(tcp_sndbuf()) + # per response chunk causes OOM at that size on BK7231N + tcp_snd_buf = "(4*TCP_MSS)" # 4×1460=5,840 (SDK: 10×1460=14,600) + tcp_wnd = "(4*TCP_MSS)" # 4×1460=5,840 (SDK: 10×1460=14,600) + + # lwIP heap — BK72XX has less headroom than RTL/LN (SDK: 32KB for both) + mem_size = 12288 if CORE.is_bk72xx else 16384 + + # TCP_SND_QUEUELEN: max pbufs queued for send buffer + # ESP-IDF formula: (4 * TCP_SND_BUF + (TCP_MSS - 1)) / TCP_MSS + # With 4×MSS: (4*5840 + 1459) / 1460 = 17 — match ESP32 + tcp_snd_queuelen = 17 # SDK: 20, ESP32: 17 + # MEMP_NUM_TCP_SEG: segment pool, must be >= TCP_SND_QUEUELEN (lwIP sanity check) + memp_num_tcp_seg = tcp_snd_queuelen # SDK: 40 + + lwip_opts: list[str] = [ + # Disable statistics — not needed for production, saves RAM + "LWIP_STATS=0", # SDK: 1 + "MEM_STATS=0", # SDK: 1 + "MEMP_STATS=0", # SDK: 1 + # TCP send buffer — ESPAsyncWebServer allocates malloc(tcp_sndbuf()) + # per response chunk. At 14.6KB this causes OOM on BK7231N (#14095) + f"TCP_SND_BUF={tcp_snd_buf}", # SDK: 10×MSS (14,600) + # TCP receive window — match send buffer ratio (ESP8266: 4×MSS) + f"TCP_WND={tcp_wnd}", # SDK: 10×MSS (14,600) + # Socket counts — auto-calculated from component registrations + # API=4 TCP, web_server=6 TCP, OTA=1 TCP, mDNS=2 UDP, etc. + f"MAX_SOCKETS_TCP={tcp_sockets}", # SDK: 12 + f"MAX_SOCKETS_UDP={udp_sockets}", # SDK: 22 + # Listening sockets — ESPHome needs API + web_server + OTA at most + f"MAX_LISTENING_SOCKETS_TCP={listening_tcp}", # SDK: 4 + # lwIP heap — SDK allocates 32KB, ESPHome needs far less + f"MEM_SIZE={mem_size}", # SDK: 32,768 + # Queued segment limits — derived from 4×MSS buffer size + f"TCP_SND_QUEUELEN={tcp_snd_queuelen}", # SDK: 20, match ESP32 + f"MEMP_NUM_TCP_SEG={memp_num_tcp_seg}", # SDK: 40, must be >= queuelen + # PCB pools — 1:1 with socket counts + f"MEMP_NUM_TCP_PCB={tcp_sockets}", # SDK: 12 + # UDP PCB pool — +2 for lwIP internal use (DHCP, DNS) + f"MEMP_NUM_UDP_PCB={udp_sockets + 2}", # SDK: 24 (22+2) + # Netconn pool — sum of all socket types + f"MEMP_NUM_NETCONN={tcp_sockets + listening_tcp + udp_sockets}", # SDK: 38 + # Netbuf pool — ESP8266 uses 0, conservative value + "MEMP_NUM_NETBUF=4", # SDK: 16 + # Inbound message pool — between ESP8266 (4) and SDK (16) + "MEMP_NUM_TCPIP_MSG_INPKT=8", # SDK: 16 + ] + cg.add_platformio_option("custom_options.lwip", lwip_opts) + + # pylint: disable=use-dict-literal async def component_to_code(config): var = cg.new_Pvariable(config[CONF_ID]) @@ -389,11 +486,12 @@ async def component_to_code(config): "custom_options.sys_config#h", _BK7231N_SYS_CONFIG_OPTIONS ) - # Disable LWIP statistics to save RAM - not needed in production - # Must explicitly disable all sub-stats to avoid redefinition warnings - cg.add_platformio_option( - "custom_options.lwip", - ["LWIP_STATS=0", "MEM_STATS=0", "MEMP_STATS=0"], - ) + # Tune lwIP for ESPHome's actual needs. + # The SDK defaults (TCP_SND_BUF=10*MSS, MAX_SOCKETS_TCP=12, MEM_SIZE=32KB) + # are wildly oversized for an IoT device. ESPAsyncWebServer allocates + # malloc(tcp_sndbuf()) per response chunk — at 14.6KB this causes silent + # OOM on memory-constrained chips like BK7231N. + # See https://github.com/esphome/esphome/issues/14095 + _configure_lwip(config) await cg.register_component(var, config) diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index f87f929615..c7fb28e766 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -56,7 +56,7 @@ def _consume_mdns_sockets(config: ConfigType) -> ConfigType: from esphome.components import socket # mDNS needs 2 sockets (IPv4 + IPv6 multicast) - socket.consume_sockets(2, "mdns")(config) + socket.consume_sockets(2, "mdns", socket.SOCKET_UDP)(config) return config diff --git a/esphome/components/socket/__init__.py b/esphome/components/socket/__init__.py index e364da78f8..5ece7a9795 100644 --- a/esphome/components/socket/__init__.py +++ b/esphome/components/socket/__init__.py @@ -1,9 +1,13 @@ from collections.abc import Callable, MutableMapping +from enum import StrEnum +import logging import esphome.codegen as cg import esphome.config_validation as cv from esphome.core import CORE +_LOGGER = logging.getLogger(__name__) + CODEOWNERS = ["@esphome/core"] CONF_IMPLEMENTATION = "implementation" @@ -13,33 +17,85 @@ IMPLEMENTATION_BSD_SOCKETS = "bsd_sockets" # Socket tracking infrastructure # Components register their socket needs and platforms read this to configure appropriately -KEY_SOCKET_CONSUMERS = "socket_consumers" +KEY_SOCKET_CONSUMERS_TCP = "socket_consumers_tcp" +KEY_SOCKET_CONSUMERS_UDP = "socket_consumers_udp" + +# Recommended minimum socket counts to ensure headroom +# Platforms should apply these (or their own) on top of get_socket_counts() +MIN_TCP_SOCKETS = 10 +MIN_UDP_SOCKETS = 6 # Wake loop threadsafe support tracking KEY_WAKE_LOOP_THREADSAFE_REQUIRED = "wake_loop_threadsafe_required" +class SocketType(StrEnum): + TCP = "tcp" + UDP = "udp" + + +# Legacy aliases +SOCKET_TCP = SocketType.TCP +SOCKET_UDP = SocketType.UDP + +_SOCKET_TYPE_KEYS = { + SocketType.TCP: KEY_SOCKET_CONSUMERS_TCP, + SocketType.UDP: KEY_SOCKET_CONSUMERS_UDP, +} + + def consume_sockets( - value: int, consumer: str + value: int, consumer: str, socket_type: SocketType = SocketType.TCP ) -> Callable[[MutableMapping], MutableMapping]: """Register socket usage for a component. Args: value: Number of sockets needed by the component consumer: Name of the component consuming the sockets + socket_type: Type of socket (SocketType.TCP or SocketType.UDP) Returns: A validator function that records the socket usage """ + typed_key = _SOCKET_TYPE_KEYS[socket_type] def _consume_sockets(config: MutableMapping) -> MutableMapping: - consumers: dict[str, int] = CORE.data.setdefault(KEY_SOCKET_CONSUMERS, {}) + consumers: dict[str, int] = CORE.data.setdefault(typed_key, {}) consumers[consumer] = consumers.get(consumer, 0) + value return config return _consume_sockets +def get_socket_counts() -> tuple[int, int]: + """Return (tcp_count, udp_count) of raw registered socket needs. + + Platforms call this during code generation to configure lwIP socket limits. + All components will have registered their needs by then. + + Platforms should apply their own minimums on top of these values. + """ + tcp_consumers = CORE.data.get(KEY_SOCKET_CONSUMERS_TCP, {}) + udp_consumers = CORE.data.get(KEY_SOCKET_CONSUMERS_UDP, {}) + tcp = sum(tcp_consumers.values()) + udp = sum(udp_consumers.values()) + + tcp_list = ", ".join( + f"{name}={count}" for name, count in sorted(tcp_consumers.items()) + ) + udp_list = ", ".join( + f"{name}={count}" for name, count in sorted(udp_consumers.items()) + ) + _LOGGER.debug( + "Socket counts: TCP=%d (%s), UDP=%d (%s)", + tcp, + tcp_list or "none", + udp, + udp_list or "none", + ) + return tcp, udp + + def require_wake_loop_threadsafe() -> None: """Mark that wake_loop_threadsafe support is required by a component. @@ -66,7 +122,7 @@ def require_wake_loop_threadsafe() -> None: CORE.data[KEY_WAKE_LOOP_THREADSAFE_REQUIRED] = True cg.add_define("USE_WAKE_LOOP_THREADSAFE") # Consume 1 socket for the shared wake notification socket - consume_sockets(1, "socket.wake_loop_threadsafe")({}) + consume_sockets(1, "socket.wake_loop_threadsafe", SOCKET_UDP)({}) CONFIG_SCHEMA = cv.Schema( diff --git a/esphome/components/udp/__init__.py b/esphome/components/udp/__init__.py index c9586d0b95..74a890076e 100644 --- a/esphome/components/udp/__init__.py +++ b/esphome/components/udp/__init__.py @@ -73,7 +73,7 @@ def _consume_udp_sockets(config: ConfigType) -> ConfigType: # UDP uses up to 2 sockets: 1 broadcast + 1 listen # Whether each is used depends on code generation, so register worst case - socket.consume_sockets(2, "udp")(config) + socket.consume_sockets(2, "udp", socket.SOCKET_UDP)(config) return config diff --git a/tests/components/socket/test_wake_loop_threadsafe.py b/tests/components/socket/test_wake_loop_threadsafe.py index b4bc95176d..a40b6068a8 100644 --- a/tests/components/socket/test_wake_loop_threadsafe.py +++ b/tests/components/socket/test_wake_loop_threadsafe.py @@ -66,12 +66,12 @@ def test_require_wake_loop_threadsafe__no_networking_does_not_consume_socket() - CORE.config = {"logger": {}} # Track initial socket consumer state - initial_consumers = CORE.data.get(socket.KEY_SOCKET_CONSUMERS, {}) + initial_udp = CORE.data.get(socket.KEY_SOCKET_CONSUMERS_UDP, {}) # Call require_wake_loop_threadsafe socket.require_wake_loop_threadsafe() # Verify no socket was consumed - consumers = CORE.data.get(socket.KEY_SOCKET_CONSUMERS, {}) - assert "socket.wake_loop_threadsafe" not in consumers - assert consumers == initial_consumers + udp_consumers = CORE.data.get(socket.KEY_SOCKET_CONSUMERS_UDP, {}) + assert "socket.wake_loop_threadsafe" not in udp_consumers + assert udp_consumers == initial_udp