From c6d423159c9f48f86a81f2bde29fac013f8b9852 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 25 Aug 2026 00:26:18 -0500 Subject: [PATCH] [esp32] Only build the mbedTLS certificate bundle when a component needs it (#18747) --- esphome/components/audio/__init__.py | 3 + esphome/components/esp32/__init__.py | 128 ++++++++++++------ esphome/components/esp32/const.py | 1 + esphome/components/http_request/__init__.py | 4 +- .../certificate_bundle_arduino_tls.yaml | 9 ++ .../esp32/config/certificate_bundle_full.yaml | 9 ++ .../certificate_bundle_http_request.yaml | 14 ++ .../config/certificate_bundle_sdkconfig.yaml | 9 ++ tests/component_tests/esp32/test_esp32.py | 58 ++++++++ tests/components/esp32/test.esp32-idf.yaml | 2 +- 10 files changed, 190 insertions(+), 47 deletions(-) create mode 100644 tests/component_tests/esp32/config/certificate_bundle_arduino_tls.yaml create mode 100644 tests/component_tests/esp32/config/certificate_bundle_full.yaml create mode 100644 tests/component_tests/esp32/config/certificate_bundle_http_request.yaml create mode 100644 tests/component_tests/esp32/config/certificate_bundle_sdkconfig.yaml diff --git a/esphome/components/audio/__init__.py b/esphome/components/audio/__init__.py index 277df0506a..2a5304be77 100644 --- a/esphome/components/audio/__init__.py +++ b/esphome/components/audio/__init__.py @@ -7,6 +7,7 @@ from esphome.components.esp32 import ( add_idf_component, add_idf_sdkconfig_option, include_builtin_idf_component, + require_certificate_bundle, ) import esphome.config_validation as cv from esphome.const import ( @@ -335,6 +336,8 @@ def _emit_memory_pair(value: str | None, psram_key: str, internal_key: str) -> N async def to_code(config: ConfigType) -> None: # Re-enable ESP-IDF's HTTP client (excluded by default to save compile time) include_builtin_idf_component("esp_http_client") + # HTTPS streams verify the server against the root certificate bundle + require_certificate_bundle() add_idf_component( name="esphome/esp-audio-libs", diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index cde0cfd68b..fca63e9a25 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -65,6 +65,7 @@ from .boards import BOARDS, STANDARD_BOARDS from .const import ( KEY_ARDUINO_LIBRARIES, KEY_BOARD, + KEY_CERT_BUNDLE, KEY_COMPONENTS, KEY_ESP32, KEY_EXCLUDE_COMPONENTS, @@ -343,6 +344,10 @@ ARDUINO_LIBRARY_IDF_COMPONENTS: dict[str, tuple[str, ...]] = { "Zigbee": ("espressif__esp-zigbee-lib", "espressif__esp-zboss-lib"), } +# Arduino libraries whose sources reference esp_crt_bundle_attach without a +# CONFIG_MBEDTLS_CERTIFICATE_BUNDLE guard, so enabling them needs the bundle. +ARDUINO_LIBRARIES_NEEDING_CERT_BUNDLE = frozenset({"NetworkClientSecure"}) + # Arduino library to Arduino library dependencies # When enabling one library, also enable its dependencies # Kconfig "select" statements don't work with CONFIG_ARDUINO_SELECTIVE_COMPILATION @@ -644,6 +649,17 @@ class RawSdkconfigValue: SdkconfigValueType = bool | int | HexInt | str | RawSdkconfigValue +def set_idf_sdkconfig_default(name: str, value: SdkconfigValueType) -> None: + """Set an sdkconfig option unless it is already set. + + For the FINAL priority reconcile jobs: they run after every to_code, + including the user's sdkconfig_options, and must not override an + existing value. + """ + if name not in CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS]: + add_idf_sdkconfig_option(name, value) + + def add_idf_sdkconfig_option(name: str, value: SdkconfigValueType): """Set an esp-idf sdkconfig value.""" CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS][name] = value @@ -788,6 +804,10 @@ def _enable_arduino_library(name: str) -> None: # Also enable any required IDF components for idf_component in ARDUINO_LIBRARY_IDF_COMPONENTS.get(name, ()): include_builtin_idf_component(idf_component) + if not ARDUINO_LIBRARIES_NEEDING_CERT_BUNDLE.isdisjoint( + {name, *ARDUINO_LIBRARY_DEPENDENCIES.get(name, ())} + ): + require_certificate_bundle() def add_extra_script(stage: str, filename: str, path: Path): @@ -1735,6 +1755,16 @@ def require_vfs_termios() -> None: CORE.data[KEY_VFS_TERMIOS_REQUIRED] = True +def require_certificate_bundle() -> None: + """Enable the mbedTLS root certificate bundle for this build. + + The bundle is off by default; components that verify TLS server + certificates (http_request, audio streaming) call this so the bundle is + compiled and gen_crt_bundle runs only when something uses it. + """ + CORE.data[KEY_ESP32][KEY_CERT_BUNDLE] = True + + def require_full_certificate_bundle() -> None: """Request the full certificate bundle instead of the common-CAs-only bundle. @@ -1744,6 +1774,7 @@ def require_full_certificate_bundle() -> None: Call this from components that need to connect to services using uncommon CAs. """ + require_certificate_bundle() CORE.data[KEY_ESP32][KEY_FULL_CERT_BUNDLE] = True @@ -2218,6 +2249,31 @@ async def _set_libc_picolibc_newlib_compat() -> None: ) +@coroutine_with_priority(CoroPriority.FINAL) +async def _reconcile_certificate_bundle_sdkconfig() -> None: + """Enable the mbedTLS certificate bundle only when something asked for it. + + Runs at FINAL priority so every require_certificate_bundle() call has + happened. Without a request the bundle is disabled, which skips + esp_crt_bundle.c, the gen_crt_bundle step and the x509_crt_bundle.S embed. + A user-supplied sdkconfig_options value takes precedence. + """ + data = CORE.data[KEY_ESP32] + enabled = data.get(KEY_CERT_BUNDLE, False) + set_idf_sdkconfig_default("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE", enabled) + if not enabled: + return + # Use CMN (common CAs) bundle by default to save ~51KB flash + # CMN covers CAs with >1% market share (~99% of websites) + # Components needing uncommon CAs can call require_full_certificate_bundle() + use_full_bundle = data.get(KEY_FULL_CERT_BUNDLE, False) + set_idf_sdkconfig_default( + "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL", use_full_bundle + ) + if not use_full_bundle: + set_idf_sdkconfig_default("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN", True) + + @coroutine_with_priority(CoroPriority.FINAL) async def _reconcile_network_sdkconfig() -> None: """Reconcile WiFi/Ethernet/Bluetooth/coexistence sdkconfig flags. @@ -2229,37 +2285,31 @@ async def _reconcile_network_sdkconfig() -> None: always takes precedence. """ net = CORE.data[KEY_ESP32].get(KEY_NETWORK_SDKCONFIG, NetworkSdkconfigData()) - opts = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] is_arduino = CORE.using_arduino - def set_opt(name: str, value: SdkconfigValueType) -> None: - # User sdkconfig_options (applied during to_code) win. - if name not in opts: - add_idf_sdkconfig_option(name, value) - # Bluetooth: only ever enable when requested. The IDF default is off. # According to the IDF docs, only one of 4.2 or 5.0 should be enabled. if net.bluetooth: - set_opt("CONFIG_BT_ENABLED", True) - set_opt("CONFIG_BT_BLE_42_FEATURES_SUPPORTED", True) - set_opt("CONFIG_BT_BLE_50_FEATURES_SUPPORTED", False) + set_idf_sdkconfig_default("CONFIG_BT_ENABLED", True) + set_idf_sdkconfig_default("CONFIG_BT_BLE_42_FEATURES_SUPPORTED", True) + set_idf_sdkconfig_default("CONFIG_BT_BLE_50_FEATURES_SUPPORTED", False) # WiFi stack: disable only when Ethernet is present and WiFi is not. WiFi # relies on the IDF default (enabled), so it is never written True here. wifi_disabled = net.ethernet and not net.wifi if wifi_disabled: - set_opt("CONFIG_ESP_WIFI_ENABLED", False) + set_idf_sdkconfig_default("CONFIG_ESP_WIFI_ENABLED", False) # Software coexistence: enable when requested (the schema only allows it # alongside WiFi). Disable only in the Ethernet-without-WiFi case. if net.software_coexistence: - set_opt("CONFIG_SW_COEXIST_ENABLE", True) + set_idf_sdkconfig_default("CONFIG_SW_COEXIST_ENABLE", True) elif wifi_disabled: - set_opt("CONFIG_SW_COEXIST_ENABLE", False) + set_idf_sdkconfig_default("CONFIG_SW_COEXIST_ENABLE", False) # SoftAP support: drop it when WiFi is used without AP mode (IDF only). if not is_arduino and net.wifi and not net.wifi_ap: - set_opt("CONFIG_ESP_WIFI_SOFTAP_SUPPORT", False) + set_idf_sdkconfig_default("CONFIG_ESP_WIFI_SOFTAP_SUPPORT", False) # LWIP DHCP server: a WiFi-AP-mode / enable_lwip_dhcp_server concern (not # coexistence). Disable when WiFi has no AP (IDF) or the enable_lwip_dhcp_server @@ -2270,7 +2320,7 @@ async def _reconcile_network_sdkconfig() -> None: if ( wifi_wants_dhcps_off or dhcp_server_disabled_by_option ) and not arduino_eth_exclusion: - set_opt("CONFIG_LWIP_DHCPS", False) + set_idf_sdkconfig_default("CONFIG_LWIP_DHCPS", False) @coroutine_with_priority(CoroPriority.FINAL) @@ -2295,29 +2345,24 @@ async def _reconcile_vfs_fatfs_sdkconfig( """Reconcile VFS/FATFS sdkconfig flags after all require_*() calls; user sdkconfig_options win.""" opts = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] - def set_opt(name: str, value: SdkconfigValueType) -> None: - # User sdkconfig_options (applied during to_code) win. - if name not in opts: - add_idf_sdkconfig_option(name, value) - # USB Serial JTAG VFS needs termios (require_vfs_termios(), e.g. logger). ~1.8KB flash when off. if CORE.data.get(KEY_VFS_TERMIOS_REQUIRED, False): - set_opt("CONFIG_VFS_SUPPORT_TERMIOS", True) + set_idf_sdkconfig_default("CONFIG_VFS_SUPPORT_TERMIOS", True) else: - set_opt("CONFIG_VFS_SUPPORT_TERMIOS", not disable_vfs_termios) + set_idf_sdkconfig_default("CONFIG_VFS_SUPPORT_TERMIOS", not disable_vfs_termios) # VFS select is only needed for UART/eventfd fds (require_vfs_select(), e.g. openthread); # sockets use lwip_select() either way. ~2.7KB flash when off. if CORE.data.get(KEY_VFS_SELECT_REQUIRED, False): - set_opt("CONFIG_VFS_SUPPORT_SELECT", True) + set_idf_sdkconfig_default("CONFIG_VFS_SUPPORT_SELECT", True) else: - set_opt("CONFIG_VFS_SUPPORT_SELECT", not disable_vfs_select) + set_idf_sdkconfig_default("CONFIG_VFS_SUPPORT_SELECT", not disable_vfs_select) # Directory functions: opendir/readdir/mkdir etc. (require_vfs_dir()). ~0.5KB flash when off. if CORE.data.get(KEY_VFS_DIR_REQUIRED, False): - set_opt("CONFIG_VFS_SUPPORT_DIR", True) + set_idf_sdkconfig_default("CONFIG_VFS_SUPPORT_DIR", True) else: - set_opt("CONFIG_VFS_SUPPORT_DIR", not disable_vfs_dir) + set_idf_sdkconfig_default("CONFIG_VFS_SUPPORT_DIR", not disable_vfs_dir) # FATFS (require_fatfs()): LFN + one volume per esp_vfs_fat mount. Defaults only; # sdkconfig_options override. FATFS_LONG_FILENAMES is a Kconfig choice -- if the user set @@ -2330,15 +2375,15 @@ async def _reconcile_vfs_fatfs_sdkconfig( user_picked_lfn = any(k in opts for k in lfn_keys) if CORE.data[KEY_ESP32].get(KEY_FATFS_REQUIRED, False): if not user_picked_lfn: - set_opt("CONFIG_FATFS_LFN_NONE", False) - set_opt("CONFIG_FATFS_LFN_HEAP", True) - set_opt("CONFIG_FATFS_MAX_LFN", 255) - set_opt("CONFIG_FATFS_VOLUME_COUNT", 4) + set_idf_sdkconfig_default("CONFIG_FATFS_LFN_NONE", False) + set_idf_sdkconfig_default("CONFIG_FATFS_LFN_HEAP", True) + set_idf_sdkconfig_default("CONFIG_FATFS_MAX_LFN", 255) + set_idf_sdkconfig_default("CONFIG_FATFS_VOLUME_COUNT", 4) elif disable_fatfs: if not user_picked_lfn: - set_opt("CONFIG_FATFS_LFN_NONE", True) + set_idf_sdkconfig_default("CONFIG_FATFS_LFN_NONE", True) # Kconfig range is [1,10]; 0 gets clamped to the default. - set_opt("CONFIG_FATFS_VOLUME_COUNT", 1) + set_idf_sdkconfig_default("CONFIG_FATFS_VOLUME_COUNT", 1) @coroutine_with_priority(CoroPriority.FINAL - 1) @@ -2525,21 +2570,11 @@ async def to_code(config): ) add_idf_sdkconfig_option("CONFIG_MBEDTLS_PSK_MODES", True) - add_idf_sdkconfig_option("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE", True) cg.add_build_flag("-Wno-nonnull-compare") - # Use CMN (common CAs) bundle by default to save ~51KB flash - # CMN covers CAs with >1% market share (~99% of websites) - # Components needing uncommon CAs can call require_full_certificate_bundle() - use_full_bundle = conf[CONF_ADVANCED].get( - CONF_USE_FULL_CERTIFICATE_BUNDLE, False - ) or CORE.data[KEY_ESP32].get(KEY_FULL_CERT_BUNDLE, False) - add_idf_sdkconfig_option( - "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL", use_full_bundle - ) - if not use_full_bundle: - add_idf_sdkconfig_option("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN", True) + if conf[CONF_ADVANCED].get(CONF_USE_FULL_CERTIFICATE_BUNDLE, False): + require_full_certificate_bundle() add_idf_sdkconfig_option(f"CONFIG_IDF_TARGET_{variant}", True) add_idf_sdkconfig_option( @@ -2929,6 +2964,9 @@ async def to_code(config): # FINAL priority: runs after every network/coexistence request_*() call CORE.add_job(_reconcile_network_sdkconfig) + # FINAL priority: runs after every require_certificate_bundle() call + CORE.add_job(_reconcile_certificate_bundle_sdkconfig) + # FINAL: require_*() calls can come from to_code at or below this priority, so an # inline read would be iteration-order-dependent; reconcile once after every job ran. CORE.add_job( @@ -2956,6 +2994,10 @@ async def to_code(config): for name, value in conf[CONF_SDKCONFIG_OPTIONS].items(): add_idf_sdkconfig_option(name, RawSdkconfigValue(value)) + # A bundle forced on through sdkconfig_options is a request like any other, + # so it still gets the CMN variant pinned. + if conf[CONF_SDKCONFIG_OPTIONS].get("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE") == "y": + require_certificate_bundle() # Components from YAML are added in a separate coroutine with FINAL priority # Schedule it to run after all other components diff --git a/esphome/components/esp32/const.py b/esphome/components/esp32/const.py index 09f458c64b..e7d8a66e7a 100644 --- a/esphome/components/esp32/const.py +++ b/esphome/components/esp32/const.py @@ -27,6 +27,7 @@ KEY_REFRESH = "refresh" KEY_PATH = "path" KEY_SUBMODULES = "submodules" KEY_EXTRA_BUILD_FILES = "extra_build_files" +KEY_CERT_BUNDLE = "cert_bundle" KEY_FULL_CERT_BUNDLE = "full_cert_bundle" KEY_NETWORK_SDKCONFIG = "network_sdkconfig" diff --git a/esphome/components/http_request/__init__.py b/esphome/components/http_request/__init__.py index 8a5aae022a..2abf097aec 100644 --- a/esphome/components/http_request/__init__.py +++ b/esphome/components/http_request/__init__.py @@ -196,9 +196,7 @@ async def to_code(config: ConfigType) -> None: # framework: # advanced: # use_full_certificate_bundle: true - esp32.add_idf_sdkconfig_option( - "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE", True - ) + esp32.require_certificate_bundle() esp32.add_idf_sdkconfig_option( "CONFIG_ESP_TLS_INSECURE", diff --git a/tests/component_tests/esp32/config/certificate_bundle_arduino_tls.yaml b/tests/component_tests/esp32/config/certificate_bundle_arduino_tls.yaml new file mode 100644 index 0000000000..68f9cf1d0f --- /dev/null +++ b/tests/component_tests/esp32/config/certificate_bundle_arduino_tls.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + libraries: + - NetworkClientSecure + +esp32: + board: esp32dev + framework: + type: arduino diff --git a/tests/component_tests/esp32/config/certificate_bundle_full.yaml b/tests/component_tests/esp32/config/certificate_bundle_full.yaml new file mode 100644 index 0000000000..179fc12f51 --- /dev/null +++ b/tests/component_tests/esp32/config/certificate_bundle_full.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + advanced: + use_full_certificate_bundle: true diff --git a/tests/component_tests/esp32/config/certificate_bundle_http_request.yaml b/tests/component_tests/esp32/config/certificate_bundle_http_request.yaml new file mode 100644 index 0000000000..b29e5de2bd --- /dev/null +++ b/tests/component_tests/esp32/config/certificate_bundle_http_request.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + password: "test_password" + +http_request: + verify_ssl: true diff --git a/tests/component_tests/esp32/config/certificate_bundle_sdkconfig.yaml b/tests/component_tests/esp32/config/certificate_bundle_sdkconfig.yaml new file mode 100644 index 0000000000..af44693806 --- /dev/null +++ b/tests/component_tests/esp32/config/certificate_bundle_sdkconfig.yaml @@ -0,0 +1,9 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + sdkconfig_options: + CONFIG_MBEDTLS_CERTIFICATE_BUNDLE: y diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index 7208318d3a..9925887c66 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -17,6 +17,7 @@ from esphome.components.esp32 import ( VARIANT_ESP32, VARIANTS, NetworkSdkconfigData, + RawSdkconfigValue, _ota_downgrade_protection_errors, _reconcile_network_sdkconfig, _reconcile_vfs_fatfs_sdkconfig, @@ -292,6 +293,63 @@ def test_default_exclusions_reincluded_by_owning_components( assert "fatfs" in excluded +_BUNDLE_OPTIONS = ( + "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE", + "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN", + "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL", +) + + +@pytest.mark.parametrize( + ("config_file", "expected"), + [ + pytest.param("exclusion_reincludes.yaml", (False, None, None), id="no_tls"), + pytest.param( + "certificate_bundle_http_request.yaml", + (True, True, False), + id="http_request", + ), + pytest.param( + "exclusion_reincludes_http_request.yaml", + (False, None, None), + id="http_request_no_verify", + ), + pytest.param( + "certificate_bundle_full.yaml", (True, None, True), id="full_option" + ), + pytest.param( + "certificate_bundle_arduino_tls.yaml", + (True, True, False), + id="arduino_network_client_secure", + ), + ], +) +def test_certificate_bundle_sdkconfig( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + config_file: str, + expected: tuple[bool | None, ...], +) -> None: + """The bundle and its CMN/FULL variant are written only when requested.""" + generate_main(component_config_path(config_file)) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert tuple(sdkconfig.get(name) for name in _BUNDLE_OPTIONS) == expected + + +def test_user_sdkconfig_certificate_bundle_wins( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """A raw sdkconfig_options bundle setting is kept and still pins CMN.""" + generate_main(component_config_path("certificate_bundle_sdkconfig.yaml")) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + value = sdkconfig["CONFIG_MBEDTLS_CERTIFICATE_BUNDLE"] + assert isinstance(value, RawSdkconfigValue) + assert value.value == "y" + assert sdkconfig.get("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN") is True + assert sdkconfig.get("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL") is False + + def test_execute_from_psram_s3_sdkconfig( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], diff --git a/tests/components/esp32/test.esp32-idf.yaml b/tests/components/esp32/test.esp32-idf.yaml index 6b77a4e171..523e614e24 100644 --- a/tests/components/esp32/test.esp32-idf.yaml +++ b/tests/components/esp32/test.esp32-idf.yaml @@ -7,7 +7,7 @@ esp32: enable_lwip_mdns_queries: true enable_lwip_bridge_interface: true disable_libc_locks_in_iram: false # Test explicit opt-out of RAM optimization - use_full_certificate_bundle: false # Test CMN bundle (default) + use_full_certificate_bundle: false # Bundle stays off without a component that needs it include_builtin_idf_components: - freertos # Test escape hatch (freertos is always included anyway) enable_full_printf: false