[esp32] Grow the default IDF component exclusion list (#18536)

esphome/components/ac_dimmer/output.py

@@ -49,6 +49,12 @@ CONFIG_SCHEMA = cv.All(
 
 
 async def to_code(config):
+    if CORE.is_esp32:
+        from esphome.components.esp32 import include_builtin_idf_component
+
+        # Re-enable the gptimer driver (excluded by default to save compile time)
+        include_builtin_idf_component("esp_driver_gptimer")
+
     if CORE.is_esp8266:
         # ac_dimmer uses setTimer1Callback which requires the waveform generator
         from esphome.components.esp8266.const import require_waveform

esphome/components/esp32/__init__.py

@@ -204,18 +204,32 @@ COMPILER_OPTIMIZATIONS = {
 # ESP-IDF components excluded by default to reduce compile time.
 # Components can be re-enabled by calling include_builtin_idf_component() in to_code().
 #
-# Cannot be excluded (dependencies of required components):
-# - "console": espressif/mdns unconditionally depends on it
-# - "sdmmc": driver -> esp_driver_sdmmc -> sdmmc dependency chain
+# Note: excluding a component only removes it from the initial build set.
+# ESP-IDF's requirement expansion adds an excluded component back when any
+# component still in the build REQUIRES it (e.g. espressif/mdns pulls
+# "console" back in, esp_http_client pulls "tcp_transport" back in), so
+# exclusions here are safe for such components and simply become no-ops in
+# builds that need them.
 DEFAULT_EXCLUDED_IDF_COMPONENTS = (
+    "app_trace",  # CPU trace/SystemView support - unused by ESPHome
     "cmock",  # Unit testing mock framework - ESPHome doesn't use IDF's testing
+    "console",  # Console REPL - unused by ESPHome; espressif/mdns pulls it back when configured
     "driver",  # Legacy driver shim - only needed by esp32_touch, esp32_can for legacy headers
+    "esp-tls",  # TLS wrapper - re-included by http_request, mqtt, web_server_idf
     "esp_adc",  # ADC driver - only needed by adc component
+    "esp_driver_cam",  # Camera driver - the esp32-camera managed component pulls it back
     "esp_driver_dac",  # DAC driver - only needed by esp32_dac component
+    "esp_driver_gptimer",  # General purpose timer - re-included by ac_dimmer, opentherm, Arduino BLE libs
+    "esp_driver_i2c",  # I2C driver - re-included by i2c; esp32-camera pulls it back itself
     "esp_driver_i2s",  # I2S driver - only needed by i2s_audio component
+    "esp_driver_ledc",  # LEDC PWM driver - re-included by ledc; esp32-camera pulls it back itself
     "esp_driver_mcpwm",  # MCPWM driver - ESPHome doesn't use motor control PWM
     "esp_driver_pcnt",  # PCNT driver - only needed by pulse_counter, hlw8012 components
     "esp_driver_rmt",  # RMT driver - only needed by remote_transmitter/receiver, neopixelbus
+    "esp_driver_sdio",  # SDIO device-mode driver - unused by ESPHome
+    "esp_driver_sdm",  # Sigma-delta modulation driver - unused by ESPHome
+    "esp_driver_sdmmc",  # SD/MMC host driver - unused by ESPHome
+    "esp_driver_sdspi",  # SD-over-SPI driver - unused by ESPHome
     "esp_driver_touch_sens",  # Touch sensor driver - only needed by esp32_touch
     "esp_driver_twai",  # TWAI/CAN driver - only needed by esp32_can component
     "esp_eth",  # Ethernet driver - only needed by ethernet component
@@ -227,11 +241,16 @@ DEFAULT_EXCLUDED_IDF_COMPONENTS = (
     "esp_local_ctrl",  # Local control over HTTPS/BLE - ESPHome has native API
     "espcoredump",  # Core dump support - ESPHome has its own debug component
     "fatfs",  # FAT filesystem - ESPHome doesn't use filesystem storage
+    "json",  # cJSON library - ESPHome uses ArduinoJson instead
     "mqtt",  # ESP-IDF MQTT library - ESPHome has its own MQTT implementation
     "openthread",  # Thread protocol - only needed by openthread component
     "perfmon",  # Xtensa performance monitor - ESPHome has its own debug component
+    "protobuf-c",  # Protobuf runtime - only used by provisioning components (also excluded)
     "protocomm",  # Protocol communication for provisioning - unused by ESPHome
+    "rt",  # POSIX realtime extensions - unused by ESPHome
+    "sdmmc",  # SD/MMC protocol layer - only used by SD drivers and fatfs (also excluded)
     "spiffs",  # SPIFFS filesystem - ESPHome doesn't use filesystem storage (IDF only)
+    "tcp_transport",  # Transport layer - esp_http_client/mqtt pull it back when re-included
     "ulp",  # ULP coprocessor - not currently used by any ESPHome component
     "unity",  # Unit testing framework - ESPHome doesn't use IDF's testing
     "wear_levelling",  # Flash wear levelling for fatfs - unused since fatfs unused

esphome/components/http_request/__init__.py

@@ -170,8 +170,11 @@ async def to_code(config: ConfigType) -> None:
         cg.add(var.set_watchdog_timeout(timeout_ms))
 
     if CORE.is_esp32:
-        # Re-enable ESP-IDF's HTTP client (excluded by default to save compile time)
+        # Re-enable ESP-IDF's HTTP client (excluded by default to save compile time).
+        # esp-tls is re-enabled too because http_request includes <esp_tls.h>
+        # directly and esp_http_client only pulls it in as a private dependency.
         esp32.include_builtin_idf_component("esp_http_client")
+        esp32.include_builtin_idf_component("esp-tls")
 
         cg.add(var.set_buffer_size_rx(config[CONF_BUFFER_SIZE_RX]))
         cg.add(var.set_buffer_size_tx(config[CONF_BUFFER_SIZE_TX]))

esphome/components/i2c/__init__.py

@@ -284,6 +284,11 @@ FINAL_VALIDATE_SCHEMA = _final_validate
 async def to_code(config):
     cg.add_global(i2c_ns.using)
     cg.add_define("USE_I2C")
+    if CORE.is_esp32:
+        from esphome.components.esp32 import include_builtin_idf_component
+
+        # Re-enable the I2C driver (excluded by default to save compile time)
+        include_builtin_idf_component("esp_driver_i2c")
     if CORE.is_host:
         var = cg.new_Pvariable(config[CONF_ID])
         await cg.register_component(var, config)

esphome/components/ledc/output.py

@@ -3,6 +3,7 @@ from typing import Any
 from esphome import automation, pins
 import esphome.codegen as cg
 from esphome.components import output
+from esphome.components.esp32 import include_builtin_idf_component
 import esphome.config_validation as cv
 from esphome.const import (
     CONF_CHANNEL,
@@ -62,6 +63,9 @@ CONFIG_SCHEMA = output.FLOAT_OUTPUT_SCHEMA.extend(
 
 
 async def to_code(config: ConfigType) -> None:
+    # Re-enable the LEDC driver (excluded by default to save compile time)
+    include_builtin_idf_component("esp_driver_ledc")
+
     gpio = await cg.gpio_pin_expression(config[CONF_PIN])
     var = cg.new_Pvariable(config[CONF_ID], gpio)
     await cg.register_component(var, config)

esphome/components/mqtt/__init__.py

@@ -361,6 +361,8 @@ async def to_code(config):
             add_idf_component(name="espressif/mqtt", ref="1.0.0")
         else:
             include_builtin_idf_component("mqtt")
+        # mqtt_client.h drags in esp_tls types; esp-tls is excluded by default
+        include_builtin_idf_component("esp-tls")
 
     cg.add_define("USE_MQTT")
     cg.add_global(mqtt_ns.using)

esphome/components/nextion/display.py

@@ -290,7 +290,9 @@ async def to_code(config):
 
         if CORE.is_esp32:
             # Re-enable ESP-IDF's HTTP client (excluded by default to save compile time)
+            # and esp-tls, whose sdkconfig options below need the component present
             esp32.include_builtin_idf_component("esp_http_client")
+            esp32.include_builtin_idf_component("esp-tls")
             esp32.add_idf_sdkconfig_option("CONFIG_ESP_TLS_INSECURE", True)
             esp32.add_idf_sdkconfig_option(
                 "CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY", True

esphome/components/web_server_idf/__init__.py

@@ -1,4 +1,7 @@
-from esphome.components.esp32 import add_idf_sdkconfig_option
+from esphome.components.esp32 import (
+    add_idf_sdkconfig_option,
+    include_builtin_idf_component,
+)
 import esphome.config_validation as cv
 
 CODEOWNERS = ["@dentra"]
@@ -12,3 +15,6 @@ CONFIG_SCHEMA = cv.All(
 async def to_code(config):
     # Increase the maximum supported size of headers section in HTTP request packet to be processed by the server
     add_idf_sdkconfig_option("CONFIG_HTTPD_MAX_REQ_HDR_LEN", 1024)
+    # Re-enable esp-tls (excluded by default to save compile time);
+    # web_server_idf.cpp includes <esp_tls_crypto.h> for digest auth
+    include_builtin_idf_component("esp-tls")

tests/component_tests/esp32/config/exclusion_reincludes.yaml

@@ -0,0 +1,20 @@
+esphome:
+  name: test
+
+esp32:
+  board: esp32dev
+  framework:
+    type: esp-idf
+
+i2c:
+  sda: 21
+  scl: 22
+
+output:
+  - platform: ledc
+    id: ledc_out
+    pin: 25
+  - platform: ac_dimmer
+    id: dimmer_out
+    gate_pin: 26
+    zero_cross_pin: 27

tests/component_tests/esp32/config/exclusion_reincludes_http_request.yaml

@@ -0,0 +1,14 @@
+esphome:
+  name: test
+
+esp32:
+  board: esp32dev
+  framework:
+    type: esp-idf
+
+wifi:
+  ssid: "test_ssid"
+  password: "test_password"
+
+http_request:
+  verify_ssl: false

tests/component_tests/esp32/config/exclusion_reincludes_mqtt.yaml

@@ -0,0 +1,14 @@
+esphome:
+  name: test
+
+esp32:
+  board: esp32dev
+  framework:
+    type: esp-idf
+
+wifi:
+  ssid: "test_ssid"
+  password: "test_password"
+
+mqtt:
+  broker: "10.0.0.1"

tests/component_tests/esp32/config/exclusion_reincludes_nextion.yaml

@@ -0,0 +1,20 @@
+esphome:
+  name: test
+
+esp32:
+  board: esp32dev
+  framework:
+    type: esp-idf
+
+wifi:
+  ssid: "test_ssid"
+  password: "test_password"
+
+uart:
+  tx_pin: 17
+  rx_pin: 16
+  baud_rate: 115200
+
+display:
+  - platform: nextion
+    tft_url: "http://10.0.0.1/display.tft"

tests/component_tests/esp32/config/exclusion_reincludes_web_server.yaml

@@ -0,0 +1,14 @@
+esphome:
+  name: test
+
+esp32:
+  board: esp32dev
+  framework:
+    type: esp-idf
+
+wifi:
+  ssid: "test_ssid"
+  password: "test_password"
+
+web_server:
+  version: 3

tests/component_tests/esp32/test_esp32.py

@@ -236,6 +236,62 @@ def test_esp32_configuration_errors(
         FINAL_VALIDATE_SCHEMA(CONFIG_SCHEMA(config))
 
 
+@pytest.mark.parametrize(
+    ("config_file", "reincluded"),
+    [
+        pytest.param(
+            "exclusion_reincludes.yaml",
+            ("esp_driver_i2c", "esp_driver_ledc", "esp_driver_gptimer"),
+            id="i2c_ledc_ac_dimmer",
+        ),
+        # esp-tls has three owners; a per-owner config makes a dropped
+        # re-include from any single one fail the test.
+        pytest.param(
+            "exclusion_reincludes_http_request.yaml",
+            ("esp-tls", "esp_http_client"),
+            id="http_request",
+        ),
+        pytest.param(
+            # "mqtt" itself is deliberately not asserted: on IDF >= 6.0 it
+            # is a managed component and never leaves the exclusion set.
+            "exclusion_reincludes_mqtt.yaml",
+            ("esp-tls",),
+            id="mqtt",
+        ),
+        pytest.param(
+            "exclusion_reincludes_web_server.yaml",
+            ("esp-tls",),
+            id="web_server_idf",
+        ),
+        pytest.param(
+            "exclusion_reincludes_nextion.yaml",
+            ("esp-tls", "esp_http_client"),
+            id="nextion",
+        ),
+    ],
+)
+def test_default_exclusions_reincluded_by_owning_components(
+    generate_main: Callable[[str | Path], str],
+    component_config_path: Callable[[str], Path],
+    config_file: str,
+    reincluded: tuple[str, ...],
+) -> None:
+    """Components whose IDF driver is excluded by default must re-include it
+    during codegen; a dropped include_builtin_idf_component() call would only
+    surface as a missing-header failure in a full compile job."""
+    from esphome.components.esp32.const import KEY_EXCLUDE_COMPONENTS
+
+    generate_main(component_config_path(config_file))
+    excluded = CORE.data[KEY_ESP32][KEY_EXCLUDE_COMPONENTS]
+
+    for name in reincluded:
+        assert name not in excluded, f"{name} should have been re-included"
+
+    # Components no part of this config touches stay excluded.
+    assert "unity" in excluded
+    assert "fatfs" in excluded
+
+
 def test_execute_from_psram_s3_sdkconfig(
     generate_main: Callable[[str | Path], str],
     component_config_path: Callable[[str], Path],