Merge branch 'esp8266-native-toolchain-plumbing' into esp8266-native-build-infra

esphome/build_gen/espidf.py

@@ -1,6 +1,7 @@
 """ESP-IDF direct build generator for ESPHome."""
 
 import json
+import logging
 from pathlib import Path
 
 from esphome.components.esp32 import (
@@ -11,6 +12,7 @@ from esphome.components.esp32 import (
 )
 import esphome.config_validation as cv
 from esphome.core import CORE
+from esphome.espidf import variant_to_idf_target
 from esphome.framework_helpers import (
     get_project_compile_flags,
     get_project_cxx_compile_flags,
@@ -18,6 +20,8 @@ from esphome.framework_helpers import (
 )
 from esphome.helpers import mkdir_p, write_file_if_changed
 
+_LOGGER = logging.getLogger(__name__)
+
 # Replaces the IDF default C++ standard (-std=gnu++2b appended to
 # CXX_COMPILE_OPTIONS by project.cmake's __build_init) with the one set via
 # cg.set_cpp_standard(). Emitted between include(project.cmake) and project(),
@@ -31,11 +35,12 @@ idf_build_set_property(CXX_COMPILE_OPTIONS "${{esphome_cxx_compile_options}}")""
 
 
 def get_available_components() -> list[str] | None:
-    """Get list of built-in ESP-IDF components from project_description.json.
+    """List the built-in ESP-IDF components from ``project_description.json``.
 
-    Excludes ``src``, IDF-managed components (``managed_components/``), and
-    converted PIO libs (``pio_components/``). Returns ``None`` if the build
-    dir or ``project_description.json`` isn't ready yet.
+    Only components below its ``idf_path/components`` count, which leaves out
+    ``src``, IDF-managed components, converted PIO libs and project local
+    ones such as the Arduino ``component_stubs``. Returns ``None`` if the
+    build dir or ``project_description.json`` isn't ready yet.
     """
     if CORE.build_path is None:
         return None
@@ -46,30 +51,24 @@ def get_available_components() -> list[str] | None:
     try:
         with project_desc.open(encoding="utf-8") as f:
             data = json.load(f)
-
-        component_info = data.get("build_component_info", {})
-
-        result = []
-        for name, info in component_info.items():
-            # Exclude our own src component
-            if name == "src":
-                continue
-
-            # Exclude IDF-managed and converted-PIO components (external).
-            comp_dir = info.get("dir", "")
-            if "managed_components" in comp_dir or "pio_components" in comp_dir:
-                continue
-
-            result.append(name)
-
-        return result
-    except (json.JSONDecodeError, OSError):
+        root = (Path(data["idf_path"]) / "components").resolve()
+        result = [
+            name
+            for name, info in data.get("build_component_info", {}).items()
+            if (comp_dir := info.get("dir"))
+            and Path(comp_dir).resolve().is_relative_to(root)
+        ]
+    except (json.JSONDecodeError, KeyError, OSError) as err:
+        _LOGGER.debug("Could not read %s: %s", project_desc, err)
         return None
+    if not result:
+        _LOGGER.warning("No ESP-IDF components found under %s", root)
+    return result
 
 
 def has_discovered_components() -> bool:
-    """Check if we have discovered components from a previous configure."""
-    return get_available_components() is not None
+    """Check if a previous configure discovered any built-in components."""
+    return bool(get_available_components())
 
 
 def _cmake_quote(value: str) -> str:
@@ -79,15 +78,17 @@ def _cmake_quote(value: str) -> str:
     return f'"{escaped}"'
 
 
-def get_project_cmakelists(minimal: bool = False) -> str:
+def get_project_cmakelists(
+    minimal: bool = False, builtin_components: list[str] | None = None
+) -> str:
     """Generate the top-level CMakeLists.txt for ESP-IDF project.
 
     When ``minimal`` is true, omit ``ESPHOME_PROJECT_BUILTIN_COMPONENTS``
     since ``project_description.json`` may be stale on the first write.
+    ``builtin_components`` supplies the discovered list (from the cache)
+    instead of reading it from ``project_description.json``.
     """
-    # Get IDF target from ESP32 variant (e.g., ESP32S3 -> esp32s3)
-    variant = get_esp32_variant()
-    idf_target = variant.lower().replace("-", "")
+    idf_target = variant_to_idf_target(get_esp32_variant())
 
     # esp_idf_size 2.x (bundled with IDF >=6.0) made NG the default and
     # removed the --ng flag; on 1.x (IDF 5.5) --ng is required to get
@@ -162,9 +163,11 @@ def get_project_cmakelists(minimal: bool = False) -> str:
         else "\n".join(
             f"idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS {name} APPEND)"
             for name in sorted(
-                set(get_available_components() or []).difference(
-                    CORE.cmake_args.get("EXCLUDE_COMPONENTS", "").split(";")
-                )
+                set(
+                    builtin_components
+                    if builtin_components is not None
+                    else get_available_components() or []
+                ).difference(CORE.cmake_args.get("EXCLUDE_COMPONENTS", "").split(";"))
             )
         )
     )
@@ -279,7 +282,9 @@ target_link_options(${{COMPONENT_LIB}} PUBLIC
 """
 
 
-def write_project(minimal: bool = False) -> None:
+def write_project(
+    minimal: bool = False, builtin_components: list[str] | None = None
+) -> None:
     """Write ESP-IDF project files."""
     mkdir_p(CORE.build_path)
     mkdir_p(CORE.relative_src_path())
@@ -287,7 +292,7 @@ def write_project(minimal: bool = False) -> None:
     # Write top-level CMakeLists.txt
     write_file_if_changed(
         CORE.relative_build_path("CMakeLists.txt"),
-        get_project_cmakelists(minimal=minimal),
+        get_project_cmakelists(minimal=minimal, builtin_components=builtin_components),
     )
 
     # Write component CMakeLists.txt in src/

esphome/components/emontx/sensor/__init__.py

@@ -7,8 +7,10 @@ from esphome.const import (
     CONF_ID,
     CONF_STATE_CLASS,
     CONF_UNIT_OF_MEASUREMENT,
+    DEVICE_CLASS_APPARENT_POWER,
     DEVICE_CLASS_CURRENT,
     DEVICE_CLASS_ENERGY,
+    DEVICE_CLASS_FREQUENCY,
     DEVICE_CLASS_POWER,
     DEVICE_CLASS_POWER_FACTOR,
     DEVICE_CLASS_TEMPERATURE,
@@ -18,8 +20,10 @@ from esphome.const import (
     UNIT_AMPERE,
     UNIT_CELSIUS,
     UNIT_EMPTY,
+    UNIT_HERTZ,
     UNIT_PULSES,
     UNIT_VOLT,
+    UNIT_VOLT_AMPS,
     UNIT_WATT,
     UNIT_WATT_HOURS,
 )
@@ -29,6 +33,32 @@ from .. import CONF_EMONTX_ID, CONF_TAG_NAME, EmonTx, emontx_ns
 
 EmonTxSensor = emontx_ns.class_("EmonTxSensor", sensor.Sensor, cg.Component)
 
+# Known emonTx/avrdb JSON tag conventions, gathered from real firmware
+# (see https://github.com/openenergymonitor/avrdb_firmware), used to decide
+# whether each tag below requires a numeric index or may also appear bare:
+#
+#   Tag family    Bare (no index)          Numeric-indexed
+#   -----------   -----------------------  ----------------------------------
+#   P (power)     no                       P1, P2, ... (multi-channel boards)
+#   E (energy)    no                       E1, E2, ...
+#   V (voltage)   Vrms (NOT matched here,  V1, V2, V3 (per-phase boards)
+#                 doesn't fit "V"+digits)
+#   I (current)   no                       I1, I2, ...
+#   T (temp.)     no                       T1, T2, ...
+#   F (frequency) F (single mains freq.)   not seen indexed
+#   PULSE         pulse (single-CT boards) PULSE1, PULSE2, ... (other variants)
+#   PF (power     not seen bare            PF1, PF2, ... (currently unused/
+#     factor)                              commented out in avrdb firmware)
+#   AP (apparent  not seen bare            AP1, AP2, ... (not an avrdb tag at
+#     power)                               all; avrdb uses "VA"+index instead,
+#                                           itself currently unused/commented
+#                                           out; "AP" is kept here for other
+#                                           firmware/integrations using it)
+#
+# This is why a bare "PULSE" resolves to proper defaults below, but bare
+# "PF"/"AP" fall back to generic defaults instead: only PULSE has a
+# confirmed bare-tag use in real, currently-shipping firmware.
+
 # Define sensor type configurations by prefix
 SENSOR_CONFIGS = {
     "P": {
@@ -63,7 +93,25 @@ SENSOR_CONFIGS = {
     },
 }
 
-# Pattern-based configurations
+# Tags reported once, without a numeric index (e.g. "F"), matched exactly
+# rather than by prefix.
+EXACT_TAG_CONFIGS = {
+    "F": {
+        CONF_UNIT_OF_MEASUREMENT: UNIT_HERTZ,
+        CONF_DEVICE_CLASS: DEVICE_CLASS_FREQUENCY,
+        CONF_STATE_CLASS: STATE_CLASS_MEASUREMENT,
+        CONF_ACCURACY_DECIMALS: 2,
+    },
+}
+
+# Pattern-based configurations. The remainder after the prefix must be a
+# non-empty numeric index (like V1/I1/E1), so e.g. "APPLE" doesn't collide
+# with the "AP" prefix and a bare "PF"/"AP" (no index) doesn't match.
+# "PULSE" is the exception: some emonTx firmware (e.g. avrdb-based single-CT
+# variants) reports a single pulse counter as a bare "pulse" tag with no
+# numeric index at all, so that pattern also accepts an empty suffix.
+PATTERNS_ALLOWING_BARE_TAG = {"PULSE"}
+
 PATTERN_CONFIGS = {
     "PULSE": {
         CONF_UNIT_OF_MEASUREMENT: UNIT_PULSES,
@@ -77,14 +125,21 @@ PATTERN_CONFIGS = {
         CONF_STATE_CLASS: STATE_CLASS_MEASUREMENT,
         CONF_ACCURACY_DECIMALS: 2,
     },
+    "AP": {
+        CONF_UNIT_OF_MEASUREMENT: UNIT_VOLT_AMPS,
+        CONF_DEVICE_CLASS: DEVICE_CLASS_APPARENT_POWER,
+        CONF_STATE_CLASS: STATE_CLASS_MEASUREMENT,
+        CONF_ACCURACY_DECIMALS: 2,
+    },
 }
 
 # BASE_SCHEMA intentionally omits state_class and accuracy_decimals defaults.
 # Passing them to sensor_schema() would register them via cv.Optional(key, default=...),
 # making them always present in the validated config dict and preventing
 # apply_tag_defaults from overriding them with the correct per-prefix values.
-# They are injected by apply_tag_defaults below, after running through
-# sensor.validate_state_class() so the value is code-generation-ready.
+# They are injected by apply_tag_defaults below, after running through the
+# same validators sensor_schema() would use (see _DEFAULT_VALIDATORS) so the
+# values are code-generation-ready.
 BASE_SCHEMA = sensor.sensor_schema(EmonTxSensor).extend(
     {
         cv.GenerateID(CONF_EMONTX_ID): cv.use_id(EmonTx),
@@ -93,30 +148,43 @@ BASE_SCHEMA = sensor.sensor_schema(EmonTxSensor).extend(
 )
 
 
+_DEFAULT_VALIDATORS = {
+    CONF_STATE_CLASS: sensor.validate_state_class,
+    CONF_DEVICE_CLASS: sensor.validate_device_class,
+    CONF_UNIT_OF_MEASUREMENT: sensor.validate_unit_of_measurement,
+}
+
+
 def _apply_defaults(config: ConfigType, defaults: dict) -> None:
     """Inject defaults into config, skipping keys already set by the user.
-    state_class values are run through validate_state_class so they are
-    code-generation-ready, matching what sensor_schema() would normally do."""
+    Values are run through the same validators sensor_schema() would use, so
+    they are code-generation-ready and a typo'd constant fails validation
+    instead of shipping silently."""
     for key, value in defaults.items():
         if key not in config:
-            if key == CONF_STATE_CLASS:
-                value = sensor.validate_state_class(value)
+            if key in _DEFAULT_VALIDATORS:
+                value = _DEFAULT_VALIDATORS[key](value)
             config[key] = value
 
 
 def apply_tag_defaults(config: ConfigType) -> ConfigType:
     """Apply defaults based on tag prefix if applicable, but don't restrict any tags."""
     tag = config[CONF_TAG_NAME]
+    tag_upper = tag.upper()
 
+    if (exact_config := EXACT_TAG_CONFIGS.get(tag_upper)) is not None:
+        _apply_defaults(config, exact_config)
+        return config
+
+    for pattern, pattern_config in PATTERN_CONFIGS.items():
+        suffix = tag_upper[len(pattern) :]
+        bare_ok = not suffix and pattern in PATTERNS_ALLOWING_BARE_TAG
+        if tag_upper.startswith(pattern) and (suffix.isdigit() or bare_ok):
+            _apply_defaults(config, pattern_config)
+            return config
+
+    # Only apply defaults for known prefixes with numeric indices (e.g. E1, V2, T3)
     if len(tag) >= 2:
-        tag_upper = tag.upper()
-
-        for pattern, pattern_config in PATTERN_CONFIGS.items():
-            if tag_upper.startswith(pattern):
-                _apply_defaults(config, pattern_config)
-                return config
-
-        # Only apply defaults for known prefixes with numeric indices (e.g. E1, V2, T3)
         prefix = tag_upper[0]
         if prefix in SENSOR_CONFIGS and tag[1:].isdigit():
             _apply_defaults(config, SENSOR_CONFIGS[prefix])

esphome/espidf/toolchain.py

@@ -1,6 +1,7 @@
 """ESP-IDF direct build API for ESPHome."""
 
 from dataclasses import dataclass, field
+import hashlib
 import json
 import logging
 import os
@@ -23,7 +24,7 @@ from esphome.core import CORE, EsphomeError
 from esphome.espidf import variant_to_idf_target
 from esphome.espidf.framework import check_esp_idf_install, get_framework_env
 from esphome.espidf.size_summary import print_summary
-from esphome.helpers import add_git_ceiling_directory
+from esphome.helpers import add_git_ceiling_directory, write_file
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -256,6 +257,106 @@ def run_reconfigure() -> int:
     return run_idf_py(*_get_sdkconfig_args(), "reconfigure")
 
 
+def _builtin_component_cache_path() -> Path | None:
+    """Cache file for this build's built-in component list.
+
+    The file lives inside the extracted framework directory so it is
+    discarded together with that exact checkout (re-extract, source
+    override, clean-all); the target and the EXCLUDE_COMPONENTS set name it.
+    The sdkconfig is not part of the key: IDF components register regardless
+    of CONFIG_* options and only gate their sources on them. A checkout
+    supplied through IDF_PATH is not managed by ESPHome and is never cached.
+    """
+    if "IDF_PATH" in os.environ:
+        return None
+    target = variant_to_idf_target(CORE.data[KEY_ESP32][KEY_VARIANT])
+    excluded = CORE.cmake_args.get("EXCLUDE_COMPONENTS", "")
+    excluded_key = hashlib.sha256(excluded.encode()).hexdigest()[:12]
+    return (
+        _get_idf_path() / ".esphome_component_lists" / f"{target}-{excluded_key}.json"
+    )
+
+
+def load_cached_builtin_components() -> list[str] | None:
+    """Return the cached built-in component list for this build, if valid.
+
+    Every name must still exist under ``$IDF_PATH/components`` so a stale
+    entry is treated as a miss instead of failing the configure.
+    """
+    if (path := _builtin_component_cache_path()) is None:
+        return None
+    try:
+        components = json.loads(path.read_text(encoding="utf-8"))
+        present = {
+            entry.name
+            for entry in (path.parents[1] / "components").iterdir()
+            if entry.is_dir()
+        }
+    except (OSError, ValueError):
+        return None
+    if (
+        isinstance(components, list)
+        and all(isinstance(c, str) for c in components)
+        and present.issuperset(components)
+    ):
+        return components
+    return None
+
+
+def save_cached_builtin_components(components: list[str]) -> None:
+    """Store a built-in component list that just configured successfully."""
+    if not components or (path := _builtin_component_cache_path()) is None:
+        return
+    try:
+        write_file(path, json.dumps(components, separators=(",", ":")))
+    except EsphomeError as err:
+        _LOGGER.warning("Could not write component list cache %s: %s", path, err)
+
+
+def _write_project_and_reconfigure(builtin_components: list[str] | None) -> int:
+    """Write the full CMakeLists.txt and run the configure for it."""
+    from esphome.build_gen.espidf import write_project
+
+    _LOGGER.info("Writing CMakeLists.txt with the built-in component list...")
+    write_project(minimal=False, builtin_components=builtin_components)
+    # Explicit reconfigure: ninja only re-runs cmake when CMakeLists.txt
+    # is strictly newer than build.ninja, which fails on coarse-mtime
+    # filesystems (#18682). Also keeps idf.py from regenerating memory.ld
+    # in testing mode.
+    return run_reconfigure()
+
+
+def _configure_project() -> int:
+    """Configure the project, discovering the built-in components if needed.
+
+    A cached component list skips the discovery configure. If the configure
+    with a cached list fails the entry is dropped and discovery runs once; a
+    list is only cached after it configured successfully.
+    """
+    from esphome.build_gen.espidf import get_available_components, write_project
+
+    if (cached := load_cached_builtin_components()) is not None:
+        _LOGGER.info("Using cached ESP-IDF component list")
+        if _write_project_and_reconfigure(cached) == 0:
+            return 0
+        _LOGGER.warning("Cached component list failed; rediscovering")
+        _builtin_component_cache_path().unlink(missing_ok=True)
+    _LOGGER.info("Discovering available ESP-IDF components...")
+    write_project(minimal=True)
+    if (rc := run_reconfigure()) != 0:
+        _LOGGER.error("Component discovery failed")
+        return rc
+    discovered = get_available_components()
+    if not discovered:
+        _LOGGER.error("Component discovery found no built-in ESP-IDF components")
+        return 1
+    if (rc := _write_project_and_reconfigure(discovered)) != 0:
+        _LOGGER.error("Reconfigure with discovered components failed")
+        return rc
+    save_cached_builtin_components(discovered)
+    return 0
+
+
 def has_outdated_files():
     """Check if the build configuration is stale.
 
@@ -382,29 +483,17 @@ def run_compile(config, verbose: bool) -> int:
     """Compile the ESP-IDF project.
 
     Uses two-phase configure to auto-discover available components:
-    1. If no previous build, configure with minimal REQUIRES to discover components
+    1. If no previous build, configure with minimal REQUIRES to discover
+       components (skipped when a cached list for this IDF/target/exclusion
+       set exists)
     2. Regenerate CMakeLists.txt with discovered components
     3. Run full build
     """
-    from esphome.build_gen.espidf import write_project
-
     # Check if we need to do discovery phase
-    if need_reconfigure():
-        _LOGGER.info("Discovering available ESP-IDF components...")
-        write_project(minimal=True)
-        rc = run_reconfigure()
-        if rc != 0:
-            _LOGGER.error("Component discovery failed")
-            return rc
-        _LOGGER.info("Regenerating CMakeLists.txt with discovered components...")
-        write_project(minimal=False)
-        # Explicit reconfigure: ninja only re-runs cmake when CMakeLists.txt
-        # is strictly newer than build.ninja, which fails on coarse-mtime
-        # filesystems (#18682). Also keeps idf.py from regenerating memory.ld
-        # in testing mode.
-        rc = run_reconfigure()
-        if rc != 0:
-            _LOGGER.error("Reconfigure with discovered components failed")
+    if not need_reconfigure():
+        _LOGGER.info("Build configuration is up to date")
+    else:
+        if (rc := _configure_project()) != 0:
             return rc
         # cmake does not rewrite CMakeCache.txt when only properties change,
         # so restamp it or every build repeats discovery. Only after success,

tests/component_tests/emontx/test_sensor_defaults.py

@@ -6,9 +6,28 @@ from esphome.components import sensor
 from esphome.components.emontx.sensor import CONFIG_SCHEMA, apply_tag_defaults
 from esphome.const import (
     CONF_ACCURACY_DECIMALS,
+    CONF_DEVICE_CLASS,
     CONF_STATE_CLASS,
+    CONF_UNIT_OF_MEASUREMENT,
+    DEVICE_CLASS_APPARENT_POWER,
+    DEVICE_CLASS_CURRENT,
+    DEVICE_CLASS_ENERGY,
+    DEVICE_CLASS_FREQUENCY,
+    DEVICE_CLASS_POWER,
+    DEVICE_CLASS_POWER_FACTOR,
+    DEVICE_CLASS_TEMPERATURE,
+    DEVICE_CLASS_VOLTAGE,
     STATE_CLASS_MEASUREMENT,
     STATE_CLASS_TOTAL_INCREASING,
+    UNIT_AMPERE,
+    UNIT_CELSIUS,
+    UNIT_EMPTY,
+    UNIT_HERTZ,
+    UNIT_PULSES,
+    UNIT_VOLT,
+    UNIT_VOLT_AMPS,
+    UNIT_WATT,
+    UNIT_WATT_HOURS,
 )
 
 
@@ -61,9 +80,25 @@ def _make_config(tag: str) -> dict:
         ("PULSE1", STATE_CLASS_TOTAL_INCREASING, 0),
         ("PULSE12", STATE_CLASS_TOTAL_INCREASING, 0),
         ("PF1", STATE_CLASS_MEASUREMENT, 2),
+        ("AP1", STATE_CLASS_MEASUREMENT, 2),
+        ("AP12", STATE_CLASS_MEASUREMENT, 2),
+        # Frequency: reported as a single, un-numbered tag
+        ("F", STATE_CLASS_MEASUREMENT, 2),
         # Unknown / free-form tags fall back to generic defaults
         ("CUSTOM1", STATE_CLASS_MEASUREMENT, 0),
         ("X", STATE_CLASS_MEASUREMENT, 0),
+        # "F1" is not the exact "F" tag, so it falls back to generic defaults
+        ("F1", STATE_CLASS_MEASUREMENT, 0),
+        # "PULSE" (no index) is how some real emonTx firmware reports a
+        # single pulse counter, so it still resolves to the PULSE defaults
+        ("PULSE", STATE_CLASS_TOTAL_INCREASING, 0),
+        # Real firmware sends this lowercase; tag_upper's case-folding must
+        # still match it against the PULSE pattern
+        ("pulse", STATE_CLASS_TOTAL_INCREASING, 0),
+        # PF/AP require a numeric index; the bare prefix alone (no index)
+        # falls back to generic defaults
+        ("PF", STATE_CLASS_MEASUREMENT, 0),
+        ("AP", STATE_CLASS_MEASUREMENT, 0),
     ],
 )
 def test_apply_tag_defaults(tag, expected_state_class, expected_decimals):
@@ -76,6 +111,80 @@ def test_apply_tag_defaults(tag, expected_state_class, expected_decimals):
     assert result[CONF_ACCURACY_DECIMALS] == expected_decimals
 
 
+@pytest.mark.parametrize(
+    ("tag", "expected_unit", "expected_device_class"),
+    [
+        # Known numeric-index prefixes
+        ("E1", UNIT_WATT_HOURS, DEVICE_CLASS_ENERGY),
+        ("E12", UNIT_WATT_HOURS, DEVICE_CLASS_ENERGY),
+        ("P1", UNIT_WATT, DEVICE_CLASS_POWER),
+        ("V1", UNIT_VOLT, DEVICE_CLASS_VOLTAGE),
+        ("I1", UNIT_AMPERE, DEVICE_CLASS_CURRENT),
+        ("T1", UNIT_CELSIUS, DEVICE_CLASS_TEMPERATURE),
+        # Known patterns
+        ("PULSE1", UNIT_PULSES, DEVICE_CLASS_ENERGY),
+        ("PULSE12", UNIT_PULSES, DEVICE_CLASS_ENERGY),
+        # Bare "PULSE" (no index), as reported by some real emonTx firmware
+        ("PULSE", UNIT_PULSES, DEVICE_CLASS_ENERGY),
+        # Real firmware sends this lowercase; tag_upper's case-folding must
+        # still match it against the PULSE pattern
+        ("pulse", UNIT_PULSES, DEVICE_CLASS_ENERGY),
+        ("PF1", UNIT_EMPTY, DEVICE_CLASS_POWER_FACTOR),
+        ("AP1", UNIT_VOLT_AMPS, DEVICE_CLASS_APPARENT_POWER),
+        ("AP12", UNIT_VOLT_AMPS, DEVICE_CLASS_APPARENT_POWER),
+        # Frequency: reported as a single, un-numbered tag
+        ("F", UNIT_HERTZ, DEVICE_CLASS_FREQUENCY),
+    ],
+)
+def test_apply_tag_defaults_unit_and_device_class(
+    tag, expected_unit, expected_device_class
+):
+    """apply_tag_defaults must inject the correct, validated unit_of_measurement
+    and device_class for each tag type when no user overrides are present."""
+    config = _make_config(tag)
+    result = apply_tag_defaults(config)
+
+    assert result[CONF_UNIT_OF_MEASUREMENT] == sensor.validate_unit_of_measurement(
+        expected_unit
+    )
+    assert result[CONF_DEVICE_CLASS] == sensor.validate_device_class(
+        expected_device_class
+    )
+
+
+@pytest.mark.parametrize(
+    "tag",
+    [
+        "CUSTOM1",
+        "X",
+        # Non-numeric suffixes must not collide with a PATTERN_CONFIGS prefix
+        # (e.g. "APPLE" starting with "AP", "PFX" starting with "PF").
+        "APPLE",
+        "PFX",
+        "PULSE_A",
+        # "F1" is not the exact "F" tag
+        "F1",
+        # Bare "PF"/"AP" (no numeric index) don't match; unlike "PULSE",
+        # real firmware never reports these without an index
+        "PF",
+        "AP",
+    ],
+)
+def test_apply_tag_defaults_unknown_tag_has_no_unit_or_device_class(tag):
+    """Unknown / free-form tags only get generic state_class and
+    accuracy_decimals defaults; unit_of_measurement and device_class are left
+    for the user to set explicitly."""
+    config = _make_config(tag)
+    result = apply_tag_defaults(config)
+
+    assert CONF_UNIT_OF_MEASUREMENT not in result
+    assert CONF_DEVICE_CLASS not in result
+    assert result[CONF_STATE_CLASS] == sensor.validate_state_class(
+        STATE_CLASS_MEASUREMENT
+    )
+    assert result[CONF_ACCURACY_DECIMALS] == 0
+
+
 @pytest.mark.parametrize(
     ("tag", "user_state_class", "user_decimals"),
     [

tests/components/emontx/validate.esp32-idf.yaml

@@ -57,6 +57,21 @@ sensor:
     name: Power Factor 1
     emontx_id: test_emontx
 
+  # Apparent power sensor (AP pattern): expects state_class=measurement,
+  # unit=VA, device_class=apparent_power, accuracy_decimals=2
+  - platform: emontx
+    tag_name: AP1
+    name: Apparent Power 1
+    emontx_id: test_emontx
+
+  # Frequency sensor (F, matched exactly, not as a prefix): expects
+  # state_class=measurement, unit=Hz, device_class=frequency,
+  # accuracy_decimals=2
+  - platform: emontx
+    tag_name: F
+    name: Frequency
+    emontx_id: test_emontx
+
   # Unknown tag: no prefix match, falls back to state_class=measurement,
   # accuracy_decimals=0
   - platform: emontx

tests/unit_tests/build_gen/test_espidf.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import json
+import logging
 from pathlib import Path
 from unittest.mock import patch
 
@@ -35,22 +36,25 @@ def _reset_core(tmp_path: Path) -> None:
     }
 
 
-def _write_project_description(tmp_path: Path, components: dict[str, str]) -> None:
+def _write_project_description(
+    tmp_path: Path, components: dict[str, str], idf_path: str = "/idf"
+) -> None:
     """Stub a project_description.json with the given component_name -> dir map."""
     build_dir = tmp_path / "build"
     build_dir.mkdir(exist_ok=True)
     (build_dir / "project_description.json").write_text(
         json.dumps(
             {
+                "idf_path": idf_path,
                 "build_component_info": {
                     name: {"dir": dir_} for name, dir_ in components.items()
-                }
+                },
             }
         )
     )
 
 
-def _render(minimal: bool = False) -> str:
+def _render(minimal: bool = False, builtin_components: list[str] | None = None) -> str:
     """Render the top-level CMakeLists with the standard variant/name patches."""
     with (
         patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"),
@@ -58,7 +62,9 @@ def _render(minimal: bool = False) -> str:
     ):
         from esphome.build_gen.espidf import get_project_cmakelists
 
-        return get_project_cmakelists(minimal=minimal)
+        return get_project_cmakelists(
+            minimal=minimal, builtin_components=builtin_components
+        )
 
 
 def test_get_available_components_returns_none_without_build_path() -> None:
@@ -77,8 +83,11 @@ def test_get_available_components_returns_none_without_project_description(
     assert get_available_components() is None
 
 
-def test_get_available_components_filters_src_managed_and_pio(tmp_path: Path) -> None:
-    """Built-ins are returned; src/, managed_components/, pio_components/ skipped."""
+def test_get_available_components_keeps_only_idf_tree_components(
+    tmp_path: Path,
+) -> None:
+    """Only components under idf_path/components are built-ins: src, managed,
+    converted PIO libs and Arduino component_stubs are all left out."""
     _write_project_description(
         tmp_path,
         {
@@ -86,6 +95,7 @@ def test_get_available_components_filters_src_managed_and_pio(tmp_path: Path) ->
             "esp_lcd": "/idf/components/esp_lcd",
             "espressif__arduino-esp32": f"{tmp_path}/managed_components/arduino",
             "JPEGDEC": f"{tmp_path}/pio_components/arduino/abc/bitbank2/JPEGDEC",
+            "cbor": f"{tmp_path}/component_stubs/cbor",
             "freertos": "/idf/components/freertos",
         },
     )
@@ -94,6 +104,75 @@ def test_get_available_components_filters_src_managed_and_pio(tmp_path: Path) ->
     assert sorted(get_available_components()) == ["esp_lcd", "freertos"]
 
 
+def test_codegen_and_configure_writes_render_the_same_cmakelists(
+    tmp_path: Path,
+) -> None:
+    """write_project() at codegen time (no list) and the configure-time write
+    (discovered list) must agree, or ninja re-runs cmake on every build."""
+    _write_project_description(
+        tmp_path,
+        {
+            "lwip": "/idf/components/lwip",
+            "cbor": f"{tmp_path}/component_stubs/cbor",
+        },
+    )
+    from esphome.build_gen.espidf import get_available_components
+
+    assert _render() == _render(builtin_components=get_available_components())
+    assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS cbor" not in _render()
+
+
+def test_get_available_components_warns_when_nothing_is_under_idf_path(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    _write_project_description(tmp_path, {"cbor": f"{tmp_path}/component_stubs/cbor"})
+    from esphome.build_gen.espidf import (
+        get_available_components,
+        has_discovered_components,
+    )
+
+    assert get_available_components() == []
+    assert "No ESP-IDF components found under" in caplog.text
+    # An empty discovery must not count as configured, or it would be latched in.
+    assert not has_discovered_components()
+
+
+def test_get_available_components_ignores_corrupt_or_unexpected_file(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+    from esphome.build_gen.espidf import (
+        get_available_components,
+        has_discovered_components,
+    )
+
+    (build_dir / "project_description.json").write_text("{not json")
+    assert get_available_components() is None
+    assert not has_discovered_components()
+    (build_dir / "project_description.json").write_text('{"build_component_info": {}}')
+    with caplog.at_level(logging.DEBUG, logger="esphome.build_gen.espidf"):
+        assert get_available_components() is None
+    assert "Could not read" in caplog.text
+
+
+def test_has_discovered_components_after_configure(tmp_path: Path) -> None:
+    _write_project_description(tmp_path, {"lwip": "/idf/components/lwip"})
+    from esphome.build_gen.espidf import has_discovered_components
+
+    assert has_discovered_components()
+
+
+def test_get_project_cmakelists_uses_supplied_builtin_components() -> None:
+    """A cached list replaces project_description.json and is still filtered
+    by EXCLUDE_COMPONENTS."""
+    with patch.dict(CORE.cmake_args, {"EXCLUDE_COMPONENTS": "fatfs;unity"}):
+        content = _render(builtin_components=["lwip", "fatfs", "esp_timer"])
+    assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_timer APPEND" in content
+    assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS lwip APPEND" in content
+    assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS fatfs APPEND" not in content
+
+
 def test_get_project_cmakelists_minimal_omits_builtin_components_property(
     tmp_path: Path,
 ) -> None:

tests/unit_tests/test_espidf_toolchain.py

@@ -2,6 +2,8 @@
 
 # pylint: disable=protected-access
 
+from collections.abc import Iterator
+from contextlib import contextmanager
 import json
 import os
 from pathlib import Path
@@ -309,6 +311,11 @@ def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> No
 
     with (
         patch.object(toolchain, "need_reconfigure", return_value=True),
+        patch.object(toolchain, "load_cached_builtin_components", return_value=None),
+        patch.object(toolchain, "save_cached_builtin_components"),
+        patch(
+            "esphome.build_gen.espidf.get_available_components", return_value=["lwip"]
+        ),
         patch("esphome.build_gen.espidf.write_project"),
         patch.object(toolchain, "run_reconfigure", return_value=0),
         patch.object(toolchain, "run_idf_py", return_value=0),
@@ -329,6 +336,11 @@ def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None:
 
     with (
         patch.object(toolchain, "need_reconfigure", return_value=True),
+        patch.object(toolchain, "load_cached_builtin_components", return_value=None),
+        patch.object(toolchain, "save_cached_builtin_components"),
+        patch(
+            "esphome.build_gen.espidf.get_available_components", return_value=["lwip"]
+        ),
         patch("esphome.build_gen.espidf.write_project"),
         patch.object(toolchain, "run_reconfigure", return_value=0),
         patch.object(toolchain, "run_idf_py", return_value=0),
@@ -354,7 +366,7 @@ def test_run_compile_reconfigures_after_full_write_outside_testing_mode(
     calls: list[tuple] = []
     reconfigures = 0
 
-    def record_write(minimal: bool = False) -> None:
+    def record_write(minimal: bool = False, builtin_components=None) -> None:
         calls.append(("write_project", minimal))
 
     def record_reconfigure() -> int:
@@ -365,6 +377,11 @@ def test_run_compile_reconfigures_after_full_write_outside_testing_mode(
 
     with (
         patch.object(toolchain, "need_reconfigure", return_value=True),
+        patch.object(toolchain, "load_cached_builtin_components", return_value=None),
+        patch.object(toolchain, "save_cached_builtin_components"),
+        patch(
+            "esphome.build_gen.espidf.get_available_components", return_value=["lwip"]
+        ),
         patch("esphome.build_gen.espidf.write_project", side_effect=record_write),
         patch.object(toolchain, "run_reconfigure", side_effect=record_reconfigure),
         patch.object(toolchain, "run_idf_py", return_value=0) as mock_build,
@@ -383,6 +400,229 @@ def test_run_compile_reconfigures_after_full_write_outside_testing_mode(
     assert cmakecache.stat().st_mtime == old
 
 
+def _record_compile_calls(
+    cached: list[str] | None,
+    saved: list[str] | None = None,
+    reconfigure_rcs: tuple[int, ...] = (),
+    cache_file: Path | None = None,
+) -> tuple[int, list[tuple]]:
+    """Run run_compile with a stubbed cache and return (rc, call log).
+
+    ``reconfigure_rcs`` overrides the exit codes of the first reconfigures;
+    later ones succeed.
+    """
+    calls: list[tuple] = []
+    rcs = iter(reconfigure_rcs)
+
+    def record_reconfigure() -> int:
+        calls.append(("run_reconfigure",))
+        return next(rcs, 0)
+
+    def record_write(minimal: bool = False, builtin_components=None) -> None:
+        calls.append(("write_project", minimal, builtin_components))
+
+    def record_save(components: list[str]) -> None:
+        calls.append(("save", components))
+
+    with (
+        patch.object(toolchain, "need_reconfigure", return_value=True),
+        patch.object(toolchain, "load_cached_builtin_components", return_value=cached),
+        patch.object(
+            toolchain, "save_cached_builtin_components", side_effect=record_save
+        ),
+        patch("esphome.build_gen.espidf.get_available_components", return_value=saved),
+        patch("esphome.build_gen.espidf.write_project", side_effect=record_write),
+        patch.object(toolchain, "run_reconfigure", side_effect=record_reconfigure),
+        patch.object(
+            toolchain, "_builtin_component_cache_path", return_value=cache_file
+        ),
+        patch.object(
+            toolchain,
+            "run_idf_py",
+            side_effect=lambda *a, **kw: calls.append(("build",)) or 0,
+        ),
+        patch.object(toolchain, "print_summary"),
+    ):
+        rc = toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False)
+    return rc, calls
+
+
+def test_run_compile_poisoned_cache_is_dropped_and_rediscovered(
+    setup_core: Path, tmp_path: Path
+) -> None:
+    """A cached list that fails the configure is deleted and discovery runs
+    once more instead of every later build failing the same way."""
+    _setup_build(setup_core)
+    cache_file = tmp_path / "esp32-abc.json"
+    cache_file.write_text("[]")
+    rc, calls = _record_compile_calls(
+        ["stale"], saved=["lwip"], reconfigure_rcs=(1,), cache_file=cache_file
+    )
+    assert rc == 0
+    assert not cache_file.exists()
+    assert calls == [
+        ("write_project", False, ["stale"]),
+        ("run_reconfigure",),
+        ("write_project", True, None),
+        ("run_reconfigure",),
+        ("write_project", False, ["lwip"]),
+        ("run_reconfigure",),
+        ("save", ["lwip"]),
+        ("build",),
+    ]
+
+
+def test_run_compile_cache_miss_discovers_and_saves(setup_core: Path) -> None:
+    """Without a cached list the discovery configure runs, the discovered list
+    feeds the full write and is cached only after that configure succeeds."""
+    _setup_build(setup_core)
+    rc, calls = _record_compile_calls(None, saved=["lwip"])
+    assert rc == 0
+    assert calls == [
+        ("write_project", True, None),
+        ("run_reconfigure",),
+        ("write_project", False, ["lwip"]),
+        ("run_reconfigure",),
+        ("save", ["lwip"]),
+        ("build",),
+    ]
+
+
+def test_run_compile_discovery_failure_stops_before_full_write(
+    setup_core: Path,
+) -> None:
+    """A failed discovery configure returns its exit code and never writes
+    the full CMakeLists, a cache entry or a build."""
+    _setup_build(setup_core)
+    rc, calls = _record_compile_calls(None, reconfigure_rcs=(2,))
+    assert rc == 2
+    assert calls == [("write_project", True, None), ("run_reconfigure",)]
+
+
+@pytest.mark.parametrize("discovered", [None, []], ids=["no_manifest", "empty"])
+def test_run_compile_fails_when_discovery_finds_nothing(
+    setup_core: Path,
+    caplog: pytest.LogCaptureFixture,
+    discovered: list[str] | None,
+) -> None:
+    _setup_build(setup_core)
+    rc, calls = _record_compile_calls(None, saved=discovered)
+    assert rc == 1
+    assert calls == [("write_project", True, None), ("run_reconfigure",)]
+    assert "found no built-in ESP-IDF components" in caplog.text
+
+
+def test_run_compile_does_not_cache_a_list_that_failed_to_configure(
+    setup_core: Path,
+) -> None:
+    _setup_build(setup_core)
+    rc, calls = _record_compile_calls(None, saved=["lwip"], reconfigure_rcs=(0, 3))
+    assert rc == 3
+    assert ("save", ["lwip"]) not in calls
+    assert ("build",) not in calls
+
+
+def test_run_compile_cache_hit_skips_discovery(setup_core: Path) -> None:
+    """A cached list goes straight to the full write; the explicit reconfigure
+    after it (#18730) still runs."""
+    _setup_build(setup_core)
+    rc, calls = _record_compile_calls(["esp_timer", "lwip"])
+    assert rc == 0
+    assert calls == [
+        ("write_project", False, ["esp_timer", "lwip"]),
+        ("run_reconfigure",),
+        ("build",),
+    ]
+
+
+@contextmanager
+def _cache_env(tmp_path: Path, excluded: str) -> Iterator[Path]:
+    """Patch everything the cache key derives from onto a temp IDF tree and
+    yield that tree's path."""
+    idf_path = tmp_path / "idf"
+    (idf_path / "components").mkdir(parents=True, exist_ok=True)
+    with (
+        patch.object(toolchain, "_get_idf_path", return_value=idf_path),
+        patch.dict(CORE.data, {KEY_ESP32: {KEY_VARIANT: "ESP32"}}),
+        patch.dict(CORE.cmake_args, {"EXCLUDE_COMPONENTS": excluded}),
+    ):
+        yield idf_path
+
+
+def test_component_cache_round_trip(setup_core: Path, tmp_path: Path) -> None:
+    """A saved list is read back until it is dropped."""
+    _setup_build(setup_core)
+    with _cache_env(tmp_path, "fatfs") as idf_path:
+        for name in ("lwip", "esp_timer"):
+            (idf_path / "components" / name).mkdir()
+        assert toolchain.load_cached_builtin_components() is None
+        toolchain.save_cached_builtin_components(["esp_timer", "lwip"])
+        assert toolchain.load_cached_builtin_components() == ["esp_timer", "lwip"]
+        toolchain._builtin_component_cache_path().unlink()
+        assert toolchain.load_cached_builtin_components() is None
+
+
+def test_component_cache_misses_on_key_change_or_missing_component(
+    setup_core: Path, tmp_path: Path
+) -> None:
+    """A different exclusion set uses another entry, an entry naming a
+    component that no longer exists is ignored, and a custom IDF_PATH is
+    never cached."""
+    _setup_build(setup_core)
+    with _cache_env(tmp_path, "fatfs") as idf_path:
+        (idf_path / "components" / "lwip").mkdir()
+        toolchain.save_cached_builtin_components(["lwip"])
+        path = toolchain._builtin_component_cache_path()
+        assert path.parent == idf_path / ".esphome_component_lists"
+        assert path.name.startswith("esp32-")
+        assert toolchain.load_cached_builtin_components() == ["lwip"]
+        with patch.dict(os.environ, {"IDF_PATH": str(idf_path)}):
+            assert toolchain.load_cached_builtin_components() is None
+    with _cache_env(tmp_path, "fatfs;unity"):
+        assert toolchain.load_cached_builtin_components() is None
+    with _cache_env(tmp_path, "fatfs") as idf_path:
+        path.write_text(json.dumps(["lwip", "gone"]))
+        assert toolchain.load_cached_builtin_components() is None
+        # A plain file with the right name is not a component directory.
+        (idf_path / "components" / "gone").write_text("not a directory")
+        assert toolchain.load_cached_builtin_components() is None
+
+
+def test_component_cache_save_skips_empty_list_or_custom_idf_path(
+    setup_core: Path, tmp_path: Path
+) -> None:
+    _setup_build(setup_core)
+    with _cache_env(tmp_path, "") as idf_path:
+        toolchain.save_cached_builtin_components([])
+        with patch.dict(os.environ, {"IDF_PATH": str(idf_path)}):
+            toolchain.save_cached_builtin_components(["lwip"])
+        assert not (idf_path / ".esphome_component_lists").exists()
+
+
+def test_component_cache_write_failure_is_logged(
+    setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> None:
+    _setup_build(setup_core)
+    with (
+        _cache_env(tmp_path, ""),
+        patch.object(toolchain, "write_file", side_effect=EsphomeError("disk full")),
+    ):
+        toolchain.save_cached_builtin_components(["lwip"])
+        assert toolchain.load_cached_builtin_components() is None
+    assert "Could not write component list cache" in caplog.text
+
+
+def test_component_cache_ignores_corrupt_file(setup_core: Path, tmp_path: Path) -> None:
+    _setup_build(setup_core)
+    with _cache_env(tmp_path, ""):
+        path = toolchain._builtin_component_cache_path()
+        path.parent.mkdir(parents=True)
+        path.write_text("{not json")
+        assert toolchain.load_cached_builtin_components() is None
+        path.write_text(json.dumps({"components": ["lwip"]}))
+        assert toolchain.load_cached_builtin_components() is None
+
+
 def test_run_compile_passes_compile_process_limit(setup_core: Path) -> None:
     """compile_process_limit is forwarded to run_idf_py as the job limit."""
     _setup_build(setup_core)