diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 5d4e6b8401..2ef89cf595 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -1,6 +1,7 @@ """ESP-IDF direct build generator for ESPHome.""" import json +import logging from pathlib import Path from esphome.components.esp32 import ( @@ -11,6 +12,7 @@ from esphome.components.esp32 import ( ) import esphome.config_validation as cv from esphome.core import CORE +from esphome.espidf import variant_to_idf_target from esphome.framework_helpers import ( get_project_compile_flags, get_project_cxx_compile_flags, @@ -18,6 +20,8 @@ from esphome.framework_helpers import ( ) from esphome.helpers import mkdir_p, write_file_if_changed +_LOGGER = logging.getLogger(__name__) + # Replaces the IDF default C++ standard (-std=gnu++2b appended to # CXX_COMPILE_OPTIONS by project.cmake's __build_init) with the one set via # cg.set_cpp_standard(). Emitted between include(project.cmake) and project(), @@ -31,11 +35,12 @@ idf_build_set_property(CXX_COMPILE_OPTIONS "${{esphome_cxx_compile_options}}")"" def get_available_components() -> list[str] | None: - """Get list of built-in ESP-IDF components from project_description.json. + """List the built-in ESP-IDF components from ``project_description.json``. - Excludes ``src``, IDF-managed components (``managed_components/``), and - converted PIO libs (``pio_components/``). Returns ``None`` if the build - dir or ``project_description.json`` isn't ready yet. + Only components below its ``idf_path/components`` count, which leaves out + ``src``, IDF-managed components, converted PIO libs and project local + ones such as the Arduino ``component_stubs``. Returns ``None`` if the + build dir or ``project_description.json`` isn't ready yet. """ if CORE.build_path is None: return None @@ -46,30 +51,24 @@ def get_available_components() -> list[str] | None: try: with project_desc.open(encoding="utf-8") as f: data = json.load(f) - - component_info = data.get("build_component_info", {}) - - result = [] - for name, info in component_info.items(): - # Exclude our own src component - if name == "src": - continue - - # Exclude IDF-managed and converted-PIO components (external). - comp_dir = info.get("dir", "") - if "managed_components" in comp_dir or "pio_components" in comp_dir: - continue - - result.append(name) - - return result - except (json.JSONDecodeError, OSError): + root = (Path(data["idf_path"]) / "components").resolve() + result = [ + name + for name, info in data.get("build_component_info", {}).items() + if (comp_dir := info.get("dir")) + and Path(comp_dir).resolve().is_relative_to(root) + ] + except (json.JSONDecodeError, KeyError, OSError) as err: + _LOGGER.debug("Could not read %s: %s", project_desc, err) return None + if not result: + _LOGGER.warning("No ESP-IDF components found under %s", root) + return result def has_discovered_components() -> bool: - """Check if we have discovered components from a previous configure.""" - return get_available_components() is not None + """Check if a previous configure discovered any built-in components.""" + return bool(get_available_components()) def _cmake_quote(value: str) -> str: @@ -79,15 +78,17 @@ def _cmake_quote(value: str) -> str: return f'"{escaped}"' -def get_project_cmakelists(minimal: bool = False) -> str: +def get_project_cmakelists( + minimal: bool = False, builtin_components: list[str] | None = None +) -> str: """Generate the top-level CMakeLists.txt for ESP-IDF project. When ``minimal`` is true, omit ``ESPHOME_PROJECT_BUILTIN_COMPONENTS`` since ``project_description.json`` may be stale on the first write. + ``builtin_components`` supplies the discovered list (from the cache) + instead of reading it from ``project_description.json``. """ - # Get IDF target from ESP32 variant (e.g., ESP32S3 -> esp32s3) - variant = get_esp32_variant() - idf_target = variant.lower().replace("-", "") + idf_target = variant_to_idf_target(get_esp32_variant()) # esp_idf_size 2.x (bundled with IDF >=6.0) made NG the default and # removed the --ng flag; on 1.x (IDF 5.5) --ng is required to get @@ -162,9 +163,11 @@ def get_project_cmakelists(minimal: bool = False) -> str: else "\n".join( f"idf_build_set_property(ESPHOME_PROJECT_BUILTIN_COMPONENTS {name} APPEND)" for name in sorted( - set(get_available_components() or []).difference( - CORE.cmake_args.get("EXCLUDE_COMPONENTS", "").split(";") - ) + set( + builtin_components + if builtin_components is not None + else get_available_components() or [] + ).difference(CORE.cmake_args.get("EXCLUDE_COMPONENTS", "").split(";")) ) ) ) @@ -279,7 +282,9 @@ target_link_options(${{COMPONENT_LIB}} PUBLIC """ -def write_project(minimal: bool = False) -> None: +def write_project( + minimal: bool = False, builtin_components: list[str] | None = None +) -> None: """Write ESP-IDF project files.""" mkdir_p(CORE.build_path) mkdir_p(CORE.relative_src_path()) @@ -287,7 +292,7 @@ def write_project(minimal: bool = False) -> None: # Write top-level CMakeLists.txt write_file_if_changed( CORE.relative_build_path("CMakeLists.txt"), - get_project_cmakelists(minimal=minimal), + get_project_cmakelists(minimal=minimal, builtin_components=builtin_components), ) # Write component CMakeLists.txt in src/ diff --git a/esphome/components/emontx/sensor/__init__.py b/esphome/components/emontx/sensor/__init__.py index 967bc4e699..56a7fb8b55 100644 --- a/esphome/components/emontx/sensor/__init__.py +++ b/esphome/components/emontx/sensor/__init__.py @@ -7,8 +7,10 @@ from esphome.const import ( CONF_ID, CONF_STATE_CLASS, CONF_UNIT_OF_MEASUREMENT, + DEVICE_CLASS_APPARENT_POWER, DEVICE_CLASS_CURRENT, DEVICE_CLASS_ENERGY, + DEVICE_CLASS_FREQUENCY, DEVICE_CLASS_POWER, DEVICE_CLASS_POWER_FACTOR, DEVICE_CLASS_TEMPERATURE, @@ -18,8 +20,10 @@ from esphome.const import ( UNIT_AMPERE, UNIT_CELSIUS, UNIT_EMPTY, + UNIT_HERTZ, UNIT_PULSES, UNIT_VOLT, + UNIT_VOLT_AMPS, UNIT_WATT, UNIT_WATT_HOURS, ) @@ -29,6 +33,32 @@ from .. import CONF_EMONTX_ID, CONF_TAG_NAME, EmonTx, emontx_ns EmonTxSensor = emontx_ns.class_("EmonTxSensor", sensor.Sensor, cg.Component) +# Known emonTx/avrdb JSON tag conventions, gathered from real firmware +# (see https://github.com/openenergymonitor/avrdb_firmware), used to decide +# whether each tag below requires a numeric index or may also appear bare: +# +# Tag family Bare (no index) Numeric-indexed +# ----------- ----------------------- ---------------------------------- +# P (power) no P1, P2, ... (multi-channel boards) +# E (energy) no E1, E2, ... +# V (voltage) Vrms (NOT matched here, V1, V2, V3 (per-phase boards) +# doesn't fit "V"+digits) +# I (current) no I1, I2, ... +# T (temp.) no T1, T2, ... +# F (frequency) F (single mains freq.) not seen indexed +# PULSE pulse (single-CT boards) PULSE1, PULSE2, ... (other variants) +# PF (power not seen bare PF1, PF2, ... (currently unused/ +# factor) commented out in avrdb firmware) +# AP (apparent not seen bare AP1, AP2, ... (not an avrdb tag at +# power) all; avrdb uses "VA"+index instead, +# itself currently unused/commented +# out; "AP" is kept here for other +# firmware/integrations using it) +# +# This is why a bare "PULSE" resolves to proper defaults below, but bare +# "PF"/"AP" fall back to generic defaults instead: only PULSE has a +# confirmed bare-tag use in real, currently-shipping firmware. + # Define sensor type configurations by prefix SENSOR_CONFIGS = { "P": { @@ -63,7 +93,25 @@ SENSOR_CONFIGS = { }, } -# Pattern-based configurations +# Tags reported once, without a numeric index (e.g. "F"), matched exactly +# rather than by prefix. +EXACT_TAG_CONFIGS = { + "F": { + CONF_UNIT_OF_MEASUREMENT: UNIT_HERTZ, + CONF_DEVICE_CLASS: DEVICE_CLASS_FREQUENCY, + CONF_STATE_CLASS: STATE_CLASS_MEASUREMENT, + CONF_ACCURACY_DECIMALS: 2, + }, +} + +# Pattern-based configurations. The remainder after the prefix must be a +# non-empty numeric index (like V1/I1/E1), so e.g. "APPLE" doesn't collide +# with the "AP" prefix and a bare "PF"/"AP" (no index) doesn't match. +# "PULSE" is the exception: some emonTx firmware (e.g. avrdb-based single-CT +# variants) reports a single pulse counter as a bare "pulse" tag with no +# numeric index at all, so that pattern also accepts an empty suffix. +PATTERNS_ALLOWING_BARE_TAG = {"PULSE"} + PATTERN_CONFIGS = { "PULSE": { CONF_UNIT_OF_MEASUREMENT: UNIT_PULSES, @@ -77,14 +125,21 @@ PATTERN_CONFIGS = { CONF_STATE_CLASS: STATE_CLASS_MEASUREMENT, CONF_ACCURACY_DECIMALS: 2, }, + "AP": { + CONF_UNIT_OF_MEASUREMENT: UNIT_VOLT_AMPS, + CONF_DEVICE_CLASS: DEVICE_CLASS_APPARENT_POWER, + CONF_STATE_CLASS: STATE_CLASS_MEASUREMENT, + CONF_ACCURACY_DECIMALS: 2, + }, } # BASE_SCHEMA intentionally omits state_class and accuracy_decimals defaults. # Passing them to sensor_schema() would register them via cv.Optional(key, default=...), # making them always present in the validated config dict and preventing # apply_tag_defaults from overriding them with the correct per-prefix values. -# They are injected by apply_tag_defaults below, after running through -# sensor.validate_state_class() so the value is code-generation-ready. +# They are injected by apply_tag_defaults below, after running through the +# same validators sensor_schema() would use (see _DEFAULT_VALIDATORS) so the +# values are code-generation-ready. BASE_SCHEMA = sensor.sensor_schema(EmonTxSensor).extend( { cv.GenerateID(CONF_EMONTX_ID): cv.use_id(EmonTx), @@ -93,30 +148,43 @@ BASE_SCHEMA = sensor.sensor_schema(EmonTxSensor).extend( ) +_DEFAULT_VALIDATORS = { + CONF_STATE_CLASS: sensor.validate_state_class, + CONF_DEVICE_CLASS: sensor.validate_device_class, + CONF_UNIT_OF_MEASUREMENT: sensor.validate_unit_of_measurement, +} + + def _apply_defaults(config: ConfigType, defaults: dict) -> None: """Inject defaults into config, skipping keys already set by the user. - state_class values are run through validate_state_class so they are - code-generation-ready, matching what sensor_schema() would normally do.""" + Values are run through the same validators sensor_schema() would use, so + they are code-generation-ready and a typo'd constant fails validation + instead of shipping silently.""" for key, value in defaults.items(): if key not in config: - if key == CONF_STATE_CLASS: - value = sensor.validate_state_class(value) + if key in _DEFAULT_VALIDATORS: + value = _DEFAULT_VALIDATORS[key](value) config[key] = value def apply_tag_defaults(config: ConfigType) -> ConfigType: """Apply defaults based on tag prefix if applicable, but don't restrict any tags.""" tag = config[CONF_TAG_NAME] + tag_upper = tag.upper() + if (exact_config := EXACT_TAG_CONFIGS.get(tag_upper)) is not None: + _apply_defaults(config, exact_config) + return config + + for pattern, pattern_config in PATTERN_CONFIGS.items(): + suffix = tag_upper[len(pattern) :] + bare_ok = not suffix and pattern in PATTERNS_ALLOWING_BARE_TAG + if tag_upper.startswith(pattern) and (suffix.isdigit() or bare_ok): + _apply_defaults(config, pattern_config) + return config + + # Only apply defaults for known prefixes with numeric indices (e.g. E1, V2, T3) if len(tag) >= 2: - tag_upper = tag.upper() - - for pattern, pattern_config in PATTERN_CONFIGS.items(): - if tag_upper.startswith(pattern): - _apply_defaults(config, pattern_config) - return config - - # Only apply defaults for known prefixes with numeric indices (e.g. E1, V2, T3) prefix = tag_upper[0] if prefix in SENSOR_CONFIGS and tag[1:].isdigit(): _apply_defaults(config, SENSOR_CONFIGS[prefix]) diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index baf316a4ee..986f9dfb8b 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -1,6 +1,7 @@ """ESP-IDF direct build API for ESPHome.""" from dataclasses import dataclass, field +import hashlib import json import logging import os @@ -23,7 +24,7 @@ from esphome.core import CORE, EsphomeError from esphome.espidf import variant_to_idf_target from esphome.espidf.framework import check_esp_idf_install, get_framework_env from esphome.espidf.size_summary import print_summary -from esphome.helpers import add_git_ceiling_directory +from esphome.helpers import add_git_ceiling_directory, write_file _LOGGER = logging.getLogger(__name__) @@ -256,6 +257,106 @@ def run_reconfigure() -> int: return run_idf_py(*_get_sdkconfig_args(), "reconfigure") +def _builtin_component_cache_path() -> Path | None: + """Cache file for this build's built-in component list. + + The file lives inside the extracted framework directory so it is + discarded together with that exact checkout (re-extract, source + override, clean-all); the target and the EXCLUDE_COMPONENTS set name it. + The sdkconfig is not part of the key: IDF components register regardless + of CONFIG_* options and only gate their sources on them. A checkout + supplied through IDF_PATH is not managed by ESPHome and is never cached. + """ + if "IDF_PATH" in os.environ: + return None + target = variant_to_idf_target(CORE.data[KEY_ESP32][KEY_VARIANT]) + excluded = CORE.cmake_args.get("EXCLUDE_COMPONENTS", "") + excluded_key = hashlib.sha256(excluded.encode()).hexdigest()[:12] + return ( + _get_idf_path() / ".esphome_component_lists" / f"{target}-{excluded_key}.json" + ) + + +def load_cached_builtin_components() -> list[str] | None: + """Return the cached built-in component list for this build, if valid. + + Every name must still exist under ``$IDF_PATH/components`` so a stale + entry is treated as a miss instead of failing the configure. + """ + if (path := _builtin_component_cache_path()) is None: + return None + try: + components = json.loads(path.read_text(encoding="utf-8")) + present = { + entry.name + for entry in (path.parents[1] / "components").iterdir() + if entry.is_dir() + } + except (OSError, ValueError): + return None + if ( + isinstance(components, list) + and all(isinstance(c, str) for c in components) + and present.issuperset(components) + ): + return components + return None + + +def save_cached_builtin_components(components: list[str]) -> None: + """Store a built-in component list that just configured successfully.""" + if not components or (path := _builtin_component_cache_path()) is None: + return + try: + write_file(path, json.dumps(components, separators=(",", ":"))) + except EsphomeError as err: + _LOGGER.warning("Could not write component list cache %s: %s", path, err) + + +def _write_project_and_reconfigure(builtin_components: list[str] | None) -> int: + """Write the full CMakeLists.txt and run the configure for it.""" + from esphome.build_gen.espidf import write_project + + _LOGGER.info("Writing CMakeLists.txt with the built-in component list...") + write_project(minimal=False, builtin_components=builtin_components) + # Explicit reconfigure: ninja only re-runs cmake when CMakeLists.txt + # is strictly newer than build.ninja, which fails on coarse-mtime + # filesystems (#18682). Also keeps idf.py from regenerating memory.ld + # in testing mode. + return run_reconfigure() + + +def _configure_project() -> int: + """Configure the project, discovering the built-in components if needed. + + A cached component list skips the discovery configure. If the configure + with a cached list fails the entry is dropped and discovery runs once; a + list is only cached after it configured successfully. + """ + from esphome.build_gen.espidf import get_available_components, write_project + + if (cached := load_cached_builtin_components()) is not None: + _LOGGER.info("Using cached ESP-IDF component list") + if _write_project_and_reconfigure(cached) == 0: + return 0 + _LOGGER.warning("Cached component list failed; rediscovering") + _builtin_component_cache_path().unlink(missing_ok=True) + _LOGGER.info("Discovering available ESP-IDF components...") + write_project(minimal=True) + if (rc := run_reconfigure()) != 0: + _LOGGER.error("Component discovery failed") + return rc + discovered = get_available_components() + if not discovered: + _LOGGER.error("Component discovery found no built-in ESP-IDF components") + return 1 + if (rc := _write_project_and_reconfigure(discovered)) != 0: + _LOGGER.error("Reconfigure with discovered components failed") + return rc + save_cached_builtin_components(discovered) + return 0 + + def has_outdated_files(): """Check if the build configuration is stale. @@ -382,29 +483,17 @@ def run_compile(config, verbose: bool) -> int: """Compile the ESP-IDF project. Uses two-phase configure to auto-discover available components: - 1. If no previous build, configure with minimal REQUIRES to discover components + 1. If no previous build, configure with minimal REQUIRES to discover + components (skipped when a cached list for this IDF/target/exclusion + set exists) 2. Regenerate CMakeLists.txt with discovered components 3. Run full build """ - from esphome.build_gen.espidf import write_project - # Check if we need to do discovery phase - if need_reconfigure(): - _LOGGER.info("Discovering available ESP-IDF components...") - write_project(minimal=True) - rc = run_reconfigure() - if rc != 0: - _LOGGER.error("Component discovery failed") - return rc - _LOGGER.info("Regenerating CMakeLists.txt with discovered components...") - write_project(minimal=False) - # Explicit reconfigure: ninja only re-runs cmake when CMakeLists.txt - # is strictly newer than build.ninja, which fails on coarse-mtime - # filesystems (#18682). Also keeps idf.py from regenerating memory.ld - # in testing mode. - rc = run_reconfigure() - if rc != 0: - _LOGGER.error("Reconfigure with discovered components failed") + if not need_reconfigure(): + _LOGGER.info("Build configuration is up to date") + else: + if (rc := _configure_project()) != 0: return rc # cmake does not rewrite CMakeCache.txt when only properties change, # so restamp it or every build repeats discovery. Only after success, diff --git a/tests/component_tests/emontx/test_sensor_defaults.py b/tests/component_tests/emontx/test_sensor_defaults.py index 00d24d282e..5c6a8e4108 100644 --- a/tests/component_tests/emontx/test_sensor_defaults.py +++ b/tests/component_tests/emontx/test_sensor_defaults.py @@ -6,9 +6,28 @@ from esphome.components import sensor from esphome.components.emontx.sensor import CONFIG_SCHEMA, apply_tag_defaults from esphome.const import ( CONF_ACCURACY_DECIMALS, + CONF_DEVICE_CLASS, CONF_STATE_CLASS, + CONF_UNIT_OF_MEASUREMENT, + DEVICE_CLASS_APPARENT_POWER, + DEVICE_CLASS_CURRENT, + DEVICE_CLASS_ENERGY, + DEVICE_CLASS_FREQUENCY, + DEVICE_CLASS_POWER, + DEVICE_CLASS_POWER_FACTOR, + DEVICE_CLASS_TEMPERATURE, + DEVICE_CLASS_VOLTAGE, STATE_CLASS_MEASUREMENT, STATE_CLASS_TOTAL_INCREASING, + UNIT_AMPERE, + UNIT_CELSIUS, + UNIT_EMPTY, + UNIT_HERTZ, + UNIT_PULSES, + UNIT_VOLT, + UNIT_VOLT_AMPS, + UNIT_WATT, + UNIT_WATT_HOURS, ) @@ -61,9 +80,25 @@ def _make_config(tag: str) -> dict: ("PULSE1", STATE_CLASS_TOTAL_INCREASING, 0), ("PULSE12", STATE_CLASS_TOTAL_INCREASING, 0), ("PF1", STATE_CLASS_MEASUREMENT, 2), + ("AP1", STATE_CLASS_MEASUREMENT, 2), + ("AP12", STATE_CLASS_MEASUREMENT, 2), + # Frequency: reported as a single, un-numbered tag + ("F", STATE_CLASS_MEASUREMENT, 2), # Unknown / free-form tags fall back to generic defaults ("CUSTOM1", STATE_CLASS_MEASUREMENT, 0), ("X", STATE_CLASS_MEASUREMENT, 0), + # "F1" is not the exact "F" tag, so it falls back to generic defaults + ("F1", STATE_CLASS_MEASUREMENT, 0), + # "PULSE" (no index) is how some real emonTx firmware reports a + # single pulse counter, so it still resolves to the PULSE defaults + ("PULSE", STATE_CLASS_TOTAL_INCREASING, 0), + # Real firmware sends this lowercase; tag_upper's case-folding must + # still match it against the PULSE pattern + ("pulse", STATE_CLASS_TOTAL_INCREASING, 0), + # PF/AP require a numeric index; the bare prefix alone (no index) + # falls back to generic defaults + ("PF", STATE_CLASS_MEASUREMENT, 0), + ("AP", STATE_CLASS_MEASUREMENT, 0), ], ) def test_apply_tag_defaults(tag, expected_state_class, expected_decimals): @@ -76,6 +111,80 @@ def test_apply_tag_defaults(tag, expected_state_class, expected_decimals): assert result[CONF_ACCURACY_DECIMALS] == expected_decimals +@pytest.mark.parametrize( + ("tag", "expected_unit", "expected_device_class"), + [ + # Known numeric-index prefixes + ("E1", UNIT_WATT_HOURS, DEVICE_CLASS_ENERGY), + ("E12", UNIT_WATT_HOURS, DEVICE_CLASS_ENERGY), + ("P1", UNIT_WATT, DEVICE_CLASS_POWER), + ("V1", UNIT_VOLT, DEVICE_CLASS_VOLTAGE), + ("I1", UNIT_AMPERE, DEVICE_CLASS_CURRENT), + ("T1", UNIT_CELSIUS, DEVICE_CLASS_TEMPERATURE), + # Known patterns + ("PULSE1", UNIT_PULSES, DEVICE_CLASS_ENERGY), + ("PULSE12", UNIT_PULSES, DEVICE_CLASS_ENERGY), + # Bare "PULSE" (no index), as reported by some real emonTx firmware + ("PULSE", UNIT_PULSES, DEVICE_CLASS_ENERGY), + # Real firmware sends this lowercase; tag_upper's case-folding must + # still match it against the PULSE pattern + ("pulse", UNIT_PULSES, DEVICE_CLASS_ENERGY), + ("PF1", UNIT_EMPTY, DEVICE_CLASS_POWER_FACTOR), + ("AP1", UNIT_VOLT_AMPS, DEVICE_CLASS_APPARENT_POWER), + ("AP12", UNIT_VOLT_AMPS, DEVICE_CLASS_APPARENT_POWER), + # Frequency: reported as a single, un-numbered tag + ("F", UNIT_HERTZ, DEVICE_CLASS_FREQUENCY), + ], +) +def test_apply_tag_defaults_unit_and_device_class( + tag, expected_unit, expected_device_class +): + """apply_tag_defaults must inject the correct, validated unit_of_measurement + and device_class for each tag type when no user overrides are present.""" + config = _make_config(tag) + result = apply_tag_defaults(config) + + assert result[CONF_UNIT_OF_MEASUREMENT] == sensor.validate_unit_of_measurement( + expected_unit + ) + assert result[CONF_DEVICE_CLASS] == sensor.validate_device_class( + expected_device_class + ) + + +@pytest.mark.parametrize( + "tag", + [ + "CUSTOM1", + "X", + # Non-numeric suffixes must not collide with a PATTERN_CONFIGS prefix + # (e.g. "APPLE" starting with "AP", "PFX" starting with "PF"). + "APPLE", + "PFX", + "PULSE_A", + # "F1" is not the exact "F" tag + "F1", + # Bare "PF"/"AP" (no numeric index) don't match; unlike "PULSE", + # real firmware never reports these without an index + "PF", + "AP", + ], +) +def test_apply_tag_defaults_unknown_tag_has_no_unit_or_device_class(tag): + """Unknown / free-form tags only get generic state_class and + accuracy_decimals defaults; unit_of_measurement and device_class are left + for the user to set explicitly.""" + config = _make_config(tag) + result = apply_tag_defaults(config) + + assert CONF_UNIT_OF_MEASUREMENT not in result + assert CONF_DEVICE_CLASS not in result + assert result[CONF_STATE_CLASS] == sensor.validate_state_class( + STATE_CLASS_MEASUREMENT + ) + assert result[CONF_ACCURACY_DECIMALS] == 0 + + @pytest.mark.parametrize( ("tag", "user_state_class", "user_decimals"), [ diff --git a/tests/components/emontx/validate.esp32-idf.yaml b/tests/components/emontx/validate.esp32-idf.yaml index 7caee78a07..882ee26fcb 100644 --- a/tests/components/emontx/validate.esp32-idf.yaml +++ b/tests/components/emontx/validate.esp32-idf.yaml @@ -57,6 +57,21 @@ sensor: name: Power Factor 1 emontx_id: test_emontx + # Apparent power sensor (AP pattern): expects state_class=measurement, + # unit=VA, device_class=apparent_power, accuracy_decimals=2 + - platform: emontx + tag_name: AP1 + name: Apparent Power 1 + emontx_id: test_emontx + + # Frequency sensor (F, matched exactly, not as a prefix): expects + # state_class=measurement, unit=Hz, device_class=frequency, + # accuracy_decimals=2 + - platform: emontx + tag_name: F + name: Frequency + emontx_id: test_emontx + # Unknown tag: no prefix match, falls back to state_class=measurement, # accuracy_decimals=0 - platform: emontx diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index 29010bcf0e..079f10ddb9 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import logging from pathlib import Path from unittest.mock import patch @@ -35,22 +36,25 @@ def _reset_core(tmp_path: Path) -> None: } -def _write_project_description(tmp_path: Path, components: dict[str, str]) -> None: +def _write_project_description( + tmp_path: Path, components: dict[str, str], idf_path: str = "/idf" +) -> None: """Stub a project_description.json with the given component_name -> dir map.""" build_dir = tmp_path / "build" build_dir.mkdir(exist_ok=True) (build_dir / "project_description.json").write_text( json.dumps( { + "idf_path": idf_path, "build_component_info": { name: {"dir": dir_} for name, dir_ in components.items() - } + }, } ) ) -def _render(minimal: bool = False) -> str: +def _render(minimal: bool = False, builtin_components: list[str] | None = None) -> str: """Render the top-level CMakeLists with the standard variant/name patches.""" with ( patch("esphome.build_gen.espidf.get_esp32_variant", return_value="ESP32"), @@ -58,7 +62,9 @@ def _render(minimal: bool = False) -> str: ): from esphome.build_gen.espidf import get_project_cmakelists - return get_project_cmakelists(minimal=minimal) + return get_project_cmakelists( + minimal=minimal, builtin_components=builtin_components + ) def test_get_available_components_returns_none_without_build_path() -> None: @@ -77,8 +83,11 @@ def test_get_available_components_returns_none_without_project_description( assert get_available_components() is None -def test_get_available_components_filters_src_managed_and_pio(tmp_path: Path) -> None: - """Built-ins are returned; src/, managed_components/, pio_components/ skipped.""" +def test_get_available_components_keeps_only_idf_tree_components( + tmp_path: Path, +) -> None: + """Only components under idf_path/components are built-ins: src, managed, + converted PIO libs and Arduino component_stubs are all left out.""" _write_project_description( tmp_path, { @@ -86,6 +95,7 @@ def test_get_available_components_filters_src_managed_and_pio(tmp_path: Path) -> "esp_lcd": "/idf/components/esp_lcd", "espressif__arduino-esp32": f"{tmp_path}/managed_components/arduino", "JPEGDEC": f"{tmp_path}/pio_components/arduino/abc/bitbank2/JPEGDEC", + "cbor": f"{tmp_path}/component_stubs/cbor", "freertos": "/idf/components/freertos", }, ) @@ -94,6 +104,75 @@ def test_get_available_components_filters_src_managed_and_pio(tmp_path: Path) -> assert sorted(get_available_components()) == ["esp_lcd", "freertos"] +def test_codegen_and_configure_writes_render_the_same_cmakelists( + tmp_path: Path, +) -> None: + """write_project() at codegen time (no list) and the configure-time write + (discovered list) must agree, or ninja re-runs cmake on every build.""" + _write_project_description( + tmp_path, + { + "lwip": "/idf/components/lwip", + "cbor": f"{tmp_path}/component_stubs/cbor", + }, + ) + from esphome.build_gen.espidf import get_available_components + + assert _render() == _render(builtin_components=get_available_components()) + assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS cbor" not in _render() + + +def test_get_available_components_warns_when_nothing_is_under_idf_path( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + _write_project_description(tmp_path, {"cbor": f"{tmp_path}/component_stubs/cbor"}) + from esphome.build_gen.espidf import ( + get_available_components, + has_discovered_components, + ) + + assert get_available_components() == [] + assert "No ESP-IDF components found under" in caplog.text + # An empty discovery must not count as configured, or it would be latched in. + assert not has_discovered_components() + + +def test_get_available_components_ignores_corrupt_or_unexpected_file( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + build_dir = tmp_path / "build" + build_dir.mkdir() + from esphome.build_gen.espidf import ( + get_available_components, + has_discovered_components, + ) + + (build_dir / "project_description.json").write_text("{not json") + assert get_available_components() is None + assert not has_discovered_components() + (build_dir / "project_description.json").write_text('{"build_component_info": {}}') + with caplog.at_level(logging.DEBUG, logger="esphome.build_gen.espidf"): + assert get_available_components() is None + assert "Could not read" in caplog.text + + +def test_has_discovered_components_after_configure(tmp_path: Path) -> None: + _write_project_description(tmp_path, {"lwip": "/idf/components/lwip"}) + from esphome.build_gen.espidf import has_discovered_components + + assert has_discovered_components() + + +def test_get_project_cmakelists_uses_supplied_builtin_components() -> None: + """A cached list replaces project_description.json and is still filtered + by EXCLUDE_COMPONENTS.""" + with patch.dict(CORE.cmake_args, {"EXCLUDE_COMPONENTS": "fatfs;unity"}): + content = _render(builtin_components=["lwip", "fatfs", "esp_timer"]) + assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS esp_timer APPEND" in content + assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS lwip APPEND" in content + assert "ESPHOME_PROJECT_BUILTIN_COMPONENTS fatfs APPEND" not in content + + def test_get_project_cmakelists_minimal_omits_builtin_components_property( tmp_path: Path, ) -> None: diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index c54daec6a4..9deb27d83c 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -2,6 +2,8 @@ # pylint: disable=protected-access +from collections.abc import Iterator +from contextlib import contextmanager import json import os from pathlib import Path @@ -309,6 +311,11 @@ def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> No with ( patch.object(toolchain, "need_reconfigure", return_value=True), + patch.object(toolchain, "load_cached_builtin_components", return_value=None), + patch.object(toolchain, "save_cached_builtin_components"), + patch( + "esphome.build_gen.espidf.get_available_components", return_value=["lwip"] + ), patch("esphome.build_gen.espidf.write_project"), patch.object(toolchain, "run_reconfigure", return_value=0), patch.object(toolchain, "run_idf_py", return_value=0), @@ -329,6 +336,11 @@ def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None: with ( patch.object(toolchain, "need_reconfigure", return_value=True), + patch.object(toolchain, "load_cached_builtin_components", return_value=None), + patch.object(toolchain, "save_cached_builtin_components"), + patch( + "esphome.build_gen.espidf.get_available_components", return_value=["lwip"] + ), patch("esphome.build_gen.espidf.write_project"), patch.object(toolchain, "run_reconfigure", return_value=0), patch.object(toolchain, "run_idf_py", return_value=0), @@ -354,7 +366,7 @@ def test_run_compile_reconfigures_after_full_write_outside_testing_mode( calls: list[tuple] = [] reconfigures = 0 - def record_write(minimal: bool = False) -> None: + def record_write(minimal: bool = False, builtin_components=None) -> None: calls.append(("write_project", minimal)) def record_reconfigure() -> int: @@ -365,6 +377,11 @@ def test_run_compile_reconfigures_after_full_write_outside_testing_mode( with ( patch.object(toolchain, "need_reconfigure", return_value=True), + patch.object(toolchain, "load_cached_builtin_components", return_value=None), + patch.object(toolchain, "save_cached_builtin_components"), + patch( + "esphome.build_gen.espidf.get_available_components", return_value=["lwip"] + ), patch("esphome.build_gen.espidf.write_project", side_effect=record_write), patch.object(toolchain, "run_reconfigure", side_effect=record_reconfigure), patch.object(toolchain, "run_idf_py", return_value=0) as mock_build, @@ -383,6 +400,229 @@ def test_run_compile_reconfigures_after_full_write_outside_testing_mode( assert cmakecache.stat().st_mtime == old +def _record_compile_calls( + cached: list[str] | None, + saved: list[str] | None = None, + reconfigure_rcs: tuple[int, ...] = (), + cache_file: Path | None = None, +) -> tuple[int, list[tuple]]: + """Run run_compile with a stubbed cache and return (rc, call log). + + ``reconfigure_rcs`` overrides the exit codes of the first reconfigures; + later ones succeed. + """ + calls: list[tuple] = [] + rcs = iter(reconfigure_rcs) + + def record_reconfigure() -> int: + calls.append(("run_reconfigure",)) + return next(rcs, 0) + + def record_write(minimal: bool = False, builtin_components=None) -> None: + calls.append(("write_project", minimal, builtin_components)) + + def record_save(components: list[str]) -> None: + calls.append(("save", components)) + + with ( + patch.object(toolchain, "need_reconfigure", return_value=True), + patch.object(toolchain, "load_cached_builtin_components", return_value=cached), + patch.object( + toolchain, "save_cached_builtin_components", side_effect=record_save + ), + patch("esphome.build_gen.espidf.get_available_components", return_value=saved), + patch("esphome.build_gen.espidf.write_project", side_effect=record_write), + patch.object(toolchain, "run_reconfigure", side_effect=record_reconfigure), + patch.object( + toolchain, "_builtin_component_cache_path", return_value=cache_file + ), + patch.object( + toolchain, + "run_idf_py", + side_effect=lambda *a, **kw: calls.append(("build",)) or 0, + ), + patch.object(toolchain, "print_summary"), + ): + rc = toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) + return rc, calls + + +def test_run_compile_poisoned_cache_is_dropped_and_rediscovered( + setup_core: Path, tmp_path: Path +) -> None: + """A cached list that fails the configure is deleted and discovery runs + once more instead of every later build failing the same way.""" + _setup_build(setup_core) + cache_file = tmp_path / "esp32-abc.json" + cache_file.write_text("[]") + rc, calls = _record_compile_calls( + ["stale"], saved=["lwip"], reconfigure_rcs=(1,), cache_file=cache_file + ) + assert rc == 0 + assert not cache_file.exists() + assert calls == [ + ("write_project", False, ["stale"]), + ("run_reconfigure",), + ("write_project", True, None), + ("run_reconfigure",), + ("write_project", False, ["lwip"]), + ("run_reconfigure",), + ("save", ["lwip"]), + ("build",), + ] + + +def test_run_compile_cache_miss_discovers_and_saves(setup_core: Path) -> None: + """Without a cached list the discovery configure runs, the discovered list + feeds the full write and is cached only after that configure succeeds.""" + _setup_build(setup_core) + rc, calls = _record_compile_calls(None, saved=["lwip"]) + assert rc == 0 + assert calls == [ + ("write_project", True, None), + ("run_reconfigure",), + ("write_project", False, ["lwip"]), + ("run_reconfigure",), + ("save", ["lwip"]), + ("build",), + ] + + +def test_run_compile_discovery_failure_stops_before_full_write( + setup_core: Path, +) -> None: + """A failed discovery configure returns its exit code and never writes + the full CMakeLists, a cache entry or a build.""" + _setup_build(setup_core) + rc, calls = _record_compile_calls(None, reconfigure_rcs=(2,)) + assert rc == 2 + assert calls == [("write_project", True, None), ("run_reconfigure",)] + + +@pytest.mark.parametrize("discovered", [None, []], ids=["no_manifest", "empty"]) +def test_run_compile_fails_when_discovery_finds_nothing( + setup_core: Path, + caplog: pytest.LogCaptureFixture, + discovered: list[str] | None, +) -> None: + _setup_build(setup_core) + rc, calls = _record_compile_calls(None, saved=discovered) + assert rc == 1 + assert calls == [("write_project", True, None), ("run_reconfigure",)] + assert "found no built-in ESP-IDF components" in caplog.text + + +def test_run_compile_does_not_cache_a_list_that_failed_to_configure( + setup_core: Path, +) -> None: + _setup_build(setup_core) + rc, calls = _record_compile_calls(None, saved=["lwip"], reconfigure_rcs=(0, 3)) + assert rc == 3 + assert ("save", ["lwip"]) not in calls + assert ("build",) not in calls + + +def test_run_compile_cache_hit_skips_discovery(setup_core: Path) -> None: + """A cached list goes straight to the full write; the explicit reconfigure + after it (#18730) still runs.""" + _setup_build(setup_core) + rc, calls = _record_compile_calls(["esp_timer", "lwip"]) + assert rc == 0 + assert calls == [ + ("write_project", False, ["esp_timer", "lwip"]), + ("run_reconfigure",), + ("build",), + ] + + +@contextmanager +def _cache_env(tmp_path: Path, excluded: str) -> Iterator[Path]: + """Patch everything the cache key derives from onto a temp IDF tree and + yield that tree's path.""" + idf_path = tmp_path / "idf" + (idf_path / "components").mkdir(parents=True, exist_ok=True) + with ( + patch.object(toolchain, "_get_idf_path", return_value=idf_path), + patch.dict(CORE.data, {KEY_ESP32: {KEY_VARIANT: "ESP32"}}), + patch.dict(CORE.cmake_args, {"EXCLUDE_COMPONENTS": excluded}), + ): + yield idf_path + + +def test_component_cache_round_trip(setup_core: Path, tmp_path: Path) -> None: + """A saved list is read back until it is dropped.""" + _setup_build(setup_core) + with _cache_env(tmp_path, "fatfs") as idf_path: + for name in ("lwip", "esp_timer"): + (idf_path / "components" / name).mkdir() + assert toolchain.load_cached_builtin_components() is None + toolchain.save_cached_builtin_components(["esp_timer", "lwip"]) + assert toolchain.load_cached_builtin_components() == ["esp_timer", "lwip"] + toolchain._builtin_component_cache_path().unlink() + assert toolchain.load_cached_builtin_components() is None + + +def test_component_cache_misses_on_key_change_or_missing_component( + setup_core: Path, tmp_path: Path +) -> None: + """A different exclusion set uses another entry, an entry naming a + component that no longer exists is ignored, and a custom IDF_PATH is + never cached.""" + _setup_build(setup_core) + with _cache_env(tmp_path, "fatfs") as idf_path: + (idf_path / "components" / "lwip").mkdir() + toolchain.save_cached_builtin_components(["lwip"]) + path = toolchain._builtin_component_cache_path() + assert path.parent == idf_path / ".esphome_component_lists" + assert path.name.startswith("esp32-") + assert toolchain.load_cached_builtin_components() == ["lwip"] + with patch.dict(os.environ, {"IDF_PATH": str(idf_path)}): + assert toolchain.load_cached_builtin_components() is None + with _cache_env(tmp_path, "fatfs;unity"): + assert toolchain.load_cached_builtin_components() is None + with _cache_env(tmp_path, "fatfs") as idf_path: + path.write_text(json.dumps(["lwip", "gone"])) + assert toolchain.load_cached_builtin_components() is None + # A plain file with the right name is not a component directory. + (idf_path / "components" / "gone").write_text("not a directory") + assert toolchain.load_cached_builtin_components() is None + + +def test_component_cache_save_skips_empty_list_or_custom_idf_path( + setup_core: Path, tmp_path: Path +) -> None: + _setup_build(setup_core) + with _cache_env(tmp_path, "") as idf_path: + toolchain.save_cached_builtin_components([]) + with patch.dict(os.environ, {"IDF_PATH": str(idf_path)}): + toolchain.save_cached_builtin_components(["lwip"]) + assert not (idf_path / ".esphome_component_lists").exists() + + +def test_component_cache_write_failure_is_logged( + setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + _setup_build(setup_core) + with ( + _cache_env(tmp_path, ""), + patch.object(toolchain, "write_file", side_effect=EsphomeError("disk full")), + ): + toolchain.save_cached_builtin_components(["lwip"]) + assert toolchain.load_cached_builtin_components() is None + assert "Could not write component list cache" in caplog.text + + +def test_component_cache_ignores_corrupt_file(setup_core: Path, tmp_path: Path) -> None: + _setup_build(setup_core) + with _cache_env(tmp_path, ""): + path = toolchain._builtin_component_cache_path() + path.parent.mkdir(parents=True) + path.write_text("{not json") + assert toolchain.load_cached_builtin_components() is None + path.write_text(json.dumps({"components": ["lwip"]})) + assert toolchain.load_cached_builtin_components() is None + + def test_run_compile_passes_compile_process_limit(setup_core: Path) -> None: """compile_process_limit is forwarded to run_idf_py as the job limit.""" _setup_build(setup_core)