From b328758634676b84839f13ad2f70469351b976dd Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Nov 2025 10:53:44 -0600 Subject: [PATCH 1/9] Revert "[core] Deduplicate identical stateless lambdas to reduce flash usage" (#12117) --- esphome/cpp_generator.py | 177 +-------------- tests/component_tests/text/test_text.py | 19 +- tests/unit_tests/test_lambda_dedup.py | 286 ------------------------ 3 files changed, 11 insertions(+), 471 deletions(-) delete mode 100644 tests/unit_tests/test_lambda_dedup.py diff --git a/esphome/cpp_generator.py b/esphome/cpp_generator.py index 4f91696ca1f..6f1af01a5be 100644 --- a/esphome/cpp_generator.py +++ b/esphome/cpp_generator.py @@ -19,21 +19,11 @@ from esphome.core import ( TimePeriodNanoseconds, TimePeriodSeconds, ) -from esphome.coroutine import CoroPriority, coroutine_with_priority from esphome.helpers import cpp_string_escape, indent_all_but_first_and_last from esphome.types import Expression, SafeExpType, TemplateArgsType from esphome.util import OrderedDict from esphome.yaml_util import ESPHomeDataBase -# Keys for lambda deduplication storage in CORE.data -_KEY_LAMBDA_DEDUP = "lambda_dedup" -_KEY_LAMBDA_DEDUP_DECLARATIONS = "lambda_dedup_declarations" - -# Regex patterns for static variable detection (compiled once) -_RE_CPP_SINGLE_LINE_COMMENT = re.compile(r"//.*?$", re.MULTILINE) -_RE_CPP_MULTI_LINE_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL) -_RE_STATIC_VARIABLE = re.compile(r"\bstatic\s+(?!cast|assert|pointer_cast)\w+\s+\w+") - class RawExpression(Expression): __slots__ = ("text",) @@ -198,7 +188,7 @@ class LambdaExpression(Expression): def __init__( self, parts, parameters, capture: str = "=", return_type=None, source=None - ) -> None: + ): self.parts = parts if not isinstance(parameters, ParameterListExpression): parameters = ParameterListExpression(*parameters) @@ -207,21 +197,16 @@ class LambdaExpression(Expression): self.capture = capture self.return_type = safe_exp(return_type) if return_type is not None else None - def format_body(self) -> str: - """Format the lambda body with source directive and content.""" - body = "" - if self.source is not None: - body += f"{self.source.as_line_directive}\n" - body += self.content - return body - - def __str__(self) -> str: + def __str__(self): # Stateless lambdas (empty capture) implicitly convert to function pointers # when assigned to function pointer types - no unary + needed cpp = f"[{self.capture}]({self.parameters})" if self.return_type is not None: cpp += f" -> {self.return_type}" - cpp += f" {{\n{self.format_body()}\n}}" + cpp += " {\n" + if self.source is not None: + cpp += f"{self.source.as_line_directive}\n" + cpp += f"{self.content}\n}}" return indent_all_but_first_and_last(cpp) @property @@ -229,37 +214,6 @@ class LambdaExpression(Expression): return "".join(str(part) for part in self.parts) -class SharedFunctionLambdaExpression(LambdaExpression): - """A lambda expression that references a shared deduplicated function. - - This class wraps a function pointer but maintains the LambdaExpression - interface so calling code works unchanged. - """ - - __slots__ = ("_func_name",) - - def __init__( - self, - func_name: str, - parameters: TemplateArgsType, - return_type: SafeExpType | None = None, - ) -> None: - # Initialize parent with empty parts since we're just a function reference - super().__init__( - [], parameters, capture="", return_type=return_type, source=None - ) - self._func_name = func_name - - def __str__(self) -> str: - # Just return the function name - it's already a function pointer - return self._func_name - - @property - def content(self) -> str: - # No content, just a function reference - return "" - - # pylint: disable=abstract-method class Literal(Expression, metaclass=abc.ABCMeta): __slots__ = () @@ -629,25 +583,6 @@ def add_global(expression: SafeExpType | Statement, prepend: bool = False): CORE.add_global(expression, prepend) -@coroutine_with_priority(CoroPriority.FINAL) -async def flush_lambda_dedup_declarations() -> None: - """Flush all deferred lambda deduplication declarations to global scope. - - This is a coroutine that runs with FINAL priority (after all components) - to ensure all referenced variables are declared before the shared - lambda functions that use them. - """ - if _KEY_LAMBDA_DEDUP_DECLARATIONS not in CORE.data: - return - - declarations = CORE.data[_KEY_LAMBDA_DEDUP_DECLARATIONS] - for func_declaration in declarations: - add_global(RawStatement(func_declaration)) - - # Clear the list so we don't add them again - CORE.data[_KEY_LAMBDA_DEDUP_DECLARATIONS] = [] - - def add_library(name: str, version: str | None, repository: str | None = None): """Add a library to the codegen library storage. @@ -721,93 +656,6 @@ async def get_variable_with_full_id(id_: ID) -> tuple[ID, "MockObj"]: return await CORE.get_variable_with_full_id(id_) -def _has_static_variables(code: str) -> bool: - """Check if code contains static variable definitions. - - Static variables in lambdas should not be deduplicated because each lambda - instance should have its own static variable state. - - Args: - code: The lambda body code to check - - Returns: - True if code contains static variable definitions - """ - # Remove C++ comments to avoid false positives - # Remove single-line comments (// ...) - code_no_comments = _RE_CPP_SINGLE_LINE_COMMENT.sub("", code) - # Remove multi-line comments (/* ... */) - code_no_comments = _RE_CPP_MULTI_LINE_COMMENT.sub("", code_no_comments) - - # Match: static - # But not: static_cast, static_assert, static_pointer_cast - return bool(_RE_STATIC_VARIABLE.search(code_no_comments)) - - -def _get_shared_lambda_name(lambda_expr: LambdaExpression) -> str | None: - """Get the shared function name for a lambda expression. - - If an identical lambda was already generated, returns the existing shared - function name. Otherwise, creates a new shared function and returns its name. - - Lambdas with static variables are not deduplicated to preserve their - independent state. - - Args: - lambda_expr: The lambda expression to deduplicate - - Returns: - The name of the shared function for this lambda (either existing or newly created), - or None if the lambda should not be deduplicated (e.g., contains static variables) - """ - # Create a unique key from the lambda content, parameters, and return type - content = lambda_expr.content - - # Don't deduplicate lambdas with static variables - each instance needs its own state - if _has_static_variables(content): - return None - param_str = str(lambda_expr.parameters) - return_str = ( - str(lambda_expr.return_type) if lambda_expr.return_type is not None else "void" - ) - - # Use tuple of (content, params, return_type) as key - lambda_key = (content, param_str, return_str) - - # Initialize deduplication storage in CORE.data if not exists - if _KEY_LAMBDA_DEDUP not in CORE.data: - CORE.data[_KEY_LAMBDA_DEDUP] = {} - # Register the flush job to run after all components (FINAL priority) - # This ensures all variables are declared before shared lambda functions - CORE.add_job(flush_lambda_dedup_declarations) - - lambda_cache = CORE.data[_KEY_LAMBDA_DEDUP] - - # Check if we've seen this lambda before - if lambda_key in lambda_cache: - # Return name of existing shared function - return lambda_cache[lambda_key] - - # First occurrence - create a shared function - # Use the cache size as the function number - func_name = f"shared_lambda_{len(lambda_cache)}" - - # Build the function declaration using lambda's body formatting - func_declaration = ( - f"{return_str} {func_name}({param_str}) {{\n{lambda_expr.format_body()}\n}}" - ) - - # Store the declaration to be added later (after all variable declarations) - # We can't add it immediately because it might reference variables not yet declared - CORE.data.setdefault(_KEY_LAMBDA_DEDUP_DECLARATIONS, []).append(func_declaration) - - # Store in cache - lambda_cache[lambda_key] = func_name - - # Return the function name (this is the first occurrence, but we still generate shared function) - return func_name - - async def process_lambda( value: Lambda, parameters: TemplateArgsType, @@ -865,19 +713,6 @@ async def process_lambda( location.line += value.content_offset else: location = None - - # Lambda deduplication: Only deduplicate stateless lambdas (empty capture). - # Stateful lambdas cannot be shared as they capture different contexts. - # Lambdas with static variables are also not deduplicated to preserve independent state. - if capture == "": - lambda_expr = LambdaExpression( - parts, parameters, capture, return_type, location - ) - func_name = _get_shared_lambda_name(lambda_expr) - if func_name is not None: - # Return a shared function reference instead of inline lambda - return SharedFunctionLambdaExpression(func_name, parameters, return_type) - return LambdaExpression(parts, parameters, capture, return_type, location) diff --git a/tests/component_tests/text/test_text.py b/tests/component_tests/text/test_text.py index 56dee205b40..bfc3131f6d3 100644 --- a/tests/component_tests/text/test_text.py +++ b/tests/component_tests/text/test_text.py @@ -1,6 +1,4 @@ -"""Tests for the text component.""" - -from esphome.core import CORE +"""Tests for the binary sensor component.""" def test_text_is_setup(generate_main): @@ -58,22 +56,15 @@ def test_text_config_value_mode_set(generate_main): assert "it_3->traits.set_mode(text::TEXT_MODE_PASSWORD);" in main_cpp -def test_text_config_lambda_is_set(generate_main) -> None: +def test_text_config_lamda_is_set(generate_main): """ - Test if lambda is set for lambda mode (optimized with stateless lambda and deduplication) + Test if lambda is set for lambda mode (optimized with stateless lambda) """ # Given # When main_cpp = generate_main("tests/component_tests/text/test_text.yaml") - # Get both global and main sections to find the shared lambda definition - full_cpp = CORE.cpp_global_section + main_cpp - # Then - # Lambda is deduplicated into a shared function (reference in main section) - assert "it_4->set_template(shared_lambda_" in main_cpp - # Lambda body should be in the code somewhere - assert 'return std::string{"Hello"};' in full_cpp - # Verify the shared lambda function is defined (in global section) - assert "esphome::optional shared_lambda_" in full_cpp + assert "it_4->set_template([]() -> esphome::optional {" in main_cpp + assert 'return std::string{"Hello"};' in main_cpp diff --git a/tests/unit_tests/test_lambda_dedup.py b/tests/unit_tests/test_lambda_dedup.py deleted file mode 100644 index bbf5f02e6df..00000000000 --- a/tests/unit_tests/test_lambda_dedup.py +++ /dev/null @@ -1,286 +0,0 @@ -"""Tests for lambda deduplication in cpp_generator.""" - -from esphome import cpp_generator as cg -from esphome.core import CORE - - -def test_deduplicate_identical_lambdas() -> None: - """Test that identical stateless lambdas are deduplicated.""" - # Create two identical lambda expressions - lambda1 = cg.LambdaExpression( - parts=["return 42;"], - parameters=[], - capture="", - return_type=cg.RawExpression("int"), - ) - - lambda2 = cg.LambdaExpression( - parts=["return 42;"], - parameters=[], - capture="", - return_type=cg.RawExpression("int"), - ) - - # Try to deduplicate them - func_name1 = cg._get_shared_lambda_name(lambda1) - func_name2 = cg._get_shared_lambda_name(lambda2) - - # Both should get the same function name (deduplication happened) - assert func_name1 == func_name2 - assert func_name1 == "shared_lambda_0" - - -def test_different_lambdas_not_deduplicated() -> None: - """Test that different lambdas get different function names.""" - lambda1 = cg.LambdaExpression( - parts=["return 42;"], - parameters=[], - capture="", - return_type=cg.RawExpression("int"), - ) - - lambda2 = cg.LambdaExpression( - parts=["return 24;"], # Different content - parameters=[], - capture="", - return_type=cg.RawExpression("int"), - ) - - func_name1 = cg._get_shared_lambda_name(lambda1) - func_name2 = cg._get_shared_lambda_name(lambda2) - - # Different lambdas should get different function names - assert func_name1 != func_name2 - assert func_name1 == "shared_lambda_0" - assert func_name2 == "shared_lambda_1" - - -def test_different_return_types_not_deduplicated() -> None: - """Test that lambdas with different return types are not deduplicated.""" - lambda1 = cg.LambdaExpression( - parts=["return 42;"], - parameters=[], - capture="", - return_type=cg.RawExpression("int"), - ) - - lambda2 = cg.LambdaExpression( - parts=["return 42;"], # Same content - parameters=[], - capture="", - return_type=cg.RawExpression("float"), # Different return type - ) - - func_name1 = cg._get_shared_lambda_name(lambda1) - func_name2 = cg._get_shared_lambda_name(lambda2) - - # Different return types = different functions - assert func_name1 != func_name2 - - -def test_different_parameters_not_deduplicated() -> None: - """Test that lambdas with different parameters are not deduplicated.""" - lambda1 = cg.LambdaExpression( - parts=["return x;"], - parameters=[("int", "x")], - capture="", - return_type=cg.RawExpression("int"), - ) - - lambda2 = cg.LambdaExpression( - parts=["return x;"], # Same content - parameters=[("float", "x")], # Different parameter type - capture="", - return_type=cg.RawExpression("int"), - ) - - func_name1 = cg._get_shared_lambda_name(lambda1) - func_name2 = cg._get_shared_lambda_name(lambda2) - - # Different parameters = different functions - assert func_name1 != func_name2 - - -def test_flush_lambda_dedup_declarations() -> None: - """Test that deferred declarations are properly stored for later flushing.""" - # Create a lambda which will create a deferred declaration - lambda1 = cg.LambdaExpression( - parts=["return 42;"], - parameters=[], - capture="", - return_type=cg.RawExpression("int"), - ) - - cg._get_shared_lambda_name(lambda1) - - # Check that declaration was stored - assert cg._KEY_LAMBDA_DEDUP_DECLARATIONS in CORE.data - assert len(CORE.data[cg._KEY_LAMBDA_DEDUP_DECLARATIONS]) == 1 - - # Verify the declaration content is correct - declaration = CORE.data[cg._KEY_LAMBDA_DEDUP_DECLARATIONS][0] - assert "shared_lambda_0" in declaration - assert "return 42;" in declaration - - # Note: The actual flushing happens via CORE.add_job with FINAL priority - # during real code generation, so we don't test that here - - -def test_shared_function_lambda_expression() -> None: - """Test SharedFunctionLambdaExpression behaves correctly.""" - shared_lambda = cg.SharedFunctionLambdaExpression( - func_name="shared_lambda_0", - parameters=[], - return_type=cg.RawExpression("int"), - ) - - # Should output just the function name - assert str(shared_lambda) == "shared_lambda_0" - - # Should have empty capture (stateless) - assert shared_lambda.capture == "" - - # Should have empty content (just a reference) - assert shared_lambda.content == "" - - -def test_lambda_deduplication_counter() -> None: - """Test that lambda counter increments correctly.""" - # Create 3 different lambdas - for i in range(3): - lambda_expr = cg.LambdaExpression( - parts=[f"return {i};"], - parameters=[], - capture="", - return_type=cg.RawExpression("int"), - ) - func_name = cg._get_shared_lambda_name(lambda_expr) - assert func_name == f"shared_lambda_{i}" - - -def test_lambda_format_body() -> None: - """Test that format_body correctly formats lambda body with source.""" - # Without source - lambda1 = cg.LambdaExpression( - parts=["return 42;"], - parameters=[], - capture="", - return_type=None, - source=None, - ) - assert lambda1.format_body() == "return 42;" - - # With source would need a proper source object, skip for now - - -def test_stateful_lambdas_not_deduplicated() -> None: - """Test that stateful lambdas (non-empty capture) are not deduplicated.""" - # _get_shared_lambda_name is only called for stateless lambdas (capture == "") - # Stateful lambdas bypass deduplication entirely in process_lambda - - # Verify that a stateful lambda would NOT get deduplicated - # by checking it's not in the stateless dedup cache - stateful_lambda = cg.LambdaExpression( - parts=["return x + y;"], - parameters=[], - capture="=", # Non-empty capture means stateful - return_type=cg.RawExpression("int"), - ) - - # Stateful lambdas should NOT be passed to _get_shared_lambda_name - # This is enforced by the `if capture == ""` check in process_lambda - # We verify the lambda has a non-empty capture - assert stateful_lambda.capture != "" - assert stateful_lambda.capture == "=" - - -def test_static_variable_detection() -> None: - """Test detection of static variables in lambda code.""" - # Should detect static variables - assert cg._has_static_variables("static int counter = 0;") - assert cg._has_static_variables("static bool flag = false; return flag;") - assert cg._has_static_variables(" static float value = 1.0; ") - - # Should NOT detect static_cast, static_assert, etc. (with underscores) - assert not cg._has_static_variables("return static_cast(value);") - assert not cg._has_static_variables("static_assert(sizeof(int) == 4);") - assert not cg._has_static_variables("auto ptr = static_pointer_cast(bar);") - - # Edge case: 'cast', 'assert', 'pointer_cast' are NOT C++ keywords - # Someone could use them as type names, but we should NOT flag them - # because they're not actually static variables with state - # NOTE: These are valid C++ but extremely unlikely in ESPHome lambdas - assert not cg._has_static_variables("static cast obj;") # 'cast' as type name - assert not cg._has_static_variables("static assert value;") # 'assert' as type name - assert not cg._has_static_variables( - "static pointer_cast ptr;" - ) # 'pointer_cast' as type - - # Should NOT detect in comments - assert not cg._has_static_variables("// static int x = 0;\nreturn 42;") - assert not cg._has_static_variables("/* static int y = 0; */ return 42;") - - # Should detect even with comments elsewhere - assert cg._has_static_variables("// comment\nstatic int x = 0;\nreturn x;") - - # Should NOT detect non-static code - assert not cg._has_static_variables("int counter = 0; return counter++;") - assert not cg._has_static_variables("return 42;") - - # Should handle newlines between static and type/variable - assert cg._has_static_variables("static int\nfoo = 0;") - assert cg._has_static_variables("static\nint\nbar = 0;") - assert cg._has_static_variables( - "static int \n foo = 0;" - ) # Mixed spaces/newlines - - -def test_lambdas_with_static_not_deduplicated() -> None: - """Test that lambdas with static variables are not deduplicated.""" - # Two identical lambdas with static variables - lambda1 = cg.LambdaExpression( - parts=["static int counter = 0; return counter++;"], - parameters=[], - capture="", - return_type=cg.RawExpression("int"), - ) - - lambda2 = cg.LambdaExpression( - parts=["static int counter = 0; return counter++;"], - parameters=[], - capture="", - return_type=cg.RawExpression("int"), - ) - - # Should return None (not deduplicated) - func_name1 = cg._get_shared_lambda_name(lambda1) - func_name2 = cg._get_shared_lambda_name(lambda2) - - assert func_name1 is None - assert func_name2 is None - - -def test_lambdas_without_static_still_deduplicated() -> None: - """Test that lambdas without static variables are still deduplicated.""" - # Two identical lambdas WITHOUT static variables - lambda1 = cg.LambdaExpression( - parts=["int counter = 0; return counter++;"], # No static - parameters=[], - capture="", - return_type=cg.RawExpression("int"), - ) - - lambda2 = cg.LambdaExpression( - parts=["int counter = 0; return counter++;"], # No static - parameters=[], - capture="", - return_type=cg.RawExpression("int"), - ) - - # Should be deduplicated (same function name) - func_name1 = cg._get_shared_lambda_name(lambda1) - func_name2 = cg._get_shared_lambda_name(lambda2) - - assert func_name1 is not None - assert func_name2 is not None - assert func_name1 == func_name2 From 12a51ff047641d7ecdc3ec1a5b532ba8bdf5e49a Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Wed, 26 Nov 2025 18:00:44 +0100 Subject: [PATCH 2/9] [packages] Fix package schema validation (#12116) Co-authored-by: J. Nick Koston --- esphome/components/packages/__init__.py | 70 ++++++++++++++----- .../component_tests/packages/test_packages.py | 39 +++++++++-- 2 files changed, 87 insertions(+), 22 deletions(-) diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 04057c07f2d..41cde0391bc 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -1,3 +1,4 @@ +import logging from pathlib import Path from esphome import git, yaml_util @@ -20,18 +21,41 @@ from esphome.const import ( ) from esphome.core import EsphomeError +_LOGGER = logging.getLogger(__name__) + DOMAIN = CONF_PACKAGES -def validate_git_package(config: dict): - if CONF_URL not in config: - return config - config = BASE_SCHEMA(config) - new_config = config +def valid_package_contents(package_config: dict): + """Validates that a package_config that will be merged looks as much as possible to a valid config + to fail early on obvious mistakes.""" + if isinstance(package_config, dict): + if CONF_URL in package_config: + # If a URL key is found, then make sure the config conforms to a remote package schema: + return REMOTE_PACKAGE_SCHEMA(package_config) + + # Validate manually since Voluptuous would regenerate dicts and lose metadata + # such as ESPHomeDataBase + for k, v in package_config.items(): + if not isinstance(k, str): + raise cv.Invalid("Package content keys must be strings") + if isinstance(v, (dict, list)): + continue # e.g. script: [] or logger: {level: debug} + if v is None: + continue # e.g. web_server: + raise cv.Invalid("Invalid component content in package definition") + return package_config + + raise cv.Invalid("Package contents must be a dict") + + +def expand_file_to_files(config: dict): if CONF_FILE in config: + new_config = config new_config[CONF_FILES] = [config[CONF_FILE]] del new_config[CONF_FILE] - return new_config + return new_config + return config def validate_yaml_filename(value): @@ -45,7 +69,7 @@ def validate_yaml_filename(value): def validate_source_shorthand(value): if not isinstance(value, str): - raise cv.Invalid("Shorthand only for strings") + raise cv.Invalid("Git URL shorthand only for strings") git_file = git.GitFile.from_shorthand(value) @@ -56,10 +80,17 @@ def validate_source_shorthand(value): if git_file.ref: conf[CONF_REF] = git_file.ref - return BASE_SCHEMA(conf) + return REMOTE_PACKAGE_SCHEMA(conf) -BASE_SCHEMA = cv.All( +def deprecate_single_package(config): + _LOGGER.warning( + "Including a single package under `packages:` is deprecated. Use a list instead." + ) + return config + + +REMOTE_PACKAGE_SCHEMA = cv.All( cv.Schema( { cv.Required(CONF_URL): cv.url, @@ -90,23 +121,30 @@ BASE_SCHEMA = cv.All( } ), cv.has_at_least_one_key(CONF_FILE, CONF_FILES), + expand_file_to_files, ) -PACKAGE_SCHEMA = cv.All( - cv.Any(validate_source_shorthand, BASE_SCHEMA, dict), validate_git_package +PACKAGE_SCHEMA = cv.Any( # A package definition is either: + validate_source_shorthand, # A git URL shorthand string that expands to a remote package schema, or + REMOTE_PACKAGE_SCHEMA, # a valid remote package schema, or + valid_package_contents, # Something that at least looks like an actual package, e.g. {wifi:{ssid: xxx}} + # which will have to be fully validated later as per each component's schema. ) -CONFIG_SCHEMA = cv.Any( +CONFIG_SCHEMA = cv.Any( # under `packages:` we can have either: cv.Schema( { - str: PACKAGE_SCHEMA, + str: PACKAGE_SCHEMA, # a named dict of package definitions, or } ), - [PACKAGE_SCHEMA], + [PACKAGE_SCHEMA], # a list of package definitions, or + cv.All( # a single package definition (deprecated) + cv.ensure_list(PACKAGE_SCHEMA), deprecate_single_package + ), ) -def _process_base_package(config: dict, skip_update: bool = False) -> dict: +def _process_remote_package(config: dict, skip_update: bool = False) -> dict: # When skip_update is True, use NEVER_REFRESH to prevent updates actual_refresh = git.NEVER_REFRESH if skip_update else config[CONF_REFRESH] repo_dir, revert = git.clone_or_update( @@ -185,7 +223,7 @@ def _process_base_package(config: dict, skip_update: bool = False) -> dict: def _process_package(package_config, config, skip_update: bool = False): recursive_package = package_config if CONF_URL in package_config: - package_config = _process_base_package(package_config, skip_update) + package_config = _process_remote_package(package_config, skip_update) if isinstance(package_config, dict): recursive_package = do_packages_pass(package_config, skip_update) return merge_config(recursive_package, config) diff --git a/tests/component_tests/packages/test_packages.py b/tests/component_tests/packages/test_packages.py index 1c4c91aa524..ac4e211fe65 100644 --- a/tests/component_tests/packages/test_packages.py +++ b/tests/component_tests/packages/test_packages.py @@ -95,7 +95,7 @@ def test_package_invalid_dict(basic_esphome, basic_wifi): @pytest.mark.parametrize( - "package", + "packages", [ {"package1": "github://esphome/non-existant-repo/file1.yml@main"}, {"package2": "github://esphome/non-existant-repo/file1.yml"}, @@ -107,12 +107,12 @@ def test_package_invalid_dict(basic_esphome, basic_wifi): ], ], ) -def test_package_shorthand(package): - CONFIG_SCHEMA(package) +def test_package_shorthand(packages): + CONFIG_SCHEMA(packages) @pytest.mark.parametrize( - "package", + "packages", [ # not github {"package1": "someplace://esphome/non-existant-repo/file1.yml@main"}, @@ -133,9 +133,9 @@ def test_package_shorthand(package): [3], ], ) -def test_package_invalid(package): +def test_package_invalid(packages): with pytest.raises(cv.Invalid): - CONFIG_SCHEMA(package) + CONFIG_SCHEMA(packages) def test_package_include(basic_wifi, basic_esphome): @@ -155,6 +155,33 @@ def test_package_include(basic_wifi, basic_esphome): assert actual == expected +def test_single_package( + basic_esphome, + basic_wifi, + caplog: pytest.LogCaptureFixture, +): + """ + Tests the simple case where a single package is added to the top-level config as is. + In this test, the CONF_WIFI config is expected to be simply added to the top-level config. + This tests the case where the user just put packages: !include package.yaml, not + part of a list or mapping of packages. + This behavior is deprecated, the test also checks if a warning is issued. + """ + config = {CONF_ESPHOME: basic_esphome, CONF_PACKAGES: {CONF_WIFI: basic_wifi}} + + expected = {CONF_ESPHOME: basic_esphome, CONF_WIFI: basic_wifi} + + with caplog.at_level("WARNING"): + actual = packages_pass(config) + + assert actual == expected + + assert ( + "Including a single package under `packages:` is deprecated. Use a list instead." + in caplog.text + ) + + def test_package_append(basic_wifi, basic_esphome): """ Tests the case where a key is present in both a package and top-level config. From 083886c4b05a95aa117ba4dfafac17b45908b681 Mon Sep 17 00:00:00 2001 From: Pawelo <81100874+pgolawsk@users.noreply.github.com> Date: Wed, 26 Nov 2025 19:06:51 +0100 Subject: [PATCH 3/9] [prometheus] Avoid generating unused light color metrics to reduce memory usage on ESP8266 (#9530) Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston --- .../prometheus/prometheus_handler.cpp | 135 ++++++++---------- .../prometheus/prometheus_handler.h | 8 ++ tests/components/prometheus/common.yaml | 27 ++++ 3 files changed, 92 insertions(+), 78 deletions(-) diff --git a/esphome/components/prometheus/prometheus_handler.cpp b/esphome/components/prometheus/prometheus_handler.cpp index 812b5478607..252b4774007 100644 --- a/esphome/components/prometheus/prometheus_handler.cpp +++ b/esphome/components/prometheus/prometheus_handler.cpp @@ -141,6 +141,24 @@ void PrometheusHandler::add_friendly_name_label_(AsyncResponseStream *stream, st } } +#ifdef USE_ESP8266 +void PrometheusHandler::print_metric_labels_(AsyncResponseStream *stream, const __FlashStringHelper *metric_name, + EntityBase *obj, std::string &area, std::string &node, + std::string &friendly_name) { +#else +void PrometheusHandler::print_metric_labels_(AsyncResponseStream *stream, const char *metric_name, EntityBase *obj, + std::string &area, std::string &node, std::string &friendly_name) { +#endif + stream->print(metric_name); + stream->print(ESPHOME_F("{id=\"")); + stream->print(relabel_id_(obj).c_str()); + add_area_label_(stream, area); + add_node_label_(stream, node); + add_friendly_name_label_(stream, friendly_name); + stream->print(ESPHOME_F("\",name=\"")); + stream->print(relabel_name_(obj).c_str()); +} + // Type-specific implementation #ifdef USE_SENSOR void PrometheusHandler::sensor_type_(AsyncResponseStream *stream) { @@ -303,13 +321,7 @@ void PrometheusHandler::light_row_(AsyncResponseStream *stream, light::LightStat if (obj->is_internal() && !this->include_internal_) return; // State - stream->print(ESPHOME_F("esphome_light_state{id=\"")); - stream->print(relabel_id_(obj).c_str()); - add_area_label_(stream, area); - add_node_label_(stream, node); - add_friendly_name_label_(stream, friendly_name); - stream->print(ESPHOME_F("\",name=\"")); - stream->print(relabel_name_(obj).c_str()); + print_metric_labels_(stream, ESPHOME_F("esphome_light_state"), obj, area, node, friendly_name); stream->print(ESPHOME_F("\"} ")); stream->print(obj->remote_values.is_on()); stream->print(ESPHOME_F("\n")); @@ -318,78 +330,45 @@ void PrometheusHandler::light_row_(AsyncResponseStream *stream, light::LightStat float brightness, r, g, b, w; color.as_brightness(&brightness); color.as_rgbw(&r, &g, &b, &w); - stream->print(ESPHOME_F("esphome_light_color{id=\"")); - stream->print(relabel_id_(obj).c_str()); - add_area_label_(stream, area); - add_node_label_(stream, node); - add_friendly_name_label_(stream, friendly_name); - stream->print(ESPHOME_F("\",name=\"")); - stream->print(relabel_name_(obj).c_str()); - stream->print(ESPHOME_F("\",channel=\"brightness\"} ")); - stream->print(brightness); - stream->print(ESPHOME_F("\n")); - stream->print(ESPHOME_F("esphome_light_color{id=\"")); - stream->print(relabel_id_(obj).c_str()); - add_area_label_(stream, area); - add_node_label_(stream, node); - add_friendly_name_label_(stream, friendly_name); - stream->print(ESPHOME_F("\",name=\"")); - stream->print(relabel_name_(obj).c_str()); - stream->print(ESPHOME_F("\",channel=\"r\"} ")); - stream->print(r); - stream->print(ESPHOME_F("\n")); - stream->print(ESPHOME_F("esphome_light_color{id=\"")); - stream->print(relabel_id_(obj).c_str()); - add_area_label_(stream, area); - add_node_label_(stream, node); - add_friendly_name_label_(stream, friendly_name); - stream->print(ESPHOME_F("\",name=\"")); - stream->print(relabel_name_(obj).c_str()); - stream->print(ESPHOME_F("\",channel=\"g\"} ")); - stream->print(g); - stream->print(ESPHOME_F("\n")); - stream->print(ESPHOME_F("esphome_light_color{id=\"")); - stream->print(relabel_id_(obj).c_str()); - add_area_label_(stream, area); - add_node_label_(stream, node); - add_friendly_name_label_(stream, friendly_name); - stream->print(ESPHOME_F("\",name=\"")); - stream->print(relabel_name_(obj).c_str()); - stream->print(ESPHOME_F("\",channel=\"b\"} ")); - stream->print(b); - stream->print(ESPHOME_F("\n")); - stream->print(ESPHOME_F("esphome_light_color{id=\"")); - stream->print(relabel_id_(obj).c_str()); - add_area_label_(stream, area); - add_node_label_(stream, node); - add_friendly_name_label_(stream, friendly_name); - stream->print(ESPHOME_F("\",name=\"")); - stream->print(relabel_name_(obj).c_str()); - stream->print(ESPHOME_F("\",channel=\"w\"} ")); - stream->print(w); - stream->print(ESPHOME_F("\n")); - // Effect - std::string effect = obj->get_effect_name(); - if (effect == "None") { - stream->print(ESPHOME_F("esphome_light_effect_active{id=\"")); - stream->print(relabel_id_(obj).c_str()); - add_area_label_(stream, area); - add_node_label_(stream, node); - add_friendly_name_label_(stream, friendly_name); - stream->print(ESPHOME_F("\",name=\"")); - stream->print(relabel_name_(obj).c_str()); - stream->print(ESPHOME_F("\",effect=\"None\"} 0\n")); - } else { - stream->print(ESPHOME_F("esphome_light_effect_active{id=\"")); - stream->print(relabel_id_(obj).c_str()); - add_area_label_(stream, area); - add_node_label_(stream, node); - add_friendly_name_label_(stream, friendly_name); - stream->print(ESPHOME_F("\",name=\"")); - stream->print(relabel_name_(obj).c_str()); + if (obj->get_traits().supports_color_capability(light::ColorCapability::BRIGHTNESS)) { + print_metric_labels_(stream, ESPHOME_F("esphome_light_color"), obj, area, node, friendly_name); + stream->print(ESPHOME_F("\",channel=\"brightness\"} ")); + stream->print(brightness); + stream->print(ESPHOME_F("\n")); + } + if (obj->get_traits().supports_color_capability(light::ColorCapability::RGB)) { + print_metric_labels_(stream, ESPHOME_F("esphome_light_color"), obj, area, node, friendly_name); + stream->print(ESPHOME_F("\",channel=\"r\"} ")); + stream->print(r); + stream->print(ESPHOME_F("\n")); + print_metric_labels_(stream, ESPHOME_F("esphome_light_color"), obj, area, node, friendly_name); + stream->print(ESPHOME_F("\",channel=\"g\"} ")); + stream->print(g); + stream->print(ESPHOME_F("\n")); + print_metric_labels_(stream, ESPHOME_F("esphome_light_color"), obj, area, node, friendly_name); + stream->print(ESPHOME_F("\",channel=\"b\"} ")); + stream->print(b); + stream->print(ESPHOME_F("\n")); + } + if (obj->get_traits().supports_color_capability(light::ColorCapability::WHITE)) { + print_metric_labels_(stream, ESPHOME_F("esphome_light_color"), obj, area, node, friendly_name); + stream->print(ESPHOME_F("\",channel=\"w\"} ")); + stream->print(w); + stream->print(ESPHOME_F("\n")); + } + // Skip effect metrics if light has no effects + if (!obj->get_effects().empty()) { + // Effect + std::string effect = obj->get_effect_name(); + print_metric_labels_(stream, ESPHOME_F("esphome_light_effect_active"), obj, area, node, friendly_name); stream->print(ESPHOME_F("\",effect=\"")); - stream->print(effect.c_str()); - stream->print(ESPHOME_F("\"} 1\n")); + // Only vary based on effect + if (effect == "None") { + stream->print(ESPHOME_F("None\"} 0\n")); + } else { + stream->print(effect.c_str()); + stream->print(ESPHOME_F("\"} 1\n")); + } } } #endif diff --git a/esphome/components/prometheus/prometheus_handler.h b/esphome/components/prometheus/prometheus_handler.h index 45cc81b899c..24243c8c98d 100644 --- a/esphome/components/prometheus/prometheus_handler.h +++ b/esphome/components/prometheus/prometheus_handler.h @@ -66,6 +66,14 @@ class PrometheusHandler : public AsyncWebHandler, public Component { void add_area_label_(AsyncResponseStream *stream, std::string &area); void add_node_label_(AsyncResponseStream *stream, std::string &node); void add_friendly_name_label_(AsyncResponseStream *stream, std::string &friendly_name); + /// Print metric name and common labels (id, area, node, friendly_name, name) +#ifdef USE_ESP8266 + void print_metric_labels_(AsyncResponseStream *stream, const __FlashStringHelper *metric_name, EntityBase *obj, + std::string &area, std::string &node, std::string &friendly_name); +#else + void print_metric_labels_(AsyncResponseStream *stream, const char *metric_name, EntityBase *obj, std::string &area, + std::string &node, std::string &friendly_name); +#endif #ifdef USE_SENSOR /// Return the type for prometheus diff --git a/tests/components/prometheus/common.yaml b/tests/components/prometheus/common.yaml index 0b90d614ddb..7ff416dccbe 100644 --- a/tests/components/prometheus/common.yaml +++ b/tests/components/prometheus/common.yaml @@ -112,6 +112,25 @@ cover: } return COVER_CLOSED; +light: + - platform: binary + name: "Binary Light" + output: test_output + - platform: monochromatic + name: "Brightness Light" + output: test_output + - platform: rgb + name: "RGB Light" + red: test_output + green: test_output + blue: test_output + - platform: rgbw + name: "RGBW Light" + red: test_output + green: test_output + blue: test_output + white: test_output + lock: - platform: template id: template_lock1 @@ -122,6 +141,14 @@ lock: return LOCK_STATE_UNLOCKED; optimistic: true +output: + - platform: template + id: test_output + type: float + write_action: + - lambda: |- + // no-op for CI/build tests + (void)state; select: - platform: template id: template_select1 From 1d59c7a838578c0055499cdeb3f2006a8908eba1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Nov 2025 12:08:49 -0600 Subject: [PATCH 4/9] [scheduler] Fix use-after-move crash in heap operations --- esphome/core/scheduler.cpp | 24 ++++++++++++------------ esphome/core/scheduler.h | 4 +++- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index 09d50ee7c81..ae486ea3b0d 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -359,8 +359,7 @@ void HOT Scheduler::call(uint32_t now) { std::unique_ptr item; { LockGuard guard{this->lock_}; - item = std::move(this->items_[0]); - this->pop_raw_(); + item = this->pop_raw_(); } const char *name = item->get_name(); @@ -401,7 +400,7 @@ void HOT Scheduler::call(uint32_t now) { // Don't run on failed components if (item->component != nullptr && item->component->is_failed()) { LockGuard guard{this->lock_}; - this->pop_raw_(); + this->recycle_item_(this->pop_raw_()); continue; } @@ -414,7 +413,7 @@ void HOT Scheduler::call(uint32_t now) { { LockGuard guard{this->lock_}; if (is_item_removed_(item.get())) { - this->pop_raw_(); + this->recycle_item_(this->pop_raw_()); this->to_remove_--; continue; } @@ -423,7 +422,7 @@ void HOT Scheduler::call(uint32_t now) { // Single-threaded or multi-threaded with atomics: can check without lock if (is_item_removed_(item.get())) { LockGuard guard{this->lock_}; - this->pop_raw_(); + this->recycle_item_(this->pop_raw_()); this->to_remove_--; continue; } @@ -443,14 +442,14 @@ void HOT Scheduler::call(uint32_t now) { LockGuard guard{this->lock_}; - auto executed_item = std::move(this->items_[0]); // Only pop after function call, this ensures we were reachable // during the function call and know if we were cancelled. - this->pop_raw_(); + auto executed_item = this->pop_raw_(); if (executed_item->remove) { - // We were removed/cancelled in the function call, stop + // We were removed/cancelled in the function call, recycle and continue this->to_remove_--; + this->recycle_item_(std::move(executed_item)); continue; } @@ -510,17 +509,18 @@ size_t HOT Scheduler::cleanup_() { if (!item->remove) break; this->to_remove_--; - this->pop_raw_(); + this->recycle_item_(this->pop_raw_()); } return this->items_.size(); } -void HOT Scheduler::pop_raw_() { +std::unique_ptr HOT Scheduler::pop_raw_() { std::pop_heap(this->items_.begin(), this->items_.end(), SchedulerItem::cmp); - // Instead of destroying, recycle the item - this->recycle_item_(std::move(this->items_.back())); + // Move the item out before popping - this is the item that was at the front of the heap + auto item = std::move(this->items_.back()); this->items_.pop_back(); + return item; } // Helper to execute a scheduler item diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index bea1503df0e..476681a787e 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -219,7 +219,9 @@ class Scheduler { // Returns the number of items remaining after cleanup // IMPORTANT: This method should only be called from the main thread (loop task). size_t cleanup_(); - void pop_raw_(); + // Remove and return the front item from the heap + // IMPORTANT: Caller must hold the scheduler lock before calling this function. + std::unique_ptr pop_raw_(); private: // Helper to cancel items by name - must be called with lock held From 877d2b914c11d4a0ef7b163be1c67e724d6d4bfb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 26 Nov 2025 16:38:39 -0600 Subject: [PATCH 5/9] tweaks --- esphome/core/scheduler.cpp | 16 ++++++++-------- esphome/core/scheduler.h | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index ae486ea3b0d..352587bf10b 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -359,7 +359,7 @@ void HOT Scheduler::call(uint32_t now) { std::unique_ptr item; { LockGuard guard{this->lock_}; - item = this->pop_raw_(); + item = this->pop_raw_locked_(); } const char *name = item->get_name(); @@ -400,7 +400,7 @@ void HOT Scheduler::call(uint32_t now) { // Don't run on failed components if (item->component != nullptr && item->component->is_failed()) { LockGuard guard{this->lock_}; - this->recycle_item_(this->pop_raw_()); + this->recycle_item_(this->pop_raw_locked_()); continue; } @@ -413,7 +413,7 @@ void HOT Scheduler::call(uint32_t now) { { LockGuard guard{this->lock_}; if (is_item_removed_(item.get())) { - this->recycle_item_(this->pop_raw_()); + this->recycle_item_(this->pop_raw_locked_()); this->to_remove_--; continue; } @@ -422,7 +422,7 @@ void HOT Scheduler::call(uint32_t now) { // Single-threaded or multi-threaded with atomics: can check without lock if (is_item_removed_(item.get())) { LockGuard guard{this->lock_}; - this->recycle_item_(this->pop_raw_()); + this->recycle_item_(this->pop_raw_locked_()); this->to_remove_--; continue; } @@ -444,7 +444,7 @@ void HOT Scheduler::call(uint32_t now) { // Only pop after function call, this ensures we were reachable // during the function call and know if we were cancelled. - auto executed_item = this->pop_raw_(); + auto executed_item = this->pop_raw_locked_(); if (executed_item->remove) { // We were removed/cancelled in the function call, recycle and continue @@ -496,7 +496,7 @@ size_t HOT Scheduler::cleanup_() { return this->items_.size(); // We must hold the lock for the entire cleanup operation because: - // 1. We're modifying items_ (via pop_raw_) which requires exclusive access + // 1. We're modifying items_ (via pop_raw_locked_) which requires exclusive access // 2. We're decrementing to_remove_ which is also modified by other threads // (though all modifications are already under lock) // 3. Other threads read items_ when searching for items to cancel in cancel_item_locked_() @@ -509,11 +509,11 @@ size_t HOT Scheduler::cleanup_() { if (!item->remove) break; this->to_remove_--; - this->recycle_item_(this->pop_raw_()); + this->recycle_item_(this->pop_raw_locked_()); } return this->items_.size(); } -std::unique_ptr HOT Scheduler::pop_raw_() { +std::unique_ptr HOT Scheduler::pop_raw_locked_() { std::pop_heap(this->items_.begin(), this->items_.end(), SchedulerItem::cmp); // Move the item out before popping - this is the item that was at the front of the heap diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 476681a787e..08e003c9fbc 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -221,7 +221,7 @@ class Scheduler { size_t cleanup_(); // Remove and return the front item from the heap // IMPORTANT: Caller must hold the scheduler lock before calling this function. - std::unique_ptr pop_raw_(); + std::unique_ptr pop_raw_locked_(); private: // Helper to cancel items by name - must be called with lock held From eb970cf44ec07ea45596f837c573e4fec7ff09d7 Mon Sep 17 00:00:00 2001 From: Jon Oberheide <506986+jonoberheide@users.noreply.github.com> Date: Wed, 26 Nov 2025 17:56:22 -0500 Subject: [PATCH 6/9] make thermostat humidification_action public (#12132) --- esphome/components/thermostat/thermostat_climate.cpp | 8 ++++---- esphome/components/thermostat/thermostat_climate.h | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/esphome/components/thermostat/thermostat_climate.cpp b/esphome/components/thermostat/thermostat_climate.cpp index 2b51f58f4ff..e79eed40551 100644 --- a/esphome/components/thermostat/thermostat_climate.cpp +++ b/esphome/components/thermostat/thermostat_climate.cpp @@ -654,7 +654,7 @@ void ThermostatClimate::trigger_supplemental_action_() { void ThermostatClimate::switch_to_humidity_control_action_(HumidificationAction action) { // setup_complete_ helps us ensure an action is called immediately after boot - if ((action == this->humidification_action_) && this->setup_complete_) { + if ((action == this->humidification_action) && this->setup_complete_) { // already in target mode return; } @@ -683,7 +683,7 @@ void ThermostatClimate::switch_to_humidity_control_action_(HumidificationAction this->prev_humidity_control_trigger_->stop_action(); this->prev_humidity_control_trigger_ = nullptr; } - this->humidification_action_ = action; + this->humidification_action = action; this->prev_humidity_control_trigger_ = trig; if (trig != nullptr) { trig->trigger(); @@ -1114,7 +1114,7 @@ bool ThermostatClimate::dehumidification_required_() { } // if we get here, the current humidity is between target + hysteresis and target - hysteresis, // so the action should not change - return this->humidification_action_ == THERMOSTAT_HUMIDITY_CONTROL_ACTION_DEHUMIDIFY; + return this->humidification_action == THERMOSTAT_HUMIDITY_CONTROL_ACTION_DEHUMIDIFY; } bool ThermostatClimate::humidification_required_() { @@ -1127,7 +1127,7 @@ bool ThermostatClimate::humidification_required_() { } // if we get here, the current humidity is between target - hysteresis and target + hysteresis, // so the action should not change - return this->humidification_action_ == THERMOSTAT_HUMIDITY_CONTROL_ACTION_HUMIDIFY; + return this->humidification_action == THERMOSTAT_HUMIDITY_CONTROL_ACTION_HUMIDIFY; } void ThermostatClimate::dump_preset_config_(const char *preset_name, const ThermostatClimateTargetTempConfig &config) { diff --git a/esphome/components/thermostat/thermostat_climate.h b/esphome/components/thermostat/thermostat_climate.h index 76391f800cb..69d2307b1c3 100644 --- a/esphome/components/thermostat/thermostat_climate.h +++ b/esphome/components/thermostat/thermostat_climate.h @@ -207,6 +207,9 @@ class ThermostatClimate : public climate::Climate, public Component { void validate_target_temperature_high(); void validate_target_humidity(); + /// The current humidification action + HumidificationAction humidification_action{THERMOSTAT_HUMIDITY_CONTROL_ACTION_NONE}; + protected: /// Override control to change settings of the climate device. void control(const climate::ClimateCall &call) override; @@ -301,9 +304,6 @@ class ThermostatClimate : public climate::Climate, public Component { /// The current supplemental action climate::ClimateAction supplemental_action_{climate::CLIMATE_ACTION_OFF}; - /// The current humidification action - HumidificationAction humidification_action_{THERMOSTAT_HUMIDITY_CONTROL_ACTION_NONE}; - /// Default standard preset to use on start up climate::ClimatePreset default_preset_{}; From caaa08d678f43ebcad29e837dfc25137f21d2776 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 27 Nov 2025 09:05:45 +1000 Subject: [PATCH 7/9] [core] Fix for missing arguments to shared_lambda (#12115) --- esphome/components/lvgl/widgets/__init__.py | 4 ++-- esphome/cpp_generator.py | 8 +------- tests/components/lvgl/lvgl-package.yaml | 12 ++++++++++++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/esphome/components/lvgl/widgets/__init__.py b/esphome/components/lvgl/widgets/__init__.py index 2e7948522e7..b1d157325be 100644 --- a/esphome/components/lvgl/widgets/__init__.py +++ b/esphome/components/lvgl/widgets/__init__.py @@ -382,7 +382,7 @@ async def set_obj_properties(w: Widget, config): clrs = join_enums(flag_clr, "LV_OBJ_FLAG_") w.clear_flag(clrs) for key, value in lambs.items(): - lamb = await cg.process_lambda(value, [], return_type=cg.bool_) + lamb = await cg.process_lambda(value, [], capture="=", return_type=cg.bool_) flag = f"LV_OBJ_FLAG_{key.upper()}" with LvConditional(call_lambda(lamb)) as cond: w.add_flag(flag) @@ -407,7 +407,7 @@ async def set_obj_properties(w: Widget, config): clears = join_enums(clears, "LV_STATE_") w.clear_state(clears) for key, value in lambs.items(): - lamb = await cg.process_lambda(value, [], return_type=cg.bool_) + lamb = await cg.process_lambda(value, [], capture="=", return_type=cg.bool_) state = f"LV_STATE_{key.upper()}" with LvConditional(call_lambda(lamb)) as cond: w.add_state(state) diff --git a/esphome/cpp_generator.py b/esphome/cpp_generator.py index 6f1af01a5be..1a47b346b77 100644 --- a/esphome/cpp_generator.py +++ b/esphome/cpp_generator.py @@ -659,7 +659,7 @@ async def get_variable_with_full_id(id_: ID) -> tuple[ID, "MockObj"]: async def process_lambda( value: Lambda, parameters: TemplateArgsType, - capture: str = "=", + capture: str = "", return_type: SafeExpType = None, ) -> LambdaExpression | None: """Process the given lambda value into a LambdaExpression. @@ -702,12 +702,6 @@ async def process_lambda( parts[i * 3 + 1] = var parts[i * 3 + 2] = "" - # All id() references are global variables in generated C++ code. - # Global variables should not be captured - they're accessible everywhere. - # Use empty capture instead of capture-by-value. - if capture == "=": - capture = "" - if isinstance(value, ESPHomeDataBase) and value.esp_range is not None: location = value.esp_range.start_mark location.line += value.content_offset diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index 30866a603c8..a5714d5639a 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -16,6 +16,18 @@ binary_sensor: platform: template - id: left_sensor platform: template + - platform: lvgl + id: button_checker + name: LVGL button + widget: button_button + on_state: + then: + - lvgl.checkbox.update: + id: checkbox_id + state: + checked: !lambda |- + auto y = x; // block inlining of one line return + return y; lvgl: log_level: debug From a2d9941c622388f827570077dc6ab45d1d80112f Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 27 Nov 2025 09:06:32 +1000 Subject: [PATCH 8/9] [lvgl] Add option to sync updates with display (#11896) Co-authored-by: J. Nick Koston --- esphome/components/lvgl/__init__.py | 4 ++ esphome/components/lvgl/defines.py | 1 + esphome/components/lvgl/lvgl_esphome.cpp | 48 +++++++++++++++++++---- esphome/components/lvgl/lvgl_esphome.h | 13 +++--- tests/components/lvgl/test.esp32-idf.yaml | 1 + 5 files changed, 55 insertions(+), 12 deletions(-) diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index eaa37b54dd2..eeabf755a60 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -276,6 +276,7 @@ async def to_code(configs): config[df.CONF_FULL_REFRESH], config[CONF_DRAW_ROUNDING], config[df.CONF_RESUME_ON_INPUT], + config[df.CONF_UPDATE_WHEN_DISPLAY_IDLE], ) await cg.register_component(lv_component, config) Widget.create(config[CONF_ID], lv_component, LvScrActType(), config) @@ -373,6 +374,9 @@ LVGL_SCHEMA = cv.All( df.CONF_DEFAULT_FONT, default="montserrat_14" ): lvalid.lv_font, cv.Optional(df.CONF_FULL_REFRESH, default=False): cv.boolean, + cv.Optional( + df.CONF_UPDATE_WHEN_DISPLAY_IDLE, default=False + ): cv.boolean, cv.Optional(CONF_DRAW_ROUNDING, default=2): cv.positive_int, cv.Optional(CONF_BUFFER_SIZE, default=0): cv.percentage, cv.Optional(CONF_LOG_LEVEL, default="WARN"): cv.one_of( diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index 6b3b9c97ef8..1fce6fa458b 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -542,6 +542,7 @@ CONF_TOUCHSCREENS = "touchscreens" CONF_TRANSPARENCY_KEY = "transparency_key" CONF_THEME = "theme" CONF_UPDATE_ON_RELEASE = "update_on_release" +CONF_UPDATE_WHEN_DISPLAY_IDLE = "update_when_display_idle" CONF_VISIBLE_ROW_COUNT = "visible_row_count" CONF_WIDGET = "widget" CONF_WIDGETS = "widgets" diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index fbcd68378c8..18226a9f57e 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -106,6 +106,7 @@ void LvglComponent::dump_config() { this->disp_drv_.hor_res, this->disp_drv_.ver_res, 100 / this->buffer_frac_, this->rotation, (int) this->draw_rounding); } + void LvglComponent::set_paused(bool paused, bool show_snow) { this->paused_ = paused; this->show_snow_ = show_snow; @@ -124,32 +125,38 @@ void LvglComponent::esphome_lvgl_init() { lv_update_event = static_cast(lv_event_register_id()); lv_api_event = static_cast(lv_event_register_id()); } + void LvglComponent::add_event_cb(lv_obj_t *obj, event_callback_t callback, lv_event_code_t event) { lv_obj_add_event_cb(obj, callback, event, nullptr); } + void LvglComponent::add_event_cb(lv_obj_t *obj, event_callback_t callback, lv_event_code_t event1, lv_event_code_t event2) { add_event_cb(obj, callback, event1); add_event_cb(obj, callback, event2); } + void LvglComponent::add_event_cb(lv_obj_t *obj, event_callback_t callback, lv_event_code_t event1, lv_event_code_t event2, lv_event_code_t event3) { add_event_cb(obj, callback, event1); add_event_cb(obj, callback, event2); add_event_cb(obj, callback, event3); } + void LvglComponent::add_page(LvPageType *page) { this->pages_.push_back(page); page->set_parent(this); lv_disp_set_default(this->disp_); page->setup(this->pages_.size() - 1); } + void LvglComponent::show_page(size_t index, lv_scr_load_anim_t anim, uint32_t time) { if (index >= this->pages_.size()) return; this->current_page_ = index; lv_scr_load_anim(this->pages_[this->current_page_]->obj, anim, time, 0, false); } + void LvglComponent::show_next_page(lv_scr_load_anim_t anim, uint32_t time) { if (this->pages_.empty() || (this->current_page_ == this->pages_.size() - 1 && !this->page_wrap_)) return; @@ -158,6 +165,7 @@ void LvglComponent::show_next_page(lv_scr_load_anim_t anim, uint32_t time) { } while (this->pages_[this->current_page_]->skip); // skip empty pages() this->show_page(this->current_page_, anim, time); } + void LvglComponent::show_prev_page(lv_scr_load_anim_t anim, uint32_t time) { if (this->pages_.empty() || (this->current_page_ == 0 && !this->page_wrap_)) return; @@ -166,8 +174,10 @@ void LvglComponent::show_prev_page(lv_scr_load_anim_t anim, uint32_t time) { } while (this->pages_[this->current_page_]->skip); // skip empty pages() this->show_page(this->current_page_, anim, time); } + size_t LvglComponent::get_current_page() const { return this->current_page_; } bool LvPageType::is_showing() const { return this->parent_->get_current_page() == this->index; } + void LvglComponent::draw_buffer_(const lv_area_t *area, lv_color_t *ptr) { auto width = lv_area_get_width(area); auto height = lv_area_get_height(area); @@ -222,7 +232,7 @@ void LvglComponent::draw_buffer_(const lv_area_t *area, lv_color_t *ptr) { } void LvglComponent::flush_cb_(lv_disp_drv_t *disp_drv, const lv_area_t *area, lv_color_t *color_p) { - if (!this->paused_) { + if (!this->is_paused()) { auto now = millis(); this->draw_buffer_(area, color_p); ESP_LOGVV(TAG, "flush_cb, area=%d/%d, %d/%d took %dms", area->x1, area->y1, lv_area_get_width(area), @@ -230,6 +240,7 @@ void LvglComponent::flush_cb_(lv_disp_drv_t *disp_drv, const lv_area_t *area, lv } lv_disp_flush_ready(disp_drv); } + IdleTrigger::IdleTrigger(LvglComponent *parent, TemplatableValue timeout) : timeout_(std::move(timeout)) { parent->add_on_idle_callback([this](uint32_t idle_time) { if (!this->is_idle_ && idle_time > this->timeout_.value()) { @@ -377,6 +388,27 @@ void LvKeyboardType::set_obj(lv_obj_t *lv_obj) { } #endif // USE_LVGL_KEYBOARD +void LvglComponent::draw_end_() { + if (this->draw_end_callback_ != nullptr) + this->draw_end_callback_->trigger(); + if (this->update_when_display_idle_) { + for (auto *disp : this->displays_) + disp->update(); + } +} + +bool LvglComponent::is_paused() const { + if (this->paused_) + return true; + if (this->update_when_display_idle_) { + for (auto *disp : this->displays_) { + if (!disp->is_idle()) + return true; + } + } + return false; +} + void LvglComponent::write_random_() { int iterations = 6 - lv_disp_get_inactive_time(this->disp_) / 60000; if (iterations <= 0) @@ -426,12 +458,13 @@ void LvglComponent::write_random_() { * presses a key or clicks on the screen. */ LvglComponent::LvglComponent(std::vector displays, float buffer_frac, bool full_refresh, - int draw_rounding, bool resume_on_input) + int draw_rounding, bool resume_on_input, bool update_when_display_idle) : draw_rounding(draw_rounding), displays_(std::move(displays)), buffer_frac_(buffer_frac), full_refresh_(full_refresh), - resume_on_input_(resume_on_input) { + resume_on_input_(resume_on_input), + update_when_display_idle_(update_when_display_idle) { lv_disp_draw_buf_init(&this->draw_buf_, nullptr, nullptr, 0); lv_disp_drv_init(&this->disp_drv_); this->disp_drv_.draw_buf = &this->draw_buf_; @@ -487,7 +520,7 @@ void LvglComponent::setup() { if (this->draw_start_callback_ != nullptr) { this->disp_drv_.render_start_cb = render_start_cb; } - if (this->draw_end_callback_ != nullptr) { + if (this->draw_end_callback_ != nullptr || this->update_when_display_idle_) { this->disp_drv_.monitor_cb = monitor_cb; } #if LV_USE_LOG @@ -509,14 +542,15 @@ void LvglComponent::setup() { void LvglComponent::update() { // update indicators - if (this->paused_) { + if (this->is_paused()) { return; } this->idle_callbacks_.call(lv_disp_get_inactive_time(this->disp_)); } + void LvglComponent::loop() { - if (this->paused_) { - if (this->show_snow_) + if (this->is_paused()) { + if (this->paused_ && this->show_snow_) this->write_random_(); } else { lv_timer_handler_run_in_period(5); diff --git a/esphome/components/lvgl/lvgl_esphome.h b/esphome/components/lvgl/lvgl_esphome.h index bd6f1fdb61a..9c82f3646bc 100644 --- a/esphome/components/lvgl/lvgl_esphome.h +++ b/esphome/components/lvgl/lvgl_esphome.h @@ -151,7 +151,7 @@ class LvglComponent : public PollingComponent { public: LvglComponent(std::vector displays, float buffer_frac, bool full_refresh, int draw_rounding, - bool resume_on_input); + bool resume_on_input, bool update_when_display_idle); static void static_flush_cb(lv_disp_drv_t *disp_drv, const lv_area_t *area, lv_color_t *color_p); float get_setup_priority() const override { return setup_priority::PROCESSOR; } @@ -171,7 +171,9 @@ class LvglComponent : public PollingComponent { // @param paused If true, pause the display. If false, resume the display. // @param show_snow If true, show the snow effect when paused. void set_paused(bool paused, bool show_snow); - bool is_paused() const { return this->paused_; } + + // Returns true if the display is explicitly paused, or a blocking display update is in progress. + bool is_paused() const; // If the display is paused and we have resume_on_input_ set to true, resume the display. void maybe_wakeup() { if (this->paused_ && this->resume_on_input_) { @@ -210,10 +212,10 @@ class LvglComponent : public PollingComponent { void set_draw_end_trigger(Trigger<> *trigger) { this->draw_end_callback_ = trigger; } protected: - // these functions are never called unless the callbacks are non-null since the - // LVGL callbacks that call them are not set unless the start/end callbacks are non-null + void draw_end_(); + // Not checking for non-null callback since the + // LVGL callback that calls it is not set in that case void draw_start_() const { this->draw_start_callback_->trigger(); } - void draw_end_() const { this->draw_end_callback_->trigger(); } void write_random_(); void draw_buffer_(const lv_area_t *area, lv_color_t *ptr); @@ -222,6 +224,7 @@ class LvglComponent : public PollingComponent { size_t buffer_frac_{1}; bool full_refresh_{}; bool resume_on_input_{}; + bool update_when_display_idle_{}; lv_disp_draw_buf_t draw_buf_{}; lv_disp_drv_t disp_drv_{}; diff --git a/tests/components/lvgl/test.esp32-idf.yaml b/tests/components/lvgl/test.esp32-idf.yaml index 2450d28eb8e..e6025e17fc6 100644 --- a/tests/components/lvgl/test.esp32-idf.yaml +++ b/tests/components/lvgl/test.esp32-idf.yaml @@ -60,6 +60,7 @@ display: update_interval: never lvgl: + update_when_display_idle: true displays: - tft_display - second_display From 927d3715c1ab16d5963ee404790b5ae142b1b613 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 27 Nov 2025 09:06:40 +1000 Subject: [PATCH 9/9] [lvgl] Allow setting text directly on a button (#11964) Co-authored-by: J. Nick Koston --- esphome/components/lvgl/__init__.py | 18 +++++++--- esphome/components/lvgl/defines.py | 22 ++++++++++-- esphome/components/lvgl/schemas.py | 35 ++++++++++++++----- esphome/components/lvgl/types.py | 16 ++++++--- esphome/components/lvgl/widgets/button.py | 42 ++++++++++++++++++++--- tests/components/lvgl/lvgl-package.yaml | 8 +++++ 6 files changed, 115 insertions(+), 26 deletions(-) diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index eeabf755a60..040661495c5 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -108,7 +108,7 @@ LV_CONF_H_FORMAT = """\ def generate_lv_conf_h(): - definitions = [as_macro(m, v) for m, v in df.lv_defines.items()] + definitions = [as_macro(m, v) for m, v in df.get_data(df.KEY_LV_DEFINES).items()] definitions.sort() return LV_CONF_H_FORMAT.format("\n".join(definitions)) @@ -140,11 +140,11 @@ def multi_conf_validate(configs: list[dict]): ) -def final_validation(configs): - if len(configs) != 1: - multi_conf_validate(configs) +def final_validation(config_list): + if len(config_list) != 1: + multi_conf_validate(config_list) global_config = full_config.get() - for config in configs: + for config in config_list: if (pages := config.get(CONF_PAGES)) and all(p[df.CONF_SKIP] for p in pages): raise cv.Invalid("At least one page must not be skipped") for display_id in config[df.CONF_DISPLAYS]: @@ -190,6 +190,14 @@ def final_validation(configs): raise cv.Invalid( f"Widget '{w}' does not have any dynamic properties to refresh", ) + # Do per-widget type final validation for update actions + for widget_type, update_configs in df.get_data(df.KEY_UPDATED_WIDGETS).items(): + for conf in update_configs: + for id_conf in conf.get(CONF_ID, ()): + name = id_conf[CONF_ID] + path = global_config.get_path_for_id(name) + widget_conf = global_config.get_config_for_path(path[:-1]) + widget_type.final_validate(name, conf, widget_conf, path[1:]) async def to_code(configs): diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index 1fce6fa458b..1d528b2f735 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any from esphome import codegen as cg, config_validation as cv from esphome.const import CONF_ITEMS -from esphome.core import ID, Lambda +from esphome.core import CORE, ID, Lambda from esphome.cpp_generator import LambdaExpression, MockObj from esphome.cpp_types import uint32 from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor @@ -20,11 +20,27 @@ from .helpers import requires_component LOGGER = logging.getLogger(__name__) lvgl_ns = cg.esphome_ns.namespace("lvgl") -lv_defines = {} # Dict of #defines to provide as build flags +DOMAIN = "lvgl" +KEY_LV_DEFINES = "lv_defines" +KEY_UPDATED_WIDGETS = "updated_widgets" + + +def get_data(key, default=None): + """ + Get a data structure from the global data store by key + :param key: A key for the data + :param default: The default data - the default is an empty dict + :return: + """ + return CORE.data.setdefault(DOMAIN, {}).setdefault( + key, default if default is not None else {} + ) def add_define(macro, value="1"): - if macro in lv_defines and lv_defines[macro] != value: + lv_defines = get_data(KEY_LV_DEFINES) + value = str(value) + if lv_defines.setdefault(macro, value) != value: LOGGER.error( "Redefinition of %s - was %s now %s", macro, lv_defines[macro], value ) diff --git a/esphome/components/lvgl/schemas.py b/esphome/components/lvgl/schemas.py index f2704f99deb..45d933c00e6 100644 --- a/esphome/components/lvgl/schemas.py +++ b/esphome/components/lvgl/schemas.py @@ -1,3 +1,5 @@ +from collections.abc import Callable + from esphome import config_validation as cv from esphome.automation import Trigger, validate_automation from esphome.components.time import RealTimeClock @@ -311,19 +313,36 @@ def automation_schema(typ: LvType): } -def base_update_schema(widget_type, parts): +def _update_widget(widget_type: WidgetType) -> Callable[[dict], dict]: """ - Create a schema for updating a widgets style properties, states and flags + During validation of update actions, create a map of action types to affected widgets + for use in final validation. + :param widget_type: + :return: + """ + + def validator(value: dict) -> dict: + df.get_data(df.KEY_UPDATED_WIDGETS).setdefault(widget_type, []).append(value) + return value + + return validator + + +def base_update_schema(widget_type: WidgetType | LvType, parts): + """ + Create a schema for updating a widget's style properties, states and flags. :param widget_type: The type of the ID :param parts: The allowable parts to specify :return: """ - return part_schema(parts).extend( + + w_type = widget_type.w_type if isinstance(widget_type, WidgetType) else widget_type + schema = part_schema(parts).extend( { cv.Required(CONF_ID): cv.ensure_list( cv.maybe_simple_value( { - cv.Required(CONF_ID): cv.use_id(widget_type), + cv.Required(CONF_ID): cv.use_id(w_type), }, key=CONF_ID, ) @@ -332,11 +351,9 @@ def base_update_schema(widget_type, parts): } ) - -def create_modify_schema(widget_type): - return base_update_schema(widget_type.w_type, widget_type.parts).extend( - widget_type.modify_schema - ) + if isinstance(widget_type, WidgetType): + schema.add_extra(_update_widget(widget_type)) + return schema def obj_schema(widget_type: WidgetType): diff --git a/esphome/components/lvgl/types.py b/esphome/components/lvgl/types.py index b99c0ad5a38..9c92ca7e986 100644 --- a/esphome/components/lvgl/types.py +++ b/esphome/components/lvgl/types.py @@ -152,18 +152,18 @@ class WidgetType: # Local import to avoid circular import from .automation import update_to_code - from .schemas import WIDGET_TYPES, create_modify_schema + from .schemas import WIDGET_TYPES, base_update_schema if not is_mock: if self.name in WIDGET_TYPES: raise EsphomeError(f"Duplicate definition of widget type '{self.name}'") WIDGET_TYPES[self.name] = self - # Register the update action automatically + # Register the update action automatically, adding widget-specific properties register_action( f"lvgl.{self.name}.update", ObjUpdateAction, - create_modify_schema(self), + base_update_schema(self, self.parts).extend(self.modify_schema), )(update_to_code) @property @@ -182,7 +182,6 @@ class WidgetType: Generate code for a given widget :param w: The widget :param config: Its configuration - :return: Generated code as a list of text lines """ async def obj_creator(self, parent: MockObjClass, config: dict): @@ -228,6 +227,15 @@ class WidgetType: """ return value + def final_validate(self, widget, update_config, widget_config, path): + """ + Allow final validation for a given widget type update action + :param widget: A widget + :param update_config: The configuration for the update action + :param widget_config: The configuration for the widget itself + :param path: The path to the widget, for error reporting + """ + class NumberType(WidgetType): def get_max(self, config: dict): diff --git a/esphome/components/lvgl/widgets/button.py b/esphome/components/lvgl/widgets/button.py index b59884ee67a..5f2910174f1 100644 --- a/esphome/components/lvgl/widgets/button.py +++ b/esphome/components/lvgl/widgets/button.py @@ -1,20 +1,52 @@ -from esphome.const import CONF_BUTTON +from esphome import config_validation as cv +from esphome.const import CONF_BUTTON, CONF_TEXT +from esphome.cpp_generator import MockObj -from ..defines import CONF_MAIN +from ..defines import CONF_MAIN, CONF_WIDGETS +from ..helpers import add_lv_use +from ..lv_validation import lv_text +from ..lvcode import lv, lv_expr +from ..schemas import TEXT_SCHEMA from ..types import LvBoolean, WidgetType +from . import Widget +from .label import label_spec lv_button_t = LvBoolean("lv_btn_t") class ButtonType(WidgetType): def __init__(self): - super().__init__(CONF_BUTTON, lv_button_t, (CONF_MAIN,), lv_name="btn") + super().__init__( + CONF_BUTTON, lv_button_t, (CONF_MAIN,), schema=TEXT_SCHEMA, lv_name="btn" + ) + + def validate(self, value): + if CONF_TEXT in value: + if CONF_WIDGETS in value: + raise cv.Invalid("Cannot use both text and widgets in a button") + add_lv_use("label") + return value def get_uses(self): return ("btn",) - async def to_code(self, w, config): - return [] + def on_create(self, var: MockObj, config: dict): + if CONF_TEXT in config: + lv.label_create(var) + return var + + async def to_code(self, w: Widget, config): + if text := config.get(CONF_TEXT): + label_widget = Widget.create( + None, lv_expr.obj_get_child(w.obj, 0), label_spec + ) + await label_widget.set_property(CONF_TEXT, await lv_text.process(text)) + + def final_validate(self, widget, update_config, widget_config, path): + if CONF_TEXT in update_config and CONF_TEXT not in widget_config: + raise cv.Invalid( + "Button must have 'text:' configured to allow updating text", path + ) button_spec = ButtonType() diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index a5714d5639a..65d629bcdf3 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -426,6 +426,14 @@ lvgl: logger.log: Long pressed repeated - buttons: - id: button_e + - button: + id: button_with_text + text: Button + on_click: + lvgl.button.update: + id: button_with_text + text: Clicked + - button: layout: 2x1 id: button_button