From 7a23a339e9a6f8234f2e8ad3339f047e3b18e27f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 19 Apr 2026 16:00:31 -0500 Subject: [PATCH 01/35] [substitutions] Fix `substitutions: !include file.yaml` regression (#15850) --- esphome/components/packages/__init__.py | 8 ++- esphome/components/substitutions/__init__.py | 35 +++++++++++-- .../15-substitutions_as_include.approved.yaml | 5 ++ .../15-substitutions_as_include.input.yaml | 5 ++ .../substitutions/15-substitutions_inc.yaml | 1 + ...ons_as_include_with_packages.approved.yaml | 5 ++ ...utions_as_include_with_packages.input.yaml | 9 ++++ ...ubstitutions_include_cli_var.approved.yaml | 6 +++ ...7-substitutions_include_cli_var.input.yaml | 8 +++ tests/unit_tests/test_substitutions.py | 51 +++++++++++++++++++ 10 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.input.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/15-substitutions_inc.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.input.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.input.yaml diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 3f3df753512..252a24061a0 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -10,6 +10,7 @@ from esphome.components.substitutions import ( ContextVars, push_context, resolve_include, + resolve_substitutions_block, substitute, ) from esphome.components.substitutions.jinja import has_jinja @@ -516,7 +517,12 @@ def do_packages_pass( if CONF_PACKAGES not in config: return config - substitutions = UserDict(config.pop(CONF_SUBSTITUTIONS, {})) + with cv.prepend_path(CONF_SUBSTITUTIONS): + substitutions = UserDict( + resolve_substitutions_block( + config.pop(CONF_SUBSTITUTIONS, {}), command_line_substitutions + ) + ) processor = _PackageProcessor( substitutions, command_line_substitutions, skip_update ) diff --git a/esphome/components/substitutions/__init__.py b/esphome/components/substitutions/__init__.py index 94aebbbfe35..e451ad5db8d 100644 --- a/esphome/components/substitutions/__init__.py +++ b/esphome/components/substitutions/__init__.py @@ -414,6 +414,34 @@ def _warn_unresolved_variables(errors: ErrList) -> None: ) +def resolve_substitutions_block( + substitutions: Any, + command_line_substitutions: dict[str, Any] | None, +) -> dict[str, Any]: + """Resolve a deferred ``substitutions: !include file.yaml`` and validate the shape. + + The caller is responsible for wrapping the call in + ``cv.prepend_path(CONF_SUBSTITUTIONS)`` for error reporting. + ``command_line_substitutions`` seeds the filename context so + ``substitutions: !include ${var}.yaml`` can reference CLI-provided vars. + """ + if isinstance(substitutions, IncludeFile): + # Single-shot resolution — matches ``_walk_packages`` for the + # ``packages: !include`` entry point. Chained includes (an include that + # itself loads another ``!include`` at the top level) are not supported. + substitutions, _ = resolve_include( + substitutions, + [], + ContextVars(command_line_substitutions or {}), + strict_undefined=False, + ) + if not isinstance(substitutions, dict): + raise cv.Invalid( + f"Substitutions must be a key to value mapping, got {type(substitutions)}" + ) + return substitutions + + def do_substitution_pass( config: OrderedDict, command_line_substitutions: dict[str, Any] | None = None ) -> OrderedDict: @@ -429,10 +457,9 @@ def do_substitution_pass( # Use merge_dicts_ordered to preserve OrderedDict type for move_to_end() substitutions = config.pop(CONF_SUBSTITUTIONS, {}) with cv.prepend_path(CONF_SUBSTITUTIONS): - if not isinstance(substitutions, dict): - raise cv.Invalid( - f"Substitutions must be a key to value mapping, got {type(substitutions)}" - ) + substitutions = resolve_substitutions_block( + substitutions, command_line_substitutions + ) substitutions = merge_dicts_ordered( substitutions, command_line_substitutions or {} ) diff --git a/tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.approved.yaml b/tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.approved.yaml new file mode 100644 index 00000000000..14aa707def5 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.approved.yaml @@ -0,0 +1,5 @@ +substitutions: + wifi_password: sub_password +wifi: + ssid: main_ssid + password: sub_password diff --git a/tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.input.yaml b/tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.input.yaml new file mode 100644 index 00000000000..5909e7bf4f7 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.input.yaml @@ -0,0 +1,5 @@ +substitutions: !include 15-substitutions_inc.yaml + +wifi: + ssid: main_ssid + password: $wifi_password diff --git a/tests/unit_tests/fixtures/substitutions/15-substitutions_inc.yaml b/tests/unit_tests/fixtures/substitutions/15-substitutions_inc.yaml new file mode 100644 index 00000000000..44d9a4b9ef2 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/15-substitutions_inc.yaml @@ -0,0 +1 @@ +wifi_password: sub_password diff --git a/tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.approved.yaml b/tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.approved.yaml new file mode 100644 index 00000000000..14aa707def5 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.approved.yaml @@ -0,0 +1,5 @@ +substitutions: + wifi_password: sub_password +wifi: + ssid: main_ssid + password: sub_password diff --git a/tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.input.yaml b/tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.input.yaml new file mode 100644 index 00000000000..a2e72f33a23 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.input.yaml @@ -0,0 +1,9 @@ +substitutions: !include 15-substitutions_inc.yaml + +packages: + wifi_pkg: + wifi: + password: $wifi_password + +wifi: + ssid: main_ssid diff --git a/tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.approved.yaml b/tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.approved.yaml new file mode 100644 index 00000000000..f1fd5fb0786 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.approved.yaml @@ -0,0 +1,6 @@ +substitutions: + subs_file: 15-substitutions_inc + wifi_password: sub_password +wifi: + ssid: main_ssid + password: sub_password diff --git a/tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.input.yaml b/tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.input.yaml new file mode 100644 index 00000000000..3248504b468 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.input.yaml @@ -0,0 +1,8 @@ +command_line_substitutions: + subs_file: 15-substitutions_inc + +substitutions: !include ${subs_file}.yaml + +wifi: + ssid: main_ssid + password: $wifi_password diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index 01c669e5420..71bbd9db86d 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -675,6 +675,57 @@ def test_include_filename_substitution_undefined_var(tmp_path: Path) -> None: substitutions.do_substitution_pass(config) +def test_do_substitution_pass_included_substitutions_must_be_mapping( + tmp_path: Path, +) -> None: + """`substitutions: !include list.yaml` where the file holds a list raises cv.Invalid. + + Locks in the shape check that runs after the deferred IncludeFile has been + resolved. + """ + parent = tmp_path / "main.yaml" + parent.write_text("") + + def loader(path: Path): + return ["not", "a", "mapping"] + + include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader) + config = OrderedDict({CONF_SUBSTITUTIONS: include}) + + with pytest.raises( + cv.Invalid, match="Substitutions must be a key to value mapping" + ): + substitutions.do_substitution_pass(config) + + +def test_do_packages_pass_included_substitutions_must_be_mapping( + tmp_path: Path, +) -> None: + """`substitutions: !include list.yaml` alongside `packages:` raises cv.Invalid. + + Without the shape check, ``UserDict(...)`` would surface a low-level + ``TypeError``; the explicit ``cv.Invalid`` points at the substitutions path. + """ + parent = tmp_path / "main.yaml" + parent.write_text("") + + def loader(path: Path): + return ["not", "a", "mapping"] + + include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader) + config = OrderedDict( + { + CONF_SUBSTITUTIONS: include, + "packages": {"noop": {"wifi": {"ssid": "main"}}}, + } + ) + + with pytest.raises( + cv.Invalid, match="Substitutions must be a key to value mapping" + ): + do_packages_pass(config) + + def test_resolve_package_undefined_var_in_include_filename(tmp_path: Path) -> None: """An undefined substitution in a package include filename raises cv.Invalid. From aad1318b4a8eceec8c8e16dd3718d26f55790697 Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Sun, 19 Apr 2026 23:09:14 +0200 Subject: [PATCH 02/35] [packages] Improve error messages with include stack and fix missing path propagation (#15844) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/packages/__init__.py | 9 ++ esphome/components/substitutions/__init__.py | 50 +++++++++++ .../component_tests/packages/test_packages.py | 88 ++++++++++++++++++- tests/unit_tests/test_substitutions.py | 34 +++++++ 4 files changed, 179 insertions(+), 2 deletions(-) diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 252a24061a0..97a53094800 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -8,7 +8,9 @@ from typing import Any from esphome import git, yaml_util from esphome.components.substitutions import ( ContextVars, + ErrList, push_context, + raise_first_undefined, resolve_include, resolve_substitutions_block, substitute, @@ -360,12 +362,19 @@ def _substitute_package_definition( if isinstance(package_config, str) or ( isinstance(package_config, dict) and is_remote_package(package_config) ): + # Collect undefined-variable errors (rather than raising strict) so the + # path walked through a remote-package dict is preserved and the user + # sees which field (url / path / ref / ...) referenced the undefined + # variable. + errors: ErrList = [] package_config = substitute( item=package_config, path=[], parent_context=context_vars or ContextVars(), strict_undefined=False, + errors=errors, ) + raise_first_undefined(errors, package_config, "package definition") return package_config diff --git a/esphome/components/substitutions/__init__.py b/esphome/components/substitutions/__init__.py index e451ad5db8d..8bbccffca16 100644 --- a/esphome/components/substitutions/__init__.py +++ b/esphome/components/substitutions/__init__.py @@ -30,6 +30,56 @@ ErrList = list[tuple[UndefinedError, SubstitutionPath, Any]] jinja = Jinja() +def raise_first_undefined( + errors: ErrList, + source: Any, + context_label: str, +) -> None: + """If *errors* is non-empty, raise ``cv.Invalid`` for the first undefined variable. + + The raised error names the missing variable, the path walked into *source* + (for nested dicts, e.g. ``url`` or ``ref``), and the YAML source location + when *source* carries one. Only the first error is surfaced; the user will + re-run after fixing it and any remaining undefined variables will be + reported then. + + ``context_label`` is the noun describing where the undefined variable + appeared (e.g. ``"package definition"``). + """ + if not errors: + return + err, err_path, err_value = errors[0] + if len(errors) > 1: + # Log any further undefined variables so debug-level output covers + # the full set, even though only the first is surfaced to the user. + extras = ", ".join( + f"{e.message} at '{'->'.join(str(p) for p in p_path)}'" + for e, p_path, _ in errors[1:] + ) + _LOGGER.debug("Additional undefined variables in %s: %s", context_label, extras) + # Prefer the location of the offending scalar (e.g. the `url:` value) over + # the enclosing package-definition dict so the message points at the exact + # line/column that carries the undefined variable. + location_node = ( + err_value + if isinstance(err_value, ESPHomeDataBase) and err_value.esp_range is not None + else source + ) + location = "" + if ( + isinstance(location_node, ESPHomeDataBase) + and location_node.esp_range is not None + ): + mark = location_node.esp_range.start_mark + # DocumentLocation.line/column are 0-based (from the YAML Mark). Render + # as 1-based to match config.line_info() and editor line numbering. + location = f" (in {mark.document} {mark.line + 1}:{mark.column + 1})" + field = f" at '{'->'.join(str(p) for p in err_path)}'" if err_path else "" + raise cv.Invalid( + f"Undefined variable in {context_label}{field}: {err.message}{location}" + ) + + def validate_substitution_key(value: Any) -> str: """Validate and normalize a substitution key, stripping a leading ``$`` if present.""" value = cv.string(value) diff --git a/tests/component_tests/packages/test_packages.py b/tests/component_tests/packages/test_packages.py index cd91c4d8cb1..0bd339efa9c 100644 --- a/tests/component_tests/packages/test_packages.py +++ b/tests/component_tests/packages/test_packages.py @@ -2,18 +2,20 @@ import logging from pathlib import Path +import re from unittest.mock import MagicMock, patch import pytest from esphome.components.packages import ( CONFIG_SCHEMA, + _substitute_package_definition, _walk_packages, do_packages_pass, is_package_definition, merge_packages, ) -from esphome.components.substitutions import do_substitution_pass +from esphome.components.substitutions import ContextVars, do_substitution_pass import esphome.config as config_module from esphome.config import resolve_extend_remove from esphome.config_helpers import Extend, Remove @@ -44,7 +46,7 @@ from esphome.const import ( ) from esphome.core import CORE from esphome.util import OrderedDict -from esphome.yaml_util import IncludeFile, add_context +from esphome.yaml_util import IncludeFile, add_context, load_yaml # Test strings TEST_DEVICE_NAME = "test_device_name" @@ -1399,3 +1401,85 @@ def test_raw_config_contains_merged_esphome_from_package(tmp_path) -> None: "CORE.raw_config should contain esphome section after package merge" ) assert CORE.raw_config[CONF_ESPHOME][CONF_NAME] == TEST_DEVICE_NAME + + +# --------------------------------------------------------------------------- +# _substitute_package_definition +# --------------------------------------------------------------------------- + + +def test_substitute_package_definition_local_dict_returned_unchanged() -> None: + """A plain local config dict is not substituted and is returned as-is.""" + pkg = {CONF_WIFI: {CONF_SSID: "test"}} + result = _substitute_package_definition(pkg, ContextVars()) + assert result is pkg + + +def test_substitute_package_definition_string_resolved_with_context() -> None: + """A string package definition has its variables substituted.""" + ctx = ContextVars({"variant": "esp32"}) + result = _substitute_package_definition("device-${variant}.yaml", ctx) + assert result == "device-esp32.yaml" + + +def test_substitute_package_definition_undefined_in_string() -> None: + """An undefined variable in a package URL string raises cv.Invalid.""" + with pytest.raises(cv.Invalid, match="Undefined variable in package definition"): + _substitute_package_definition( + "github://org/repo/${undefined_var}/pkg.yaml", ContextVars() + ) + + +def test_substitute_package_definition_undefined_in_remote_dict_field() -> None: + """An undefined variable inside a remote-dict field names the offending field.""" + with pytest.raises(cv.Invalid) as exc_info: + _substitute_package_definition( + {CONF_URL: "github://${typo}/repo"}, ContextVars() + ) + err = str(exc_info.value) + assert "'typo' is undefined" in err + assert CONF_URL in err + + +def test_substitute_package_definition_undefined_in_remote_dict_non_first_field() -> ( + None +): + """The field path joins correctly for non-first dict fields (e.g. ``ref``).""" + with pytest.raises(cv.Invalid) as exc_info: + _substitute_package_definition( + { + CONF_URL: "github://org/repo", + CONF_REF: "branch-${branch_typo}", + }, + ContextVars(), + ) + err = str(exc_info.value) + assert "'branch_typo' is undefined" in err + assert CONF_REF in err + + +def test_substitute_package_definition_includes_source_location(tmp_path: Path) -> None: + """A package loaded from YAML surfaces file/line/col in the cv.Invalid message. + + Line/column are rendered 1-based (matching config.line_info() and editor + line numbering) and point at the offending scalar, not the enclosing dict. + """ + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text( + "packages:\n broken: github://org/repo/${undefined_var}/pkg.yaml\n" + ) + config = load_yaml(yaml_file) + package_config = config[CONF_PACKAGES]["broken"] + + with pytest.raises(cv.Invalid) as exc_info: + _substitute_package_definition(package_config, ContextVars()) + + err = str(exc_info.value) + assert "main.yaml" in err + # The offending value lives on line 2 (1-based). Column depends on the YAML + # loader, so we only pin line and check that a 1-based column is present. + match = re.search(r"main\.yaml (\d+):(\d+)", err) + assert match, err + line, col = int(match.group(1)), int(match.group(2)) + assert line == 2, f"expected 1-based line 2, got {line} (err={err!r})" + assert col >= 1, f"expected 1-based column ≥ 1, got {col} (err={err!r})" diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index 71bbd9db86d..3599e703d9d 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -14,6 +14,7 @@ from esphome.components.packages import ( do_packages_pass, merge_packages, ) +from esphome.components.substitutions.jinja import UndefinedError from esphome.config import resolve_extend_remove from esphome.config_helpers import Extend, merge_config import esphome.config_validation as cv @@ -675,6 +676,39 @@ def test_include_filename_substitution_undefined_var(tmp_path: Path) -> None: substitutions.do_substitution_pass(config) +def test_raise_first_undefined_logs_extras_at_debug( + caplog: pytest.LogCaptureFixture, +) -> None: + """Only the first undefined error is raised; extras are logged at debug.""" + errors: substitutions.ErrList = [ + (UndefinedError("'a' is undefined"), ["url"], None), + (UndefinedError("'b' is undefined"), ["ref"], None), + (UndefinedError("'c' is undefined"), ["path"], None), + ] + + with ( + caplog.at_level(logging.DEBUG, logger="esphome.components.substitutions"), + pytest.raises(cv.Invalid) as exc_info, + ): + substitutions.raise_first_undefined(errors, None, "package definition") + + # First error is surfaced as the cv.Invalid message. + raised = str(exc_info.value) + assert "'a' is undefined" in raised + assert "'b' is undefined" not in raised + assert "'c' is undefined" not in raised + + # Remaining errors are captured via debug logging for troubleshooting. + assert "Additional undefined variables in package definition" in caplog.text + assert "'b' is undefined at 'ref'" in caplog.text + assert "'c' is undefined at 'path'" in caplog.text + + +def test_raise_first_undefined_noop_on_empty() -> None: + """An empty errors list is a no-op — no exception, no log.""" + substitutions.raise_first_undefined([], None, "package definition") + + def test_do_substitution_pass_included_substitutions_must_be_mapping( tmp_path: Path, ) -> None: From 1847666e75950c50eb4be05564e0ca262ff852af Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 19 Apr 2026 17:05:27 -0500 Subject: [PATCH 03/35] [core] Default PollingComponent() to not run when codegen is bypassed (#15832) --- esphome/core/component.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/core/component.h b/esphome/core/component.h index 717ca362576..67db5423af9 100644 --- a/esphome/core/component.h +++ b/esphome/core/component.h @@ -601,7 +601,7 @@ class Component { */ class PollingComponent : public Component { public: - PollingComponent() : PollingComponent(1) {} + PollingComponent() : PollingComponent(SCHEDULER_DONT_RUN) {} /** Initialize this polling component with the given update interval in ms. * From cb90ac45c35975a882795013a69a6ac613185a58 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Apr 2026 07:50:11 -1000 Subject: [PATCH 04/35] [core] Fix app_state_ status bits clobbered for non-looping components (#15658) --- esphome/core/application.cpp | 34 ++- esphome/core/application.h | 24 ++- esphome/core/component.cpp | 13 ++ esphome/core/component.h | 5 + tests/integration/fixtures/status_flags.yaml | 141 +++++++++++++ tests/integration/test_status_flags.py | 209 +++++++++++++++++++ 6 files changed, 416 insertions(+), 10 deletions(-) create mode 100644 tests/integration/fixtures/status_flags.yaml create mode 100644 tests/integration/test_status_flags.py diff --git a/esphome/core/application.cpp b/esphome/core/application.cpp index cd758598801..0c17c701615 100644 --- a/esphome/core/application.cpp +++ b/esphome/core/application.cpp @@ -85,8 +85,12 @@ void Application::setup() { if (component->can_proceed()) continue; + // Force the status LED to blink WARNING while we wait for a slow + // component to come up. Cleared after setup() finishes if no real + // component has warning set. + this->app_state_ |= STATUS_LED_WARNING; + do { - uint8_t new_app_state = STATUS_LED_WARNING; uint32_t now = millis(); // Process pending loop enables to handle GPIO interrupts during setup @@ -96,17 +100,26 @@ void Application::setup() { // Update loop_component_start_time_ right before calling each component this->loop_component_start_time_ = millis(); this->components_[j]->call(); - new_app_state |= this->components_[j]->get_component_state(); - this->app_state_ |= new_app_state; this->feed_wdt(); } this->after_loop_tasks_(); - this->app_state_ = new_app_state; yield(); } while (!component->can_proceed() && !component->is_failed()); } + // Setup is complete. Reconcile STATUS_LED_WARNING: the slow-setup path + // above may have forced it on, and any status_clear_warning() calls + // from components during setup were intentional no-ops (gated by + // APP_STATE_SETUP_COMPLETE). Walk components once here to pick up the + // real state. STATUS_LED_ERROR is never artificially forced, so its + // clear path always works and needs no reconciliation. Finally, set + // APP_STATE_SETUP_COMPLETE so subsequent warning clears go through + // the normal walk-and-clear path. + if (!this->any_component_has_status_flag_(STATUS_LED_WARNING)) + this->app_state_ &= ~STATUS_LED_WARNING; + this->app_state_ |= APP_STATE_SETUP_COMPLETE; + ESP_LOGI(TAG, "setup() finished successfully!"); #ifdef USE_SETUP_PRIORITY_OVERRIDE @@ -211,6 +224,19 @@ void HOT Application::feed_wdt(uint32_t time) { #endif } } +bool Application::any_component_has_status_flag_(uint8_t flag) const { + // Walk all components (not just looping ones) so non-looping components' + // status bits are respected. Only called from the slow-path clear helpers + // (status_clear_warning_slow_path_ / status_clear_error_slow_path_) on an + // actual set→clear transition, so walking O(N) here is paid once per + // transition — not once per loop iteration. + for (auto *component : this->components_) { + if ((component->get_component_state() & flag) != 0) + return true; + } + return false; +} + void Application::reboot() { ESP_LOGI(TAG, "Forcing a reboot"); for (auto &component : std::ranges::reverse_view(this->components_)) { diff --git a/esphome/core/application.h b/esphome/core/application.h index 6b2969b4907..0150bb6646a 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -401,7 +401,18 @@ class Application { */ void teardown_components(uint32_t timeout_ms); - uint8_t get_app_state() const { return this->app_state_; } + /// Return the public app state status bits (STATUS_LED_* only). + /// Internal bookkeeping bits like APP_STATE_SETUP_COMPLETE are masked + /// out so external readers (status_led components, etc.) never see them. + uint8_t get_app_state() const { return this->app_state_ & ~APP_STATE_SETUP_COMPLETE; } + + /// True once Application::setup() has finished walking all components + /// and finalized the initial status flags. Before this point, the + /// slow-setup busy-wait may be forcing STATUS_LED_WARNING on, and + /// status_clear_* intentionally skips its walk-and-clear step so the + /// forced bit doesn't get wiped. Stored as a free bit on app_state_ + /// (bit 6) to avoid costing additional RAM. + bool is_setup_complete() const { return (this->app_state_ & APP_STATE_SETUP_COMPLETE) != 0; } // Helper macro for entity getter method declarations #ifdef USE_DEVICES @@ -577,6 +588,12 @@ class Application { bool is_socket_ready_(int fd) const { return FD_ISSET(fd, &this->read_fds_); } #endif + /// Walk all registered components looking for any whose component_state_ + /// has the given flag set. Used by Component::status_clear_*_slow_path_() + /// (which is a friend) to decide whether to clear the corresponding bit on + /// this->app_state_ (the app-wide "any component has this status" indicator). + bool any_component_has_status_flag_(uint8_t flag) const; + /// Register a component, detecting loop() override at compile time. /// Uses HasLoopOverride which handles ambiguous &T::loop from multiple inheritance. template void register_component_(T *comp) { @@ -838,8 +855,6 @@ inline void ESPHOME_ALWAYS_INLINE Application::before_loop_tasks_(uint32_t loop_ } inline void ESPHOME_ALWAYS_INLINE Application::loop() { - uint8_t new_app_state = 0; - // Get the initial loop time at the start uint32_t last_op_end_time = millis(); @@ -859,13 +874,10 @@ inline void ESPHOME_ALWAYS_INLINE Application::loop() { // Use the finish method to get the current time as the end time last_op_end_time = guard.finish(); } - new_app_state |= component->get_component_state(); - this->app_state_ |= new_app_state; this->feed_wdt(last_op_end_time); } this->after_loop_tasks_(); - this->app_state_ = new_app_state; #ifdef USE_RUNTIME_STATS // Process any pending runtime stats printing after all components have run diff --git a/esphome/core/component.cpp b/esphome/core/component.cpp index deda42b0a7d..8949b4b76dc 100644 --- a/esphome/core/component.cpp +++ b/esphome/core/component.cpp @@ -411,10 +411,23 @@ void Component::status_set_error(const LogString *message) { } void Component::status_clear_warning_slow_path_() { this->component_state_ &= ~STATUS_LED_WARNING; + // Clear the app-wide STATUS_LED_WARNING bit only if setup has finished + // AND no other component still has it set. During setup the forced + // STATUS_LED_WARNING (from the slow-setup busy-wait) must not be wiped + // by a transient component clear — Application::setup() reconciles + // the warning bit once at the end before setting APP_STATE_SETUP_COMPLETE. + // The set path is unchanged (set_status_flag_ still writes directly). + if (App.is_setup_complete() && !App.any_component_has_status_flag_(STATUS_LED_WARNING)) + App.app_state_ &= ~STATUS_LED_WARNING; ESP_LOGW(TAG, "%s cleared Warning flag", LOG_STR_ARG(this->get_component_log_str())); } void Component::status_clear_error_slow_path_() { this->component_state_ &= ~STATUS_LED_ERROR; + // STATUS_LED_ERROR is never artificially forced — it only ever lands + // in app_state_ via a real set_status_flag_ call. So the walk-and-clear + // path is always safe, including during setup. + if (!App.any_component_has_status_flag_(STATUS_LED_ERROR)) + App.app_state_ &= ~STATUS_LED_ERROR; ESP_LOGE(TAG, "%s cleared Error flag", LOG_STR_ARG(this->get_component_log_str())); } void Component::status_momentary_warning(const char *name, uint32_t length) { diff --git a/esphome/core/component.h b/esphome/core/component.h index e2b7aa85d3a..3307c5ae76e 100644 --- a/esphome/core/component.h +++ b/esphome/core/component.h @@ -89,6 +89,11 @@ inline constexpr uint8_t STATUS_LED_WARNING = 0x08; inline constexpr uint8_t STATUS_LED_ERROR = 0x10; // Component loop override flag uses bit 5 (set at registration time) inline constexpr uint8_t COMPONENT_HAS_LOOP = 0x20; +// Bit 6 on Application::app_state_ (ONLY) — set at the end of +// Application::setup(). Component::status_clear_*_slow_path_() uses this to +// decide whether to propagate clears to App.app_state_. Never set on a +// Component's component_state_. +inline constexpr uint8_t APP_STATE_SETUP_COMPLETE = 0x40; // Remove before 2026.8.0 enum class RetryResult { DONE, RETRY }; diff --git a/tests/integration/fixtures/status_flags.yaml b/tests/integration/fixtures/status_flags.yaml new file mode 100644 index 00000000000..cb118dcc84c --- /dev/null +++ b/tests/integration/fixtures/status_flags.yaml @@ -0,0 +1,141 @@ +esphome: + name: status-flags-test + +host: +api: + actions: + # Warning flag services for sensor_a + - action: set_warning_a + then: + - lambda: "id(sensor_a)->status_set_warning();" + - component.update: app_warning_bit + - component.update: app_error_bit + - action: clear_warning_a + then: + - lambda: "id(sensor_a)->status_clear_warning();" + - component.update: app_warning_bit + - component.update: app_error_bit + + # Warning flag services for sensor_b + - action: set_warning_b + then: + - lambda: "id(sensor_b)->status_set_warning();" + - component.update: app_warning_bit + - component.update: app_error_bit + - action: clear_warning_b + then: + - lambda: "id(sensor_b)->status_clear_warning();" + - component.update: app_warning_bit + - component.update: app_error_bit + + # Error flag services for sensor_a + - action: set_error_a + then: + - lambda: "id(sensor_a)->status_set_error();" + - component.update: app_warning_bit + - component.update: app_error_bit + - action: clear_error_a + then: + - lambda: "id(sensor_a)->status_clear_error();" + - component.update: app_warning_bit + - component.update: app_error_bit + + # Error flag services for sensor_b + - action: set_error_b + then: + - lambda: "id(sensor_b)->status_set_error();" + - component.update: app_warning_bit + - component.update: app_error_bit + - action: clear_error_b + then: + - lambda: "id(sensor_b)->status_clear_error();" + - component.update: app_warning_bit + - component.update: app_error_bit + + # Snapshot of the status_led_light's output state for observation. + - action: snapshot_led + then: + - component.update: status_led_writes + - component.update: status_led_last_state + +logger: + +# Tracks each write to the fake status_led output. +globals: + - id: status_led_write_count + type: uint32_t + restore_value: no + initial_value: "0" + - id: status_led_last_write + type: bool + restore_value: no + initial_value: "false" + +# Fake binary output — status_led_light writes to this instead of a pin. +# Every write bumps a counter and records the last value, both of which +# are exposed below so the test can verify status_led_light's loop is +# actually reading App.get_app_state() and responding. +output: + - platform: template + id: fake_status_led + type: binary + write_action: + - globals.set: + id: status_led_write_count + value: !lambda "return id(status_led_write_count) + 1;" + - globals.set: + id: status_led_last_write + value: !lambda "return state;" + +# Actual status_led_light component under test. +light: + - platform: status_led + name: Status LED + id: status_led_light_id + output: fake_status_led + +sensor: + # Two components that the test will toggle warning/error flags on. + - platform: template + name: Sensor A + id: sensor_a + update_interval: 24h + lambda: return 1.0; + - platform: template + name: Sensor B + id: sensor_b + update_interval: 24h + lambda: return 2.0; + + # Expose App.app_state_'s STATUS_LED_WARNING / STATUS_LED_ERROR bits + # as 0.0 / 1.0. force_update ensures every manual component.update + # publishes even if the value is unchanged. + - platform: template + name: App Warning Bit + id: app_warning_bit + update_interval: 24h + force_update: true + lambda: |- + return (App.get_app_state() & STATUS_LED_WARNING) != 0 ? 1.0 : 0.0; + - platform: template + name: App Error Bit + id: app_error_bit + update_interval: 24h + force_update: true + lambda: |- + return (App.get_app_state() & STATUS_LED_ERROR) != 0 ? 1.0 : 0.0; + + # Observables for the fake status_led output. + - platform: template + name: Status LED Writes + id: status_led_writes + update_interval: 24h + force_update: true + lambda: return id(status_led_write_count); + - platform: template + name: Status LED Last State + id: status_led_last_state + update_interval: 24h + force_update: true + lambda: |- + return id(status_led_last_write) ? 1.0 : 0.0; diff --git a/tests/integration/test_status_flags.py b/tests/integration/test_status_flags.py new file mode 100644 index 00000000000..ffbc7c7f634 --- /dev/null +++ b/tests/integration/test_status_flags.py @@ -0,0 +1,209 @@ +"""Integration tests for Component::status_set/clear_warning/error propagation. + +Verifies that toggling STATUS_LED_WARNING / STATUS_LED_ERROR on individual +components correctly updates the app-wide bits on Application::app_state_, +AND that the status_led_light component actually responds to those bits +by writing to its output (the full chain from component.status_set_warning +→ App.app_state_ → status_led_light.loop() reading get_app_state()). + +Exercises the multi-component OR semantics (the app bit stays set while +any component still has the flag, and only clears when the last component +clears its bit), the independence of warning and error, and the actual +status_led_light read of the bits via a fake template output that counts +writes. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from .state_utils import InitialStateHelper, SensorTracker, build_key_to_entity_mapping +from .types import APIClientConnectedFactory, RunCompiledFunction + +# Time to let the host-mode main loop run so status_led_light.loop() can +# execute enough iterations to produce measurable write-count changes on +# the fake template output. 300 ms is well above the minimum needed. +STATUS_LED_SETTLE_S = 0.3 + + +@pytest.mark.asyncio +async def test_status_flags( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + async with run_compiled(yaml_config), api_client_connected() as client: + entities, services = await client.list_entities_services() + + # Map every custom API service by name for the test to execute. + svc = {s.name: s for s in services} + for name in ( + "set_warning_a", + "clear_warning_a", + "set_warning_b", + "clear_warning_b", + "set_error_a", + "clear_error_a", + "set_error_b", + "clear_error_b", + "snapshot_led", + ): + assert name in svc, f"service {name} not registered" + + # Track every sensor we care about. SensorTracker gives us + # expect(value) / expect_any() futures that resolve when a + # matching state arrives; much simpler than manual bookkeeping. + tracker = SensorTracker( + [ + "app_warning_bit", + "app_error_bit", + "status_led_writes", + "status_led_last_state", + ] + ) + tracker.key_to_sensor.update( + build_key_to_entity_mapping(entities, list(tracker.sensor_states.keys())) + ) + + # Swallow initial state broadcasts so the test only reacts to + # state changes triggered by our service calls. + initial_state_helper = InitialStateHelper(entities) + client.subscribe_states(initial_state_helper.on_state_wrapper(tracker.on_state)) + try: + await initial_state_helper.wait_for_initial_states() + except TimeoutError: + pytest.fail("Timeout waiting for initial states") + + async def call(name: str) -> None: + await client.execute_service(svc[name], {}) + + async def call_and_expect_bits( + service_name: str, *, warning: float, error: float + ) -> None: + """Execute a service and wait for both app bit sensors to match. + + Each bit-toggling service calls component.update on both + app_warning_bit and app_error_bit, so both sensors publish. + """ + futures = tracker.expect_all( + {"app_warning_bit": warning, "app_error_bit": error} + ) + await call(service_name) + await tracker.await_all(futures) + + async def snapshot_led_writes() -> int: + """Trigger a publish of the fake status_led output counter and return it.""" + future = tracker.expect_any("status_led_writes") + await call("snapshot_led") + await tracker.await_change(future, "status_led_writes") + return int(tracker.sensor_states["status_led_writes"][-1]) + + # ---- Baseline: everything clean ---- + await call_and_expect_bits("clear_warning_a", warning=0.0, error=0.0) + + # ================================================================ + # Part 1 — STATUS_LED_WARNING propagation to App.app_state_ + # ================================================================ + + # Single component set/clear + await call_and_expect_bits("set_warning_a", warning=1.0, error=0.0) + await call_and_expect_bits("clear_warning_a", warning=0.0, error=0.0) + + # Multi-component OR: both set, clear A, bit stays (B still has it), clear B, gone + await call_and_expect_bits("set_warning_a", warning=1.0, error=0.0) + await call_and_expect_bits("set_warning_b", warning=1.0, error=0.0) + await call_and_expect_bits("clear_warning_a", warning=1.0, error=0.0) + await call_and_expect_bits("clear_warning_b", warning=0.0, error=0.0) + + # Opposite clear order + await call_and_expect_bits("set_warning_a", warning=1.0, error=0.0) + await call_and_expect_bits("set_warning_b", warning=1.0, error=0.0) + await call_and_expect_bits("clear_warning_b", warning=1.0, error=0.0) + await call_and_expect_bits("clear_warning_a", warning=0.0, error=0.0) + + # ================================================================ + # Part 2 — STATUS_LED_ERROR propagation (same scenarios) + # ================================================================ + + await call_and_expect_bits("set_error_a", warning=0.0, error=1.0) + await call_and_expect_bits("clear_error_a", warning=0.0, error=0.0) + + await call_and_expect_bits("set_error_a", warning=0.0, error=1.0) + await call_and_expect_bits("set_error_b", warning=0.0, error=1.0) + await call_and_expect_bits("clear_error_a", warning=0.0, error=1.0) + await call_and_expect_bits("clear_error_b", warning=0.0, error=0.0) + + # ================================================================ + # Part 3 — warning and error are independent + # ================================================================ + + await call_and_expect_bits("set_warning_a", warning=1.0, error=0.0) + await call_and_expect_bits("set_error_b", warning=1.0, error=1.0) + await call_and_expect_bits("clear_warning_a", warning=0.0, error=1.0) + await call_and_expect_bits("clear_error_b", warning=0.0, error=0.0) + + # ================================================================ + # Part 4 — status_led_light actually reads App.app_state_ + # ================================================================ + # The fake status_led_light output increments status_led_write_count + # on every write. status_led_light::loop() writes its output on every + # iteration while an error/warning bit is set, so after holding a + # warning for ~300 ms we should see the counter move significantly. + # This is the end-to-end proof that the bits we set above actually + # reach status_led_light and drive its behavior. + + count_before_warning = await snapshot_led_writes() + await call_and_expect_bits("set_warning_a", warning=1.0, error=0.0) + # Let status_led_light's loop run long enough to toggle the pin + # several times (it reads get_app_state() every main loop iteration). + await asyncio.sleep(STATUS_LED_SETTLE_S) + count_after_warning = await snapshot_led_writes() + assert count_after_warning > count_before_warning, ( + "status_led_light did not respond to STATUS_LED_WARNING being set: " + f"write count stayed at {count_before_warning} → {count_after_warning}. " + "The full chain Component::status_set_warning → App.app_state_ → " + "status_led_light::loop reading get_app_state() is broken." + ) + await call_and_expect_bits("clear_warning_a", warning=0.0, error=0.0) + + # Same check for ERROR + count_before_error = await snapshot_led_writes() + await call_and_expect_bits("set_error_a", warning=0.0, error=1.0) + await asyncio.sleep(STATUS_LED_SETTLE_S) + count_after_error = await snapshot_led_writes() + assert count_after_error > count_before_error, ( + "status_led_light did not respond to STATUS_LED_ERROR being set: " + f"write count stayed at {count_before_error} → {count_after_error}. " + ) + await call_and_expect_bits("clear_error_a", warning=0.0, error=0.0) + + # ---- Set → clear → re-set round-trip ---- + # After clearing, status_led_light stops writing (steady state). + # Re-setting the flag must make it resume. This guards against a + # future idle optimization (e.g. #15642) where status_led disables + # its own loop when idle: if the re-enable path were broken, the + # second set would not produce writes. + # + # Snapshot AFTER the clear to avoid counting writes that were still + # in-flight from the error-set phase. + count_after_clear = await snapshot_led_writes() + await asyncio.sleep(STATUS_LED_SETTLE_S) + count_after_idle = await snapshot_led_writes() + assert count_after_idle - count_after_clear <= 5, ( + "status_led_light kept writing after warning/error was cleared: " + f"count grew from {count_after_clear} to {count_after_idle}. " + "Expected it to stop writing once all status bits were clear." + ) + # Re-set warning — writes must resume. + await call_and_expect_bits("set_warning_a", warning=1.0, error=0.0) + await asyncio.sleep(STATUS_LED_SETTLE_S) + count_after_reset = await snapshot_led_writes() + assert count_after_reset > count_after_idle + 5, ( + "status_led_light did not resume writing after re-setting " + f"STATUS_LED_WARNING: count went from {count_after_idle} to " + f"{count_after_reset}. If an idle optimization disabled the " + "loop, the re-enable path may be broken." + ) + await call_and_expect_bits("clear_warning_a", warning=0.0, error=0.0) From 4479212008f551f38885c3f4a278befb9a400b67 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 14 Apr 2026 09:08:30 -1000 Subject: [PATCH 05/35] [core] Inline feed_wdt hot path with out-of-line slow path (#15656) Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- esphome/core/application.cpp | 34 ++++++++++++++++++++-------------- esphome/core/application.h | 36 +++++++++++++++++++++++++++++------- esphome/core/scheduler.cpp | 8 +++++++- 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/esphome/core/application.cpp b/esphome/core/application.cpp index 0c17c701615..1c732307054 100644 --- a/esphome/core/application.cpp +++ b/esphome/core/application.cpp @@ -209,21 +209,27 @@ void Application::process_dump_config_() { this->dump_config_at_++; } -void HOT Application::feed_wdt(uint32_t time) { - static uint32_t last_feed = 0; - // Use provided time if available, otherwise get current time - uint32_t now = time ? time : millis(); - // Compare in milliseconds (3ms threshold) - if (now - last_feed > 3) { - arch_feed_wdt(); - last_feed = now; -#ifdef USE_STATUS_LED - if (status_led::global_status_led != nullptr) { - status_led::global_status_led->call(); - } -#endif +void Application::feed_wdt() { + // Cold entry: callers without a millis() timestamp in hand. Fetches the + // time and takes the same rate-limit path as feed_wdt_with_time(). + uint32_t now = millis(); + if (now - this->last_wdt_feed_ > WDT_FEED_INTERVAL_MS) { + this->feed_wdt_slow_(now); } } + +void HOT Application::feed_wdt_slow_(uint32_t time) { + // Callers (both feed_wdt() and feed_wdt_with_time()) have already + // confirmed the WDT_FEED_INTERVAL_MS rate limit was exceeded. + arch_feed_wdt(); + this->last_wdt_feed_ = time; +#ifdef USE_STATUS_LED + if (status_led::global_status_led != nullptr) { + status_led::global_status_led->call(); + } +#endif +} + bool Application::any_component_has_status_flag_(uint8_t flag) const { // Walk all components (not just looping ones) so non-looping components' // status bits are respected. Only called from the slow-path clear helpers @@ -325,7 +331,7 @@ void Application::teardown_components(uint32_t timeout_ms) { while (pending_count > 0 && (now - start_time) < timeout_ms) { // Feed watchdog during teardown to prevent triggering - this->feed_wdt(now); + this->feed_wdt_with_time(now); // Process components and compact the array, keeping only those still pending size_t still_pending = 0; diff --git a/esphome/core/application.h b/esphome/core/application.h index 0150bb6646a..60087d527d2 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -385,7 +385,24 @@ class Application { void schedule_dump_config() { this->dump_config_at_ = 0; } - void feed_wdt(uint32_t time = 0); + /// Minimum interval between real arch_feed_wdt() calls. Chosen to keep the + /// rate of HAL pokes low while still being small enough that any plausible + /// watchdog timeout (seconds) has orders of magnitude of safety margin. + static constexpr uint32_t WDT_FEED_INTERVAL_MS = 3; + + /// Feed the task watchdog. Cold entry — callers without a millis() + /// timestamp in hand. Out of line to keep call sites tiny. + void feed_wdt(); + + /// Feed the task watchdog, hot entry. Callers that already have a + /// millis() timestamp pay only a load + sub + branch on the common + /// (no-op) path. The actual arch feed + status LED update live in + /// feed_wdt_slow_. + void ESPHOME_ALWAYS_INLINE feed_wdt_with_time(uint32_t time) { + if (static_cast(time - this->last_wdt_feed_) > WDT_FEED_INTERVAL_MS) [[unlikely]] { + this->feed_wdt_slow_(time); + } + } void reboot(); @@ -632,7 +649,10 @@ class Application { /// Caller must ensure dump_config_at_ < components_.size(). void __attribute__((noinline)) process_dump_config_(); - void feed_wdt_arch_(); + /// Slow path for feed_wdt(): actually calls arch_feed_wdt(), updates + /// last_wdt_feed_, and re-dispatches the status LED. Out of line so the + /// inline wrapper stays tiny. + void feed_wdt_slow_(uint32_t time); /// Perform a delay while also monitoring socket file descriptors for readiness #ifdef USE_HOST @@ -686,6 +706,7 @@ class Application { // 4-byte members uint32_t last_loop_{0}; uint32_t loop_component_start_time_{0}; + uint32_t last_wdt_feed_{0}; // millis() of most recent arch_feed_wdt(); rate-limits feed_wdt() hot path #ifdef USE_HOST int max_fd_{-1}; // Highest file descriptor number for select() @@ -830,12 +851,13 @@ inline void ESPHOME_ALWAYS_INLINE Application::before_loop_tasks_(uint32_t loop_ this->drain_wake_notifications_(); #endif - // Process scheduled tasks + // Process scheduled tasks. Scheduler::call now feeds the watchdog itself + // after each scheduled item that actually runs, so we no longer need an + // unconditional feed here — when Scheduler::call has no work to do, the + // only elapsed time is a sleep wake + a few instructions, and when it does + // have work, it fed the wdt as it went. this->scheduler.call(loop_start_time); - // Feed the watchdog timer - this->feed_wdt(loop_start_time); - // Process any pending enable_loop requests from ISRs // This must be done before marking in_loop_ = true to avoid race conditions if (this->has_pending_enable_loop_requests_) { @@ -874,7 +896,7 @@ inline void ESPHOME_ALWAYS_INLINE Application::loop() { // Use the finish method to get the current time as the end time last_op_end_time = guard.finish(); } - this->feed_wdt(last_op_end_time); + this->feed_wdt_with_time(last_op_end_time); } this->after_loop_tasks_(); diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index dff50b03ef0..3e75a680643 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -739,7 +739,13 @@ uint32_t HOT Scheduler::execute_item_(SchedulerItem *item, uint32_t now) { App.set_current_component(item->component); WarnIfComponentBlockingGuard guard{item->component, now}; item->callback(); - return guard.finish(); + uint32_t end = guard.finish(); + // Feed the watchdog after each scheduled item (both main heap and defer + // queue paths go through here). A run of back-to-back callbacks cannot + // starve the wdt. The inline fast path is a load + sub + branch — nearly + // free when the 3 ms rate limit hasn't elapsed. + App.feed_wdt_with_time(end); + return end; } // Common implementation for cancel operations - handles locking From 6857e1ceb4533f20a2e07d7cd08aa6e768c030ee Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Wed, 15 Apr 2026 14:32:11 -0400 Subject: [PATCH 06/35] [st7789v] Fix swapped offset_width/offset_height in model presets (#15755) --- esphome/components/st7789v/display.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/esphome/components/st7789v/display.py b/esphome/components/st7789v/display.py index 85414237cfe..745c37f47dd 100644 --- a/esphome/components/st7789v/display.py +++ b/esphome/components/st7789v/display.py @@ -45,8 +45,8 @@ MODELS = { presets={ CONF_HEIGHT: 240, CONF_WIDTH: 135, - CONF_OFFSET_HEIGHT: 52, - CONF_OFFSET_WIDTH: 40, + CONF_OFFSET_HEIGHT: 40, + CONF_OFFSET_WIDTH: 52, CONF_CS_PIN: "GPIO5", CONF_DC_PIN: "GPIO16", CONF_RESET_PIN: "GPIO23", @@ -68,8 +68,8 @@ MODELS = { presets={ CONF_HEIGHT: 280, CONF_WIDTH: 240, - CONF_OFFSET_HEIGHT: 0, - CONF_OFFSET_WIDTH: 20, + CONF_OFFSET_HEIGHT: 20, + CONF_OFFSET_WIDTH: 0, } ), "ADAFRUIT_S2_TFT_FEATHER_240X135": model_spec( @@ -77,8 +77,8 @@ MODELS = { presets={ CONF_HEIGHT: 240, CONF_WIDTH: 135, - CONF_OFFSET_HEIGHT: 52, - CONF_OFFSET_WIDTH: 40, + CONF_OFFSET_HEIGHT: 40, + CONF_OFFSET_WIDTH: 52, CONF_CS_PIN: "GPIO7", CONF_DC_PIN: "GPIO39", CONF_RESET_PIN: "GPIO40", @@ -89,8 +89,8 @@ MODELS = { presets={ CONF_HEIGHT: 320, CONF_WIDTH: 170, - CONF_OFFSET_HEIGHT: 35, - CONF_OFFSET_WIDTH: 0, + CONF_OFFSET_HEIGHT: 0, + CONF_OFFSET_WIDTH: 35, CONF_ROTATION: 270, CONF_CS_PIN: "GPIO10", CONF_DC_PIN: "GPIO13", @@ -102,8 +102,8 @@ MODELS = { presets={ CONF_HEIGHT: 320, CONF_WIDTH: 172, - CONF_OFFSET_HEIGHT: 34, - CONF_OFFSET_WIDTH: 0, + CONF_OFFSET_HEIGHT: 0, + CONF_OFFSET_WIDTH: 34, CONF_ROTATION: 90, CONF_CS_PIN: "GPIO21", CONF_DC_PIN: "GPIO22", From 4047d5af5fab0956c9ef782947b99c8c30accd6d Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Wed, 15 Apr 2026 14:32:33 -0400 Subject: [PATCH 07/35] [sx126x][sx127x] Fix frequency precision loss from float32 codegen (#15753) --- esphome/components/sx126x/__init__.py | 4 ++-- esphome/components/sx127x/__init__.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/esphome/components/sx126x/__init__.py b/esphome/components/sx126x/__init__.py index 08f4c0fb882..b8696158fe5 100644 --- a/esphome/components/sx126x/__init__.py +++ b/esphome/components/sx126x/__init__.py @@ -200,11 +200,11 @@ CONFIG_SCHEMA = ( cv.hex_int, cv.Range(min=0, max=0xFFFF) ), cv.Optional(CONF_DEVIATION, default="5kHz"): cv.All( - cv.frequency, cv.float_range(min=0, max=100000) + cv.frequency, cv.int_range(min=0, max=100000) ), cv.Required(CONF_DIO1_PIN): pins.gpio_input_pin_schema, cv.Required(CONF_FREQUENCY): cv.All( - cv.frequency, cv.float_range(min=137.0e6, max=1020.0e6) + cv.frequency, cv.int_range(min=int(137e6), max=int(1020e6)) ), cv.Required(CONF_HW_VERSION): cv.one_of( "sx1261", "sx1262", "sx1268", "llcc68", lower=True diff --git a/esphome/components/sx127x/__init__.py b/esphome/components/sx127x/__init__.py index 7f554fbf841..8fa72471927 100644 --- a/esphome/components/sx127x/__init__.py +++ b/esphome/components/sx127x/__init__.py @@ -197,11 +197,11 @@ CONFIG_SCHEMA = ( cv.Optional(CONF_CODING_RATE, default="CR_4_5"): cv.enum(CODING_RATE), cv.Optional(CONF_CRC_ENABLE, default=False): cv.boolean, cv.Optional(CONF_DEVIATION, default="5kHz"): cv.All( - cv.frequency, cv.float_range(min=0, max=100000) + cv.frequency, cv.int_range(min=0, max=100000) ), cv.Optional(CONF_DIO0_PIN): pins.internal_gpio_input_pin_schema, cv.Required(CONF_FREQUENCY): cv.All( - cv.frequency, cv.float_range(min=137.0e6, max=1020.0e6) + cv.frequency, cv.int_range(min=int(137e6), max=int(1020e6)) ), cv.Required(CONF_MODULATION): cv.enum(MOD), cv.Optional(CONF_ON_PACKET): automation.validate_automation(single=True), From 5a2b7546f666abc106d22860c3f2bea5ad1b1a64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Apr 2026 09:07:49 -1000 Subject: [PATCH 08/35] Bump aioesphomeapi from 44.15.0 to 44.16.0 (#15757) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 726a0a221ae..b14d5f5f5a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ platformio==6.1.19 esptool==5.2.0 click==8.3.2 esphome-dashboard==20260408.1 -aioesphomeapi==44.15.0 +aioesphomeapi==44.16.0 zeroconf==0.148.0 puremagic==1.30 ruamel.yaml==0.19.1 # dashboard_import From af1aaba547d28639521b07d2bbe763f8c8b9e963 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 16 Apr 2026 23:19:42 +1000 Subject: [PATCH 09/35] [lvgl] Clean the build if lv_conf.h changes (#15777) --- esphome/components/lvgl/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index b6421dc43d7..ac0363ca696 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -44,6 +44,7 @@ from esphome.core import CORE, ID, Lambda from esphome.cpp_generator import MockObj from esphome.final_validate import full_config from esphome.helpers import write_file_if_changed +from esphome.writer import clean_build from esphome.yaml_util import load_yaml from . import defines as df, helpers, lv_validation as lvalid, widgets @@ -451,7 +452,8 @@ async def to_code(configs): df.add_define(f"LV_DRAW_SW_SUPPORT_{fmt}", "1") lv_conf_h_file = CORE.relative_src_path(LV_CONF_FILENAME) - write_file_if_changed(lv_conf_h_file, generate_lv_conf_h()) + if write_file_if_changed(lv_conf_h_file, generate_lv_conf_h()): + clean_build(clear_pio_cache=False) cg.add_build_flag("-DLV_CONF_H=1") # handle windows paths in a way that doesn't break the generated C++ lv_conf_h_path = Path(lv_conf_h_file).as_posix() From 92c99a7d41bfd69389519f8628feaf63d95ec957 Mon Sep 17 00:00:00 2001 From: Boris Krivonog Date: Thu, 16 Apr 2026 15:35:24 +0200 Subject: [PATCH 10/35] [mitsubishi_cn105] use HEAT_COOL mode to enable temperature slider (#15748) --- .../mitsubishi_cn105_climate.cpp | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp index 40ddb88a795..284339e57f9 100644 --- a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp +++ b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp @@ -7,7 +7,7 @@ namespace esphome::mitsubishi_cn105 { static const char *const TAG = "mitsubishi_cn105.climate"; static constexpr std::array MODE_MAP{ - std::pair{MitsubishiCN105::Mode::AUTO, climate::CLIMATE_MODE_AUTO}, + std::pair{MitsubishiCN105::Mode::AUTO, climate::CLIMATE_MODE_HEAT_COOL}, std::pair{MitsubishiCN105::Mode::HEAT, climate::CLIMATE_MODE_HEAT}, std::pair{MitsubishiCN105::Mode::DRY, climate::CLIMATE_MODE_DRY}, std::pair{MitsubishiCN105::Mode::COOL, climate::CLIMATE_MODE_COOL}, @@ -76,23 +76,13 @@ void MitsubishiCN105Climate::loop() { climate::ClimateTraits MitsubishiCN105Climate::traits() { climate::ClimateTraits traits; - traits.set_supported_modes({ - climate::CLIMATE_MODE_OFF, - climate::CLIMATE_MODE_COOL, - climate::CLIMATE_MODE_HEAT, - climate::CLIMATE_MODE_DRY, - climate::CLIMATE_MODE_FAN_ONLY, - climate::CLIMATE_MODE_AUTO, - }); + for (const auto &p : MODE_MAP) { + traits.add_supported_mode(p.second); + } - traits.set_supported_fan_modes({ - climate::CLIMATE_FAN_AUTO, - climate::CLIMATE_FAN_QUIET, - climate::CLIMATE_FAN_LOW, - climate::CLIMATE_FAN_MEDIUM, - climate::CLIMATE_FAN_MIDDLE, - climate::CLIMATE_FAN_HIGH, - }); + for (const auto &p : FAN_MODE_MAP) { + traits.add_supported_fan_mode(p.second); + } traits.set_visual_min_temperature(16.0f); traits.set_visual_max_temperature(31.0f); From 914ed10bccee4e21b19916c7b10736124337a457 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 16 Apr 2026 03:36:55 -1000 Subject: [PATCH 11/35] [core] Diagnose missing cg.templatable in codegen for TEMPLATABLE_VALUE fields (#15758) --- esphome/core/automation.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/esphome/core/automation.h b/esphome/core/automation.h index eb270bfee26..468ea3b382c 100644 --- a/esphome/core/automation.h +++ b/esphome/core/automation.h @@ -62,6 +62,18 @@ template class TemplatableFn { !std::convertible_to, T> || !std::default_initializable) = delete; + // Reject raw (non-callable) values with a helpful diagnostic pointing at the Python-side fix. + // TemplatableFn stores only a function pointer (4 bytes), so constants must be wrapped in a + // stateless lambda by codegen. External components hitting this error should use + // `cg.templatable(value, args, type)` in their Python __init__.py before passing to the setter. + template TemplatableFn(V) requires(!std::invocable) && (!std::convertible_to) { + static_assert(sizeof(V) == 0, "Missing cg.templatable(...) in Python codegen for this TEMPLATABLE_VALUE " + "field. The wrapper was always required; it worked by accident because the old " + "TemplatableValue implicitly converted raw constants. TemplatableFn cannot. See " + "https://developers.esphome.io/blog/2026/04/09/" + "templatablefn-4-byte-templatable-storage-for-trivially-copyable-types/"); + } + bool has_value() const { return this->f_ != nullptr; } T value(X... x) const { return this->f_ ? this->f_(x...) : T{}; } From aa80bdbbc68ab65c19c05002b57ac630f393892a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 16 Apr 2026 03:40:22 -1000 Subject: [PATCH 12/35] [time] Fix RTC is_valid() rejecting valid times after day_of_year cleanup (#15763) --- esphome/components/bm8563/bm8563.cpp | 2 +- esphome/components/ds1307/ds1307.cpp | 2 +- esphome/components/pcf85063/pcf85063.cpp | 2 +- esphome/components/pcf8563/pcf8563.cpp | 2 +- esphome/components/rx8130/rx8130.cpp | 2 +- esphome/core/time.h | 8 ++- tests/components/time/is_valid.cpp | 72 ++++++++++++++++++++++++ 7 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 tests/components/time/is_valid.cpp diff --git a/esphome/components/bm8563/bm8563.cpp b/esphome/components/bm8563/bm8563.cpp index 062094c036f..d911301c9d3 100644 --- a/esphome/components/bm8563/bm8563.cpp +++ b/esphome/components/bm8563/bm8563.cpp @@ -63,7 +63,7 @@ void BM8563::read_time() { rtc_time.day_of_week, rtc_time.hour, rtc_time.minute, rtc_time.second); rtc_time.recalc_timestamp_utc(false); - if (!rtc_time.is_valid()) { + if (!rtc_time.is_valid(/*check_day_of_week=*/true, /*check_day_of_year=*/false)) { ESP_LOGE(TAG, "Invalid RTC time, not syncing to system clock."); return; } diff --git a/esphome/components/ds1307/ds1307.cpp b/esphome/components/ds1307/ds1307.cpp index 8fff4213b4c..ba2ad6032f4 100644 --- a/esphome/components/ds1307/ds1307.cpp +++ b/esphome/components/ds1307/ds1307.cpp @@ -44,7 +44,7 @@ void DS1307Component::read_time() { .year = uint16_t(ds1307_.reg.year + 10u * ds1307_.reg.year_10 + 2000), }; rtc_time.recalc_timestamp_utc(false); - if (!rtc_time.is_valid()) { + if (!rtc_time.is_valid(/*check_day_of_week=*/true, /*check_day_of_year=*/false)) { ESP_LOGE(TAG, "Invalid RTC time, not syncing to system clock."); return; } diff --git a/esphome/components/pcf85063/pcf85063.cpp b/esphome/components/pcf85063/pcf85063.cpp index 1cf28a49557..000de1433cb 100644 --- a/esphome/components/pcf85063/pcf85063.cpp +++ b/esphome/components/pcf85063/pcf85063.cpp @@ -44,7 +44,7 @@ void PCF85063Component::read_time() { .year = uint16_t(pcf85063_.reg.year + 10u * pcf85063_.reg.year_10 + 2000), }; rtc_time.recalc_timestamp_utc(false); - if (!rtc_time.is_valid()) { + if (!rtc_time.is_valid(/*check_day_of_week=*/true, /*check_day_of_year=*/false)) { ESP_LOGE(TAG, "Invalid RTC time, not syncing to system clock."); return; } diff --git a/esphome/components/pcf8563/pcf8563.cpp b/esphome/components/pcf8563/pcf8563.cpp index b748f0156aa..50003ca378b 100644 --- a/esphome/components/pcf8563/pcf8563.cpp +++ b/esphome/components/pcf8563/pcf8563.cpp @@ -44,7 +44,7 @@ void PCF8563Component::read_time() { .year = uint16_t(pcf8563_.reg.year + 10u * pcf8563_.reg.year_10 + 2000), }; rtc_time.recalc_timestamp_utc(false); - if (!rtc_time.is_valid()) { + if (!rtc_time.is_valid(/*check_day_of_week=*/true, /*check_day_of_year=*/false)) { ESP_LOGE(TAG, "Invalid RTC time, not syncing to system clock."); return; } diff --git a/esphome/components/rx8130/rx8130.cpp b/esphome/components/rx8130/rx8130.cpp index 3b704d2551f..0aa6e86d31e 100644 --- a/esphome/components/rx8130/rx8130.cpp +++ b/esphome/components/rx8130/rx8130.cpp @@ -81,7 +81,7 @@ void RX8130Component::read_time() { .year = static_cast(bcd2dec(date[6]) + 2000), }; rtc_time.recalc_timestamp_utc(false); - if (!rtc_time.is_valid()) { + if (!rtc_time.is_valid(/*check_day_of_week=*/true, /*check_day_of_year=*/false)) { ESP_LOGE(TAG, "Invalid RTC time, not syncing to system clock."); return; } diff --git a/esphome/core/time.h b/esphome/core/time.h index ed474320381..0b67b7b3fcd 100644 --- a/esphome/core/time.h +++ b/esphome/core/time.h @@ -76,8 +76,12 @@ struct ESPTime { /// @copydoc strftime(const std::string &format) std::string strftime(const char *format); - /// Check if this ESPTime is valid (all fields in range and year is greater than or equal to 2019) - bool is_valid() const { return this->year >= 2019 && this->fields_in_range(); } + /// Check if this ESPTime is valid (year >= 2019 and the requested fields are in range). + /// @param check_day_of_week validate day_of_week (not always available when constructing from date/time fields) + /// @param check_day_of_year validate day_of_year (not always available when constructing from date/time fields) + bool is_valid(bool check_day_of_week = true, bool check_day_of_year = true) const { + return this->year >= 2019 && this->fields_in_range(check_day_of_week, check_day_of_year); + } /// Check if time fields are in range. /// @param check_day_of_week validate day_of_week (not always available when constructing from date/time fields) diff --git a/tests/components/time/is_valid.cpp b/tests/components/time/is_valid.cpp new file mode 100644 index 00000000000..9148c0e8d6b --- /dev/null +++ b/tests/components/time/is_valid.cpp @@ -0,0 +1,72 @@ +// Regression tests for ESPTime::is_valid() optional checks. +// +// The RTC components (ds1307, bm8563, pcf85063, pcf8563, rx8130) read date/time +// fields from hardware but do not populate day_of_year. They call +// recalc_timestamp_utc(false) -- which skips day_of_year -- and then is_valid(). +// These tests ensure the is_valid() overload can skip day_of_year validation so +// RTCs don't log "Invalid RTC time, not syncing to system clock." for valid times. + +#include +#include "esphome/core/time.h" + +namespace esphome::testing { + +// Build an ESPTime that mirrors what the RTC components construct: all fields +// populated from hardware except day_of_year (left zero-initialized). +static ESPTime make_rtc_like_time() { + ESPTime t{}; + t.second = 30; + t.minute = 15; + t.hour = 12; + t.day_of_week = 4; // thursday + t.day_of_month = 15; + t.month = 4; + t.year = 2026; + // day_of_year intentionally left at 0 -- RTCs don't compute it. + return t; +} + +TEST(ESPTimeIsValid, DefaultRejectsZeroDayOfYear) { + // Default is_valid() checks day_of_year; zero-init is out of range. + ESPTime t = make_rtc_like_time(); + EXPECT_FALSE(t.is_valid()); +} + +TEST(ESPTimeIsValid, SkipDayOfYearAcceptsRTCLikeTime) { + // RTC code path: skip day_of_year validation. + ESPTime t = make_rtc_like_time(); + EXPECT_TRUE(t.is_valid(/*check_day_of_week=*/true, /*check_day_of_year=*/false)); +} + +TEST(ESPTimeIsValid, SkipDayOfYearStillRejectsOutOfRangeFields) { + ESPTime t = make_rtc_like_time(); + t.hour = 25; + EXPECT_FALSE(t.is_valid(/*check_day_of_week=*/true, /*check_day_of_year=*/false)); +} + +TEST(ESPTimeIsValid, SkipDayOfYearStillRejectsYearBefore2019) { + ESPTime t = make_rtc_like_time(); + t.year = 2000; + EXPECT_FALSE(t.is_valid(/*check_day_of_week=*/true, /*check_day_of_year=*/false)); +} + +TEST(ESPTimeIsValid, SkipBothDayChecksAcceptsGPSLikeTime) { + // GPS path (gps_time.cpp) populates neither day_of_week nor day_of_year. + ESPTime t{}; + t.second = 30; + t.minute = 15; + t.hour = 12; + t.day_of_month = 15; + t.month = 4; + t.year = 2026; + EXPECT_TRUE(t.is_valid(/*check_day_of_week=*/false, /*check_day_of_year=*/false)); + EXPECT_FALSE(t.is_valid()); // default still rejects +} + +TEST(ESPTimeIsValid, FullyPopulatedAcceptsWithDefaults) { + ESPTime t = make_rtc_like_time(); + t.day_of_year = 105; + EXPECT_TRUE(t.is_valid()); +} + +} // namespace esphome::testing From 227dfa3730692c2ecf1931cdee1131bb08b4893f Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 16 Apr 2026 14:36:06 -0400 Subject: [PATCH 13/35] [qmc5883l] Move per-update log line from DEBUG to VERBOSE (#15781) --- esphome/components/qmc5883l/qmc5883l.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/qmc5883l/qmc5883l.cpp b/esphome/components/qmc5883l/qmc5883l.cpp index bc2adb5cfe5..d0488d0c9f7 100644 --- a/esphome/components/qmc5883l/qmc5883l.cpp +++ b/esphome/components/qmc5883l/qmc5883l.cpp @@ -100,7 +100,7 @@ void QMC5883LComponent::update() { // ROL_PNT in setup and reading 7 bytes starting at the status register. // If status and all three axes are desired, using ROL_PNT saves you 3 bytes. // But simply not reading status saves you 4 bytes always and is much simpler. - if (ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_DEBUG) { + if (ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_VERBOSE) { err = this->read_register(QMC5883L_REGISTER_STATUS, &status, 1); if (err != i2c::ERROR_OK) { char buf[32]; @@ -165,7 +165,7 @@ void QMC5883LComponent::update() { temp = int16_t(raw_temp) * 0.01f; } - ESP_LOGD(TAG, "Got x=%0.02fµT y=%0.02fµT z=%0.02fµT heading=%0.01f° temperature=%0.01f°C status=%u", x, y, z, heading, + ESP_LOGV(TAG, "Got x=%0.02fµT y=%0.02fµT z=%0.02fµT heading=%0.01f° temperature=%0.01f°C status=%u", x, y, z, heading, temp, status); if (this->x_sensor_ != nullptr) From 81fb6712fef424a0273237786b2a4e4898f57742 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 16 Apr 2026 08:45:33 -1000 Subject: [PATCH 14/35] [bundle] Force-resolve nested IncludeFile during file discovery (#15762) --- esphome/bundle.py | 78 ++++++++- .../fixtures/bundle/bundle_test.yaml | 6 +- .../fixtures/bundle/common/wifi.yaml | 2 + tests/unit_tests/test_bundle.py | 150 +++++++++++++++++- 4 files changed, 226 insertions(+), 10 deletions(-) create mode 100644 tests/unit_tests/fixtures/bundle/common/wifi.yaml diff --git a/esphome/bundle.py b/esphome/bundle.py index b6816c7c95e..efa80acc8cc 100644 --- a/esphome/bundle.py +++ b/esphome/bundle.py @@ -151,8 +151,8 @@ class ConfigBundleCreator: def __init__(self, config: dict[str, Any]) -> None: self._config = config - self._config_dir = CORE.config_dir - self._config_path = CORE.config_path + self._config_dir = Path(CORE.config_dir).resolve() + self._config_path = Path(CORE.config_path).resolve() self._files: list[BundleFile] = [] self._seen_paths: set[Path] = set() self._secrets_paths: set[Path] = set() @@ -258,21 +258,36 @@ class ConfigBundleCreator: def _discover_yaml_includes(self) -> None: """Discover YAML files loaded during config parsing. - We track files by wrapping _load_yaml_internal. The config has already - been loaded at this point (bundle is a POST_CONFIG_ACTION), so we - re-load just to discover the file list. + Deliberately uses a fresh re-parse and force-loads every deferred + ``IncludeFile`` to include *all* potentially-reachable includes, + even branches not selected by the local substitutions. Bundles are + meant to be compiled on another system where command-line + substitution overrides may choose a different branch — e.g. + ``!include network/${eth_model}/config.yaml`` must ship every + candidate so the remote build can pick any one. + + Entries with unresolved substitution variables in the filename + path are skipped with a warning (they cannot be resolved without + the substitution pass). Secrets files are tracked separately so we can filter them to only include the keys this config actually references. """ + # Must be a fresh parse: IncludeFile.load() caches its result in + # _content, and we discover files by listening for loader calls. On + # an already-parsed tree the cache is populated, .load() returns + # without calling the loader, the listener never fires, and the + # referenced files would be silently dropped from the bundle. with yaml_util.track_yaml_loads() as loaded_files: try: - yaml_util.load_yaml(self._config_path) + data = yaml_util.load_yaml(self._config_path) except EsphomeError: _LOGGER.debug( "Bundle: re-loading YAML for include discovery failed, " "proceeding with partial file list" ) + else: + _force_load_include_files(data) for fpath in loaded_files: if fpath == self._config_path.resolve(): @@ -608,6 +623,57 @@ def _add_bytes_to_tar(tar: tarfile.TarFile, name: str, data: bytes) -> None: tar.addfile(info, io.BytesIO(data)) +def _force_load_include_files(obj: Any, _seen: set[int] | None = None) -> None: + """Recursively resolve any ``IncludeFile`` instances in a YAML tree. + + Nested ``!include`` returns a deferred ``IncludeFile`` that is only + resolved during the substitution pass. During bundle discovery we need + the referenced files to actually load so the ``track_yaml_loads`` + listener fires for them. + + ``IncludeFile`` instances with unresolved substitution variables in the + filename cannot be loaded — we skip and warn about those. + """ + if _seen is None: + _seen = set() + + if isinstance(obj, yaml_util.IncludeFile): + if id(obj) in _seen: + return + _seen.add(id(obj)) + if obj.has_unresolved_expressions(): + _LOGGER.warning( + "Bundle: cannot resolve !include %s (referenced from %s) " + "with substitutions in path", + obj.file, + obj.parent_file, + ) + return + try: + loaded = obj.load() + except EsphomeError as err: + _LOGGER.warning( + "Bundle: failed to load !include %s (referenced from %s): %s", + obj.file, + obj.parent_file, + err, + ) + return + _force_load_include_files(loaded, _seen) + elif isinstance(obj, dict): + if id(obj) in _seen: + return + _seen.add(id(obj)) + for value in obj.values(): + _force_load_include_files(value, _seen) + elif isinstance(obj, (list, tuple)): + if id(obj) in _seen: + return + _seen.add(id(obj)) + for item in obj: + _force_load_include_files(item, _seen) + + def _resolve_include_path(include_path: Any) -> Path | None: """Resolve an include path to absolute, skipping system includes.""" if isinstance(include_path, str) and include_path.startswith("<"): diff --git a/tests/unit_tests/fixtures/bundle/bundle_test.yaml b/tests/unit_tests/fixtures/bundle/bundle_test.yaml index f834a8d867f..247f5cc8bb1 100644 --- a/tests/unit_tests/fixtures/bundle/bundle_test.yaml +++ b/tests/unit_tests/fixtures/bundle/bundle_test.yaml @@ -11,9 +11,9 @@ esp32: logger: <<: !include common/base.yaml -wifi: - ssid: !secret wifi_ssid - password: !secret wifi_password +# Plain nested !include — deferred as an IncludeFile until the substitution +# pass. The bundle must force-resolve it to pick up common/wifi.yaml. +wifi: !include common/wifi.yaml api: diff --git a/tests/unit_tests/fixtures/bundle/common/wifi.yaml b/tests/unit_tests/fixtures/bundle/common/wifi.yaml new file mode 100644 index 00000000000..d7e7b3cd450 --- /dev/null +++ b/tests/unit_tests/fixtures/bundle/common/wifi.yaml @@ -0,0 +1,2 @@ +ssid: !secret wifi_ssid +password: !secret wifi_password diff --git a/tests/unit_tests/test_bundle.py b/tests/unit_tests/test_bundle.py index b8b2d0ffd1a..89bf1a33b3e 100644 --- a/tests/unit_tests/test_bundle.py +++ b/tests/unit_tests/test_bundle.py @@ -5,8 +5,10 @@ from __future__ import annotations import io import json from pathlib import Path +import shutil import tarfile from typing import Any +from unittest.mock import patch import pytest @@ -20,6 +22,7 @@ from esphome.bundle import ( _add_bytes_to_tar, _default_target_dir, _find_used_secret_keys, + _force_load_include_files, extract_bundle, is_bundle_path, prepare_bundle_for_compile, @@ -485,7 +488,7 @@ def test_read_bundle_manifest_minimal(tmp_path: Path) -> None: result = read_bundle_manifest(bundle_path) assert result.esphome_version == "unknown" - assert result.files == [] + assert not result.files assert result.has_secrets is False @@ -862,6 +865,117 @@ def test_discover_files_skips_missing_directory(tmp_path: Path) -> None: assert len(files) == 1 +def test_discover_files_nested_include(tmp_path: Path) -> None: + """Nested !include files (e.g. wifi: !include wifi.yaml) are bundled.""" + config_dir = _setup_config_dir(tmp_path) + (config_dir / "test.yaml").write_text( + "esphome:\n name: test\nwifi: !include wifi.yaml\n" + ) + (config_dir / "wifi.yaml").write_text('ssid: "a"\npassword: "b"\n') + + creator = ConfigBundleCreator({}) + files = creator.discover_files() + + paths = [f.path for f in files] + assert "test.yaml" in paths + assert "wifi.yaml" in paths + + +def test_discover_files_deeply_nested_include(tmp_path: Path) -> None: + """Chains of !include (a includes b includes c) are fully resolved.""" + config_dir = _setup_config_dir(tmp_path) + (config_dir / "test.yaml").write_text( + "esphome:\n name: test\nwifi: !include level1.yaml\n" + ) + (config_dir / "level1.yaml").write_text("nested: !include level2.yaml\n") + (config_dir / "level2.yaml").write_text('value: "leaf"\n') + + creator = ConfigBundleCreator({}) + files = creator.discover_files() + + paths = [f.path for f in files] + assert "level1.yaml" in paths + assert "level2.yaml" in paths + + +def test_discover_files_nested_include_unresolved_substitution( + tmp_path: Path, +) -> None: + """!include with substitution vars in path cannot be resolved; skipped gracefully.""" + config_dir = _setup_config_dir(tmp_path) + (config_dir / "test.yaml").write_text( + "esphome:\n name: test\nwifi: !include ${platform}.yaml\n" + ) + + creator = ConfigBundleCreator({}) + # Should not raise + files = creator.discover_files() + + paths = [f.path for f in files] + assert "test.yaml" in paths + + +def test_discover_files_nested_include_load_failure( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A nested !include pointing at a missing file is logged and skipped.""" + config_dir = _setup_config_dir(tmp_path) + (config_dir / "test.yaml").write_text( + "esphome:\n name: test\nwifi: !include missing.yaml\n" + ) + + creator = ConfigBundleCreator({}) + files = creator.discover_files() + + paths = [f.path for f in files] + assert "test.yaml" in paths + assert any( + "failed to load !include" in r.message and "missing.yaml" in r.message + for r in caplog.records + ) + + +def test_force_load_skips_duplicate_include_file() -> None: + """The same IncludeFile referenced twice is only loaded once.""" + + class _StubInclude: + """Mimics yaml_util.IncludeFile minimally for _force_load testing.""" + + def __init__(self) -> None: + self.file = Path("dup.yaml") + self.parent_file = Path("root.yaml") + self.load_calls = 0 + + def has_unresolved_expressions(self) -> bool: + return False + + def load(self) -> dict[str, Any]: + self.load_calls += 1 + return {} + + stub = _StubInclude() + # Same instance appears twice — second visit must hit the _seen guard. + tree = {"a": stub, "b": [stub]} + + with patch("esphome.bundle.yaml_util.IncludeFile", _StubInclude): + _force_load_include_files(tree) + + assert stub.load_calls == 1 + + +def test_force_load_handles_cyclic_containers() -> None: + """Cyclic dict/list references don't cause infinite recursion.""" + cyclic_dict: dict[str, Any] = {} + cyclic_dict["self"] = cyclic_dict + + cyclic_list: list[Any] = [] + cyclic_list.append(cyclic_list) + + # Should return without recursing forever + _force_load_include_files(cyclic_dict) + _force_load_include_files(cyclic_list) + + def test_discover_files_yaml_reload_failure( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1008,6 +1122,40 @@ def test_discover_files_walk_tuple_values(tmp_path: Path) -> None: assert "a.pem" in paths +# --------------------------------------------------------------------------- +# ConfigBundleCreator - fixture-based end-to-end +# --------------------------------------------------------------------------- + + +def test_discover_files_fixture_config(fixture_path: Path, tmp_path: Path) -> None: + """Use the real ``fixtures/bundle/`` tree as an end-to-end reproducer. + + The fixture config uses ``wifi: !include common/wifi.yaml`` — a plain + nested !include that is returned as a deferred ``IncludeFile`` and only + resolved during the substitution pass. Before this fix, bundle discovery + never ran substitutions, so ``common/wifi.yaml`` was silently missing + from the bundle. + """ + # Copy the fixture tree into a tmp dir so the test doesn't rely on the + # source repo being writable and so we can set CORE.config_path freely. + src = fixture_path / "bundle" + dst = tmp_path / "bundle" + shutil.copytree(src, dst) + + CORE.config_path = dst / "bundle_test.yaml" + + creator = ConfigBundleCreator({}) + files = creator.discover_files() + paths = {f.path for f in files} + + # Root and top-level !secret-referenced files + assert "bundle_test.yaml" in paths + assert "secrets.yaml" in paths + # The nested !include — this is what regressed when IncludeFile became + # deferred (PR #12213). + assert "common/wifi.yaml" in paths + + # --------------------------------------------------------------------------- # ConfigBundleCreator - create_bundle # --------------------------------------------------------------------------- From 9cb2b562b9e6706dc00117a7ac6299730eebc100 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 16 Apr 2026 09:01:55 -1000 Subject: [PATCH 15/35] [ili9xxx] Guard against null buffer in display_() when allocation fails (#15786) --- esphome/components/ili9xxx/ili9xxx_defines.h | 2 ++ esphome/components/ili9xxx/ili9xxx_display.cpp | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/esphome/components/ili9xxx/ili9xxx_defines.h b/esphome/components/ili9xxx/ili9xxx_defines.h index f4c5aad957c..70e0937f790 100644 --- a/esphome/components/ili9xxx/ili9xxx_defines.h +++ b/esphome/components/ili9xxx/ili9xxx_defines.h @@ -1,5 +1,7 @@ #pragma once +#include + namespace esphome { namespace ili9xxx { diff --git a/esphome/components/ili9xxx/ili9xxx_display.cpp b/esphome/components/ili9xxx/ili9xxx_display.cpp index a3eff901d30..11acb8a73a4 100644 --- a/esphome/components/ili9xxx/ili9xxx_display.cpp +++ b/esphome/components/ili9xxx/ili9xxx_display.cpp @@ -229,6 +229,10 @@ void ILI9XXXDisplay::update() { } void ILI9XXXDisplay::display_() { + // buffer may be null if allocation failed + if (this->buffer_ == nullptr) { + return; + } // check if something was displayed if ((this->x_high_ < this->x_low_) || (this->y_high_ < this->y_low_)) { return; From 722cfae04c4ce7745701f783f2f9c75fed2243f7 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 16 Apr 2026 15:07:04 -0400 Subject: [PATCH 16/35] [esp32] Accept unquoted minimum_chip_revision values (#15785) --- esphome/components/esp32/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 7b3f9da3da6..a68614cb436 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1222,7 +1222,7 @@ FRAMEWORK_SCHEMA = cv.Schema( cv.Optional(CONF_IGNORE_EFUSE_CUSTOM_MAC, default=False): cv.boolean, cv.Optional(CONF_IGNORE_EFUSE_MAC_CRC, default=False): cv.boolean, cv.Optional(CONF_MINIMUM_CHIP_REVISION): cv.one_of( - *ESP32_CHIP_REVISIONS + *ESP32_CHIP_REVISIONS, string=True ), cv.Optional(CONF_SRAM1_AS_IRAM, default=False): cv.boolean, # DHCP server is needed for WiFi AP mode. When WiFi component is used, From b167b64f06ea724824ea5601ab024bac42c235a8 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Thu, 16 Apr 2026 18:16:58 -0400 Subject: [PATCH 17/35] [lvgl] Guard lv_image_set_src wrapper with LV_USE_IMAGE (#15789) --- esphome/components/lvgl/lvgl_esphome.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/esphome/components/lvgl/lvgl_esphome.h b/esphome/components/lvgl/lvgl_esphome.h index 3ba258b1a2b..3ec1d247d8e 100644 --- a/esphome/components/lvgl/lvgl_esphome.h +++ b/esphome/components/lvgl/lvgl_esphome.h @@ -76,16 +76,17 @@ inline void lv_style_set_text_font(lv_style_t *style, const font::Font *font) { } #endif #if defined(USE_LVGL_IMAGE) && defined(USE_IMAGE) -// Shortcut / overload, so that the source of an image can easily be updated -// from within a lambda. -inline void lv_image_set_src(lv_obj_t *obj, image::Image *image) { lv_image_set_src(obj, image->get_lv_image_dsc()); } +#if LV_USE_IMAGE +// Shortcut / overload, so that the source of an image widget can easily be updated from within a lambda. +inline void lv_image_set_src(lv_obj_t *obj, image::Image *image) { ::lv_image_set_src(obj, image->get_lv_image_dsc()); } +#endif // LV_USE_IMAGE inline void lv_obj_set_style_bitmap_mask_src(lv_obj_t *obj, image::Image *image, lv_style_selector_t selector) { - lv_obj_set_style_bitmap_mask_src(obj, image->get_lv_image_dsc(), selector); + ::lv_obj_set_style_bitmap_mask_src(obj, image->get_lv_image_dsc(), selector); } inline void lv_obj_set_style_bg_image_src(lv_obj_t *obj, image::Image *image, lv_style_selector_t selector) { - lv_obj_set_style_bg_image_src(obj, image->get_lv_image_dsc(), selector); + ::lv_obj_set_style_bg_image_src(obj, image->get_lv_image_dsc(), selector); } #endif // USE_LVGL_IMAGE #ifdef USE_LVGL_ANIMIMG From c3e739eba9c73c34ce843c7b6cc1bdcee11355c4 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Fri, 17 Apr 2026 21:17:16 +1000 Subject: [PATCH 18/35] [mipi_spi] Drawing fixes for native display (#15802) --- esphome/components/mipi_spi/display.py | 6 +- esphome/components/mipi_spi/mipi_spi.h | 14 +- .../mipi_spi/test_final_validate.py | 185 ++++++++++++++++++ 3 files changed, 195 insertions(+), 10 deletions(-) create mode 100644 tests/component_tests/mipi_spi/test_final_validate.py diff --git a/esphome/components/mipi_spi/display.py b/esphome/components/mipi_spi/display.py index 42c7ec22249..364ada90463 100644 --- a/esphome/components/mipi_spi/display.py +++ b/esphome/components/mipi_spi/display.py @@ -195,7 +195,7 @@ def model_schema(config): "big_endian", "little_endian", lower=True ), model.option(CONF_COLOR_DEPTH, 16): cv.one_of(*color_depth, lower=True), - model.option(CONF_DRAW_ROUNDING, 2): power_of_two, + model.option(CONF_DRAW_ROUNDING, 1): power_of_two, model.option(CONF_PIXEL_MODE, DISPLAY_16BIT): cv.one_of( *pixel_modes, lower=True ), @@ -297,9 +297,9 @@ def _final_validate(config): buffer_size = color_depth // 8 * width * height // frac # Target a buffer size of 20kB, except for large displays, which shouldn't end up here - fraction = min(20000.0, buffer_size // 16) / buffer_size + fraction = min(20000.0, buffer_size // 4) / buffer_size config[CONF_BUFFER_SIZE] = 1.0 / next( - x for x in range(2, 17) if fraction >= 1 / x + (x for x in range(2, 8) if fraction >= 1 / x), 8 ) diff --git a/esphome/components/mipi_spi/mipi_spi.h b/esphome/components/mipi_spi/mipi_spi.h index 2242be6c17b..f2923458937 100644 --- a/esphome/components/mipi_spi/mipi_spi.h +++ b/esphome/components/mipi_spi/mipi_spi.h @@ -546,13 +546,12 @@ class MipiSpiBuffer : public MipiSpistart_line_ = 0; this->start_line_ < this->get_height_internal(); - this->start_line_ += this->get_height_internal() / FRACTION) { + auto increment = (this->get_height_internal() / FRACTION / ROUNDING) * ROUNDING; + for (this->start_line_ = 0; this->start_line_ < this->get_height_internal(); this->start_line_ = this->end_line_) { #if ESPHOME_LOG_LEVEL == ESPHOME_LOG_LEVEL_VERBOSE auto lap = millis(); #endif - this->end_line_ = - clamp_at_most(this->start_line_ + this->get_height_internal() / FRACTION, this->get_height_internal()); + this->end_line_ = clamp_at_most(this->start_line_ + increment, this->get_height_internal()); if (this->auto_clear_enabled_) { this->clear(); } @@ -574,12 +573,13 @@ class MipiSpiBuffer : public MipiSpix_low_ = this->x_low_ / ROUNDING * ROUNDING; this->y_low_ = this->y_low_ / ROUNDING * ROUNDING; - this->x_high_ = (this->x_high_ + ROUNDING) / ROUNDING * ROUNDING - 1; - this->y_high_ = (this->y_high_ + ROUNDING) / ROUNDING * ROUNDING - 1; + this->x_high_ = round_buffer(this->x_high_ + 1) - 1; + this->y_high_ = clamp_at_most(round_buffer(this->y_high_ + 1) - 1, this->end_line_ - 1); int w = this->x_high_ - this->x_low_ + 1; int h = this->y_high_ - this->y_low_ + 1; this->write_to_display_(this->x_low_, this->y_low_, w, h, this->buffer_, this->x_low_, - this->y_low_ - this->start_line_, round_buffer(this->get_width_internal()) - w); + this->y_low_ - this->start_line_, + round_buffer(this->get_width_internal()) - w - this->x_low_); // invalidate watermarks this->x_low_ = this->get_width_internal(); this->y_low_ = this->get_height_internal(); diff --git a/tests/component_tests/mipi_spi/test_final_validate.py b/tests/component_tests/mipi_spi/test_final_validate.py new file mode 100644 index 00000000000..8c45b47752c --- /dev/null +++ b/tests/component_tests/mipi_spi/test_final_validate.py @@ -0,0 +1,185 @@ +"""Tests for the _final_validate buffer size calculation in mipi_spi.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from esphome.components.display import CONF_SHOW_TEST_CARD +from esphome.components.esp32 import KEY_BOARD, KEY_VARIANT, VARIANT_ESP32 +from esphome.components.mipi_spi.display import CONFIG_SCHEMA, FINAL_VALIDATE_SCHEMA +from esphome.const import CONF_BUFFER_SIZE, PlatformFramework +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + + +def _validated(config: ConfigType) -> ConfigType: + """Run the component config schema followed by the final validation.""" + config = CONFIG_SCHEMA(config) + FINAL_VALIDATE_SCHEMA(config) + return config + + +def _custom_config( + width: int, + height: int, + color_depth: str | int | None = None, + **extra: Any, +) -> ConfigType: + """Build a minimal valid custom-model config with the given dimensions.""" + config: ConfigType = { + "model": "custom", + "dc_pin": 18, + "dimensions": {"width": width, "height": height}, + "init_sequence": [[0xA0, 0x01]], + } + if color_depth is not None: + config["color_depth"] = color_depth + config.update(extra) + return config + + +# The auto buffer-size selection inside _final_validate targets ~20 kB of +# pixel buffer. For a buffer of ``depth_bytes * width * height``, it picks the +# smallest integer ``x`` in range(2, 8) such that +# ``min(20000, buffer // 4) / buffer >= 1 / x`` (falling back to ``x = 8``). +# The test cases below cover the full range of possible outcomes (1/4 .. 1/8). +@pytest.mark.parametrize( + ("width", "height", "color_depth", "expected"), + [ + # 16-bit color depth -- buffer = 2 * width * height + # 128*160*2 = 40960 B -> fraction = 10240/40960 = 0.25 -> x = 4 + pytest.param(128, 160, "16bit", 1.0 / 4, id="16bit_tiny"), + # 200*224*2 = 89600 B -> fraction = 20000/89600 ≈ 0.2232 -> x = 5 + pytest.param(200, 224, "16bit", 1.0 / 5, id="16bit_small"), + # 240*224*2 = 107520 B -> fraction ≈ 0.1860 -> x = 6 + pytest.param(240, 224, "16bit", 1.0 / 6, id="16bit_medium"), + # 200*320*2 = 128000 B -> fraction = 0.15625 -> x = 7 + pytest.param(200, 320, "16bit", 1.0 / 7, id="16bit_large"), + # 240*320*2 = 153600 B -> fraction ≈ 0.1302 -> default x = 8 + pytest.param(240, 320, "16bit", 1.0 / 8, id="16bit_xlarge"), + # 320*480*2 = 307200 B -> fraction ≈ 0.0651 -> default x = 8 + pytest.param(320, 480, "16bit", 1.0 / 8, id="16bit_huge"), + # 8-bit color depth -- buffer = width * height + # 320*240 = 76800 B -> fraction = 19200/76800 = 0.25 -> x = 4 + pytest.param(320, 240, "8bit", 1.0 / 4, id="8bit_tiny"), + # 400*224 = 89600 B -> fraction ≈ 0.2232 -> x = 5 + pytest.param(400, 224, "8bit", 1.0 / 5, id="8bit_small"), + # 480*224 = 107520 B -> fraction ≈ 0.1860 -> x = 6 + pytest.param(480, 224, "8bit", 1.0 / 6, id="8bit_medium"), + # 400*320 = 128000 B -> fraction = 0.15625 -> x = 7 + pytest.param(400, 320, "8bit", 1.0 / 7, id="8bit_large"), + # 480*320 = 153600 B -> fraction ≈ 0.1302 -> default x = 8 + pytest.param(480, 320, "8bit", 1.0 / 8, id="8bit_xlarge"), + ], +) +def test_buffer_size_auto_selected( + width: int, + height: int, + color_depth: str, + expected: float, + set_core_config: SetCoreConfigCallable, +) -> None: + """Without PSRAM or an explicit buffer_size, a fraction is chosen from the display size. + + Without any drawing method and without LVGL, final validation also auto-enables + ``show_test_card``, which in turn makes the component require a buffer and therefore + triggers the buffer-size selection path. + """ + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_BOARD: "esp32dev", KEY_VARIANT: VARIANT_ESP32}, + ) + + config = _validated(_custom_config(width, height, color_depth)) + + # Sanity check: final validation should have enabled the test card for us, + # which is what causes the buffer-size calculation to actually run. + assert config.get(CONF_SHOW_TEST_CARD) is True + assert config[CONF_BUFFER_SIZE] == pytest.approx(expected) + + +@pytest.mark.parametrize( + "buffer_size", + [0.125, 0.25, 0.5, 1.0], + ids=["one_eighth", "one_quarter", "half", "full"], +) +def test_explicit_buffer_size_is_preserved( + buffer_size: float, + set_core_config: SetCoreConfigCallable, +) -> None: + """An explicitly configured buffer_size is never overridden by final validation.""" + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_BOARD: "esp32dev", KEY_VARIANT: VARIANT_ESP32}, + ) + + config = _validated( + _custom_config(240, 320, "16bit", buffer_size=buffer_size), + ) + + assert config[CONF_BUFFER_SIZE] == pytest.approx(buffer_size) + + +def test_buffer_size_not_set_when_psram_enabled( + set_core_config: SetCoreConfigCallable, + set_component_config, +) -> None: + """When PSRAM is enabled the auto buffer-size selection is skipped.""" + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_BOARD: "esp32dev", KEY_VARIANT: VARIANT_ESP32}, + ) + # Presence of the psram domain in the full config is what _final_validate checks. + set_component_config("psram", True) + + config = _validated(_custom_config(240, 320, "16bit")) + + assert CONF_BUFFER_SIZE not in config + + +def test_buffer_size_not_set_when_buffer_not_required( + set_core_config: SetCoreConfigCallable, + set_component_config, +) -> None: + """With LVGL present and no drawing methods, no buffer fraction is chosen. + + LVGL suppresses the automatic show_test_card injection, which means + ``requires_buffer`` is False and the early-return branch fires. + """ + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_BOARD: "esp32dev", KEY_VARIANT: VARIANT_ESP32}, + ) + set_component_config("lvgl", []) + + config = _validated(_custom_config(240, 320, "16bit")) + + assert CONF_BUFFER_SIZE not in config + # And no test card should have been auto-enabled either. + assert not config.get(CONF_SHOW_TEST_CARD) + + +def test_buffer_size_selected_when_lvgl_with_test_card( + set_core_config: SetCoreConfigCallable, + set_component_config, +) -> None: + """LVGL present + an explicit drawing method still triggers buffer sizing. + + When LVGL is enabled, ``show_test_card`` is not injected automatically, + but users can still request it explicitly -- in that case ``requires_buffer`` + is True and the buffer-size heuristic still runs. + """ + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_BOARD: "esp32dev", KEY_VARIANT: VARIANT_ESP32}, + ) + set_component_config("lvgl", []) + + # 128x160 @ 16bit -> expected 1/4 (see test_buffer_size_auto_selected). + config = _validated( + _custom_config(128, 160, "16bit", show_test_card=True), + ) + + assert config[CONF_BUFFER_SIZE] == pytest.approx(1.0 / 4) From f5806818cd4b087fa282d1c3b98f618f8c8e3c10 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Fri, 17 Apr 2026 22:11:05 +1000 Subject: [PATCH 19/35] [image] Fix byte order handling (#15800) --- esphome/components/image/__init__.py | 17 ++++++++++------- esphome/components/image/image.cpp | 2 +- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/esphome/components/image/__init__.py b/esphome/components/image/__init__.py index 4a5fcc385e8..7db50597e6f 100644 --- a/esphome/components/image/__init__.py +++ b/esphome/components/image/__init__.py @@ -28,7 +28,6 @@ from esphome.const import ( CONF_URL, ) from esphome.core import CORE, HexInt -from esphome.final_validate import full_config _LOGGER = logging.getLogger(__name__) @@ -676,12 +675,16 @@ def _final_validate(config): :param config: :return: """ - fv = full_config.get() - if "lvgl" in fv and not all(CONF_BYTE_ORDER in x for x in config): - config = config.copy() - for c in config: - if not c.get(CONF_BYTE_ORDER): - c[CONF_BYTE_ORDER] = "LITTLE_ENDIAN" + config = config.copy() + for c in config: + if byte_order := c.get(CONF_BYTE_ORDER): + if byte_order == "BIG_ENDIAN": + _LOGGER.warning( + "The image '%s' is configured with big-endian byte order, little-endian is expected", + c.get(CONF_FILE), + ) + else: + c[CONF_BYTE_ORDER] = "LITTLE_ENDIAN" return config diff --git a/esphome/components/image/image.cpp b/esphome/components/image/image.cpp index a6f9e35e2e5..5b4ed6968c6 100644 --- a/esphome/components/image/image.cpp +++ b/esphome/components/image/image.cpp @@ -189,7 +189,7 @@ Color Image::get_rgb_pixel_(int x, int y) const { } Color Image::get_rgb565_pixel_(int x, int y) const { const uint8_t *pos = this->data_start_ + (x + y * this->width_) * this->bpp_ / 8; - uint16_t rgb565 = encode_uint16(progmem_read_byte(pos), progmem_read_byte(pos + 1)); + uint16_t rgb565 = encode_uint16(progmem_read_byte(pos + 1), progmem_read_byte(pos)); auto r = (rgb565 & 0xF800) >> 11; auto g = (rgb565 & 0x07E0) >> 5; auto b = rgb565 & 0x001F; From b26601a3dcfc284153b6c5405dcfe91f6356d8c9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Apr 2026 02:45:50 -1000 Subject: [PATCH 20/35] [core] coerce set_interval(0) / update_interval: 0ms to 1ms (#15799) --- esphome/config_validation.py | 21 +++++- esphome/core/scheduler.cpp | 13 ++++ .../scheduler_interval_zero_coerced.yaml | 27 ++++++++ .../test_scheduler_interval_zero_coerced.py | 67 +++++++++++++++++++ tests/unit_tests/test_config_validation.py | 28 ++++++++ 5 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 tests/integration/fixtures/scheduler_interval_zero_coerced.yaml create mode 100644 tests/integration/test_scheduler_interval_zero_coerced.py diff --git a/esphome/config_validation.py b/esphome/config_validation.py index 31cfb41a6d9..bf53013d9ba 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -943,7 +943,26 @@ def time_period_in_minutes_(value): def update_interval(value): if value == "never": return TimePeriodMilliseconds(milliseconds=SCHEDULER_DONT_RUN) - return positive_time_period_milliseconds(value) + result = positive_time_period_milliseconds(value) + # 0ms was historically (mis)used as a pseudo-loop() mechanism for + # PollingComponents. Under the hood it calls set_interval(0), which + # causes Scheduler::call() to spin (WDT reset in the field). Coerce + # to 1ms so existing configs keep working at ~1kHz instead of + # spinning. Don't hard-fail so configs don't break on upgrade; + # authors should migrate to HighFrequencyLoopRequester (C++) for + # true run-every-loop behaviour. + if result.total_milliseconds == 0: + _LOGGER.warning( + "update_interval of 0ms is not supported - coercing to 1ms. " + "A literal 0ms schedule would spin the main loop (the scheduled " + "item would always be due, so the scheduler would never yield " + "back) and trigger a watchdog reset. Set update_interval to a " + "non-zero value such as 1ms or higher. (Custom C++ components " + "that need true run-every-loop behaviour should override loop() " + "with HighFrequencyLoopRequester instead.)" + ) + return TimePeriodMilliseconds(milliseconds=1) + return result time_period = Any(time_period_str_unit, time_period_str_colon, time_period_dict) diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index 3e75a680643..7e6ad19ac70 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -144,6 +144,19 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type return; } + // An interval of 0 means "fire every tick forever," which is misuse: the + // item would always be due, causing Scheduler::call() to spin and starve + // the main loop (WDT reset in the field). Coerce to 1ms so existing code + // using update_interval=0ms as a pseudo-loop() continues to work at ~1kHz, + // and warn so authors can migrate to HighFrequencyLoopRequester which is + // the intended mechanism for running fast in the main loop. Zero-delay + // timeouts (defer) remain legitimate one-shots and are not affected. + if (type == SchedulerItem::INTERVAL && delay == 0) [[unlikely]] { + ESP_LOGE(TAG, "[%s] set_interval(0) would spin main loop - coercing to 1ms (use HighFrequencyLoopRequester)", + component ? LOG_STR_ARG(component->get_component_log_str()) : LOG_STR_LITERAL("?")); + delay = 1; + } + // Take lock early to protect scheduler_item_pool_ access and retry-cancelled check LockGuard guard{this->lock_}; diff --git a/tests/integration/fixtures/scheduler_interval_zero_coerced.yaml b/tests/integration/fixtures/scheduler_interval_zero_coerced.yaml new file mode 100644 index 00000000000..13be55d6176 --- /dev/null +++ b/tests/integration/fixtures/scheduler_interval_zero_coerced.yaml @@ -0,0 +1,27 @@ +esphome: + name: sched-interval-zero + +host: +api: +logger: + level: DEBUG + +globals: + - id: fire_count + type: int + initial_value: "0" + +interval: + # Deliberately configure 0ms — this path goes through the C++ + # Scheduler::set_timer_common_ coercion (not the Python cv.update_interval + # path, since interval: doesn't call cv.update_interval — it's an intervals + # component schema, not a PollingComponent's update_interval). + # Expected: scheduler coerces to 1ms at registration, emits ESP_LOGE, + # fires at ~1kHz instead of spinning. + - interval: 0ms + then: + - lambda: |- + id(fire_count) += 1; + if (id(fire_count) == 50) { + ESP_LOGI("test", "ZERO_INTERVAL_50_FIRES_REACHED"); + } diff --git a/tests/integration/test_scheduler_interval_zero_coerced.py b/tests/integration/test_scheduler_interval_zero_coerced.py new file mode 100644 index 00000000000..f71c0f7281f --- /dev/null +++ b/tests/integration/test_scheduler_interval_zero_coerced.py @@ -0,0 +1,67 @@ +"""Test that Scheduler::set_timer_common_ coerces interval=0 to 1ms. + +Regression test for the scheduler busy-loop when interval=0 was passed +literally. Without the coercion, Scheduler::call() would spin forever +because the item's next_execution == now_64 after re-scheduling, failing +the loop's `> now_64` break condition. The device would fail to yield +back to the main loop and trigger a WDT reset. + +With the coercion, interval=0 becomes interval=1 and the scheduler +fires at ~1kHz (bounded by the loop), the main loop continues to run, +and the device stays responsive to API calls. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_scheduler_interval_zero_coerced( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """interval=0ms must be coerced to 1ms and not starve the main loop.""" + loop = asyncio.get_running_loop() + reached_50: asyncio.Future[None] = loop.create_future() + coerce_warning: asyncio.Future[None] = loop.create_future() + + def on_log_line(line: str) -> None: + if "ZERO_INTERVAL_50_FIRES_REACHED" in line and not reached_50.done(): + reached_50.set_result(None) + if "would spin main loop" in line and not coerce_warning.done(): + coerce_warning.set_result(None) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + # The API-client connection itself is evidence that the main loop + # is not starved — if set_interval(0) were spinning we could not + # get here at all. + device_info = await client.device_info() + assert device_info is not None + assert device_info.name == "sched-interval-zero" + + # Coerce warning must fire at registration + try: + await asyncio.wait_for(coerce_warning, timeout=5.0) + except TimeoutError: + pytest.fail("Expected coerce warning 'would spin main loop' not seen") + + # The coerced 1ms interval should fire 50 times quickly — this + # confirms the callback actually runs (not just registered) and the + # scheduler yields back to the main loop each time. + try: + await asyncio.wait_for(reached_50, timeout=5.0) + except TimeoutError: + pytest.fail( + "Coerced interval=0→1ms did not reach 50 fires within 5s, " + "which would indicate either the coercion failed or the " + "main loop is still being starved." + ) diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index ac84ce7cc89..f038272d8b0 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -24,6 +24,7 @@ from esphome.const import ( PLATFORM_LN882X, PLATFORM_RP2040, PLATFORM_RTL87XX, + SCHEDULER_DONT_RUN, ) from esphome.core import CORE, HexInt, Lambda @@ -765,3 +766,30 @@ def test_percentage_validators__raw_number_above_one_without_percent_sign( config_validation.unbounded_percentage(value) with pytest.raises(Invalid, match="percent sign"): config_validation.unbounded_possibly_negative_percentage(value) + + +def test_update_interval__coerces_zero_to_one_ms( + caplog: pytest.LogCaptureFixture, +) -> None: + """update_interval: 0ms must be coerced to 1ms (not rejected) because a + literal 0ms schedule causes Scheduler::call() to spin. Coercion keeps + existing configs compiling on upgrade while emitting a user-facing + warning that directs them to set a non-zero value.""" + with caplog.at_level("WARNING"): + result = config_validation.update_interval("0ms") + assert result.total_milliseconds == 1 + assert "update_interval of 0ms is not supported" in caplog.text + assert "1ms" in caplog.text + + +def test_update_interval__preserves_nonzero_values() -> None: + """Non-zero update_interval values must pass through unchanged.""" + assert config_validation.update_interval("1ms").total_milliseconds == 1 + assert config_validation.update_interval("50ms").total_milliseconds == 50 + assert config_validation.update_interval("60s").total_milliseconds == 60000 + + +def test_update_interval__never_passes_through() -> None: + """update_interval: never must still map to SCHEDULER_DONT_RUN.""" + result = config_validation.update_interval("never") + assert result.total_milliseconds == SCHEDULER_DONT_RUN From ed5852c2d6a64b7618bdabb35c67f650afd59fce Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Fri, 17 Apr 2026 10:05:30 -0400 Subject: [PATCH 21/35] [ethernet] Fix SPI3_HOST default breaking compile on variants without SPI3 (#15809) Co-authored-by: J. Nick Koston --- .../components/ethernet/ethernet_component.h | 2 +- .../ethernet/test.esp32-c3-idf.yaml | 19 +++++++++++++++++++ ...720.esp32-idf.yaml => test.esp32-idf.yaml} | 0 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 tests/components/ethernet/test.esp32-c3-idf.yaml rename tests/components/ethernet/{test-lan8720.esp32-idf.yaml => test.esp32-idf.yaml} (100%) diff --git a/esphome/components/ethernet/ethernet_component.h b/esphome/components/ethernet/ethernet_component.h index 3a878423156..17c84ee9543 100644 --- a/esphome/components/ethernet/ethernet_component.h +++ b/esphome/components/ethernet/ethernet_component.h @@ -221,7 +221,7 @@ class EthernetComponent final : public Component { int reset_pin_{-1}; int phy_addr_spi_{-1}; int clock_speed_; - spi_host_device_t interface_{SPI3_HOST}; + spi_host_device_t interface_{SPI2_HOST}; #ifdef USE_ETHERNET_SPI_POLLING_SUPPORT uint32_t polling_interval_{0}; #endif diff --git a/tests/components/ethernet/test.esp32-c3-idf.yaml b/tests/components/ethernet/test.esp32-c3-idf.yaml new file mode 100644 index 00000000000..b7b95875c6d --- /dev/null +++ b/tests/components/ethernet/test.esp32-c3-idf.yaml @@ -0,0 +1,19 @@ +ethernet: + type: W5500 + clk_pin: 6 + mosi_pin: 7 + miso_pin: 2 + cs_pin: 10 + interrupt_pin: 3 + reset_pin: 4 + clock_speed: 10Mhz + manual_ip: + static_ip: 192.168.178.56 + gateway: 192.168.178.1 + subnet: 255.255.255.0 + domain: .local + mac_address: "02:AA:BB:CC:DD:01" + on_connect: + - logger.log: "Ethernet connected!" + on_disconnect: + - logger.log: "Ethernet disconnected!" diff --git a/tests/components/ethernet/test-lan8720.esp32-idf.yaml b/tests/components/ethernet/test.esp32-idf.yaml similarity index 100% rename from tests/components/ethernet/test-lan8720.esp32-idf.yaml rename to tests/components/ethernet/test.esp32-idf.yaml From 9841deec31cb1d70a19228fb6691ba5b6acfaa16 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Fri, 17 Apr 2026 09:31:31 -0500 Subject: [PATCH 22/35] [core] Fix DelayAction compile error with non-const reference args (#15814) --- esphome/core/base_automation.h | 4 +++- tests/components/http_request/http_request.yaml | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/esphome/core/base_automation.h b/esphome/core/base_automation.h index 11133d39739..17f937d10d9 100644 --- a/esphome/core/base_automation.h +++ b/esphome/core/base_automation.h @@ -205,7 +205,9 @@ template class DelayAction : public Action, public Compon } else { // For delays with arguments, capture by value to preserve argument values // Arguments must be copied because original references may be invalid after delay - auto f = [this, x...]() { this->play_next_(x...); }; + // `mutable` is required so captured copies of non-const reference args (e.g. std::string&) + // are passed as non-const lvalues to play_next_(const Ts&...) where Ts may be `T&` + auto f = [this, x...]() mutable { this->play_next_(x...); }; App.scheduler.set_timer_common_(this, Scheduler::SchedulerItem::TIMEOUT, Scheduler::NameType::NUMERIC_ID_INTERNAL, nullptr, static_cast(InternalSchedulerID::DELAY_ACTION), this->delay_.value(x...), std::move(f), diff --git a/tests/components/http_request/http_request.yaml b/tests/components/http_request/http_request.yaml index 13ca5ceba03..ef67671c91d 100644 --- a/tests/components/http_request/http_request.yaml +++ b/tests/components/http_request/http_request.yaml @@ -45,6 +45,11 @@ esphome: args: - response->status_code - body.c_str() + - delay: 1s + - logger.log: + format: "After delay, body still: %s" + args: + - body.c_str() http_request: useragent: esphome/tagreader From 1d88027618d06662b8b4df1e1d047922ff1330dc Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Fri, 17 Apr 2026 16:14:54 -0400 Subject: [PATCH 23/35] [esp32] Downgrade unneeded `ignore_pin_validation_error` to a warning (#15811) --- esphome/components/esp32/gpio.py | 12 +++- tests/component_tests/esp32/test_esp32.py | 79 ++++++++++++++++++++++- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/esphome/components/esp32/gpio.py b/esphome/components/esp32/gpio.py index a7180cbcd7e..36dd44155ae 100644 --- a/esphome/components/esp32/gpio.py +++ b/esphome/components/esp32/gpio.py @@ -172,10 +172,16 @@ def validate_gpio_pin(pin): exc, ) else: - # Throw an exception if used for a pin that would not have resulted - # in a validation error anyway! + # `ignore_pin_validation_error` only suppresses an error raised by the + # variant's pin_validation above (e.g. SPI flash/PSRAM pins, invalid pin + # numbers). If that didn't raise, the option is a no-op -- warn so the + # user can clean it up, but don't block the build. if ignore_pin_validation_warning: - raise cv.Invalid(f"GPIO{pin[CONF_NUMBER]} is not a reserved pin") + _LOGGER.warning( + "GPIO%d has no validation errors to ignore; " + "remove `ignore_pin_validation_error: true` from this pin.", + pin[CONF_NUMBER], + ) return pin diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index bd4f9828ce5..ac492e27529 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -8,10 +8,16 @@ from typing import Any import pytest -from esphome.components.esp32 import VARIANTS -from esphome.components.esp32.const import KEY_ESP32, KEY_SDKCONFIG_OPTIONS +from esphome.components.esp32 import VARIANT_ESP32, VARIANTS +from esphome.components.esp32.const import KEY_ESP32, KEY_SDKCONFIG_OPTIONS, KEY_VARIANT +from esphome.components.esp32.gpio import validate_gpio_pin import esphome.config_validation as cv -from esphome.const import CONF_ESPHOME, PlatformFramework +from esphome.const import ( + CONF_ESPHOME, + CONF_IGNORE_PIN_VALIDATION_ERROR, + CONF_NUMBER, + PlatformFramework, +) from esphome.core import CORE from tests.component_tests.types import SetCoreConfigCallable @@ -149,6 +155,73 @@ def test_execute_from_psram_p4_sdkconfig( assert "CONFIG_SPIRAM_RODATA" not in sdkconfig +def test_ignore_pin_validation_error_on_clean_pin_warns( + set_core_config: SetCoreConfigCallable, + caplog: pytest.LogCaptureFixture, +) -> None: + """A pin that passes validation but sets `ignore_pin_validation_error: true` + should log a warning nudging the user to remove the flag, and not raise.""" + set_core_config( + PlatformFramework.ESP32_IDF, platform_data={KEY_VARIANT: VARIANT_ESP32} + ) + + pin = {CONF_NUMBER: 4, CONF_IGNORE_PIN_VALIDATION_ERROR: True} + with caplog.at_level("WARNING"): + result = validate_gpio_pin(pin) + + assert result[CONF_NUMBER] == 4 + assert "GPIO4 has no validation errors to ignore" in caplog.text + + +def test_ignore_pin_validation_error_on_dirty_pin_suppresses( + set_core_config: SetCoreConfigCallable, + caplog: pytest.LogCaptureFixture, +) -> None: + """A pin that fails validation with `ignore_pin_validation_error: true` should + log the suppression warning and not raise (existing behavior).""" + set_core_config( + PlatformFramework.ESP32_IDF, platform_data={KEY_VARIANT: VARIANT_ESP32} + ) + + # GPIO6 is a flash pin on ESP32 -> pin_validation raises cv.Invalid + pin = {CONF_NUMBER: 6, CONF_IGNORE_PIN_VALIDATION_ERROR: True} + with caplog.at_level("WARNING"): + result = validate_gpio_pin(pin) + + assert result[CONF_NUMBER] == 6 + assert "Ignoring validation error on pin 6" in caplog.text + + +def test_dirty_pin_without_ignore_flag_raises( + set_core_config: SetCoreConfigCallable, +) -> None: + """A pin that fails validation without the ignore flag should still raise.""" + set_core_config( + PlatformFramework.ESP32_IDF, platform_data={KEY_VARIANT: VARIANT_ESP32} + ) + + pin = {CONF_NUMBER: 6, CONF_IGNORE_PIN_VALIDATION_ERROR: False} + with pytest.raises(cv.Invalid, match="flash interface"): + validate_gpio_pin(pin) + + +def test_clean_pin_without_ignore_flag_does_not_warn( + set_core_config: SetCoreConfigCallable, + caplog: pytest.LogCaptureFixture, +) -> None: + """A clean pin without the ignore flag should pass silently.""" + set_core_config( + PlatformFramework.ESP32_IDF, platform_data={KEY_VARIANT: VARIANT_ESP32} + ) + + pin = {CONF_NUMBER: 4, CONF_IGNORE_PIN_VALIDATION_ERROR: False} + with caplog.at_level("WARNING"): + result = validate_gpio_pin(pin) + + assert result[CONF_NUMBER] == 4 + assert "has no validation errors to ignore" not in caplog.text + + def test_execute_from_psram_disabled_sdkconfig( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], From e2dfef5ddce725e7f3323e43749452c4ed93c324 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 18 Apr 2026 06:42:45 +1000 Subject: [PATCH 24/35] [runtime_image] Fix RGB order (#15813) --- esphome/components/runtime_image/runtime_image.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/runtime_image/runtime_image.cpp b/esphome/components/runtime_image/runtime_image.cpp index fa42b53496b..4c7f1bfb6f3 100644 --- a/esphome/components/runtime_image/runtime_image.cpp +++ b/esphome/components/runtime_image/runtime_image.cpp @@ -127,9 +127,9 @@ void RuntimeImage::draw_pixel(int x, int y, const Color &color) { uint32_t pos = this->get_position_(x, y); Color mapped_color = color; this->map_chroma_key(mapped_color); - this->buffer_[pos + 0] = mapped_color.r; + this->buffer_[pos + 0] = mapped_color.b; this->buffer_[pos + 1] = mapped_color.g; - this->buffer_[pos + 2] = mapped_color.b; + this->buffer_[pos + 2] = mapped_color.r; if (this->transparency_ == image::TRANSPARENCY_ALPHA_CHANNEL) { this->buffer_[pos + 3] = color.w; } From 6d5340f253009a1421329bcff3c6251ca0e6aa7d Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Sat, 18 Apr 2026 09:17:28 +1000 Subject: [PATCH 25/35] [lvgl] Fix crash with snow on rotated display (#15822) --- esphome/components/lvgl/lvgl_esphome.cpp | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index ce9b013dcf1..d8248e4aa4e 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -642,26 +642,28 @@ void LvglComponent::write_random_() { int iterations = 6 - lv_display_get_inactive_time(this->disp_) / 60000; if (iterations <= 0) iterations = 1; + int16_t width = lv_display_get_horizontal_resolution(this->disp_); + int16_t height = lv_display_get_vertical_resolution(this->disp_); while (iterations-- != 0) { - int32_t col = random_uint32() % this->width_; + int32_t col = random_uint32() % width; col = col / this->draw_rounding * this->draw_rounding; - int32_t row = random_uint32() % this->height_; + int32_t row = random_uint32() % height; row = row / this->draw_rounding * this->draw_rounding; // size will be between 8 and 32, and a multiple of draw_rounding int32_t size = (random_uint32() % 25 + 8) / this->draw_rounding * this->draw_rounding; - lv_area_t area{col, row, col + size - 1, row + size - 1}; + lv_area_t area{.x1 = col, .y1 = row, .x2 = col + size - 1, .y2 = row + size - 1}; // clip to display bounds just in case - if (area.x2 >= this->width_) - area.x2 = this->width_ - 1; - if (area.y2 >= this->height_) - area.y2 = this->height_ - 1; + if (area.x2 >= width) + area.x2 = width - 1; + if (area.y2 >= height) + area.y2 = height - 1; // line_len can't exceed 1024, and minimum buffer size is 2048, so this won't overflow the buffer size_t line_len = lv_area_get_width(&area) * lv_area_get_height(&area) / 2; for (size_t i = 0; i != line_len; i++) { - ((uint32_t *) (this->draw_buf_))[i] = random_uint32(); + reinterpret_cast(this->draw_buf_)[i] = random_uint32(); } - this->draw_buffer_(&area, (lv_color_data *) this->draw_buf_); + this->draw_buffer_(&area, reinterpret_cast(this->draw_buf_)); } } From 08ac61ae9424821ac868d817971313d1e8508394 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 18 Apr 2026 06:58:54 -0500 Subject: [PATCH 26/35] [core] Feed WDT unconditionally in main loop to fix empty-config panic (#15830) --- esphome/core/application.h | 26 +++++++++++++++++--------- esphome/core/scheduler.cpp | 5 ++++- esphome/core/scheduler.h | 3 ++- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/esphome/core/application.h b/esphome/core/application.h index 60087d527d2..db8af735bd4 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -641,7 +641,7 @@ class Application { void enable_component_loop_(Component *component); void enable_pending_loops_(); void activate_looping_component_(uint16_t index); - inline void ESPHOME_ALWAYS_INLINE before_loop_tasks_(uint32_t loop_start_time); + inline uint32_t ESPHOME_ALWAYS_INLINE before_loop_tasks_(uint32_t loop_start_time); inline void ESPHOME_ALWAYS_INLINE after_loop_tasks_() { this->in_loop_ = false; } /// Process dump_config output one component per loop iteration. @@ -845,18 +845,15 @@ inline void Application::drain_wake_notifications_() { } #endif // USE_HOST -inline void ESPHOME_ALWAYS_INLINE Application::before_loop_tasks_(uint32_t loop_start_time) { +inline uint32_t ESPHOME_ALWAYS_INLINE Application::before_loop_tasks_(uint32_t loop_start_time) { #ifdef USE_HOST // Drain wake notifications first to clear socket for next wake this->drain_wake_notifications_(); #endif - // Process scheduled tasks. Scheduler::call now feeds the watchdog itself - // after each scheduled item that actually runs, so we no longer need an - // unconditional feed here — when Scheduler::call has no work to do, the - // only elapsed time is a sleep wake + a few instructions, and when it does - // have work, it fed the wdt as it went. - this->scheduler.call(loop_start_time); + // Scheduler::call feeds the WDT per item and returns the timestamp of the + // last fired item, or the input unchanged when nothing ran. + uint32_t last_op_end_time = this->scheduler.call(loop_start_time); // Process any pending enable_loop requests from ISRs // This must be done before marking in_loop_ = true to avoid race conditions @@ -874,13 +871,24 @@ inline void ESPHOME_ALWAYS_INLINE Application::before_loop_tasks_(uint32_t loop_ // Mark that we're in the loop for safe reentrant modifications this->in_loop_ = true; + return last_op_end_time; } inline void ESPHOME_ALWAYS_INLINE Application::loop() { // Get the initial loop time at the start uint32_t last_op_end_time = millis(); - this->before_loop_tasks_(last_op_end_time); + // Returned timestamp keeps us monotonic with last_wdt_feed_ (advanced by + // the scheduler's per-item feeds) without an extra millis() call. + last_op_end_time = this->before_loop_tasks_(last_op_end_time); + // Guarantee a WDT touch every tick — covers configs with no looping + // components and no scheduler work, where the per-item / per-component + // feeds never fire. Rate-limited inline fast path, ~free when unneeded. + this->feed_wdt_with_time(last_op_end_time); +#ifdef USE_RUNTIME_STATS + uint32_t loop_before_end_us = micros(); + uint64_t loop_before_scheduled_us = ComponentRuntimeStats::global_recorded_us - loop_recorded_snap; +#endif for (this->current_loop_index_ = 0; this->current_loop_index_ < this->looping_components_active_end_; this->current_loop_index_++) { diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index 7e6ad19ac70..b0eaa670ac2 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -533,7 +533,7 @@ void HOT Scheduler::process_defer_queue_slow_path_(uint32_t &now) { } #endif /* not ESPHOME_THREAD_SINGLE */ -void HOT Scheduler::call(uint32_t now) { +uint32_t HOT Scheduler::call(uint32_t now) { #ifndef ESPHOME_THREAD_SINGLE this->process_defer_queue_(now); #endif /* not ESPHOME_THREAD_SINGLE */ @@ -703,6 +703,9 @@ void HOT Scheduler::call(uint32_t now) { this->debug_verify_no_leak_(); } #endif + // execute_item_() advances `now` as items fire; return it so the caller + // stays monotonic with last_wdt_feed_. + return now; } void HOT Scheduler::process_to_add_slow_path_() { LockGuard guard{this->lock_}; diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 43a3ec7049b..00a7f269538 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -129,7 +129,8 @@ class Scheduler { // Execute all scheduled items that are ready // @param now Fresh timestamp from millis() - must not be stale/cached - void call(uint32_t now); + // @return Timestamp of the last item that ran, or `now` unchanged if none ran. + uint32_t call(uint32_t now); // Move items from to_add_ into the main heap. // IMPORTANT: This method should only be called from the main thread (loop task). From 78701debec764ae9e0ba6339eb27b8ff96a8ba46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Apr 2026 12:32:36 +0000 Subject: [PATCH 27/35] Bump aioesphomeapi from 44.16.0 to 44.16.1 (#15836) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b14d5f5f5a3..25638c0e7bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ platformio==6.1.19 esptool==5.2.0 click==8.3.2 esphome-dashboard==20260408.1 -aioesphomeapi==44.16.0 +aioesphomeapi==44.16.1 zeroconf==0.148.0 puremagic==1.30 ruamel.yaml==0.19.1 # dashboard_import From 602305b20de2ab6da44baa4e113eb3016115365e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sat, 18 Apr 2026 08:04:51 -0500 Subject: [PATCH 28/35] [core] Default PollingComponent() to 1ms when codegen is bypassed (#15831) --- esphome/core/component.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/core/component.h b/esphome/core/component.h index 3307c5ae76e..9e339dce649 100644 --- a/esphome/core/component.h +++ b/esphome/core/component.h @@ -593,7 +593,7 @@ class Component { */ class PollingComponent : public Component { public: - PollingComponent() : PollingComponent(0) {} + PollingComponent() : PollingComponent(1) {} /** Initialize this polling component with the given update interval in ms. * From ef780886c35c019fdbdb9c0e4a1fe343958897e0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Sun, 19 Apr 2026 16:00:31 -0500 Subject: [PATCH 29/35] [substitutions] Fix `substitutions: !include file.yaml` regression (#15850) --- esphome/components/packages/__init__.py | 8 ++- esphome/components/substitutions/__init__.py | 35 +++++++++++-- .../15-substitutions_as_include.approved.yaml | 5 ++ .../15-substitutions_as_include.input.yaml | 5 ++ .../substitutions/15-substitutions_inc.yaml | 1 + ...ons_as_include_with_packages.approved.yaml | 5 ++ ...utions_as_include_with_packages.input.yaml | 9 ++++ ...ubstitutions_include_cli_var.approved.yaml | 6 +++ ...7-substitutions_include_cli_var.input.yaml | 8 +++ tests/unit_tests/test_substitutions.py | 51 +++++++++++++++++++ 10 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.input.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/15-substitutions_inc.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.input.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.approved.yaml create mode 100644 tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.input.yaml diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 3f3df753512..252a24061a0 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -10,6 +10,7 @@ from esphome.components.substitutions import ( ContextVars, push_context, resolve_include, + resolve_substitutions_block, substitute, ) from esphome.components.substitutions.jinja import has_jinja @@ -516,7 +517,12 @@ def do_packages_pass( if CONF_PACKAGES not in config: return config - substitutions = UserDict(config.pop(CONF_SUBSTITUTIONS, {})) + with cv.prepend_path(CONF_SUBSTITUTIONS): + substitutions = UserDict( + resolve_substitutions_block( + config.pop(CONF_SUBSTITUTIONS, {}), command_line_substitutions + ) + ) processor = _PackageProcessor( substitutions, command_line_substitutions, skip_update ) diff --git a/esphome/components/substitutions/__init__.py b/esphome/components/substitutions/__init__.py index c0bd9d7be97..7aace9bb648 100644 --- a/esphome/components/substitutions/__init__.py +++ b/esphome/components/substitutions/__init__.py @@ -414,6 +414,34 @@ def _warn_unresolved_variables(errors: ErrList) -> None: ) +def resolve_substitutions_block( + substitutions: Any, + command_line_substitutions: dict[str, Any] | None, +) -> dict[str, Any]: + """Resolve a deferred ``substitutions: !include file.yaml`` and validate the shape. + + The caller is responsible for wrapping the call in + ``cv.prepend_path(CONF_SUBSTITUTIONS)`` for error reporting. + ``command_line_substitutions`` seeds the filename context so + ``substitutions: !include ${var}.yaml`` can reference CLI-provided vars. + """ + if isinstance(substitutions, IncludeFile): + # Single-shot resolution — matches ``_walk_packages`` for the + # ``packages: !include`` entry point. Chained includes (an include that + # itself loads another ``!include`` at the top level) are not supported. + substitutions, _ = resolve_include( + substitutions, + [], + ContextVars(command_line_substitutions or {}), + strict_undefined=False, + ) + if not isinstance(substitutions, dict): + raise cv.Invalid( + f"Substitutions must be a key to value mapping, got {type(substitutions)}" + ) + return substitutions + + def do_substitution_pass( config: OrderedDict, command_line_substitutions: dict[str, Any] | None = None ) -> OrderedDict: @@ -429,10 +457,9 @@ def do_substitution_pass( # Use merge_dicts_ordered to preserve OrderedDict type for move_to_end() substitutions = config.pop(CONF_SUBSTITUTIONS, {}) with cv.prepend_path(CONF_SUBSTITUTIONS): - if not isinstance(substitutions, dict): - raise cv.Invalid( - f"Substitutions must be a key to value mapping, got {type(substitutions)}" - ) + substitutions = resolve_substitutions_block( + substitutions, command_line_substitutions + ) substitutions = merge_dicts_ordered( substitutions, command_line_substitutions or {} ) diff --git a/tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.approved.yaml b/tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.approved.yaml new file mode 100644 index 00000000000..14aa707def5 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.approved.yaml @@ -0,0 +1,5 @@ +substitutions: + wifi_password: sub_password +wifi: + ssid: main_ssid + password: sub_password diff --git a/tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.input.yaml b/tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.input.yaml new file mode 100644 index 00000000000..5909e7bf4f7 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/15-substitutions_as_include.input.yaml @@ -0,0 +1,5 @@ +substitutions: !include 15-substitutions_inc.yaml + +wifi: + ssid: main_ssid + password: $wifi_password diff --git a/tests/unit_tests/fixtures/substitutions/15-substitutions_inc.yaml b/tests/unit_tests/fixtures/substitutions/15-substitutions_inc.yaml new file mode 100644 index 00000000000..44d9a4b9ef2 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/15-substitutions_inc.yaml @@ -0,0 +1 @@ +wifi_password: sub_password diff --git a/tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.approved.yaml b/tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.approved.yaml new file mode 100644 index 00000000000..14aa707def5 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.approved.yaml @@ -0,0 +1,5 @@ +substitutions: + wifi_password: sub_password +wifi: + ssid: main_ssid + password: sub_password diff --git a/tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.input.yaml b/tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.input.yaml new file mode 100644 index 00000000000..a2e72f33a23 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/16-substitutions_as_include_with_packages.input.yaml @@ -0,0 +1,9 @@ +substitutions: !include 15-substitutions_inc.yaml + +packages: + wifi_pkg: + wifi: + password: $wifi_password + +wifi: + ssid: main_ssid diff --git a/tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.approved.yaml b/tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.approved.yaml new file mode 100644 index 00000000000..f1fd5fb0786 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.approved.yaml @@ -0,0 +1,6 @@ +substitutions: + subs_file: 15-substitutions_inc + wifi_password: sub_password +wifi: + ssid: main_ssid + password: sub_password diff --git a/tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.input.yaml b/tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.input.yaml new file mode 100644 index 00000000000..3248504b468 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/17-substitutions_include_cli_var.input.yaml @@ -0,0 +1,8 @@ +command_line_substitutions: + subs_file: 15-substitutions_inc + +substitutions: !include ${subs_file}.yaml + +wifi: + ssid: main_ssid + password: $wifi_password diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index 01c669e5420..71bbd9db86d 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -675,6 +675,57 @@ def test_include_filename_substitution_undefined_var(tmp_path: Path) -> None: substitutions.do_substitution_pass(config) +def test_do_substitution_pass_included_substitutions_must_be_mapping( + tmp_path: Path, +) -> None: + """`substitutions: !include list.yaml` where the file holds a list raises cv.Invalid. + + Locks in the shape check that runs after the deferred IncludeFile has been + resolved. + """ + parent = tmp_path / "main.yaml" + parent.write_text("") + + def loader(path: Path): + return ["not", "a", "mapping"] + + include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader) + config = OrderedDict({CONF_SUBSTITUTIONS: include}) + + with pytest.raises( + cv.Invalid, match="Substitutions must be a key to value mapping" + ): + substitutions.do_substitution_pass(config) + + +def test_do_packages_pass_included_substitutions_must_be_mapping( + tmp_path: Path, +) -> None: + """`substitutions: !include list.yaml` alongside `packages:` raises cv.Invalid. + + Without the shape check, ``UserDict(...)`` would surface a low-level + ``TypeError``; the explicit ``cv.Invalid`` points at the substitutions path. + """ + parent = tmp_path / "main.yaml" + parent.write_text("") + + def loader(path: Path): + return ["not", "a", "mapping"] + + include = yaml_util.IncludeFile(parent, "subs.yaml", None, loader) + config = OrderedDict( + { + CONF_SUBSTITUTIONS: include, + "packages": {"noop": {"wifi": {"ssid": "main"}}}, + } + ) + + with pytest.raises( + cv.Invalid, match="Substitutions must be a key to value mapping" + ): + do_packages_pass(config) + + def test_resolve_package_undefined_var_in_include_filename(tmp_path: Path) -> None: """An undefined substitution in a package include filename raises cv.Invalid. From 1862c6115f51e89587dd8a1dd9462ae7fac18464 Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Sun, 19 Apr 2026 23:09:14 +0200 Subject: [PATCH 30/35] [packages] Improve error messages with include stack and fix missing path propagation (#15844) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- esphome/components/packages/__init__.py | 9 ++ esphome/components/substitutions/__init__.py | 50 +++++++++++ .../component_tests/packages/test_packages.py | 88 ++++++++++++++++++- tests/unit_tests/test_substitutions.py | 34 +++++++ 4 files changed, 179 insertions(+), 2 deletions(-) diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 252a24061a0..97a53094800 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -8,7 +8,9 @@ from typing import Any from esphome import git, yaml_util from esphome.components.substitutions import ( ContextVars, + ErrList, push_context, + raise_first_undefined, resolve_include, resolve_substitutions_block, substitute, @@ -360,12 +362,19 @@ def _substitute_package_definition( if isinstance(package_config, str) or ( isinstance(package_config, dict) and is_remote_package(package_config) ): + # Collect undefined-variable errors (rather than raising strict) so the + # path walked through a remote-package dict is preserved and the user + # sees which field (url / path / ref / ...) referenced the undefined + # variable. + errors: ErrList = [] package_config = substitute( item=package_config, path=[], parent_context=context_vars or ContextVars(), strict_undefined=False, + errors=errors, ) + raise_first_undefined(errors, package_config, "package definition") return package_config diff --git a/esphome/components/substitutions/__init__.py b/esphome/components/substitutions/__init__.py index 7aace9bb648..0144c13c017 100644 --- a/esphome/components/substitutions/__init__.py +++ b/esphome/components/substitutions/__init__.py @@ -30,6 +30,56 @@ ErrList = list[tuple[UndefinedError, SubstitutionPath, Any]] jinja = Jinja() +def raise_first_undefined( + errors: ErrList, + source: Any, + context_label: str, +) -> None: + """If *errors* is non-empty, raise ``cv.Invalid`` for the first undefined variable. + + The raised error names the missing variable, the path walked into *source* + (for nested dicts, e.g. ``url`` or ``ref``), and the YAML source location + when *source* carries one. Only the first error is surfaced; the user will + re-run after fixing it and any remaining undefined variables will be + reported then. + + ``context_label`` is the noun describing where the undefined variable + appeared (e.g. ``"package definition"``). + """ + if not errors: + return + err, err_path, err_value = errors[0] + if len(errors) > 1: + # Log any further undefined variables so debug-level output covers + # the full set, even though only the first is surfaced to the user. + extras = ", ".join( + f"{e.message} at '{'->'.join(str(p) for p in p_path)}'" + for e, p_path, _ in errors[1:] + ) + _LOGGER.debug("Additional undefined variables in %s: %s", context_label, extras) + # Prefer the location of the offending scalar (e.g. the `url:` value) over + # the enclosing package-definition dict so the message points at the exact + # line/column that carries the undefined variable. + location_node = ( + err_value + if isinstance(err_value, ESPHomeDataBase) and err_value.esp_range is not None + else source + ) + location = "" + if ( + isinstance(location_node, ESPHomeDataBase) + and location_node.esp_range is not None + ): + mark = location_node.esp_range.start_mark + # DocumentLocation.line/column are 0-based (from the YAML Mark). Render + # as 1-based to match config.line_info() and editor line numbering. + location = f" (in {mark.document} {mark.line + 1}:{mark.column + 1})" + field = f" at '{'->'.join(str(p) for p in err_path)}'" if err_path else "" + raise cv.Invalid( + f"Undefined variable in {context_label}{field}: {err.message}{location}" + ) + + def validate_substitution_key(value: Any) -> str: """Validate and normalize a substitution key, stripping a leading ``$`` if present.""" value = cv.string(value) diff --git a/tests/component_tests/packages/test_packages.py b/tests/component_tests/packages/test_packages.py index cd91c4d8cb1..0bd339efa9c 100644 --- a/tests/component_tests/packages/test_packages.py +++ b/tests/component_tests/packages/test_packages.py @@ -2,18 +2,20 @@ import logging from pathlib import Path +import re from unittest.mock import MagicMock, patch import pytest from esphome.components.packages import ( CONFIG_SCHEMA, + _substitute_package_definition, _walk_packages, do_packages_pass, is_package_definition, merge_packages, ) -from esphome.components.substitutions import do_substitution_pass +from esphome.components.substitutions import ContextVars, do_substitution_pass import esphome.config as config_module from esphome.config import resolve_extend_remove from esphome.config_helpers import Extend, Remove @@ -44,7 +46,7 @@ from esphome.const import ( ) from esphome.core import CORE from esphome.util import OrderedDict -from esphome.yaml_util import IncludeFile, add_context +from esphome.yaml_util import IncludeFile, add_context, load_yaml # Test strings TEST_DEVICE_NAME = "test_device_name" @@ -1399,3 +1401,85 @@ def test_raw_config_contains_merged_esphome_from_package(tmp_path) -> None: "CORE.raw_config should contain esphome section after package merge" ) assert CORE.raw_config[CONF_ESPHOME][CONF_NAME] == TEST_DEVICE_NAME + + +# --------------------------------------------------------------------------- +# _substitute_package_definition +# --------------------------------------------------------------------------- + + +def test_substitute_package_definition_local_dict_returned_unchanged() -> None: + """A plain local config dict is not substituted and is returned as-is.""" + pkg = {CONF_WIFI: {CONF_SSID: "test"}} + result = _substitute_package_definition(pkg, ContextVars()) + assert result is pkg + + +def test_substitute_package_definition_string_resolved_with_context() -> None: + """A string package definition has its variables substituted.""" + ctx = ContextVars({"variant": "esp32"}) + result = _substitute_package_definition("device-${variant}.yaml", ctx) + assert result == "device-esp32.yaml" + + +def test_substitute_package_definition_undefined_in_string() -> None: + """An undefined variable in a package URL string raises cv.Invalid.""" + with pytest.raises(cv.Invalid, match="Undefined variable in package definition"): + _substitute_package_definition( + "github://org/repo/${undefined_var}/pkg.yaml", ContextVars() + ) + + +def test_substitute_package_definition_undefined_in_remote_dict_field() -> None: + """An undefined variable inside a remote-dict field names the offending field.""" + with pytest.raises(cv.Invalid) as exc_info: + _substitute_package_definition( + {CONF_URL: "github://${typo}/repo"}, ContextVars() + ) + err = str(exc_info.value) + assert "'typo' is undefined" in err + assert CONF_URL in err + + +def test_substitute_package_definition_undefined_in_remote_dict_non_first_field() -> ( + None +): + """The field path joins correctly for non-first dict fields (e.g. ``ref``).""" + with pytest.raises(cv.Invalid) as exc_info: + _substitute_package_definition( + { + CONF_URL: "github://org/repo", + CONF_REF: "branch-${branch_typo}", + }, + ContextVars(), + ) + err = str(exc_info.value) + assert "'branch_typo' is undefined" in err + assert CONF_REF in err + + +def test_substitute_package_definition_includes_source_location(tmp_path: Path) -> None: + """A package loaded from YAML surfaces file/line/col in the cv.Invalid message. + + Line/column are rendered 1-based (matching config.line_info() and editor + line numbering) and point at the offending scalar, not the enclosing dict. + """ + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text( + "packages:\n broken: github://org/repo/${undefined_var}/pkg.yaml\n" + ) + config = load_yaml(yaml_file) + package_config = config[CONF_PACKAGES]["broken"] + + with pytest.raises(cv.Invalid) as exc_info: + _substitute_package_definition(package_config, ContextVars()) + + err = str(exc_info.value) + assert "main.yaml" in err + # The offending value lives on line 2 (1-based). Column depends on the YAML + # loader, so we only pin line and check that a 1-based column is present. + match = re.search(r"main\.yaml (\d+):(\d+)", err) + assert match, err + line, col = int(match.group(1)), int(match.group(2)) + assert line == 2, f"expected 1-based line 2, got {line} (err={err!r})" + assert col >= 1, f"expected 1-based column ≥ 1, got {col} (err={err!r})" diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index 71bbd9db86d..3599e703d9d 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -14,6 +14,7 @@ from esphome.components.packages import ( do_packages_pass, merge_packages, ) +from esphome.components.substitutions.jinja import UndefinedError from esphome.config import resolve_extend_remove from esphome.config_helpers import Extend, merge_config import esphome.config_validation as cv @@ -675,6 +676,39 @@ def test_include_filename_substitution_undefined_var(tmp_path: Path) -> None: substitutions.do_substitution_pass(config) +def test_raise_first_undefined_logs_extras_at_debug( + caplog: pytest.LogCaptureFixture, +) -> None: + """Only the first undefined error is raised; extras are logged at debug.""" + errors: substitutions.ErrList = [ + (UndefinedError("'a' is undefined"), ["url"], None), + (UndefinedError("'b' is undefined"), ["ref"], None), + (UndefinedError("'c' is undefined"), ["path"], None), + ] + + with ( + caplog.at_level(logging.DEBUG, logger="esphome.components.substitutions"), + pytest.raises(cv.Invalid) as exc_info, + ): + substitutions.raise_first_undefined(errors, None, "package definition") + + # First error is surfaced as the cv.Invalid message. + raised = str(exc_info.value) + assert "'a' is undefined" in raised + assert "'b' is undefined" not in raised + assert "'c' is undefined" not in raised + + # Remaining errors are captured via debug logging for troubleshooting. + assert "Additional undefined variables in package definition" in caplog.text + assert "'b' is undefined at 'ref'" in caplog.text + assert "'c' is undefined at 'path'" in caplog.text + + +def test_raise_first_undefined_noop_on_empty() -> None: + """An empty errors list is a no-op — no exception, no log.""" + substitutions.raise_first_undefined([], None, "package definition") + + def test_do_substitution_pass_included_substitutions_must_be_mapping( tmp_path: Path, ) -> None: From 36812591ebf2bc00930618ae5eca8a78ab51bb1d Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Mon, 20 Apr 2026 10:20:56 +1200 Subject: [PATCH 31/35] Bump version to 2026.4.1 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index 6f1f3d0e7e1..deb57df1d3e 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.4.0 +PROJECT_NUMBER = 2026.4.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index 95fd488dca6..1f5b3b6c577 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.4.0" +__version__ = "2026.4.1" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From bab9cd3e7a9b6ebfd2a0ac1ab0a9dd10371d6529 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 16 Apr 2026 14:07:26 -1000 Subject: [PATCH 32/35] [runtime_stats] Track main loop active time and report overhead (#15743) --- .../runtime_stats/runtime_stats.cpp | 99 +++++++++++++++---- .../components/runtime_stats/runtime_stats.h | 41 ++++++++ esphome/core/application.h | 23 +++++ esphome/core/component.cpp | 4 + esphome/core/component.h | 8 ++ tests/integration/test_runtime_stats.py | 33 +++++++ 6 files changed, 189 insertions(+), 19 deletions(-) diff --git a/esphome/components/runtime_stats/runtime_stats.cpp b/esphome/components/runtime_stats/runtime_stats.cpp index 06714b5a446..9ed141155a5 100644 --- a/esphome/components/runtime_stats/runtime_stats.cpp +++ b/esphome/components/runtime_stats/runtime_stats.cpp @@ -32,40 +32,101 @@ void RuntimeStatsCollector::log_stats_() { " Period stats (last %" PRIu32 "ms): %zu active components", this->log_interval_, count); - if (count == 0) { - return; + // Sum component time so we can derive main-loop overhead + // (active loop time minus time attributable to component loop()s). + // Period sum iterates the active-in-period subset; total sum must iterate + // all components since total_active_time_us_ includes iterations where + // currently-idle components previously ran. + uint64_t period_component_sum_us = 0; + for (size_t i = 0; i < count; i++) { + period_component_sum_us += sorted[i]->runtime_stats_.period_time_us; + } + uint64_t total_component_sum_us = 0; + for (auto *component : components) { + total_component_sum_us += component->runtime_stats_.total_time_us; } - // Sort by period runtime (descending) - std::sort(sorted, sorted + count, compare_period_time); + if (count > 0) { + // Sort by period runtime (descending) + std::sort(sorted, sorted + count, compare_period_time); - // Log top components by period runtime - for (size_t i = 0; i < count; i++) { - const auto &stats = sorted[i]->runtime_stats_; - ESP_LOGI(TAG, " %s: count=%" PRIu32 ", avg=%.3fms, max=%.2fms, total=%.1fms", - LOG_STR_ARG(sorted[i]->get_component_log_str()), stats.period_count, - stats.period_count > 0 ? stats.period_time_us / (float) stats.period_count / 1000.0f : 0.0f, - stats.period_max_time_us / 1000.0f, stats.period_time_us / 1000.0f); + // Log top components by period runtime + for (size_t i = 0; i < count; i++) { + const auto &stats = sorted[i]->runtime_stats_; + ESP_LOGI(TAG, " %s: count=%" PRIu32 ", avg=%.3fms, max=%.2fms, total=%.1fms", + LOG_STR_ARG(sorted[i]->get_component_log_str()), stats.period_count, + stats.period_count > 0 ? stats.period_time_us / (float) stats.period_count / 1000.0f : 0.0f, + stats.period_max_time_us / 1000.0f, stats.period_time_us / 1000.0f); + } + } + + // Main-loop overhead for the period: active wall time minus component time. + // active = sum of per-iteration loop time excluding yield/sleep. + if (this->period_active_count_ > 0) { + uint64_t active = this->period_active_time_us_; + uint64_t overhead = active > period_component_sum_us ? active - period_component_sum_us : 0; + // Use double for µs→ms conversion so multi-day uptimes (where total + // microsecond counters exceed float's ~7-digit mantissa) keep resolution. + ESP_LOGI(TAG, + " main_loop: iters=%" PRIu64 ", active_avg=%.3fms, active_max=%.2fms, active_total=%.1fms, " + "overhead_total=%.1fms", + this->period_active_count_, + static_cast(active) / static_cast(this->period_active_count_) / 1000.0, + static_cast(this->period_active_max_us_) / 1000.0, static_cast(active) / 1000.0, + static_cast(overhead) / 1000.0); + uint64_t before = this->period_before_time_us_; + uint64_t tail = this->period_tail_time_us_; + uint64_t accounted = before + tail; + uint64_t inter = overhead > accounted ? overhead - accounted : 0; + ESP_LOGI(TAG, " main_loop_overhead_section: before=%.1fms, tail=%.1fms, inter_component=%.1fms", + static_cast(before) / 1000.0, static_cast(tail) / 1000.0, + static_cast(inter) / 1000.0); } // Log total stats since boot (only for active components - idle ones haven't changed) ESP_LOGI(TAG, " Total stats (since boot): %zu active components", count); - // Re-sort by total runtime for all-time stats - std::sort(sorted, sorted + count, compare_total_time); + if (count > 0) { + // Re-sort by total runtime for all-time stats + std::sort(sorted, sorted + count, compare_total_time); - for (size_t i = 0; i < count; i++) { - const auto &stats = sorted[i]->runtime_stats_; - ESP_LOGI(TAG, " %s: count=%" PRIu32 ", avg=%.3fms, max=%.2fms, total=%.1fms", - LOG_STR_ARG(sorted[i]->get_component_log_str()), stats.total_count, - stats.total_count > 0 ? stats.total_time_us / (float) stats.total_count / 1000.0f : 0.0f, - stats.total_max_time_us / 1000.0f, stats.total_time_us / 1000.0); + for (size_t i = 0; i < count; i++) { + const auto &stats = sorted[i]->runtime_stats_; + ESP_LOGI(TAG, " %s: count=%" PRIu32 ", avg=%.3fms, max=%.2fms, total=%.1fms", + LOG_STR_ARG(sorted[i]->get_component_log_str()), stats.total_count, + stats.total_count > 0 ? stats.total_time_us / (float) stats.total_count / 1000.0f : 0.0f, + stats.total_max_time_us / 1000.0f, stats.total_time_us / 1000.0); + } + } + + if (this->total_active_count_ > 0) { + uint64_t active = this->total_active_time_us_; + uint64_t overhead = active > total_component_sum_us ? active - total_component_sum_us : 0; + ESP_LOGI(TAG, + " main_loop: iters=%" PRIu64 ", active_avg=%.3fms, active_max=%.2fms, active_total=%.1fms, " + "overhead_total=%.1fms", + this->total_active_count_, + static_cast(active) / static_cast(this->total_active_count_) / 1000.0, + static_cast(this->total_active_max_us_) / 1000.0, static_cast(active) / 1000.0, + static_cast(overhead) / 1000.0); + uint64_t before = this->total_before_time_us_; + uint64_t tail = this->total_tail_time_us_; + uint64_t accounted = before + tail; + uint64_t inter = overhead > accounted ? overhead - accounted : 0; + ESP_LOGI(TAG, " main_loop_overhead_section: before=%.1fms, tail=%.1fms, inter_component=%.1fms", + static_cast(before) / 1000.0, static_cast(tail) / 1000.0, + static_cast(inter) / 1000.0); } // Reset period stats for (auto *component : components) { component->runtime_stats_.reset_period(); } + this->period_active_count_ = 0; + this->period_active_time_us_ = 0; + this->period_active_max_us_ = 0; + this->period_before_time_us_ = 0; + this->period_tail_time_us_ = 0; } bool RuntimeStatsCollector::compare_period_time(Component *a, Component *b) { diff --git a/esphome/components/runtime_stats/runtime_stats.h b/esphome/components/runtime_stats/runtime_stats.h index 3c2c9f78add..82e0fb7c618 100644 --- a/esphome/components/runtime_stats/runtime_stats.h +++ b/esphome/components/runtime_stats/runtime_stats.h @@ -29,6 +29,31 @@ class RuntimeStatsCollector { // Process any pending stats printing (should be called after component loop) void process_pending_stats(uint32_t current_time); + // Record the wall time of one main loop iteration excluding the yield/sleep. + // Called once per loop from Application::loop(). + // active_us = total time between loop start and just before yield. + // before_us = time spent in before_loop_tasks_ (scheduler + ISR enable_loop). + // tail_us = time spent in after_loop_tasks_ + the trailing record/stats prefix. + // Residual overhead at log time = active − Σ(component) − before − tail, + // which captures per-iteration inter-component bookkeeping (set_current_component, + // WarnIfComponentBlockingGuard construction/destruction, feed_wdt_with_time calls, + // the for-loop itself). + void record_loop_active(uint32_t active_us, uint32_t before_us, uint32_t tail_us) { + this->period_active_count_++; + this->period_active_time_us_ += active_us; + if (active_us > this->period_active_max_us_) + this->period_active_max_us_ = active_us; + this->total_active_count_++; + this->total_active_time_us_ += active_us; + if (active_us > this->total_active_max_us_) + this->total_active_max_us_ = active_us; + + this->period_before_time_us_ += before_us; + this->total_before_time_us_ += before_us; + this->period_tail_time_us_ += tail_us; + this->total_tail_time_us_ += tail_us; + } + protected: void log_stats_(); // Static comparators — member functions have friend access, lambdas do not @@ -37,6 +62,22 @@ class RuntimeStatsCollector { uint32_t log_interval_; uint32_t next_log_time_{0}; + + // Main loop active-time stats (wall time per iteration, excluding yield/sleep). + // Counters are uint64_t — at sub-millisecond loop times a uint32_t can wrap in + // a few weeks of uptime, which is well within ESPHome device lifetimes. + uint64_t period_active_count_{0}; + uint64_t period_active_time_us_{0}; + uint32_t period_active_max_us_{0}; + uint64_t total_active_count_{0}; + uint64_t total_active_time_us_{0}; + uint32_t total_active_max_us_{0}; + + // Split of overhead sections — accumulated per iteration. + uint64_t period_before_time_us_{0}; + uint64_t total_before_time_us_{0}; + uint64_t period_tail_time_us_{0}; + uint64_t total_tail_time_us_{0}; }; } // namespace runtime_stats diff --git a/esphome/core/application.h b/esphome/core/application.h index db8af735bd4..19245ab203b 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -875,6 +875,17 @@ inline uint32_t ESPHOME_ALWAYS_INLINE Application::before_loop_tasks_(uint32_t l } inline void ESPHOME_ALWAYS_INLINE Application::loop() { +#ifdef USE_RUNTIME_STATS + // Capture the start of the active (non-sleeping) portion of this iteration. + // Used to derive main-loop overhead = active time − Σ(component time) − + // before/tail splits recorded below. + uint32_t loop_active_start_us = micros(); + // Snapshot the cumulative component-recorded time so we can subtract the + // slice that the scheduler spends inside its own WarnIfComponentBlockingGuard + // (scheduler.cpp) — that time is already counted in per-component stats, + // so charging it again to "before" would double-count. + uint64_t loop_recorded_snap = ComponentRuntimeStats::global_recorded_us; +#endif // Get the initial loop time at the start uint32_t last_op_end_time = millis(); @@ -907,12 +918,24 @@ inline void ESPHOME_ALWAYS_INLINE Application::loop() { this->feed_wdt_with_time(last_op_end_time); } +#ifdef USE_RUNTIME_STATS + uint32_t loop_tail_start_us = micros(); +#endif this->after_loop_tasks_(); #ifdef USE_RUNTIME_STATS // Process any pending runtime stats printing after all components have run // This ensures stats printing doesn't affect component timing measurements if (global_runtime_stats != nullptr) { + uint32_t loop_now_us = micros(); + // Subtract scheduled-component time from the "before" bucket so it is + // not double-counted (it is already attributed to per-component stats). + uint32_t loop_before_wall_us = loop_before_end_us - loop_active_start_us; + uint32_t loop_before_overhead_us = loop_before_wall_us > loop_before_scheduled_us + ? loop_before_wall_us - static_cast(loop_before_scheduled_us) + : 0; + global_runtime_stats->record_loop_active(loop_now_us - loop_active_start_us, loop_before_overhead_us, + loop_now_us - loop_tail_start_us); global_runtime_stats->process_pending_stats(last_op_end_time); } #endif diff --git a/esphome/core/component.cpp b/esphome/core/component.cpp index 8949b4b76dc..e33652482ea 100644 --- a/esphome/core/component.cpp +++ b/esphome/core/component.cpp @@ -506,6 +506,10 @@ void PollingComponent::stop_poller() { uint32_t PollingComponent::get_update_interval() const { return this->update_interval_; } +#ifdef USE_RUNTIME_STATS +uint64_t ComponentRuntimeStats::global_recorded_us = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +#endif + void __attribute__((noinline, cold)) WarnIfComponentBlockingGuard::warn_blocking(Component *component, uint32_t blocking_time) { bool should_warn; diff --git a/esphome/core/component.h b/esphome/core/component.h index 9e339dce649..717ca362576 100644 --- a/esphome/core/component.h +++ b/esphome/core/component.h @@ -116,6 +116,13 @@ struct ComponentRuntimeStats { uint64_t total_time_us{0}; uint32_t total_max_time_us{0}; + // Cumulative sum of every record_time() duration since boot, across all + // components. Used by Application::loop() to snapshot time spent inside + // WarnIfComponentBlockingGuard (including guards constructed by the + // scheduler at scheduler.cpp) so main-loop overhead accounting can + // subtract scheduled-callback time from the before_loop_tasks_ wall time. + static uint64_t global_recorded_us; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + void record_time(uint32_t duration_us) { this->period_count++; this->period_time_us += duration_us; @@ -125,6 +132,7 @@ struct ComponentRuntimeStats { this->total_time_us += duration_us; if (duration_us > this->total_max_time_us) this->total_max_time_us = duration_us; + global_recorded_us += duration_us; } void reset_period() { this->period_count = 0; diff --git a/tests/integration/test_runtime_stats.py b/tests/integration/test_runtime_stats.py index 9e93035d835..bd7f36341d2 100644 --- a/tests/integration/test_runtime_stats.py +++ b/tests/integration/test_runtime_stats.py @@ -26,6 +26,7 @@ async def test_runtime_stats( # Track component stats component_stats_found = set() + main_loop_lines: list[dict[str, str]] = [] # Patterns to match - need to handle ANSI color codes and timestamps # The log format is: [HH:MM:SS][color codes][I][tag]: message @@ -34,6 +35,14 @@ async def test_runtime_stats( component_pattern = re.compile( r"^\[[^\]]+\].*?\s+([\w.]+):\s+count=(\d+),\s+avg=([\d.]+)ms" ) + # Main loop overhead line emitted by runtime_stats + main_loop_pattern = re.compile( + r"main_loop:\s+iters=(?P\d+),\s+" + r"active_avg=(?P[\d.]+)ms,\s+" + r"active_max=(?P[\d.]+)ms,\s+" + r"active_total=(?P[\d.]+)ms,\s+" + r"overhead_total=(?P[\d.]+)ms" + ) def check_output(line: str) -> None: """Check log output for runtime stats messages.""" @@ -54,6 +63,11 @@ async def test_runtime_stats( component_name = match.group(1) component_stats_found.add(component_name) + # Check for main_loop overhead line + ml_match = main_loop_pattern.search(line) + if ml_match: + main_loop_lines.append(ml_match.groupdict()) + async with ( run_compiled(yaml_config, line_callback=check_output), api_client_connected() as client, @@ -86,3 +100,22 @@ async def test_runtime_stats( assert "template.switch" in component_stats_found, ( f"Expected template.switch stats, found: {component_stats_found}" ) + + # Verify the main_loop overhead line is emitted (at least once for + # the period section and once for the total section, per log cycle). + assert len(main_loop_lines) >= 2, ( + f"Expected at least 2 main_loop lines, got {len(main_loop_lines)}" + ) + for fields in main_loop_lines: + assert int(fields["iters"]) > 0, f"iters should be > 0: {fields}" + assert float(fields["active_total"]) > 0.0, ( + f"active_total should be > 0: {fields}" + ) + assert float(fields["active_avg"]) >= 0.0, ( + f"active_avg should be >= 0: {fields}" + ) + # overhead_total is derived and may be 0 if components dominate, + # but the field must still be present and parseable as a float. + assert float(fields["overhead_total"]) >= 0.0, ( + f"overhead_total should be >= 0: {fields}" + ) From e5f6a734ba46884cafeed83e6e735cba0c901a2d Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Mon, 20 Apr 2026 10:08:07 +1000 Subject: [PATCH 33/35] [lvgl] Fix angles for arc (#15860) --- esphome/components/lvgl/widgets/arc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/esphome/components/lvgl/widgets/arc.py b/esphome/components/lvgl/widgets/arc.py index 9eaf3dadce4..ac993cc3829 100644 --- a/esphome/components/lvgl/widgets/arc.py +++ b/esphome/components/lvgl/widgets/arc.py @@ -77,8 +77,11 @@ class ArcType(NumberType): # start_angle and end_angle are mapped to bg_start_angle and bg_end_angle prop = str(prop) if prop.endswith("_angle"): - prop = "bg_" + prop - await w.set_property(prop, config, processor=validator) + await w.set_property( + "bg_" + prop, await validator.process(config.get(prop)) + ) + else: + await w.set_property(prop, config, processor=validator) if CONF_ADJUSTABLE in config: if not config[CONF_ADJUSTABLE]: lv_obj.remove_style(w.obj, nullptr, LV_PART.KNOB) From 0d3db2b6701fb7e87cd41f9f8c60445b06575618 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Mon, 20 Apr 2026 10:08:07 +1000 Subject: [PATCH 34/35] [lvgl] Fix angles for arc (#15860) --- esphome/components/lvgl/widgets/arc.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/esphome/components/lvgl/widgets/arc.py b/esphome/components/lvgl/widgets/arc.py index 9eaf3dadce4..ac993cc3829 100644 --- a/esphome/components/lvgl/widgets/arc.py +++ b/esphome/components/lvgl/widgets/arc.py @@ -77,8 +77,11 @@ class ArcType(NumberType): # start_angle and end_angle are mapped to bg_start_angle and bg_end_angle prop = str(prop) if prop.endswith("_angle"): - prop = "bg_" + prop - await w.set_property(prop, config, processor=validator) + await w.set_property( + "bg_" + prop, await validator.process(config.get(prop)) + ) + else: + await w.set_property(prop, config, processor=validator) if CONF_ADJUSTABLE in config: if not config[CONF_ADJUSTABLE]: lv_obj.remove_style(w.obj, nullptr, LV_PART.KNOB) From 78aabf257fdb3e6d490fc79715d34471fc99836a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 20 Apr 2026 11:52:56 +0200 Subject: [PATCH 35/35] [core] wake_loop_threadsafe() forces a component-phase iteration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the Phase A / Phase B split in this PR, an external producer that called wake_loop_threadsafe() (MQTT RX, USB RX, BLE event, espnow, camera, mWW, speakers, USB host/CDC, lwip socket, enable_loop_soon_any_context) only got Phase A — the component phase stayed gated by loop_interval_, so the producer's component loop() could be delayed by up to loop_interval_ ms before draining its queued work. That breaks the long-standing semantic of wake_loop_threadsafe(). Add a wake_request flag set by every wake_loop_* entry point and exchange-cleared at the gate in Application::loop(). When the flag is set, force Phase B regardless of loop_interval_. Storage is conditional on the threading model: - ESPHOME_THREAD_MULTI_ATOMICS: std::atomic (uint8_t, not bool, because GCC on Xtensa generates an indirect call for atomic ops — same workaround as scheduler.h) - ESPHOME_THREAD_SINGLE / ESPHOME_THREAD_MULTI_NO_ATOMICS: volatile uint8_t (8-bit aligned loads/stores are atomic on every supported MCU; the platform signal that follows wake_request_set provides the cross-thread/cross-core memory barrier) Helpers (wake_request_set / wake_request_take) are always_inline so IRAM_ATTR call sites stay in IRAM. Set BEFORE the platform signal so the consumer is guaranteed to see the flag on its next gate check. Adds an integration test that raises loop_interval_ to 2s, snapshots a counting component's loop count, spawns a std::thread that calls App.wake_loop_threadsafe() after 50ms, and asserts the count increments inside a 500ms observation window. Without the fix the count would not move for ~2s. --- esphome/core/application.h | 24 ++++-- esphome/core/wake.cpp | 16 ++++ esphome/core/wake.h | 51 ++++++++++++- .../wake_test_component/__init__.py | 19 +++++ .../wake_test_component.cpp | 19 +++++ .../wake_test_component/wake_test_component.h | 27 +++++++ .../fixtures/wake_loop_forces_phase_b.yaml | 52 +++++++++++++ .../test_wake_loop_forces_phase_b.py | 76 +++++++++++++++++++ 8 files changed, 277 insertions(+), 7 deletions(-) create mode 100644 tests/integration/fixtures/external_components/wake_test_component/__init__.py create mode 100644 tests/integration/fixtures/external_components/wake_test_component/wake_test_component.cpp create mode 100644 tests/integration/fixtures/external_components/wake_test_component/wake_test_component.h create mode 100644 tests/integration/fixtures/wake_loop_forces_phase_b.yaml create mode 100644 tests/integration/test_wake_loop_forces_phase_b.py diff --git a/esphome/core/application.h b/esphome/core/application.h index 3752731e998..c87d4eb3313 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -660,12 +660,20 @@ inline void ESPHOME_ALWAYS_INLINE Application::loop() { uint32_t loop_tail_start_us = loop_before_end_us; #endif - // Gate the component phase on loop_interval_ or an active high-frequency - // request. When a scheduler wake preempts sleep early, this gate keeps the - // component phase from running more often than loop_interval_. + // Gate the component phase on loop_interval_, an active high-frequency + // request, or an explicit wake from a background producer. A scheduler-only + // wake (e.g. set_interval firing under a raised loop_interval_) leaves the + // component phase gated; an external producer that called wake_loop_* + // (MQTT RX, USB RX, BLE event, etc.) needs the component phase to actually + // run so its component's loop() can drain the queued work — that is the + // long-standing semantic of wake_loop_threadsafe(), and the wake_request + // flag preserves it. wake_request_take() exchange-clears the flag; wakes + // that arrive during Phase B re-set it and run Phase B again on the next + // iteration. const bool high_frequency = HighFrequencyLoopRequester::is_high_frequency(); const uint32_t elapsed = now - this->last_loop_; - const bool do_component_phase = high_frequency || (elapsed >= this->loop_interval_); + const bool woke = esphome::wake_request_take(); + const bool do_component_phase = high_frequency || woke || (elapsed >= this->loop_interval_); if (do_component_phase) { this->before_component_phase_(); @@ -716,8 +724,12 @@ inline void ESPHOME_ALWAYS_INLINE Application::loop() { #endif // Compute sleep: bounded by time-until-next-component-phase and the - // scheduler's next deadline. When a scheduler event wakes us early we - // re-enter loop(), Phase A services it, and the component phase stays gated. + // scheduler's next deadline. When a scheduler timer fires it re-enters + // loop(), Phase A services it, and the component phase stays gated by + // loop_interval_. When a background producer calls wake_loop_threadsafe() + // it sets the wake_request flag and wakes select() / the task notification; + // the gate above sees the flag and runs Phase B too so the producer's + // component can drain its queued work without waiting up to loop_interval_. // // Re-read HighFrequencyLoopRequester::is_high_frequency() here instead of // reusing the cached `high_frequency` captured above: a component calling diff --git a/esphome/core/wake.cpp b/esphome/core/wake.cpp index 3709fa88ac7..cebc4d04b7f 100644 --- a/esphome/core/wake.cpp +++ b/esphome/core/wake.cpp @@ -12,9 +12,22 @@ namespace esphome { +// === Wake-requested flag storage === +#ifdef ESPHOME_THREAD_MULTI_ATOMICS +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +std::atomic g_wake_requested{0}; +#else +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +volatile uint8_t g_wake_requested = 0; +#endif + // === ESP32 / LibreTiny — IRAM_ATTR entry points === #if defined(USE_ESP32) || defined(USE_LIBRETINY) void IRAM_ATTR wake_loop_isrsafe(BaseType_t *px_higher_priority_task_woken) { + // ISR-safe: set flag before notify so the wake is visible on the next gate + // check. wake_request_set() is just an aligned 8-bit store / atomic store + // and is safe from IRAM. + wake_request_set(); esphome_main_task_notify_from_isr(px_higher_priority_task_woken); } void IRAM_ATTR wake_loop_any_context() { wake_main_task_any_context(); } @@ -72,6 +85,9 @@ void wakeable_delay(uint32_t ms) { // === Host (UDP loopback socket) === #ifdef USE_HOST void wake_loop_threadsafe() { + // Set flag before sending so the consumer's gate check on the next loop() + // entry observes the wake regardless of select() scheduling. + wake_request_set(); if (App.wake_socket_fd_ >= 0) { const char dummy = 1; ::send(App.wake_socket_fd_, &dummy, 1, 0); diff --git a/esphome/core/wake.h b/esphome/core/wake.h index 77a38d429e5..41b7ab33b51 100644 --- a/esphome/core/wake.h +++ b/esphome/core/wake.h @@ -7,6 +7,10 @@ #include "esphome/core/defines.h" #include "esphome/core/hal.h" +#ifdef ESPHOME_THREAD_MULTI_ATOMICS +#include +#endif + #if defined(USE_ESP32) || defined(USE_LIBRETINY) #include "esphome/core/main_task.h" #endif @@ -25,12 +29,48 @@ namespace esphome { extern volatile bool g_main_loop_woke; #endif +// === wake_request flag — signals Application::loop() that a producer queued +// work for some component's loop() to drain (MQTT RX, USB RX, BLE event, etc.) +// and the component phase should run this tick instead of being held off by +// the loop_interval_ gate. Set by every wake_loop_* entry point; consumed +// (via exchange-and-clear) at the gate in Application::loop(). === +// +// std::atomic rather than std::atomic because GCC on Xtensa +// generates an indirect function call for atomic ops instead of inlining +// them — same workaround applied in scheduler.h for the SchedulerItem::remove +// flag. On non-atomic platforms a volatile uint8_t suffices: 8-bit aligned +// loads/stores are atomic on every supported MCU, and the platform signal +// that follows wake_request_set() (FreeRTOS task-notify, esp_schedule, socket +// send) provides the cross-thread/cross-core memory barrier. +#ifdef ESPHOME_THREAD_MULTI_ATOMICS +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +extern std::atomic g_wake_requested; + +__attribute__((always_inline)) inline void wake_request_set() { g_wake_requested.store(1, std::memory_order_release); } +__attribute__((always_inline)) inline bool wake_request_take() { + return g_wake_requested.exchange(0, std::memory_order_acquire) != 0; +} +#else +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +extern volatile uint8_t g_wake_requested; + +__attribute__((always_inline)) inline void wake_request_set() { g_wake_requested = 1; } +__attribute__((always_inline)) inline bool wake_request_take() { + uint8_t v = g_wake_requested; + g_wake_requested = 0; + return v != 0; +} +#endif + // === ESP32 / LibreTiny (FreeRTOS) === #if defined(USE_ESP32) || defined(USE_LIBRETINY) /// Wake the main loop from any context (ISR or task). /// always_inline so callers placed in IRAM keep the whole wake path in IRAM. __attribute__((always_inline)) inline void wake_main_task_any_context() { + // Set the wake-requested flag BEFORE the task notification so the consumer + // (Application::loop() gate) is guaranteed to see it on its next gate check. + wake_request_set(); if (in_isr_context()) { BaseType_t px_higher_priority_task_woken = pdFALSE; esphome_main_task_notify_from_isr(&px_higher_priority_task_woken); @@ -50,7 +90,10 @@ __attribute__((always_inline)) inline void wake_main_task_any_context() { void wake_loop_isrsafe(BaseType_t *px_higher_priority_task_woken); void wake_loop_any_context(); -inline void wake_loop_threadsafe() { esphome_main_task_notify(); } +inline void wake_loop_threadsafe() { + wake_request_set(); + esphome_main_task_notify(); +} namespace internal { inline void wakeable_delay(uint32_t ms) { @@ -67,6 +110,9 @@ inline void wakeable_delay(uint32_t ms) { /// Inline implementation — IRAM callers inline this directly. inline void ESPHOME_ALWAYS_INLINE wake_loop_impl() { + // Set the wake-requested flag BEFORE esp_schedule so the consumer is + // guaranteed to see it on its next gate check. + wake_request_set(); g_main_loop_woke = true; esp_schedule(); } @@ -98,6 +144,9 @@ inline void wakeable_delay(uint32_t ms) { #elif defined(USE_RP2040) inline void wake_loop_any_context() { + // Set the wake-requested flag BEFORE the SEV so the consumer is guaranteed + // to see it on its next gate check. + wake_request_set(); g_main_loop_woke = true; __sev(); } diff --git a/tests/integration/fixtures/external_components/wake_test_component/__init__.py b/tests/integration/fixtures/external_components/wake_test_component/__init__.py new file mode 100644 index 00000000000..ce241678897 --- /dev/null +++ b/tests/integration/fixtures/external_components/wake_test_component/__init__.py @@ -0,0 +1,19 @@ +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome.const import CONF_ID + +CODEOWNERS = ["@esphome/tests"] + +wake_test_component_ns = cg.esphome_ns.namespace("wake_test_component") +WakeTestComponent = wake_test_component_ns.class_("WakeTestComponent", cg.Component) + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(): cv.declare_id(WakeTestComponent), + } +).extend(cv.COMPONENT_SCHEMA) + + +async def to_code(config): + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) diff --git a/tests/integration/fixtures/external_components/wake_test_component/wake_test_component.cpp b/tests/integration/fixtures/external_components/wake_test_component/wake_test_component.cpp new file mode 100644 index 00000000000..b58f1c9adc8 --- /dev/null +++ b/tests/integration/fixtures/external_components/wake_test_component/wake_test_component.cpp @@ -0,0 +1,19 @@ +#include "wake_test_component.h" +#include "esphome/core/application.h" +#include "esphome/core/log.h" +#include +#include + +namespace esphome::wake_test_component { + +static const char *const TAG = "wake_test_component"; + +void WakeTestComponent::start_async_wake() { + ESP_LOGI(TAG, "Spawning async wake thread (50ms delay)"); + std::thread([] { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + App.wake_loop_threadsafe(); + }).detach(); +} + +} // namespace esphome::wake_test_component diff --git a/tests/integration/fixtures/external_components/wake_test_component/wake_test_component.h b/tests/integration/fixtures/external_components/wake_test_component/wake_test_component.h new file mode 100644 index 00000000000..c8e4e0a89fe --- /dev/null +++ b/tests/integration/fixtures/external_components/wake_test_component/wake_test_component.h @@ -0,0 +1,27 @@ +#pragma once + +#include "esphome/core/component.h" +#include + +namespace esphome::wake_test_component { + +class WakeTestComponent : public Component { + public: + void setup() override {} + void loop() override { this->loop_count_.fetch_add(1, std::memory_order_relaxed); } + + int get_loop_count() const { return this->loop_count_.load(std::memory_order_relaxed); } + + // Spawn a detached thread that sleeps briefly then calls + // App.wake_loop_threadsafe(). Used by the integration test to verify a + // cross-thread wake forces a component-phase iteration even when + // loop_interval_ has been raised high enough to gate it off otherwise. + void start_async_wake(); + + float get_setup_priority() const override { return setup_priority::DATA; } + + protected: + std::atomic loop_count_{0}; +}; + +} // namespace esphome::wake_test_component diff --git a/tests/integration/fixtures/wake_loop_forces_phase_b.yaml b/tests/integration/fixtures/wake_loop_forces_phase_b.yaml new file mode 100644 index 00000000000..d97ab8514f6 --- /dev/null +++ b/tests/integration/fixtures/wake_loop_forces_phase_b.yaml @@ -0,0 +1,52 @@ +esphome: + name: wake-loop-phase-b + on_boot: + priority: -100 + then: + - lambda: |- + // Raise loop_interval_ to 2000ms. Without the wake-request flag, + // a wake_loop_threadsafe() call would only run Phase A (scheduler) + // and leave the component phase gated for ~2s. + App.set_loop_interval(2000); + # Let boot transients settle. + - delay: 1000ms + - lambda: |- + // Snapshot the loop counter, then ask the component to spawn a + // background thread that calls App.wake_loop_threadsafe() after + // ~50ms. With the fix, that wake forces Phase B on the next tick + // and the counter increments well within the 500ms observation + // window below. + id(count_at_start) = id(wake_counter)->get_loop_count(); + id(start_time) = millis(); + id(wake_counter)->start_async_wake(); + ESP_LOGI("test", "WAKE_STARTED count=%d", id(count_at_start)); + # Observation window must be much shorter than loop_interval_ (2000ms) + # so a "false pass" isn't possible by simply waiting out the gate. + - delay: 500ms + - lambda: |- + int count_now = id(wake_counter)->get_loop_count(); + int delta = count_now - id(count_at_start); + uint32_t elapsed = millis() - id(start_time); + ESP_LOGI("test", "WAKE_RESULT delta=%d elapsed=%u", delta, elapsed); + +host: +api: +logger: + level: INFO + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + components: [wake_test_component] + +globals: + - id: count_at_start + type: int + initial_value: "0" + - id: start_time + type: uint32_t + initial_value: "0" + +wake_test_component: + id: wake_counter diff --git a/tests/integration/test_wake_loop_forces_phase_b.py b/tests/integration/test_wake_loop_forces_phase_b.py new file mode 100644 index 00000000000..5f05f07dd82 --- /dev/null +++ b/tests/integration/test_wake_loop_forces_phase_b.py @@ -0,0 +1,76 @@ +"""Test that wake_loop_threadsafe() forces a component-phase iteration. + +Regression test for the wake-request flag added to Application::loop()'s +Phase A / Phase B gate. Background producers (MQTT RX, USB RX, BLE event, +etc.) call App.wake_loop_threadsafe() expecting their component's loop() +to drain queued work; if the component phase stays gated by loop_interval_, +the work waits up to loop_interval_ ms instead of running on the next tick. + +Setup: +- App.set_loop_interval(2000) — a wide gate that would clearly mask the bug. +- A test component spawns a detached std::thread that sleeps 50 ms and then + calls App.wake_loop_threadsafe() from a non-main thread. +- The on_boot block snapshots the component's loop counter before/after a + 500 ms observation window. + +Without the fix, delta=0 (the gate holds Phase B for ~2 s). +With the fix, delta>=1 (the wake forces Phase B within one tick of the wake). +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +import re + +import pytest + +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_wake_loop_forces_phase_b( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """A wake_loop_threadsafe() call from a background thread must trigger the + component phase within the next tick, even when loop_interval_ is raised + well above the observation window.""" + external_components_path = str( + Path(__file__).parent / "fixtures" / "external_components" + ) + yaml_config = yaml_config.replace( + "EXTERNAL_COMPONENT_PATH", external_components_path + ) + + loop = asyncio.get_running_loop() + result: asyncio.Future[tuple[int, int]] = loop.create_future() + + def on_log_line(line: str) -> None: + match = re.search(r"WAKE_RESULT delta=(\d+) elapsed=(\d+)", line) + if match and not result.done(): + result.set_result((int(match.group(1)), int(match.group(2)))) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + device_info = await client.device_info() + assert device_info is not None + assert device_info.name == "wake-loop-phase-b" + + try: + delta, elapsed = await asyncio.wait_for(result, timeout=15.0) + except TimeoutError: + pytest.fail("WAKE_RESULT marker never appeared") + + # Without the fix, delta would be 0 — loop_interval_=2000ms held + # Phase B off for the full 500ms observation window. With the fix + # the wake from the background thread (~50ms after start) forces + # Phase B on the next tick, so the counter increments at least once. + assert delta >= 1, ( + f"wake_loop_threadsafe() from a background thread should force " + f"Phase B within the next tick; observed delta={delta} after " + f"{elapsed}ms with loop_interval_=2000ms" + )