Merge remote-tracking branch 'origin/decouple_scheduler_loop_cadence' into integration

tests/component_tests/packages/test_packages.py

@@ -2,18 +2,20 @@
 
 import logging
 from pathlib import Path
+import re
 from unittest.mock import MagicMock, patch
 
 import pytest
 
 from esphome.components.packages import (
     CONFIG_SCHEMA,
+    _substitute_package_definition,
     _walk_packages,
     do_packages_pass,
     is_package_definition,
     merge_packages,
 )
-from esphome.components.substitutions import do_substitution_pass
+from esphome.components.substitutions import ContextVars, do_substitution_pass
 import esphome.config as config_module
 from esphome.config import resolve_extend_remove
 from esphome.config_helpers import Extend, Remove
@@ -44,7 +46,7 @@ from esphome.const import (
 )
 from esphome.core import CORE
 from esphome.util import OrderedDict
-from esphome.yaml_util import IncludeFile, add_context
+from esphome.yaml_util import IncludeFile, add_context, load_yaml
 
 # Test strings
 TEST_DEVICE_NAME = "test_device_name"
@@ -1399,3 +1401,85 @@ def test_raw_config_contains_merged_esphome_from_package(tmp_path) -> None:
         "CORE.raw_config should contain esphome section after package merge"
     )
     assert CORE.raw_config[CONF_ESPHOME][CONF_NAME] == TEST_DEVICE_NAME
+
+
+# ---------------------------------------------------------------------------
+# _substitute_package_definition
+# ---------------------------------------------------------------------------
+
+
+def test_substitute_package_definition_local_dict_returned_unchanged() -> None:
+    """A plain local config dict is not substituted and is returned as-is."""
+    pkg = {CONF_WIFI: {CONF_SSID: "test"}}
+    result = _substitute_package_definition(pkg, ContextVars())
+    assert result is pkg
+
+
+def test_substitute_package_definition_string_resolved_with_context() -> None:
+    """A string package definition has its variables substituted."""
+    ctx = ContextVars({"variant": "esp32"})
+    result = _substitute_package_definition("device-${variant}.yaml", ctx)
+    assert result == "device-esp32.yaml"
+
+
+def test_substitute_package_definition_undefined_in_string() -> None:
+    """An undefined variable in a package URL string raises cv.Invalid."""
+    with pytest.raises(cv.Invalid, match="Undefined variable in package definition"):
+        _substitute_package_definition(
+            "github://org/repo/${undefined_var}/pkg.yaml", ContextVars()
+        )
+
+
+def test_substitute_package_definition_undefined_in_remote_dict_field() -> None:
+    """An undefined variable inside a remote-dict field names the offending field."""
+    with pytest.raises(cv.Invalid) as exc_info:
+        _substitute_package_definition(
+            {CONF_URL: "github://${typo}/repo"}, ContextVars()
+        )
+    err = str(exc_info.value)
+    assert "'typo' is undefined" in err
+    assert CONF_URL in err
+
+
+def test_substitute_package_definition_undefined_in_remote_dict_non_first_field() -> (
+    None
+):
+    """The field path joins correctly for non-first dict fields (e.g. ``ref``)."""
+    with pytest.raises(cv.Invalid) as exc_info:
+        _substitute_package_definition(
+            {
+                CONF_URL: "github://org/repo",
+                CONF_REF: "branch-${branch_typo}",
+            },
+            ContextVars(),
+        )
+    err = str(exc_info.value)
+    assert "'branch_typo' is undefined" in err
+    assert CONF_REF in err
+
+
+def test_substitute_package_definition_includes_source_location(tmp_path: Path) -> None:
+    """A package loaded from YAML surfaces file/line/col in the cv.Invalid message.
+
+    Line/column are rendered 1-based (matching config.line_info() and editor
+    line numbering) and point at the offending scalar, not the enclosing dict.
+    """
+    yaml_file = tmp_path / "main.yaml"
+    yaml_file.write_text(
+        "packages:\n  broken: github://org/repo/${undefined_var}/pkg.yaml\n"
+    )
+    config = load_yaml(yaml_file)
+    package_config = config[CONF_PACKAGES]["broken"]
+
+    with pytest.raises(cv.Invalid) as exc_info:
+        _substitute_package_definition(package_config, ContextVars())
+
+    err = str(exc_info.value)
+    assert "main.yaml" in err
+    # The offending value lives on line 2 (1-based). Column depends on the YAML
+    # loader, so we only pin line and check that a 1-based column is present.
+    match = re.search(r"main\.yaml (\d+):(\d+)", err)
+    assert match, err
+    line, col = int(match.group(1)), int(match.group(2))
+    assert line == 2, f"expected 1-based line 2, got {line} (err={err!r})"
+    assert col >= 1, f"expected 1-based column ≥ 1, got {col} (err={err!r})"

tests/integration/fixtures/external_components/wake_test_component/__init__.py

@@ -0,0 +1,19 @@
+import esphome.codegen as cg
+import esphome.config_validation as cv
+from esphome.const import CONF_ID
+
+CODEOWNERS = ["@esphome/tests"]
+
+wake_test_component_ns = cg.esphome_ns.namespace("wake_test_component")
+WakeTestComponent = wake_test_component_ns.class_("WakeTestComponent", cg.Component)
+
+CONFIG_SCHEMA = cv.Schema(
+    {
+        cv.GenerateID(): cv.declare_id(WakeTestComponent),
+    }
+).extend(cv.COMPONENT_SCHEMA)
+
+
+async def to_code(config):
+    var = cg.new_Pvariable(config[CONF_ID])
+    await cg.register_component(var, config)

tests/integration/fixtures/external_components/wake_test_component/wake_test_component.cpp

@@ -0,0 +1,19 @@
+#include "wake_test_component.h"
+#include "esphome/core/application.h"
+#include "esphome/core/log.h"
+#include <chrono>
+#include <thread>
+
+namespace esphome::wake_test_component {
+
+static const char *const TAG = "wake_test_component";
+
+void WakeTestComponent::start_async_wake() {
+  ESP_LOGI(TAG, "Spawning async wake thread (50ms delay)");
+  std::thread([] {
+    std::this_thread::sleep_for(std::chrono::milliseconds(50));
+    App.wake_loop_threadsafe();
+  }).detach();
+}
+
+}  // namespace esphome::wake_test_component

tests/integration/fixtures/external_components/wake_test_component/wake_test_component.h

@@ -0,0 +1,27 @@
+#pragma once
+
+#include "esphome/core/component.h"
+#include <atomic>
+
+namespace esphome::wake_test_component {
+
+class WakeTestComponent : public Component {
+ public:
+  void setup() override {}
+  void loop() override { this->loop_count_.fetch_add(1, std::memory_order_relaxed); }
+
+  int get_loop_count() const { return this->loop_count_.load(std::memory_order_relaxed); }
+
+  // Spawn a detached thread that sleeps briefly then calls
+  // App.wake_loop_threadsafe(). Used by the integration test to verify a
+  // cross-thread wake forces a component-phase iteration even when
+  // loop_interval_ has been raised high enough to gate it off otherwise.
+  void start_async_wake();
+
+  float get_setup_priority() const override { return setup_priority::DATA; }
+
+ protected:
+  std::atomic<int> loop_count_{0};
+};
+
+}  // namespace esphome::wake_test_component

tests/integration/fixtures/wake_loop_forces_phase_b.yaml

@@ -0,0 +1,52 @@
+esphome:
+  name: wake-loop-phase-b
+  on_boot:
+    priority: -100
+    then:
+      - lambda: |-
+          // Raise loop_interval_ to 2000ms. Without the wake-request flag,
+          // a wake_loop_threadsafe() call would only run Phase A (scheduler)
+          // and leave the component phase gated for ~2s.
+          App.set_loop_interval(2000);
+      # Let boot transients settle.
+      - delay: 1000ms
+      - lambda: |-
+          // Snapshot the loop counter, then ask the component to spawn a
+          // background thread that calls App.wake_loop_threadsafe() after
+          // ~50ms. With the fix, that wake forces Phase B on the next tick
+          // and the counter increments well within the 500ms observation
+          // window below.
+          id(count_at_start) = id(wake_counter)->get_loop_count();
+          id(start_time) = millis();
+          id(wake_counter)->start_async_wake();
+          ESP_LOGI("test", "WAKE_STARTED count=%d", id(count_at_start));
+      # Observation window must be much shorter than loop_interval_ (2000ms)
+      # so a "false pass" isn't possible by simply waiting out the gate.
+      - delay: 500ms
+      - lambda: |-
+          int count_now = id(wake_counter)->get_loop_count();
+          int delta = count_now - id(count_at_start);
+          uint32_t elapsed = millis() - id(start_time);
+          ESP_LOGI("test", "WAKE_RESULT delta=%d elapsed=%u", delta, elapsed);
+
+host:
+api:
+logger:
+  level: INFO
+
+external_components:
+  - source:
+      type: local
+      path: EXTERNAL_COMPONENT_PATH
+    components: [wake_test_component]
+
+globals:
+  - id: count_at_start
+    type: int
+    initial_value: "0"
+  - id: start_time
+    type: uint32_t
+    initial_value: "0"
+
+wake_test_component:
+  id: wake_counter

tests/integration/test_wake_loop_forces_phase_b.py

@@ -0,0 +1,76 @@
+"""Test that wake_loop_threadsafe() forces a component-phase iteration.
+
+Regression test for the wake-request flag added to Application::loop()'s
+Phase A / Phase B gate. Background producers (MQTT RX, USB RX, BLE event,
+etc.) call App.wake_loop_threadsafe() expecting their component's loop()
+to drain queued work; if the component phase stays gated by loop_interval_,
+the work waits up to loop_interval_ ms instead of running on the next tick.
+
+Setup:
+- App.set_loop_interval(2000) — a wide gate that would clearly mask the bug.
+- A test component spawns a detached std::thread that sleeps 50 ms and then
+  calls App.wake_loop_threadsafe() from a non-main thread.
+- The on_boot block snapshots the component's loop counter before/after a
+  500 ms observation window.
+
+Without the fix, delta=0 (the gate holds Phase B for ~2 s).
+With the fix, delta>=1 (the wake forces Phase B within one tick of the wake).
+"""
+
+from __future__ import annotations
+
+import asyncio
+from pathlib import Path
+import re
+
+import pytest
+
+from .types import APIClientConnectedFactory, RunCompiledFunction
+
+
+@pytest.mark.asyncio
+async def test_wake_loop_forces_phase_b(
+    yaml_config: str,
+    run_compiled: RunCompiledFunction,
+    api_client_connected: APIClientConnectedFactory,
+) -> None:
+    """A wake_loop_threadsafe() call from a background thread must trigger the
+    component phase within the next tick, even when loop_interval_ is raised
+    well above the observation window."""
+    external_components_path = str(
+        Path(__file__).parent / "fixtures" / "external_components"
+    )
+    yaml_config = yaml_config.replace(
+        "EXTERNAL_COMPONENT_PATH", external_components_path
+    )
+
+    loop = asyncio.get_running_loop()
+    result: asyncio.Future[tuple[int, int]] = loop.create_future()
+
+    def on_log_line(line: str) -> None:
+        match = re.search(r"WAKE_RESULT delta=(\d+) elapsed=(\d+)", line)
+        if match and not result.done():
+            result.set_result((int(match.group(1)), int(match.group(2))))
+
+    async with (
+        run_compiled(yaml_config, line_callback=on_log_line),
+        api_client_connected() as client,
+    ):
+        device_info = await client.device_info()
+        assert device_info is not None
+        assert device_info.name == "wake-loop-phase-b"
+
+        try:
+            delta, elapsed = await asyncio.wait_for(result, timeout=15.0)
+        except TimeoutError:
+            pytest.fail("WAKE_RESULT marker never appeared")
+
+        # Without the fix, delta would be 0 — loop_interval_=2000ms held
+        # Phase B off for the full 500ms observation window. With the fix
+        # the wake from the background thread (~50ms after start) forces
+        # Phase B on the next tick, so the counter increments at least once.
+        assert delta >= 1, (
+            f"wake_loop_threadsafe() from a background thread should force "
+            f"Phase B within the next tick; observed delta={delta} after "
+            f"{elapsed}ms with loop_interval_=2000ms"
+        )

tests/unit_tests/test_substitutions.py

@@ -14,6 +14,7 @@ from esphome.components.packages import (
     do_packages_pass,
     merge_packages,
 )
+from esphome.components.substitutions.jinja import UndefinedError
 from esphome.config import resolve_extend_remove
 from esphome.config_helpers import Extend, merge_config
 import esphome.config_validation as cv
@@ -675,6 +676,39 @@ def test_include_filename_substitution_undefined_var(tmp_path: Path) -> None:
         substitutions.do_substitution_pass(config)
 
 
+def test_raise_first_undefined_logs_extras_at_debug(
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """Only the first undefined error is raised; extras are logged at debug."""
+    errors: substitutions.ErrList = [
+        (UndefinedError("'a' is undefined"), ["url"], None),
+        (UndefinedError("'b' is undefined"), ["ref"], None),
+        (UndefinedError("'c' is undefined"), ["path"], None),
+    ]
+
+    with (
+        caplog.at_level(logging.DEBUG, logger="esphome.components.substitutions"),
+        pytest.raises(cv.Invalid) as exc_info,
+    ):
+        substitutions.raise_first_undefined(errors, None, "package definition")
+
+    # First error is surfaced as the cv.Invalid message.
+    raised = str(exc_info.value)
+    assert "'a' is undefined" in raised
+    assert "'b' is undefined" not in raised
+    assert "'c' is undefined" not in raised
+
+    # Remaining errors are captured via debug logging for troubleshooting.
+    assert "Additional undefined variables in package definition" in caplog.text
+    assert "'b' is undefined at 'ref'" in caplog.text
+    assert "'c' is undefined at 'path'" in caplog.text
+
+
+def test_raise_first_undefined_noop_on_empty() -> None:
+    """An empty errors list is a no-op — no exception, no log."""
+    substitutions.raise_first_undefined([], None, "package definition")
+
+
 def test_do_substitution_pass_included_substitutions_must_be_mapping(
     tmp_path: Path,
 ) -> None: