diff --git a/.clang-tidy.hash b/.clang-tidy.hash index 5c7eab517b..cd61d9ec48 100644 --- a/.clang-tidy.hash +++ b/.clang-tidy.hash @@ -1 +1 @@ -f31f13994768b5b07e29624406c9b053bf4bb26e1623ac2bc1e9d4a9477502d6 +d48687d988ae2a94a9973226df773478a7db1d52133545f07aa05e34fc678dcf diff --git a/.github/workflows/auto-label-pr.yml b/.github/workflows/auto-label-pr.yml index 27ddfe5911..f5214a5ce0 100644 --- a/.github/workflows/auto-label-pr.yml +++ b/.github/workflows/auto-label-pr.yml @@ -20,7 +20,7 @@ env: jobs: label: runs-on: ubuntu-latest - if: github.event.action != 'labeled' || github.event.sender.type != 'Bot' + if: github.event.pull_request.state == 'open' && (github.event.action != 'labeled' || github.event.sender.type != 'Bot') steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/esphome/components/canbus/__init__.py b/esphome/components/canbus/__init__.py index 7d3bf78f49..fcd342ad38 100644 --- a/esphome/components/canbus/__init__.py +++ b/esphome/components/canbus/__init__.py @@ -162,7 +162,6 @@ async def canbus_action_to_code(config, action_id, template_arg, args): await cg.register_parented(var, config[CONF_CANBUS_ID]) if (can_id := config.get(CONF_CAN_ID)) is not None: - can_id = await cg.templatable(can_id, args, cg.uint32) cg.add(var.set_can_id(can_id)) cg.add(var.set_use_extended_id(config[CONF_USE_EXTENDED_ID])) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index f27690c97b..cd38c82dd8 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -671,11 +671,12 @@ def _is_framework_url(source: str) -> bool: # The default/recommended arduino framework version # - https://github.com/espressif/arduino-esp32/releases ARDUINO_FRAMEWORK_VERSION_LOOKUP = { - "recommended": cv.Version(3, 3, 7), - "latest": cv.Version(3, 3, 7), - "dev": cv.Version(3, 3, 7), + "recommended": cv.Version(3, 3, 8), + "latest": cv.Version(3, 3, 8), + "dev": cv.Version(3, 3, 8), } ARDUINO_PLATFORM_VERSION_LOOKUP = { + cv.Version(3, 3, 8): cv.Version(55, 3, 38), cv.Version(3, 3, 7): cv.Version(55, 3, 37), cv.Version(3, 3, 6): cv.Version(55, 3, 36), cv.Version(3, 3, 5): cv.Version(55, 3, 35), @@ -695,6 +696,7 @@ ARDUINO_PLATFORM_VERSION_LOOKUP = { # These versions correspond to pioarduino/esp-idf releases # See: https://github.com/pioarduino/esp-idf/releases ARDUINO_IDF_VERSION_LOOKUP = { + cv.Version(3, 3, 8): cv.Version(5, 5, 4), cv.Version(3, 3, 7): cv.Version(5, 5, 3, "1"), cv.Version(3, 3, 6): cv.Version(5, 5, 2), cv.Version(3, 3, 5): cv.Version(5, 5, 2), @@ -714,17 +716,15 @@ ARDUINO_IDF_VERSION_LOOKUP = { # The default/recommended esp-idf framework version # - https://github.com/espressif/esp-idf/releases ESP_IDF_FRAMEWORK_VERSION_LOOKUP = { - "recommended": cv.Version(5, 5, 3, "1"), - "latest": cv.Version(5, 5, 3, "1"), + "recommended": cv.Version(5, 5, 4), + "latest": cv.Version(5, 5, 4), "dev": cv.Version(5, 5, 4), } ESP_IDF_PLATFORM_VERSION_LOOKUP = { cv.Version( 6, 0, 0 ): "https://github.com/pioarduino/platform-espressif32.git#prep_IDF6", - cv.Version( - 5, 5, 4 - ): "https://github.com/pioarduino/platform-espressif32.git#develop", + cv.Version(5, 5, 4): cv.Version(55, 3, 38), cv.Version(5, 5, 3, "1"): cv.Version(55, 3, 37), cv.Version(5, 5, 3): cv.Version(55, 3, 37), cv.Version(5, 5, 2): cv.Version(55, 3, 37), @@ -744,8 +744,8 @@ ESP_IDF_PLATFORM_VERSION_LOOKUP = { # The platform-espressif32 version # - https://github.com/pioarduino/platform-espressif32/releases PLATFORM_VERSION_LOOKUP = { - "recommended": cv.Version(55, 3, 37), - "latest": cv.Version(55, 3, 37), + "recommended": cv.Version(55, 3, 38), + "latest": cv.Version(55, 3, 38), "dev": "https://github.com/pioarduino/platform-espressif32.git#develop", } diff --git a/esphome/components/esp32/boards.py b/esphome/components/esp32/boards.py index 2bd08e7c39..2c73fe7d08 100644 --- a/esphome/components/esp32/boards.py +++ b/esphome/components/esp32/boards.py @@ -1960,6 +1960,10 @@ BOARDS = { "name": "Hornbill ESP32 Minima", "variant": VARIANT_ESP32, }, + "huidu_hd_wf1": { + "name": "Huidu HD-WF1", + "variant": VARIANT_ESP32S2, + }, "huidu_hd_wf2": { "name": "Huidu HD-WF2", "variant": VARIANT_ESP32S3, @@ -2028,6 +2032,10 @@ BOARDS = { "name": "LilyGo T-Display-S3", "variant": VARIANT_ESP32S3, }, + "lilygo-t-energy-s3": { + "name": "LilyGo T-Energy-S3", + "variant": VARIANT_ESP32S3, + }, "lilygo-t3-s3": { "name": "LilyGo T3-S3", "variant": VARIANT_ESP32S3, @@ -2289,10 +2297,18 @@ BOARDS = { "name": "S.ODI Ultra v1", "variant": VARIANT_ESP32, }, + "seeed_xiao_esp32_s3_plus": { + "name": "Seeed Studio XIAO ESP32S3 Plus", + "variant": VARIANT_ESP32S3, + }, "seeed_xiao_esp32c3": { "name": "Seeed Studio XIAO ESP32C3", "variant": VARIANT_ESP32C3, }, + "seeed_xiao_esp32c5": { + "name": "Seeed Studio XIAO ESP32C5", + "variant": VARIANT_ESP32C5, + }, "seeed_xiao_esp32c6": { "name": "Seeed Studio XIAO ESP32C6", "variant": VARIANT_ESP32C6, diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index f6f6204f4c..b6421dc43d 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -341,7 +341,7 @@ async def to_code(configs): df.LOGGER.info("LVGL will use hardware rotation via display driver") else: rotation_type = RotationType.ROTATION_SOFTWARE - if get_esp32_variant() == VARIANT_ESP32P4: + if CORE.is_esp32 and get_esp32_variant() == VARIANT_ESP32P4: df.LOGGER.info("LVGL will use software rotation (PPA accelerated)") else: df.LOGGER.info("LVGL will use software rotation") diff --git a/esphome/components/mdns/__init__.py b/esphome/components/mdns/__init__.py index 79d355e8ae..7c36295e8d 100644 --- a/esphome/components/mdns/__init__.py +++ b/esphome/components/mdns/__init__.py @@ -170,7 +170,7 @@ async def to_code(config): cg.add_library("LEAmDNS", None) if CORE.is_esp32: - add_idf_component(name="espressif/mdns", ref="1.10.0") + add_idf_component(name="espressif/mdns", ref="1.11.0") cg.add_define("USE_MDNS") diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 04db690c6f..3f3df75351 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -45,6 +45,18 @@ def is_remote_package(package_config: dict) -> bool: return CONF_URL in package_config +def is_package_definition(value: object) -> bool: + """Returns True if the value looks like a package definition rather than a config fragment. + + Package definitions are IncludeFile objects, git URL shorthand strings, or + remote package dicts (containing a ``url:`` key). Config fragments are + plain dicts that represent component configuration. + """ + return isinstance(value, (yaml_util.IncludeFile, str)) or ( + isinstance(value, dict) and is_remote_package(value) + ) + + def valid_package_contents(package_config: dict) -> dict: """Validate that a package looks like a plausible ESPHome config fragment. @@ -309,20 +321,23 @@ def _walk_packages( return config packages = config[CONF_PACKAGES] - if not isinstance(packages, (dict, list)): - raise cv.Invalid( - f"Packages must be a key to value mapping or list, got {type(packages)} instead" - ) - with cv.prepend_path(CONF_PACKAGES): + if isinstance(packages, yaml_util.IncludeFile): + # If the packages key is an IncludeFile, resolve it first before processing. + packages, _ = resolve_include(packages, [], context, strict_undefined=False) + if not isinstance(packages, (dict, list)): + raise cv.Invalid( + f"Packages must be a key to value mapping or list, got {type(packages)} instead" + ) + if not isinstance(packages, dict): _walk_package_list(packages, callback, context) elif (result := _walk_package_dict(packages, callback, context)) is not None: - if not validate_deprecated: + if not validate_deprecated or any( + is_package_definition(v) for v in packages.values() + ): raise result # Fallback: treat the dict as a single deprecated package. - # Note: this catches *any* cv.Invalid from the callback, which may - # mask real validation errors in named package dicts. # This block can be removed once the single-package # deprecation period (2026.7.0) is over. config[CONF_PACKAGES] = [packages] @@ -461,6 +476,9 @@ class _PackageProcessor: self, package_config: dict | str, context_vars: ContextVars | None ) -> dict: """Resolve a single package and recurse into any nested packages.""" + from_remote = isinstance(package_config, dict) and is_remote_package( + package_config + ) package_config = self.resolve_package(package_config, context_vars) self.collect_substitutions(package_config) @@ -470,7 +488,18 @@ class _PackageProcessor: # Push context from !include vars on the package root and on the packages key context_vars = push_context(package_config, context_vars) context_vars = push_context(package_config[CONF_PACKAGES], context_vars) - return _walk_packages(package_config, self.process_package, context_vars) + # Disable the deprecated single-package fallback for remote + # packages. _process_remote_package returns dicts with + # already-resolved values that is_package_definition cannot + # distinguish from config fragments, so the fallback would + # always fire and mask real errors with wrong paths + # (packages->0 instead of packages->). + return _walk_packages( + package_config, + self.process_package, + context_vars, + validate_deprecated=not from_remote, + ) def do_packages_pass( diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 1e40fef2dc..bf42730e67 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -14,7 +14,7 @@ dependencies: espressif/esp32-camera: version: 2.1.6 espressif/mdns: - version: 1.10.0 + version: 1.11.0 espressif/esp_wifi_remote: version: 1.4.0 rules: diff --git a/esphome/platformio_api.py b/esphome/platformio_api.py index cb080b2a95..e9719f7dcd 100644 --- a/esphome/platformio_api.py +++ b/esphome/platformio_api.py @@ -5,104 +5,15 @@ import os from pathlib import Path import re import subprocess -import time -from typing import Any +import sys from esphome.const import CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, KEY_CORE from esphome.core import CORE, EsphomeError -from esphome.util import run_external_command, run_external_process +from esphome.util import run_external_process _LOGGER = logging.getLogger(__name__) -def patch_structhash(): - # Patch platformio's structhash to not recompile the entire project when files are - # removed/added. This might have unintended consequences, but this improves compile - # times greatly when adding/removing components and a simple clean build solves - # all issues - from platformio.run import cli, helpers - - def patched_clean_build_dir(build_dir, *args): - from platformio import fs - from platformio.project.helpers import get_project_dir - - platformio_ini = Path(get_project_dir()) / "platformio.ini" - - build_dir = Path(build_dir) - - # if project's config is modified - if ( - build_dir.is_dir() - and platformio_ini.stat().st_mtime > build_dir.stat().st_mtime - ): - fs.rmtree(build_dir) - - if not build_dir.is_dir(): - build_dir.mkdir(parents=True) - - helpers.clean_build_dir = patched_clean_build_dir - cli.clean_build_dir = patched_clean_build_dir - - -def patch_file_downloader(): - """Patch PlatformIO's FileDownloader to retry on PackageException errors. - - PlatformIO's FileDownloader uses HTTPSession which lacks built-in retry - for 502/503 errors. We add retries with exponential backoff and close the - session between attempts to force a fresh TCP connection, which may route - to a different CDN edge node. - """ - from platformio.package.download import FileDownloader - from platformio.package.exception import PackageException - - if getattr(FileDownloader.__init__, "_esphome_patched", False): - return - - original_init = FileDownloader.__init__ - - def patched_init(self, *args: Any, **kwargs: Any) -> None: - max_retries = 5 - - for attempt in range(max_retries): - try: - original_init(self, *args, **kwargs) - return - except PackageException as e: - if attempt < max_retries - 1: - # Exponential backoff: 2, 4, 8, 16 seconds - delay = 2 ** (attempt + 1) - _LOGGER.warning( - "Package download failed: %s. " - "Retrying in %d seconds... (attempt %d/%d)", - str(e), - delay, - attempt + 1, - max_retries, - ) - # Close the response and session to free resources - # and force a new TCP connection on retry, which may - # route to a different CDN edge node - # pylint: disable=protected-access,broad-except - try: - if ( - hasattr(self, "_http_response") - and self._http_response is not None - ): - self._http_response.close() - if hasattr(self, "_http_session"): - self._http_session.close() - except Exception: - pass - # pylint: enable=protected-access,broad-except - time.sleep(delay) - else: - # Final attempt - re-raise - raise - - patched_init._esphome_patched = True # type: ignore[attr-defined] # pylint: disable=protected-access - FileDownloader.__init__ = patched_init - - IGNORE_LIB_WARNINGS = f"(?:{'|'.join(['Hash', 'Update'])})" FILTER_PLATFORMIO_LINES = [ r"Verbose mode can be enabled via `-v, --verbose` option.*", @@ -142,20 +53,6 @@ FILTER_PLATFORMIO_LINES = [ ] -class PlatformioLogFilter(logging.Filter): - """Filter to suppress noisy platformio log messages.""" - - _PATTERN = re.compile( - r"|".join(r"(?:" + pattern + r")" for pattern in FILTER_PLATFORMIO_LINES) - ) - - def filter(self, record: logging.LogRecord) -> bool: - # Only filter messages from platformio-related loggers - if "platformio" not in record.name.lower(): - return True - return self._PATTERN.match(record.getMessage()) is None - - def run_platformio_cli(*args, **kwargs) -> str | int: os.environ["PLATFORMIO_FORCE_COLOR"] = "true" os.environ["PLATFORMIO_BUILD_DIR"] = str(CORE.relative_pioenvs_path().absolute()) @@ -166,30 +63,12 @@ def run_platformio_cli(*args, **kwargs) -> str | int: os.environ.setdefault("PYTHONWARNINGS", "ignore::SyntaxWarning") # Increase uv retry count to handle transient network errors (default is 3) os.environ.setdefault("UV_HTTP_RETRIES", "10") - cmd = ["platformio"] + list(args) + cmd = [sys.executable, "-m", "esphome.platformio_runner"] + list(args) if not CORE.verbose: kwargs["filter_lines"] = FILTER_PLATFORMIO_LINES - if os.environ.get("ESPHOME_USE_SUBPROCESS") is not None: - return run_external_process(*cmd, **kwargs) - - import platformio.__main__ - - patch_structhash() - patch_file_downloader() - - # Add log filter to suppress noisy platformio messages - log_filter = PlatformioLogFilter() if not CORE.verbose else None - if log_filter: - for handler in logging.getLogger().handlers: - handler.addFilter(log_filter) - try: - return run_external_command(platformio.__main__.main, *cmd, **kwargs) - finally: - if log_filter: - for handler in logging.getLogger().handlers: - handler.removeFilter(log_filter) + return run_external_process(*cmd, **kwargs) def run_platformio_cli_run(config, verbose, *args, **kwargs) -> str | int: diff --git a/esphome/platformio_runner.py b/esphome/platformio_runner.py new file mode 100644 index 0000000000..408d49d1a6 --- /dev/null +++ b/esphome/platformio_runner.py @@ -0,0 +1,114 @@ +"""Subprocess entry point that applies ESPHome's PlatformIO patches. + +Invoked via ``python -m esphome.platformio_runner`` instead of +``python -m platformio`` so that the patches (incremental rebuild +preservation, download retries) apply inside the subprocess. Running +PlatformIO in a subprocess keeps its ``sys.path`` mutations and other +global state from leaking into the ESPHome process. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +import sys +import time +from typing import Any + +_LOGGER = logging.getLogger(__name__) + + +def patch_structhash() -> None: + """Avoid full rebuilds when files are added or removed. + + PlatformIO clears the build dir whenever its structure hash changes. + We replace that with an mtime check against ``platformio.ini`` so + incremental builds are preserved unless the project config changed. + """ + from platformio.run import cli, helpers + + def patched_clean_build_dir(build_dir, *_args): + from platformio import fs + from platformio.project.helpers import get_project_dir + + platformio_ini = Path(get_project_dir()) / "platformio.ini" + build_dir = Path(build_dir) + + if ( + build_dir.is_dir() + and platformio_ini.stat().st_mtime > build_dir.stat().st_mtime + ): + fs.rmtree(build_dir) + + if not build_dir.is_dir(): + build_dir.mkdir(parents=True) + + helpers.clean_build_dir = patched_clean_build_dir + cli.clean_build_dir = patched_clean_build_dir + + +def patch_file_downloader() -> None: + """Retry PlatformIO package downloads with exponential backoff. + + PlatformIO's ``FileDownloader`` uses an ``HTTPSession`` without built-in + retry for 502/503 errors. We wrap ``__init__`` to retry on + ``PackageException`` and close the session between attempts so a new + TCP connection can route to a different CDN edge node. + """ + from platformio.package.download import FileDownloader + from platformio.package.exception import PackageException + + if getattr(FileDownloader.__init__, "_esphome_patched", False): + return + + original_init = FileDownloader.__init__ + + def patched_init(self, *args: Any, **kwargs: Any) -> None: + max_retries = 5 + + for attempt in range(max_retries): + try: + original_init(self, *args, **kwargs) + return + except PackageException as e: + if attempt < max_retries - 1: + delay = 2 ** (attempt + 1) + _LOGGER.warning( + "Package download failed: %s. " + "Retrying in %d seconds... (attempt %d/%d)", + str(e), + delay, + attempt + 1, + max_retries, + ) + # pylint: disable=protected-access,broad-except + try: + if ( + hasattr(self, "_http_response") + and self._http_response is not None + ): + self._http_response.close() + if hasattr(self, "_http_session"): + self._http_session.close() + except Exception: + pass + # pylint: enable=protected-access,broad-except + time.sleep(delay) + else: + raise + + patched_init._esphome_patched = True # type: ignore[attr-defined] # pylint: disable=protected-access + FileDownloader.__init__ = patched_init + + +def main() -> int: + patch_structhash() + patch_file_downloader() + + import platformio.__main__ + + return platformio.__main__.main() or 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/platformio.ini b/platformio.ini index e0f7c7d443..7d17628a8f 100644 --- a/platformio.ini +++ b/platformio.ini @@ -133,10 +133,10 @@ extra_scripts = post:esphome/components/esp8266/post_build.py.script ; This are common settings for the ESP32 (all variants) using Arduino. [common:esp32-arduino] extends = common:arduino -platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.37/platform-espressif32.zip +platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.38/platform-espressif32.zip platform_packages = - pioarduino/framework-arduinoespressif32@https://github.com/espressif/arduino-esp32/releases/download/3.3.7/esp32-core-3.3.7.tar.xz - pioarduino/framework-espidf@https://github.com/pioarduino/esp-idf/releases/download/v5.5.3.1/esp-idf-v5.5.3.1.tar.xz + pioarduino/framework-arduinoespressif32@https://github.com/espressif/arduino-esp32/releases/download/3.3.8/esp32-core-3.3.8.tar.xz + pioarduino/framework-espidf@https://github.com/pioarduino/esp-idf/releases/download/v5.5.4/esp-idf-v5.5.4.tar.xz framework = arduino, espidf ; Arduino as an ESP-IDF component lib_deps = @@ -169,9 +169,9 @@ extra_scripts = post:esphome/components/esp32/post_build.py.script ; This are common settings for the ESP32 (all variants) using IDF. [common:esp32-idf] extends = common:idf -platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.37/platform-espressif32.zip +platform = https://github.com/pioarduino/platform-espressif32/releases/download/55.03.38/platform-espressif32.zip platform_packages = - pioarduino/framework-espidf@https://github.com/pioarduino/esp-idf/releases/download/v5.5.3.1/esp-idf-v5.5.3.1.tar.xz + pioarduino/framework-espidf@https://github.com/pioarduino/esp-idf/releases/download/v5.5.4/esp-idf-v5.5.4.tar.xz framework = espidf lib_deps = diff --git a/pyproject.toml b/pyproject.toml index 2e3a247768..a744286e88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,6 @@ classifiers = [ "Topic :: Home Automation", ] -# Python 3.14 is not supported on Windows, see https://github.com/zephyrproject-rtos/windows-curses/issues/76 requires-python = ">=3.11.0,<3.15" dynamic = ["dependencies", "optional-dependencies", "version"] diff --git a/tests/component_tests/packages/test_packages.py b/tests/component_tests/packages/test_packages.py index 0893c7dcbb..cd91c4d8cb 100644 --- a/tests/component_tests/packages/test_packages.py +++ b/tests/component_tests/packages/test_packages.py @@ -1,11 +1,18 @@ """Tests for the packages component.""" +import logging from pathlib import Path from unittest.mock import MagicMock, patch import pytest -from esphome.components.packages import CONFIG_SCHEMA, do_packages_pass, merge_packages +from esphome.components.packages import ( + CONFIG_SCHEMA, + _walk_packages, + do_packages_pass, + is_package_definition, + merge_packages, +) from esphome.components.substitutions import do_substitution_pass import esphome.config as config_module from esphome.config import resolve_extend_remove @@ -37,7 +44,7 @@ from esphome.const import ( ) from esphome.core import CORE from esphome.util import OrderedDict -from esphome.yaml_util import add_context +from esphome.yaml_util import IncludeFile, add_context # Test strings TEST_DEVICE_NAME = "test_device_name" @@ -79,6 +86,44 @@ def packages_pass(config): return config +_INCLUDE_FILE = "INCLUDE_FILE" + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + # IncludeFile objects are package definitions + (_INCLUDE_FILE, True), + # Git URL shorthand strings are package definitions + ("github://esphome/firmware/base.yaml@main", True), + # Remote package dicts (with url key) are package definitions + ({"url": "https://github.com/esphome/firmware", "file": "base.yaml"}, True), + # Plain config dicts are NOT package definitions (they are config fragments) + ({"wifi": {"ssid": "test"}}, False), + # None is not a package definition + (None, False), + # Lists are not package definitions + ([{"wifi": {"ssid": "test"}}], False), + # Empty dicts are not package definitions + ({}, False), + ], + ids=[ + "include_file", + "git_shorthand", + "remote_package", + "config_fragment", + "none", + "list", + "empty_dict", + ], +) +def test_is_package_definition(value: object, expected: bool) -> None: + """Test that is_package_definition correctly identifies package definitions.""" + if value is _INCLUDE_FILE: + value = MagicMock(spec=IncludeFile) + assert is_package_definition(value) is expected + + def test_package_unused(basic_esphome, basic_wifi) -> None: """ Ensures do_package_pass does not change a config if packages aren't used. @@ -1061,6 +1106,51 @@ def test_packages_invalid_type_raises() -> None: do_packages_pass(config) +@patch("esphome.components.packages.resolve_include") +def test_packages_include_file_resolves_to_list(mock_resolve_include) -> None: + """When packages: is an IncludeFile that resolves to a list, it is processed correctly.""" + include_file = MagicMock(spec=IncludeFile) + package_content = {CONF_WIFI: {CONF_SSID: TEST_PACKAGE_WIFI_SSID}} + mock_resolve_include.return_value = ([package_content], None) + + config = {CONF_PACKAGES: include_file} + result = do_packages_pass(config) + result = merge_packages(result) + + assert result == {CONF_WIFI: {CONF_SSID: TEST_PACKAGE_WIFI_SSID}} + + +@patch("esphome.components.packages.resolve_include") +def test_packages_include_file_resolves_to_dict(mock_resolve_include) -> None: + """When packages: is an IncludeFile that resolves to a dict, it is processed correctly.""" + include_file = MagicMock(spec=IncludeFile) + package_content = {CONF_WIFI: {CONF_SSID: TEST_PACKAGE_WIFI_SSID}} + mock_resolve_include.return_value = ({"network": package_content}, None) + + config = {CONF_PACKAGES: include_file} + result = do_packages_pass(config) + result = merge_packages(result) + + assert result == {CONF_WIFI: {CONF_SSID: TEST_PACKAGE_WIFI_SSID}} + + +@patch("esphome.components.packages.resolve_include") +def test_packages_include_file_resolves_to_invalid_type_raises( + mock_resolve_include, +) -> None: + """When packages: is an IncludeFile that resolves to an invalid type, cv.Invalid is raised.""" + include_file = MagicMock(spec=IncludeFile) + mock_resolve_include.return_value = ("not_a_dict_or_list", None) + + config = {CONF_PACKAGES: include_file} + with pytest.raises( + cv.Invalid, match="Packages must be a key to value mapping or list" + ) as exc_info: + do_packages_pass(config) + + assert exc_info.value.path == [CONF_PACKAGES] + + @pytest.mark.parametrize( "invalid_package", [ @@ -1107,6 +1197,134 @@ def test_invalid_package_contents_masked_by_deprecation( do_packages_pass(config) +def test_named_dict_with_include_files_no_false_deprecation_warning( + caplog: pytest.LogCaptureFixture, +) -> None: + """Package errors in named dicts must not trigger the deprecated fallback.""" + good_include = MagicMock(spec=IncludeFile) + bad_include = MagicMock(spec=IncludeFile) + + config = { + CONF_PACKAGES: { + "good_pkg": good_include, + "bad_pkg": bad_include, + }, + } + + call_count = 0 + + def failing_callback(package_config: dict, context: object) -> dict: + nonlocal call_count + call_count += 1 + if call_count == 1: + # First package processes fine + return {CONF_WIFI: {CONF_SSID: "test"}} + # Second package has an error (e.g. jinja syntax error) + raise cv.Invalid("simulated jinja error in bad_pkg") + + with ( + caplog.at_level(logging.WARNING), + pytest.raises(cv.Invalid, match="simulated jinja error"), + ): + _walk_packages(config, failing_callback) + + # Must NOT emit the deprecated single-package warning + assert "deprecated" not in caplog.text.lower() + + +def test_validate_deprecated_false_raises_directly( + caplog: pytest.LogCaptureFixture, +) -> None: + """With validate_deprecated=False, errors raise directly without fallback. + + This is the codepath used for remote packages where _process_remote_package + returns already-resolved dicts that is_package_definition cannot detect. + """ + config = { + CONF_PACKAGES: { + "pkg_a": {CONF_WIFI: {CONF_SSID: "test"}}, + "pkg_b": {CONF_WIFI: {CONF_SSID: "test2"}}, + }, + } + + call_count = 0 + + def failing_callback(package_config: dict, context: object) -> dict: + nonlocal call_count + call_count += 1 + if call_count == 1: + return package_config + raise cv.Invalid("nested error") + + with ( + caplog.at_level(logging.WARNING), + pytest.raises(cv.Invalid, match="nested error"), + ): + _walk_packages(config, failing_callback, validate_deprecated=False) + + assert "deprecated" not in caplog.text.lower() + + +def test_error_on_first_declared_package_still_detected() -> None: + """When the first declared package errors, it's the last processed in reverse. + + All other entries are already resolved to dicts, but the failing entry + retains its original IncludeFile value since assignment was skipped. + """ + config = { + CONF_PACKAGES: { + "first_pkg": MagicMock(spec=IncludeFile), + "second_pkg": MagicMock(spec=IncludeFile), + "third_pkg": MagicMock(spec=IncludeFile), + }, + } + + call_count = 0 + + def fail_on_last(package_config: dict, context: object) -> dict: + nonlocal call_count + call_count += 1 + # Reverse iteration: third_pkg (1), second_pkg (2), first_pkg (3) + if call_count < 3: + return {CONF_WIFI: {CONF_SSID: "test"}} + raise cv.Invalid("error in first_pkg") + + with pytest.raises(cv.Invalid, match="error in first_pkg"): + _walk_packages(config, fail_on_last) + + +def test_deprecated_single_package_fallback_still_works( + caplog: pytest.LogCaptureFixture, +) -> None: + """The deprecated single-package form still falls back at the top level. + + When a dict's values are plain config fragments (not package definitions) + and the callback fails, the deprecated fallback wraps the dict in a list + and retries with a deprecation warning. + """ + config = { + CONF_PACKAGES: { + CONF_WIFI: {CONF_SSID: "test", CONF_PASSWORD: "secret"}, + }, + } + + attempt = 0 + + def fail_then_succeed(package_config: dict, context: object) -> dict: + nonlocal attempt + attempt += 1 + if attempt == 1: + # First attempt: treating as named dict fails + raise cv.Invalid("not a valid package") + # Second attempt: after fallback wraps as list, succeeds + return package_config + + with caplog.at_level(logging.WARNING): + _walk_packages(config, fail_then_succeed) + + assert "deprecated" in caplog.text.lower() + + def test_merge_packages_invalid_nested_type_raises() -> None: """Invalid nested packages type during merge raises cv.Invalid.""" config = { diff --git a/tests/components/canbus/common.yaml b/tests/components/canbus/common.yaml index 8bddeb7409..e779f7f078 100644 --- a/tests/components/canbus/common.yaml +++ b/tests/components/canbus/common.yaml @@ -50,6 +50,13 @@ button: - platform: template name: Canbus Actions on_press: + - canbus.send: + can_id: 0x601 + data: [0, 1, 2] + - canbus.send: + can_id: 0x1FFFFFFF + use_extended_id: true + data: [0, 1, 2] - canbus.send: "abc" - canbus.send: [0, 1, 2] - canbus.send: !lambda return {0, 1, 2}; diff --git a/tests/components/lvgl/test.host.yaml b/tests/components/lvgl/test.host.yaml index f84156c9d8..6328648fe3 100644 --- a/tests/components/lvgl/test.host.yaml +++ b/tests/components/lvgl/test.host.yaml @@ -20,6 +20,7 @@ lvgl: - id: lvgl_0 default_font: space16 displays: sdl0 + rotation: 180 top_layer: - id: lvgl_1 diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index 1a1bfffd03..dfd4305c4d 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -84,9 +84,9 @@ def mock_decode_pc() -> Generator[Mock, None, None]: @pytest.fixture -def mock_run_external_command() -> Generator[Mock, None, None]: - """Mock run_external_command for platformio_api.""" - with patch("esphome.platformio_api.run_external_command") as mock: +def mock_run_external_process() -> Generator[Mock, None, None]: + """Mock run_external_process for platformio_api.""" + with patch("esphome.platformio_api.run_external_process") as mock: yield mock diff --git a/tests/unit_tests/fixtures/substitutions/13-packages_as_included_list.approved.yaml b/tests/unit_tests/fixtures/substitutions/13-packages_as_included_list.approved.yaml new file mode 100644 index 0000000000..7863def190 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/13-packages_as_included_list.approved.yaml @@ -0,0 +1,3 @@ +wifi: + password: pkg_password + ssid: main_ssid diff --git a/tests/unit_tests/fixtures/substitutions/13-packages_as_included_list.input.yaml b/tests/unit_tests/fixtures/substitutions/13-packages_as_included_list.input.yaml new file mode 100644 index 0000000000..7a3b4970db --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/13-packages_as_included_list.input.yaml @@ -0,0 +1,4 @@ +packages: !include 13-packages_list.yaml + +wifi: + ssid: main_ssid diff --git a/tests/unit_tests/fixtures/substitutions/13-packages_list.yaml b/tests/unit_tests/fixtures/substitutions/13-packages_list.yaml new file mode 100644 index 0000000000..23161db3d3 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/13-packages_list.yaml @@ -0,0 +1,2 @@ +- wifi: + password: pkg_password diff --git a/tests/unit_tests/fixtures/substitutions/14-packages_as_included_dict.approved.yaml b/tests/unit_tests/fixtures/substitutions/14-packages_as_included_dict.approved.yaml new file mode 100644 index 0000000000..7863def190 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/14-packages_as_included_dict.approved.yaml @@ -0,0 +1,3 @@ +wifi: + password: pkg_password + ssid: main_ssid diff --git a/tests/unit_tests/fixtures/substitutions/14-packages_as_included_dict.input.yaml b/tests/unit_tests/fixtures/substitutions/14-packages_as_included_dict.input.yaml new file mode 100644 index 0000000000..8b9fc5ec3a --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/14-packages_as_included_dict.input.yaml @@ -0,0 +1,4 @@ +packages: !include 14-packages_dict.yaml + +wifi: + ssid: main_ssid diff --git a/tests/unit_tests/fixtures/substitutions/14-packages_dict.yaml b/tests/unit_tests/fixtures/substitutions/14-packages_dict.yaml new file mode 100644 index 0000000000..55e8b38a43 --- /dev/null +++ b/tests/unit_tests/fixtures/substitutions/14-packages_dict.yaml @@ -0,0 +1,3 @@ +network: + wifi: + password: pkg_password diff --git a/tests/unit_tests/test_platformio_api.py b/tests/unit_tests/test_platformio_api.py index e1b3908c24..ddc4e45c84 100644 --- a/tests/unit_tests/test_platformio_api.py +++ b/tests/unit_tests/test_platformio_api.py @@ -1,7 +1,8 @@ """Tests for platformio_api.py path functions.""" +# pylint: disable=protected-access + import json -import logging import os from pathlib import Path import shutil @@ -10,7 +11,7 @@ from unittest.mock import MagicMock, Mock, call, patch import pytest -from esphome import platformio_api +from esphome import platformio_api, platformio_runner from esphome.core import CORE, EsphomeError @@ -281,13 +282,13 @@ def test_run_idedata_raises_on_invalid_json( def test_run_platformio_cli_sets_environment_variables( - setup_core: Path, mock_run_external_command: Mock + setup_core: Path, mock_run_external_process: Mock ) -> None: """Test run_platformio_cli sets correct environment variables.""" CORE.build_path = str(setup_core / "build" / "test") with patch.dict(os.environ, {}, clear=False): - mock_run_external_command.return_value = 0 + mock_run_external_process.return_value = 0 platformio_api.run_platformio_cli("test", "arg") # Check environment variables were set @@ -300,10 +301,12 @@ def test_run_platformio_cli_sets_environment_variables( assert "PLATFORMIO_LIBDEPS_DIR" in os.environ assert "PYTHONWARNINGS" in os.environ - # Check command was called correctly - mock_run_external_command.assert_called_once() - args = mock_run_external_command.call_args[0] - assert "platformio" in args + # Check command was called correctly — runs PlatformIO as a subprocess + # via the esphome.platformio_runner entry point. + mock_run_external_process.assert_called_once() + args = mock_run_external_process.call_args[0] + assert "-m" in args + assert "esphome.platformio_runner" in args assert "test" in args assert "arg" in args @@ -444,7 +447,7 @@ def test_patch_structhash(setup_core: Path) -> None: }, ): # Call patch_structhash - platformio_api.patch_structhash() + platformio_runner.patch_structhash() # Verify both modules had clean_build_dir patched # Check that clean_build_dir was set on both modules @@ -496,7 +499,7 @@ def test_patched_clean_build_dir_removes_outdated(setup_core: Path) -> None: }, ): # Call patch_structhash to install the patched function - platformio_api.patch_structhash() + platformio_runner.patch_structhash() # Call the patched function mock_helpers.clean_build_dir(str(build_dir), []) @@ -546,7 +549,7 @@ def test_patched_clean_build_dir_keeps_updated(setup_core: Path) -> None: }, ): # Call patch_structhash to install the patched function - platformio_api.patch_structhash() + platformio_runner.patch_structhash() # Call the patched function mock_helpers.clean_build_dir(str(build_dir), []) @@ -594,7 +597,7 @@ def test_patched_clean_build_dir_creates_missing(setup_core: Path) -> None: }, ): # Call patch_structhash to install the patched function - platformio_api.patch_structhash() + platformio_runner.patch_structhash() # Call the patched function mock_helpers.clean_build_dir(str(build_dir), []) @@ -719,7 +722,7 @@ def test_patch_file_downloader_succeeds_first_try() -> None: ), }, ): - platformio_api.patch_file_downloader() + platformio_runner.patch_file_downloader() from platformio.package.download import FileDownloader @@ -758,7 +761,7 @@ def test_patch_file_downloader_retries_on_failure() -> None: ), patch("time.sleep") as mock_sleep, ): - platformio_api.patch_file_downloader() + platformio_runner.patch_file_downloader() from platformio.package.download import FileDownloader @@ -799,7 +802,7 @@ def test_patch_file_downloader_raises_after_max_retries() -> None: ), patch("time.sleep") as mock_sleep, ): - platformio_api.patch_file_downloader() + platformio_runner.patch_file_downloader() from platformio.package.download import FileDownloader @@ -847,7 +850,7 @@ def test_patch_file_downloader_closes_session_and_response_between_retries() -> ), patch("time.sleep"), ): - platformio_api.patch_file_downloader() + platformio_runner.patch_file_downloader() from platformio.package.download import FileDownloader @@ -882,9 +885,9 @@ def test_patch_file_downloader_idempotent() -> None: }, ): # Patch multiple times - platformio_api.patch_file_downloader() - platformio_api.patch_file_downloader() - platformio_api.patch_file_downloader() + platformio_runner.patch_file_downloader() + platformio_runner.patch_file_downloader() + platformio_runner.patch_file_downloader() from platformio.package.download import FileDownloader @@ -895,19 +898,18 @@ def test_patch_file_downloader_idempotent() -> None: assert call_count == 1 -def test_platformio_log_filter_allows_non_platformio_messages() -> None: - """Test that non-platformio logger messages are allowed through.""" - log_filter = platformio_api.PlatformioLogFilter() - record = logging.LogRecord( - name="esphome.core", - level=logging.INFO, - pathname="", - lineno=0, - msg="Some esphome message", - args=(), - exc_info=None, +def _filter_through_redirect(line: str) -> str: + """Write a line through RedirectText with FILTER_PLATFORMIO_LINES and return what passes.""" + import io + + from esphome.util import RedirectText + + captured = io.StringIO() + redirect = RedirectText( + captured, filter_lines=platformio_api.FILTER_PLATFORMIO_LINES ) - assert log_filter.filter(record) is True + redirect.write(line + "\n") + return captured.getvalue() @pytest.mark.parametrize( @@ -930,19 +932,9 @@ def test_platformio_log_filter_allows_non_platformio_messages() -> None: "Memory Usage -> https://bit.ly/pio-memory-usage", ], ) -def test_platformio_log_filter_blocks_noisy_messages(msg: str) -> None: - """Test that noisy platformio messages are filtered out.""" - log_filter = platformio_api.PlatformioLogFilter() - record = logging.LogRecord( - name="platformio.builder", - level=logging.INFO, - pathname="", - lineno=0, - msg=msg, - args=(), - exc_info=None, - ) - assert log_filter.filter(record) is False +def test_filter_platformio_lines_blocks_noisy_messages(msg: str) -> None: + """Test that noisy platformio output lines are filtered out by RedirectText.""" + assert _filter_through_redirect(msg) == "" @pytest.mark.parametrize( @@ -954,39 +946,6 @@ def test_platformio_log_filter_blocks_noisy_messages(msg: str) -> None: "warning: unused variable", ], ) -def test_platformio_log_filter_allows_other_platformio_messages(msg: str) -> None: - """Test that non-noisy platformio messages are allowed through.""" - log_filter = platformio_api.PlatformioLogFilter() - record = logging.LogRecord( - name="platformio.builder", - level=logging.INFO, - pathname="", - lineno=0, - msg=msg, - args=(), - exc_info=None, - ) - assert log_filter.filter(record) is True - - -@pytest.mark.parametrize( - "logger_name", - [ - "PLATFORMIO.builder", - "PlatformIO.core", - "platformio.run", - ], -) -def test_platformio_log_filter_case_insensitive_logger_name(logger_name: str) -> None: - """Test that platformio logger name matching is case insensitive.""" - log_filter = platformio_api.PlatformioLogFilter() - record = logging.LogRecord( - name=logger_name, - level=logging.INFO, - pathname="", - lineno=0, - msg="Found 5 compatible libraries", - args=(), - exc_info=None, - ) - assert log_filter.filter(record) is False +def test_filter_platformio_lines_allows_other_messages(msg: str) -> None: + """Test that non-noisy platformio output lines pass through RedirectText.""" + assert _filter_through_redirect(msg) == msg + "\n"