diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 3b39d519c4..977fe9428d 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -8,7 +8,7 @@ contact_links: url: https://github.com/esphome/esphome-webserver/issues/new/choose about: Report an issue with the ESPHome web server. - name: Report an issue with the ESPHome Builder / Dashboard - url: https://github.com/esphome/dashboard/issues/new/choose + url: https://github.com/esphome/device-builder/issues/new/choose about: Report an issue with the ESPHome Builder / Dashboard. - name: Report an issue with the ESPHome API client url: https://github.com/esphome/aioesphomeapi/issues/new/choose diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 30aa511e29..71dedd65aa 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -182,8 +182,8 @@ jobs: contents: read # actions/checkout to load the test configs strategy: fail-fast: false - # Cap concurrency so this smoke test doesn't hog all the shared runners. - max-parallel: 2 + # Modest cap so this smoke test leaves room on the shared runner pool. + max-parallel: 8 matrix: # One entry per distinct toolchain. ESP32 variants (c3/c6/s2/s3/p4) # share a toolchain bundle, so esp32 is exercised on the base variant diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e020e90018..735ba73c99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,7 @@ jobs: python -m venv venv . venv/bin/activate python --version - uv pip install -r requirements.txt -r requirements_dev.txt -r requirements_test.txt pre-commit + uv pip install -r requirements.txt -r requirements_dev.txt -r requirements_test.txt uv pip install -e . determine-jobs: @@ -208,69 +208,34 @@ jobs: run: script/ci-suggest-changes if: always() - pre-commit-ci-lite: - name: pre-commit.ci lite + lint-format: + name: Check lint and formatting runs-on: ubuntu-latest needs: - - common - determine-jobs if: github.event_name == 'pull_request' && !startsWith(github.base_ref, 'beta') && !startsWith(github.base_ref, 'release') && needs.determine-jobs.outputs.core-ci == 'true' steps: - name: Check out code from GitHub uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Restore Python - uses: ./.github/actions/restore-python + - name: Run prek + uses: j178/prek-action@4e14d07f9231acabce116ccfca13b13dd9755ece # v3.0.0 with: - python-version: ${{ env.DEFAULT_PYTHON }} - cache-key: ${{ needs.common.outputs.cache-key }} - # Inlined from esphome/pre-commit-action with a restore-only cache - # step: the pre-commit-seed-cache job owns saving this cache, so - # pull request runs never write per-PR copies. - - name: Restore pre-commit cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/pre-commit - # Must match the key pre-commit-seed-cache saves - # yamllint disable-line rule:line-length - key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} - - name: Run pre-commit + # Keep in sync with requirements_test.txt. + prek-version: "0.4.11" + # This job only runs on pull requests, so nothing ever populates + # the cache on dev. Every run would miss and then write a per-pull + # request copy, which is what the old seed-cache job existed to + # avoid. Building the hooks from scratch takes seconds, so skip it. + cache: false env: - SKIP: pylint,ci-custom - run: | - python -m pip install pre-commit - pre-commit run --show-diff-on-failure --color=always --all-files + PREK_SKIP: pylint,ci-custom + # Pushes any fixes the hooks made back to the pull request. This step + # must keep its default name: the GitHub App that performs the push + # locates the workflow run by that name. - uses: pre-commit-ci/lite-action@5d6cc0eb514c891a40562a58a8e71576c5c7fb43 # v1.1.0 if: always() - - pre-commit-seed-cache: - name: Seed pre-commit cache - runs-on: ubuntu-latest - needs: - - common - # Saves a dev-scoped pre-commit cache that pull request runs can - # restore, since pre-commit.ci lite itself never runs on dev pushes. - if: github.event_name == 'push' && github.ref == 'refs/heads/dev' - steps: - - name: Check out code from GitHub - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Restore Python - uses: ./.github/actions/restore-python with: - python-version: ${{ env.DEFAULT_PYTHON }} - cache-key: ${{ needs.common.outputs.cache-key }} - - name: Cache pre-commit environments - id: cache-pre-commit - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/pre-commit - # Must match the restore key in pre-commit-ci-lite - # yamllint disable-line rule:line-length - key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} - - name: Install pre-commit hook environments - if: steps.cache-pre-commit.outputs.cache-hit != 'true' - run: | - python -m pip install pre-commit - pre-commit install-hooks + msg: apply automatic formatting fixes pytest: name: Run pytest @@ -501,7 +466,7 @@ jobs: echo "binary=$BINARY" >> $GITHUB_OUTPUT - name: Run CodSpeed benchmarks - uses: CodSpeedHQ/action@88472375d0a4572cf70a9f1fe3a4e0ab8da1b924 # v5.0.1 + uses: CodSpeedHQ/action@0ca9cbbf4623b599a6c3ed4fc8a922942705d9f1 # v5.0.2 with: run: | . venv/bin/activate @@ -551,7 +516,6 @@ jobs: ESPHOME_SDK_NRF_PREFIX: ~/.esphome-sdk-nrf strategy: fail-fast: false - max-parallel: 2 matrix: include: - id: clang-tidy @@ -742,7 +706,6 @@ jobs: ESPHOME_ESP_IDF_PREFIX: ~/.esphome-idf strategy: fail-fast: false - max-parallel: 3 matrix: include: - id: clang-tidy @@ -822,7 +785,6 @@ jobs: ESPHOME_ESP_IDF_PREFIX: ~/.esphome-idf strategy: fail-fast: false - max-parallel: 3 matrix: include: - id: clang-tidy @@ -909,7 +871,7 @@ jobs: ESPHOME_SDK_NRF_PREFIX: ~/.esphome-sdk-nrf strategy: fail-fast: false - max-parallel: ${{ (startsWith(github.base_ref, 'beta') || startsWith(github.base_ref, 'release')) && 8 || 4 }} + max-parallel: ${{ (startsWith(github.base_ref, 'beta') || startsWith(github.base_ref, 'release')) && 32 || 16 }} matrix: batch: ${{ fromJson(needs.determine-jobs.outputs.component-test-batches) }} steps: @@ -1456,16 +1418,15 @@ jobs: ci-status: name: CI Status runs-on: ubuntu-24.04 - # Listed in the same order the jobs are defined above. Two jobs are + # Listed in the same order the jobs are defined above. One job is # deliberately left out: "benchmarks" reports through CodSpeed rather than - # this check, and "pre-commit-seed-cache" only populates a cache on pushes - # to dev. + # this check. needs: - common - determine-jobs - ci-custom - pylint - - pre-commit-ci-lite + - lint-format - pytest - codecov-empty-upload - integration-tests diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1586ead2e6..2751529222 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,7 +56,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4 + uses: github/codeql-action/init@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -84,6 +84,6 @@ jobs: exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4 + uses: github/codeql-action/analyze@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 8ec88238a9..3c471b6efb 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -16,7 +16,7 @@ jobs: # No GITHUB_TOKEN permissions: the reusable workflow mints an ESPHome # GitHub App token so the labels, comments and closures come from # esphome[bot] instead of github-actions[bot]. - uses: esphome/workflows/.github/workflows/stale.yml@203cea60ebfd18e2b966e57750750e0417a9feec # main + uses: esphome/workflows/.github/workflows/stale.yml@61fd37a044cad4e9aa4303027b2a61b6a34da855 # main secrets: ESPHOME_GITHUB_APP_PRIVATE_KEY: ${{ secrets.ESPHOME_GITHUB_APP_PRIVATE_KEY }} with: diff --git a/.github/workflows/sync-device-classes.yml b/.github/workflows/sync-device-classes.yml index 5d250b97eb..a299e76584 100644 --- a/.github/workflows/sync-device-classes.yml +++ b/.github/workflows/sync-device-classes.yml @@ -44,7 +44,7 @@ jobs: - name: Set up uv # An order of magnitude faster than pip on cold boots, with its # own wheel cache. ``--system`` (below) installs into the - # setup-python interpreter so subsequent ``pre-commit`` / + # setup-python interpreter so subsequent ``prek`` / # ``script/run-in-env.py`` steps find the deps without a # ``uv run`` prefix. uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 @@ -58,19 +58,19 @@ jobs: - name: Install Home Assistant run: | uv pip install --system -e lib/home-assistant - uv pip install --system -r requirements.txt -r requirements_test.txt pre-commit + uv pip install --system -r requirements.txt -r requirements_test.txt - name: Sync run: | python ./script/sync-device_class.py - - name: Apply pre-commit auto-fixes + - name: Apply prek auto-fixes # First pass: let formatters (ruff, end-of-file-fixer, etc.) modify - # files. pre-commit exits non-zero whenever a hook touches anything, + # files. prek exits non-zero whenever a hook touches anything, # which would otherwise abort the workflow before the auto-fixes # can flow into the sync PR. # - # SKIP: + # PREK_SKIP: # - no-commit-to-branch is a local guard against committing on # dev/release/beta; CI runs on dev by definition, and # peter-evans/create-pull-request creates the branch itself. @@ -79,18 +79,18 @@ jobs: # the runtime deps (HA + requirements*.txt); main CI already # gates pylint on real PRs. env: - SKIP: pylint,no-commit-to-branch - run: python script/run-in-env.py pre-commit run --all-files || true + PREK_SKIP: pylint,no-commit-to-branch + run: python script/run-in-env.py prek run --all-files || true - - name: Verify pre-commit clean + - name: Verify prek clean # Second pass: re-run all hooks against the now-fixed tree. # Auto-fixers exit 0 (nothing to change); any remaining failure # from a check-only hook (flake8 / yamllint / ci-custom) is a - # real issue and fails the workflow loudly. Same SKIP list as + # real issue and fails the workflow loudly. Same PREK_SKIP list as # above for the same reasons. env: - SKIP: pylint,no-commit-to-branch - run: python script/run-in-env.py pre-commit run --all-files + PREK_SKIP: pylint,no-commit-to-branch + run: python script/run-in-env.py prek run --all-files - name: Commit changes uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 diff --git a/.gitignore b/.gitignore index de3e4fa68e..fdb75824fb 100644 --- a/.gitignore +++ b/.gitignore @@ -133,6 +133,8 @@ CTestTestfile.cmake .gcc-flags.json config/ +# Test fixture config/ directories are tracked (the rule above is the dashboard dir) +!tests/component_tests/**/config/ tests/build/ tests/.esphome/ /.temp-clang-tidy.cpp diff --git a/AGENTS.md b/AGENTS.md index b067482d18..40381030cb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -412,7 +412,7 @@ This document provides essential context for AI models interacting with this pro * **Configuration:** * `pyproject.toml`: Defines the Python project metadata and dependencies. * `platformio.ini`: Configures the PlatformIO build environments for different microcontrollers. - * `.pre-commit-config.yaml`: Configures the pre-commit hooks for linting and formatting. + * `.pre-commit-config.yaml`: Configures the lint and format hooks, run by `prek`. * **CI/CD Pipeline:** Defined in `.github/workflows`. * **Static Analysis & Development:** * `esphome/core/defines.h`: A comprehensive header file containing all `#define` directives that can be added by components using `cg.add_define()` in Python. This file is used exclusively for development, static analysis tools, and CI testing - it is not used during runtime compilation. When developing components that add new defines, they must be added to this file to ensure proper IDE support and static analysis coverage. The file includes feature flags, build configurations, and platform-specific defines that help static analyzers understand the complete codebase without needing to compile for specific platforms. @@ -420,7 +420,7 @@ This document provides essential context for AI models interacting with this pro ## 6. Development & Testing Workflow * **Local Development Environment:** Use the provided Docker container or create a Python virtual environment and install dependencies from `requirements_dev.txt`. -* **Running Commands:** Use the `script/run-in-env.py` script to execute commands within the project's virtual environment. For example, to run the linter: `python3 script/run-in-env.py pre-commit run`. +* **Running Commands:** Use the `script/run-in-env.py` script to execute commands within the project's virtual environment. For example, to run the linter: `python3 script/run-in-env.py prek run`. * **Testing:** * **Python:** Run unit tests with `pytest`. * **C++:** Use `clang-tidy` for static analysis. @@ -493,7 +493,7 @@ This document provides essential context for AI models interacting with this pro 1. **Fork & Branch:** Create a new branch based on the `dev` branch (always use `git checkout -b dev` to ensure you're branching from `dev`, not the currently checked out branch). 2. **Make Changes:** Adhere to all coding conventions and patterns. 3. **Test:** Create component tests for all supported platforms and run the full test suite locally. - 4. **Lint:** Run `pre-commit` to ensure code is compliant. + 4. **Lint:** Run `prek` to ensure code is compliant. 5. **Commit:** Commit your changes. There is no strict format for commit messages. 6. **Pull Request:** Submit a PR against the `dev` branch. The Pull Request title must start with a `[tag]` prefix. For component work, use the component name (e.g., `[display] Fix bug`, `[abc123] Add new component`); for changes to shared/core code that isn't tied to a single component, use `[core]` (e.g., `[core] Add validator`). Update documentation, examples, and add `CODEOWNERS` entries as needed. Pull requests should always be made using the `.github/PULL_REQUEST_TEMPLATE.md` template - fill out all sections completely without removing any parts of the template. diff --git a/CODEOWNERS b/CODEOWNERS index cb1be26a61..d2e26edca3 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -288,6 +288,7 @@ esphome/components/ld2412/* @Rihan9 esphome/components/ld2420/* @descipher esphome/components/ld2450/* @hareeshmu esphome/components/ld24xx/* @kbx81 +esphome/components/ld6002b/* @hepter esphome/components/ledc/* @OttoWinter esphome/components/libretiny/* @kuba2k2 esphome/components/libretiny_pwm/* @kuba2k2 @@ -296,6 +297,7 @@ esphome/components/lightwaverf/* @max246 esphome/components/lilygo_t5_47/touchscreen/* @jesserockz esphome/components/lm75b/* @beormund esphome/components/ln882h_ble/* @Bl00d-B0b +esphome/components/ln882h_ble_tracker/* @Bl00d-B0b esphome/components/ln882x/* @lamauny esphome/components/lock/* @esphome/core esphome/components/logger/* @esphome/core @@ -350,6 +352,7 @@ esphome/components/mlx90393/* @functionpointer esphome/components/mlx90614/* @jesserockz esphome/components/mmc5603/* @benhoff esphome/components/mmc5983/* @agoode +esphome/components/modbus_client/* @exciton esphome/components/modbus_controller/* @martgras esphome/components/modbus_controller/binary_sensor/* @martgras esphome/components/modbus_controller/number/* @martgras @@ -438,6 +441,7 @@ esphome/components/rp2/* @jesserockz esphome/components/rp2040_ble/* @bdraco esphome/components/rp2040_pio_led_strip/* @Papa-DMan esphome/components/rp2040_pwm/* @jesserockz +esphome/components/rp2_ble_tracker/* @bdraco esphome/components/rpi_dpi_rgb/* @clydebarrow esphome/components/rtl87xx/* @kuba2k2 esphome/components/rtttl/* @glmnet @ximex diff --git a/docker/Dockerfile b/docker/Dockerfile index 5f84226b87..e928fd37ca 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.9.1 +RUN uv pip install --no-cache-dir esphome-device-builder==1.9.3 RUN \ platformio settings set enable_telemetry No \ diff --git a/esphome/__main__.py b/esphome/__main__.py index e56b504398..cb45dd7c5f 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2,16 +2,12 @@ import argparse from collections.abc import Callable from contextlib import suppress -from datetime import datetime import functools -import getpass import importlib import logging import os from pathlib import Path import re -import shutil -import subprocess import sys import time from typing import Protocol @@ -19,16 +15,18 @@ from typing import Protocol # Note: Do not import modules from esphome.components here, as this would # cause them to be loaded before external components are processed, resulting # in the built-in version being used instead of the external component one. -from esphome import const +from esphome import const, platform_hooks from esphome.const import ( ALLOWED_NAME_CHARS, ARGUMENT_HELP_DEVICE, + BUNDLE_EXTENSION, CONF_API, CONF_AUTH, CONF_BAUD_RATE, CONF_BROKER, CONF_DEASSERT_RTS_DTR, CONF_DISABLED, + CONF_DISCOVER_IP, CONF_ESPHOME, CONF_LEVEL, CONF_LOG_TOPIC, @@ -48,6 +46,8 @@ from esphome.const import ( CONF_WEB_SERVER, CONF_WIFI, ENV_NOGITIGNORE, + KEY_ESP32, + KEY_VARIANT, SECRETS_FILES, Toolchain, ) @@ -55,6 +55,7 @@ from esphome.core import CORE, EsphomeError, coroutine from esphome.enum import StrEnum from esphome.helpers import get_bool_env, indent, is_ip_address from esphome.log import AnsiFore, color, setup_log +from esphome.stacktrace import LogLineProcessor from esphome.types import ConfigType from esphome.upload_targets import PortType, get_port_type from esphome.util import ( @@ -484,8 +485,6 @@ def has_web_server_ota() -> bool: def has_mqtt_ip_lookup() -> bool: """Check if MQTT is available and IP lookup is supported.""" - from esphome.components.mqtt import CONF_DISCOVER_IP - if CONF_MQTT not in CORE.config: return False # Default Enabled @@ -618,6 +617,8 @@ def _resolve_network_devices( def run_miniterm(config: ConfigType, port: str, args) -> int: + from datetime import datetime + from aioesphomeapi import LogParser import serial @@ -630,18 +631,9 @@ def run_miniterm(config: ConfigType, port: str, args) -> int: return 1 _LOGGER.info("Starting log output from %s with baud rate %s", port, baud_rate) - process_stacktrace = None - - try: - module = importlib.import_module("esphome.components." + CORE.target_platform) - process_stacktrace = module.process_stacktrace - except (AttributeError, ImportError): - _LOGGER.info( - 'Stacktrace analysis is unavailable: no compatible analyzer found for target platform "%s".', - CORE.target_platform, - ) - - backtrace_state = False + # Decoder resolution, crash isolation, and disable-after-failure + # all live in LogLineProcessor, shared with the API log path. + processor = LogLineProcessor(config, CORE.target_platform) ser = serial.Serial() ser.baudrate = baud_rate ser.port = port @@ -681,11 +673,7 @@ def run_miniterm(config: ConfigType, port: str, args) -> int: "utf8", "backslashreplace" ) safe_print(parser.parse_line(line, time_str)) - - if process_stacktrace is not None: - backtrace_state = process_stacktrace( - config, line, backtrace_state - ) + processor.process_line(line) except serial.SerialException: _LOGGER.error("Serial port closed!") return 0 @@ -930,9 +918,10 @@ def upload_using_esptool( mcu = "esp8266" if CORE.is_esp32: - from esphome.components.esp32 import get_esp32_variant - - mcu = get_esp32_variant().lower() + # Same lookup as esp32.get_esp32_variant(), read directly so the + # serial upload path does not import the esp32 package; both the + # validator and the warm-cache apply_to_core populate this key. + mcu = CORE.data[KEY_ESP32][KEY_VARIANT].lower() line_callbacks: list[Callable[[str], str | None]] = [] if ( @@ -986,6 +975,8 @@ def upload_using_esptool( def upload_using_platformio(config: ConfigType, port: str) -> int: + import shutil + from esphome.platformio import toolchain # RP2040 platform-raspberrypi build recipe expects firmware.bin.signed for @@ -1023,6 +1014,8 @@ def upload_using_picotool(config: ConfigType) -> int: the mass storage copy approach that causes "disk not ejected properly" warnings on macOS. """ + import subprocess + from esphome.platformio import toolchain idedata = toolchain.get_idedata(config) @@ -1129,6 +1122,8 @@ def check_permissions(port: str): "the USB cable can be used for data and is not a power-only cable." ) if not (os.access(port, os.R_OK | os.W_OK)): + import getpass + raise EsphomeError( "You do not have read or write permission on the selected serial port. " "To resolve this issue, you can add your user to the dialout group " @@ -1141,12 +1136,11 @@ def upload_program( config: ConfigType, args: ArgsProtocol, devices: list[str] ) -> tuple[int, str | None]: host = devices[0] - try: - module = importlib.import_module("esphome.components." + CORE.target_platform) - if module.upload_program(config, args, host): - return 0, host - except AttributeError: - pass + platform_upload = platform_hooks.get_platform_hook( + CORE.target_platform, "upload_program" + ) + if platform_upload is not None and platform_upload(config, args, host): + return 0, host port_type = get_port_type(host) @@ -1406,12 +1400,11 @@ def _should_subscribe_states(args: ArgsProtocol) -> bool: def show_logs(config: ConfigType, args: ArgsProtocol, devices: list[str]) -> int | None: - try: - module = importlib.import_module("esphome.components." + CORE.target_platform) - if module.show_logs(config, args, devices): - return 0 - except AttributeError: - pass + platform_show_logs = platform_hooks.get_platform_hook( + CORE.target_platform, "show_logs" + ) + if platform_show_logs is not None and platform_show_logs(config, args, devices): + return 0 if "logger" not in config: raise EsphomeError("Logger is not configured!") @@ -1429,7 +1422,7 @@ def show_logs(config: ConfigType, args: ArgsProtocol, devices: list[str]) -> int if has_api() and ( network_devices := _resolve_network_devices(devices, config, args) ): - from esphome.components.api.client import run_logs + from esphome.api_client import run_logs return run_logs( config, @@ -1713,7 +1706,7 @@ def command_clean(args: ArgsProtocol, config: ConfigType) -> int | None: def command_bundle(args: ArgsProtocol, config: ConfigType) -> int | None: - from esphome.bundle import BUNDLE_EXTENSION, ConfigBundleCreator + from esphome.bundle import ConfigBundleCreator creator = ConfigBundleCreator(config) @@ -2563,10 +2556,11 @@ def run_esphome(argv): return 0 # Bundle support: if the configuration is a .esphomebundle, extract it - # and rewrite conf_path to the extracted YAML config. - from esphome.bundle import is_bundle_path, prepare_bundle_for_compile + # and rewrite conf_path to the extracted YAML config. The suffix check + # stays inline so the ordinary run never imports esphome.bundle. + if conf_path.name.lower().endswith(BUNDLE_EXTENSION): + from esphome.bundle import prepare_bundle_for_compile - if is_bundle_path(conf_path): _LOGGER.info("Extracting config bundle %s...", conf_path) conf_path = prepare_bundle_for_compile(conf_path) # Update the argument so downstream code sees the extracted path @@ -2607,6 +2601,8 @@ def run_esphome(argv): config = read_config( command_line_substitutions, skip_external_update=skip_external, + # Snapshot only needed by `esphome config --no-defaults`. + snapshot_user_config=getattr(args, "no_defaults", False), ) # Refresh the cache so the next upload/logs hits the fast path # instead of re-running read_config. Skip when the storage diff --git a/esphome/api_client.py b/esphome/api_client.py new file mode 100644 index 0000000000..a75f219b17 --- /dev/null +++ b/esphome/api_client.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import asyncio +from contextlib import suppress +import logging +from typing import TYPE_CHECKING, Any +import warnings + +# Suppress protobuf version warnings +with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=UserWarning, message=".*Protobuf gencode version.*" + ) + from aioesphomeapi import APIClient, parse_log_message + from aioesphomeapi.log_runner import async_run + +from esphome.const import CONF_ENCRYPTION, CONF_KEY, CONF_PORT, __version__ +from esphome.core import CORE +from esphome.stacktrace import LogLineProcessor +from esphome.util import safe_print + +if TYPE_CHECKING: + from aioesphomeapi.api_pb2 import ( + SubscribeLogsResponse, # pylint: disable=no-name-in-module + ) + + +_LOGGER = logging.getLogger(__name__) + + +async def async_run_logs( + config: dict[str, Any], + addresses: list[str], + subscribe_states: bool = True, +) -> None: + """Run the logs command in the event loop.""" + from datetime import datetime + + conf = config["api"] + name = config["esphome"]["name"] + port: int = int(conf[CONF_PORT]) + noise_psk: str | None = None + if (encryption := conf.get(CONF_ENCRYPTION)) and (key := encryption.get(CONF_KEY)): + noise_psk = key + + _LOGGER.info( + "Starting log output from %s using esphome API", " or ".join(addresses) + ) + + cli = APIClient( + addresses[0], # Primary address for compatibility + port, + "", # Password auth removed in 2026.1.0 + client_info=f"ESPHome Logs {__version__}", + noise_psk=noise_psk, + addresses=addresses, # Pass all addresses for automatic retry + provide_time=False, + ) + + # Decoder resolution policy lives in LogLineProcessor. + processor = LogLineProcessor(config, CORE.target_platform) + + def on_log(msg: SubscribeLogsResponse) -> None: + """Handle a new log message.""" + time_ = datetime.now().astimezone() + message: bytes = msg.message + text = message.decode("utf8", "backslashreplace") + nanoseconds = time_.microsecond // 1000 + timestamp = ( + f"[{time_.hour:02}:{time_.minute:02}:{time_.second:02}.{nanoseconds:03}]" + ) + for parsed_msg in parse_log_message(text, timestamp): + # safe_print handles the dashboard \033 escaping and falls back + # to backslashreplace encoding on stdouts that can't represent + # the wifi signal-bar block characters (Windows redirected + # cp1252 pipe). + safe_print(parsed_msg) + for raw_line in text.splitlines(): + processor.process_line(raw_line) + + # Safe to fall back to plaintext here only for this diagnostics use + # case: the stream is one-way from device to client, and this code + # never accepts commands or acts on any message the device sends. + # An on-path attacker could still both inject fabricated log lines + # and passively read the device's log output (and any state data + # delivered when subscribe_states is enabled), so this does lose + # confidentiality as well as authentication/integrity. That tradeoff + # is acceptable for operator-visible logs, which aioesphomeapi also + # warns may come from an unverified device. Never mirror this opt-in + # for any connection that sends data to the device or uses Home + # Assistant actions. + stop = await async_run( + cli, + on_log, + name=name, + subscribe_states=subscribe_states, + allow_plaintext_fallback=True, + # A top-level ``deep_sleep:`` block means the device is only awake + # briefly; cap the reconnect backoff so a wake window is not missed. + deep_sleep="deep_sleep" in config, + ) + try: + await asyncio.Event().wait() + finally: + await stop() + + +def run_logs( + config: dict[str, Any], + addresses: list[str], + subscribe_states: bool = True, +) -> None: + """Run the logs command.""" + with suppress(KeyboardInterrupt): + asyncio.run( + async_run_logs(config, addresses, subscribe_states=subscribe_states) + ) diff --git a/esphome/async_thread.py b/esphome/async_thread.py index 3972d735f5..3296d65af6 100644 --- a/esphome/async_thread.py +++ b/esphome/async_thread.py @@ -11,43 +11,136 @@ from __future__ import annotations import asyncio from collections.abc import Awaitable, Callable +from itertools import count +import logging import threading +from typing import cast + +_LOGGER = logging.getLogger(__name__) + +# How long the orphan watcher waits for an abandoned coroutine before giving +# up, so a hung operation does not park a watcher thread forever. +ORPHAN_WAIT_TIMEOUT = 300.0 + + +_runner_ids = count(1) + + +class AsyncDispatchTimeout(TimeoutError): + """The caller stopped waiting; the coroutine was abandoned. + + A subclass so callers can tell the dispatcher's own expiry apart from a + ``TimeoutError`` raised inside the coroutine, while existing + ``except TimeoutError`` handlers keep working. + """ class AsyncThreadRunner[T](threading.Thread): """Run an async coroutine in a daemon thread and expose its result. - The runner catches all exceptions from the coroutine and stores them in - ``exception`` so ``event`` is always set — this prevents callers waiting - on ``event`` from hanging forever when the coroutine crashes. - - Typical usage:: - - runner = AsyncThreadRunner(lambda: my_coro(arg)) - runner.start() - if not runner.event.wait(timeout=5.0): - ... # timed out - if runner.exception is not None: - raise runner.exception - result = runner.result + ``event`` is always set, even when the coroutine crashes, so waiters + never hang; ``completed`` distinguishes a delivered result (even a + legitimate ``None``) from a coroutine that never finished. Prefer + :func:`run_async`; use this class directly only when a failure should + degrade to a default value instead of raising. """ def __init__(self, coro_factory: Callable[[], Awaitable[T]]) -> None: - super().__init__(daemon=True) + super().__init__(daemon=True, name=f"async-thread-runner-{next(_runner_ids)}") self._coro_factory = coro_factory self.result: T | None = None self.exception: BaseException | None = None + self.completed = False self.event = threading.Event() async def _runner(self) -> None: try: self.result = await self._coro_factory() - except Exception as exc: # noqa: BLE001 # pylint: disable=broad-except - # Capture all exceptions so ``event`` is always set — otherwise a - # crash would hang the waiter forever. + # Distinguishes a delivered result from "never ran", since None + # is a valid result value. + self.completed = True + except BaseException as exc: # noqa: BLE001 # pylint: disable=broad-except + # Capture everything, including BaseException — otherwise a + # cancellation or SystemExit would leave ``exception`` unset and + # waiters would mistake the empty ``result`` for success. self.exception = exc finally: self.event.set() def run(self) -> None: - asyncio.run(self._runner()) + try: + asyncio.run(self._runner()) + except BaseException as exc: # noqa: BLE001 # pylint: disable=broad-except + # asyncio.run itself can fail before _runner executes (e.g. loop + # creation under fd exhaustion); record it so waiters never hang. + # A failure during loop cleanup after the coroutine completed + # must not clobber the delivered result, hence the guard. + if self.exception is None and not self.completed: + self.exception = exc + else: + _LOGGER.debug( + "Event loop teardown failed after outcome recorded", + exc_info=True, + ) + finally: + self.event.set() + + +def run_async[T]( + coro_factory: Callable[[], Awaitable[T]], + timeout: float | None = None, + on_orphan: Callable[[T], None] | None = None, +) -> T: + """Run a coroutine in a daemon-thread event loop and return its result. + + Raises :class:`AsyncDispatchTimeout` if the coroutine does not finish + within ``timeout`` seconds; the thread is abandoned and exits with the + interpreter. If the abandoned coroutine later produces a result, + ``on_orphan`` (if given) is called with it so resources such as a + connected socket can be released; delivery is best effort and bounded + by ``ORPHAN_WAIT_TIMEOUT``. + """ + runner: AsyncThreadRunner[T] = AsyncThreadRunner(coro_factory) + runner.start() + if not runner.event.wait(timeout): + + def _cleanup() -> None: + if not runner.event.wait(ORPHAN_WAIT_TIMEOUT): + # The one state where a resource can genuinely leak; leave + # a trace so a recurring hang is attributable. + _LOGGER.info( + "Orphan watcher gave up after %.0fs; a late result may leak", + ORPHAN_WAIT_TIMEOUT, + ) + return + if not runner.completed: + # The only place an abandoned thread's real error surfaces; + # without it a late failure hides behind the TimeoutError. + # INFO, not DEBUG: it fires at most once per abandoned + # operation and the cause may not reproduce on a rerun. + _LOGGER.info( + "Abandoned async operation failed", + exc_info=runner.exception, + ) + return + if (result := runner.result) is None: + return + if on_orphan is None: + _LOGGER.debug("Discarding late result; no on_orphan handler") + return + try: + on_orphan(result) + except Exception: # pylint: disable=broad-except + # INFO, not DEBUG: a failed release means a real leak, and + # it fires at most once per abandoned operation. + _LOGGER.info("Error releasing orphaned result", exc_info=True) + + threading.Thread( + target=_cleanup, daemon=True, name="async-orphan-cleanup" + ).start() + raise AsyncDispatchTimeout("Timed out waiting for async operation") + if (exc := runner.exception) is not None: + raise exc + if not runner.completed: + raise RuntimeError("Async operation finished without a result or an exception") + return cast("T", runner.result) diff --git a/esphome/bundle.py b/esphome/bundle.py index dcaea03646..b633c5ca4f 100644 --- a/esphome/bundle.py +++ b/esphome/bundle.py @@ -20,6 +20,7 @@ from typing import Any from esphome import const, yaml_util from esphome.const import ( + BUNDLE_EXTENSION, CONF_ESPHOME, CONF_EXTERNAL_COMPONENTS, CONF_INCLUDES, @@ -29,12 +30,12 @@ from esphome.const import ( CONF_TYPE, ) from esphome.core import CORE, EsphomeError +from esphome.util import filter_yaml_files _LOGGER = logging.getLogger(__name__) DOMAIN = "bundle" -BUNDLE_EXTENSION = ".esphomebundle.tar.gz" MANIFEST_FILENAME = "manifest.json" CURRENT_MANIFEST_VERSION = 1 MAX_DECOMPRESSED_SIZE = 500 * 1024 * 1024 # 500 MB @@ -128,6 +129,9 @@ class BundleData: """Files components asked to include, keyed under DOMAIN in CORE.data.""" extra_files: list[Path] = field(default_factory=list) + # Directories whose YAML files are scanned for !secret references but + # never bundled, e.g. git package checkouts the builder re-fetches. + secret_scan_dirs: set[Path] = field(default_factory=set) # Original config dir parsed from an extracted bundle's manifest.json, # kept in the path flavor of the machine the bundle was created on. # The checked flag makes the manifest lookup happen at most once per run; @@ -155,6 +159,30 @@ def add_bundle_file(path: Path) -> None: _get_data().extra_files.append(CORE.relative_config_path(path)) +def add_secret_scan_dir(path: Path) -> None: + """Register a directory to scan for ``!secret`` references when bundling. + + The directory's files are not added to the bundle. Components call this + for YAML the build consumes without bundling it — such as git-fetched + packages, which the builder re-fetches — so the secrets those files + reference are still shipped in the filtered secrets file. + + A relative path is taken as relative to the config directory. + """ + if not path.is_absolute(): + path = CORE.relative_config_path(path) + _get_data().secret_scan_dirs.add(path) + + +def _secret_scan_yaml_files() -> list[Path]: + """Return the YAML files inside registered secret-scan directories.""" + return filter_yaml_files( + f + for scan_dir in _get_data().secret_scan_dirs + for f in yaml_util.find_files(scan_dir, "*") + ) + + # Windows paths start with a drive letter or contain backslashes; POSIX # paths do neither in practice, so this is how the flavor of a recorded # path string is recognized on any host. @@ -310,6 +338,7 @@ class ConfigBundleCreator: yaml_sources = [ bf.source for bf in files if bf.source.suffix in (".yaml", ".yml") ] + yaml_sources.extend(_secret_scan_yaml_files()) used_secret_keys = _find_used_secret_keys(yaml_sources) filtered_secrets = self._build_filtered_secrets(used_secret_keys) @@ -394,6 +423,13 @@ class ConfigBundleCreator: """ discovered = yaml_util.discover_user_yaml_files(self._config_path) self._secrets_paths.update(discovered.secrets) + # A !secret inside a file this re-parse does not reach (for example + # a git-fetched package the builder re-fetches) still resolves + # against the config-dir secrets.yaml at build time, so always + # consider that file; filtering no-ops when no key matches. + default_secrets = self._config_dir / yaml_util.SECRET_YAML + if default_secrets.is_file(): + self._secrets_paths.add(default_secrets.resolve()) config_resolved = self._config_path.resolve() for fpath in discovered.files: if fpath == config_resolved: @@ -719,11 +755,6 @@ def _validate_tar_members(tar: tarfile.TarFile, target_dir: Path) -> None: ) -def is_bundle_path(path: Path) -> bool: - """Check if a path looks like a bundle file.""" - return path.name.lower().endswith(BUNDLE_EXTENSION) - - def _add_bytes_to_tar(tar: tarfile.TarFile, name: str, data: bytes) -> None: """Add in-memory bytes to a tar archive with deterministic metadata.""" info = tarfile.TarInfo(name=name) diff --git a/esphome/codegen.py b/esphome/codegen.py index 0694eb4d84..2430f17f3a 100644 --- a/esphome/codegen.py +++ b/esphome/codegen.py @@ -49,11 +49,13 @@ from esphome.cpp_helpers import ( # noqa: F401 build_registry_entry, build_registry_list, extract_registry_entry_config, + get_slot_count, gpio_pin_expression, past_safe_mode, register_component, register_parented, set_setup_priority, + slot_counter, ) from esphome.cpp_types import ( # noqa: F401 NAN, diff --git a/esphome/compiled_config.py b/esphome/compiled_config.py index f4fd205285..1bcd567b84 100644 --- a/esphome/compiled_config.py +++ b/esphome/compiled_config.py @@ -61,7 +61,14 @@ def load_compiled_config(conf_path: Path) -> ConfigType | None: from esphome import yaml_util try: - config = yaml_util.load_yaml(cache_path, clear_secrets=False) + # Fast path never validates or generates code - no source ranges + # needed (see load_yaml). Callers must not feed this config into + # read_config/write_cpp: the esp_range consumers in config.py and + # cpp_generator.py are isinstance-guarded and would degrade + # silently (wrong error/lambda locations) instead of raising. + config = yaml_util.load_yaml( + cache_path, clear_secrets=False, track_document_range=False + ) except Exception: # noqa: BLE001 # pylint: disable=broad-except return None diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 0719cee352..8ec94df1db 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -13,6 +13,7 @@ from esphome.const import ( CONF_CAPTURE_RESPONSE, CONF_DATA, CONF_DATA_TEMPLATE, + CONF_ENCRYPTION, CONF_EVENT, CONF_ID, CONF_KEY, @@ -102,7 +103,6 @@ SERVICE_ARG_FALLBACK_TYPES: dict[str, MockObj] = { for name, t in _SERVICE_ARG_SCALAR_TYPES.items() }, } -CONF_ENCRYPTION = "encryption" CONF_BATCH_DELAY = "batch_delay" CONF_CUSTOM_SERVICES = "custom_services" CONF_HOMEASSISTANT_SERVICES = "homeassistant_services" diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index f49ec8b01d..49db8e645d 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -443,7 +443,7 @@ void APIConnection::on_disconnect_response() { uint16_t APIConnection::fill_and_encode_entity_state(EntityBase *entity, StateResponseProtoMessage &msg, CalculateSizeFn size_fn, MessageEncodeFn encode_fn, APIConnection *conn, uint32_t remaining_size) { - msg.key = entity->get_object_id_hash(); + msg.key = entity->get_entity_key(); #ifdef USE_DEVICES msg.device_id = entity->get_device_id(); #endif @@ -454,7 +454,7 @@ uint16_t APIConnection::fill_and_encode_entity_info(EntityBase *entity, InfoResp CalculateSizeFn size_fn, MessageEncodeFn encode_fn, APIConnection *conn, uint32_t remaining_size) { // Set common fields that are shared by all entity types - msg.key = entity->get_object_id_hash(); + msg.key = entity->get_entity_key(); if (entity->has_own_name()) { msg.name = entity->get_name(); @@ -1144,7 +1144,7 @@ void APIConnection::try_send_camera_image_() { bool done = this->image_reader_->available() == to_send; CameraImageResponse msg; - msg.key = camera::Camera::instance()->get_object_id_hash(); + msg.key = camera::Camera::instance()->get_entity_key(); msg.set_data(this->image_reader_->peek_data_buffer(), to_send); msg.done = done; #ifdef USE_DEVICES diff --git a/esphome/components/api/client.py b/esphome/components/api/client.py index 7f07146dba..5e1c88b2ca 100644 --- a/esphome/components/api/client.py +++ b/esphome/components/api/client.py @@ -1,177 +1,10 @@ -from __future__ import annotations +"""Backward-compatibility shim; the log client lives in esphome.api_client. -import asyncio -from datetime import datetime -import importlib -import logging -from typing import TYPE_CHECKING, Any -import warnings +Importing this module executes the whole api component package, which pulls +in the validation stack. CLI code paths should import esphome.api_client +directly so the logs fast path stays light. +""" -# Suppress protobuf version warnings -with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", category=UserWarning, message=".*Protobuf gencode version.*" - ) - from aioesphomeapi import APIClient, parse_log_message - from aioesphomeapi.log_runner import async_run +from esphome.api_client import async_run_logs, run_logs -import contextlib - -from esphome.const import CONF_KEY, CONF_PORT, __version__ -from esphome.core import CORE -from esphome.util import safe_print - -from . import CONF_ENCRYPTION - -if TYPE_CHECKING: - from aioesphomeapi.api_pb2 import ( - SubscribeLogsResponse, # pylint: disable=no-name-in-module - ) - - -_LOGGER = logging.getLogger(__name__) - - -class _LogLineProcessor: - """Feeds incoming log lines to the stack-trace decoder. - - Two responsibilities beyond just calling the decoder: - 1. Catch everything the decoder can raise. aioesphomeapi isolates - exceptions raised by log handlers, so an escaping one no longer - kills the session, but it does log a full traceback per line. A - crash dump carries a PC line plus one per backtrace frame, so the - tracebacks bury the dump the user is trying to read. Decoding is a - diagnostic nicety; nothing it raises is worth that noise. - 2. Disable decoding after the first failure. _decode_pc shells out to - the toolchain to resolve addr2line, which is expensive; a single - crash dump can contain many PC/BT lines and we don't want to retry - the failing subprocess for each one. This only works if every - failure is caught, which is why 1 is not narrowed to EsphomeError. - """ - - def __init__(self, config: dict[str, Any], platform_handler: Any | None) -> None: - self._config = config - self._platform_handler = platform_handler - self._decode_enabled = True - self.backtrace_state = False - - def process_line(self, raw_line: str) -> None: - if not self._decode_enabled: - return - try: - if self._platform_handler is not None: - self.backtrace_state = self._platform_handler( - self._config, raw_line, self.backtrace_state - ) - except Exception as exc: # noqa: BLE001 # pylint: disable=broad-except - self._decode_enabled = False - self.backtrace_state = False - # _run_idedata raises EsphomeError with no message; fall back - # to a generic explanation when str(exc) is empty. - detail = str(exc) or "build artifacts not found locally" - _LOGGER.debug("Stack-trace decoding failed", exc_info=True) - _LOGGER.warning( - "Crash trace decoding unavailable: %s. " - "Run 'esphome compile' for this device to enable PC decoding.", - detail, - ) - - -async def async_run_logs( - config: dict[str, Any], - addresses: list[str], - subscribe_states: bool = True, -) -> None: - """Run the logs command in the event loop.""" - conf = config["api"] - name = config["esphome"]["name"] - port: int = int(conf[CONF_PORT]) - noise_psk: str | None = None - if (encryption := conf.get(CONF_ENCRYPTION)) and (key := encryption.get(CONF_KEY)): - noise_psk = key - - if len(addresses) == 1: - _LOGGER.info("Starting log output from %s using esphome API", addresses[0]) - else: - _LOGGER.info( - "Starting log output from %s using esphome API", " or ".join(addresses) - ) - - cli = APIClient( - addresses[0], # Primary address for compatibility - port, - "", # Password auth removed in 2026.1.0 - client_info=f"ESPHome Logs {__version__}", - noise_psk=noise_psk, - addresses=addresses, # Pass all addresses for automatic retry - provide_time=False, - ) - - # Try platform-specific stacktrace handler first, fall back to generic - platform_process_stacktrace = None - try: - module = importlib.import_module("esphome.components." + CORE.target_platform) - platform_process_stacktrace = module.process_stacktrace - except (AttributeError, ImportError): - _LOGGER.info( - 'Stacktrace analysis is unavailable: no compatible analyzer found for target platform "%s".', - CORE.target_platform, - ) - - processor = _LogLineProcessor(config, platform_process_stacktrace) - - def on_log(msg: SubscribeLogsResponse) -> None: - """Handle a new log message.""" - time_ = datetime.now().astimezone() - message: bytes = msg.message - text = message.decode("utf8", "backslashreplace") - nanoseconds = time_.microsecond // 1000 - timestamp = ( - f"[{time_.hour:02}:{time_.minute:02}:{time_.second:02}.{nanoseconds:03}]" - ) - for parsed_msg in parse_log_message(text, timestamp): - # safe_print handles the dashboard \033 escaping and falls back - # to backslashreplace encoding on stdouts that can't represent - # the wifi signal-bar block characters (Windows redirected - # cp1252 pipe). - safe_print(parsed_msg) - for raw_line in text.splitlines(): - processor.process_line(raw_line) - - # Safe to fall back to plaintext here only for this diagnostics use - # case: the stream is one-way from device to client, and this code - # never accepts commands or acts on any message the device sends. - # An on-path attacker could still both inject fabricated log lines - # and passively read the device's log output (and any state data - # delivered when subscribe_states is enabled), so this does lose - # confidentiality as well as authentication/integrity. That tradeoff - # is acceptable for operator-visible logs, which aioesphomeapi also - # warns may come from an unverified device. Never mirror this opt-in - # for any connection that sends data to the device or uses Home - # Assistant actions. - stop = await async_run( - cli, - on_log, - name=name, - subscribe_states=subscribe_states, - allow_plaintext_fallback=True, - # A top-level ``deep_sleep:`` block means the device is only awake - # briefly; cap the reconnect backoff so a wake window is not missed. - deep_sleep="deep_sleep" in config, - ) - try: - await asyncio.Event().wait() - finally: - await stop() - - -def run_logs( - config: dict[str, Any], - addresses: list[str], - subscribe_states: bool = True, -) -> None: - """Run the logs command.""" - with contextlib.suppress(KeyboardInterrupt): - asyncio.run( - async_run_logs(config, addresses, subscribe_states=subscribe_states) - ) +__all__ = ["async_run_logs", "run_logs"] diff --git a/esphome/components/bk72xx/__init__.py b/esphome/components/bk72xx/__init__.py index 3ffab0f3a5..ee9bf1e0d4 100644 --- a/esphome/components/bk72xx/__init__.py +++ b/esphome/components/bk72xx/__init__.py @@ -51,7 +51,11 @@ def _set_core_data(config): return config -CONFIG_SCHEMA = libretiny.BASE_SCHEMA +# extend({}) makes this platform's own schema instance: BASE_SCHEMA is shared +# by every LibreTiny platform, and prepending this platform's _set_core_data +# onto the shared object would run it for every platform's validation once two +# platform modules are imported in one process (device-builder, tests). +CONFIG_SCHEMA = libretiny.BASE_SCHEMA.extend({}) PIN_SCHEMA = libretiny.gpio.BASE_PIN_SCHEMA diff --git a/esphome/components/bk72xx_ble/__init__.py b/esphome/components/bk72xx_ble/__init__.py index 29e2a6b13d..23f3d06184 100644 --- a/esphome/components/bk72xx_ble/__init__.py +++ b/esphome/components/bk72xx_ble/__init__.py @@ -45,6 +45,11 @@ CONFIG_SCHEMA = cv.Schema( ).extend(cv.COMPONENT_SCHEMA) +# Once per registered scan listener; sizes the controller's StaticVector +# listener storage. +request_scan_listener_slot = cg.slot_counter("BK72XX_BLE_SCAN_LISTENER_COUNT") + + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) diff --git a/esphome/components/bk72xx_ble/bk72xx_ble.cpp b/esphome/components/bk72xx_ble/bk72xx_ble.cpp index a5ecaf4abb..69c7df0b96 100644 --- a/esphome/components/bk72xx_ble/bk72xx_ble.cpp +++ b/esphome/components/bk72xx_ble/bk72xx_ble.cpp @@ -179,8 +179,10 @@ void BK72xxBLE::loop() { if (report == nullptr) return; do { +#ifdef BK72XX_BLE_SCAN_LISTENER_COUNT for (auto *listener : this->scan_listeners_) listener->on_scan_report(*report); +#endif this->report_pool_.release(report); } while ((report = this->report_queue_.pop()) != nullptr); diff --git a/esphome/components/bk72xx_ble/bk72xx_ble.h b/esphome/components/bk72xx_ble/bk72xx_ble.h index 2654f4e68e..4e615af159 100644 --- a/esphome/components/bk72xx_ble/bk72xx_ble.h +++ b/esphome/components/bk72xx_ble/bk72xx_ble.h @@ -6,10 +6,10 @@ #include "esphome/core/component.h" #include "esphome/core/event_pool.h" +#include "esphome/core/helpers.h" #include "esphome/core/lock_free_queue.h" #include -#include namespace esphome::bk72xx_ble { @@ -62,8 +62,12 @@ class BK72xxBLE final : public Component { /// Controller BLE address, least-significant octet first (BLE convention). void get_mac_lsb_first(uint8_t out[6]) const; +#ifdef BK72XX_BLE_SCAN_LISTENER_COUNT /// Register a consumer for scan reports (delivered on the main task via loop()). + /// Storage is codegen-sized: the consumer's codegen requests a slot via + /// request_scan_listener_slot(), which emits BK72XX_BLE_SCAN_LISTENER_COUNT. void register_scan_listener(BLEScanListener *listener) { this->scan_listeners_.push_back(listener); } +#endif /// Start the controller scan. Interval/window are in BLE units (0.625 ms). /// Enables the stack first if needed. Returns false on controller failure. @@ -78,7 +82,11 @@ class BK72xxBLE final : public Component { protected: void resolve_mac_(); - std::vector scan_listeners_; +#ifdef BK72XX_BLE_SCAN_LISTENER_COUNT + // Codegen-sized: no heap allocation, no std::vector template instantiation — + // the same StaticVector pattern as the tracker's ble_device_base listeners. + StaticVector scan_listeners_; +#endif // Report ring: the BDK notice callback (BLE task) allocates a report from the // pool, fills it and pushes the pointer; loop() pops, dispatches and releases. // Lock-free SPSC, zero allocation at steady state — the esp32_ble pattern. diff --git a/esphome/components/bk72xx_ble_tracker/__init__.py b/esphome/components/bk72xx_ble_tracker/__init__.py index e53e8e13d7..e7f8ed92ba 100644 --- a/esphome/components/bk72xx_ble_tracker/__init__.py +++ b/esphome/components/bk72xx_ble_tracker/__init__.py @@ -12,19 +12,30 @@ Scan modes: Use this when the radio is dedicated to BLE. continuous: false — a started scan runs for `duration` ms, then stops. The FIRST start is external too: nothing in this component - starts a non-continuous scan on boot, so until the - automation actions land (follow-up PR) the radio stays - idle. start_scan() is called from code (e.g. an api - client-connected automation) so the single-core radio - can service WiFi in between scans. + starts a non-continuous scan on boot — the radio stays + idle until bk72xx_ble_tracker.start_scan fires (e.g. + from an api client-connected automation), so the + single-core radio can service WiFi in between scans. """ +from esphome import automation import esphome.codegen as cg from esphome.components import bk72xx_ble, ble_device_base, ota -from esphome.components.const import CONF_SCAN_PARAMETERS, CONF_WINDOW +from esphome.components.ble_device_base import automation as ble_automation +from esphome.components.const import CONF_ON_SCAN_END, CONF_SCAN_PARAMETERS, CONF_WINDOW import esphome.config_validation as cv -from esphome.const import CONF_CONTINUOUS, CONF_DURATION, CONF_ID, CONF_INTERVAL -from esphome.core import CORE, CoroPriority, coroutine_with_priority +from esphome.const import ( + CONF_CONTINUOUS, + CONF_DURATION, + CONF_ID, + CONF_INTERVAL, + CONF_MANUFACTURER_ID, + CONF_ON_BLE_ADVERTISE, + CONF_ON_BLE_MANUFACTURER_DATA_ADVERTISE, + CONF_ON_BLE_SERVICE_DATA_ADVERTISE, + CONF_SERVICE_UUID, +) +from esphome.core import ID from esphome.types import ConfigType CONF_BK72XX_BLE_ID = "bk72xx_ble_id" @@ -33,11 +44,21 @@ DEPENDENCIES = ["bk72xx"] AUTO_LOAD = ["ble_device_base", "bk72xx_ble"] CODEOWNERS = ["@Bl00d-B0b"] +ble_device_base.register_hub_provider("bk72xx_ble_tracker") + bk72xx_ble_tracker_ns = cg.esphome_ns.namespace("bk72xx_ble_tracker") BK72xxBLETracker = bk72xx_ble_tracker_ns.class_( "BK72xxBLETracker", ble_device_base.BLEHub, cg.Component ) +StartScanAction = bk72xx_ble_tracker_ns.class_("StartScanAction", automation.Action) +StopScanAction = bk72xx_ble_tracker_ns.class_("StopScanAction", automation.Action) + +ESPBTAdvertiseTrigger = ble_automation.ESPBTAdvertiseTrigger +BLEServiceDataAdvertiseTrigger = ble_automation.BLEServiceDataAdvertiseTrigger +BLEManufacturerDataAdvertiseTrigger = ble_automation.BLEManufacturerDataAdvertiseTrigger +BLEEndOfScanTrigger = ble_automation.BLEEndOfScanTrigger + # interval defaults to the BK reference scan rate — 100 ms with the shared 30 ms # window, a 30 % duty cycle. Converted to the controller's 0.625 ms BLE units in @@ -49,18 +70,77 @@ CONFIG_SCHEMA = cv.Schema( cv.GenerateID(): cv.declare_id(BK72xxBLETracker), cv.GenerateID(CONF_BK72XX_BLE_ID): cv.use_id(bk72xx_ble.BK72xxBLE), cv.Optional(CONF_SCAN_PARAMETERS, default={}): SCAN_PARAMETERS_SCHEMA, + cv.Optional(CONF_ON_BLE_ADVERTISE): ble_automation.advertise_trigger_schema( + ESPBTAdvertiseTrigger + ), + cv.Optional( + CONF_ON_BLE_SERVICE_DATA_ADVERTISE + ): ble_automation.uuid_trigger_schema( + BLEServiceDataAdvertiseTrigger, + {cv.Required(CONF_SERVICE_UUID): ble_device_base.bt_uuid}, + ), + cv.Optional( + CONF_ON_BLE_MANUFACTURER_DATA_ADVERTISE + ): ble_automation.uuid_trigger_schema( + BLEManufacturerDataAdvertiseTrigger, + {cv.Required(CONF_MANUFACTURER_ID): ble_device_base.bt_uuid}, + ), + cv.Optional(CONF_ON_SCAN_END): ble_automation.scan_end_trigger_schema( + BLEEndOfScanTrigger + ), } ).extend(cv.COMPONENT_SCHEMA) -# Runs at FINAL priority so every BLE sensor has registered through -# ble_device_base (and any tracker-owned listeners have been counted) before -# the StaticVector size is emitted. Same pattern as esp32_ble_tracker. -@coroutine_with_priority(CoroPriority.FINAL) -async def _emit_listener_count() -> None: - count = ble_device_base.get_listener_count() - if count > 0: - cg.add_define("ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT", count) +@automation.register_action( + "bk72xx_ble_tracker.start_scan", + StartScanAction, + cv.Schema( + { + cv.GenerateID(): cv.use_id(BK72xxBLETracker), + # Optional with no default, unlike esp32_ble_tracker: omitting it + # keeps whatever scan_parameters.continuous configured, instead of + # silently forcing one-shot. + cv.Optional(CONF_CONTINUOUS): cv.templatable(cv.boolean), + } + ), + synchronous=True, +) +async def start_scan_action_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: list, +) -> cg.MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + if (continuous := config.get(CONF_CONTINUOUS)) is not None: + template_ = await cg.templatable(continuous, args, cg.bool_) + cg.add(var.set_continuous(template_)) + return var + + +@automation.register_action( + "bk72xx_ble_tracker.stop_scan", + StopScanAction, + automation.maybe_simple_id( + cv.Schema( + { + cv.GenerateID(): cv.use_id(BK72xxBLETracker), + } + ) + ), + synchronous=True, +) +async def stop_scan_action_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: list, +) -> cg.MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + return var async def to_code(config: ConfigType) -> None: @@ -69,6 +149,9 @@ async def to_code(config: ConfigType) -> None: parent = await cg.get_variable(config[CONF_BK72XX_BLE_ID]) cg.add(var.set_parent(parent)) + # The tracker registers itself as a controller scan listener in setup(); + # request the codegen-sized StaticVector slot for it. + bk72xx_ble.request_scan_listener_slot() # Get notified when an OTA update starts, to pause scanning (esp32_ble_tracker parity) ota.request_ota_state_listeners() @@ -77,6 +160,23 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_scan_interval(ble_device_base.to_ble_units(scan[CONF_INTERVAL]))) cg.add(var.set_scan_window(ble_device_base.to_ble_units(scan[CONF_WINDOW]))) cg.add(var.set_scan_duration(scan[CONF_DURATION].total_milliseconds)) - cg.add(var.set_scan_continuous(scan[CONF_CONTINUOUS])) + cg.add(var.set_configured_continuous(scan[CONF_CONTINUOUS])) - CORE.add_job(_emit_listener_count) + for conf in config.get(CONF_ON_BLE_ADVERTISE, []): + await ble_automation.advertise_trigger_to_code(conf, var) + + for trigger_key, uuid_key, setter_prefix in ( + (CONF_ON_BLE_SERVICE_DATA_ADVERTISE, CONF_SERVICE_UUID, "set_service_uuid"), + ( + CONF_ON_BLE_MANUFACTURER_DATA_ADVERTISE, + CONF_MANUFACTURER_ID, + "set_manufacturer_uuid", + ), + ): + for conf in config.get(trigger_key, []): + await ble_automation.uuid_trigger_to_code( + conf, var, uuid_key, setter_prefix + ) + + for conf in config.get(CONF_ON_SCAN_END, []): + await ble_automation.scan_end_trigger_to_code(conf, var) diff --git a/esphome/components/bk72xx_ble_tracker/automation.h b/esphome/components/bk72xx_ble_tracker/automation.h new file mode 100644 index 0000000000..9017d19d71 --- /dev/null +++ b/esphome/components/bk72xx_ble_tracker/automation.h @@ -0,0 +1,48 @@ +// Automation triggers and actions for bk72xx_ble_tracker: triggers are the +// neutral ble_device_base classes; only the scan-control actions are +// platform-specific. + +#pragma once + +#ifdef USE_LIBRETINY + +#include "bk72xx_ble_tracker.h" + +#include "esphome/core/automation.h" +#include "esphome/core/helpers.h" + +namespace esphome::bk72xx_ble_tracker { + +template class StartScanAction final : public Action, public Parented { + public: + TEMPLATABLE_VALUE(bool, continuous) + void play(const Ts &...x) override { + // With continuous: set, the action wins. Without it, the configured value + // is used - stop_scan() clears the runtime flag permanently, so a bare + // stop_scan/start_scan pair would otherwise never resume continuous mode. + const bool want = + this->continuous_.has_value() ? this->continuous_.value(x...) : this->parent_->configured_continuous(); + if (this->parent_->scan_running()) { + // Same mode on a running scan is a no-op (esp32 parity): re-anchoring + // the duration window here would let a repeated action keep a one-shot + // scan alive forever. A real mode switch re-anchors so a change to + // one-shot runs a full duration from now. + if (want != this->parent_->scan_continuous()) { + this->parent_->set_scan_continuous(want); + this->parent_->restart_scan_duration(); + } + return; + } + this->parent_->set_scan_continuous(want); + this->parent_->start_scan(); + } +}; + +template class StopScanAction final : public Action, public Parented { + public: + void play(const Ts &...x) override { this->parent_->stop_scan(); } +}; + +} // namespace esphome::bk72xx_ble_tracker + +#endif // USE_LIBRETINY diff --git a/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.cpp b/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.cpp index eba440d84d..c859f22c61 100644 --- a/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.cpp +++ b/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.cpp @@ -171,9 +171,11 @@ void BK72xxBLETracker::on_scan_report(const bk72xx_ble::BLEScanReport &report) { ble_device_base::ESPBTDevice device; device.from_scan_result(report.mac, report.rssi, report.addr_type, report.data, report.data_len); bool found = false; - for (auto *listener : this->listeners_) - if (listener->parse_device(device)) + for (auto *listener : this->listeners_) { + if (listener->parse_device(device)) { found = true; + } + } // Mirror esp32_ble_tracker: log a newly-seen device only when nothing claimed // it and the scan is one-shot (continuous scans would spam). if (!found && !this->scan_continuous_) @@ -208,6 +210,17 @@ void BK72xxBLETracker::start_scan() { this->try_start_with_backoff_(millis(), /* force= */ true); } +void BK72xxBLETracker::restart_scan_duration() { + if (!this->scan_running_) + return; + // Re-anchor only the one-shot duration clock. scan_period_start_ (the + // continuous-mode on_scan_end period) is deliberately left alone: a + // start_scan action fired more often than scan_duration_ would otherwise + // suppress on_scan_end indefinitely — and absence detection (ble_rssi's NAN + // publish) rides on that period. + this->scan_start_time_ = millis(); +} + void BK72xxBLETracker::stop_scan() { this->scan_continuous_ = false; this->scan_requested_ = false; // also cancels a pending (not yet successful) start diff --git a/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.h b/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.h index 3d32798622..67e4467c77 100644 --- a/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.h +++ b/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.h @@ -70,7 +70,22 @@ class BK72xxBLETracker : public Component, void set_scan_interval(uint32_t scan_interval) { this->scan_interval_ = scan_interval; } void set_scan_window(uint32_t scan_window) { this->scan_window_ = scan_window; } void set_scan_duration(uint32_t scan_duration) { this->scan_duration_ = scan_duration; } + /// Set from YAML (scan_parameters.continuous); also the value + /// configured_continuous() reports and a bare start_scan action restores. + void set_configured_continuous(bool scan_continuous) { + this->scan_continuous_ = scan_continuous; + this->scan_continuous_configured_ = scan_continuous; + } + /// Runtime control (esp32_ble_tracker lambda parity): does not change the + /// configured value, so configured_continuous() still reports what YAML + /// asked for. void set_scan_continuous(bool scan_continuous) { this->scan_continuous_ = scan_continuous; } + bool scan_continuous() const { return this->scan_continuous_; } + bool configured_continuous() const { return this->scan_continuous_configured_; } + /// Re-anchor the one-shot duration clock of a running scan to now — used + /// when an action changes the scan mode without stopping the radio. The + /// continuous-mode on_scan_end period is deliberately not touched. + void restart_scan_duration(); // ---- Public scan control ---- // Mirrors esp32_ble_tracker: set_scan_continuous() + start_scan() / stop_scan(). @@ -91,7 +106,14 @@ class BK72xxBLETracker : public Component, // controller never solicits scan responses and never merges them; consumers // relying on scan-response fields (device names) get them only where the // receiver merges per address (Home Assistant does). No GATT client either. - return {.active_scan = false, .merges_scan_response = false, .gatt = false}; + // scan_mode_switch stays false for the same reason: with no active-scan + // path there is no mode to switch to. + return {.active_scan = false, .merges_scan_response = false, .gatt = false, .scan_mode_switch = false}; + } + bool request_scan_mode(bool active) override { + // Passive-only controller: a passive request is already honored, an active + // one cannot be. + return !active; } // The controller stores the address LSB-first (BLE convention); the contract // wants printable (MSB-first) order. @@ -127,6 +149,7 @@ class BK72xxBLETracker : public Component, uint32_t scan_window_{48}; // 48 × 0.625 ms = 30 ms (30/100 = 30 %) uint32_t scan_duration_{300000}; bool scan_continuous_{true}; + bool scan_continuous_configured_{true}; // YAML value; stop_scan() must not lose it #ifdef USE_OTA_STATE_LISTENER bool scan_continuous_before_ota_{false}; // continuous mode saved at OTA start, restored on OTA failure bool scan_requested_before_ota_{false}; // pending one-shot latch saved at OTA start, re-latched on OTA failure diff --git a/esphome/components/ble_client/__init__.py b/esphome/components/ble_client/__init__.py index 56ac2ea147..1ef7967fa8 100644 --- a/esphome/components/ble_client/__init__.py +++ b/esphome/components/ble_client/__init__.py @@ -9,6 +9,7 @@ from esphome.const import ( CONF_ID, CONF_MAC_ADDRESS, CONF_NAME, + CONF_NOTIFY, CONF_ON_CONNECT, CONF_ON_DISCONNECT, CONF_SERVICE_UUID, @@ -16,11 +17,46 @@ from esphome.const import ( CONF_VALUE, ) from esphome.core import ID +from esphome.types import ConfigType AUTO_LOAD = ["esp32_ble_client"] CODEOWNERS = ["@buxtronix", "@clydebarrow"] DEPENDENCIES = ["esp32_ble_tracker"] +CONF_DESCRIPTOR_UUID = "descriptor_uuid" +CONF_ON_NOTIFY = "on_notify" + + +def validate_descriptor_not_notify(config: ConfigType) -> ConfigType: + """Reject descriptor_uuid combined with notify or on_notify. + + BLE descriptors cannot send notifications; only characteristics can, and + ESP-IDF has no descriptor variant of esp_ble_gattc_register_for_notify. + """ + if CONF_DESCRIPTOR_UUID in config and ( + config.get(CONF_NOTIFY) or CONF_ON_NOTIFY in config + ): + raise cv.Invalid( + f"'{CONF_DESCRIPTOR_UUID}' cannot be used with '{CONF_NOTIFY}' or " + f"'{CONF_ON_NOTIFY}': BLE descriptors cannot send notifications; remove " + f"'{CONF_DESCRIPTOR_UUID}' to receive characteristic notifications, or " + f"remove '{CONF_NOTIFY}' and '{CONF_ON_NOTIFY}' to poll the descriptor" + ) + return config + + +def notify_from_on_notify(config: ConfigType) -> ConfigType: + """Enable notifications when an on_notify automation is configured. + + The triggers have no registration path of their own; without notify the + automation would validate but never fire. + """ + if CONF_ON_NOTIFY in config and not config[CONF_NOTIFY]: + config = config.copy() + config[CONF_NOTIFY] = True + return config + + ble_client_ns = cg.esphome_ns.namespace("ble_client") BLEClient = ble_client_ns.class_("BLEClient", esp32_ble_client.BLEClientBase) BLEClientNode = ble_client_ns.class_("BLEClientNode") diff --git a/esphome/components/ble_client/sensor/__init__.py b/esphome/components/ble_client/sensor/__init__.py index 0975640ece..7764955d89 100644 --- a/esphome/components/ble_client/sensor/__init__.py +++ b/esphome/components/ble_client/sensor/__init__.py @@ -14,13 +14,16 @@ from esphome.const import ( UNIT_DECIBEL_MILLIWATT, ) -from .. import ble_client_ns +from .. import ( + CONF_DESCRIPTOR_UUID, + CONF_ON_NOTIFY, + ble_client_ns, + notify_from_on_notify, + validate_descriptor_not_notify, +) DEPENDENCIES = ["ble_client"] -CONF_DESCRIPTOR_UUID = "descriptor_uuid" - -CONF_ON_NOTIFY = "on_notify" TYPE_CHARACTERISTIC = "characteristic" TYPE_RSSI = "rssi" @@ -85,6 +88,8 @@ CONFIG_SCHEMA = cv.All( }, lower=True, ), + validate_descriptor_not_notify, + notify_from_on_notify, ) diff --git a/esphome/components/ble_client/sensor/ble_sensor.cpp b/esphome/components/ble_client/sensor/ble_sensor.cpp index 60992f282e..5dbb7e42ed 100644 --- a/esphome/components/ble_client/sensor/ble_sensor.cpp +++ b/esphome/components/ble_client/sensor/ble_sensor.cpp @@ -61,7 +61,7 @@ void BLESensor::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_t ga break; } this->handle = chr->handle; - if (this->descr_uuid_.get_uuid().len > 0) { + if (this->descr_uuid_.is_set()) { auto *descr = chr->get_descriptor(this->descr_uuid_); if (descr == nullptr) { this->status_set_warning(); diff --git a/esphome/components/ble_client/text_sensor/__init__.py b/esphome/components/ble_client/text_sensor/__init__.py index 0f53cccdad..820f60845d 100644 --- a/esphome/components/ble_client/text_sensor/__init__.py +++ b/esphome/components/ble_client/text_sensor/__init__.py @@ -9,13 +9,16 @@ from esphome.const import ( CONF_TRIGGER_ID, ) -from .. import ble_client_ns +from .. import ( + CONF_DESCRIPTOR_UUID, + CONF_ON_NOTIFY, + ble_client_ns, + notify_from_on_notify, + validate_descriptor_not_notify, +) DEPENDENCIES = ["ble_client"] -CONF_DESCRIPTOR_UUID = "descriptor_uuid" - -CONF_ON_NOTIFY = "on_notify" adv_data_t = cg.std_vector.template(cg.uint8) adv_data_t_const_ref = adv_data_t.operator("ref").operator("const") @@ -48,7 +51,9 @@ CONFIG_SCHEMA = cv.All( } ) .extend(cv.polling_component_schema("60s")) - .extend(ble_client.BLE_CLIENT_SCHEMA) + .extend(ble_client.BLE_CLIENT_SCHEMA), + validate_descriptor_not_notify, + notify_from_on_notify, ) diff --git a/esphome/components/ble_client/text_sensor/ble_text_sensor.cpp b/esphome/components/ble_client/text_sensor/ble_text_sensor.cpp index 6f09281922..ed2b0a63a0 100644 --- a/esphome/components/ble_client/text_sensor/ble_text_sensor.cpp +++ b/esphome/components/ble_client/text_sensor/ble_text_sensor.cpp @@ -61,7 +61,7 @@ void BLETextSensor::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_ break; } this->handle = chr->handle; - if (this->descr_uuid_.get_uuid().len > 0) { + if (this->descr_uuid_.is_set()) { auto *descr = chr->get_descriptor(this->descr_uuid_); if (descr == nullptr) { this->status_set_warning(); diff --git a/esphome/components/ble_device_base/__init__.py b/esphome/components/ble_device_base/__init__.py index 8100b41d99..a52286f456 100644 --- a/esphome/components/ble_device_base/__init__.py +++ b/esphome/components/ble_device_base/__init__.py @@ -8,32 +8,43 @@ ESPBLEiBeacon / ESPBTDeviceListener, in ble_device.h) and the tracker contract BLE consumers (sensor components, bluetooth_proxy) bind to whichever tracker the configuration declares via `cv.use_id(BLEHub)` — ESPHome resolves any declared subclass, so there is no platform table here and no dependency in either -direction. A sensor appends inject_ble_hub to its CONFIG_SCHEMA (via cv.All) and -calls register_ble_device() in to_code; a tracker component subclasses BLEHub -(C++ and codegen class). Adding a new BLE chip requires only a new tracker -component. +direction. A sensor extends BLE_DEVICE_SCHEMA in its CONFIG_SCHEMA (so an +explicit ble_hub_id: is a declared key even on strict schemas) and calls +register_ble_device() in to_code; a tracker component subclasses BLEHub (C++ +and codegen class) and MUST call register_hub_provider() at import time — +without it _require_hub rejects configs that bind through the generated id +(an explicit ble_hub_id: bypasses the registry). Adding a new BLE chip +requires only a new in-tree tracker component; out-of-tree BLE hubs are +not supported. AES-CCM decryption for encrypted advertisements is provided portably in ble_aes_ccm.h. """ +from collections.abc import Callable import re import esphome.codegen as cg from esphome.components.const import CONF_WINDOW import esphome.config_validation as cv -from esphome.const import CONF_ACTIVE, CONF_CONTINUOUS, CONF_DURATION, CONF_INTERVAL -from esphome.core import CORE +from esphome.const import ( + CONF_ACTIVE, + CONF_CONTINUOUS, + CONF_DURATION, + CONF_INTERVAL, + KEY_TARGET_PLATFORM, +) +from esphome.core import CORE, ID, KEY_CORE from esphome.types import ConfigType CODEOWNERS = ["@Bl00d-B0b"] CONF_BLE_HUB_ID = "ble_hub_id" -# CORE.data key: number of parsed-advertisement listeners registered in this -# build. Trackers whose codegen sizes storage at compile time (esp32's -# StaticVector count define) read it in their final coroutine. -KEY_BLE_LISTENER_COUNT = "ble_device_base_listener_count" +# Number of parsed-advertisement listeners registered in this build; read via +# cg.get_slot_count() by esp32_ble_tracker's feature coupling. +LISTENER_COUNT_DEFINE = "ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT" + ble_device_base_ns = cg.esphome_ns.namespace("ble_device_base") @@ -45,17 +56,92 @@ BLEHub = ble_device_base_ns.class_("BLEHub") ESPBTDeviceListener = ble_device_base_ns.class_("ESPBTDeviceListener") -def inject_ble_hub(config: ConfigType) -> ConfigType: - """Validator: auto-resolve the configured BLE tracker into the config. +# Config keys that provide a BLEHub, registered by each tracker component at +# import time (a tracker's module is imported iff it can end up in the build). +# Used only to phrase an actionable error when a BLE consumer is configured +# without any tracker — the binding itself resolves any BLEHub subclass and +# needs no platform table. Out-of-tree BLE hubs are not supported; the +# registry and the messages below deal in in-tree trackers only. +_HUB_PROVIDERS: set[str] = set() - Append via cv.All to a BLE consumer's CONFIG_SCHEMA. Uses cv.GenerateID + - cv.use_id(BLEHub): an omitted id resolves to the single declared tracker on - any platform; multiple trackers can be disambiguated with an explicit - ble_hub_id. - """ - return cv.Schema( - {cv.GenerateID(CONF_BLE_HUB_ID): cv.use_id(BLEHub)}, extra=cv.ALLOW_EXTRA - )(config) +# The in-tree trackers per target platform, so the missing-tracker error names +# them even in a fresh process where no tracker module has been imported yet (a +# consumer imports only ble_device_base, so the registry is empty exactly in +# the most common failure: the tracker was simply forgotten). Filtered by the +# current platform so an esp32 config is not told to add a Beken tracker; an +# unknown/absent platform falls back to every in-tree name. +_IN_TREE_HUB_PROVIDERS: dict[str, str] = { + "esp32": "esp32_ble_tracker", + "bk72xx": "bk72xx_ble_tracker", + "rp2": "rp2_ble_tracker", + "ln882x": "ln882h_ble_tracker", +} + + +def register_hub_provider(component: str) -> None: + """Called at import time by every component whose config key declares a BLEHub.""" + _HUB_PROVIDERS.add(component) + + +def _require_hub(value: ID) -> ID: + # Without this check a missing tracker surfaces at ID resolution as + # "Couldn't find any component that can be used for 'ble_device_base::BLEHub'" + # — a C++ class name the user never types. Component final validation cannot + # phrase it better: the ID pass runs first and its error skips all later + # steps. All explicitly configured components are loaded before any schema + # validates, so a registered provider in loaded_integrations is exact here. + if value.id is not None: + # Explicit ble_hub_id: — the user is pointing at a specific hub (the + # multi-hub disambiguation case). Let the ID pass judge it; its error + # names the missing id, which is accurate. + return value + if not _HUB_PROVIDERS & CORE.loaded_integrations: + # Defensive lookup rather than CORE.target_platform: the property + # raises when no platform is registered, and this message must never + # be the thing that crashes. In a real run the platform is always set + # (LoadTargetPlatformValidationStep runs before any other domain), so + # the unfiltered all-platforms fallback is reachable only from tests. + platform = CORE.data.get(KEY_CORE, {}).get(KEY_TARGET_PLATFORM) + if platform is not None and platform not in _IN_TREE_HUB_PROVIDERS: + # Known platform with no in-tree hub (esp8266, host, rtl87xx, …): + # listing the other platforms' trackers would misdirect, and + # out-of-tree BLE hubs are not supported. + raise cv.Invalid( + f"No BLE tracker exists for {platform}; BLE components are " + "not supported on this platform" + ) + in_tree = ( + {tracker} + if (tracker := _IN_TREE_HUB_PROVIDERS.get(platform)) + else set(_IN_TREE_HUB_PROVIDERS.values()) + ) + # in_tree only: _HUB_PROVIDERS is import-time state that outlives + # CORE.reset() in a long-lived process (dashboard), so a tracker from + # an earlier build of another platform must not leak into the message. + # The gate above is immune — loaded_integrations resets per run. + names = ", ".join(sorted(in_tree)) + raise cv.Invalid(f"No BLE tracker configured — add one of: {names}") + return value + + +# Schema fragment binding a consumer to the configured BLE tracker: extend a +# consumer's CONFIG_SCHEMA with this so ble_hub_id: is a declared key — a +# trailing validator after a PREVENT_EXTRA schema would reject the explicit +# form before ever running. An omitted id resolves to the single declared +# tracker on any platform; multiple trackers are disambiguated with an +# explicit ble_hub_id. +BLE_DEVICE_SCHEMA = cv.Schema( + {cv.GenerateID(CONF_BLE_HUB_ID): cv.All(cv.use_id(BLEHub), _require_hub)} +) + + +def rename_legacy_hub_id(component: str) -> Callable[[ConfigType], ConfigType]: + """Transitional alias for the pre-migration binding key: esp32_ble_id -> + ble_hub_id. Warns and auto-migrates until removal; every migrated platform + prepends this to its CONFIG_SCHEMA so existing configs keep validating.""" + return cv.rename_key( + "esp32_ble_id", CONF_BLE_HUB_ID, removed_in="2027.2.0", component=component + ) def request_irk_support() -> None: @@ -64,16 +150,29 @@ def request_irk_support() -> None: cg.add_define("USE_BLE_DEVICE_IRK") -def get_listener_count() -> int: - """Number of parsed listeners registered so far (for tracker codegen).""" - return CORE.data.get(KEY_BLE_LISTENER_COUNT, 0) +# Number of GATT client connection slots in this build; sizes the platform +# backend's connection storage. +GATT_CLIENT_COUNT_DEFINE = "ESPHOME_BLE_GATT_CLIENT_COUNT" + +_request_gatt_connection_slot = cg.slot_counter(GATT_CLIENT_COUNT_DEFINE) + + +def request_gatt_client() -> None: + """Compile in the neutral GATT client contract (ble_gatt_client.h) and + claim one connection slot. Called by bluetooth_proxy once per connection + it instantiates on a hub platform.""" + cg.add_define("USE_BLE_GATT_CLIENT") + _request_gatt_connection_slot() + + +_request_listener_slot = cg.slot_counter(LISTENER_COUNT_DEFINE) async def register_ble_device(var: cg.MockObj, config: ConfigType) -> cg.MockObj: """Register `var` as a parsed-advertisement listener on the configured hub.""" hub = await cg.get_variable(config[CONF_BLE_HUB_ID]) cg.add(hub.register_listener(var)) - CORE.data[KEY_BLE_LISTENER_COUNT] = CORE.data.get(KEY_BLE_LISTENER_COUNT, 0) + 1 + _request_listener_slot() return var @@ -220,3 +319,25 @@ def as_hex_array(value: str) -> cg.RawExpression: def as_reversed_hex_array(value: str) -> cg.RawExpression: return _hex_array_expression(value, reverse=True) + + +def add_service_uuid(var: cg.MockObj, service_uuid: str) -> None: + """Emit the width-matched service-UUID setter for a consumer. + + 16-/32-bit UUIDs go out as plain hex literals, 128-bit as a reversed byte + array (BLE wire order). Shared here so every sensor platform dispatches the + same way instead of carrying its own if/elif copy. + """ + if len(service_uuid) == len(BT_UUID16_FORMAT): + cg.add(var.set_service_uuid16(as_hex(service_uuid))) + elif len(service_uuid) == len(BT_UUID32_FORMAT): + cg.add(var.set_service_uuid32(as_hex(service_uuid))) + elif len(service_uuid) == len(BT_UUID128_FORMAT): + cg.add(var.set_service_uuid128(as_reversed_hex_array(service_uuid))) + else: + # bt_uuid restricts lengths to exactly these three formats; if that + # ever loosens, fail the build instead of emitting no setter (a + # sensor whose match_by_ is unset silently never matches). ValueError, + # not cv.Invalid: this runs from to_code, after validation, where + # voluptuous errors surface as raw tracebacks. + raise ValueError(f"Unsupported UUID format: {service_uuid}") diff --git a/esphome/components/ble_device_base/automation.h b/esphome/components/ble_device_base/automation.h new file mode 100644 index 0000000000..507b3278d9 --- /dev/null +++ b/esphome/components/ble_device_base/automation.h @@ -0,0 +1,117 @@ +// Platform-neutral BLE advertisement triggers: ESPBTDeviceListener subclasses +// registered on a BLEHub, exposed by each tracker under its own automation +// names. parse_device()'s return feeds the "Found device" suppression. + +#pragma once + +#include "ble_device.h" +#include "ble_hub.h" + +#include "esphome/core/automation.h" +#include "esphome/core/helpers.h" + +#include +#include + +namespace esphome::ble_device_base { + +// on_ble_advertise: fires on every BLE advertisement, optionally filtered to one or more MACs. +class ESPBTAdvertiseTrigger final : public Trigger, public ESPBTDeviceListener { + public: + explicit ESPBTAdvertiseTrigger(BLEHub *parent) { parent->register_listener(this); } + + void set_addresses(std::initializer_list addresses) { this->addresses_ = addresses; } + + bool parse_device(const ESPBTDevice &device) override { + if (!this->addresses_.empty() && std::find(this->addresses_.begin(), this->addresses_.end(), + device.address_uint64()) == this->addresses_.end()) { + return false; + } + this->trigger(device); + return true; + } + + protected: + FixedVector addresses_; +}; + +// on_ble_service_data_advertise: fires when an advertisement contains service +// data for the given UUID. Optional single-MAC filter. +class BLEServiceDataAdvertiseTrigger final : public Trigger, public ESPBTDeviceListener { + public: + explicit BLEServiceDataAdvertiseTrigger(BLEHub *parent) { parent->register_listener(this); } + + void set_service_uuid16(uint64_t uuid) { this->uuid_ = ESPBTUUID::from_uint16(static_cast(uuid)); } + void set_service_uuid32(uint64_t uuid) { this->uuid_ = ESPBTUUID::from_uint32(static_cast(uuid)); } + void set_service_uuid128(const uint8_t *uuid) { this->uuid_ = ESPBTUUID::from_raw(uuid); } + + void set_address(uint64_t address) { + this->address_ = address; + this->has_address_ = true; + } + + bool parse_device(const ESPBTDevice &device) override { + if (this->has_address_ && device.address_uint64() != this->address_) { + return false; + } + for (const auto &sd : device.get_service_datas()) { + if (sd.uuid == this->uuid_) { + this->trigger(sd.data); + return true; + } + } + return false; + } + + protected: + ESPBTUUID uuid_{}; + uint64_t address_{0}; + bool has_address_{false}; +}; + +// on_ble_manufacturer_data_advertise: fires when an advertisement contains +// manufacturer data for the given ID. Optional single-MAC filter. +class BLEManufacturerDataAdvertiseTrigger final : public Trigger, public ESPBTDeviceListener { + public: + explicit BLEManufacturerDataAdvertiseTrigger(BLEHub *parent) { parent->register_listener(this); } + + void set_manufacturer_uuid16(uint64_t uuid) { this->uuid_ = ESPBTUUID::from_uint16(static_cast(uuid)); } + void set_manufacturer_uuid32(uint64_t uuid) { this->uuid_ = ESPBTUUID::from_uint32(static_cast(uuid)); } + void set_manufacturer_uuid128(const uint8_t *uuid) { this->uuid_ = ESPBTUUID::from_raw(uuid); } + + void set_address(uint64_t address) { + this->address_ = address; + this->has_address_ = true; + } + + bool parse_device(const ESPBTDevice &device) override { + if (this->has_address_ && device.address_uint64() != this->address_) { + return false; + } + for (const auto &md : device.get_manufacturer_datas()) { + if (md.uuid == this->uuid_) { + this->trigger(md.data); + return true; + } + } + return false; + } + + protected: + ESPBTUUID uuid_{}; + uint64_t address_{0}; + bool has_address_{false}; +}; + +// on_scan_end: fires whenever a scan period ends (duration elapsed or stop +// requested). A listener whose on_scan_end() hook fires the trigger — never +// claims devices (parse_device always returns false). +class BLEEndOfScanTrigger final : public Trigger<>, public ESPBTDeviceListener { + public: + explicit BLEEndOfScanTrigger(BLEHub *parent) { parent->register_listener(this); } + + bool parse_device(const ESPBTDevice &device) override { return false; } + void on_scan_end() override { this->trigger(); } +}; + +} // namespace esphome::ble_device_base diff --git a/esphome/components/ble_device_base/automation.py b/esphome/components/ble_device_base/automation.py new file mode 100644 index 0000000000..eb63061a8d --- /dev/null +++ b/esphome/components/ble_device_base/automation.py @@ -0,0 +1,128 @@ +"""Shared codegen for the neutral BLE advertisement triggers (automation.h).""" + +from typing import Any + +from esphome import automation +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome.const import CONF_MAC_ADDRESS, CONF_TRIGGER_ID +from esphome.cpp_generator import MockObjClass +from esphome.types import ConfigType + +from . import ( + BT_UUID16_FORMAT, + BT_UUID32_FORMAT, + BT_UUID128_FORMAT, + LISTENER_COUNT_DEFINE, + as_hex, + as_reversed_hex_array, + ble_device_base_ns, +) + +adv_data_t = cg.std_vector.template(cg.uint8) +adv_data_t_const_ref = adv_data_t.operator("ref").operator("const") +ESPBTDeviceConstRef = ( + ble_device_base_ns.class_("ESPBTDevice").operator("ref").operator("const") +) + +ESPBTAdvertiseTrigger = ble_device_base_ns.class_( + "ESPBTAdvertiseTrigger", automation.Trigger.template(ESPBTDeviceConstRef) +) +BLEServiceDataAdvertiseTrigger = ble_device_base_ns.class_( + "BLEServiceDataAdvertiseTrigger", automation.Trigger.template(adv_data_t_const_ref) +) +BLEManufacturerDataAdvertiseTrigger = ble_device_base_ns.class_( + "BLEManufacturerDataAdvertiseTrigger", + automation.Trigger.template(adv_data_t_const_ref), +) +BLEEndOfScanTrigger = ble_device_base_ns.class_( + "BLEEndOfScanTrigger", automation.Trigger.template() +) + +# UUID string length -> setter width. 16/32-bit go out as plain hex literals, +# 128-bit as a reversed byte array (BLE wire order). Keyed exhaustively so an +# impossible length fails as a KeyError instead of silently picking a width +# (bt_uuid validation upstream only ever produces these three). +_UUID_WIDTHS = { + len(BT_UUID16_FORMAT): "16", + len(BT_UUID32_FORMAT): "32", + len(BT_UUID128_FORMAT): "128", +} + + +def uuid_trigger_schema( + trigger_class: MockObjClass, extra: dict[Any, Any] | None = None +): + """Schema for a UUID-filtered trigger — pairs with uuid_trigger_to_code(). + + `extra` carries the required UUID key (a cv marker, so a dict rather than + **kwargs); the optional single-mac filter is what uuid_trigger_to_code() + reads back. + """ + return automation.validate_automation( + { + cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(trigger_class), + cv.Optional(CONF_MAC_ADDRESS): cv.mac_address, + **(extra or {}), + } + ) + + +def advertise_trigger_schema(trigger_class: MockObjClass): + """on_ble_advertise schema: multi-mac list filter, unlike the single-mac + uuid_trigger_schema() — pairs with advertise_trigger_to_code().""" + return automation.validate_automation( + { + cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(trigger_class), + cv.Optional(CONF_MAC_ADDRESS): cv.ensure_list(cv.mac_address), + } + ) + + +def scan_end_trigger_schema(trigger_class: MockObjClass): + """on_scan_end schema: id only — pairs with scan_end_trigger_to_code().""" + return automation.validate_automation( + {cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(trigger_class)} + ) + + +# Triggers register as ble_device_base listeners in their constructors; count +# them where they are created so no backend can undercount the StaticVector +# (push_back past capacity drops silently). Shares the define with +# register_ble_device() via the core slot-counter factory. +_count_listener = cg.slot_counter(LISTENER_COUNT_DEFINE) + + +async def advertise_trigger_to_code(conf: ConfigType, var: cg.MockObj) -> None: + """Build an on_ble_advertise trigger (optional multi-mac filter).""" + trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) + if (macs := conf.get(CONF_MAC_ADDRESS)) is not None: + cg.add(trigger.set_addresses([it.as_hex for it in macs])) + await automation.build_automation(trigger, [(ESPBTDeviceConstRef, "x")], conf) + _count_listener() + + +async def scan_end_trigger_to_code(conf: ConfigType, var: cg.MockObj) -> None: + """Build an on_scan_end trigger.""" + trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) + await automation.build_automation(trigger, [], conf) + _count_listener() + + +async def uuid_trigger_to_code( + conf: ConfigType, var: cg.MockObj, key: str, setter_prefix: str +) -> None: + """Build a UUID-filtered advertise trigger. + + The UUID width picks the setter: 16-/32-bit go out as a plain hex literal, + 128-bit as a reversed byte array (BLE wire order). + """ + trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) + uuid = conf[key] + width = _UUID_WIDTHS[len(uuid)] + value = as_hex(uuid) if width != "128" else as_reversed_hex_array(uuid) + cg.add(getattr(trigger, f"{setter_prefix}{width}")(value)) + if (mac := conf.get(CONF_MAC_ADDRESS)) is not None: + cg.add(trigger.set_address(mac.as_hex)) + await automation.build_automation(trigger, [(adv_data_t_const_ref, "x")], conf) + _count_listener() diff --git a/esphome/components/ble_device_base/ble_client_state.cpp b/esphome/components/ble_device_base/ble_client_state.cpp new file mode 100644 index 0000000000..55817024b5 --- /dev/null +++ b/esphome/components/ble_device_base/ble_client_state.cpp @@ -0,0 +1,26 @@ +#include "ble_client_state.h" + +namespace esphome::ble_device_base { + +const char *client_state_to_string(ClientState state) { + switch (state) { + case ClientState::INIT: + return "INIT"; + case ClientState::DISCONNECTING: + return "DISCONNECTING"; + case ClientState::IDLE: + return "IDLE"; + case ClientState::DISCOVERED: + return "DISCOVERED"; + case ClientState::CONNECTING: + return "CONNECTING"; + case ClientState::CONNECTED: + return "CONNECTED"; + case ClientState::ESTABLISHED: + return "ESTABLISHED"; + default: + return "UNKNOWN"; + } +} + +} // namespace esphome::ble_device_base diff --git a/esphome/components/ble_device_base/ble_client_state.h b/esphome/components/ble_device_base/ble_client_state.h new file mode 100644 index 0000000000..58f7d84fad --- /dev/null +++ b/esphome/components/ble_device_base/ble_client_state.h @@ -0,0 +1,53 @@ +// ble_client_state.h +// +// Platform-neutral GATT client connection state types, shared by every +// platform's GATT client implementation (esp32_ble_client, bluetooth_connection +// backends). Moved here from esp32_ble_tracker, which re-exports them under its +// own namespace for backward compatibility. + +#pragma once + +#include + +namespace esphome::ble_device_base { + +/// ESPHome-private errors for the API's plain-int error fields, outside the +/// ATT code range so they cannot be mistaken for spec errors. -1 is +/// understood by API clients as "not connected". Shared by every GATT +/// client backend. +static constexpr int GATT_ERR_NOT_CONNECTED = -1; +static constexpr int GATT_ERR_NO_MEMORY = -2; + +enum class ClientState : uint8_t { + // Connection is allocated + INIT, + // Client is disconnecting + DISCONNECTING, + // Connection is idle, no device detected. + IDLE, + // Device advertisement found. + DISCOVERED, + // Connection in progress. + CONNECTING, + // Initial connection established. + CONNECTED, + // The client and sub-clients have completed setup. + ESTABLISHED, +}; + +// Helper function to convert ClientState to string +const char *client_state_to_string(ClientState state); + +enum class ConnectionType : uint8_t { + // The default connection type, we hold all the services in ram + // for the duration of the connection. + V1, + // The client has a cache of the services and mtu so we should not + // fetch them again + V3_WITH_CACHE, + // The client does not need the services and mtu once we send them + // so we should wipe them from memory as soon as we send them + V3_WITHOUT_CACHE +}; + +} // namespace esphome::ble_device_base diff --git a/esphome/components/ble_device_base/ble_device.cpp b/esphome/components/ble_device_base/ble_device.cpp index 9c3e1d4397..fc5bf5c1e0 100644 --- a/esphome/components/ble_device_base/ble_device.cpp +++ b/esphome/components/ble_device_base/ble_device.cpp @@ -8,6 +8,7 @@ #include "ble_aes_ccm.h" #include "esphome/core/defines.h" +#include "esphome/core/hal.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" @@ -98,6 +99,8 @@ ESPBTUUID ESPBTUUID::from_raw(const char *data, size_t length) { #ifdef USE_ESP32 ESPBTUUID ESPBTUUID::from_uuid(esp_bt_uuid_t uuid) { + if (uuid.len == 0) // the unset sentinel get_uuid() emits + return {}; if (uuid.len == ESP_UUID_LEN_16) return ESPBTUUID::from_uint16(uuid.uuid.uuid16); if (uuid.len == ESP_UUID_LEN_32) @@ -108,6 +111,10 @@ ESPBTUUID ESPBTUUID::from_uuid(esp_bt_uuid_t uuid) { esp_bt_uuid_t ESPBTUUID::get_uuid() const { esp_bt_uuid_t ret; switch (this->type_) { + case Type::UNSET: + ret.len = 0; + memset(&ret.uuid, 0, sizeof(ret.uuid)); + break; case Type::UUID16: ret.len = ESP_UUID_LEN_16; ret.uuid.uuid16 = this->uuid_.uuid16; @@ -129,7 +136,7 @@ void ESPBTDevice::parse_scan_rst(const esp32_ble::BLEScanResult &scan_result) { this->scan_result_ = &scan_result; // BLEScanResult's bda is most-significant octet first; the neutral ingest // takes the BLE controller (LSB-first) order, so reverse — address_uint64()/ - // address_str() then produce exactly the historical esp32 values. + // address_str_to() then produce exactly the historical esp32 values. uint8_t mac_lsb_first[6]; for (uint8_t i = 0; i < 6; i++) mac_lsb_first[i] = scan_result.bda[5 - i]; @@ -139,7 +146,8 @@ void ESPBTDevice::parse_scan_rst(const esp32_ble::BLEScanResult &scan_result) { #endif // USE_ESP32 ESPBTUUID ESPBTUUID::as_128bit() const { - if (this->type_ == Type::UUID128) + // Widening an unset UUID stays unset; expanding it would produce a set 0x0000 base UUID. + if (this->type_ == Type::UNSET || this->type_ == Type::UUID128) return *this; uint8_t data[16]; this->to_128bit_(data); @@ -149,6 +157,8 @@ ESPBTUUID ESPBTUUID::as_128bit() const { bool ESPBTUUID::contains(uint8_t data1, uint8_t data2) const { // Adjacent byte-pair search — identical semantics to esp32_ble::ESPBTUUID::contains. switch (this->type_) { + case Type::UNSET: + return false; case Type::UUID16: return (this->uuid_.uuid16 >> 8) == data2 && (this->uuid_.uuid16 & 0xFF) == data1; case Type::UUID32: @@ -173,6 +183,9 @@ const char *ESPBTUUID::to_str(char *buf) const { // Identical output format to esp32_ble::ESPBTUUID::to_str. char *pos = buf; switch (this->type_) { + case Type::UNSET: + memcpy(buf, "None", 5); + return buf; case Type::UUID16: *pos++ = '0'; *pos++ = 'x'; @@ -207,6 +220,7 @@ const char *ESPBTUUID::to_str(char *buf) const { void ESPBTUUID::to_128bit_(uint8_t out[16]) const { // Bluetooth Base UUID 00000000-0000-1000-8000-00805F9B34FB (LSB-first), with the 16/32-bit // value placed at bytes 12..; identical expansion to esp32_ble::ESPBTUUID::as_128bit(). + // Callers screen out UNSET first (operator==, as_128bit); it would expand like 0x0000. static const uint8_t BASE[16] = {0xFB, 0x34, 0x9B, 0x5F, 0x80, 0x00, 0x00, 0x80, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; if (this->type_ == Type::UUID128) { @@ -223,6 +237,8 @@ void ESPBTUUID::to_128bit_(uint8_t out[16]) const { bool ESPBTUUID::operator==(const ESPBTUUID &other) const { if (this->type_ == other.type_) { switch (this->type_) { + case Type::UNSET: + return true; case Type::UUID16: return this->uuid_.uuid16 == other.uuid_.uuid16; case Type::UUID32: @@ -232,6 +248,9 @@ bool ESPBTUUID::operator==(const ESPBTUUID &other) const { } return false; } + // Unset never equals a set UUID; 0x0000 is a valid value, distinct from "not configured". + if (this->type_ == Type::UNSET || other.type_ == Type::UNSET) + return false; // Different widths: expand both to the 128-bit Bluetooth Base UUID form and compare, so a // configured 16/32-bit UUID matches the equivalent 128-bit advertisement (esp32 parity). uint8_t a[16]; @@ -247,16 +266,21 @@ bool ESPBTUUID::operator==(const ESPBTUUID &other) const { ESPBLEiBeacon::ESPBLEiBeacon(const uint8_t *data) { memcpy(&this->beacon_data_, data, sizeof(this->beacon_data_)); } -optional ESPBLEiBeacon::from_manufacturer_data(const ServiceData &data) { +optional ESPBLEiBeacon::from_manufacturer_data(const ServiceData &data, bool *prefix_rejected) { // iBeacon manufacturer specific data (after company-ID bytes have been stripped): // [0x02][0x15][16-byte UUID][2-byte major][2-byte minor][1-byte power] = exactly 23 bytes - // Parity with esp32_ble_tracker: gate on the Apple company ID and length only. - // (Checking the 0x02/0x15 sub-type prefix would be stricter, but is a behavior - // change; it belongs to a follow-up, not this refactor.) if (!data.uuid.contains(0x4C, 0x00)) // Apple company ID 0x004C return {}; if (data.data.size() != 23) return {}; + // Require the iBeacon sub-type/length prefix — stricter than the legacy + // esp32 parser, which accepted any 23-byte Apple payload and surfaced + // non-iBeacon frames as garbage beacons. + if (data.data[0] != 0x02 || data.data[1] != 0x15) { + if (prefix_rejected != nullptr) + *prefix_rejected = true; + return {}; + } return ESPBLEiBeacon(data.data.data()); } @@ -264,6 +288,44 @@ optional ESPBLEiBeacon::from_manufacturer_data(const ServiceData // ESPBTDevice // --------------------------------------------------------------------------- +optional ESPBTDevice::get_ibeacon() const { + bool prefix_rejected = false; + uint8_t rejected_sub_type = 0; + uint8_t rejected_len = 0; + for (const auto &it : this->manufacturer_datas_) { + bool rejected = false; + auto res = ESPBLEiBeacon::from_manufacturer_data(it, &rejected); + if (res.has_value()) + return res; + if (rejected && !prefix_rejected) { + prefix_rejected = true; + rejected_sub_type = it.data[0]; + rejected_len = it.data[1]; + } + } + if (prefix_rejected) { + // Only when no beacon was found at all: these frames were accepted before + // the prefix check, so their disappearance must be observable at the + // default log level. Throttled so a chatty non-iBeacon Apple advertiser + // cannot flood the log; a different address may bypass the shared window + // so that advertiser cannot mask the device that actually regressed — but + // with a 1 s floor, or two alternating advertisers log every frame. + static uint32_t last_log = 0; + static uint64_t last_addr = 0; + const uint32_t now = millis(); + const uint64_t addr = this->address_uint64(); + const uint32_t since = now - last_log; + if (last_log == 0 || since > 60000 || (addr != last_addr && since > 1000)) { + last_log = now; + last_addr = addr; + char addr_buf[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; + ESP_LOGD(TAG, "%s: 23-byte Apple frame without iBeacon prefix ignored (sub-type 0x%02X len 0x%02X)", + this->address_str_to(addr_buf), rejected_sub_type, rejected_len); + } + } + return {}; +} + const char *ESPBTDevice::address_type_str() const { switch (this->address_type_) { case BLE_ADDR_TYPE_PUBLIC: @@ -287,7 +349,8 @@ void ESPBTDevice::from_scan_result(const uint8_t *mac, int rssi, uint8_t addr_ty this->address_[i] = mac[5 - i]; this->address_type_ = addr_type; this->rssi_ = rssi; - this->name_.clear(); + this->name_len_ = 0; + this->name_[0] = '\0'; this->service_uuids_.clear(); this->manufacturer_datas_.clear(); this->service_datas_.clear(); @@ -303,7 +366,7 @@ void ESPBTDevice::from_scan_result(const uint8_t *mac, int rssi, uint8_t addr_ty " Address: %s (%s)\n" " RSSI: %d\n" " Name: '%s'", - this->address_str_to(addr_buf), this->address_type_str(), this->rssi_, this->name_.c_str()); + this->address_str_to(addr_buf), this->address_type_str(), this->rssi_, this->name_); for (auto &it : this->tx_powers_) { ESP_LOGVV(TAG, " TX Power: %d", it); } @@ -346,6 +409,7 @@ void ESPBTDevice::from_scan_result(const uint8_t *mac, int rssi, uint8_t addr_ty #endif // ESPHOME_LOG_HAS_VERY_VERBOSE } +// Remove before 2027.2.0 std::string ESPBTDevice::address_str() const { char buf[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; return std::string(this->address_str_to(buf)); @@ -414,8 +478,12 @@ void ESPBTDevice::parse_adv_(const uint8_t *payload, uint16_t len) { // Keep the longest name seen — a merged adv + scan-response frame may carry both the // shortened and the complete name, and the shortened form must never replace the // complete one (same rule as esp32_ble_tracker's parse_adv_). - if (ad_data_len > this->name_.length()) - this->name_.assign(reinterpret_cast(ad_data), ad_data_len); + if (ad_data_len > this->name_len_) { + uint8_t name_len = ad_data_len > MAX_ADV_NAME_LEN ? MAX_ADV_NAME_LEN : static_cast(ad_data_len); + memcpy(this->name_, ad_data, name_len); + this->name_[name_len] = '\0'; + this->name_len_ = name_len; + } break; case 0x0A: // TX Power Level diff --git a/esphome/components/ble_device_base/ble_device.h b/esphome/components/ble_device_base/ble_device.h index 2d2cb5796b..fba1fe2347 100644 --- a/esphome/components/ble_device_base/ble_device.h +++ b/esphome/components/ble_device_base/ble_device.h @@ -14,6 +14,7 @@ #include "esphome/core/defines.h" #include "esphome/core/helpers.h" +#include "esphome/core/string_ref.h" #include #include @@ -85,14 +86,17 @@ class ESPBTUUID { bool operator==(const ESPBTUUID &other) const; bool operator!=(const ESPBTUUID &other) const { return !(*this == other); } - /// Write "0xABCD" / "0xABCDEF01" / the dashed 128-bit form into buf - /// (>= UUID_STR_LEN bytes) and return buf. + /// Write "0xABCD" / "0xABCDEF01" / the dashed 128-bit form, or "None" for an + /// unset UUID, into buf (>= UUID_STR_LEN bytes) and return buf. const char *to_str(char *buf) const; #if defined(__cpp_lib_span) const char *to_str(std::span output) const { return this->to_str(output.data()); } #endif - enum class Type : uint8_t { UUID16, UUID32, UUID128 }; + // UNSET is the default-constructed state; get_uuid() reports it as len 0 (the historical sentinel). + enum class Type : uint8_t { UNSET, UUID16, UUID32, UUID128 }; Type type() const { return this->type_; } + /// True if a UUID has been configured (not default-constructed). + bool is_set() const { return this->type_ != Type::UNSET; } uint16_t uuid16() const { return this->uuid_.uuid16; } uint32_t uuid32() const { return this->uuid_.uuid32; } const uint8_t *uuid128() const { return this->uuid_.uuid128; } @@ -101,7 +105,7 @@ class ESPBTUUID { // Expand to the 128-bit Bluetooth Base UUID byte form (out is 16 bytes, little-endian). void to_128bit_(uint8_t out[16]) const; - Type type_{Type::UUID16}; + Type type_{Type::UNSET}; union { uint16_t uuid16; uint32_t uuid32; @@ -126,7 +130,12 @@ class ESPBLEiBeacon { public: ESPBLEiBeacon() { memset(&this->beacon_data_, 0, sizeof(this->beacon_data_)); } explicit ESPBLEiBeacon(const uint8_t *data); - static optional from_manufacturer_data(const ServiceData &data); + /// prefix_rejected: caller must initialise to false; set to true ONLY when a + /// 23-byte Apple frame was refused for lacking the 0x02/0x15 iBeacon prefix — + /// the case the legacy esp32 parser accepted. Never written on accept or on + /// the non-Apple/wrong-size rejects. The caller with the device address does + /// the logging (see ESPBTDevice::get_ibeacon()). + static optional from_manufacturer_data(const ServiceData &data, bool *prefix_rejected = nullptr); uint16_t get_major() const { return byteswap(this->beacon_data_.major); } uint16_t get_minor() const { return byteswap(this->beacon_data_.minor); } @@ -158,6 +167,13 @@ inline uint64_t mac_lsb_first_to_uint64(const uint8_t *mac) { return addr; } +/// Unpack a uint64 BLE address into printable (MSB-first) byte order — +/// the order bd_addr_t / esp_bd_addr_t style APIs expect. +inline void uint64_to_mac_msb_first(uint64_t address, uint8_t out[6]) { + for (int i = 0; i < 6; i++) + out[i] = (address >> ((5 - i) * 8)) & 0xFF; +} + // --------------------------------------------------------------------------- // ESPBTDevice — parsed BLE advertisement // --------------------------------------------------------------------------- @@ -172,8 +188,9 @@ class ESPBTDevice { static constexpr size_t MAC_ADDRESS_PRETTY_BUFFER_SIZE = esphome::MAC_ADDRESS_PRETTY_BUFFER_SIZE; /// Return MAC as "XX:XX:XX:XX:XX:XX" string. + ESPDEPRECATED("Use address_str_to() instead. Removed in 2027.2.0.", "2026.8.0") std::string address_str() const; - /// Buffer overload: writes "XX:XX:XX:XX:XX:XX\0" into buf (>= 18 bytes), returns buf. + /// Writes "XX:XX:XX:XX:XX:XX\0" into buf (>= MAC_ADDRESS_PRETTY_BUFFER_SIZE bytes), returns buf. const char *address_str_to(char *buf) const; #if defined(__cpp_lib_span) const char *address_str_to(std::span buf) const { @@ -189,6 +206,8 @@ class ESPBTDevice { // Historical esp32 signature: consumers assign the result to esp_ble_addr_type_t. esp_ble_addr_type_t get_address_type() const { return static_cast(this->address_type_); } /// Historical esp32 ingest (esp32 builds only): parse an ESP-IDF scan result. + /// Prefer ESPBTDevice::from_scan_result(); deprecation is a follow-up pending + /// consumer feedback on the raw scan-result fields. void parse_scan_rst(const esp32_ble::BLEScanResult &scan_result); // Exposed through a function for use in lambdas const esp32_ble::BLEScanResult &get_scan_result() const { return *scan_result_; } @@ -200,7 +219,9 @@ class ESPBTDevice { const char *address_type_str() const; int get_rssi() const { return rssi_; } - const std::string &get_name() const { return name_; } + /// Advertised name as a view into the fixed buffer (always NUL-terminated, + /// so c_str() is safe); converts implicitly to std::string where needed. + StringRef get_name() const { return StringRef(this->name_, this->name_len_); } const std::vector &get_service_uuids() const { return service_uuids_; } const std::vector &get_manufacturer_datas() const { return manufacturer_datas_; } @@ -214,22 +235,22 @@ class ESPBTDevice { /// decryptor; compiled only when a sensor configures irk: (request_irk_support). bool resolve_irk(const uint8_t *irk) const; - optional get_ibeacon() const { - for (const auto &it : this->manufacturer_datas_) { - auto res = ESPBLEiBeacon::from_manufacturer_data(it); - if (res.has_value()) - return res; - } - return {}; - } + optional get_ibeacon() const; protected: void parse_adv_(const uint8_t *payload, uint16_t len); + // Max name bytes in a legacy advertisement AD element (31-byte PDU minus + // the 2-byte element header); every in-tree tracker scans legacy PDUs only. + static constexpr uint8_t MAX_ADV_NAME_LEN = 29; + uint8_t address_[6]{0}; uint8_t address_type_{0}; int rssi_{0}; - std::string name_{}; + // Fixed buffer instead of std::string: no per-advertisement heap churn on + // the scan path, and no libstdc++ string/exception machinery in the image. + char name_[MAX_ADV_NAME_LEN + 1]{}; + uint8_t name_len_{0}; std::vector service_uuids_{}; std::vector manufacturer_datas_{}; std::vector service_datas_{}; diff --git a/esphome/components/ble_device_base/ble_gatt_client.h b/esphome/components/ble_device_base/ble_gatt_client.h new file mode 100644 index 0000000000..ef28f7672a --- /dev/null +++ b/esphome/components/ble_device_base/ble_gatt_client.h @@ -0,0 +1,140 @@ +// ble_gatt_client.h +// +// Platform-neutral GATT client connection contract. +// +// A platform's GATT client backend (bluetooth_connection/esp32, +// bluetooth_connection/rp2) implements BLEGattConnection; consumers +// (bluetooth_proxy) drive it through this interface and receive +// completions through GattClientEventListener. All listener callbacks are +// delivered on the ESPHome main loop; borrowed data pointers are valid only +// for the duration of the call. +// +// Error domain (plain int, forwarded to the API without translation): +// 0 success +// 1..0x11 ATT error codes (Bluetooth spec; BTstack and Bluedroid agree) +// GATT_ERR_NOT_CONNECTED (-1) no connection to the peer (on esp32 a raw +// ESP_FAIL from the stack shares this value; both read as a +// failed, unusable connection on the client side) +// GATT_ERR_NO_MEMORY (-2) backend storage exhausted +// anything else: platform stack error/status code, surfaced opaquely. +// Connection events carry HCI status/disconnect reason codes (same code +// space on every controller). + +#pragma once + +#include "esphome/core/defines.h" + +#ifdef USE_BLE_GATT_CLIENT + +#include "ble_client_state.h" +#include "ble_device.h" + +#include + +namespace esphome::ble_device_base { + +// Materialized GATT database of a connected peer, discovered by the backend +// and streamed to the API by the consumer. Flat arrays with index ranges +// (not pointers): a service owns characteristics +// [first_characteristic, first_characteristic + characteristic_count) and a +// characteristic owns descriptors [first_descriptor, ...) — discovery is +// depth-first, so the ranges are naturally contiguous. +struct GattDescriptor { + ESPBTUUID uuid; + uint16_t handle; +}; + +struct GattCharacteristic { + ESPBTUUID uuid; + uint16_t value_handle; + // Needed to rebuild the stack's characteristic object for CCCD operations. + uint16_t end_handle; + uint8_t properties; // Bluetooth spec property bitfield + uint16_t first_descriptor; + uint16_t descriptor_count; +}; + +struct GattService { + ESPBTUUID uuid; + uint16_t start_handle; + uint16_t end_handle; + uint16_t first_characteristic; + uint16_t characteristic_count; +}; + +/// Borrowed view of the backend-owned service table. Valid from a successful +/// on_service_discovery_done() until release_services(). Characteristics and +/// descriptors are reached through the per-service/per-characteristic index +/// ranges; the array totals let a consumer bounds-check those ranges instead +/// of trusting the backend's discovery bookkeeping blindly. +struct GattServiceTable { + const GattService *services{nullptr}; + const GattCharacteristic *characteristics{nullptr}; + const GattDescriptor *descriptors{nullptr}; + uint16_t service_count{0}; + uint16_t characteristic_count{0}; + uint16_t descriptor_count{0}; +}; + +/// Completion/event sink for a GATT connection. Implemented by the consumer +/// (bluetooth_proxy's connection wrapper). Every callback runs on the main loop. +class GattClientEventListener { + public: + virtual ~GattClientEventListener() = default; + + /// Connected (with negotiated MTU) or disconnected/connect-failed + /// (error = HCI status or disconnect reason). + virtual void on_connection_state(bool connected, uint16_t mtu, int error) = 0; + /// Service discovery finished; on success the service table is populated. + virtual void on_service_discovery_done(int error) = 0; + /// Characteristic or descriptor read finished. data/len valid during the call. + virtual void on_read_result(uint16_t handle, const uint8_t *data, uint16_t len, int error) = 0; + /// Characteristic write-with-response or descriptor write finished. + virtual void on_write_result(uint16_t handle, int error) = 0; + /// Notification/indication registration state changed. + virtual void on_notify_state(uint16_t handle, bool enabled, int error) = 0; + /// Notification/indication data from the peer. data/len valid during the call. + virtual void on_notify_data(uint16_t handle, const uint8_t *data, uint16_t len) = 0; +}; + +/// One GATT client connection slot. Operations return 0 when accepted +/// (completion arrives via the listener) or a synchronous error code +/// (busy, not connected, stack rejection). One operation may be outstanding +/// at a time; callers see a synchronous error otherwise. +class BLEGattConnection { + public: + virtual ~BLEGattConnection() = default; + + void set_listener(GattClientEventListener *listener) { this->listener_ = listener; } + + /// Start connecting to a peer. addr_type is a BLE_ADDR_TYPE_* constant + /// (ble_device.h). Completion: on_connection_state(). + virtual int connect(uint64_t address, uint8_t addr_type) = 0; + /// Disconnect (or cancel a connect in progress). Completion: on_connection_state(). + virtual int disconnect() = 0; + /// Discover the peer's services/characteristics/descriptors into the + /// service table. Completion: on_service_discovery_done(). + virtual int discover_services() = 0; + virtual int read_characteristic(uint16_t handle) = 0; + virtual int write_characteristic(uint16_t handle, const uint8_t *data, uint16_t len, bool response) = 0; + virtual int read_descriptor(uint16_t handle) = 0; + virtual int write_descriptor(uint16_t handle, const uint8_t *data, uint16_t len) = 0; + /// Enable/disable delivery of on_notify_data() for a characteristic value + /// handle. Local registration only — the CCCD write is the API client's + /// responsibility (it arrives as a plain write_descriptor). + virtual int notify_characteristic(uint16_t handle, bool enable) = 0; + virtual int update_connection_params(uint16_t min_interval, uint16_t max_interval, uint16_t latency, + uint16_t timeout) = 0; + + /// Backend-owned service table (see GattServiceTable lifetime). + virtual GattServiceTable get_service_table() = 0; + /// Free the transient service table storage. Call after streaming. + virtual void release_services() = 0; + + protected: + GattClientEventListener *listener_{nullptr}; +}; + +} // namespace esphome::ble_device_base + +#endif // USE_BLE_GATT_CLIENT diff --git a/esphome/components/ble_device_base/ble_hub.h b/esphome/components/ble_device_base/ble_hub.h index c02b491237..3870e9833e 100644 --- a/esphome/components/ble_device_base/ble_hub.h +++ b/esphome/components/ble_device_base/ble_hub.h @@ -56,9 +56,15 @@ struct HubCapabilities { /// frame. When false, consumers relying on scan-response fields (e.g. names) /// may only see them where the receiver merges per address (Home Assistant does). bool merges_scan_response; - /// GATT client connections are available (today: esp32 only, but a chip SDK - /// gaining GATT support only has to flip this bit). + /// GATT client connections are available: the platform has a + /// bluetooth_connection backend implementing ble_device_base::BLEGattConnection + /// (ble_gatt_client.h). Today: esp32; rp2 follows with its BTstack backend. bool gatt; + /// request_scan_mode() is honored at runtime. Distinct from active_scan: + /// a passive-only controller (bk72xx) can never switch, and a hub may + /// support active scanning yet still refuse the runtime switch + /// (esp32_ble_tracker drives its mode through its own tracker API). + bool scan_mode_switch; }; class BLEHub { @@ -79,6 +85,17 @@ class BLEHub { virtual bool scan_running() = 0; /// True when the current/configured scan mode is active (scan requests sent). virtual bool scan_active() = 0; + /// Request a scan-mode change (active = send scan requests). Returns false + /// when the hub cannot honor the request; the caller reports the real state + /// back to its subscriber. A hub that returns true applies the mode + /// immediately: a running scan is restarted with the new mode, an idle one + /// picks it up on its next start. The default cannot-change keeps hubs + /// without a mode switch (and out-of-tree trackers) building unchanged. + /// Independent of HubCapabilities::active_scan: that bit describes what the + /// CONTROLLER can do; whether this method honors requests is advertised by + /// HubCapabilities::scan_mode_switch, so consumers can gate features on the + /// switch without probing. + virtual bool request_scan_mode(bool active) { return false; } }; } // namespace esphome::ble_device_base diff --git a/esphome/components/ble_scanner/ble_scanner.h b/esphome/components/ble_scanner/ble_scanner.h index 106171d38f..b4e4488646 100644 --- a/esphome/components/ble_scanner/ble_scanner.h +++ b/esphome/components/ble_scanner/ble_scanner.h @@ -23,7 +23,7 @@ class BLEScanner final : public text_sensor::TextSensor, // Escape special characters in the device name for valid JSON. Control characters stay in the \u00XX form this // sensor has always published. char escaped_name[128]; - json_escape_into_buffer(escaped_name, StringRef(device.get_name()), /*short_control_escapes=*/false); + json_escape_into_buffer(escaped_name, device.get_name(), /*short_control_escapes=*/false); char buf[256]; snprintf(buf, sizeof(buf), "{\"timestamp\":%" PRId64 ",\"address\":\"%s\",\"rssi\":%d,\"name\":\"%s\"}", diff --git a/esphome/components/bluetooth_proxy/__init__.py b/esphome/components/bluetooth_proxy/__init__.py index ad7528c156..bb05f1b21f 100644 --- a/esphome/components/bluetooth_proxy/__init__.py +++ b/esphome/components/bluetooth_proxy/__init__.py @@ -1,14 +1,50 @@ +import functools import logging import esphome.codegen as cg -from esphome.components import esp32_ble, esp32_ble_client, esp32_ble_tracker -from esphome.components.esp32 import add_idf_sdkconfig_option -from esphome.components.esp32_ble import BTLoggers +from esphome.components import ble_device_base import esphome.config_validation as cv -from esphome.const import CONF_ACTIVE, CONF_ID +from esphome.const import CONF_ACTIVE, CONF_ID, PLATFORM_LN882X, PLATFORM_RP2 +from esphome.core import CORE +from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor +from esphome.types import ConfigType -AUTO_LOAD = ["esp32_ble_client", "esp32_ble_tracker"] -DEPENDENCIES = ["api", "esp32"] +# The esp32 BLE stack (esp32_ble, esp32_ble_client, esp32_ble_tracker) is +# imported lazily inside _esp32_config_schema()/_to_code_esp32(): importing +# those modules registers esp32-only automations (ble.enable, ble.disable, ...) +# as a side effect, and a module-scope import would leak them into every +# platform's registry the moment a config declares `bluetooth_proxy:` — +# degrading "Unable to find action" config errors into C++ compile failures. + + +def AUTO_LOAD(config: ConfigType | None = None) -> list[str]: + """Components to auto-load for the platform being compiled. + + Callable with no argument so tooling that resolves AUTO_LOAD without a + target platform (the device-builder catalog sync does exactly this) gets + the union of every arm instead of an empty list — which is what lets it + keep cross-referencing the esp32 BLE stack. A real build always has a + target platform set, so it takes one of the concrete branches. + """ + if CORE.is_esp32: + return ["esp32_ble_client", "esp32_ble_tracker"] + if CORE.target_platform in _HUB_PLATFORMS: + return ["ble_device_base"] + # No target platform, or one this component does not support: tooling + # resolving the manifest (including the host-pinned dependency resolver) — + # expose every arm so the closure keeps the esp32 BLE stack. + return ["ble_device_base", "esp32_ble_client", "esp32_ble_tracker"] + + +# Platforms with an in-tree ble_device_base BLE tracker hub whose controller +# supports active scanning. Passive-only hubs (bk72xx) are deliberately NOT +# admitted yet: every current client (aioesphomeapi, bleak-esphome, Home +# Assistant) assumes an ESPHome proxy can scan actively, so a passive-only +# proxy would be misdriven — bk72xx follows once the API carries a feature +# flag clients can trust (FEATURE_ACTIVE_SCAN + a version flag, separate PRs). +_HUB_PLATFORMS = (PLATFORM_LN882X, PLATFORM_RP2) + +DEPENDENCIES = ["api"] CODEOWNERS = ["@jesserockz", "@bdraco"] _LOGGER = logging.getLogger(__name__) @@ -20,65 +56,209 @@ DEFAULT_CONNECTION_SLOTS = 3 bluetooth_proxy_ns = cg.esphome_ns.namespace("bluetooth_proxy") -BluetoothProxy = bluetooth_proxy_ns.class_( - "BluetoothProxy", esp32_ble_tracker.ESPBTDeviceListener, cg.Component -) -BluetoothConnection = bluetooth_proxy_ns.class_( - "BluetoothConnection", esp32_ble_client.BLEClientBase -) +BluetoothProxy = bluetooth_proxy_ns.class_("BluetoothProxy", cg.Component) -CONNECTION_SCHEMA = esp32_ble_tracker.ESP_BLE_DEVICE_SCHEMA.extend( - { - cv.GenerateID(): cv.declare_id(BluetoothConnection), - } -).extend(cv.COMPONENT_SCHEMA) +# Mirrors esp32_ble.IDF_MAX_CONNECTIONS as a literal so the statically walkable +# CONFIG_SCHEMA below can state the connection_slots range without importing the +# esp32 BLE stack. tests/component_tests/bluetooth_proxy/ pins the two together. +_IDF_MAX_CONNECTIONS = 9 -def validate_connections(config): - if CONF_CONNECTIONS in config: - if not config[CONF_ACTIVE]: - raise cv.Invalid( - "Connections can only be used if the proxy is set to active" - ) - elif config[CONF_ACTIVE]: - connection_slots: int = config[CONF_CONNECTION_SLOTS] - esp32_ble.consume_connection_slots(connection_slots, "bluetooth_proxy")(config) +@functools.cache +def _esp32_config_schema() -> cv.All: + """Build the esp32 schema, importing the esp32 BLE stack only when used.""" + from esphome.components import esp32_ble, esp32_ble_client, esp32_ble_tracker - return { - **config, - CONF_CONNECTIONS: [CONNECTION_SCHEMA({}) for _ in range(connection_slots)], + if esp32_ble.IDF_MAX_CONNECTIONS != _IDF_MAX_CONNECTIONS: + raise cv.Invalid( + f"bluetooth_proxy's connection-slot limit mirror " + f"({_IDF_MAX_CONNECTIONS}) is out of sync with " + f"esp32_ble.IDF_MAX_CONNECTIONS ({esp32_ble.IDF_MAX_CONNECTIONS}); " + f"update _IDF_MAX_CONNECTIONS in bluetooth_proxy/__init__.py" + ) + + BluetoothConnection = bluetooth_proxy_ns.class_( + "BluetoothConnection", esp32_ble_client.BLEClientBase + ) + CONNECTION_SCHEMA = esp32_ble_tracker.ESP_BLE_DEVICE_SCHEMA.extend( + { + cv.GenerateID(): cv.declare_id(BluetoothConnection), } + ).extend(cv.COMPONENT_SCHEMA) + + def validate_connections(config): + if CONF_CONNECTIONS in config: + if not config[CONF_ACTIVE]: + raise cv.Invalid( + "Connections can only be used if the proxy is set to active" + ) + elif config[CONF_ACTIVE]: + connection_slots: int = config[CONF_CONNECTION_SLOTS] + esp32_ble.consume_connection_slots(connection_slots, "bluetooth_proxy")( + config + ) + + return { + **config, + CONF_CONNECTIONS: [ + CONNECTION_SCHEMA({}) for _ in range(connection_slots) + ], + } + return config + + return cv.All( + ( + cv.Schema( + { + **_COMMON_SCHEMA_KEYS, + cv.Optional(CONF_ACTIVE, default=True): cv.boolean, + cv.Optional(CONF_CACHE_SERVICES, default=True): cv.boolean, + cv.Optional( + CONF_CONNECTION_SLOTS, + default=DEFAULT_CONNECTION_SLOTS, + ): cv.All( + cv.positive_int, + cv.Range(min=1, max=esp32_ble.IDF_MAX_CONNECTIONS), + ), + cv.Optional(CONF_CONNECTIONS): cv.All( + cv.ensure_list(CONNECTION_SCHEMA), + cv.Length(min=1, max=esp32_ble.IDF_MAX_CONNECTIONS), + ), + } + ) + .extend(esp32_ble_tracker.ESP_BLE_DEVICE_SCHEMA) + .extend(cv.COMPONENT_SCHEMA) + ), + validate_connections, + ) + + +def _validate_no_active(config: ConfigType) -> ConfigType: + if config[CONF_ACTIVE]: + raise cv.Invalid( + "Active connections are not supported on this platform; the proxy " + "forwards advertisements only (set active: false)" + ) return config -CONFIG_SCHEMA = cv.All( - ( - cv.Schema( - { - cv.GenerateID(): cv.declare_id(BluetoothProxy), - cv.Optional(CONF_ACTIVE, default=True): cv.boolean, - cv.Optional(CONF_CACHE_SERVICES, default=True): cv.boolean, - cv.Optional( - CONF_CONNECTION_SLOTS, - default=DEFAULT_CONNECTION_SLOTS, - ): cv.All( - cv.positive_int, - cv.Range(min=1, max=esp32_ble.IDF_MAX_CONNECTIONS), - ), - cv.Optional(CONF_CONNECTIONS): cv.All( - cv.ensure_list(CONNECTION_SCHEMA), - cv.Length(min=1, max=esp32_ble.IDF_MAX_CONNECTIONS), - ), - } - ) - .extend(esp32_ble_tracker.ESP_BLE_DEVICE_SCHEMA) - .extend(cv.COMPONENT_SCHEMA) - ), - validate_connections, +# Advertisement-only proxy on a neutral BLE hub: the hub's raw-advertisement +# callback feeds the same API batching. GATT/active connections are excluded at +# compile time — only the esp32 build compiles the connection stack; nothing +# reads HubCapabilities::gatt at runtime for this today. +# Keys both platform schemas must declare identically; each arm spreads this +# dict so the shared surface cannot drift. CONF_ACTIVE deliberately stays +# per-arm: its default differs (esp32 True, hub arms False — no GATT). +_COMMON_SCHEMA_KEYS = { + cv.GenerateID(): cv.declare_id(BluetoothProxy), +} + +_BLE_HUB_CONFIG_SCHEMA = cv.All( + cv.Schema( + { + **_COMMON_SCHEMA_KEYS, + # Declared directly (BLE_DEVICE_SCHEMA-style): appending a validator + # after a strict schema rejects an explicit `ble_hub_id` before it + # runs, and that key is the documented way to disambiguate once a + # platform has two trackers. + cv.GenerateID(ble_device_base.CONF_BLE_HUB_ID): cv.use_id( + ble_device_base.BLEHub + ), + cv.Optional(CONF_ACTIVE, default=False): cv.boolean, + } + ).extend(cv.COMPONENT_SCHEMA), + _validate_no_active, ) -async def to_code(config): +@schema_extractor("schema") +def _validate_platform(config: ConfigType) -> ConfigType: + """Apply the schema for the platform actually being compiled. + + esp32 keeps the full GATT proxy; every other platform gets the + advertisement-only shape, which rejects the connection-oriented options + above because its schema does not define them. + """ + if config is SCHEMA_EXTRACT: + # The language-schema dumper runs without a platform. Expose the esp32 + # shape so `connections`, the ids and every default stay in the + # generated schema the editor and dashboard consume. + return _esp32_config_schema() + if CORE.is_esp32: + return _esp32_config_schema()(config) + if CORE.target_platform not in _HUB_PLATFORMS: + # Fail here with the actual reason. Without this gate the error surfaces + # later as an unresolvable hub ID ("Are you missing a hub declaration?") + # on platforms where no hub component can be declared. + raise cv.Invalid( + f"bluetooth_proxy is not supported on {CORE.target_platform}: no " + "active-scan-capable BLE tracker hub is available for this " + "platform. It runs on esp32 (full proxy), and the ln882x and rp2 " + "families (advertisement-only)." + ) + return _BLE_HUB_CONFIG_SCHEMA(config) + + +def _reject_connection_keys_off_esp32(config: ConfigType) -> ConfigType: + """Reject connection-oriented options by name on hub-only platforms. + + Runs before the walkable schema below so the user gets "this option does + not exist here" instead of the option's esp32 value range (which would + imply a smaller number is accepted). + """ + if not isinstance(config, dict) or CORE.is_esp32 or CORE.target_platform is None: + return config + if CORE.target_platform not in _HUB_PLATFORMS: + # No proxy of any kind exists here: fall through so _validate_platform + # reports "not supported on {platform}" instead of a key-level message + # implying an advertisement-only proxy is available. + return config + for key in (CONF_CONNECTION_SLOTS, CONF_CACHE_SERVICES, CONF_CONNECTIONS): + if key in config: + raise cv.Invalid( + f"'{key}' requires active connection support, which needs the " + "esp32 GATT stack; this platform runs the advertisement-only " + "proxy and has no such option", + path=[key], + ) + return config + + +# CONFIG_SCHEMA stays a statically walkable schema: tooling (the dashboard's +# field-range extractor among others) introspects it to discover options and +# their bounds, which a bare dispatch function would hide. It carries the scalar +# keys with no defaults; _validate_platform then runs the real per-platform +# schema, which applies the defaults and rejects options the platform does not +# support. +# +# It deliberately does NOT declare `connections`: this outer schema runs before +# the per-platform one, so any key it transforms is transformed twice. Running +# CONNECTION_SCHEMA twice re-validates an already-generated ID through +# declare_id(), which (unlike use_id) has no guard for an ID instance and +# rejects it as empty. extra=ALLOW_EXTRA passes `connections` through untouched +# for _ESP32_CONFIG_SCHEMA to validate exactly once. +CONFIG_SCHEMA = cv.All( + _reject_connection_keys_off_esp32, + cv.Schema( + { + cv.Optional(CONF_ACTIVE): cv.boolean, + cv.Optional(CONF_CACHE_SERVICES): cv.boolean, + cv.Optional(CONF_CONNECTION_SLOTS): cv.All( + cv.positive_int, + cv.Range(min=1, max=_IDF_MAX_CONNECTIONS), + ), + }, + extra=cv.ALLOW_EXTRA, + ), + _validate_platform, +) + + +async def _to_code_esp32(config: ConfigType) -> None: + from esphome.components import esp32_ble, esp32_ble_tracker + from esphome.components.esp32 import add_idf_sdkconfig_option + from esphome.components.esp32_ble import BTLoggers + # Register the loggers this component needs esp32_ble.register_bt_logger(BTLoggers.GATT, BTLoggers.L2CAP, BTLoggers.SMP) @@ -87,17 +267,12 @@ async def to_code(config): cg.add(var.set_active(config[CONF_ACTIVE])) await esp32_ble_tracker.register_raw_ble_device(var, config) + await esp32_ble_tracker.register_scanner_state_listener(var, config) # Define max connections for protobuf fixed array connection_count = len(config.get(CONF_CONNECTIONS, [])) cg.add_define("BLUETOOTH_PROXY_MAX_CONNECTIONS", connection_count) - # Define batch size for BLE advertisements - # Each advertisement is up to 80 bytes when packaged (including protocol overhead) - # 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload - # This achieves ~97% WiFi MTU utilization while staying under the limit - cg.add_define("BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE", 16) - for connection_conf in config.get(CONF_CONNECTIONS, []): connection_var = cg.new_Pvariable(connection_conf[CONF_ID]) await cg.register_component(connection_var, connection_conf) @@ -107,4 +282,30 @@ async def to_code(config): if config.get(CONF_CACHE_SERVICES): add_idf_sdkconfig_option("CONFIG_BT_GATTC_CACHE_NVS_FLASH", True) + +async def _to_code_ble_hub(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + + cg.add(var.set_active(config[CONF_ACTIVE])) + hub = await cg.get_variable(config[ble_device_base.CONF_BLE_HUB_ID]) + cg.add(var.set_ble_hub(hub)) + + # The api component sizes BluetoothConnectionsFreeResponse.allocated with + # this define whenever a proxy is present; no connections off-esp32. + cg.add_define("BLUETOOTH_PROXY_MAX_CONNECTIONS", 0) + + +async def to_code(config: ConfigType) -> None: + if CORE.is_esp32: + await _to_code_esp32(config) + else: + await _to_code_ble_hub(config) + + # Define batch size for BLE advertisements + # Each advertisement is up to 80 bytes when packaged (including protocol overhead) + # 16 advertisements × 80 bytes (worst case) = 1280 bytes out of ~1320 bytes usable payload + # This achieves ~97% WiFi MTU utilization while staying under the limit + cg.add_define("BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE", 16) + cg.add_define("USE_BLUETOOTH_PROXY") diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index f1a30cdfa2..e681030611 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -1,6 +1,9 @@ #include "bluetooth_proxy.h" +#ifdef USE_BLUETOOTH_PROXY + #include "esphome/components/api/api_server.h" +#include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/macros.h" #include "esphome/core/application.h" @@ -8,8 +11,6 @@ #include #include -#ifdef USE_ESP32 - namespace esphome::bluetooth_proxy { static const char *const TAG = "bluetooth_proxy"; @@ -23,14 +24,14 @@ static_assert(sizeof(((api::BluetoothLERawAdvertisement *) nullptr)->data) == 62 BluetoothProxy::BluetoothProxy() { global_bluetooth_proxy = this; } +#ifdef USE_ESP32 + void BluetoothProxy::setup() { this->connections_free_response_.limit = BLUETOOTH_PROXY_MAX_CONNECTIONS; this->connections_free_response_.free = BLUETOOTH_PROXY_MAX_CONNECTIONS; // Capture the configured scan mode from YAML before any API changes this->configured_scan_active_ = this->parent_->get_scan_active(); - - this->parent_->add_scanner_state_listener(this); } void BluetoothProxy::on_scanner_state(esp32_ble_tracker::ScannerState state) { @@ -50,6 +51,62 @@ void BluetoothProxy::send_bluetooth_scanner_state_(esp32_ble_tracker::ScannerSta this->api_connection_->send_message(resp); } +#else // !USE_ESP32 + +void BluetoothProxy::setup() { + this->connections_free_response_.limit = 0; + this->connections_free_response_.free = 0; + + // Capture the configured scan mode from YAML before any API changes + this->configured_scan_active_ = this->hub_->scan_active(); + this->last_scan_running_ = this->hub_->scan_running(); + + // The hub delivers raw advertisements on the ESPHome main loop: + // mac is least-significant octet first (BLE controller convention). + this->hub_->set_raw_advertisement_callback({this, [](void *self, const ble_device_base::RawAdvertisement &adv) { + static_cast(self)->on_raw_advertisement_(adv); + }}); +} + +void BluetoothProxy::on_raw_advertisement_(const ble_device_base::RawAdvertisement &raw) { + if (!api::global_api_server->is_connected() || this->api_connection_ == nullptr) + return; + + auto &adv = this->response_.advertisements[this->response_.advertisements_len]; + // raw.mac is LSB-first; this yields the same uint64 the esp32 proxy sends. + adv.address = ble_device_base::mac_lsb_first_to_uint64(raw.mac); + adv.rssi = raw.rssi; + adv.address_type = raw.addr_type; + uint8_t length = raw.data_len > sizeof(adv.data) ? sizeof(adv.data) : static_cast(raw.data_len); + adv.data_len = length; + std::memcpy(adv.data, raw.data, length); + + this->response_.advertisements_len++; + + ESP_LOGV(TAG, "Queuing raw packet from %02X:%02X:%02X:%02X:%02X:%02X, length %d. RSSI: %d dB", raw.mac[5], raw.mac[4], + raw.mac[3], raw.mac[2], raw.mac[1], raw.mac[0], length, raw.rssi); + + // Flush if we have reached BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE + if (this->response_.advertisements_len >= BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE) { + this->flush_pending_advertisements_(); + } +} + +void BluetoothProxy::send_bluetooth_scanner_state_() { + api::BluetoothScannerStateResponse resp; + resp.state = this->hub_->scan_running() ? api::enums::BluetoothScannerState::BLUETOOTH_SCANNER_STATE_RUNNING + : api::enums::BluetoothScannerState::BLUETOOTH_SCANNER_STATE_IDLE; + resp.mode = this->hub_->scan_active() ? api::enums::BluetoothScannerMode::BLUETOOTH_SCANNER_MODE_ACTIVE + : api::enums::BluetoothScannerMode::BLUETOOTH_SCANNER_MODE_PASSIVE; + resp.configured_mode = this->configured_scan_active_ + ? api::enums::BluetoothScannerMode::BLUETOOTH_SCANNER_MODE_ACTIVE + : api::enums::BluetoothScannerMode::BLUETOOTH_SCANNER_MODE_PASSIVE; + this->api_connection_->send_message(resp); +} + +#endif // USE_ESP32 + +#ifdef USE_ESP32 void BluetoothProxy::log_connection_request_ignored_(BluetoothConnection *connection, espbt::ClientState state) { ESP_LOGW(TAG, "[%d] [%s] Connection request ignored, state: %s", connection->get_connection_index(), connection->address_str(), espbt::client_state_to_string(state)); @@ -58,6 +115,7 @@ void BluetoothProxy::log_connection_request_ignored_(BluetoothConnection *connec void BluetoothProxy::log_connection_info_(BluetoothConnection *connection, const char *message) { ESP_LOGI(TAG, "[%d] [%s] Connecting %s", connection->get_connection_index(), connection->address_str(), message); } +#endif // USE_ESP32 void BluetoothProxy::log_not_connected_gatt_(const char *action, const char *type) { ESP_LOGW(TAG, "Cannot %s GATT %s, not connected", action, type); @@ -69,6 +127,8 @@ void BluetoothProxy::handle_gatt_not_connected_(uint64_t address, uint16_t handl this->send_gatt_error(address, handle, ESP_GATT_NOT_CONNECTED); } +#ifdef USE_ESP32 + #ifdef USE_ESP32_BLE_DEVICE bool BluetoothProxy::parse_device(const esp32_ble_tracker::ESPBTDevice &device) { // This method should never be called since bluetooth_proxy always uses raw advertisements @@ -109,18 +169,38 @@ bool BluetoothProxy::parse_devices(const esp32_ble::BLEScanResult *scan_results, return true; } +#endif // USE_ESP32 + void BluetoothProxy::log_advertisement_flush_() { ESP_LOGV(TAG, "Sent batch of %u BLE advertisements", this->response_.advertisements_len); } void BluetoothProxy::dump_config() { +#ifdef USE_ESP32 ESP_LOGCONFIG(TAG, "Bluetooth Proxy:\n" " Active: %s\n" " Connections: %d", YESNO(this->active_), this->connection_count_); +#else + // Advertisement-only: print configured facts. dump_config runs right after + // setup, before the radio is up, so live scan state would always read + // "stopped" here — the loop's BluetoothScannerStateResponse carries the + // changing value instead. + char mac_str[18]; + this->get_bluetooth_mac_address_pretty(mac_str); + ESP_LOGCONFIG(TAG, + "Bluetooth Proxy:\n" + " Mode: advertisement-only (no GATT connections)\n" + " Configured scan: %s\n" + " Adapter MAC: %s", + this->configured_scan_active_ ? "active" : "passive", + mac_str[0] != '\0' ? mac_str : "unavailable (adapter not up yet)"); +#endif } +#ifdef USE_ESP32 + void BluetoothProxy::loop() { // Run advertisement flush / connection cleanup every 100ms uint32_t now = App.get_loop_component_start_time(); @@ -254,13 +334,8 @@ void BluetoothProxy::bluetooth_device_request(const api::BluetoothDeviceRequest esp_bd_addr_t address; uint64_to_bd_addr(msg.address, address); esp_err_t ret = esp_ble_gattc_cache_clean(address); - api::BluetoothDeviceClearCacheResponse call; - call.address = msg.address; - call.success = ret == ESP_OK; - call.error = ret; - - this->api_connection_->send_message(call); - + // Shares the sender with the neutral path, which also null-checks api_connection_. + this->send_device_clear_cache(msg.address, ret == ESP_OK, ret); break; } case api::enums::BLUETOOTH_DEVICE_REQUEST_TYPE_CONNECT: { @@ -378,6 +453,124 @@ void BluetoothProxy::bluetooth_set_connection_params(const api::BluetoothSetConn this->api_connection_->send_message(resp); } +void BluetoothProxy::bluetooth_scanner_set_mode(bool active) { + if (this->parent_->get_scan_active() == active) { + return; + } + ESP_LOGD(TAG, "Setting scanner mode to %s", active ? "active" : "passive"); + this->parent_->set_scan_active(active); + this->parent_->stop_scan(); + this->parent_->set_scan_continuous( + true); // Set this to true to automatically start scanning again when it has cleaned up. +} + +#else // !USE_ESP32 + +// Advertisement-only proxy. GATT client connections are excluded at compile +// time — this whole arm is selected by #ifdef USE_ESP32, and nothing consults +// HubCapabilities at runtime today — so every connection-oriented request is +// answered with a clean error instead of silence, and Home Assistant treats +// the proxy as passive. + +void BluetoothProxy::loop() { + // Run advertisement flush / scanner-state poll every 100ms + uint32_t now = App.get_loop_component_start_time(); + if (now - this->last_advertisement_flush_time_ < 100) + return; + this->last_advertisement_flush_time_ = now; + + if (!api::global_api_server->is_connected() || this->api_connection_ == nullptr) + return; + + // The hub has no scanner-state listener interface; poll and report on change. + bool running = this->hub_->scan_running(); + if (running != this->last_scan_running_) { + this->last_scan_running_ = running; + this->send_bluetooth_scanner_state_(); + } + + this->flush_pending_advertisements_(); +} + +void BluetoothProxy::bluetooth_device_request(const api::BluetoothDeviceRequest &msg) { + switch (msg.request_type) { + case api::enums::BLUETOOTH_DEVICE_REQUEST_TYPE_CONNECT_V3_WITH_CACHE: + case api::enums::BLUETOOTH_DEVICE_REQUEST_TYPE_CONNECT_V3_WITHOUT_CACHE: + case api::enums::BLUETOOTH_DEVICE_REQUEST_TYPE_CONNECT: + ESP_LOGW(TAG, "Active connections are not supported on this platform"); + this->send_device_connection(msg.address, false, 0, ESP_GATT_NOT_CONNECTED); + break; + case api::enums::BLUETOOTH_DEVICE_REQUEST_TYPE_DISCONNECT: + // Not an error: the device is already disconnected, which is the requested state. + this->send_device_connection(msg.address, false); + this->send_connections_free(); + break; + case api::enums::BLUETOOTH_DEVICE_REQUEST_TYPE_PAIR: + this->send_device_pairing(msg.address, false, ESP_GATT_NOT_CONNECTED); + break; + case api::enums::BLUETOOTH_DEVICE_REQUEST_TYPE_UNPAIR: + this->send_device_unpairing(msg.address, false, ESP_GATT_NOT_CONNECTED); + break; + case api::enums::BLUETOOTH_DEVICE_REQUEST_TYPE_CLEAR_CACHE: + this->send_device_clear_cache(msg.address, false, ESP_GATT_NOT_CONNECTED); + break; + } +} + +void BluetoothProxy::bluetooth_gatt_read(const api::BluetoothGATTReadRequest &msg) { + this->handle_gatt_not_connected_(msg.address, msg.handle, "read", "characteristic"); +} + +void BluetoothProxy::bluetooth_gatt_write(const api::BluetoothGATTWriteRequest &msg) { + this->handle_gatt_not_connected_(msg.address, msg.handle, "write", "characteristic"); +} + +void BluetoothProxy::bluetooth_gatt_read_descriptor(const api::BluetoothGATTReadDescriptorRequest &msg) { + this->handle_gatt_not_connected_(msg.address, msg.handle, "read", "descriptor"); +} + +void BluetoothProxy::bluetooth_gatt_write_descriptor(const api::BluetoothGATTWriteDescriptorRequest &msg) { + this->handle_gatt_not_connected_(msg.address, msg.handle, "write", "descriptor"); +} + +void BluetoothProxy::bluetooth_gatt_send_services(const api::BluetoothGATTGetServicesRequest &msg) { + this->handle_gatt_not_connected_(msg.address, 0, "get", "services"); +} + +void BluetoothProxy::bluetooth_gatt_notify(const api::BluetoothGATTNotifyRequest &msg) { + this->handle_gatt_not_connected_(msg.address, msg.handle, "notify", "characteristic"); +} + +void BluetoothProxy::bluetooth_set_connection_params(const api::BluetoothSetConnectionParamsRequest &msg) { + if (this->api_connection_ == nullptr) + return; + api::BluetoothSetConnectionParamsResponse resp; + resp.address = msg.address; + resp.error = ESP_GATT_NOT_CONNECTED; + this->api_connection_->send_message(resp); +} + +void BluetoothProxy::bluetooth_scanner_set_mode(bool active) { + if (this->hub_->scan_active() != active) { + ESP_LOGD(TAG, "Setting scanner mode to %s", active ? "active" : "passive"); + if (!this->hub_->request_scan_mode(active)) { + // Passive-only controller asked for active scanning; the state report + // below carries the real, unchanged mode so the subscriber does not + // assume the change happened. + ESP_LOGW(TAG, "Scanner mode %s not supported by this tracker", active ? "active" : "passive"); + } + } + if (this->api_connection_ != nullptr) { + // Keep loop()'s change detector in step with the state sent here, so a + // failed restart (scan_running_ dropped by the tracker) is not reported + // twice — once now and again on the next tick. + this->last_scan_running_ = this->hub_->scan_running(); + this->send_bluetooth_scanner_state_(); + } +} + +#endif // USE_ESP32 + void BluetoothProxy::subscribe_api_connection(api::APIConnection *api_connection, uint32_t flags) { if (this->api_connection_ != nullptr && this->api_connection_ != api_connection) { // A previous subscriber still holds the slot. This is almost always a stale @@ -392,9 +585,13 @@ void BluetoothProxy::subscribe_api_connection(api::APIConnection *api_connection this->api_connection_->get_peername_to(old_peername)); } this->api_connection_ = api_connection; +#ifdef USE_ESP32 this->parent_->recalculate_advertisement_parser_types(); - this->send_bluetooth_scanner_state_(this->parent_->get_scanner_state()); +#else + this->last_scan_running_ = this->hub_->scan_running(); + this->send_bluetooth_scanner_state_(); +#endif } void BluetoothProxy::unsubscribe_api_connection(api::APIConnection *api_connection) { @@ -403,10 +600,12 @@ void BluetoothProxy::unsubscribe_api_connection(api::APIConnection *api_connecti return; } this->api_connection_ = nullptr; +#ifdef USE_ESP32 this->parent_->recalculate_advertisement_parser_types(); +#endif } -void BluetoothProxy::send_device_connection(uint64_t address, bool connected, uint16_t mtu, esp_err_t error) { +void BluetoothProxy::send_device_connection(uint64_t address, bool connected, uint16_t mtu, proxy_err_t error) { if (this->api_connection_ == nullptr) return; api::BluetoothDeviceConnectionResponse call; @@ -434,7 +633,7 @@ void BluetoothProxy::send_gatt_services_done(uint64_t address) { this->api_connection_->send_message(call); } -void BluetoothProxy::send_gatt_error(uint64_t address, uint16_t handle, esp_err_t error) { +void BluetoothProxy::send_gatt_error(uint64_t address, uint16_t handle, proxy_err_t error) { if (this->api_connection_ == nullptr) return; api::BluetoothGATTErrorResponse call; @@ -444,7 +643,7 @@ void BluetoothProxy::send_gatt_error(uint64_t address, uint16_t handle, esp_err_ this->api_connection_->send_message(call); } -void BluetoothProxy::send_device_pairing(uint64_t address, bool paired, esp_err_t error) { +void BluetoothProxy::send_device_pairing(uint64_t address, bool paired, proxy_err_t error) { if (this->api_connection_ == nullptr) return; api::BluetoothDevicePairingResponse call; @@ -455,7 +654,7 @@ void BluetoothProxy::send_device_pairing(uint64_t address, bool paired, esp_err_ this->api_connection_->send_message(call); } -void BluetoothProxy::send_device_unpairing(uint64_t address, bool success, esp_err_t error) { +void BluetoothProxy::send_device_unpairing(uint64_t address, bool success, proxy_err_t error) { if (this->api_connection_ == nullptr) return; api::BluetoothDeviceUnpairingResponse call; @@ -466,19 +665,21 @@ void BluetoothProxy::send_device_unpairing(uint64_t address, bool success, esp_e this->api_connection_->send_message(call); } -void BluetoothProxy::bluetooth_scanner_set_mode(bool active) { - if (this->parent_->get_scan_active() == active) { +// Shared by both platform paths: the neutral bluetooth_device_request() uses it to +// answer a clear-cache request with a clean error, so it must not be esp32-guarded. +void BluetoothProxy::send_device_clear_cache(uint64_t address, bool success, proxy_err_t error) { + if (this->api_connection_ == nullptr) return; - } - ESP_LOGD(TAG, "Setting scanner mode to %s", active ? "active" : "passive"); - this->parent_->set_scan_active(active); - this->parent_->stop_scan(); - this->parent_->set_scan_continuous( - true); // Set this to true to automatically start scanning again when it has cleaned up. + api::BluetoothDeviceClearCacheResponse call; + call.address = address; + call.success = success; + call.error = error; + + this->api_connection_->send_message(call); } BluetoothProxy *global_bluetooth_proxy = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) } // namespace esphome::bluetooth_proxy -#endif // USE_ESP32 +#endif // USE_BLUETOOTH_PROXY diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.h b/esphome/components/bluetooth_proxy/bluetooth_proxy.h index 2b6d29da43..fd1f1839c9 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.h +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.h @@ -1,6 +1,8 @@ #pragma once -#ifdef USE_ESP32 +#include "esphome/core/defines.h" + +#ifdef USE_BLUETOOTH_PROXY #include #include @@ -8,11 +10,14 @@ #include "esphome/components/api/api_connection.h" #include "esphome/components/api/api_pb2.h" -#include "esphome/components/esp32_ble_client/ble_client_base.h" -#include "esphome/components/esp32_ble_tracker/esp32_ble_tracker.h" #include "esphome/core/automation.h" #include "esphome/core/component.h" -#include "esphome/core/defines.h" + +#include "esphome/components/ble_device_base/ble_client_state.h" + +#ifdef USE_ESP32 +#include "esphome/components/esp32_ble_client/ble_client_base.h" +#include "esphome/components/esp32_ble_tracker/esp32_ble_tracker.h" #include "bluetooth_connection.h" @@ -20,14 +25,31 @@ #include #endif #include +#else +#include "esphome/components/ble_device_base/ble_hub.h" +#endif // USE_ESP32 namespace esphome::bluetooth_proxy { -static constexpr esp_err_t ESP_GATT_NOT_CONNECTED = -1; +// Proxy-owned error type for the API error fields, which are plain integers on +// the wire. Aliases esp_err_t on esp32 (where the values come from IDF calls); +// a bare int elsewhere. Owning the name instead of probing for esp_err_t keeps +// the header independent of how a hub platform's SDK spells its error type. +#ifdef USE_ESP32 +using proxy_err_t = esp_err_t; +static constexpr proxy_err_t PROXY_OK = ESP_OK; +#else +using proxy_err_t = int; +static constexpr proxy_err_t PROXY_OK = 0; +#endif + +static constexpr proxy_err_t ESP_GATT_NOT_CONNECTED = ble_device_base::GATT_ERR_NOT_CONNECTED; static constexpr int DONE_SENDING_SERVICES = -2; static constexpr int INIT_SENDING_SERVICES = -3; +#ifdef USE_ESP32 using namespace esp32_ble_client; +#endif // Legacy versions: // Version 1: Initial version without active connections @@ -53,21 +75,28 @@ enum BluetoothProxySubscriptionFlag : uint32_t { SUBSCRIPTION_RAW_ADVERTISEMENTS = 1 << 0, }; +#ifdef USE_ESP32 class BluetoothProxy final : public esp32_ble_tracker::ESPBTDeviceListener, public esp32_ble_tracker::BLEScannerStateListener, public Component { friend class BluetoothConnection; // Allow connection to update connections_free_response_ +#else +class BluetoothProxy final : public Component { +#endif public: BluetoothProxy(); +#ifdef USE_ESP32 #ifdef USE_ESP32_BLE_DEVICE bool parse_device(const esp32_ble_tracker::ESPBTDevice &device) override; #endif bool parse_devices(const esp32_ble::BLEScanResult *scan_results, size_t count) override; + esp32_ble_tracker::AdvertisementParserType get_advertisement_parser_type() override; +#endif // USE_ESP32 void dump_config() override; void setup() override; void loop() override; - esp32_ble_tracker::AdvertisementParserType get_advertisement_parser_type() override; +#ifdef USE_ESP32 // maybe_unused: in a passive proxy (active: false) MAX is 0, the body below is removed, and connection is unused. void register_connection([[maybe_unused]] BluetoothConnection *connection) { // Guard the always-false comparison (-Wtype-limits) in a passive proxy (active: false), where MAX is 0. @@ -78,6 +107,14 @@ class BluetoothProxy final : public esp32_ble_tracker::ESPBTDeviceListener, } #endif } +#else + void set_ble_hub(ble_device_base::BLEHub *hub) { this->hub_ = hub; } + // Run after the hub's setup() (the trackers use AFTER_WIFI): setup() below + // snapshots scan_active()/scan_running() and installs the raw callback, and + // the BLEHub contract does not promise those are settled any earlier than + // the hub's own setup(). + float get_setup_priority() const override { return setup_priority::AFTER_WIFI - 1.0f; } +#endif // USE_ESP32 void bluetooth_device_request(const api::BluetoothDeviceRequest &msg); void bluetooth_gatt_read(const api::BluetoothGATTReadRequest &msg); @@ -92,17 +129,18 @@ class BluetoothProxy final : public esp32_ble_tracker::ESPBTDeviceListener, void unsubscribe_api_connection(api::APIConnection *api_connection); api::APIConnection *get_api_connection() { return this->api_connection_; } - void send_device_connection(uint64_t address, bool connected, uint16_t mtu = 0, esp_err_t error = ESP_OK); + void send_device_connection(uint64_t address, bool connected, uint16_t mtu = 0, proxy_err_t error = PROXY_OK); void send_connections_free(); void send_connections_free(api::APIConnection *api_connection); void send_gatt_services_done(uint64_t address); - void send_gatt_error(uint64_t address, uint16_t handle, esp_err_t error); - void send_device_pairing(uint64_t address, bool paired, esp_err_t error = ESP_OK); - void send_device_unpairing(uint64_t address, bool success, esp_err_t error = ESP_OK); - void send_device_clear_cache(uint64_t address, bool success, esp_err_t error = ESP_OK); + void send_gatt_error(uint64_t address, uint16_t handle, proxy_err_t error); + void send_device_pairing(uint64_t address, bool paired, proxy_err_t error = PROXY_OK); + void send_device_unpairing(uint64_t address, bool success, proxy_err_t error = PROXY_OK); + void send_device_clear_cache(uint64_t address, bool success, proxy_err_t error = PROXY_OK); void bluetooth_scanner_set_mode(bool active); +#ifdef USE_ESP32 static void uint64_to_bd_addr(uint64_t address, esp_bd_addr_t bd_addr) { bd_addr[0] = (address >> 40) & 0xff; bd_addr[1] = (address >> 32) & 0xff; @@ -111,12 +149,15 @@ class BluetoothProxy final : public esp32_ble_tracker::ESPBTDeviceListener, bd_addr[4] = (address >> 8) & 0xff; bd_addr[5] = (address >> 0) & 0xff; } +#endif void set_active(bool active) { this->active_ = active; } bool has_active() { return this->active_; } +#ifdef USE_ESP32 /// BLEScannerStateListener interface void on_scanner_state(esp32_ble_tracker::ScannerState state) override; +#endif uint32_t get_legacy_version() const { if (this->active_) { @@ -129,7 +170,17 @@ class BluetoothProxy final : public esp32_ble_tracker::ESPBTDeviceListener, uint32_t flags = 0; flags |= BluetoothProxyFeature::FEATURE_PASSIVE_SCAN; flags |= BluetoothProxyFeature::FEATURE_RAW_ADVERTISEMENTS; +#ifdef USE_ESP32 flags |= BluetoothProxyFeature::FEATURE_STATE_AND_MODE; +#else + // Advertise mode switching only where the hub honors request_scan_mode(); + // scan_mode_switch is the capability bit for exactly that (#18079) — + // active_scan alone is not enough, a hub may support active scanning yet + // refuse the runtime switch. + if (this->hub_->get_capabilities().scan_mode_switch) { + flags |= BluetoothProxyFeature::FEATURE_STATE_AND_MODE; + } +#endif if (this->active_) { flags |= BluetoothProxyFeature::FEATURE_ACTIVE_CONNECTIONS; flags |= BluetoothProxyFeature::FEATURE_REMOTE_CACHING; @@ -142,16 +193,37 @@ class BluetoothProxy final : public esp32_ble_tracker::ESPBTDeviceListener, } void get_bluetooth_mac_address_pretty(std::span output) { +#ifdef USE_ESP32 const uint8_t *mac = esp_bt_dev_get_address(); if (mac != nullptr) { format_mac_addr_upper(mac, output.data()); } else { output[0] = '\0'; } +#else + uint8_t mac[6] = {}; + this->hub_->get_adapter_mac(mac); + // Mirror the esp32 arm's unavailable -> empty-string fallback: some hubs + // (rp2040's BTstack) only learn the address once the link layer is up, and + // report all-zero until then. + bool nonzero = false; + for (uint8_t b : mac) + nonzero |= b != 0; + if (nonzero) { + format_mac_addr_upper(mac, output.data()); + } else { + output[0] = '\0'; + } +#endif } protected: +#ifdef USE_ESP32 void send_bluetooth_scanner_state_(esp32_ble_tracker::ScannerState state); +#else + void send_bluetooth_scanner_state_(); + void on_raw_advertisement_(const ble_device_base::RawAdvertisement &raw); +#endif /// Caller must ensure api_connection_ is non-null and API server is connected. void flush_pending_advertisements_() { @@ -165,9 +237,11 @@ class BluetoothProxy final : public esp32_ble_tracker::ESPBTDeviceListener, } void log_advertisement_flush_(); +#ifdef USE_ESP32 BluetoothConnection *get_connection_(uint64_t address, bool reserve); void log_connection_request_ignored_(BluetoothConnection *connection, espbt::ClientState state); void log_connection_info_(BluetoothConnection *connection, const char *message); +#endif void log_not_connected_gatt_(const char *action, const char *type); void handle_gatt_not_connected_(uint64_t address, uint16_t handle, const char *action, const char *type); @@ -175,8 +249,12 @@ class BluetoothProxy final : public esp32_ble_tracker::ESPBTDeviceListener, // Group 1: Pointers (4 bytes each, naturally aligned) api::APIConnection *api_connection_{nullptr}; +#ifdef USE_ESP32 // Group 2: Fixed-size array of connection pointers std::array connections_{}; +#else + ble_device_base::BLEHub *hub_{nullptr}; +#endif // BLE advertisement batching api::BluetoothLERawAdvertisementsResponse response_; @@ -191,11 +269,13 @@ class BluetoothProxy final : public esp32_ble_tracker::ESPBTDeviceListener, bool active_; uint8_t connection_count_{0}; bool configured_scan_active_{false}; // Configured scan mode from YAML - // 3 bytes used, 1 byte padding +#ifndef USE_ESP32 + bool last_scan_running_{false}; // Last scanner state reported to the subscriber +#endif }; extern BluetoothProxy *global_bluetooth_proxy; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) } // namespace esphome::bluetooth_proxy -#endif // USE_ESP32 +#endif // USE_BLUETOOTH_PROXY diff --git a/esphome/components/bthome_mithermometer/bthome_ble.cpp b/esphome/components/bthome_mithermometer/bthome_ble.cpp index ff38ab1740..66f147c266 100644 --- a/esphome/components/bthome_mithermometer/bthome_ble.cpp +++ b/esphome/components/bthome_mithermometer/bthome_ble.cpp @@ -25,6 +25,9 @@ static constexpr size_t BTHOME_NONCE_SIZE = 13; static constexpr size_t BTHOME_MIC_SIZE = 4; static constexpr size_t BTHOME_COUNTER_SIZE = 4; +// Both callers are log macros (LOGCONFIG / LOGVV); below CONFIG level they +// compile away and an ungated helper trips -Wunused-function. +#if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_CONFIG static const char *format_mac_address(std::span buffer, uint64_t address) { std::array mac{}; for (size_t i = 0; i < MAC_ADDRESS_SIZE; i++) { @@ -34,6 +37,7 @@ static const char *format_mac_address(std::span= ESPHOME_LOG_LEVEL_CONFIG static bool get_bthome_value_length(uint8_t obj_type, size_t &value_length) { switch (obj_type) { diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index 7476385563..44878274d6 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -27,6 +27,7 @@ CONF_LOOP = "loop" CONF_NOX_INDEX = "nox_index" CONF_ON_PACKET = "on_packet" CONF_ON_RECEIVE = "on_receive" +CONF_ON_SCAN_END = "on_scan_end" CONF_ON_STATE_CHANGE = "on_state_change" CONF_PARITY = "parity" CONF_RECEIVER_FREQUENCY = "receiver_frequency" @@ -36,6 +37,7 @@ CONF_SCAN_PARAMETERS = "scan_parameters" CONF_SHA256 = "sha256" CONF_STATE_SAVE_INTERVAL = "state_save_interval" CONF_STOP_BITS = "stop_bits" +CONF_TARGET_COUNT = "target_count" CONF_USE_PSRAM = "use_psram" CONF_VOC_INDEX = "voc_index" CONF_VOLUME_INCREMENT = "volume_increment" diff --git a/esphome/components/epaper_spi/epaper_spi_t133a01.cpp b/esphome/components/epaper_spi/epaper_spi_t133a01.cpp index 5735333761..95d1fcb484 100644 --- a/esphome/components/epaper_spi/epaper_spi_t133a01.cpp +++ b/esphome/components/epaper_spi/epaper_spi_t133a01.cpp @@ -311,9 +311,12 @@ bool HOT EPaperT133A01::transfer_data() { this->current_data_index_ = half; if (millis() - start_time > MAX_TRANSFER_TIME) { - return false; + break; } } + if (half < total_rows) { + return false; + } ESP_LOGD(TAG, "CS phase done"); this->disable(); this->cs_pin_->digital_write(true); // deselect CS @@ -346,9 +349,12 @@ bool HOT EPaperT133A01::transfer_data() { this->current_data_index_ = half; if (millis() - start_time > MAX_TRANSFER_TIME) { - return false; + break; } } + if (half < total_rows * 2) { + return false; + } ESP_LOGD(TAG, "CS1 phase done"); this->disable(); this->cs1_pin_->digital_write(true); // deselect CS1 diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 6491b9a5e6..e31d0352e9 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -119,6 +119,7 @@ CONF_SIGNING_SCHEME = "signing_scheme" CONF_SRAM1_AS_IRAM = "sram1_as_iram" CONF_SUBTYPE = "subtype" CONF_VERIFICATION_KEY = "verification_key" +CONF_VERIFICATION_KEYS = "verification_keys" ARDUINO_FRAMEWORK_NAME = "framework-arduinoespressif32" ARDUINO_FRAMEWORK_PKG = f"pioarduino/{ARDUINO_FRAMEWORK_NAME}" @@ -141,12 +142,22 @@ ASSERTION_LEVELS = { "SILENT": "CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_SILENT", } +SIGNING_SCHEME_RSA3072 = "rsa3072" +SIGNING_SCHEME_ECDSA256 = "ecdsa256" +SIGNING_SCHEME_ECDSA_V1 = "ecdsa_v1" + SIGNING_SCHEMES = { - "rsa3072": "CONFIG_SECURE_SIGNED_APPS_RSA_SCHEME", - "ecdsa256": "CONFIG_SECURE_SIGNED_APPS_ECDSA_V2_SCHEME", - "ecdsa_v1": "CONFIG_SECURE_SIGNED_APPS_ECDSA_SCHEME", + SIGNING_SCHEME_RSA3072: "CONFIG_SECURE_SIGNED_APPS_RSA_SCHEME", + SIGNING_SCHEME_ECDSA256: "CONFIG_SECURE_SIGNED_APPS_ECDSA_V2_SCHEME", + SIGNING_SCHEME_ECDSA_V1: "CONFIG_SECURE_SIGNED_APPS_ECDSA_SCHEME", } +# A Secure Boot v2 image carries at most three signature blocks, and hardware +# secure boot exposes three eFuse key slots. The trusted-key list isn't bound by +# the per-image limit (an incoming image need only match one trusted key), but +# cap it at three to mirror those hardware limits. +SIGNED_OTA_MAX_KEYS = 3 + # Chip variants that only support one V2 signing scheme. # Based on SOC_SECURE_BOOT_V2_RSA / SOC_SECURE_BOOT_V2_ECC in soc_caps.h. # Variants not listed in either set support both RSA and ECDSA V2 @@ -628,7 +639,6 @@ class NetworkSdkconfigData: wifi_ap: bool = False # WiFi AP mode configured ethernet: bool = False # Ethernet component active bluetooth: bool = False # any BLE component active - ble_42: bool = False # BLE 4.2 features needed software_coexistence: bool = False # WiFi/BT software coexistence requested # esp32 advanced enable_lwip_dhcp_server option (True/False/None=unset) enable_lwip_dhcp_server: bool | None = None @@ -654,12 +664,10 @@ def request_ethernet() -> None: _network_sdkconfig().ethernet = True -def request_bluetooth(ble_42: bool = False) -> None: - """Request the Bluetooth controller. Pass ble_42=True for 4.2 features.""" +def request_bluetooth() -> None: + """Request the Bluetooth controller.""" net = _network_sdkconfig() net.bluetooth = True - if ble_42: - net.ble_42 = True def request_software_coexistence() -> None: @@ -1167,11 +1175,99 @@ def _ota_downgrade_protection_errors( return errs +def _sbv2_rsa_key_digest(path: Path) -> bytes: + """SHA-256 of a public key's Secure Boot v2 signature-block key region. + + This hashes the 776-byte {n, e, rinv, m'} region exactly as the ROM lays it + out -- i.e. the value the device computes per signature block and the one + ``espsecure digest-sbv2-public-key`` prints, not a hash of the DER key. + """ + import hashlib + import struct + + from cryptography.exceptions import UnsupportedAlgorithm + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.hazmat.primitives.serialization import ( + load_pem_private_key, + load_pem_public_key, + ) + + data = path.read_bytes() + try: + if b"PUBLIC KEY" in data: + public_key = load_pem_public_key(data) + else: + # verification_keys only needs the public half; warn so the private + # key doesn't end up committed alongside the config. + _LOGGER.warning( + "'%s' is a private key, but '%s' needs only the public key. Use a " + "public-key PEM or the 64-hex digest (espsecure " + "digest-sbv2-public-key) so the private key stays out of your config.", + path, + CONF_VERIFICATION_KEYS, + ) + public_key = load_pem_private_key(data, password=None).public_key() + except (ValueError, TypeError, UnsupportedAlgorithm) as err: + raise cv.Invalid(f"Could not load key '{path}': {err}") from err + if not isinstance(public_key, rsa.RSAPublicKey) or public_key.key_size != 3072: + raise cv.Invalid( + f"'{CONF_VERIFICATION_KEYS}' entries must be RSA-3072 keys; " + f"'{path}' is not." + ) + numbers = public_key.public_numbers() + n, e = numbers.n, numbers.e + m = (-pow(n, -1, 1 << 32)) & 0xFFFFFFFF + rinv = (1 << (public_key.key_size * 2)) % n + blob = struct.pack( + "<384sI384sI", + n.to_bytes(384, "big")[::-1], + e, + rinv.to_bytes(384, "big")[::-1], + m, + ) + return hashlib.sha256(blob).digest() + + +def _validate_trusted_key(value: Any) -> str: + """Normalize a trusted key to its 64-hex-char signature-block digest. + + Accepts either the digest directly (so CI can inject it without shipping a + key file) or a PEM key file whose digest is computed here. Typed ``Any`` + because YAML hands validators the parsed value -- e.g. an unquoted ``0x...`` + digest arrives as an int, which the guard below rejects with advice to quote. + """ + # An unquoted 0x... or all-digit digest is parsed by YAML as an int before it + # reaches here, so it never looks like a string digest -- reject it clearly + # rather than letting it fall through to cv.file_ as a bogus path. + if not isinstance(value, str): + raise cv.Invalid( + f"Expected a key file path or a 64-character hex digest, got {value!r}. " + f"Quote the digest so YAML keeps it as text (an unquoted '0x...' or " + f"all-digit value is parsed as a number)." + ) + stripped = value.strip() + if re.fullmatch(r"[0-9A-Fa-f]{64}", stripped): + return stripped.lower() + # An all-hex value that isn't exactly 64 chars is a mangled digest, not a + # path: a truncated or 0x-prefixed CI variable would otherwise fall through + # and fail as "file not found", pointing at the wrong problem. + if re.fullmatch(r"(?:0x)?[0-9A-Fa-f]+", stripped): + raise cv.Invalid( + f"'{stripped}' looks like a key digest but must be exactly 64 hex " + f"characters (a SHA-256, no '0x' prefix); check for truncation." + ) + return _sbv2_rsa_key_digest(cv.file_(value)).hex() + + _SIGNED_OTA_VERIFICATION_SCHEMA = cv.Schema( { cv.Optional(CONF_SIGNING_KEY): cv.file_, cv.Optional(CONF_VERIFICATION_KEY): cv.file_, - cv.Optional(CONF_SIGNING_SCHEME, default="rsa3072"): cv.one_of( + cv.Optional(CONF_VERIFICATION_KEYS): cv.All( + cv.ensure_list(_validate_trusted_key), + cv.Length(min=1, max=SIGNED_OTA_MAX_KEYS), + ), + cv.Optional(CONF_SIGNING_SCHEME, default=SIGNING_SCHEME_RSA3072): cv.one_of( *SIGNING_SCHEMES, lower=True ), } @@ -1204,9 +1300,15 @@ def _validate_signed_ota_keys(config: ConfigType) -> ConfigType: block appended to each image, so verifying externally-signed binaries needs no key in the config at all -- omitting both keys selects that external-signing mode. + + For external RSA (rsa3072, no signing key), an optional 'verification_keys' + list names the keys the running app trusts. ESPHome then verifies OTA + signatures against that compiled-in set instead of IDF's single-block + check, which enables key rotation and multi-provider backup keys. """ has_signing_key = CONF_SIGNING_KEY in config has_verification_key = CONF_VERIFICATION_KEY in config + has_verification_keys = CONF_VERIFICATION_KEYS in config scheme = config[CONF_SIGNING_SCHEME] if has_signing_key and has_verification_key: raise cv.Invalid( @@ -1214,7 +1316,35 @@ def _validate_signed_ota_keys(config: ConfigType) -> ConfigType: f"'{CONF_VERIFICATION_KEY}', not both.", path=[CONF_VERIFICATION_KEY], ) - if scheme == "ecdsa_v1": + if has_verification_keys: + if scheme != SIGNING_SCHEME_RSA3072: + raise cv.Invalid( + f"'{CONF_VERIFICATION_KEYS}' is only used with signing scheme " + f"'rsa3072' (externally-signed RSA images). With '{scheme}' the " + f"public key travels in each image's signature block.", + path=[CONF_VERIFICATION_KEYS], + ) + if has_signing_key: + raise cv.Invalid( + f"'{CONF_VERIFICATION_KEYS}' verifies externally-signed images " + f"and cannot be combined with '{CONF_SIGNING_KEY}' (which signs " + f"during the build). Provide one or the other.", + path=[CONF_VERIFICATION_KEYS], + ) + if has_verification_key: + raise cv.Invalid( + f"Provide at most one of '{CONF_VERIFICATION_KEY}' and " + f"'{CONF_VERIFICATION_KEYS}', not both.", + path=[CONF_VERIFICATION_KEYS], + ) + keys = config[CONF_VERIFICATION_KEYS] + if len(set(keys)) != len(keys): + raise cv.Invalid( + f"'{CONF_VERIFICATION_KEYS}' entries must be unique (duplicate " + f"keys add nothing and waste a trusted-set slot).", + path=[CONF_VERIFICATION_KEYS], + ) + if scheme == SIGNING_SCHEME_ECDSA_V1: if not has_signing_key and not has_verification_key: raise cv.Invalid( f"Signing scheme 'ecdsa_v1' requires either '{CONF_SIGNING_KEY}' " @@ -1370,7 +1500,10 @@ def final_validate(config): ] # V1 ECDSA is only available on the original ESP32 - if scheme == "ecdsa_v1" and variant not in SIGNED_OTA_V1_ECDSA_VARIANTS: + if ( + scheme == SIGNING_SCHEME_ECDSA_V1 + and variant not in SIGNED_OTA_V1_ECDSA_VARIANTS + ): errs.append( cv.Invalid( f"Signing scheme 'ecdsa_v1' is only supported on " @@ -1383,7 +1516,9 @@ def final_validate(config): # On ESP32, V2 RSA requires minimum_chip_revision >= 3.0 # Note: string comparison works here because cv.one_of constrains # min_rev to known ESP32_CHIP_REVISIONS values ("0.0".."3.1"). - if scheme == "rsa3072" and (min_rev is None or min_rev < "3.0"): + if scheme == SIGNING_SCHEME_RSA3072 and ( + min_rev is None or min_rev < "3.0" + ): errs.append( cv.Invalid( f"Signing scheme 'rsa3072' on {VARIANT_FRIENDLY[variant]} " @@ -1394,7 +1529,7 @@ def final_validate(config): ) ) # ESP32 does not support V2 ECDSA (no SOC_SECURE_BOOT_V2_ECC) - elif scheme == "ecdsa256": + elif scheme == SIGNING_SCHEME_ECDSA256: errs.append( cv.Invalid( f"Signing scheme 'ecdsa256' is not supported on " @@ -1404,7 +1539,11 @@ def final_validate(config): ) ) # V1 on rev 3.0+ -- suggest V2 RSA for stronger security - elif scheme == "ecdsa_v1" and min_rev is not None and min_rev >= "3.0": + elif ( + scheme == SIGNING_SCHEME_ECDSA_V1 + and min_rev is not None + and min_rev >= "3.0" + ): _LOGGER.info( "Using Secure Boot V1 ECDSA on %s rev %s. " "Consider using 'rsa3072' (Secure Boot V2 RSA) for " @@ -1415,8 +1554,14 @@ def final_validate(config): else: # Non-ESP32 variants: check V2 scheme-variant compatibility scheme_variant_conflicts = { - "ecdsa256": (SIGNED_OTA_V2_RSA_ONLY_VARIANTS, "rsa3072"), - "rsa3072": (SIGNED_OTA_V2_ECC_ONLY_VARIANTS, "ecdsa256"), + SIGNING_SCHEME_ECDSA256: ( + SIGNED_OTA_V2_RSA_ONLY_VARIANTS, + SIGNING_SCHEME_RSA3072, + ), + SIGNING_SCHEME_RSA3072: ( + SIGNED_OTA_V2_ECC_ONLY_VARIANTS, + SIGNING_SCHEME_ECDSA256, + ), } if ( conflict := scheme_variant_conflicts.get(scheme) @@ -2055,12 +2200,12 @@ async def _reconcile_network_sdkconfig() -> None: if name not in opts: add_idf_sdkconfig_option(name, value) - # Bluetooth: only ever enable when requested. The IDF default is off and - # nothing sets these False today, so never write False here. + # Bluetooth: only ever enable when requested. The IDF default is off. + # According to the IDF docs, only one of 4.2 or 5.0 should be enabled. if net.bluetooth: set_opt("CONFIG_BT_ENABLED", True) - if net.ble_42: - set_opt("CONFIG_BT_BLE_42_FEATURES_SUPPORTED", True) + set_opt("CONFIG_BT_BLE_42_FEATURES_SUPPORTED", True) + set_opt("CONFIG_BT_BLE_50_FEATURES_SUPPORTED", False) # WiFi stack: disable only when Ethernet is present and WiFi is not. WiFi # relies on the IDF default (enabled), so it is never written True here. @@ -2193,6 +2338,8 @@ async def to_code(config): cg.set_cpp_standard("gnu++20") cg.add_build_flag("-DUSE_ESP32") cg.add_define("USE_NATIVE_64BIT_TIME") + # NVS finds stored preferences by key, so preference key migration is possible + cg.add_define("USE_PREFERENCE_KEY_LOOKUP") cg.add_build_flag("-Wl,-z,noexecstack") # Deferred so KEY_COMPONENTS is fully populated -- see the coroutine. CORE.add_job(_finalize_arduino_aware_flags) @@ -2557,9 +2704,70 @@ async def to_code(config): # Enable signed app verification without hardware secure boot if signed_ota := advanced.get(CONF_SIGNED_OTA_VERIFICATION): add_idf_sdkconfig_option("CONFIG_SECURE_SIGNED_APPS_NO_SECURE_BOOT", True) - add_idf_sdkconfig_option("CONFIG_SECURE_SIGNED_ON_UPDATE_NO_SECURE_BOOT", True) scheme = signed_ota[CONF_SIGNING_SCHEME] + # For externally-signed RSA images with a declared 'verification_keys' + # list, ESPHome verifies the OTA signature itself instead of using IDF's + # on-update check. IDF only matches the incoming image's first signature + # block against the running app's first, which blocks key rotation and + # multi-provider backup keys; ESPHome accepts an image signed by any key + # in the compiled-in trusted set. Without 'verification_keys' there is no + # trust anchor, so fall back to IDF's built-in check. + # The build still produces the padded unsigned image (via SECURE_ + # SIGNED_APPS_NO_SECURE_BOOT above); only the on-update check moves. + # SECURE_SIGNED_ON_UPDATE_NO_SECURE_BOOT defaults to y under + # SECURE_SIGNED_APPS_NO_SECURE_BOOT, so it must be set explicitly: + # False to hand verification to ESPHome, True to keep IDF's check. + # Setting it False also drives the hidden CONFIG_SECURE_SIGNED_APPS to + # n; the 4 KiB padding and reserved signature sector the verifier + # depends on survive only because --secure-pad-v2 keys off + # CONFIG_SECURE_SIGNED_APPS_RSA_SCHEME (set below), not that symbol. + external_rsa = ( + scheme == SIGNING_SCHEME_RSA3072 and CONF_SIGNING_KEY not in signed_ota + ) + verification_keys = signed_ota.get(CONF_VERIFICATION_KEYS) + # verification_keys is accepted only for external RSA (rsa3072 with no + # signing_key), enforced in _validate_signed_ota_keys. Assert the + # post-condition so validator/codegen drift fails the build loudly + # instead of silently dropping the declared trust anchor and downgrading + # to IDF's single-block check. + assert not verification_keys or external_rsa + multi_key = external_rsa and verification_keys + # Turning IDF's on-update check off is global -- it also drops the + # signature check from esp_ota_set_boot_partition() on the partition-table + # path and safe_mode's recovery rollback. Both deliberately select an + # already-installed image (or an MD5-checked partition table), not a + # freshly-downloaded one, so ESPHome's verifier only needs to cover the + # app and bootloader OTA paths, where a new image is actually written. + add_idf_sdkconfig_option( + "CONFIG_SECURE_SIGNED_ON_UPDATE_NO_SECURE_BOOT", not multi_key + ) + if multi_key: + cg.add_define("USE_OTA_SIGNED_VERIFICATION_MULTI_KEY") + # Compile the trusted key digests in as the immutable trust anchor. + # Each is the SHA-256 of a key's signature-block region; the verifier + # accepts an OTA whose signature block matches one of these. + digests = [bytes.fromhex(k) for k in verification_keys] + # Echo the resolved digests so a stale or mistyped key (which builds + # cleanly but leaves the device updatable only by serial reflash) is + # visible in the build log. + _LOGGER.info( + "Signed OTA verification trusts %d key digest(s): %s", + len(digests), + ", ".join(d.hex() for d in digests), + ) + cg.add_define("OTA_TRUSTED_KEY_COUNT", len(digests)) + cg.add_define( + "OTA_TRUSTED_KEY_DIGESTS", + cg.RawExpression( + "{" + + ",".join( + "{" + ",".join(f"0x{b:02x}" for b in d) + "}" for d in digests + ) + + "}" + ), + ) + for key, flag in SIGNING_SCHEMES.items(): add_idf_sdkconfig_option(flag, scheme == key) diff --git a/esphome/components/esp32/const.py b/esphome/components/esp32/const.py index 248f84c6bc..09f458c64b 100644 --- a/esphome/components/esp32/const.py +++ b/esphome/components/esp32/const.py @@ -1,9 +1,22 @@ import esphome.codegen as cg -KEY_ESP32 = "esp32" +# Re-exported for the many esp32-side users; defined in esphome.const +# and esphome.espidf so the upload/logs fast path can use them without +# importing this package. +from esphome.const import ( # noqa: F401 # pylint: disable=unused-import + KEY_ESP32, + KEY_FLASH_SIZE, + KEY_IDF_VERSION, + KEY_VARIANT, +) + +# Back compat for external components only; in-tree callers import it +# from esphome.espidf directly. +from esphome.espidf import ( # noqa: F401 # pylint: disable=unused-import + variant_to_idf_target, +) + KEY_BOARD = "board" -KEY_FLASH_SIZE = "flash_size" -KEY_VARIANT = "variant" KEY_SDKCONFIG_OPTIONS = "sdkconfig_options" KEY_COMPONENTS = "components" KEY_EXCLUDE_COMPONENTS = "exclude_components" @@ -15,7 +28,6 @@ KEY_PATH = "path" KEY_SUBMODULES = "submodules" KEY_EXTRA_BUILD_FILES = "extra_build_files" KEY_FULL_CERT_BUNDLE = "full_cert_bundle" -KEY_IDF_VERSION = "idf_version" KEY_NETWORK_SDKCONFIG = "network_sdkconfig" VARIANT_ESP32 = "ESP32" @@ -64,9 +76,4 @@ VARIANT_FRIENDLY = { } -def variant_to_idf_target(variant: str) -> str: - """Map an esp32 variant name (e.g. "ESP32S3") to its ESP-IDF target name.""" - return variant.lower().replace("-", "") - - esp32_ns = cg.esphome_ns.namespace("esp32") diff --git a/esphome/components/esp32/crash_handler.cpp b/esphome/components/esp32/crash_handler.cpp index 4c0f430daf..1b054dcc49 100644 --- a/esphome/components/esp32/crash_handler.cpp +++ b/esphome/components/esp32/crash_handler.cpp @@ -4,6 +4,7 @@ #ifdef USE_ESP32_CRASH_HANDLER #include "crash_handler.h" +#include "esphome/core/build_info_data.h" #include "esphome/core/log.h" #include @@ -122,7 +123,7 @@ static uint8_t IRAM_ATTR capture_riscv_backtrace(RvExcFrame *frame, uint32_t *ou // Magic is second to validate the data. Remaining fields can change between versions. // Version is uint32_t because it would be padded to 4 bytes anyway before the next // uint32_t field, so we use the full width rather than wasting 3 bytes of padding. -static constexpr uint32_t CRASH_DATA_VERSION = 3; +static constexpr uint32_t CRASH_DATA_VERSION = 4; struct RawCrashData { uint32_t version; uint32_t magic; @@ -134,6 +135,7 @@ struct RawCrashData { uint32_t backtrace[MAX_BACKTRACE]; uint32_t cause; // Architecture-specific: exccause (Xtensa) or mcause (RISC-V) uint32_t fault_addr; // Faulting memory address: excvaddr (Xtensa) or mtval (RISC-V) + uint32_t build_time; // ESPHOME_BUILD_TIME of the firmware that captured this record uint8_t crashed_core; #if SOC_CPU_CORES_NUM > 1 static_assert(SOC_CPU_CORES_NUM == 2, "Dual-core logic assumes exactly 2 cores"); @@ -152,6 +154,16 @@ namespace esphome::esp32 { static const char *const TAG = "esp32.crash"; +// RAM copy of the build timestamp. The generated constant lives in flash, +// which the panic handler must not read (cache may be disabled during +// cache-error panics), so the wrapper stamps the record from this mirror +// instead. Filled during C++ dynamic initialization, well before arch_init(); +// ESPHOME_BUILD_TIME itself is constant-initialized, so the read is ordered. +// Unqualified name on purpose: the runtime header declares it in namespace +// esphome, while the static-analysis stub defines it as a macro. +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +static uint32_t s_current_build_time = static_cast(ESPHOME_BUILD_TIME); + void crash_handler_read_and_clear() { if (s_raw_crash_data.magic == CRASH_MAGIC && s_raw_crash_data.version == CRASH_DATA_VERSION) { s_crash_data_valid = true; @@ -331,6 +343,66 @@ static int append_addrs_to_hint(char *buf, int size, int pos, const uint32_t *ad return pos; } +// Register holding the faulting memory address, named as in ESP-IDF's live +// register dump. The lowercase form is for old-build reports, where the +// stacktrace decoders must not match the line. +#if CONFIG_IDF_TARGET_ARCH_XTENSA +static const char *const FAULT_ADDR_REG = "EXCVADDR"; +static const char *const FAULT_ADDR_REG_LOWER = "excvaddr"; +#elif CONFIG_IDF_TARGET_ARCH_RISCV +static const char *const FAULT_ADDR_REG = "MTVAL"; +static const char *const FAULT_ADDR_REG_LOWER = "mtval"; +#endif + +// Whether the fault address is meaningful — real CPU faults only, not +// aborts/watchdogs or SoC-level pseudo exceptions. +static bool has_fault_addr() { + return s_raw_crash_data.exception == PANIC_EXCEPTION_FAULT && !s_raw_crash_data.pseudo_excause; +} + +// Append both cores' backtrace addresses to buf; returns the new position. +static int append_all_backtraces(char *buf, int size, int pos) { + pos = append_addrs_to_hint(buf, size, pos, s_raw_crash_data.backtrace, s_raw_crash_data.backtrace_count, + s_raw_crash_data.reg_frame_count); +#if SOC_CPU_CORES_NUM > 1 + pos = append_addrs_to_hint(buf, size, pos, s_raw_crash_data.other_backtrace, s_raw_crash_data.other_backtrace_count, + s_raw_crash_data.other_reg_frame_count); +#endif + return pos; +} + +// The record was captured by a different firmware build (it survives soft +// resets, including the OTA reboot), so symbolizing its addresses against the +// current ELF would produce misleading symbols. Print them with lowercase +// labels the stacktrace decoders deliberately do not match, and skip the +// addr2line hint. One line per address so nothing is lost to a shared buffer. +// No is_return_addr() filtering here: it would inspect the current build's +// code bytes, which say nothing about addresses captured by the old build. +static uint8_t log_foreign_backtrace(const uint32_t *addrs, uint8_t count, uint8_t bt_num) { + for (uint8_t i = 0; i < count; i++) { + ESP_LOGE(TAG, " bt%d: 0x%08" PRIX32, bt_num++, addrs[i]); + } + return bt_num; +} + +static void log_foreign_addresses() { + ESP_LOGE(TAG, " Captured by a different firmware build; addresses belong to that build's ELF"); + ESP_LOGE(TAG, " pc: 0x%08" PRIX32, s_raw_crash_data.pc); + if (has_fault_addr()) { + ESP_LOGE(TAG, " %s: 0x%08" PRIX32, FAULT_ADDR_REG_LOWER, s_raw_crash_data.fault_addr); + } + uint8_t bt_num = log_foreign_backtrace(s_raw_crash_data.backtrace, s_raw_crash_data.backtrace_count, 0); +#if SOC_CPU_CORES_NUM > 1 + if (s_raw_crash_data.other_backtrace_count > 0) { + // Lowercase like the address labels: carries no address, matches no decoder. + ESP_LOGE(TAG, " other core (%d):", 1 - s_raw_crash_data.crashed_core); + log_foreign_backtrace(s_raw_crash_data.other_backtrace, s_raw_crash_data.other_backtrace_count, bt_num); + } +#else + (void) bt_num; // Single-core targets have no second list to continue numbering into. +#endif +} + // Intentionally uses separate ESP_LOGE calls per line instead of combining into // one multi-line log message. This ensures each address appears as its own line // on the serial console, making it possible to see partial output if the device @@ -348,18 +420,17 @@ void crash_handler_log() { ESP_LOGE(TAG, " Reason: %s", get_exception_type()); } ESP_LOGE(TAG, " Crashed core: %d", s_raw_crash_data.crashed_core); + if (s_raw_crash_data.build_time != s_current_build_time) { + // Captured by a different firmware build: the record survives soft resets + // including the OTA reboot, so its addresses belong to a previous ELF. + log_foreign_addresses(); + return; + } ESP_LOGE(TAG, " PC: 0x%08" PRIX32 " (fault location)", s_raw_crash_data.pc); - // Faulting memory address — only meaningful for real CPU faults, not - // aborts/watchdogs or SoC-level pseudo exceptions. Uses the same register - // name as ESP-IDF's live register dump for the architecture (EXCVADDR on - // Xtensa, MTVAL on RISC-V) so the CLI decodes it when it happens to be a - // code address. - if (s_raw_crash_data.exception == PANIC_EXCEPTION_FAULT && !s_raw_crash_data.pseudo_excause) { -#if CONFIG_IDF_TARGET_ARCH_XTENSA - ESP_LOGE(TAG, " EXCVADDR: 0x%08" PRIX32 " (faulting address)", s_raw_crash_data.fault_addr); -#elif CONFIG_IDF_TARGET_ARCH_RISCV - ESP_LOGE(TAG, " MTVAL: 0x%08" PRIX32 " (faulting address)", s_raw_crash_data.fault_addr); -#endif + // Uses the same register name as ESP-IDF's live register dump so the CLI + // decodes the address when it happens to be a code address. + if (has_fault_addr()) { + ESP_LOGE(TAG, " %s: 0x%08" PRIX32 " (faulting address)", FAULT_ADDR_REG, s_raw_crash_data.fault_addr); } log_backtrace(s_raw_crash_data.backtrace, s_raw_crash_data.backtrace_count, s_raw_crash_data.reg_frame_count); @@ -375,14 +446,7 @@ void crash_handler_log() { // Build addr2line hint with all captured addresses for easy copy-paste char hint[256]; int pos = snprintf(hint, sizeof(hint), "Use: addr2line -pfiaC -e firmware.elf 0x%08" PRIX32, s_raw_crash_data.pc); - pos = append_addrs_to_hint(hint, sizeof(hint), pos, s_raw_crash_data.backtrace, s_raw_crash_data.backtrace_count, - s_raw_crash_data.reg_frame_count); -#if SOC_CPU_CORES_NUM > 1 - append_addrs_to_hint(hint, sizeof(hint), pos, s_raw_crash_data.other_backtrace, - s_raw_crash_data.other_backtrace_count, s_raw_crash_data.other_reg_frame_count); -#else - (void) pos; // There is no second-core append on single-core targets, so pos would otherwise be unread. -#endif + append_all_backtraces(hint, sizeof(hint), pos); ESP_LOGE(TAG, "%s", hint); } @@ -408,6 +472,11 @@ void IRAM_ATTR __wrap_esp_panic_handler(panic_info_t *info) { // Zero unconditionally so a null frame doesn't leave stale .noinit data from a previous boot s_raw_crash_data.cause = 0; s_raw_crash_data.fault_addr = 0; + // Record which build's ELF the captured addresses belong to (RAM read, panic-safe). + // Still 0 if the panic precedes C++ dynamic initialization, so such a crash + // reports as a foreign build — conservative: addresses are shown raw instead + // of decoded. + s_raw_crash_data.build_time = esphome::esp32::s_current_build_time; #if SOC_CPU_CORES_NUM > 1 s_raw_crash_data.other_backtrace_count = 0; s_raw_crash_data.other_reg_frame_count = 0; diff --git a/esphome/components/esp32/preferences.cpp b/esphome/components/esp32/preferences.cpp index dc2b40455c..f3d5844cd7 100644 --- a/esphome/components/esp32/preferences.cpp +++ b/esphome/components/esp32/preferences.cpp @@ -176,12 +176,21 @@ ESPPreferenceObject ESP32Preferences::make_preference(size_t length, uint32_t ty } s_open_err = ESP_OK; } - auto *pref = new ESP32PreferenceBackend(); // NOLINT(cppcoreguidelines-owning-memory) - pref->nvs_handle = this->nvs_handle; - pref->key = type; - pref->in_flash = true; + // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) + return ESPPreferenceObject(new ESP32PreferenceBackend(this->make_backend_(type))); +} - return ESPPreferenceObject(pref); +ESP32PreferenceBackend ESP32Preferences::make_backend_(uint32_t type) const { + // in_flash keeps its default of true, selecting the NVS path + ESP32PreferenceBackend backend; + backend.nvs_handle = this->nvs_handle; + backend.key = type; + return backend; +} + +bool ESP32Preferences::load_from_key(uint32_t type, uint8_t *data, size_t len) { + ESP32PreferenceBackend backend = this->make_backend_(type); + return backend.load(data, len); } #ifdef USE_ESP32_RTC_PREFERENCES_STORAGE diff --git a/esphome/components/esp32/preferences.h b/esphome/components/esp32/preferences.h index 864d22312b..9125843958 100644 --- a/esphome/components/esp32/preferences.h +++ b/esphome/components/esp32/preferences.h @@ -23,12 +23,15 @@ class ESP32Preferences final : public PreferencesMixin { ESPPreferenceObject make_preference(size_t length, uint32_t type, bool in_flash); // Two-argument form defaults to NVS (flash) storage, preserving historic ESP32 behavior. ESPPreferenceObject make_preference(size_t length, uint32_t type); + /// One-shot read of a stored preference by key, without allocating a backend + bool load_from_key(uint32_t type, uint8_t *data, size_t len); bool sync(); bool reset(); uint32_t nvs_handle; protected: + ESP32PreferenceBackend make_backend_(uint32_t type) const; bool is_changed_(uint32_t nvs_handle, const NVSData &to_save, const char *key_str); #ifdef USE_ESP32_RTC_PREFERENCES_STORAGE diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py index c8613963b9..935d8b1b7e 100644 --- a/esphome/components/esp32_ble/__init__.py +++ b/esphome/components/esp32_ble/__init__.py @@ -1,5 +1,4 @@ from collections.abc import Callable, MutableMapping -from dataclasses import dataclass from enum import Enum import logging from typing import Any @@ -32,7 +31,7 @@ from esphome.const import ( CONF_NAME, CONF_NAME_ADD_MAC_SUFFIX, ) -from esphome.core import CORE, CoroPriority, TimePeriod, coroutine_with_priority +from esphome.core import CORE, TimePeriod import esphome.final_validate as fv from esphome.types import ConfigType @@ -134,18 +133,21 @@ def _get_required_loggers() -> set[BTLoggers]: return CORE.data.setdefault(ESP32_BLE_REQUIRED_LOGGERS_KEY, set()) -# Dataclass for handler registration counts -@dataclass -class HandlerCounts: - gap_event: int = 0 - gap_scan_event: int = 0 - gattc_event: int = 0 - gatts_event: int = 0 - ble_status_event: int = 0 - - -# Track handler registration counts for StaticVector sizing -_handler_counts = HandlerCounts() +# Handler slot counters sizing the StaticCallbackManager storage in ble.h; +# one request per register_* call below. +_request_gap_event_slot = cg.slot_counter("ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT") +_request_gap_scan_event_slot = cg.slot_counter( + "ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT" +) +_request_gattc_event_slot = cg.slot_counter( + "ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT" +) +_request_gatts_event_slot = cg.slot_counter( + "ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT" +) +_request_ble_status_event_slot = cg.slot_counter( + "ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT" +) def _add_callback( @@ -171,8 +173,8 @@ def _add_callback( def register_gap_event_handler(parent_var: cg.MockObj, handler_var: cg.MockObj) -> None: - """Register a GAP event handler and track the count.""" - _handler_counts.gap_event += 1 + """Register a GAP event handler and request a handler slot.""" + _request_gap_event_slot() _add_callback( parent_var, "add_gap_event_callback", @@ -185,8 +187,8 @@ def register_gap_event_handler(parent_var: cg.MockObj, handler_var: cg.MockObj) def register_gap_scan_event_handler( parent_var: cg.MockObj, handler_var: cg.MockObj ) -> None: - """Register a GAP scan event handler and track the count.""" - _handler_counts.gap_scan_event += 1 + """Register a GAP scan event handler and request a handler slot.""" + _request_gap_scan_event_slot() _add_callback( parent_var, "add_gap_scan_event_callback", @@ -199,8 +201,8 @@ def register_gap_scan_event_handler( def register_gattc_event_handler( parent_var: cg.MockObj, handler_var: cg.MockObj ) -> None: - """Register a GATTc event handler and track the count.""" - _handler_counts.gattc_event += 1 + """Register a GATTc event handler and request a handler slot.""" + _request_gattc_event_slot() _add_callback( parent_var, "add_gattc_event_callback", @@ -213,8 +215,8 @@ def register_gattc_event_handler( def register_gatts_event_handler( parent_var: cg.MockObj, handler_var: cg.MockObj ) -> None: - """Register a GATTs event handler and track the count.""" - _handler_counts.gatts_event += 1 + """Register a GATTs event handler and request a handler slot.""" + _request_gatts_event_slot() _add_callback( parent_var, "add_gatts_event_callback", @@ -227,8 +229,8 @@ def register_gatts_event_handler( def register_ble_status_event_handler( parent_var: cg.MockObj, handler_var: cg.MockObj ) -> None: - """Register a BLE status event handler and track the count.""" - _handler_counts.ble_status_event += 1 + """Register a BLE status event handler and request a handler slot.""" + _request_ble_status_event_slot() _add_callback( parent_var, "add_ble_status_event_callback", @@ -518,36 +520,6 @@ def final_validation(config): FINAL_VALIDATE_SCHEMA = final_validation -# This needs to be run as a job with CoroPriority.FINAL priority so that all components have -# a chance to register their handlers before the counts are added to defines. -@coroutine_with_priority(CoroPriority.FINAL) -async def _add_ble_handler_defines(): - # Add defines for StaticVector sizing based on handler registration counts - # Only define if count > 0 to avoid allocating unnecessary memory - if _handler_counts.gap_event > 0: - cg.add_define( - "ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT", _handler_counts.gap_event - ) - if _handler_counts.gap_scan_event > 0: - cg.add_define( - "ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT", - _handler_counts.gap_scan_event, - ) - if _handler_counts.gattc_event > 0: - cg.add_define( - "ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT", _handler_counts.gattc_event - ) - if _handler_counts.gatts_event > 0: - cg.add_define( - "ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT", _handler_counts.gatts_event - ) - if _handler_counts.ble_status_event > 0: - cg.add_define( - "ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT", - _handler_counts.ble_status_event, - ) - - async def to_code(config): var = cg.new_Pvariable(config[CONF_ID]) cg.add(var.set_enable_on_boot(config[CONF_ENABLE_ON_BOOT])) @@ -576,7 +548,7 @@ async def to_code(config): max_connections = config.get(CONF_MAX_CONNECTIONS, DEFAULT_MAX_CONNECTIONS) cg.add_define("USE_ESP32_BLE_MAX_CONNECTIONS", max_connections) - request_bluetooth(ble_42=True) + request_bluetooth() # When PSRAM and BT are used together, Bluedroid should prefer SPIRAM for # heap allocations and use dynamic (heap-based) environment memory tables @@ -633,9 +605,6 @@ async def to_code(config): cg.add_define("USE_ESP32_BLE_ADVERTISING") cg.add_define("USE_ESP32_BLE_UUID") - # Schedule the handler defines to be added after all components register - CORE.add_job(_add_ble_handler_defines) - @automation.register_condition("ble.enabled", BLEEnabledCondition, cv.Schema({})) async def ble_enabled_to_code(config, condition_id, template_arg, args): diff --git a/esphome/components/esp32_ble_beacon/__init__.py b/esphome/components/esp32_ble_beacon/__init__.py index 7a59cce19b..d762255040 100644 --- a/esphome/components/esp32_ble_beacon/__init__.py +++ b/esphome/components/esp32_ble_beacon/__init__.py @@ -86,4 +86,4 @@ async def to_code(config): cg.add_define("USE_ESP32_BLE_ADVERTISING") - request_bluetooth(ble_42=True) + request_bluetooth() diff --git a/esphome/components/esp32_ble_tracker/__init__.py b/esphome/components/esp32_ble_tracker/__init__.py index 7ffde76429..b8f49d4fbd 100644 --- a/esphome/components/esp32_ble_tracker/__init__.py +++ b/esphome/components/esp32_ble_tracker/__init__.py @@ -1,12 +1,11 @@ from __future__ import annotations -from dataclasses import dataclass import logging from esphome import automation import esphome.codegen as cg from esphome.components import ble_device_base, esp32_ble, ota -from esphome.components.const import CONF_SCAN_PARAMETERS, CONF_WINDOW +from esphome.components.const import CONF_ON_SCAN_END, CONF_SCAN_PARAMETERS, CONF_WINDOW from esphome.components.esp32 import ( add_idf_sdkconfig_option, request_bluetooth, @@ -44,8 +43,9 @@ AUTO_LOAD = ["ble_device_base", "esp32_ble"] DEPENDENCIES = ["esp32"] CODEOWNERS = ["@bdraco"] +ble_device_base.register_hub_provider("esp32_ble_tracker") + CONF_ESP32_BLE_ID = "esp32_ble_id" -CONF_ON_SCAN_END = "on_scan_end" CONF_SOFTWARE_COEXISTENCE = "software_coexistence" _LOGGER = logging.getLogger(__name__) @@ -56,16 +56,8 @@ class BLEFeatures(StrEnum): ESP_BT_DEVICE = "ESP_BT_DEVICE" -# Dataclass for registration counts -@dataclass -class RegistrationCounts: - listeners: int = 0 - clients: int = 0 - - -# CORE.data keys for state management +# CORE.data key for state management ESP32_BLE_TRACKER_REQUIRED_FEATURES_KEY = "esp32_ble_tracker_required_features" -ESP32_BLE_TRACKER_REGISTRATION_COUNTS_KEY = "esp32_ble_tracker_registration_counts" def _get_required_features() -> set[BLEFeatures]: @@ -73,11 +65,13 @@ def _get_required_features() -> set[BLEFeatures]: return CORE.data.setdefault(ESP32_BLE_TRACKER_REQUIRED_FEATURES_KEY, set()) -def _get_registration_counts() -> RegistrationCounts: - """Get the registration counts from CORE.data.""" - return CORE.data.setdefault( - ESP32_BLE_TRACKER_REGISTRATION_COUNTS_KEY, RegistrationCounts() - ) +# Slot counters sizing the tracker's StaticVector storage; one request per +# registered listener, client, or scanner state listener. +_request_listener_slot = cg.slot_counter("ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT") +_request_client_slot = cg.slot_counter("ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT") +_request_scanner_state_listener_slot = cg.slot_counter( + "ESPHOME_ESP32_BLE_TRACKER_SCANNER_STATE_LISTENER_COUNT" +) def register_ble_features(features: set[BLEFeatures]) -> None: @@ -239,17 +233,15 @@ async def to_code(config): ): register_ble_features({BLEFeatures.ESP_BT_DEVICE}) - registration_counts = _get_registration_counts() - for conf in config.get(CONF_ON_BLE_ADVERTISE, []): - registration_counts.listeners += 1 + _request_listener_slot() trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) if CONF_MAC_ADDRESS in conf: addr_list = [it.as_hex for it in conf[CONF_MAC_ADDRESS]] cg.add(trigger.set_addresses(addr_list)) await automation.build_automation(trigger, [(ESPBTDeviceConstRef, "x")], conf) for conf in config.get(CONF_ON_BLE_SERVICE_DATA_ADVERTISE, []): - registration_counts.listeners += 1 + _request_listener_slot() trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) if len(conf[CONF_SERVICE_UUID]) == len(bt_uuid16_format): cg.add(trigger.set_service_uuid16(as_hex(conf[CONF_SERVICE_UUID]))) @@ -262,7 +254,7 @@ async def to_code(config): cg.add(trigger.set_address(conf[CONF_MAC_ADDRESS].as_hex)) await automation.build_automation(trigger, [(adv_data_t_const_ref, "x")], conf) for conf in config.get(CONF_ON_BLE_MANUFACTURER_DATA_ADVERTISE, []): - registration_counts.listeners += 1 + _request_listener_slot() trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) if len(conf[CONF_MANUFACTURER_ID]) == len(bt_uuid16_format): cg.add(trigger.set_manufacturer_uuid16(as_hex(conf[CONF_MANUFACTURER_ID]))) @@ -275,7 +267,7 @@ async def to_code(config): cg.add(trigger.set_address(conf[CONF_MAC_ADDRESS].as_hex)) await automation.build_automation(trigger, [(adv_data_t_const_ref, "x")], conf) for conf in config.get(CONF_ON_SCAN_END, []): - registration_counts.listeners += 1 + _request_listener_slot() trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], var) await automation.build_automation(trigger, [], conf) @@ -308,28 +300,14 @@ async def _add_ble_features(): required_features = _get_required_features() # Sensors registered through the neutral ble_device_base path (BLEHub) need # the parsed-device pipeline compiled in, exactly like esp32-path listeners. - neutral_listener_count = ble_device_base.get_listener_count() - if neutral_listener_count > 0: + if cg.get_slot_count(ble_device_base.LISTENER_COUNT_DEFINE): + # The neutral (BLEHub) listener count define itself is emitted by + # ble_device_base's own job; only the feature coupling lives here. required_features.add(BLEFeatures.ESP_BT_DEVICE) - # StaticVector sizing for the neutral (BLEHub) listener list — same - # pattern as the esp32-path registration counts below. - cg.add_define("ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT", neutral_listener_count) if BLEFeatures.ESP_BT_DEVICE in required_features: cg.add_define("USE_ESP32_BLE_DEVICE") cg.add_define("USE_ESP32_BLE_UUID") - # Add defines for StaticVector sizing based on registration counts - # Only define if count > 0 to avoid allocating unnecessary memory - registration_counts = _get_registration_counts() - if registration_counts.listeners > 0: - cg.add_define( - "ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT", registration_counts.listeners - ) - if registration_counts.clients > 0: - cg.add_define( - "ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT", registration_counts.clients - ) - ESP32_BLE_START_SCAN_ACTION_SCHEMA = cv.Schema( { @@ -382,7 +360,7 @@ async def register_ble_device( var: cg.SafeExpType, config: ConfigType ) -> cg.SafeExpType: register_ble_features({BLEFeatures.ESP_BT_DEVICE}) - _get_registration_counts().listeners += 1 + _request_listener_slot() paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) cg.add(paren.register_listener(var)) return var @@ -390,7 +368,7 @@ async def register_ble_device( async def register_client(var: cg.SafeExpType, config: ConfigType) -> cg.SafeExpType: register_ble_features({BLEFeatures.ESP_BT_DEVICE}) - _get_registration_counts().clients += 1 + _request_client_slot() paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) cg.add(paren.register_client(var)) return var @@ -404,7 +382,7 @@ async def register_raw_ble_device( This does NOT register the ESP_BT_DEVICE feature, meaning ESPBTDevice will not be compiled in if this is the only registration method used. """ - _get_registration_counts().listeners += 1 + _request_listener_slot() paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) cg.add(paren.register_listener(var)) return var @@ -418,7 +396,21 @@ async def register_raw_client( This does NOT register the ESP_BT_DEVICE feature, meaning ESPBTDevice will not be compiled in if this is the only registration method used. """ - _get_registration_counts().clients += 1 + _request_client_slot() paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) cg.add(paren.register_client(var)) return var + + +async def register_scanner_state_listener( + var: cg.SafeExpType, config: ConfigType +) -> cg.SafeExpType: + """Register a listener for scanner state changes. + + The slot request here is what sizes the tracker's listener storage; a + build with no registrations compiles the storage out entirely. + """ + _request_scanner_state_listener_slot() + paren = await cg.get_variable(config[CONF_ESP32_BLE_ID]) + cg.add(paren.add_scanner_state_listener(var)) + return var diff --git a/esphome/components/esp32_ble_tracker/automation.h b/esphome/components/esp32_ble_tracker/automation.h index b653325f56..541b63b2fd 100644 --- a/esphome/components/esp32_ble_tracker/automation.h +++ b/esphome/components/esp32_ble_tracker/automation.h @@ -3,6 +3,8 @@ #include "esphome/core/automation.h" #include "esphome/components/esp32_ble_tracker/esp32_ble_tracker.h" +#include + #ifdef USE_ESP32 namespace esphome::esp32_ble_tracker { diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp index 0c1a98c4be..8418fc3fec 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp @@ -36,27 +36,6 @@ static const char *const TAG = "esp32_ble_tracker"; ESP32BLETracker *global_esp32_ble_tracker = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -const char *client_state_to_string(ClientState state) { - switch (state) { - case ClientState::INIT: - return "INIT"; - case ClientState::DISCONNECTING: - return "DISCONNECTING"; - case ClientState::IDLE: - return "IDLE"; - case ClientState::DISCOVERED: - return "DISCOVERED"; - case ClientState::CONNECTING: - return "CONNECTING"; - case ClientState::CONNECTED: - return "CONNECTED"; - case ClientState::ESTABLISHED: - return "ESTABLISHED"; - default: - return "UNKNOWN"; - } -} - float ESP32BLETracker::get_setup_priority() const { return setup_priority::AFTER_BLUETOOTH; } void ESP32BLETracker::setup() { @@ -443,9 +422,11 @@ void ESP32BLETracker::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_i void ESP32BLETracker::set_scanner_state_(ScannerState state) { this->scanner_state_ = state; this->state_version_++; +#ifdef ESPHOME_ESP32_BLE_TRACKER_SCANNER_STATE_LISTENER_COUNT for (auto *listener : this->scanner_state_listeners_) { listener->on_scanner_state(state); } +#endif } void ESP32BLETracker::dump_config() { @@ -499,6 +480,8 @@ void ESP32BLETracker::process_scan_result_(const BLEScanResult &scan_result) { if (this->parse_advertisements_) { #ifdef USE_ESP32_BLE_DEVICE ESPBTDevice device; + // The historical ingest keeps the raw scan-result fields populated for + // external components. device.parse_scan_rst(scan_result); bool found = false; diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h index b0357289f1..88642fff6b 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.h @@ -8,7 +8,6 @@ #include #include #include -#include #ifdef USE_ESP32 @@ -19,6 +18,7 @@ #include #include +#include "esphome/components/ble_device_base/ble_client_state.h" #include "esphome/components/ble_device_base/ble_device.h" #include "esphome/components/ble_device_base/ble_hub.h" #include "esphome/components/esp32_ble/ble.h" @@ -89,22 +89,11 @@ struct ClientStateCounts { bool operator!=(const ClientStateCounts &other) const { return !(*this == other); } }; -enum class ClientState : uint8_t { - // Connection is allocated - INIT, - // Client is disconnecting - DISCONNECTING, - // Connection is idle, no device detected. - IDLE, - // Device advertisement found. - DISCOVERED, - // Connection in progress. - CONNECTING, - // Initial connection established. - CONNECTED, - // The client and sub-clients have completed setup. - ESTABLISHED, -}; +// The client connection state types are owned by the platform-neutral +// ble_device_base layer; re-exported here for backward compatibility. +using ClientState = ble_device_base::ClientState; +using ConnectionType = ble_device_base::ConnectionType; +using ble_device_base::client_state_to_string; enum class ScannerState { // Scanner is idle, init state @@ -129,21 +118,6 @@ class BLEScannerStateListener { virtual void on_scanner_state(ScannerState state) = 0; }; -// Helper function to convert ClientState to string -const char *client_state_to_string(ClientState state); - -enum class ConnectionType : uint8_t { - // The default connection type, we hold all the services in ram - // for the duration of the connection. - V1, - // The client has a cache of the services and mtu so we should not - // fetch them again - V3_WITH_CACHE, - // The client does not need the services and mtu once we send them - // so we should wipe them from memory as soon as we send them - V3_WITHOUT_CACHE -}; - /// Base class for BLE GATT clients that connect to remote devices. /// /// State Change Tracking Design: @@ -243,7 +217,10 @@ class ESP32BLETracker final : public Component, this->raw_advertisement_callback_ = callback; } ble_device_base::HubCapabilities get_capabilities() const override { - return {/* active_scan = */ true, /* merges_scan_response = */ true, /* gatt = */ true}; + // scan_mode_switch is false: the mode is driven through this tracker's own + // API (set_scan_active + restart), not the neutral request_scan_mode(). + return {/* active_scan = */ true, /* merges_scan_response = */ true, /* gatt = */ true, + /* scan_mode_switch = */ false}; } void get_adapter_mac(uint8_t out[6]) override; bool scan_running() override { return this->scanner_state_ == ScannerState::RUNNING; } @@ -265,10 +242,15 @@ class ESP32BLETracker final : public Component, void on_ota_global_state(ota::OTAState state, float progress, uint8_t error, ota::OTAComponent *comp) override; #endif - /// Add a listener for scanner state changes +#ifdef ESPHOME_ESP32_BLE_TRACKER_SCANNER_STATE_LISTENER_COUNT + /// Add a listener for scanner state changes. Only compiled when a consumer + /// requested a slot in codegen: register through + /// esp32_ble_tracker.register_scanner_state_listener() in your component's + /// to_code, which requests the slot and emits this call. void add_scanner_state_listener(BLEScannerStateListener *listener) { this->scanner_state_listeners_.push_back(listener); } +#endif ScannerState get_scanner_state() const { return this->scanner_state_; } protected: @@ -335,7 +317,10 @@ class ESP32BLETracker final : public Component, #ifdef ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT StaticVector clients_; #endif - std::vector scanner_state_listeners_; +#ifdef ESPHOME_ESP32_BLE_TRACKER_SCANNER_STATE_LISTENER_COUNT + StaticVector + scanner_state_listeners_; +#endif // Parsed listeners registered through the neutral BLEHub contract (migrated // sensors); dispatched alongside listeners_. #ifdef ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT diff --git a/esphome/components/esp32_camera_web_server/camera_web_server.cpp b/esphome/components/esp32_camera_web_server/camera_web_server.cpp index 7527bbf7e4..88579e9632 100644 --- a/esphome/components/esp32_camera_web_server/camera_web_server.cpp +++ b/esphome/components/esp32_camera_web_server/camera_web_server.cpp @@ -13,7 +13,9 @@ namespace esphome::esp32_camera_web_server { -static const int IMAGE_REQUEST_TIMEOUT = 5000; +static const uint32_t IMAGE_REQUEST_TIMEOUT = 5000; +// How often streaming_handler_ reports its throughput. +static const uint32_t STREAM_STATS_INTERVAL = 5000; static const char *const TAG = "esp32_camera_web_server"; #define PART_BOUNDARY "123456789000000000000987654321" @@ -113,10 +115,31 @@ std::shared_ptr CameraWebServer::wait_for_image_() std::shared_ptr image; image.swap(this->image_); - if (!image) { - // retry as we might still be fetching image - xSemaphoreTake(this->semaphore_, IMAGE_REQUEST_TIMEOUT / portTICK_PERIOD_MS); + if (image) + return image; + + // Keep waiting until a frame really shows up, rather than trusting a single + // take() to mean one is there. + // + // on_camera_image() gives the semaphore for every frame it accepts, but the + // swap above hands frames out without taking it, so as soon as the camera is + // faster than this task for one frame the (binary) semaphore is left + // signalled by a frame that has already been consumed. The next take() then + // returns immediately with nothing to swap in, and the caller reports a lost + // frame and closes the stream -- after an arbitrary number of good frames, + // which is exactly when the camera happens to fall behind for one iteration. + // + // running_ is re-checked on every pass so a shutdown or a client that went + // away is noticed straight away instead of after the full timeout. + const uint32_t start = millis(); + while (this->running_) { + uint32_t elapsed = millis() - start; + if (elapsed >= IMAGE_REQUEST_TIMEOUT) + break; + xSemaphoreTake(this->semaphore_, pdMS_TO_TICKS(IMAGE_REQUEST_TIMEOUT - elapsed)); image.swap(this->image_); + if (image) + break; } return image; @@ -170,8 +193,14 @@ esp_err_t CameraWebServer::streaming_handler_(struct httpd_req *req) { return res; } - uint32_t last_frame = millis(); uint32_t frames = 0; + // Frame statistics are aggregated over STREAM_STATS_INTERVAL rather than + // logged per frame. A line per frame comes out of this (non-main) task tens + // of times a second, and formatting and buffering it costs more than the + // stream it is reporting on. + uint32_t stats_since = millis(); + uint32_t stats_frames = 0; + uint32_t stats_bytes = 0; camera::Camera::instance()->start_stream(esphome::camera::WEB_REQUESTER); @@ -179,7 +208,10 @@ esp_err_t CameraWebServer::streaming_handler_(struct httpd_req *req) { auto image = this->wait_for_image_(); if (!image) { - ESP_LOGW(TAG, "STREAM: failed to acquire frame"); + // A shutdown is not a lost frame: wait_for_image_() returns empty as soon + // as running_ clears, and the loop condition below ends the stream anyway. + if (this->running_) + ESP_LOGW(TAG, "STREAM: failed to acquire frame"); res = ESP_FAIL; } if (res == ESP_OK) { @@ -194,14 +226,29 @@ esp_err_t CameraWebServer::streaming_handler_(struct httpd_req *req) { } if (res == ESP_OK) { frames++; - int64_t frame_time = millis() - last_frame; - last_frame = millis(); - - ESP_LOGD(TAG, "MJPG: %" PRIu32 "B %" PRIu32 "ms (%.1ffps)", (uint32_t) image->get_data_length(), - (uint32_t) frame_time, 1000.0 / (uint32_t) frame_time); + stats_frames++; + stats_bytes += image->get_data_length(); + uint32_t elapsed = millis() - stats_since; + if (elapsed >= STREAM_STATS_INTERVAL) { + ESP_LOGD(TAG, "MJPG: %.1ffps, %" PRIu32 "B/frame (%" PRIu32 " frames)", stats_frames * 1000.0f / elapsed, + stats_bytes / stats_frames, stats_frames); + stats_since = millis(); + stats_frames = 0; + stats_bytes = 0; + } } } + // Report whatever did not fill a whole interval, so a stream that only ran for + // a second or two still says what it managed rather than nothing at all. + if (stats_frames > 0) { + uint32_t elapsed = millis() - stats_since; + if (elapsed == 0) + elapsed = 1; + ESP_LOGD(TAG, "MJPG: %.1ffps, %" PRIu32 "B/frame (%" PRIu32 " frames)", stats_frames * 1000.0f / elapsed, + stats_bytes / stats_frames, stats_frames); + } + if (!frames) { res = httpd_send_all(req, STREAM_ERROR, strlen(STREAM_ERROR)); } diff --git a/esphome/components/esp32_rmt_led_strip/led_strip.cpp b/esphome/components/esp32_rmt_led_strip/led_strip.cpp index ed2a8c5a68..95391ef100 100644 --- a/esphome/components/esp32_rmt_led_strip/led_strip.cpp +++ b/esphome/components/esp32_rmt_led_strip/led_strip.cpp @@ -255,11 +255,11 @@ light::ESPColorView ESP32RMTLEDStripLightOutput::get_view_internal(int32_t index break; } uint8_t multiplier = this->is_rgbw_ || this->is_wrgb_ ? 4 : 3; - uint8_t white = this->is_wrgb_ ? 0 : 3; + uint8_t white = this->is_wrgb_ ? 0 : this->white_index_; - return {this->buf_ + (index * multiplier) + r + this->is_wrgb_, - this->buf_ + (index * multiplier) + g + this->is_wrgb_, - this->buf_ + (index * multiplier) + b + this->is_wrgb_, + return {this->buf_ + (index * multiplier) + r + (white <= r), + this->buf_ + (index * multiplier) + g + (white <= g), + this->buf_ + (index * multiplier) + b + (white <= b), this->is_rgbw_ || this->is_wrgb_ ? this->buf_ + (index * multiplier) + white : nullptr, &this->effect_data_[index], &this->correction_}; @@ -295,11 +295,22 @@ void ESP32RMTLEDStripLightOutput::dump_config() { rgb_order = "UNKNOWN"; break; } + if (this->is_rgbw_ || this->is_wrgb_) { + char rgbw_order[5]; + uint8_t white = this->is_wrgb_ ? 0 : this->white_index_; + uint8_t rgb_index = 0; + for (uint8_t i = 0; i < 4; i++) { + rgbw_order[i] = i == white ? 'W' : rgb_order[rgb_index++]; + } + rgbw_order[4] = '\0'; + ESP_LOGCONFIG(TAG, " RGBW Order: %s", rgbw_order); + } else { + ESP_LOGCONFIG(TAG, " RGB Order: %s", rgb_order); + } ESP_LOGCONFIG(TAG, - " RGB Order: %s\n" " Max refresh rate: %" PRIu32 "\n" " Number of LEDs: %u", - rgb_order, this->max_refresh_rate_.value_or(0), this->num_leds_); + this->max_refresh_rate_.value_or(0), this->num_leds_); } float ESP32RMTLEDStripLightOutput::get_setup_priority() const { return setup_priority::HARDWARE; } diff --git a/esphome/components/esp32_rmt_led_strip/led_strip.h b/esphome/components/esp32_rmt_led_strip/led_strip.h index d7ba2aafbf..3e31309bff 100644 --- a/esphome/components/esp32_rmt_led_strip/led_strip.h +++ b/esphome/components/esp32_rmt_led_strip/led_strip.h @@ -52,6 +52,11 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight { void set_num_leds(uint16_t num_leds) { this->num_leds_ = num_leds; } void set_is_rgbw(bool is_rgbw) { this->is_rgbw_ = is_rgbw; } void set_is_wrgb(bool is_wrgb) { this->is_wrgb_ = is_wrgb; } + void set_rgbw_order(uint8_t white_index) { + this->is_rgbw_ = true; + this->is_wrgb_ = false; + this->white_index_ = white_index; + } void set_use_dma(bool use_dma) { this->use_dma_ = use_dma; } void set_use_psram(bool use_psram) { this->use_psram_ = use_psram; } @@ -91,6 +96,8 @@ class ESP32RMTLEDStripLightOutput final : public light::AddressableLight { uint16_t num_leds_; bool is_rgbw_{false}; bool is_wrgb_{false}; + // An index after the RGB channels makes offset adjustment a no-op for three-channel strips. + uint8_t white_index_{3}; bool use_dma_{false}; bool use_psram_{false}; bool invert_out_{false}; diff --git a/esphome/components/esp32_rmt_led_strip/light.py b/esphome/components/esp32_rmt_led_strip/light.py index 1c6943b003..2722a9b656 100644 --- a/esphome/components/esp32_rmt_led_strip/light.py +++ b/esphome/components/esp32_rmt_led_strip/light.py @@ -20,6 +20,7 @@ from esphome.const import ( CONF_RMT_SYMBOLS, CONF_USE_DMA, ) +from esphome.types import ConfigType _LOGGER = logging.getLogger(__name__) @@ -62,6 +63,7 @@ CHIPSETS = { } CONF_IS_WRGB = "is_wrgb" +CONF_RGBW_ORDER = "rgbw_order" CONF_BIT0_HIGH = "bit0_high" CONF_BIT0_LOW = "bit0_low" CONF_BIT1_HIGH = "bit1_high" @@ -70,6 +72,26 @@ CONF_RESET_HIGH = "reset_high" CONF_RESET_LOW = "reset_low" +def _validate_rgbw_order(value: str) -> str: + value = cv.string(value).upper() + if len(value) != 4 or set(value) != set("RGBW"): + raise cv.Invalid("RGBW order must be a permutation of RGBW") + return value + + +def _split_rgbw_order(rgbw_order: str) -> tuple[str, int]: + return rgbw_order.replace("W", ""), rgbw_order.index("W") + + +def _validate_rgbw_order_exclusivity(config: ConfigType) -> ConfigType: + if CONF_RGBW_ORDER in config and (config[CONF_IS_RGBW] or config[CONF_IS_WRGB]): + raise cv.Invalid( + f"'{CONF_RGBW_ORDER}' cannot be used with '{CONF_IS_RGBW}' or " + f"'{CONF_IS_WRGB}'" + ) + return config + + CONFIG_SCHEMA = cv.All( esp32.only_on_variant( unsupported=list(esp32_rmt.VARIANTS_NO_RMT), @@ -80,7 +102,8 @@ CONFIG_SCHEMA = cv.All( cv.GenerateID(CONF_OUTPUT_ID): cv.declare_id(ESP32RMTLEDStripLightOutput), cv.Required(CONF_PIN): pins.internal_gpio_output_pin_schema, cv.Required(CONF_NUM_LEDS): cv.positive_not_null_int, - cv.Required(CONF_RGB_ORDER): cv.enum(RGB_ORDERS, upper=True), + cv.Optional(CONF_RGB_ORDER): cv.enum(RGB_ORDERS, upper=True), + cv.Optional(CONF_RGBW_ORDER): _validate_rgbw_order, cv.SplitDefault( CONF_RMT_SYMBOLS, esp32=192, @@ -130,6 +153,8 @@ CONFIG_SCHEMA = cv.All( } ).extend(cv.COMPONENT_SCHEMA), cv.has_exactly_one_key(CONF_CHIPSET, CONF_BIT0_HIGH), + cv.has_exactly_one_key(CONF_RGB_ORDER, CONF_RGBW_ORDER), + _validate_rgbw_order_exclusivity, ) @@ -173,9 +198,14 @@ async def to_code(config): ) ) - cg.add(var.set_rgb_order(config[CONF_RGB_ORDER])) - cg.add(var.set_is_rgbw(config[CONF_IS_RGBW])) - cg.add(var.set_is_wrgb(config[CONF_IS_WRGB])) + if (rgbw_order := config.get(CONF_RGBW_ORDER)) is not None: + rgb_order, white_index = _split_rgbw_order(rgbw_order) + cg.add(var.set_rgb_order(RGB_ORDERS[rgb_order])) + cg.add(var.set_rgbw_order(white_index)) + else: + cg.add(var.set_rgb_order(config[CONF_RGB_ORDER])) + cg.add(var.set_is_rgbw(config[CONF_IS_RGBW])) + cg.add(var.set_is_wrgb(config[CONF_IS_WRGB])) cg.add(var.set_use_psram(config[CONF_USE_PSRAM])) cg.add(var.set_rmt_symbols(config[CONF_RMT_SYMBOLS])) if CONF_USE_DMA in config: diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 618bf775a0..1f7159919d 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -299,6 +299,7 @@ async def to_code(config): "pre:testing_mode.py", "pre:exclude_updater.py", "pre:exclude_waveform.py", + "pre:relocate_ratetable.py", ] if not enable_scanf_float: extra_scripts.append("pre:remove_float_scanf.py") @@ -451,6 +452,7 @@ def copy_files() -> None: "exclude_updater", "exclude_waveform", "remove_float_scanf", + "relocate_ratetable", ): copy_file_if_changed( dir / f"{script}.py.script", diff --git a/esphome/components/esp8266/const.py b/esphome/components/esp8266/const.py index 229ac61f24..3e89ab989f 100644 --- a/esphome/components/esp8266/const.py +++ b/esphome/components/esp8266/const.py @@ -1,4 +1,8 @@ import esphome.codegen as cg + +# Re-exported from the shared definition; here it indexes the BOARDS +# metadata dicts, whose entries in boards.py spell the literal. +from esphome.const import KEY_FLASH_SIZE # noqa: F401 # pylint: disable=unused-import from esphome.core import CORE KEY_ESP8266 = "esp8266" @@ -8,7 +12,6 @@ CONF_RESTORE_FROM_FLASH = "restore_from_flash" CONF_EARLY_PIN_INIT = "early_pin_init" CONF_ENABLE_SERIAL = "enable_serial" CONF_ENABLE_SERIAL1 = "enable_serial1" -KEY_FLASH_SIZE = "flash_size" KEY_WAVEFORM_REQUIRED = "waveform_required" KEY_SERIAL_REQUIRED = "serial_required" KEY_SERIAL1_REQUIRED = "serial1_required" diff --git a/esphome/components/esp8266/relocate_ratetable.py.script b/esphome/components/esp8266/relocate_ratetable.py.script new file mode 100644 index 0000000000..c9d0ba166d --- /dev/null +++ b/esphome/components/esp8266/relocate_ratetable.py.script @@ -0,0 +1,70 @@ +# pylint: disable=E0602 +Import("env") # noqa + +# Move the NONOS SDK wifi rate tables from flash to DRAM +# +# libnet80211.a ships its 802.11b/11g rate tables in the .irom.text section +# of ieee80211_phy.o (440 bytes of pure data, no relocations). The Arduino +# core linker script places .irom.text in flash, but the SDK reads these +# tables with byte loads and ets_memcpy from the wifi RX path while parsing +# beacons. Byte access to flash-mapped memory from that context misbehaves +# and crashes with StoreProhibited in ROM memcpy (PC 0x4000df64): +# +# scan_parse_beacon -> cnx_update_bss_more -> ieee80211_phy_init +# -> ieee80211_setup_ratetable -> ets_memcpy -> crash +# +# See https://github.com/espressif/ESP8266_NONOS_SDK/issues/320 (1000+ +# reports). The SDK is abandoned so the fix from +# https://github.com/espressif/ESP8266_NONOS_SDK/pull/345 was never merged; +# we apply the same linker rule here: place ieee80211_phy.o's .irom.text +# inside the DRAM .data output section so the tables are copied to RAM at +# boot. Costs 440 bytes of DRAM. +# +# The rule is inserted into the working linker script that PlatformIO +# generates in the build directory (local.eagle.app.v6.common.ld). SDK +# package files are never modified. + +import re +from os.path import join + +RULE = "*libnet80211.a:ieee80211_phy.o(.irom.text .irom.text.*)" +# Match the whole line: "_data_start" is also a substring of the +# "_dport0_data_start" line in the earlier .dport0.data section +ANCHOR = re.compile(r"^\s*_data_start = ABSOLUTE\(\.\);", re.MULTILINE) + + +def relocate_ratetable(source, target, env): + """Insert the rate table DRAM rule into the generated linker script. + + Runs as a pre-action of the link step; the linker script is a declared + dependency of the elf, so it has already been generated at this point. + """ + ld_path = join(env.subst("$BUILD_DIR"), "ld", "local.eagle.app.v6.common.ld") + with open(ld_path, encoding="utf-8") as f: + contents = f.read() + + if RULE in contents: + return # Already patched (incremental build) + + match = ANCHOR.search(contents) + if match is None: + raise RuntimeError( + f"ESPHome: '_data_start' anchor not found in {ld_path}; " + "cannot apply wifi rate table DRAM relocation " + "(has the Arduino core linker script changed?)" + ) + + insert_pos = match.end() + patched = ( + contents[:insert_pos] + + "\n /* ESPHome: wifi rate tables must live in DRAM, see NONOS SDK issue 320 */" + + f"\n {RULE}" + + contents[insert_pos:] + ) + with open(ld_path, "w", encoding="utf-8") as f: + f.write(patched) + print("ESPHome: Relocated wifi rate tables to DRAM (fixes beacon parse crash)") + + +# Register the callback to run before the link step +env.AddPreAction("$BUILD_DIR/${PROGNAME}.elf", relocate_ratetable) diff --git a/esphome/components/espnow/__init__.py b/esphome/components/espnow/__init__.py index c6c90ed67a..373ef345d1 100644 --- a/esphome/components/espnow/__init__.py +++ b/esphome/components/espnow/__init__.py @@ -13,7 +13,7 @@ from esphome.const import ( CONF_TRIGGER_ID, CONF_WIFI, ) -from esphome.core import HexInt +from esphome.core import CORE, HexInt from esphome.types import ConfigType CODEOWNERS = ["@jesserockz"] @@ -151,6 +151,10 @@ async def to_code(config): cg.add_define("USE_ESPNOW") cg.add_define("USE_ESPNOW_MAX_PAYLOAD_SIZE", config[CONF_MAX_PAYLOAD_SIZE]) + + if CONF_WIFI in CORE.config: + # Track the Wi-Fi channel via connect events instead of polling every loop + wifi.request_wifi_connect_state_listener() if wifi_channel := config.get(CONF_CHANNEL): cg.add(var.set_wifi_channel(wifi_channel)) diff --git a/esphome/components/espnow/espnow_component.cpp b/esphome/components/espnow/espnow_component.cpp index f28d7f3354..df9a1b8668 100644 --- a/esphome/components/espnow/espnow_component.cpp +++ b/esphome/components/espnow/espnow_component.cpp @@ -7,7 +7,6 @@ #include #include -#include "esphome/core/application.h" #include "esphome/core/defines.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" @@ -75,6 +74,7 @@ void on_send_report(const uint8_t *mac_addr, esp_now_send_status_t status) if (packet == nullptr) { // No events available - queue is full or we're out of memory global_esp_now->receive_packet_queue_.increment_dropped_count(); + global_esp_now->enable_loop_soon_any_context(); return; } @@ -90,8 +90,8 @@ void on_send_report(const uint8_t *mac_addr, esp_now_send_status_t status) // Push always succeeds: pool is sized to queue capacity (SIZE-1), so if // allocate() returned non-null, the queue cannot be full. - // Wake main loop immediately to process ESP-NOW send event - App.wake_loop_threadsafe(); + // Re-enable and wake the main loop to process the ESP-NOW send event + global_esp_now->enable_loop_soon_any_context(); } void on_data_received(const esp_now_recv_info_t *info, const uint8_t *data, int size) { @@ -101,6 +101,7 @@ void on_data_received(const esp_now_recv_info_t *info, const uint8_t *data, int // larger frame would overflow packet_.receive.data. if (size < 0 || size > ESPNOW_MAX_DATA_LEN) { global_esp_now->receive_packet_queue_.increment_dropped_count(); + global_esp_now->enable_loop_soon_any_context(); return; } @@ -109,6 +110,7 @@ void on_data_received(const esp_now_recv_info_t *info, const uint8_t *data, int if (packet == nullptr) { // No events available - queue is full or we're out of memory global_esp_now->receive_packet_queue_.increment_dropped_count(); + global_esp_now->enable_loop_soon_any_context(); return; } @@ -120,8 +122,8 @@ void on_data_received(const esp_now_recv_info_t *info, const uint8_t *data, int // Push always succeeds: pool is sized to queue capacity (SIZE-1), so if // allocate() returned non-null, the queue cannot be full. - // Wake main loop immediately to process ESP-NOW receive event - App.wake_loop_threadsafe(); + // Re-enable and wake the main loop to process the ESP-NOW receive event + global_esp_now->enable_loop_soon_any_context(); } ESPNowComponent::ESPNowComponent() { global_esp_now = this; } @@ -156,6 +158,11 @@ bool ESPNowComponent::is_wifi_enabled() { } void ESPNowComponent::setup() { +#if defined(USE_WIFI) && defined(USE_WIFI_CONNECT_STATE_LISTENERS) + if (wifi::global_wifi_component != nullptr) { + wifi::global_wifi_component->add_connect_state_listener(this); + } +#endif if (this->enable_on_boot_) { this->enable_(); } else { @@ -163,6 +170,19 @@ void ESPNowComponent::setup() { } } +#if defined(USE_WIFI) && defined(USE_WIFI_CONNECT_STATE_LISTENERS) +void ESPNowComponent::on_wifi_connect_state(StringRef ssid, std::span bssid) { + if (ssid.empty()) { + return; // Disconnected; the channel is only meaningful while associated + } + uint8_t old_channel = this->wifi_channel_; + this->get_wifi_channel(); + if (this->wifi_channel_ != old_channel) { + ESP_LOGI(TAG, "WiFi channel changed from %d to %d", old_channel, this->wifi_channel_); + } +} +#endif + void ESPNowComponent::enable() { if (this->state_ == ESPNOW_STATE_ENABLED) return; @@ -254,15 +274,6 @@ void ESPNowComponent::apply_wifi_channel() { } void ESPNowComponent::loop() { -#ifdef USE_WIFI - if (wifi::global_wifi_component != nullptr && wifi::global_wifi_component->is_connected()) { - int32_t new_channel = wifi::global_wifi_component->get_wifi_channel(); - if (new_channel != this->wifi_channel_) { - ESP_LOGI(TAG, "Wifi Channel is changed from %d to %" PRId32 ".", this->wifi_channel_, new_channel); - this->wifi_channel_ = new_channel; - } - } -#endif // Process received packets ESPNowPacket *packet = this->receive_packet_queue_.pop(); while (packet != nullptr) { @@ -348,6 +359,15 @@ void ESPNowComponent::loop() { if (send_dropped > 0) { ESP_LOGW(TAG, "Dropped %u send packets (queue full)", send_dropped); } + + // Nothing left to do; sleep until a callback or send() re-enables the loop. + // A packet in flight (current_send_packet_) needs no loop time even when more + // packets are queued behind it: the send callback re-enables the loop when + // the result arrives, and the SENT event handler above starts the next send. + if (this->receive_packet_queue_.empty() && + (this->current_send_packet_ != nullptr || this->send_packet_queue_.empty())) { + this->disable_loop(); + } } uint8_t ESPNowComponent::get_wifi_channel() { @@ -390,6 +410,9 @@ esp_err_t ESPNowComponent::send(const uint8_t *peer_address, const uint8_t *payl packet->load_data(peer_address, payload, size, callback); // Push the packet to the send queue this->send_packet_queue_.push(packet); + // Loop may be disabled while idle; re-enable it to send the packet + // (any-context variant so callers off the main loop are safe too) + this->enable_loop_soon_any_context(); return ESP_OK; } diff --git a/esphome/components/espnow/espnow_component.h b/esphome/components/espnow/espnow_component.h index d95255c5df..af693b47cf 100644 --- a/esphome/components/espnow/espnow_component.h +++ b/esphome/components/espnow/espnow_component.h @@ -2,6 +2,7 @@ #include "esphome/core/automation.h" #include "esphome/core/component.h" +#include "esphome/core/defines.h" #ifdef USE_ESP32 @@ -9,6 +10,10 @@ #include "esphome/core/lock_free_queue.h" #include "espnow_packet.h" +#if defined(USE_WIFI) && defined(USE_WIFI_CONNECT_STATE_LISTENERS) +#include "esphome/components/wifi/wifi_component.h" +#endif + #include #include @@ -88,7 +93,11 @@ class ESPNowBroadcastHandler { virtual bool on_broadcast(const ESPNowRecvInfo &info, const uint8_t *data, uint16_t size) = 0; }; +#if defined(USE_WIFI) && defined(USE_WIFI_CONNECT_STATE_LISTENERS) +class ESPNowComponent final : public Component, public wifi::WiFiConnectStateListener { +#else class ESPNowComponent final : public Component { +#endif public: ESPNowComponent(); void setup() override; @@ -114,6 +123,11 @@ class ESPNowComponent final : public Component { void set_auto_add_peer(bool value) { this->auto_add_peer_ = value; } +#if defined(USE_WIFI) && defined(USE_WIFI_CONNECT_STATE_LISTENERS) + // WiFiConnectStateListener interface: refresh the cached channel after each (re)connect + void on_wifi_connect_state(StringRef ssid, std::span bssid) override; +#endif + void enable(); void disable(); bool is_disabled() const { return this->state_ == ESPNOW_STATE_DISABLED; }; diff --git a/esphome/components/host/__init__.py b/esphome/components/host/__init__.py index 795c1a556d..b6a3b8b615 100644 --- a/esphome/components/host/__init__.py +++ b/esphome/components/host/__init__.py @@ -43,6 +43,8 @@ CONFIG_SCHEMA = cv.All( async def to_code(config): cg.add_build_flag("-DUSE_HOST") cg.add_define("USE_NATIVE_64BIT_TIME") + # The prefs file finds stored preferences by key, so key migration is possible + cg.add_define("USE_PREFERENCE_KEY_LOOKUP") cg.add_define("USE_ESPHOME_HOST_MAC_ADDRESS", config[CONF_MAC_ADDRESS].parts) cg.add_build_flag("-std=gnu++20") cg.add_define("ESPHOME_BOARD", "host") diff --git a/esphome/components/host/preferences.h b/esphome/components/host/preferences.h index 5f723e0675..b591fa0aab 100644 --- a/esphome/components/host/preferences.h +++ b/esphome/components/host/preferences.h @@ -27,6 +27,9 @@ class HostPreferences final : public PreferencesMixin { return true; } + /// One-shot read of a stored preference by key, without allocating a backend + bool load_from_key(uint32_t type, uint8_t *data, size_t len) { return this->load(type, data, len); } + bool load(uint32_t key, uint8_t *data, size_t len) { if (len > 255) return false; diff --git a/esphome/components/infrared/infrared.cpp b/esphome/components/infrared/infrared.cpp index 9b97995a96..288b1e5c40 100644 --- a/esphome/components/infrared/infrared.cpp +++ b/esphome/components/infrared/infrared.cpp @@ -154,12 +154,8 @@ bool Infrared::on_receive(remote_base::RemoteReceiveData data) { // Forward received IR data to API server #if defined(USE_API) && defined(USE_IR_RF) if (api::global_api_server != nullptr) { -#ifdef USE_DEVICES - uint32_t device_id = this->get_device_id(); -#else - uint32_t device_id = 0; -#endif - api::global_api_server->send_infrared_rf_receive_event(device_id, this->get_object_id_hash(), &data.get_raw_data()); + api::global_api_server->send_infrared_rf_receive_event(this->get_device_id_or_zero(), this->get_entity_key(), + &data.get_raw_data()); } #endif return false; // Don't consume the event, allow other listeners to process it diff --git a/esphome/components/ld2450/sensor.py b/esphome/components/ld2450/sensor.py index ce58cedf11..ae13900e7a 100644 --- a/esphome/components/ld2450/sensor.py +++ b/esphome/components/ld2450/sensor.py @@ -1,5 +1,6 @@ import esphome.codegen as cg from esphome.components import sensor +from esphome.components.const import CONF_TARGET_COUNT import esphome.config_validation as cv from esphome.const import ( CONF_ANGLE, @@ -21,7 +22,6 @@ DEPENDENCIES = ["ld2450"] CONF_MOVING_TARGET_COUNT = "moving_target_count" CONF_STILL_TARGET_COUNT = "still_target_count" -CONF_TARGET_COUNT = "target_count" ICON_ACCOUNT_GROUP = "mdi:account-group" ICON_ACCOUNT_SWITCH = "mdi:account-switch" diff --git a/esphome/components/ld6002b/__init__.py b/esphome/components/ld6002b/__init__.py new file mode 100644 index 0000000000..af074fc7ea --- /dev/null +++ b/esphome/components/ld6002b/__init__.py @@ -0,0 +1,73 @@ +from esphome import pins +import esphome.codegen as cg +from esphome.components import uart +import esphome.config_validation as cv +from esphome.const import CONF_ID, CONF_WAKEUP_PIN + +from .const import CONF_AUTO_WAKE, CONF_WAKEUP_PULSE + +CODEOWNERS = ["@hepter"] +DEPENDENCIES = ["uart"] +MULTI_CONF = True + +ld6002b_ns = cg.esphome_ns.namespace("ld6002b") +LD6002BComponent = ld6002b_ns.class_("LD6002BComponent", cg.Component, uart.UARTDevice) + + +def _validate_wakeup_options(config): + """Reject wake options that would silently do nothing. + + Runs before the schema so the defaults for the keys below have not been + filled in yet and an explicit user value is still distinguishable from one. + """ + if not isinstance(config, dict): + return config + if CONF_WAKEUP_PIN in config: + return config + for key in (CONF_AUTO_WAKE, CONF_WAKEUP_PULSE): + if key in config: + raise cv.Invalid( + f"'{key}' requires '{CONF_WAKEUP_PIN}' to be configured", path=[key] + ) + return config + + +CONFIG_SCHEMA = cv.All( + _validate_wakeup_options, + cv.Schema( + { + cv.GenerateID(): cv.declare_id(LD6002BComponent), + cv.Optional(CONF_WAKEUP_PIN): pins.gpio_output_pin_schema, + cv.Optional( + CONF_WAKEUP_PULSE, default="50ms" + ): cv.positive_time_period_milliseconds, + cv.Optional(CONF_AUTO_WAKE, default=True): cv.boolean, + } + ) + .extend(uart.UART_DEVICE_SCHEMA) + .extend(cv.COMPONENT_SCHEMA), +) + +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "ld6002b", + baud_rate=115200, + require_tx=True, + require_rx=True, + data_bits=8, + parity="NONE", + stop_bits=1, +) + + +async def to_code(config): + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await uart.register_uart_device(var, config) + + if wakeup_pin_config := config.get(CONF_WAKEUP_PIN): + pin = await cg.gpio_pin_expression(wakeup_pin_config) + cg.add(var.set_wakeup_pin(pin)) + + cg.add(var.set_wakeup_pulse_ms(config[CONF_WAKEUP_PULSE].total_milliseconds)) + + cg.add(var.set_auto_wake(config[CONF_AUTO_WAKE])) diff --git a/esphome/components/ld6002b/binary_sensor.py b/esphome/components/ld6002b/binary_sensor.py new file mode 100644 index 0000000000..319ace6f5d --- /dev/null +++ b/esphome/components/ld6002b/binary_sensor.py @@ -0,0 +1,38 @@ +import esphome.codegen as cg +from esphome.components import binary_sensor +import esphome.config_validation as cv +from esphome.const import CONF_TARGET, DEVICE_CLASS_OCCUPANCY + +from . import LD6002BComponent +from .const import CONF_LD6002B_ID, MAX_TARGETS + +DEPENDENCIES = ["ld6002b"] + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(CONF_LD6002B_ID): cv.use_id(LD6002BComponent), + cv.Optional(CONF_TARGET): binary_sensor.binary_sensor_schema( + device_class=DEVICE_CLASS_OCCUPANCY, + ), + } +).extend( + { + cv.Optional(f"target_{i + 1}"): binary_sensor.binary_sensor_schema( + device_class=DEVICE_CLASS_OCCUPANCY, + ) + for i in range(MAX_TARGETS) + } +) + + +async def to_code(config): + hub = await cg.get_variable(config[CONF_LD6002B_ID]) + + if target_config := config.get(CONF_TARGET): + sens = await binary_sensor.new_binary_sensor(target_config) + cg.add(hub.set_presence_binary_sensor(sens)) + + for i in range(MAX_TARGETS): + if target_config := config.get(f"target_{i + 1}"): + sens = await binary_sensor.new_binary_sensor(target_config) + cg.add(hub.set_target_presence_binary_sensor(i, sens)) diff --git a/esphome/components/ld6002b/const.py b/esphome/components/ld6002b/const.py new file mode 100644 index 0000000000..4419a92d23 --- /dev/null +++ b/esphome/components/ld6002b/const.py @@ -0,0 +1,8 @@ +CONF_AUTO_WAKE = "auto_wake" +CONF_CLUSTER_ID = "cluster_id" +CONF_DOPPLER_INDEX = "doppler_index" +CONF_LD6002B_ID = "ld6002b_id" +CONF_WAKEUP_PULSE = "wakeup_pulse" +CONF_Z = "z" + +MAX_TARGETS = 3 diff --git a/esphome/components/ld6002b/ld6002b.cpp b/esphome/components/ld6002b/ld6002b.cpp new file mode 100644 index 0000000000..2a09e98c25 --- /dev/null +++ b/esphome/components/ld6002b/ld6002b.cpp @@ -0,0 +1,590 @@ +#include "ld6002b.h" +#include "esphome/core/log.h" +#include +#include +#include +#include + +namespace esphome::ld6002b { + +static const char *const TAG = "ld6002b"; + +static constexpr uint8_t TF_SOF = 0x01; +static constexpr uint32_t SETUP_DELAY_MS = 100; + +// Command/message types +static constexpr uint16_t TYPE_CONTROL = 0x0201; + +static constexpr uint16_t TYPE_REPORT_TARGET = 0x0A04; + +// Control command values for TYPE_CONTROL +static constexpr uint32_t CMD_POINT_CLOUD_ON = 0x06; +static constexpr uint32_t CMD_POINT_CLOUD_OFF = 0x07; +static constexpr uint32_t CMD_TARGET_DISPLAY_ON = 0x08; +static constexpr uint32_t CMD_TARGET_DISPLAY_OFF = 0x09; + +static constexpr uint16_t TARGET_DATA_LEN = 20; // x,y,z,dop_idx,cluster_id + +#ifdef ESPHOME_LOG_HAS_VERBOSE +static const char *control_command_name(uint32_t command) { + switch (command) { + case CMD_POINT_CLOUD_ON: + return "point_cloud_on"; + case CMD_POINT_CLOUD_OFF: + return "point_cloud_off"; + case CMD_TARGET_DISPLAY_ON: + return "target_display_on"; + case CMD_TARGET_DISPLAY_OFF: + return "target_display_off"; + default: + return "unknown"; + } +} +#endif + +uint16_t LD6002BComponent::read_u16_be(const uint8_t *data) { return (static_cast(data[0]) << 8) | data[1]; } + +uint32_t LD6002BComponent::read_u32_le(const uint8_t *data) { + return static_cast(data[0]) | (static_cast(data[1]) << 8) | + (static_cast(data[2]) << 16) | (static_cast(data[3]) << 24); +} + +int32_t LD6002BComponent::read_int32_le(const uint8_t *data) { + uint32_t raw = read_u32_le(data); + int32_t value; + std::memcpy(&value, &raw, sizeof(value)); + return value; +} + +float LD6002BComponent::read_f32_le(const uint8_t *data) { + uint32_t raw = read_u32_le(data); + float value; + std::memcpy(&value, &raw, sizeof(value)); + return value; +} + +void LD6002BComponent::write_u32_le(uint8_t *data, uint32_t value) { + data[0] = value & 0xFF; + data[1] = (value >> 8) & 0xFF; + data[2] = (value >> 16) & 0xFF; + data[3] = (value >> 24) & 0xFF; +} + +void LD6002BComponent::setup() { + // One allocation for the component lifetime; the parser reuses it for the header and every payload. + RAMAllocator allocator; + this->data_buf_ = allocator.allocate(DEFAULT_MAX_DATA_LEN); + if (this->data_buf_ == nullptr) { + this->mark_failed(LOG_STR("Failed to allocate frame buffer")); + return; + } + if (this->wakeup_pin_ != nullptr) { + this->wakeup_pin_->setup(); + this->wakeup_pin_->digital_write(true); + } + + this->set_timeout(SETUP_DELAY_MS, [this]() { + bool want_target_stream = false; +#ifdef USE_SENSOR + want_target_stream = want_target_stream || this->target_count_sensor_ != nullptr; + if (!want_target_stream) { + for (const auto &target : this->targets_) { + if (target.x != nullptr || target.y != nullptr || target.z != nullptr || target.dop_idx != nullptr || + target.cluster_id != nullptr) { + want_target_stream = true; + break; + } + } + } +#endif +#ifdef USE_BINARY_SENSOR + want_target_stream = want_target_stream || this->presence_binary_sensor_ != nullptr; + if (!want_target_stream) { + for (auto *sensor : this->target_presence_) { + if (sensor != nullptr) { + want_target_stream = true; + break; + } + } + } +#endif + if (want_target_stream) { + this->send_control_command_(CMD_TARGET_DISPLAY_ON); + } + + this->send_control_command_(CMD_POINT_CLOUD_OFF); + }); +} + +void LD6002BComponent::dump_config() { + ESP_LOGCONFIG(TAG, + "HLK-LD6002B:\n" + " Auto wake: %s", + this->auto_wake_ ? "true" : "false"); + if (this->wakeup_pin_ != nullptr) { + LOG_PIN(" Wake-up Pin: ", this->wakeup_pin_); + ESP_LOGCONFIG(TAG, " Wake Pulse: %ums", this->wakeup_pulse_ms_); + } +#ifdef USE_SENSOR + LOG_SENSOR(" ", "Target Count", this->target_count_sensor_); + for (auto &target : this->targets_) { + LOG_SENSOR(" ", "Target X", target.x); + LOG_SENSOR(" ", "Target Y", target.y); + LOG_SENSOR(" ", "Target Z", target.z); + LOG_SENSOR(" ", "Target Doppler Index", target.dop_idx); + LOG_SENSOR(" ", "Target Cluster ID", target.cluster_id); + } +#endif +#ifdef USE_BINARY_SENSOR + LOG_BINARY_SENSOR(" ", "Presence", this->presence_binary_sensor_); + for (uint8_t i = 0; i < MAX_TARGETS; i++) { + LOG_BINARY_SENSOR(" ", "Target Presence", this->target_presence_[i]); + } +#endif +} + +void LD6002BComponent::loop() { + while (this->available()) { + uint8_t byte = this->read(); + this->parse_byte_(byte); + } + this->process_command_queue_(); +} + +void LD6002BComponent::reset_parser_() { + this->parse_state_ = ParseState::SOF; + this->header_pos_ = 0; + this->header_xor_ = 0; + this->data_len_ = 0; + this->data_pos_ = 0; + this->data_xor_ = 0; + this->discard_remaining_ = 0; + this->frame_oversize_ = false; +} + +void LD6002BComponent::parse_byte_(uint8_t byte) { + switch (this->parse_state_) { + case ParseState::DISCARD: + // discard_remaining_ is unsigned: an unguarded decrement at zero would swallow 4 GB of stream. + if (this->discard_remaining_ > 0) { + this->discard_remaining_--; + } + if (this->discard_remaining_ == 0) { + this->reset_parser_(); + } + return; + case ParseState::SOF: + if (byte != TF_SOF) + return; + this->header_pos_ = 0; + this->header_xor_ = 0; + this->header_xor_ ^= byte; + this->parse_state_ = ParseState::HEADER; + return; + case ParseState::HEADER: + if (this->header_pos_ < 6) { + this->data_buf_[this->header_pos_] = byte; + this->header_xor_ ^= byte; + this->header_pos_++; + if (this->header_pos_ == 6) { + this->frame_id_ = read_u16_be(this->data_buf_); + this->data_len_ = read_u16_be(this->data_buf_ + 2); + this->frame_type_ = read_u16_be(this->data_buf_ + 4); + // The length is only trustworthy once the header checksum has been verified, so just + // remember that the frame is oversized and let the HCK state act on it. + this->frame_oversize_ = this->data_len_ > DEFAULT_MAX_DATA_LEN; + this->parse_state_ = ParseState::HCK; + } + } + return; + case ParseState::HCK: { + uint8_t expected = static_cast(~this->header_xor_); + if (byte != expected) { + ESP_LOGV(TAG, "Header checksum mismatch"); + this->reset_parser_(); + return; + } + if (this->frame_oversize_) { + ESP_LOGW(TAG, "Frame too large: %u", this->data_len_); + // The header is verified, so the length can be trusted: skip the payload and its checksum. + this->discard_remaining_ = static_cast(this->data_len_) + 1; + this->parse_state_ = ParseState::DISCARD; + return; + } + if (this->data_len_ == 0) { + this->handle_frame_(this->frame_type_, nullptr, 0); + this->reset_parser_(); + } else { + this->data_pos_ = 0; + this->data_xor_ = 0; + this->parse_state_ = ParseState::DATA; + } + return; + } + case ParseState::DATA: + this->data_buf_[this->data_pos_++] = byte; + this->data_xor_ ^= byte; + if (this->data_pos_ >= this->data_len_) { + this->parse_state_ = ParseState::DCK; + } + return; + case ParseState::DCK: { + uint8_t expected = static_cast(~this->data_xor_); + if (byte == expected) { + this->handle_frame_(this->frame_type_, this->data_buf_, this->data_len_); + } else { + ESP_LOGV(TAG, "Data checksum mismatch"); + } + this->reset_parser_(); + return; + } + } +} + +void LD6002BComponent::handle_frame_(uint16_t type, const uint8_t *data, uint16_t len) { + this->last_traffic_ms_ = millis(); + if (this->stale_ack_count_ > 0 && millis() - this->stale_ack_ms_ > STALE_ACK_MAX_AGE_MS) { + this->stale_ack_count_ = 0; + } + // ACKs carry no id and arrive in send order: debt from earlier attempts is paid before the active command. + if (len == 0 && this->stale_ack_count_ > 0 && this->stale_ack_type_ == type) { + this->stale_ack_count_--; + ESP_LOGV(TAG, "Ignoring ACK for command 0x%04X from an earlier attempt (module frame 0x%04X)", type, + this->frame_id_); + return; + } + if (len == 0 && this->command_active_ && this->command_sent_ && type == this->active_command_.type) { + ESP_LOGV(TAG, "ACK for command 0x%04X (module frame 0x%04X)", type, this->frame_id_); + // This settles one expected reply; the rest stay owed and become the debt for the next command. + this->send_generation_++; + this->stale_ack_type_ = type; + this->stale_ack_count_ = this->acks_expected_ > 0 ? static_cast(this->acks_expected_ - 1) : 0; + this->stale_ack_ms_ = millis(); + this->command_active_ = false; + this->command_sent_ = false; + this->last_send_ms_ = 0; + this->process_command_queue_(); + return; + } + + switch (type) { + case TYPE_REPORT_TARGET: + this->handle_target_report_(data, len); + break; + default: + break; + } +} + +void LD6002BComponent::handle_target_report_(const uint8_t *data, uint16_t len) { + if (len < 4) + return; + + uint32_t target_num = read_u32_le(data); + uint16_t available = (len - 4) / TARGET_DATA_LEN; + // Un-narrowed: a report of e.g. 256 targets must not truncate to 0 and read as "absent". + const uint32_t reported = std::min(target_num, available); + uint8_t count = static_cast(std::min(reported, MAX_TARGETS)); + + // The module re-sorts its array by cluster id, so slots key on the id to track the person. + std::array wire_cluster{}; + std::array wire_placed{}; + std::array slot_seen{}; + std::array slot_wire{}; + for (uint8_t i = 0; i < count; i++) { + uint16_t cluster_offset = 4 + (i * TARGET_DATA_LEN) + 16; + wire_cluster[i] = static_cast(read_u32_le(data + cluster_offset)); + } + for (uint8_t i = 0; i < count; i++) { + for (uint8_t s = 0; s < MAX_TARGETS; s++) { + if (this->slot_occupied_[s] && !slot_seen[s] && this->slot_cluster_[s] == wire_cluster[i]) { + slot_seen[s] = true; + wire_placed[i] = true; + slot_wire[s] = i; + break; + } + } + } + for (uint8_t s = 0; s < MAX_TARGETS; s++) { + if (!slot_seen[s]) { + this->slot_occupied_[s] = false; + } + } + for (uint8_t i = 0; i < count; i++) { + if (wire_placed[i]) { + continue; + } + for (uint8_t s = 0; s < MAX_TARGETS; s++) { + if (!this->slot_occupied_[s]) { + this->slot_occupied_[s] = true; + this->slot_cluster_[s] = wire_cluster[i]; + slot_wire[s] = i; + break; + } + } + } + +#ifdef USE_SENSOR + if (this->target_count_sensor_ != nullptr) { + if (reported != this->last_target_count_) { + this->target_count_sensor_->publish_state(reported); + this->last_target_count_ = reported; + } + } +#endif + + this->target_presence_any_ = (reported > 0); +#ifdef USE_BINARY_SENSOR + if (this->presence_binary_sensor_ != nullptr) { + this->presence_binary_sensor_->publish_state(this->target_presence_any_); + } +#endif + + for (uint8_t i = 0; i < MAX_TARGETS; i++) { + bool has_target = this->slot_occupied_[i]; + if (has_target) { +#ifdef USE_SENSOR + uint16_t offset = 4 + (slot_wire[i] * TARGET_DATA_LEN); + float x = read_f32_le(data + offset + 0); + float y = read_f32_le(data + offset + 4); + float z = read_f32_le(data + offset + 8); + int32_t dop_idx = read_int32_le(data + offset + 12); + int32_t cluster_id = this->slot_cluster_[i]; + TargetSensors &target = this->targets_[i]; + if (target.x != nullptr) { + target.x->publish_state(x); + } + if (target.y != nullptr) { + target.y->publish_state(y); + } + if (target.z != nullptr) { + target.z->publish_state(z); + } + if (target.dop_idx != nullptr) { + target.dop_idx->publish_state(static_cast(dop_idx)); + } + if (target.cluster_id != nullptr) { + if (!this->last_cluster_id_valid_[i] || cluster_id != this->last_cluster_id_[i]) { + target.cluster_id->publish_state(static_cast(cluster_id)); + this->last_cluster_id_[i] = cluster_id; + this->last_cluster_id_valid_[i] = true; + } + } +#endif + } else { +#ifdef USE_SENSOR + TargetSensors &target = this->targets_[i]; + if (this->last_target_presence_[i]) { + if (target.x != nullptr) { + target.x->publish_state(NAN); + } + if (target.y != nullptr) { + target.y->publish_state(NAN); + } + if (target.z != nullptr) { + target.z->publish_state(NAN); + } + if (target.dop_idx != nullptr) { + target.dop_idx->publish_state(NAN); + } + if (target.cluster_id != nullptr) { + target.cluster_id->publish_state(NAN); + } + // The slot is free: the next person's id is new even when it repeats this one. + this->last_cluster_id_valid_[i] = false; + } +#endif + } +#ifdef USE_BINARY_SENSOR + if (this->target_presence_[i] != nullptr) { + // publish_state() already skips unchanged states, no manual de-dup needed. + this->target_presence_[i]->publish_state(has_target); + } +#endif +#ifdef USE_SENSOR + this->last_target_presence_[i] = has_target; +#endif + } +} + +void LD6002BComponent::queue_command_(uint16_t type, const uint8_t *data, uint8_t len) { + if (len > CMD_MAX_DATA_LEN) { + ESP_LOGW(TAG, "Command data too large: %u", len); + return; + } + if (this->cmd_count_ >= CMD_QUEUE_SIZE) { + ESP_LOGW(TAG, "Command queue full, dropping command 0x%04X", type); + return; + } + + PendingCommand &cmd = this->cmd_queue_[this->cmd_tail_]; + cmd.type = type; + cmd.len = len; + if (len > 0 && data != nullptr) { + std::memcpy(cmd.data.data(), data, len); + } + + this->cmd_tail_ = (this->cmd_tail_ + 1) % CMD_QUEUE_SIZE; + this->cmd_count_++; + this->process_command_queue_(); +} + +void LD6002BComponent::process_command_queue_() { + uint32_t now = millis(); + if (this->command_active_) { + // A sleeping module consumes the opening attempt as its wake-up instead of answering it. + const uint32_t ack_timeout = this->attempts_sent_ <= 1 ? CMD_FIRST_ACK_TIMEOUT_MS : CMD_ACK_TIMEOUT_MS; + if (this->command_sent_ && now - this->last_send_ms_ >= ack_timeout) { + const uint32_t active_control_command = + (this->active_command_.type == TYPE_CONTROL && this->active_command_.len >= 4) + ? read_u32_le(this->active_command_.data.data()) + : 0; + if (this->retries_left_ > 0) { +#ifdef ESPHOME_LOG_HAS_VERBOSE + if (active_control_command != 0) { + ESP_LOGV(TAG, "Retrying %s (0x%02" PRIX32 "), %u attempt(s) remaining", + control_command_name(active_control_command), active_control_command, this->retries_left_); + } else { + // Writes without a control subcommand (hold delay, z-range) had no retry trace at all. + ESP_LOGV(TAG, "Retrying command 0x%04X, %u attempt(s) remaining", this->active_command_.type, + this->retries_left_); + } +#endif + this->command_sent_ = false; + this->last_send_ms_ = 0; + this->send_command_(this->active_command_.type, this->active_command_.data.data(), this->active_command_.len); + this->retries_left_--; + } else { + if (active_control_command != 0) { + ESP_LOGW(TAG, "Command 0x%04X subcommand 0x%02" PRIX32 " timed out", this->active_command_.type, + active_control_command); + } else { + ESP_LOGW(TAG, "Command 0x%04X timed out", this->active_command_.type); + } + // A reply may still be in flight for the attempt we just gave up on, so carry one over as + // debt rather than clearing the ledger, or that late ACK would retire the successor. Only + // one: reaching this point means nothing was answered at all, so the older attempts are + // speculative, and carrying them would swallow the successor's own replies. + const uint16_t owed = (this->stale_ack_type_ == this->active_command_.type ? this->stale_ack_count_ : 0) + + (this->acks_expected_ > 0 ? 1 : 0); + this->stale_ack_type_ = this->active_command_.type; + this->stale_ack_count_ = static_cast(std::min(owed, 255)); + this->stale_ack_ms_ = now; + this->send_generation_++; + this->command_active_ = false; + this->command_sent_ = false; + this->last_send_ms_ = 0; + } + } + return; + } + + if (this->cmd_count_ == 0) + return; + + this->active_command_ = this->cmd_queue_[this->cmd_head_]; + this->cmd_head_ = (this->cmd_head_ + 1) % CMD_QUEUE_SIZE; + this->cmd_count_--; + + this->send_generation_++; + this->retries_left_ = CMD_MAX_RETRIES; + this->command_active_ = true; + this->command_sent_ = false; + this->last_send_ms_ = 0; + this->attempts_sent_ = 0; + this->acks_expected_ = 0; + if (this->stale_ack_type_ != this->active_command_.type) { + this->stale_ack_count_ = 0; + } + this->send_command_(this->active_command_.type, this->active_command_.data.data(), this->active_command_.len); +} + +void LD6002BComponent::send_command_(uint16_t type, const uint8_t *data, uint8_t len) { + this->send_command_internal_(type, data, len, true); +} + +void LD6002BComponent::send_command_internal_(uint16_t type, const uint8_t *data, uint8_t len, bool track) { + if (len > CMD_MAX_DATA_LEN) { + ESP_LOGW(TAG, "Command data too large: %u", len); + if (track) { + // Release the slot: an unwritten command is never acked and never times out. + this->command_active_ = false; + this->command_sent_ = false; + this->last_send_ms_ = 0; + } + return; + } + + // Anonymous timeouts never replace each other; with a pulse already pending the module is waking anyway. + if (this->auto_wake_ && this->wakeup_pin_ != nullptr && !this->wake_pulse_pending_) { + // Snapshot the payload: the deferred write must not depend on state a completing command changes. + if (len > 0 && data != nullptr) { + std::memcpy(this->wake_scratch_.data(), data, len); + } + this->wake_pulse_pending_ = true; + this->wakeup_pin_->digital_write(false); + const uint8_t generation = this->send_generation_; + this->set_timeout(this->wakeup_pulse_ms_, [this, type, len, track, generation]() { + this->wakeup_pin_->digital_write(true); + this->wake_pulse_pending_ = false; + // Anonymous timeouts are never cancelled, so a tracked pulse whose command has since been + // retired must not transmit: the frame would land after its successor and be booked to it. + if (track && generation != this->send_generation_) { + return; + } + this->write_frame_(type, (len > 0) ? this->wake_scratch_.data() : nullptr, len, track); + }); + return; + } + + this->write_frame_(type, data, len, track); +} + +void LD6002BComponent::write_frame_(uint16_t type, const uint8_t *data, uint8_t len, bool track) { + uint16_t frame_id = this->next_frame_id_++ & 0x7FFF; + frame_id |= 0x8000; + + uint8_t header_xor = 0; + auto write_header = [&](uint8_t b) { + this->write_byte(b); + header_xor ^= b; + }; + + write_header(TF_SOF); + write_header((frame_id >> 8) & 0xFF); + write_header(frame_id & 0xFF); + write_header((len >> 8) & 0xFF); + write_header(len & 0xFF); + write_header((type >> 8) & 0xFF); + write_header(type & 0xFF); + + this->write_byte(static_cast(~header_xor)); + + if (len > 0 && data != nullptr) { + uint8_t data_xor = 0; + for (uint8_t i = 0; i < len; i++) { + this->write_byte(data[i]); + data_xor ^= data[i]; + } + this->write_byte(static_cast(~data_xor)); + } + const uint32_t now = millis(); + if (track) { + // A frame sent to a module that has had time to fall asleep is its wake-up, and goes unanswered. + if (this->last_traffic_ms_ != 0 && now - this->last_traffic_ms_ < MODULE_AWAKE_MS) { + this->acks_expected_++; + } + this->last_send_ms_ = now; + this->command_sent_ = true; + this->attempts_sent_++; + } + this->last_traffic_ms_ = now; +} + +void LD6002BComponent::send_control_command_(uint32_t command) { + uint8_t data[4]; + write_u32_le(data, command); + this->queue_command_(TYPE_CONTROL, data, sizeof(data)); +} + +} // namespace esphome::ld6002b diff --git a/esphome/components/ld6002b/ld6002b.h b/esphome/components/ld6002b/ld6002b.h new file mode 100644 index 0000000000..8bbfb9f6e4 --- /dev/null +++ b/esphome/components/ld6002b/ld6002b.h @@ -0,0 +1,191 @@ +#pragma once + +#include "esphome/core/defines.h" +#include "esphome/core/component.h" +#include "esphome/core/helpers.h" +#include "esphome/core/gpio.h" +#include "esphome/components/uart/uart.h" +#ifdef USE_SENSOR +#include "esphome/components/sensor/sensor.h" +#endif +#ifdef USE_BINARY_SENSOR +#include "esphome/components/binary_sensor/binary_sensor.h" +#endif + +#include + +namespace esphome::ld6002b { + +static constexpr uint8_t MAX_TARGETS = 3; +static constexpr size_t DEFAULT_MAX_DATA_LEN = 1024; +// Largest protocol payload is TYPE_SET_AREA: int32 area id + 6 floats = 28 bytes. +static constexpr size_t CMD_MAX_DATA_LEN = 28; + +#ifdef USE_SENSOR +struct TargetSensors { + sensor::Sensor *x{nullptr}; + sensor::Sensor *y{nullptr}; + sensor::Sensor *z{nullptr}; + sensor::Sensor *dop_idx{nullptr}; + sensor::Sensor *cluster_id{nullptr}; +}; + +#endif + +class LD6002BComponent : public Component, public uart::UARTDevice { + public: + void setup() override; + void loop() override; + void dump_config() override; + float get_setup_priority() const override { return setup_priority::DATA; } + + void set_wakeup_pin(GPIOPin *pin) { this->wakeup_pin_ = pin; } + void set_wakeup_pulse_ms(uint32_t ms) { this->wakeup_pulse_ms_ = ms; } + void set_auto_wake(bool enable) { this->auto_wake_ = enable; } + +#ifdef USE_SENSOR + void set_target_count_sensor(sensor::Sensor *sensor) { this->target_count_sensor_ = sensor; } + + void set_target_x_sensor(uint8_t target, sensor::Sensor *sensor) { + if (target >= MAX_TARGETS) + return; + this->targets_[target].x = sensor; + } + void set_target_y_sensor(uint8_t target, sensor::Sensor *sensor) { + if (target >= MAX_TARGETS) + return; + this->targets_[target].y = sensor; + } + void set_target_z_sensor(uint8_t target, sensor::Sensor *sensor) { + if (target >= MAX_TARGETS) + return; + this->targets_[target].z = sensor; + } + void set_target_dop_idx_sensor(uint8_t target, sensor::Sensor *sensor) { + if (target >= MAX_TARGETS) + return; + this->targets_[target].dop_idx = sensor; + } + void set_target_cluster_id_sensor(uint8_t target, sensor::Sensor *sensor) { + if (target >= MAX_TARGETS) + return; + this->targets_[target].cluster_id = sensor; + } +#endif + +#ifdef USE_BINARY_SENSOR + void set_presence_binary_sensor(binary_sensor::BinarySensor *sensor) { this->presence_binary_sensor_ = sensor; } + void set_target_presence_binary_sensor(uint8_t target, binary_sensor::BinarySensor *sensor) { + if (target >= MAX_TARGETS) + return; + this->target_presence_[target] = sensor; + } +#endif + + protected: + enum class ParseState : uint8_t { SOF, HEADER, HCK, DATA, DCK, DISCARD }; + + struct PendingCommand { + uint16_t type{0}; + uint8_t len{0}; + std::array data{}; + }; + + void parse_byte_(uint8_t byte); + void reset_parser_(); + void handle_frame_(uint16_t type, const uint8_t *data, uint16_t len); + void handle_target_report_(const uint8_t *data, uint16_t len); + + void queue_command_(uint16_t type, const uint8_t *data, uint8_t len); + void process_command_queue_(); + void send_command_(uint16_t type, const uint8_t *data, uint8_t len); + void send_command_internal_(uint16_t type, const uint8_t *data, uint8_t len, bool track); + void write_frame_(uint16_t type, const uint8_t *data, uint8_t len, bool track); + void send_control_command_(uint32_t command); + + static uint16_t read_u16_be(const uint8_t *data); + static uint32_t read_u32_le(const uint8_t *data); + static int32_t read_int32_le(const uint8_t *data); + static float read_f32_le(const uint8_t *data); + static void write_u32_le(uint8_t *data, uint32_t value); + +#ifdef USE_SENSOR + std::array targets_{}; + sensor::Sensor *target_count_sensor_{nullptr}; +#endif +#ifdef USE_BINARY_SENSOR + binary_sensor::BinarySensor *presence_binary_sensor_{nullptr}; + std::array target_presence_{}; +#endif + + GPIOPin *wakeup_pin_{nullptr}; + uint32_t wakeup_pulse_ms_{50}; + bool auto_wake_{true}; + + ParseState parse_state_{ParseState::SOF}; + uint8_t header_pos_{0}; + uint8_t header_xor_{0}; + uint16_t data_len_{0}; + uint16_t frame_type_{0}; + uint16_t frame_id_{0}; + uint16_t data_pos_{0}; + uint8_t data_xor_{0}; + uint32_t discard_remaining_{0}; + bool frame_oversize_{false}; + uint8_t *data_buf_{nullptr}; + uint16_t next_frame_id_{0}; + + // Sized for the boot burst: with every platform configured, setup() enqueues + // roughly ten GET/config commands back to back before the first ack lands. + static constexpr uint8_t CMD_QUEUE_SIZE = 16; + static constexpr uint32_t CMD_ACK_TIMEOUT_MS = 300; + // A sleeping module consumes the first frame to wake and answers only the one after it. + static constexpr uint32_t CMD_FIRST_ACK_TIMEOUT_MS = 600; + // How long the module stays awake after any frame, and so still answers the next one. + static constexpr uint32_t MODULE_AWAKE_MS = 10000; + static constexpr uint8_t CMD_MAX_RETRIES = 3; + // A reply cannot trail the frame that earned it for longer than this; the field worst case is ~726ms. + static constexpr uint32_t STALE_ACK_MAX_AGE_MS = 1000; + + std::array cmd_queue_{}; + uint8_t cmd_head_{0}; + uint8_t cmd_tail_{0}; + uint8_t cmd_count_{0}; + bool command_active_{false}; + bool command_sent_{false}; + PendingCommand active_command_{}; + // Frame a pending wake pulse will write, snapshotted because active_command_ may move on first. + std::array wake_scratch_{}; + bool wake_pulse_pending_{false}; + uint8_t retries_left_{0}; + uint32_t last_send_ms_{0}; + // Last frame seen in either direction; any traffic keeps the module awake. + uint32_t last_traffic_ms_{0}; + // Frames transmitted for the command in flight, including retries; drives the retry budget. + uint8_t attempts_sent_{0}; + // Subset of those the module can actually answer: a frame that woke it is consumed, not replied to. + uint8_t acks_expected_{0}; + // ACKs still owed for superseded attempts; they carry no id, only their arrival order. + uint16_t stale_ack_type_{0}; + uint8_t stale_ack_count_{0}; + // When that debt was booked, so a debt no reply can still settle expires instead of eating a live ACK. + uint32_t stale_ack_ms_{0}; + // Bumped whenever the active command changes, so a deferred send can tell it was retired. + uint8_t send_generation_{0}; + + // Which person owns each target_N slot, so a slot survives the module re-sorting its array. + std::array slot_cluster_{}; + std::array slot_occupied_{}; + + bool target_presence_any_{false}; + +#ifdef USE_SENSOR + std::array last_target_presence_{}; // one-shot NAN clear for target sensors + // A cluster id names a person, so like the counts it is published on change, not per frame. + std::array last_cluster_id_{}; + std::array last_cluster_id_valid_{}; + uint32_t last_target_count_{0xFFFFFFFF}; +#endif +}; + +} // namespace esphome::ld6002b diff --git a/esphome/components/ld6002b/sensor.py b/esphome/components/ld6002b/sensor.py new file mode 100644 index 0000000000..3aa9b0f98a --- /dev/null +++ b/esphome/components/ld6002b/sensor.py @@ -0,0 +1,105 @@ +import esphome.codegen as cg +from esphome.components import sensor +from esphome.components.const import CONF_TARGET_COUNT +import esphome.config_validation as cv +from esphome.const import ( + CONF_X, + CONF_Y, + DEVICE_CLASS_DISTANCE, + STATE_CLASS_MEASUREMENT, + UNIT_METER, +) + +from . import LD6002BComponent +from .const import ( + CONF_CLUSTER_ID, + CONF_DOPPLER_INDEX, + CONF_LD6002B_ID, + CONF_Z, + MAX_TARGETS, +) + +DEPENDENCIES = ["ld6002b"] + +# The ld2450 defaults for a streamed value: hold the last reading for a second so a +# dropped frame does not read as absence, then rate-limit what reaches the frontend. +_VALUE_SENSOR_FILTERS = [ + { + "timeout": { + "timeout": cv.TimePeriod(milliseconds=1000), + "value": "last", + } + }, + {"throttle_with_priority": cv.TimePeriod(milliseconds=1000)}, +] + +TARGET_SCHEMA = cv.Schema( + { + cv.Optional(CONF_X): sensor.sensor_schema( + unit_of_measurement=UNIT_METER, + accuracy_decimals=2, + device_class=DEVICE_CLASS_DISTANCE, + filters=_VALUE_SENSOR_FILTERS, + state_class=STATE_CLASS_MEASUREMENT, + ), + cv.Optional(CONF_Y): sensor.sensor_schema( + unit_of_measurement=UNIT_METER, + accuracy_decimals=2, + device_class=DEVICE_CLASS_DISTANCE, + filters=_VALUE_SENSOR_FILTERS, + state_class=STATE_CLASS_MEASUREMENT, + ), + cv.Optional(CONF_Z): sensor.sensor_schema( + unit_of_measurement=UNIT_METER, + accuracy_decimals=2, + device_class=DEVICE_CLASS_DISTANCE, + filters=_VALUE_SENSOR_FILTERS, + state_class=STATE_CLASS_MEASUREMENT, + ), + cv.Optional(CONF_DOPPLER_INDEX): sensor.sensor_schema( + accuracy_decimals=0, + filters=_VALUE_SENSOR_FILTERS, + state_class=STATE_CLASS_MEASUREMENT, + ), + cv.Optional(CONF_CLUSTER_ID): sensor.sensor_schema( + accuracy_decimals=0, + ), + } +) + + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(CONF_LD6002B_ID): cv.use_id(LD6002BComponent), + cv.Optional(CONF_TARGET_COUNT): sensor.sensor_schema( + accuracy_decimals=0, + state_class=STATE_CLASS_MEASUREMENT, + ), + } +).extend({cv.Optional(f"target_{i + 1}"): TARGET_SCHEMA for i in range(MAX_TARGETS)}) + + +async def to_code(config): + hub = await cg.get_variable(config[CONF_LD6002B_ID]) + + if target_count_config := config.get(CONF_TARGET_COUNT): + sens = await sensor.new_sensor(target_count_config) + cg.add(hub.set_target_count_sensor(sens)) + + for i in range(MAX_TARGETS): + if target_config := config.get(f"target_{i + 1}"): + if x_config := target_config.get(CONF_X): + sens = await sensor.new_sensor(x_config) + cg.add(hub.set_target_x_sensor(i, sens)) + if y_config := target_config.get(CONF_Y): + sens = await sensor.new_sensor(y_config) + cg.add(hub.set_target_y_sensor(i, sens)) + if z_config := target_config.get(CONF_Z): + sens = await sensor.new_sensor(z_config) + cg.add(hub.set_target_z_sensor(i, sens)) + if doppler_index_config := target_config.get(CONF_DOPPLER_INDEX): + sens = await sensor.new_sensor(doppler_index_config) + cg.add(hub.set_target_dop_idx_sensor(i, sens)) + if cluster_id_config := target_config.get(CONF_CLUSTER_ID): + sens = await sensor.new_sensor(cluster_id_config) + cg.add(hub.set_target_cluster_id_sensor(i, sens)) diff --git a/esphome/components/libretiny/__init__.py b/esphome/components/libretiny/__init__.py index 7dbce7a07c..c51af373b3 100644 --- a/esphome/components/libretiny/__init__.py +++ b/esphome/components/libretiny/__init__.py @@ -461,6 +461,8 @@ async def component_to_code(config): # setup board config cg.add_platformio_option("board", config[CONF_BOARD]) cg.add_build_flag("-DUSE_LIBRETINY") + # FlashDB finds stored preferences by key, so preference key migration is possible + cg.add_define("USE_PREFERENCE_KEY_LOOKUP") cg.add_build_flag(f"-DUSE_{config[CONF_COMPONENT_ID].upper()}") cg.add_build_flag(f"-DUSE_LIBRETINY_VARIANT_{config[CONF_FAMILY]}") cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) diff --git a/esphome/components/libretiny/generate_components.py b/esphome/components/libretiny/generate_components.py index 791a2659a9..4997878657 100644 --- a/esphome/components/libretiny/generate_components.py +++ b/esphome/components/libretiny/generate_components.py @@ -65,6 +65,10 @@ def _set_core_data(config): return config +# extend({}) makes this platform's own schema instance: BASE_SCHEMA is shared +# by every LibreTiny platform, and prepending this platform's _set_core_data +# onto the shared object would run it for every platform's validation once two +# platform modules are imported in one process (device-builder, tests). CONFIG_SCHEMA = {SCHEMA} PIN_SCHEMA = {PIN_SCHEMA} @@ -117,7 +121,7 @@ VAR_GPIO_PIN = "validate_pin" VAR_GPIO_USAGE = "validate_usage" # lines for code snippets -SCHEMA_BASE = "libretiny.BASE_SCHEMA" +SCHEMA_BASE = "libretiny.BASE_SCHEMA.extend({})" SCHEMA_EXTRA = f"libretiny.BASE_SCHEMA.extend({VAR_SCHEMA})" PIN_SCHEMA_BASE = "libretiny.gpio.BASE_PIN_SCHEMA" PIN_SCHEMA_EXTRA = f"libretiny.BASE_PIN_SCHEMA.extend({VAR_PIN_SCHEMA})" diff --git a/esphome/components/libretiny/preferences.cpp b/esphome/components/libretiny/preferences.cpp index 313b36d31e..d0bd3bf26b 100644 --- a/esphome/components/libretiny/preferences.cpp +++ b/esphome/components/libretiny/preferences.cpp @@ -70,12 +70,21 @@ void LibreTinyPreferences::open() { } ESPPreferenceObject LibreTinyPreferences::make_preference(size_t length, uint32_t type) { - auto *pref = new LibreTinyPreferenceBackend(); // NOLINT(cppcoreguidelines-owning-memory) - pref->db = &this->db; - pref->blob = &this->blob; - pref->key = type; + // NOLINTNEXTLINE(cppcoreguidelines-owning-memory) + return ESPPreferenceObject(new LibreTinyPreferenceBackend(this->make_backend_(type))); +} - return ESPPreferenceObject(pref); +LibreTinyPreferenceBackend LibreTinyPreferences::make_backend_(uint32_t type) { + LibreTinyPreferenceBackend backend; + backend.key = type; + backend.db = &this->db; + backend.blob = &this->blob; + return backend; +} + +bool LibreTinyPreferences::load_from_key(uint32_t type, uint8_t *data, size_t len) { + LibreTinyPreferenceBackend backend = this->make_backend_(type); + return backend.load(data, len); } bool LibreTinyPreferences::sync() { diff --git a/esphome/components/libretiny/preferences.h b/esphome/components/libretiny/preferences.h index 8365d590c2..fd86c48b20 100644 --- a/esphome/components/libretiny/preferences.h +++ b/esphome/components/libretiny/preferences.h @@ -16,6 +16,8 @@ class LibreTinyPreferences final : public PreferencesMixin return this->make_preference(length, type); } ESPPreferenceObject make_preference(size_t length, uint32_t type); + /// One-shot read of a stored preference by key, without allocating a backend + bool load_from_key(uint32_t type, uint8_t *data, size_t len); bool sync(); bool reset(); @@ -23,6 +25,7 @@ class LibreTinyPreferences final : public PreferencesMixin struct fdb_blob blob; protected: + LibreTinyPreferenceBackend make_backend_(uint32_t type); bool is_changed_(fdb_kvdb_t db, const NVSData &to_save, const char *key_str); }; diff --git a/esphome/components/ln882h_ble/__init__.py b/esphome/components/ln882h_ble/__init__.py index f499dbc6fd..aadc1f3b2f 100644 --- a/esphome/components/ln882h_ble/__init__.py +++ b/esphome/components/ln882h_ble/__init__.py @@ -31,6 +31,11 @@ CONFIG_SCHEMA = cv.Schema( ).extend(cv.COMPONENT_SCHEMA) +# Once per registered scan listener; sizes the controller's StaticVector +# listener storage. +request_scan_listener_slot = cg.slot_counter("LN882H_BLE_SCAN_LISTENER_COUNT") + + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) diff --git a/esphome/components/ln882h_ble/ln882h_ble.cpp b/esphome/components/ln882h_ble/ln882h_ble.cpp index dd8fee390d..152ca571e9 100644 --- a/esphome/components/ln882h_ble/ln882h_ble.cpp +++ b/esphome/components/ln882h_ble/ln882h_ble.cpp @@ -1,11 +1,17 @@ // ln882h_ble.cpp // // BLE controller support for the LN882H (LibreTiny lightning-ln882h family) — -// the platform analog of esp32_ble / rp2040_ble. Owns the one-time stack -// bring-up (rw_init + the ln_* app init sequence) and the controller BLE -// address (persistent KV entry, WiFi-MAC-derived once). Consumers -// (ln882h_ble_tracker) build on this component and contain no SDK calls of -// their own. +// the platform analog of esp32_ble / rp2040_ble. Owns everything that talks to +// the LN882H BLE SDK: +// - one-time stack bring-up (rw_init + the ln_* app init sequence), +// - the controller BLE address (persistent KV entry, WiFi-MAC-derived once), +// - the raw controller scan primitives (ln_ble_scan_start/stop), +// - the scan-report ring: the SDK's rw-task event callback decodes each +// report (including the controller's RSSI sign quirk) into a fixed pool +// and pushes it on a lock-free SPSC queue; loop() drains, dispatches on +// the main task and returns reports to the pool — the same EventPool + +// LockFreeQueue handoff esp32_ble uses, zero allocation at steady state. +// Consumers contain no SDK calls of their own. // // BLE stack init and scan lifecycle mirror the SDK's ble_app usage. The BLE // stack itself is compiled and linked by the LibreTiny lightning-ln882h builder @@ -51,6 +57,9 @@ void ln_ble_scan_actv_creat(void); void ln_ble_scan_start(void *scan_param); void ln_ble_scan_stop(void); +using ble_evt_cb_t = void (*)(void *param); +void ln_ble_evt_mgr_reg_evt(int evt_id, ble_evt_cb_t cb); + } // extern "C" // ln_bd_addr_v_t mirrors the SDK's ln_bd_addr_t (ln_ble_app_defines.h) and is @@ -64,9 +73,10 @@ static_assert(alignof(struct ln_bd_addr_v_t) == 1, "ln_bd_addr_v_t must stay byt // CLK_G_BLE — hal/hal_clock.h clock gate bit for the BLE block // BLE_EVT_ID_SCAN_REPORT — ble/ble_evt.h event id for scan reports // GAPM_* — ble/mac/ble/hl/api/gapm_task.h, enums gapm_scan_type / -// gapm_dup_filter_pol / gapm_scan_prop +// gapm_dup_filter_pol / gapm_scan_prop / gapm_adv_report_info // --------------------------------------------------------------------------- static constexpr uint32_t CLK_G_BLE = 1u << 0; +static constexpr int BLE_EVT_ID_SCAN_REPORT = 3; // WiFi/BLE packet-traffic-indication (PTI) arbitration register. The LN882H SDK // exposes no symbolic name for this register; the address and value replicate @@ -93,6 +103,20 @@ static constexpr uint8_t GAPM_SCAN_TYPE_OBSERVER = 2; static constexpr uint8_t GAPM_DUP_FILT_DIS = 0; // gapm_scan_prop bits: PHY_1M = 1<<0, PHY_CODED = 1<<1, ACTIVE_1M = 1<<2, ACTIVE_CODED = 1<<3. static constexpr uint8_t GAPM_SCAN_PROP_PHY_1M_BIT = 1 << 0; +static constexpr uint8_t GAPM_SCAN_PROP_ACTIVE_1M_BIT = 1 << 2; + +// GAPM extended-advertising report types (bits 2:0 of ble_scan_report_t::info). +// 0 = ADV_EXT (extended advertisement), 1 = ADV_LEG (legacy advertisement), +// 2 = SCAN_RSP_EXT (scan response to extended adv), 3 = SCAN_RSP_LEG (scan response to legacy adv). +static constexpr uint8_t GAPM_REPORT_TYPE_ADV_LEG = 1; +static constexpr uint8_t GAPM_REPORT_TYPE_SCAN_RSP_LEG = 3; +// Bit 5 of ble_scan_report_t::info: the advertisement is scannable, i.e. a scan +// response may follow (enum gapm_adv_report_info, GAPM_REPORT_INFO_SCAN_ADV_BIT). +static constexpr uint8_t GAPM_REPORT_INFO_SCAN_ADV_BIT = 1u << 5; + +// --------------------------------------------------------------------------- +// SDK struct layouts +// --------------------------------------------------------------------------- // Scan parameter block passed to ln_ble_scan_start(); mirrors the SDK layout, // with the pad byte explicit so the whole block zero-initialises. @@ -110,6 +134,29 @@ struct le_scan_parameters_t { // NOLINT(readability-identifier-naming) - mirror static_assert(sizeof(le_scan_parameters_t) == 8, "le_scan_parameters_t must match the SDK layout"); static_assert(offsetof(le_scan_parameters_t, scan_intv) == 4, "unexpected padding in le_scan_parameters_t"); +// Scan report delivered by the BLE_EVT_ID_SCAN_REPORT event. Layout verified on +// hardware against the prebuilt BLE stack LibreTiny links: its report carries no +// PHY fields and stores the advertisement data inline (flexible array), unlike +// the newer upstream SDK header (which adds phy_prim/phy_second and a data pointer). +struct ble_scan_report_t { // NOLINT(readability-identifier-naming) - mirrors the SDK type name + uint8_t actv_idx; + uint8_t info; + uint8_t trans_addr_type; + uint8_t trans_addr[6]; + uint8_t target_addr_type; + uint8_t target_addr[6]; + int8_t tx_pwr; + int8_t rssi; // signed dBm, range -127..+20 (ble_evt_scan_report_t from ln_ble_event_manager.h) + uint16_t length; + uint8_t data[0]; +}; +// Pin the layout of the hand-mirrored report struct too: the comment above +// notes a newer SDK header uses a different layout (PHY fields + data pointer), +// so silent drift here would corrupt every decoded advertisement. +static_assert(sizeof(ble_scan_report_t) == 20, "ble_scan_report_t must match the linked BLE stack's layout"); +static_assert(offsetof(ble_scan_report_t, length) == 18, "unexpected padding in ble_scan_report_t"); +static_assert(offsetof(ble_scan_report_t, data) == 20, "advertisement data must follow the header inline"); + // --------------------------------------------------------------------------- // __sprintf weak stub // @@ -133,11 +180,95 @@ namespace esphome::ln882h_ble { static const char *const TAG = "ln882h_ble"; +// The SDK event callback is a plain C function pointer with no user argument, +// so it reaches the (single) component instance through a file-static pointer. +static LN882HBLE *s_ble = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +// Scan parameter blocks handed to ln_ble_scan_start(void *). static storage: +// the SDK may retain the pointer past the call (the block travels into a GAPM +// message consumed later by the rw task), so a stack-local would leave the +// controller reading a dead frame. Double-buffered: consecutive starts (the +// enable() probe followed by the first real scan, or a parameter restart) +// alternate blocks, so a rewrite can never race a previous block that is still +// in flight — correct under either reading of SDK retention. All writers run +// on the main task. +static le_scan_parameters_t s_scan_params[2]{}; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +static uint8_t s_scan_params_idx = 0; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +static le_scan_parameters_t *next_scan_params() { + s_scan_params_idx ^= 1; + return &s_scan_params[s_scan_params_idx]; +} + +// --------------------------------------------------------------------------- +// Scan-report event callback — runs in the SDK's rw task context. +// Decode the report (hardware-verified struct layout + the RSSI sign fix), +// copy it into the queue and return; all dispatch happens in loop() on the +// main task. +// --------------------------------------------------------------------------- +static void ble_scan_callback(void *param) { + if (s_ble == nullptr || param == nullptr) + return; + const auto *info = reinterpret_cast(param); + + // Only legacy framing is supported (see scan_start(): legacy 1M PHY only): + // an extended report does not fit BLEScanReport::data and would reach + // consumers as a truncated legacy frame. Reject before allocating so these + // do not burn pool slots either. + const uint8_t report_type = info->info & 0x07; + if (report_type != GAPM_REPORT_TYPE_ADV_LEG && report_type != GAPM_REPORT_TYPE_SCAN_RSP_LEG) { + s_ble->count_rejected_report(); + return; + } + + // Fill the pool slot in place (the bk72xx_ble shape): no report on the rw + // task's stack — its size is fixed by the prebuilt stack — one copy of the + // payload instead of two, and only data_len bytes ever leave this frame. + BLEScanReport *slot = s_ble->allocate_scan_report(); + if (slot == nullptr) + return; // no slot — counted as dropped in allocate_scan_report() + + // BLE RSSI sign fix. The LN882H controller intermittently reports the RSSI with + // a flipped sign: a real -58 dBm arrives as +58, above the SDK's documented + // -127..+20 dBm maximum. Recover it by negating any value above +20 (verified + // on-device: the out-of-range positives cluster at the magnitude of each + // device's real readings). This is the ONLY LN882H-specific RSSI handling — + // downstream the value is used exactly like on ESP32. + const int8_t raw = info->rssi; + + memcpy(slot->mac, info->trans_addr, 6); + slot->rssi = (raw > 20) ? static_cast(-raw) : raw; + slot->addr_type = info->trans_addr_type; + slot->is_scan_response = report_type == GAPM_REPORT_TYPE_SCAN_RSP_LEG; + slot->scannable = (info->info & GAPM_REPORT_INFO_SCAN_ADV_BIT) != 0; + slot->data_len = (info->length <= sizeof(slot->data)) ? static_cast(info->length) + : static_cast(sizeof(slot->data)); + memcpy(slot->data, info->data, slot->data_len); + + s_ble->push_scan_report(slot); +} + +BLEScanReport *LN882HBLE::allocate_scan_report() { + BLEScanReport *slot = this->report_pool_.allocate(); + if (slot == nullptr) { + // No slot: pool exhausted (queue full) or the pool's on-demand RAM + // allocation failed; count and drop either way. + this->report_queue_.increment_dropped_count(); + } + return slot; +} + +void LN882HBLE::push_scan_report(BLEScanReport *report) { + // Cannot fail: the pool is sized to the queue capacity. + this->report_queue_.push(report); +} + // --------------------------------------------------------------------------- // Component lifecycle // --------------------------------------------------------------------------- void LN882HBLE::setup() { + s_ble = this; // Resolve the MAC early so get_mac_lsb_first() is valid for consumers before // the stack is up. The KV load also happens here (no stack dependency). this->resolve_mac_(); @@ -175,23 +306,70 @@ void LN882HBLE::enable() { delay(10); // Prime the scan activity with a short probe start/stop — the SDK's scan - // manager completes activity creation on the first start. - // static: its address is handed to ln_ble_scan_start(void *), which may - // retain it past this call. - static le_scan_parameters_t probe_p{}; - probe_p.type = GAPM_SCAN_TYPE_OBSERVER; - probe_p.prop = GAPM_SCAN_PROP_PHY_1M_BIT; - probe_p.dup_filt_pol = GAPM_DUP_FILT_DIS; - probe_p.scan_intv = 160; - probe_p.scan_wd = 16; - ln_ble_scan_start(&probe_p); + // manager completes activity creation on the first start. Uses the shared + // static parameter block (see s_scan_params for the lifetime rationale). + le_scan_parameters_t *probe = next_scan_params(); + probe->type = GAPM_SCAN_TYPE_OBSERVER; + probe->prop = GAPM_SCAN_PROP_PHY_1M_BIT; + probe->dup_filt_pol = GAPM_DUP_FILT_DIS; + probe->scan_intv = 160; + probe->scan_wd = 16; + ln_ble_scan_start(probe); delay(10); ln_ble_scan_stop(); + // Register the scan-report event exactly once, after the event manager is up. + // Repeated registration corrupts the SDK's event registry (verified on + // hardware), which is why this lives here and not in scan_start(). + ln_ble_evt_mgr_reg_evt(BLE_EVT_ID_SCAN_REPORT, ble_scan_callback); + this->state_ = BLEComponentState::ACTIVE; ESP_LOGD(TAG, "BLE stack initialised"); } +void LN882HBLE::loop() { + // Log dropped reports before the empty-queue return: a drop can also mean + // EventPool::allocate() failed on heap exhaustion, and that can happen with + // the queue empty — from the very first report on. Checking here keeps that + // failure visible instead of producing a scanner that is silently dead. + uint16_t dropped = this->report_queue_.get_and_reset_dropped_count(); + if (dropped > 0) + ESP_LOGW(TAG, "Dropped %u scan reports (queue full or out of memory for a report slot)", dropped); + // Drain the lock-free ring filled by the rw task; all per-report work runs + // here on the main task, then the report returns to the pool. + BLEScanReport *report = this->report_queue_.pop(); + if (report != nullptr) { + this->reject_diagnosis_done_ = true; + do { +#ifdef LN882H_BLE_SCAN_LISTENER_COUNT + for (auto *listener : this->scan_listeners_) + listener->on_scan_report(*report); +#endif + this->report_pool_.release(report); + } while ((report = this->report_queue_.pop()) != nullptr); + } + + // Rejected-report accounting AFTER the drain: a stray non-legacy frame + // arriving ahead of the first good one must not latch the dead-scanner + // warning; the threshold keeps one-off boot noise below it while a truly + // dead scanner (~200 reports/s all rejected) crosses it within a second. + // Avoid the sub-word CAS in the common case (LockFreeQueue's dropped-count + // pattern): rejects are rare, the load is cheap. + uint16_t rejected = this->rejected_reports_.load(std::memory_order_relaxed); + if (rejected > 0) { + rejected = this->rejected_reports_.exchange(0, std::memory_order_relaxed); + if (!this->reject_diagnosis_done_) { + this->rejected_before_delivery_ += rejected; + if (this->rejected_before_delivery_ >= REJECTED_DEAD_SCANNER_THRESHOLD) { + this->reject_diagnosis_done_ = true; + ESP_LOGW(TAG, "Rejected %u scan reports before any was delivered - unexpected report encoding?", + static_cast(this->rejected_before_delivery_)); + } + } + ESP_LOGV(TAG, "Rejected %u non-legacy scan reports", rejected); + } +} + void LN882HBLE::get_mac_lsb_first(uint8_t out[6]) const { memcpy(out, this->ble_mac_, sizeof(this->ble_mac_)); } void LN882HBLE::dump_config() { @@ -246,6 +424,50 @@ void LN882HBLE::resolve_mac_() { memcpy(this->ble_mac_, bt_addr.addr, 6); } +// --------------------------------------------------------------------------- +// Controller scan primitives +// --------------------------------------------------------------------------- + +void LN882HBLE::scan_start(uint16_t interval, uint16_t window, bool active) { + if (!this->is_active()) + this->enable(); + + if (this->scanning_) { + // Already scanning - stop first so this call cleanly restarts with the new + // parameters (re-entry guard). Give the GAPM stop the same settle time + // enable() grants between consecutive GAPM operations before restarting. + this->scan_stop(); + delay(10); // NOLINT — restart-only, mirrors enable()'s inter-operation settle + } + + // Double-buffered static block — see s_scan_params for the lifetime rationale. + le_scan_parameters_t *p = next_scan_params(); + p->dup_filt_pol = GAPM_DUP_FILT_DIS; + p->type = GAPM_SCAN_TYPE_OBSERVER; + p->scan_intv = interval; + p->scan_wd = window; + // Legacy 1M PHY only: consumers size their buffers for legacy advertisements + // (62 B); coded/extended PHY (up to 255 B) would be silently truncated. + p->prop = GAPM_SCAN_PROP_PHY_1M_BIT; + if (active) + p->prop |= GAPM_SCAN_PROP_ACTIVE_1M_BIT; + + ln_ble_scan_start(p); + // ln_ble_scan_start() returns void, so this tracks the requested state, not a + // confirmed one — a controller-side failure surfaces as an idle scanner (no + // reports), which the consumer's start retry/backoff owns. + this->scanning_ = true; +} + +void LN882HBLE::scan_stop() { + // No-op when idle, as documented: the guard keeps a redundant SDK stop off + // the GAPM path (scan_start()'s re-entry guard calls this while scanning). + if (!this->scanning_) + return; + ln_ble_scan_stop(); + this->scanning_ = false; +} + } // namespace esphome::ln882h_ble #endif // USE_LN882H_BLE diff --git a/esphome/components/ln882h_ble/ln882h_ble.h b/esphome/components/ln882h_ble/ln882h_ble.h index 0e3e311341..2186822208 100644 --- a/esphome/components/ln882h_ble/ln882h_ble.h +++ b/esphome/components/ln882h_ble/ln882h_ble.h @@ -5,7 +5,11 @@ #ifdef USE_LN882H_BLE #include "esphome/core/component.h" +#include "esphome/core/event_pool.h" +#include "esphome/core/helpers.h" +#include "esphome/core/lock_free_queue.h" +#include #include namespace esphome::ln882h_ble { @@ -16,9 +20,56 @@ enum class BLEComponentState : uint8_t { ACTIVE, }; +/// One scan report from the controller, decoded from the SDK's rw-task event +/// (RSSI already sign-corrected). +struct BLEScanReport { + uint8_t mac[6]; // as the controller delivers it (LSB-first) + int8_t rssi; // signed dBm (-127..+20) + uint8_t addr_type; + bool is_scan_response; // report is a scan response (active scan) + bool scannable; // advertisement may be followed by a scan response + uint8_t data_len; // bytes valid in data[] (<= 62) + // Each report carries ONE frame — a legacy advertisement (<=31 B) or a scan + // response (<=31 B) — delivered split, exactly as the SDK reports them. The + // TRACKER merges the pair into a single frame before any consumer sees it + // (Bluedroid semantics, HubCapabilities::merges_scan_response). 62 is twice + // the legacy maximum: defensive headroom for the data_len clamp, and the + // same width as the merged framing downstream. + uint8_t data[62]; + + // EventPool contract: nothing is heap-allocated inside a report. + void release() {} +}; + +/// Consumer interface for controller scan reports. on_scan_report() always runs +/// on the ESPHome main task: reports are queued from the SDK's rw task and +/// drained by the controller's loop(), so consumers never deal with cross-task +/// state (the esp32_ble event-queue pattern). +class BLEScanListener { + public: + virtual void on_scan_report(const BLEScanReport &report) = 0; + + protected: + ~BLEScanListener() = default; // deletion via this interface is not part of the contract +}; + +// Maximum reports buffered between the rw task and loop(). Sized from the +// measured worst case, not copied: WiFi/BLE coexistence delays rw-task report +// delivery by up to ~136 ms on this device (see the tracker's pending-adv +// timeout rationale), and a busy 2.4 GHz environment delivers ~200-400 +// reports/s — a stall plus one loop() interval buffers ~30-60 reports, so 63 +// usable slots absorb it with margin. ~4.7 KB at high water, reached only +// during such stalls. +static constexpr uint8_t MAX_SCAN_REPORT_QUEUE_SIZE = 64; + +// Rejected frames tolerated before the first delivered report without +// declaring the scanner dead (boot-time stray extended frames are normal). +static constexpr uint16_t REJECTED_DEAD_SCANNER_THRESHOLD = 16; + class LN882HBLE final : public Component { public: void setup() override; + void loop() override; void dump_config() override; float get_setup_priority() const override; @@ -37,12 +88,64 @@ class LN882HBLE final : public Component { /// (the bk72xx sibling exposes the same accessor). void get_mac_lsb_first(uint8_t out[6]) const; +#ifdef LN882H_BLE_SCAN_LISTENER_COUNT + /// Register a consumer for scan reports (delivered on the main task via loop()). + /// Storage is codegen-sized: the consumer's codegen requests a slot via + /// request_scan_listener_slot(), which emits LN882H_BLE_SCAN_LISTENER_COUNT. + void register_scan_listener(BLEScanListener *listener) { this->scan_listeners_.push_back(listener); } +#endif + + /// Start the controller scan. Interval/window are in BLE units (0.625 ms); + /// active enables scan requests on the 1M PHY. Enables the stack first if + /// needed. Scans the legacy 1M PHY only (extended/coded PHY advertisements + /// exceed the legacy 62-byte framing consumers are sized for). + void scan_start(uint16_t interval, uint16_t window, bool active); + /// Stop the controller scan (no-op when not scanning). + void scan_stop(); + + /// Internal, SDK rw-task event-callback context: allocate a pool slot for a + /// scan report. Returns nullptr (and counts the drop) when the queue is full; + /// the callback fills the slot in place — no intermediate copy. + BLEScanReport *allocate_scan_report(); + /// Internal: hand a filled slot to the main-task queue (cannot fail — the + /// pool is sized to the queue capacity). + void push_scan_report(BLEScanReport *report); + /// Internal, rw-task context: count a report rejected by the legacy-only + /// filter, so a wrong assumption about the stack's report encoding shows up + /// in verbose logs instead of as a scanner that silently reports nothing. + void count_rejected_report() { this->rejected_reports_.fetch_add(1, std::memory_order_relaxed); } + protected: void resolve_mac_(); +#ifdef LN882H_BLE_SCAN_LISTENER_COUNT + // Codegen-sized: no heap allocation, no std::vector template instantiation — + // the same StaticVector pattern as the tracker's ble_device_base listeners. + StaticVector scan_listeners_; +#endif + // Report ring: the SDK event callback (rw task) allocates a report from the + // pool, fills it and pushes the pointer; loop() pops, dispatches and releases. + // Lock-free SPSC, zero allocation at steady state — the esp32_ble pattern. + // Overflow drops the NEWEST report (allocate fails, producer counts and + // returns) — under a coexistence stall the freshest advertisements are lost + // while queued ones drain. Deliberate: matches esp32_ble, and dropping from + // the head would need consumer-side locking this design exists to avoid. + esphome::LockFreeQueue report_queue_; + // Pool sized to queue capacity (SIZE-1): the ring reserves one slot, so + // allocate() returns nullptr before push() can fail. This prevents leaking a + // pool slot on a failed push and keeps release() off the producer path. + esphome::EventPool report_pool_; + // Reports rejected by the legacy-only filter (rw-task producer, main-task + // consumer via exchange in loop()). + std::atomic rejected_reports_{0}; uint8_t ble_mac_[6]{0}; // controller (LSB-first) order, as ln_bd_addr_t stores it BLEComponentState state_{BLEComponentState::STATE_OFF}; bool enable_on_boot_{false}; + bool scanning_{false}; // controller scan running (re-entry guard for scan_start) + // Dead-scanner diagnosis: done once a report is delivered or the one-shot + // warning has fired, whichever comes first. + bool reject_diagnosis_done_{false}; + uint32_t rejected_before_delivery_{0}; // drives the dead-scanner warning }; } // namespace esphome::ln882h_ble diff --git a/esphome/components/ln882h_ble_tracker/__init__.py b/esphome/components/ln882h_ble_tracker/__init__.py new file mode 100644 index 0000000000..ceb2aeffec --- /dev/null +++ b/esphome/components/ln882h_ble_tracker/__init__.py @@ -0,0 +1,166 @@ +"""LN882H BLE scanner implementing the ble_device_base BLEHub contract on +top of the ln882h_ble controller. With continuous: false nothing scans until +an explicit start_scan() call.""" + +from esphome import automation +import esphome.codegen as cg +from esphome.components import ble_device_base, ln882h_ble, ota +from esphome.components.ble_device_base import automation as ble_automation +from esphome.components.const import CONF_ON_SCAN_END, CONF_SCAN_PARAMETERS, CONF_WINDOW +import esphome.config_validation as cv +from esphome.const import ( + CONF_ACTIVE, + CONF_CONTINUOUS, + CONF_DURATION, + CONF_ID, + CONF_INTERVAL, + CONF_MANUFACTURER_ID, + CONF_ON_BLE_ADVERTISE, + CONF_ON_BLE_MANUFACTURER_DATA_ADVERTISE, + CONF_ON_BLE_SERVICE_DATA_ADVERTISE, + CONF_SERVICE_UUID, +) +from esphome.core import ID +from esphome.types import ConfigType + +CONF_LN882H_BLE_ID = "ln882h_ble_id" + +DEPENDENCIES = ["ln882x"] +AUTO_LOAD = ["ble_device_base", "ln882h_ble"] +CODEOWNERS = ["@Bl00d-B0b"] + +ble_device_base.register_hub_provider("ln882h_ble_tracker") + +ln882h_ble_tracker_ns = cg.esphome_ns.namespace("ln882h_ble_tracker") +LN882HBLETracker = ln882h_ble_tracker_ns.class_( + "LN882HBLETracker", ble_device_base.BLEHub, cg.Component +) + +StartScanAction = ln882h_ble_tracker_ns.class_("StartScanAction", automation.Action) +StopScanAction = ln882h_ble_tracker_ns.class_("StopScanAction", automation.Action) + +ESPBTAdvertiseTrigger = ble_automation.ESPBTAdvertiseTrigger +BLEServiceDataAdvertiseTrigger = ble_automation.BLEServiceDataAdvertiseTrigger +BLEManufacturerDataAdvertiseTrigger = ble_automation.BLEManufacturerDataAdvertiseTrigger +BLEEndOfScanTrigger = ble_automation.BLEEndOfScanTrigger + + +# LN882H SDK reference scan rate: 100 ms interval / 50 ms window (50 % duty). +SCAN_PARAMETERS_SCHEMA = ble_device_base.scan_parameters_schema( + "100ms", window_default="50ms", supports_active=True +) + + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(): cv.declare_id(LN882HBLETracker), + cv.GenerateID(CONF_LN882H_BLE_ID): cv.use_id(ln882h_ble.LN882HBLE), + cv.Optional(CONF_SCAN_PARAMETERS, default={}): SCAN_PARAMETERS_SCHEMA, + cv.Optional(CONF_ON_BLE_ADVERTISE): ble_automation.advertise_trigger_schema( + ESPBTAdvertiseTrigger + ), + cv.Optional( + CONF_ON_BLE_SERVICE_DATA_ADVERTISE + ): ble_automation.uuid_trigger_schema( + BLEServiceDataAdvertiseTrigger, + {cv.Required(CONF_SERVICE_UUID): ble_device_base.bt_uuid}, + ), + cv.Optional( + CONF_ON_BLE_MANUFACTURER_DATA_ADVERTISE + ): ble_automation.uuid_trigger_schema( + BLEManufacturerDataAdvertiseTrigger, + {cv.Required(CONF_MANUFACTURER_ID): ble_device_base.bt_uuid}, + ), + cv.Optional(CONF_ON_SCAN_END): ble_automation.scan_end_trigger_schema( + BLEEndOfScanTrigger + ), + } +).extend(cv.COMPONENT_SCHEMA) + + +@automation.register_action( + "ln882h_ble_tracker.start_scan", + StartScanAction, + cv.Schema( + { + cv.GenerateID(): cv.use_id(LN882HBLETracker), + cv.Optional(CONF_CONTINUOUS): cv.templatable(cv.boolean), + } + ), + synchronous=True, +) +async def start_scan_action_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: list, +) -> cg.MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + if (continuous := config.get(CONF_CONTINUOUS)) is not None: + template_ = await cg.templatable(continuous, args, cg.bool_) + cg.add(var.set_continuous(template_)) + return var + + +@automation.register_action( + "ln882h_ble_tracker.stop_scan", + StopScanAction, + automation.maybe_simple_id( + cv.Schema( + { + cv.GenerateID(): cv.use_id(LN882HBLETracker), + } + ) + ), + synchronous=True, +) +async def stop_scan_action_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: list, +) -> cg.MockObj: + var = cg.new_Pvariable(action_id, template_arg) + await cg.register_parented(var, config[CONF_ID]) + return var + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + + parent = await cg.get_variable(config[CONF_LN882H_BLE_ID]) + cg.add(var.set_parent(parent)) + # The tracker registers itself as a controller scan listener in setup(); + # request the codegen-sized StaticVector slot for it. + ln882h_ble.request_scan_listener_slot() + + # Get notified when an OTA update starts, to pause scanning (esp32_ble_tracker parity) + ota.request_ota_state_listeners() + + scan = config[CONF_SCAN_PARAMETERS] + cg.add(var.set_scan_interval(ble_device_base.to_ble_units(scan[CONF_INTERVAL]))) + cg.add(var.set_scan_window(ble_device_base.to_ble_units(scan[CONF_WINDOW]))) + cg.add(var.set_scan_duration(scan[CONF_DURATION].total_milliseconds)) + cg.add(var.set_scan_active(scan[CONF_ACTIVE])) + cg.add(var.set_configured_continuous(scan[CONF_CONTINUOUS])) + + for conf in config.get(CONF_ON_BLE_ADVERTISE, []): + await ble_automation.advertise_trigger_to_code(conf, var) + + for trigger_key, uuid_key, setter_prefix in ( + (CONF_ON_BLE_SERVICE_DATA_ADVERTISE, CONF_SERVICE_UUID, "set_service_uuid"), + ( + CONF_ON_BLE_MANUFACTURER_DATA_ADVERTISE, + CONF_MANUFACTURER_ID, + "set_manufacturer_uuid", + ), + ): + for conf in config.get(trigger_key, []): + await ble_automation.uuid_trigger_to_code( + conf, var, uuid_key, setter_prefix + ) + + for conf in config.get(CONF_ON_SCAN_END, []): + await ble_automation.scan_end_trigger_to_code(conf, var) diff --git a/esphome/components/ln882h_ble_tracker/automation.h b/esphome/components/ln882h_ble_tracker/automation.h new file mode 100644 index 0000000000..8b211384a0 --- /dev/null +++ b/esphome/components/ln882h_ble_tracker/automation.h @@ -0,0 +1,47 @@ +// Scan-control actions for ln882h_ble_tracker. The automation triggers are the +// neutral ble_device_base classes (ble_device_base/automation.h). + +#pragma once + +#ifdef USE_LIBRETINY + +#include "ln882h_ble_tracker.h" + +#include "esphome/core/automation.h" +#include "esphome/core/helpers.h" + +namespace esphome::ln882h_ble_tracker { + +template class StartScanAction final : public Action, public Parented { + public: + TEMPLATABLE_VALUE(bool, continuous) + void play(const Ts &...x) override { + // With continuous: set, the action wins. Without it, the configured value + // is used - stop_scan() clears the runtime flag permanently, so a bare + // stop_scan/start_scan pair would otherwise never resume continuous mode. + const bool want = + this->continuous_.has_value() ? this->continuous_.value(x...) : this->parent_->configured_continuous(); + if (this->parent_->scan_running()) { + // Same mode on a running scan is a no-op (esp32 parity): re-anchoring + // the duration window here would let a repeated action keep a one-shot + // scan alive forever. A real mode switch re-anchors so a change to + // one-shot runs a full duration from now. + if (want != this->parent_->scan_continuous()) { + this->parent_->set_scan_continuous(want); + this->parent_->restart_scan_duration(); + } + return; + } + this->parent_->set_scan_continuous(want); + this->parent_->start_scan(); + } +}; + +template class StopScanAction final : public Action, public Parented { + public: + void play(const Ts &...x) override { this->parent_->stop_scan(); } +}; + +} // namespace esphome::ln882h_ble_tracker + +#endif // USE_LIBRETINY diff --git a/esphome/components/ln882h_ble_tracker/ln882h_ble_tracker.cpp b/esphome/components/ln882h_ble_tracker/ln882h_ble_tracker.cpp new file mode 100644 index 0000000000..90be341820 --- /dev/null +++ b/esphome/components/ln882h_ble_tracker/ln882h_ble_tracker.cpp @@ -0,0 +1,381 @@ +#ifdef USE_LIBRETINY + +#include "ln882h_ble_tracker.h" + +#include +#include + +#include "esphome/core/hal.h" +#include "esphome/core/log.h" + +namespace esphome::ln882h_ble_tracker { + +static const char *const TAG = "ln882h_ble_tracker"; + +static constexpr float BLE_SCAN_UNIT_MS = 0.625f; + +// --------------------------------------------------------------------------- +// Component lifecycle +// --------------------------------------------------------------------------- + +void LN882HBLETracker::setup() { + // Receive the controller's scan reports; the controller queues them from the + // rw task and delivers here on the main task. + this->parent_->register_scan_listener(this); + // scan_running_ check: an on_boot start_scan action (priority 600) runs + // before this setup() (200) and enable_loop() is a no-op pre-setup — parking + // the loop here would strand that already-running scan. + if (!this->scan_continuous_ && !this->scan_running_ && !this->pending_start_) { + // Say so once: with continuous: false nothing scans until an explicit + // start_scan() — silence here reads as a broken scanner. + ESP_LOGD(TAG, "Scanning not started (continuous: false) - waiting for an explicit start_scan()"); + // Nothing to time until then; start_scan_() re-enables the loop. + this->disable_loop(); + } +#ifdef USE_OTA_STATE_LISTENER + // Pause scanning while an OTA update is in flight — on the single-core LN882H the + // BLE scan competes with the OTA flash writes. Mirrors esp32_ble_tracker. + ota::get_global_ota_callback()->add_global_state_listener(this); +#endif +} + +#ifdef USE_OTA_STATE_LISTENER +void LN882HBLETracker::on_ota_global_state(ota::OTAState state, float progress, uint8_t error, + ota::OTAComponent *comp) { + if (state == ota::OTA_STARTED) { + this->scan_continuous_before_ota_ = this->scan_continuous_; + this->scan_running_before_ota_ = this->scan_running_; + this->stop_scan(); + } else if (state == ota::OTA_ERROR || state == ota::OTA_ABORT) { + // On success the device reboots, so restore only on a failed/aborted + // update. Continuous mode resumes via loop()'s idle branch; a one-shot + // scan that was running is restarted explicitly (bk72xx sibling parity — + // stop_scan() cleared it and nothing else would bring it back). + if (this->scan_continuous_before_ota_) { + this->scan_continuous_ = true; + this->enable_loop(); // stop_scan() disabled it; loop()'s idle branch restarts the scan + } else if (this->scan_running_before_ota_) { + this->start_scan(); + } + this->scan_continuous_before_ota_ = false; + this->scan_running_before_ota_ = false; + } +} +#endif // USE_OTA_STATE_LISTENER + +void LN882HBLETracker::loop() { + if (this->pending_start_) { + // A start_scan latched before the controller's setup(); safe now — loop() + // only runs after every component set up. + this->pending_start_ = false; + if (!this->scan_running_) { + this->start_scan_(); + } + } + // Flush pending scannable advertisements whose scan response never arrived + // (device didn't answer / frame lost) — delivered unmerged after the timeout. + // Main-task only, like every consumer of pending_adv_. + const uint32_t now = millis(); + if (this->pending_count_ != 0) { + for (auto &p : this->pending_adv_) { + if (p.used && now - p.stored_ms > PENDING_ADV_TIMEOUT_MS) { + p.used = false; + this->pending_count_--; + this->process_adv_(p.mac, p.rssi, p.addr_type, p.data, p.data_len, /*raw_only=*/false); + } + } + } + + if (this->scan_continuous_) { + if (!this->scan_running_) { + this->start_scan_(); + // start_scan_() re-anchors scan_period_start_ from a later millis() than + // the cached `now`; resume the period timer next iteration. + return; + } + // Period timer: once per scan_duration_ window, restart the controller scan + // and fire on_scan_end(), mirroring esp32_ble_tracker::cleanup_scan_state_(). + // The restart is the recovery path for the coexistence failure documented in + // the header. scan_start() re-enters cleanly on its own: it stops an + // in-flight scan and grants the controller's 10 ms GAPM settle before + // restarting — an explicit scan_stop() first would clear the controller's + // re-entry guard and skip that settle. + if (now - this->scan_period_start_ >= this->scan_duration_) { + ESP_LOGD(TAG, "Scan period elapsed - restarting scan"); + this->parent_->scan_start(this->scan_interval_, this->scan_window_, this->scan_active_); + // Keep both clocks anchored to the restart: a runtime switch to + // non-continuous then times out the current period, not the whole run. + this->scan_start_time_ = now; + this->end_scan_period_(now); + } + return; + } + + // Non-continuous mode: run for scan_duration_ ms, then stop and fire on_scan_end. + // Restart is driven externally (e.g. wifi: on_connect:). + if (this->scan_running_ && now - this->scan_start_time_ >= this->scan_duration_) { + this->stop_scan_(); + } +} + +bool LN882HBLETracker::request_scan_mode(bool active) { + if (this->scan_active_ == active) + return true; + this->scan_active_ = active; + ESP_LOGD(TAG, "Scan mode %s", active ? "active" : "passive"); + // scan_start() re-enters cleanly (stops + GAPM settle). No on_scan_end and + // no period reset: the scan logically continues, only the mode changes. + if (this->scan_running_) { + this->parent_->scan_start(this->scan_interval_, this->scan_window_, this->scan_active_); + } + return true; +} + +void LN882HBLETracker::dump_config() { + ESP_LOGCONFIG(TAG, + "LN882H BLE Tracker:\n" + " Scan Duration: %" PRIu32 " s\n" + " Scan Interval: %.0f ms (%" PRIu16 " BLE units)\n" + " Scan Window: %.0f ms (%" PRIu16 " BLE units)\n" + " Scan Type: %s\n" + " Continuous Scanning: %s", + this->scan_duration_ / 1000, this->scan_interval_ * BLE_SCAN_UNIT_MS, this->scan_interval_, + this->scan_window_ * BLE_SCAN_UNIT_MS, this->scan_window_, this->scan_active_ ? "ACTIVE" : "PASSIVE", + YESNO(this->scan_continuous_)); +} + +// --------------------------------------------------------------------------- +// Adv/scan-response demux with Bluedroid-style merge: the LN controller +// delivers the pair as separate reports; a scannable advertisement is held +// until its scan response arrives and delivered as one merged frame. +// --------------------------------------------------------------------------- + +void LN882HBLETracker::on_scan_report(const ln882h_ble::BLEScanReport &report) { + if (report.is_scan_response) { + this->deliver_scan_rsp_(report); + return; + } + // Stash only while the scan runs: after a one-shot stop the loop is + // disabled and nothing would sweep the table, so a late report would + // surface minutes later as a fresh advertisement. + if (this->scan_running_ && this->scan_active_ && report.scannable) { + this->stash_adv_(report); + return; + } + this->process_adv_(report.mac, report.rssi, report.addr_type, report.data, report.data_len, /*raw_only=*/false); +} + +// Hold a scannable advertisement, waiting (≤ PENDING_ADV_TIMEOUT_MS) for its +// scan response. +void LN882HBLETracker::stash_adv_(const ln882h_ble::BLEScanReport &report) { + // One pass: find a same-device entry (deliver + reuse) while remembering the + // first free slot as the fallback. + PendingAdv *slot = nullptr; + PendingAdv *free_slot = nullptr; + for (auto &p : this->pending_adv_) { + if (!p.used) { + if (free_slot == nullptr) + free_slot = &p; + continue; + } + if (p.addr_type == report.addr_type && memcmp(p.mac, report.mac, 6) == 0) { + // Same device advertised again before its scan response arrived — deliver + // the previous advertisement (its scan response is not coming) and reuse + // the slot, so no frame is ever lost. + p.used = false; + this->pending_count_--; + this->process_adv_(p.mac, p.rssi, p.addr_type, p.data, p.data_len, /*raw_only=*/false); + slot = &p; + break; + } + } + if (slot == nullptr) + slot = free_slot; + if (slot == nullptr) { + // Table full — degrade gracefully: deliver the advertisement unmerged. + this->process_adv_(report.mac, report.rssi, report.addr_type, report.data, report.data_len, /*raw_only=*/false); + return; + } + slot->used = true; + this->pending_count_++; + memcpy(slot->mac, report.mac, 6); + slot->addr_type = report.addr_type; + slot->rssi = report.rssi; + slot->data_len = (report.data_len <= sizeof(slot->data)) ? report.data_len : sizeof(slot->data); + memcpy(slot->data, report.data, slot->data_len); + slot->stored_ms = millis(); +} + +// Scan response arrived: merge it with the pending advertisement from the same +// device into ONE frame (ESP-IDF/Bluedroid semantics). +void LN882HBLETracker::deliver_scan_rsp_(const ln882h_ble::BLEScanReport &report) { + // Fast-out on the empty table (loop()/flush use the same guard); this is + // the hottest caller. + if (this->pending_count_ != 0) { + for (auto &p : this->pending_adv_) { + if (p.used && p.addr_type == report.addr_type && memcmp(p.mac, report.mac, 6) == 0) { + // Append in place: the slot is released on delivery, so its 62-byte + // buffer (legacy adv + scan response) holds the merged frame directly. + const uint8_t room = sizeof(p.data) - p.data_len; + const uint8_t add = (report.data_len <= room) ? report.data_len : room; + memcpy(p.data + p.data_len, report.data, add); + p.used = false; + this->pending_count_--; + // The advertisement's RSSI, not the scan response's: every unmerged path + // reports the advertisement's measurement, so a device's RSSI must not + // jump between two measurements depending on merge timing. + this->process_adv_(report.mac, p.rssi, report.addr_type, p.data, p.data_len + add, /*raw_only=*/false); + return; + } + } + } + // Unmatched scan-response: goes out on the raw callback only (HA merges per + // address); local listeners/triggers receive each advertisement exactly once + // via the merged/plain path above. + this->process_adv_(report.mac, report.rssi, report.addr_type, report.data, report.data_len, /*raw_only=*/true); +} + +void LN882HBLETracker::process_adv_(const uint8_t *mac, int8_t rssi, uint8_t addr_type, const uint8_t *data, + uint8_t data_len, bool raw_only) { + // Raw callback (the raw-advertisement path). Both full advertisements and + // unmatched scan responses (raw_only) are forwarded. + if (this->raw_advertisement_callback_.is_set()) { + const ble_device_base::RawAdvertisement adv{ + .mac = mac, .data = data, .data_len = data_len, .rssi = rssi, .addr_type = addr_type}; + this->raw_advertisement_callback_.invoke(adv); + } + +#ifdef ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT + // Scan-response-only frames are never parsed for local sensors/triggers. + if (raw_only) + return; + ble_device_base::ESPBTDevice device; + device.from_scan_result(mac, rssi, addr_type, data, data_len); + // The listener list holds sensors AND this tracker's automation triggers + // (the triggers are listeners, exactly like esp32_ble_tracker), so one + // loop feeds both and ORs into `found`. + bool found = false; + for (auto *listener : this->listeners_) { + if (listener->parse_device(device)) { + found = true; + } + } + // Mirror esp32_ble_tracker: log a newly-seen device only when nothing claimed + // it and the scan is one-shot (continuous scans would spam). + if (!found && !this->scan_continuous_) + this->discovered_log_.log_device(TAG, device); +#endif // ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT +} + +// --------------------------------------------------------------------------- +// Public scan actions +// --------------------------------------------------------------------------- + +void LN882HBLETracker::start_scan() { + // Mirrors esp32_ble_tracker::start_scan(): caller sets scan_continuous_ via + // set_scan_continuous() first, then calls start_scan() to begin scanning. + if (!this->parent_->is_ready()) { + // An on_boot automation (priority 600) runs before the controller's + // setup() has resolved the BLE MAC; scan_start() now would rw_init() the + // all-zero address and bring BLE up before WiFi. Latch; loop() applies + // the start once every setup() has run. + this->pending_start_ = true; + return; + } + if (!this->scan_running_) { + this->start_scan_(); + } +} + +void LN882HBLETracker::restart_scan_duration() { + if (!this->scan_running_) + return; + // Re-anchor only the one-shot duration clock. scan_period_start_ (the + // continuous-mode on_scan_end period) is deliberately left alone: a + // start_scan action fired more often than scan_duration_ would otherwise + // suppress on_scan_end indefinitely — and absence detection (ble_rssi's NAN + // publish) rides on that period. + this->scan_start_time_ = millis(); +} + +void LN882HBLETracker::stop_scan() { + // Cancel a start latched before the controller's setup(); without this an + // on_boot start_scan/stop_scan pair would still start at the first loop(). + this->pending_start_ = false; + this->scan_continuous_ = false; + this->stop_scan_(); +} + +// --------------------------------------------------------------------------- +// Internal scan start / stop +// --------------------------------------------------------------------------- + +void LN882HBLETracker::start_scan_() { + if (this->scan_running_) + return; + + // The controller enables the stack on first use and owns the report queue; + // this call is all the SDK interaction the tracker ever needs. + this->parent_->scan_start(this->scan_interval_, this->scan_window_, this->scan_active_); + const uint32_t now = millis(); + this->scan_running_ = true; + this->scan_start_time_ = now; + this->enable_loop(); // an idle non-continuous tracker disabled it in stop_scan_() + // Log every explicit start at DEBUG — stop_scan_() logs every stop at DEBUG, and + // in non-continuous mode each period is an explicit start, so asymmetric logging + // would read as the scanner failing to come back up. + ESP_LOGD(TAG, "BLE scan started (%s, window=%.0fms, interval=%.0fms)", this->scan_active_ ? "active" : "passive", + this->scan_window_ * BLE_SCAN_UNIT_MS, this->scan_interval_ * BLE_SCAN_UNIT_MS); + // Re-anchor the on_scan_end period to every successful start, so a restart + // later than scan_duration (e.g. a failed OTA restoring continuous mode) + // does not fire on_scan_end before an advertisement can arrive. + this->scan_period_start_ = now; +} + +void LN882HBLETracker::stop_scan_() { + if (!this->scan_running_) + return; + this->parent_->scan_stop(); + this->scan_running_ = false; + // DEBUG like start_scan_() — a per-period stop at INFO would read as the + // scanner failing to come back up. + ESP_LOGD(TAG, "BLE scan stopped"); + this->end_scan_period_(millis()); // also resets the period clock so on_scan_end does not double-fire + // scan_running_ re-check: an on_scan_end automation runs synchronously inside + // end_scan_period_() and may have called start_scan() — parking the loop then + // would leave the radio scanning with no period timing or pending-adv sweep. + if (!this->scan_continuous_ && !this->scan_running_) { + // Nothing left to time; start_scan_() re-enables the loop. + this->disable_loop(); + } +} + +// Close a scan period: deliver held advertisements whose scan response never +// came (unmerged) BEFORE on_scan_end fires, then re-anchor the period clock. +void LN882HBLETracker::end_scan_period_(uint32_t now) { + this->flush_pending_adv_(); +#ifdef ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT + for (auto *listener : this->listeners_) + listener->on_scan_end(); + this->discovered_log_.clear(); // reset per-scan "Found device" dedup (esp32_ble_tracker parity) +#endif + this->scan_period_start_ = now; +} + +// Deliver every held advertisement now (scan period/scan is ending): unmerged +// delivery, same as the timeout path in loop(). Main-task only. +void LN882HBLETracker::flush_pending_adv_() { + if (this->pending_count_ == 0) + return; + for (auto &p : this->pending_adv_) { + if (p.used) { + p.used = false; + this->process_adv_(p.mac, p.rssi, p.addr_type, p.data, p.data_len, /*raw_only=*/false); + } + } + this->pending_count_ = 0; +} + +} // namespace esphome::ln882h_ble_tracker + +#endif // USE_LIBRETINY diff --git a/esphome/components/ln882h_ble_tracker/ln882h_ble_tracker.h b/esphome/components/ln882h_ble_tracker/ln882h_ble_tracker.h new file mode 100644 index 0000000000..1ad36c40d4 --- /dev/null +++ b/esphome/components/ln882h_ble_tracker/ln882h_ble_tracker.h @@ -0,0 +1,194 @@ +// BLE scanner for LN882H: implements ble_device_base::BLEHub on top of the +// ln882h_ble controller (which owns all SDK calls and delivers scan reports on +// the main task). Scan policy lives here: parameters, period timers with +// per-period restart, and the adv+scan-response merge. + +#pragma once + +#ifdef USE_LIBRETINY + +#include "esphome/components/ble_device_base/ble_device.h" +#include "esphome/components/ble_device_base/ble_hub.h" +#include "esphome/components/ln882h_ble/ln882h_ble.h" +#include "esphome/core/component.h" +#include "esphome/core/helpers.h" + +#include + +#ifdef USE_OTA_STATE_LISTENER +#include "esphome/components/ota/ota_backend.h" +#endif + +namespace esphome::ln882h_ble_tracker { + +// --------------------------------------------------------------------------- +// LN882HBLETracker +// --------------------------------------------------------------------------- + +class LN882HBLETracker : public Component, + public ble_device_base::BLEHub, + public Parented, + public ln882h_ble::BLEScanListener +#ifdef USE_OTA_STATE_LISTENER + , + public ota::OTAGlobalStateListener +#endif +{ + public: + // ---- ESPHome Component ---- + void setup() override; + void loop() override; + void dump_config() override; + float get_setup_priority() const override { return setup_priority::AFTER_WIFI; } + +#ifdef USE_OTA_STATE_LISTENER + // Pause scanning while an OTA update runs (single-core WiFi/BLE/flash contention); + // mirrors esp32_ble_tracker. + void on_ota_global_state(ota::OTAState state, float progress, uint8_t error, ota::OTAComponent *comp) override; +#endif + + // ---- YAML configuration setters ---- + void set_scan_active(bool scan_active) { this->scan_active_ = scan_active; } + void set_scan_interval(uint16_t scan_interval) { this->scan_interval_ = scan_interval; } + void set_scan_window(uint16_t scan_window) { this->scan_window_ = scan_window; } + void set_scan_duration(uint32_t scan_duration) { this->scan_duration_ = scan_duration; } + /// Set from YAML (scan_parameters.continuous); also the value + /// configured_continuous() reports and a bare start_scan action restores. + void set_configured_continuous(bool scan_continuous) { + this->scan_continuous_ = scan_continuous; + this->scan_continuous_configured_ = scan_continuous; + } + /// Runtime control (esp32_ble_tracker lambda parity): does not change the + /// configured value, so configured_continuous() still reports what YAML + /// asked for. + void set_scan_continuous(bool scan_continuous) { this->scan_continuous_ = scan_continuous; } + bool scan_continuous() const { return this->scan_continuous_; } + bool configured_continuous() const { return this->scan_continuous_configured_; } + /// Re-anchor the one-shot duration clock of a running scan to now — used + /// when an action changes the scan mode without stopping the radio. The + /// continuous-mode on_scan_end period is deliberately not touched. + void restart_scan_duration(); + + // ---- Public scan control ---- + // Mirrors esp32_ble_tracker: set_scan_continuous() + start_scan() / stop_scan(). + void start_scan(); + void stop_scan(); + + // ---- ble_device_base::BLEHub contract ---- + void register_listener(ble_device_base::ESPBTDeviceListener *listener) override { +#ifdef ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT + this->listeners_.push_back(listener); +#endif + } + void set_raw_advertisement_callback(ble_device_base::RawAdvertisementCallback callback) override { + this->raw_advertisement_callback_ = callback; + } + ble_device_base::HubCapabilities get_capabilities() const override { + // The LN882H controller supports active scanning; adv + scan response arrive + // as separate reports and are merged by this tracker (Bluedroid semantics). + // The SDK's GATT client is not exposed. + // scan_mode_switch: request_scan_mode() is implemented (restart-if-running). + return {.active_scan = true, .merges_scan_response = true, .gatt = false, .scan_mode_switch = true}; + } + // The controller stores the address LSB-first (BLE convention); the contract + // wants printable (MSB-first) order. + void get_adapter_mac(uint8_t out[6]) override { + uint8_t mac[6]; + this->parent_->get_mac_lsb_first(mac); + for (int i = 0; i < 6; i++) + out[i] = mac[5 - i]; + } + bool scan_running() override { return this->scan_running_; } + bool scan_active() override { return this->scan_active_; } + bool request_scan_mode(bool active) override; + + // ---- ln882h_ble::BLEScanListener ---- + // Delivered by the controller's loop() on the ESPHome main task — the + // rw-task → main-task handoff already happened in the controller's queue. + // Demultiplexes advertisements vs scan responses and drives the merge. + void on_scan_report(const ln882h_ble::BLEScanReport &report) override; + + protected: + // Bluedroid-style adv + scan-response merging (ESP-IDF concatenates both into + // one result before ESPHome sees it; the LN controller reports them separately): + // a scannable advertisement is held here briefly, its scan response is appended + // on arrival and the pair is delivered as ONE merged frame. Held entries whose + // scan response never arrives are flushed by loop() after PENDING_ADV_TIMEOUT_MS. + // All of this runs on the main task (the controller queue already crossed tasks), + // so no locking is involved. + void stash_adv_(const ln882h_ble::BLEScanReport &report); + void deliver_scan_rsp_(const ln882h_ble::BLEScanReport &report); + // Dispatch one (possibly merged) advertisement: the raw + // callback, and — unless raw_only — parsing for listeners/triggers. raw_only + // marks unmatched scan-response frames: forwarded on the raw callback only, + // never to local sensors/triggers (HA merges per address). + void process_adv_(const uint8_t *mac, int8_t rssi, uint8_t addr_type, const uint8_t *data, uint8_t data_len, + bool raw_only); + void start_scan_(); + void stop_scan_(); + // Close a scan period: flush held advertisements (unmerged) BEFORE + // on_scan_end fires, then re-anchor the period clock to `now`. + void end_scan_period_(uint32_t now); + void flush_pending_adv_(); + + bool scan_running_{false}; + bool scan_active_{false}; + // Defaults are the LN882H SDK's recommended scan parameters + // (ln_ble_scan.h: SCAN_INTERVAL_DEF 0xA0, SCAN_WINDOW_DEF 0x50 → 50 % duty). + // uint16_t matches the controller's scan_start() parameters. + uint16_t scan_interval_{160}; // 160 × 0.625 ms = 100 ms (SDK SCAN_INTERVAL_DEF) + uint16_t scan_window_{80}; // 80 × 0.625 ms = 50 ms (SDK SCAN_WINDOW_DEF; 50/100 = 50 %) + uint32_t scan_duration_{300000}; + bool scan_continuous_{true}; + bool pending_start_{false}; // start_scan() latched before the controller's setup() + bool scan_continuous_configured_{true}; // YAML value; stop_scan() must not lose it +#ifdef USE_OTA_STATE_LISTENER + bool scan_continuous_before_ota_{false}; // continuous mode saved at OTA start, restored on OTA failure + bool scan_running_before_ota_{false}; // one-shot scan running at OTA start, restarted on OTA failure +#endif + uint32_t scan_start_time_{0}; + + // Pending scannable advertisements awaiting their scan response (active scan). + // 62 bytes = legacy adv (31) + scan response (31), the same merged maximum as + // ESP-IDF delivers on ESP32. Main-task only. + struct PendingAdv { + bool used{false}; + uint8_t mac[6]; + uint8_t addr_type; + int8_t rssi; + uint8_t data_len; // <= sizeof(data) + uint8_t data[62]; + uint32_t stored_ms; + }; + // Sized for the unanswered case: a pair that IS answered normally matches + // within one queue drain, so a slot is held for the full timeout only by + // scannable devices that never reply. 8 concurrent such advertisers before + // the merge degrades (frames still delivered, just unmerged) at ~80 B each. + static constexpr size_t MAX_PENDING_ADV = 8; + // On air a scan response follows its advertisement by T_IFS (150 µs) — the + // timeout only covers HOST-side report queuing in rw_task under WiFi/BLE + // coexistence, measured on-device at up to ~136 ms. 300 ms = >2x that margin, + // while staying below any device's re-advertising period. + static constexpr uint32_t PENDING_ADV_TIMEOUT_MS = 300; + PendingAdv pending_adv_[MAX_PENDING_ADV]; + // Occupied pending_adv_ slots — lets loop()'s timeout sweep skip the table + // in the common case (empty: passive scan, or every pair already matched). + uint8_t pending_count_{0}; + + uint32_t scan_period_start_{0}; // millis() at start of current scan period; used to rate-limit on_scan_end() + + ble_device_base::RawAdvertisementCallback raw_advertisement_callback_{}; +#ifdef ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT + // Parsed-advertisement consumers registered through ble_device_base. + // Codegen-sized: no heap allocation, no std::vector template instantiations. + StaticVector listeners_; + // Per-period "Found device" DEBUG log with MAC dedup — shared implementation + // in ble_device_base, identical output on every tracker backend. Guarded like + // its only writer so a no-listener build does not carry an unused vector. + ble_device_base::DiscoveredDeviceLog discovered_log_{}; +#endif +}; + +} // namespace esphome::ln882h_ble_tracker + +#endif // USE_LIBRETINY diff --git a/esphome/components/ln882x/__init__.py b/esphome/components/ln882x/__init__.py index 9c91827522..6da5a4969c 100644 --- a/esphome/components/ln882x/__init__.py +++ b/esphome/components/ln882x/__init__.py @@ -51,7 +51,11 @@ def _set_core_data(config): return config -CONFIG_SCHEMA = libretiny.BASE_SCHEMA +# extend({}) makes this platform's own schema instance: BASE_SCHEMA is shared +# by every LibreTiny platform, and prepending this platform's _set_core_data +# onto the shared object would run it for every platform's validation once two +# platform modules are imported in one process (device-builder, tests). +CONFIG_SCHEMA = libretiny.BASE_SCHEMA.extend({}) PIN_SCHEMA = libretiny.gpio.BASE_PIN_SCHEMA diff --git a/esphome/components/logger/__init__.py b/esphome/components/logger/__init__.py index 77a875dd8f..f307f5d5d1 100644 --- a/esphome/components/logger/__init__.py +++ b/esphome/components/logger/__init__.py @@ -410,10 +410,16 @@ async def _late_logger_init(config: ConfigType) -> None: from esphome.components.esp8266.const import enable_serial, enable_serial1 hw_uart = config.get(CONF_HARDWARE_UART, UART0) - if has_serial_logging and hw_uart in (UART0, UART0_SWAP): + if not has_serial_logging: + # No serial logging: stub out ROM ets_putc so stray output (newlib + # stdout, lwIP diagnostics) cannot block on a slow or shared UART0. + # ets_putc always writes to the physical UART and cannot be disabled + # through uart_set_debug(); see __wrap_ets_putc in logger_esp8266.cpp. + cg.add_build_flag("-Wl,--wrap=ets_putc") + elif hw_uart in (UART0, UART0_SWAP): cg.add_define("USE_ESP8266_LOGGER_SERIAL") enable_serial() - elif has_serial_logging and hw_uart == UART1: + elif hw_uart == UART1: cg.add_define("USE_ESP8266_LOGGER_SERIAL1") enable_serial1() diff --git a/esphome/components/logger/logger_esp8266.cpp b/esphome/components/logger/logger_esp8266.cpp index 5797b03ba7..ac71ba8e3b 100644 --- a/esphome/components/logger/logger_esp8266.cpp +++ b/esphome/components/logger/logger_esp8266.cpp @@ -49,4 +49,17 @@ const LogString *Logger::get_uart_selection_() { } } // namespace esphome::logger + +#if !defined(USE_ESP8266_LOGGER_SERIAL) && !defined(USE_ESP8266_LOGGER_SERIAL1) +// With serial logging disabled, ROM ets_putc still writes to the physical UART0 +// at whatever baud rate a uart bus configured there; uart_set_debug(UART_NO) +// only silences the installable putc1 hook, not ets_putc itself. Blocking +// writes at a low baud rate (for example 4800 for a power monitoring chip) can +// starve the soft watchdog. All linked callers (newlib stdout, lwIP +// diagnostics, postmortem dumps) are redirected here by -Wl,--wrap=ets_putc. +// IRAM_ATTR because the ROM original is callable with the flash cache +// disabled (for example from newlib's _write_r, which is placed in IRAM). +extern "C" void IRAM_ATTR __wrap_ets_putc(char) {} +#endif + #endif diff --git a/esphome/components/lvgl/animation.h b/esphome/components/lvgl/animation.h index 1e0abce358..26bb433f87 100644 --- a/esphome/components/lvgl/animation.h +++ b/esphome/components/lvgl/animation.h @@ -21,12 +21,22 @@ class LvAnimationTiming { class LvAnimationTimingRoundTrip : public LvAnimationTiming { public: + // moving_length_ is the fraction of progress spent moving in each direction, in (0, 0.5]. + // Callers must pass pause in [0, 1) -- pause == 1.0 would make moving_length_ zero and divide by zero below. + LvAnimationTimingRoundTrip(float pause) : moving_length_((1.0f - pause) / 2.0f) {} float map_progress(float value) override { - value *= 2.0f; - if (value > 1.0f) - return 2.0f - value; - return value; + if (value < this->moving_length_) { + return value / this->moving_length_; + } + if (value > 1.0f - this->moving_length_) { + return (1.0f - value) / this->moving_length_; + } + // pause in the middle + return 1.0f; } + + protected: + float moving_length_{}; }; class LvAnimationTimingGravity : public LvAnimationTiming { diff --git a/esphome/components/lvgl/animation.py b/esphome/components/lvgl/animation.py index 2b1500f2c4..95d45de5ea 100644 --- a/esphome/components/lvgl/animation.py +++ b/esphome/components/lvgl/animation.py @@ -42,6 +42,7 @@ LvAnimationTimingRoundTrip = lvgl_ns.class_("LvAnimationTimingRoundTrip") LvAnimationTimingEaseInOut = lvgl_ns.class_("LvAnimationTimingEaseInOut") CONF_BOUNCE = "bounce" +CONF_PAUSE = "pause" def timing_class(name, extras=None): @@ -60,10 +61,20 @@ TIMING_SCHEMA = cv.maybe_simple_value( cv.typed_schema( dict( [ - timing_class("round_trip"), + timing_class( + "round_trip", + { + cv.Optional(CONF_PAUSE, default=0.0): cv.All( + cv.percentage, + cv.float_range( + min=0.0, max=1.0, min_included=True, max_included=False + ), + ) + }, + ), timing_class( "ease_in_out", - {cv.Optional(CONF_WEIGHT, default=2.0): lv_positive_float}, + {cv.Optional(CONF_WEIGHT, default=1.0): cv.zero_to_one_float}, ), timing_class( "gravity", diff --git a/esphome/components/lvgl/automation.py b/esphome/components/lvgl/automation.py index b7c90a5c51..cad065adee 100644 --- a/esphome/components/lvgl/automation.py +++ b/esphome/components/lvgl/automation.py @@ -5,7 +5,14 @@ from esphome import automation from esphome.automation import StatelessLambdaAction import esphome.codegen as cg import esphome.config_validation as cv -from esphome.const import CONF_ACTION, CONF_GROUP, CONF_ID, CONF_ROTATION, CONF_TIMEOUT +from esphome.const import ( + CONF_ACTION, + CONF_GROUP, + CONF_ID, + CONF_POSITION, + CONF_ROTATION, + CONF_TIMEOUT, +) from esphome.core import Lambda from esphome.cpp_generator import TemplateArguments, get_variable from esphome.cpp_types import nullptr @@ -28,6 +35,7 @@ from .defines import ( get_focused_widgets, get_options, get_refreshed_widgets, + literal, ) from .layout import layout_validator from .lv_validation import lv_bool, lv_milliseconds, lv_rotation @@ -36,6 +44,7 @@ from .lvcode import ( UPDATE_EVENT, LambdaContext, LocalVariable, + LvConditional, LvglComponent, ReturnStatement, add_line_marks, @@ -376,6 +385,48 @@ async def obj_show_to_code(config, action_id, template_arg, args): return await action_to_code(widgets, do_show, action_id, template_arg, args) +SET_Z_INDEX_SCHEMA = cv.Schema( + { + cv.Required(CONF_ID): cv.ensure_list( + cv.maybe_simple_value( + {cv.Required(CONF_ID): cv.use_id(lv_obj_t)}, + key=CONF_ID, + ) + ), + cv.Required(CONF_POSITION): cv.Any( + cv.one_of("TOP", "BOTTOM", "UP", "DOWN", upper=True), cv.int_ + ), + } +) + + +@automation.register_action( + "lvgl.widget.set_z_index", ObjUpdateAction, SET_Z_INDEX_SCHEMA, synchronous=True +) +async def obj_set_z_index_to_code(config, action_id, template_arg, args): + position = config[CONF_POSITION] + + async def do_set_z_index(widget: Widget): + if position == "TOP": + lv_obj.move_foreground(widget.obj) + elif position == "BOTTOM": + lv_obj.move_background(widget.obj) + elif position == "UP": + lv_obj.move_to_index( + widget.obj, literal(f"{lv_expr.obj_get_index(widget.obj)} + 1") + ) + elif position == "DOWN": + with LvConditional(f"{lv_expr.obj_get_index(widget.obj)} > 0"): + lv_obj.move_to_index( + widget.obj, literal(f"{lv_expr.obj_get_index(widget.obj)} - 1") + ) + else: + lv_obj.move_to_index(widget.obj, position) + + widgets = [widget.outer or widget for widget in await get_widgets(config[CONF_ID])] + return await action_to_code(widgets, do_set_z_index, action_id, template_arg, args) + + def focused_id(value): value = cv.use_id(lv_pseudo_button_t)(value) get_focused_widgets().add(value) diff --git a/esphome/components/mcp4461/mcp4461.cpp b/esphome/components/mcp4461/mcp4461.cpp index e83a6847d6..abc74b9e6d 100644 --- a/esphome/components/mcp4461/mcp4461.cpp +++ b/esphome/components/mcp4461/mcp4461.cpp @@ -21,7 +21,18 @@ void Mcp4461Component::setup() { auto init_val = this->reg_[i].initial_value; if (init_val.has_value()) { uint16_t initial_state = static_cast(*init_val * 256.0f); - this->write_wiper_level_(i, initial_state); + if (i > 3) { + // NV wiper: an unconditional write would cost one EEPROM erase/write cycle on EVERY + // boot. Only write when the stored value actually differs — and always write when + // the read itself failed (a failed read returns 0, which would silently skip the + // write whenever initial_value is 0). + bool read_ok = false; + if (this->read_wiper_level_(i, &read_ok) != initial_state || !read_ok) { + this->write_wiper_level_(i, initial_state); + } + } else { + this->write_wiper_level_(i, initial_state); + } } if (this->reg_[i].enabled) { this->reg_[i].state = this->read_wiper_level_(i); @@ -34,6 +45,23 @@ void Mcp4461Component::setup() { } } } + // Push the YAML terminal configuration to the TCON registers. TCON is volatile — on POR + // the chip restores wiper levels from the NV registers but resets TCON to "all terminals + // connected", so any terminal_a/b/w disables from the config MUST be written here. + for (uint8_t t = 0; t < 2; t++) { + Mcp4461TerminalIdx terminal_connector = static_cast(t); + uint8_t terminal_byte = this->calc_terminal_connector_byte_(terminal_connector); + this->set_terminal_register_(terminal_connector, terminal_byte); + } +} + +void Mcp4461Component::set_nonvolatile(Mcp4461WiperIdx wiper, uint32_t write_delay_ms) { + uint8_t wiper_idx = static_cast(wiper); + if (wiper_idx > 3) { + return; // NV channels E-H are the persistence target themselves + } + this->reg_[wiper_idx].nonvolatile = true; + this->reg_[wiper_idx].nonvolatile_write_delay_ms = write_delay_ms; } void Mcp4461Component::set_initial_value(Mcp4461WiperIdx wiper, float initial_value) { @@ -77,9 +105,12 @@ void Mcp4461Component::dump_config() { // so also invalid for nonvolatile. For these, only print current level. // reworked to be a one-line intentionally, as output would not be in order if (i < 4) { - ESP_LOGCONFIG(TAG, " ├── Volatile wiper [%u] level: %u, Status: %s, HW: %s, A: %s, B: %s, W: %s", i, - this->reg_[i].state, ONOFF(this->reg_[i].enabled), ONOFF(this->reg_[i].terminal_hw), - ONOFF(this->reg_[i].terminal_a), ONOFF(this->reg_[i].terminal_b), ONOFF(this->reg_[i].terminal_w)); + ESP_LOGCONFIG(TAG, + " ├── Volatile wiper [%u] level: %u, Status: %s, HW: %s, " + "A: %s, B: %s, W: %s, NV: %s", + i, this->reg_[i].state, ONOFF(this->reg_[i].enabled), ONOFF(this->reg_[i].terminal_hw), + ONOFF(this->reg_[i].terminal_a), ONOFF(this->reg_[i].terminal_b), ONOFF(this->reg_[i].terminal_w), + ONOFF(this->reg_[i].nonvolatile)); } else { ESP_LOGCONFIG(TAG, " ├── Nonvolatile wiper [%u] level: %u", i, this->reg_[i].state); } @@ -92,8 +123,10 @@ void Mcp4461Component::loop() { } for (uint8_t i = 0; i < 8; i++) { if (this->reg_[i].update_level) { - // set wiper i state if changed - if (this->reg_[i].state != this->read_wiper_level_(i)) { + // set wiper i state if changed — a failed read (returns 0) must not suppress the + // write when the target state is 0, same hardening as the NV read-compare paths + bool read_ok = false; + if (this->reg_[i].state != this->read_wiper_level_(i, &read_ok) || !read_ok) { this->write_wiper_level_(i, this->reg_[i].state); } } @@ -112,6 +145,67 @@ void Mcp4461Component::loop() { } this->reg_[i].update_terminal = false; } + this->process_nonvolatile_dirty_(); +} + +void Mcp4461Component::process_nonvolatile_dirty_() { + const uint32_t now = millis(); + for (uint8_t i = 0; i < 4; i++) { + if (!this->reg_[i].nonvolatile || !this->reg_[i].nonvolatile_dirty) { + continue; + } + if ((now - this->reg_[i].last_level_change_ms) < this->reg_[i].nonvolatile_write_delay_ms) { + continue; // still settling — debounce window not over yet + } + // Never block the loop on a still-running EEPROM cycle (t_WC up to 10 ms); datasheet: + // during an EEPROM write only volatile commands are accepted. Retry on the next loop. + if (this->is_writing_()) { + continue; + } + // Clear the dirty flag on success — and equally when WP or WiperLock block the write + // permanently, instead of retrying forever. + if (this->store_level_nonvolatile_(static_cast(i)) || this->write_protected_ || + this->reg_[i].wiper_lock_active) { + this->reg_[i].nonvolatile_dirty = false; + } else { + // Transient failure (e.g. I2C error): without this, the retry fires on every single + // loop() iteration, spamming a warning each time. Re-arming the timestamp reuses the + // stability delay as a natural retry backoff. + this->reg_[i].last_level_change_ms = now; + } + } +} + +bool Mcp4461Component::store_level_nonvolatile_(Mcp4461WiperIdx wiper) { + if (this->is_failed()) { + ESP_LOGE(TAG, "%s", LOG_STR_ARG(this->get_message_string(this->error_code_))); + return false; + } + uint8_t wiper_idx = static_cast(wiper); + if (wiper_idx > 3) { + // E-H ARE the nonvolatile registers — keep this consistent with the other guards + // instead of failing silently (reachable via the store_nonvolatile action). + ESP_LOGW(TAG, "%s", LOG_STR_ARG(this->get_message_string(MCP4461_PROHIBITED_FOR_NONVOLATILE))); + return false; + } + if (this->reg_[wiper_idx].wiper_lock_active) { + ESP_LOGW(TAG, "%s", LOG_STR_ARG(this->get_message_string(MCP4461_WIPER_LOCKED))); + return false; + } + const uint16_t level = this->reg_[wiper_idx].state; + // Skip the EEPROM cycle entirely when the NV register already holds the value. A failed + // read must NOT count as a match (it returns 0): fall through to the write instead — if + // the bus is really down, the write fails too and the dirty flag stays set for a retry. + bool read_ok = false; + if (this->read_wiper_level_(wiper_idx + 4, &read_ok) == level && read_ok) { + return true; + } + ESP_LOGV(TAG, "Persisting wiper %u level %u to nonvolatile register", wiper_idx, level); + if (!this->mcp4461_write_(this->get_wiper_address_(wiper_idx + 4), level, true)) { + ESP_LOGW(TAG, "Error persisting wiper %u level %u", wiper_idx, level); + return false; + } + return true; } uint8_t Mcp4461Component::get_status_register_() { @@ -210,7 +304,10 @@ uint16_t Mcp4461Component::get_wiper_level_(Mcp4461WiperIdx wiper) { return this->read_wiper_level_(wiper_idx); } -uint16_t Mcp4461Component::read_wiper_level_(uint8_t wiper_idx) { +uint16_t Mcp4461Component::read_wiper_level_(uint8_t wiper_idx, bool *ok) { + if (ok != nullptr) { + *ok = false; + } uint8_t addr = this->get_wiper_address_(wiper_idx); uint8_t reg = addr | static_cast(Mcp4461Commands::READ); if (wiper_idx > 3) { @@ -225,6 +322,9 @@ uint16_t Mcp4461Component::read_wiper_level_(uint8_t wiper_idx) { ESP_LOGW(TAG, "Error fetching %swiper %u value", (wiper_idx > 3) ? "nonvolatile " : "", wiper_idx); return 0; } + if (ok != nullptr) { + *ok = true; + } return buf; } @@ -265,6 +365,10 @@ bool Mcp4461Component::set_wiper_level_(Mcp4461WiperIdx wiper, uint16_t value) { ESP_LOGV(TAG, "Setting MCP4461 wiper %u to %u", wiper_idx, value); this->reg_[wiper_idx].state = value; this->reg_[wiper_idx].update_level = true; + if (this->reg_[wiper_idx].nonvolatile) { + this->reg_[wiper_idx].nonvolatile_dirty = true; + this->reg_[wiper_idx].last_level_change_ms = millis(); + } return true; } @@ -335,6 +439,12 @@ bool Mcp4461Component::increase_wiper_(Mcp4461WiperIdx wiper) { ESP_LOGW(TAG, "%s", LOG_STR_ARG(this->get_message_string(MCP4461_WIPER_LOCKED))); return false; } + if (wiper_idx > 3) { + // Datasheet: increment commands are only valid for the volatile wiper registers — + // the chip NACKs them on nonvolatile addresses. + ESP_LOGW(TAG, "%s", LOG_STR_ARG(this->get_message_string(MCP4461_PROHIBITED_FOR_NONVOLATILE))); + return false; + } if (this->reg_[wiper_idx].state == 256) { ESP_LOGV(TAG, "Maximum wiper level reached, further increase of wiper %u prohibited", wiper_idx); return false; @@ -349,6 +459,10 @@ bool Mcp4461Component::increase_wiper_(Mcp4461WiperIdx wiper) { return false; } this->reg_[wiper_idx].state++; + if (this->reg_[wiper_idx].nonvolatile) { + this->reg_[wiper_idx].nonvolatile_dirty = true; + this->reg_[wiper_idx].last_level_change_ms = millis(); + } return true; } @@ -366,6 +480,12 @@ bool Mcp4461Component::decrease_wiper_(Mcp4461WiperIdx wiper) { ESP_LOGW(TAG, "%s", LOG_STR_ARG(this->get_message_string(MCP4461_WIPER_LOCKED))); return false; } + if (wiper_idx > 3) { + // Datasheet: decrement commands are only valid for the volatile wiper registers — + // the chip NACKs them on nonvolatile addresses. + ESP_LOGW(TAG, "%s", LOG_STR_ARG(this->get_message_string(MCP4461_PROHIBITED_FOR_NONVOLATILE))); + return false; + } if (this->reg_[wiper_idx].state == 0) { ESP_LOGV(TAG, "Minimum wiper level reached, further decrease of wiper %u prohibited", wiper_idx); return false; @@ -380,11 +500,18 @@ bool Mcp4461Component::decrease_wiper_(Mcp4461WiperIdx wiper) { return false; } this->reg_[wiper_idx].state--; + if (this->reg_[wiper_idx].nonvolatile) { + this->reg_[wiper_idx].nonvolatile_dirty = true; + this->reg_[wiper_idx].last_level_change_ms = millis(); + } return true; } uint8_t Mcp4461Component::calc_terminal_connector_byte_(Mcp4461TerminalIdx terminal_connector) { - uint8_t i = static_cast(terminal_connector) <= 1 ? 0 : 2; + // TCON0 covers wipers 0/1 (A/B), TCON1 covers wipers 2/3 (C/D). The enum only holds + // 0 and 1, so the old `<= 1 ? 0 : 2` collapsed to always-0 and built TCON1 from + // channels A/B's flags — mirror the (correct) read path in update_terminal_register_(). + uint8_t i = static_cast(terminal_connector) == 0 ? 0 : 2; uint8_t new_value_byte = 0; new_value_byte += static_cast(this->reg_[i].terminal_b); new_value_byte += static_cast(this->reg_[i].terminal_w) << 1; @@ -471,6 +598,12 @@ void Mcp4461Component::enable_terminal_(Mcp4461WiperIdx wiper, char terminal) { return; } uint8_t wiper_idx = static_cast(wiper); + if (wiper_idx > 3) { + // Terminal control only exists for the volatile wipers; loop() would otherwise emit + // an unrelated TCON write and silently drop the request. + ESP_LOGW(TAG, "%s", LOG_STR_ARG(this->get_message_string(MCP4461_PROHIBITED_FOR_NONVOLATILE))); + return; + } ESP_LOGV(TAG, "Enabling terminal %c of wiper %u", terminal, wiper_idx); switch (terminal) { case 'h': @@ -498,6 +631,10 @@ void Mcp4461Component::disable_terminal_(Mcp4461WiperIdx wiper, char terminal) { return; } uint8_t wiper_idx = static_cast(wiper); + if (wiper_idx > 3) { + ESP_LOGW(TAG, "%s", LOG_STR_ARG(this->get_message_string(MCP4461_PROHIBITED_FOR_NONVOLATILE))); + return; + } ESP_LOGV(TAG, "Disabling terminal %c of wiper %u", terminal, wiper_idx); switch (terminal) { case 'h': diff --git a/esphome/components/mcp4461/mcp4461.h b/esphome/components/mcp4461/mcp4461.h index a577a4b482..933d92c1fa 100644 --- a/esphome/components/mcp4461/mcp4461.h +++ b/esphome/components/mcp4461/mcp4461.h @@ -17,6 +17,16 @@ struct WiperState { bool wiper_lock_active = false; bool update_level = false; bool update_terminal = false; + // Nonvolatile persistence (volatile wipers 0-3 only): when enabled, every level change is + // mirrored into the chip's NV wiper register after nonvolatile_write_delay of stability, so + // the chip restores it on power-on. The delay both debounces bursts (e.g. light transitions + // writing dozens of levels per second) and protects the EEPROM's limited endurance — + // without it, every intermediate step would cost one of the ~1M erase/write cycles and + // stall the bus for up to t_WC (10 ms) each. + bool nonvolatile = false; + uint32_t nonvolatile_write_delay_ms = 1000; + bool nonvolatile_dirty = false; + uint32_t last_level_change_ms = 0; }; // default wiper state is 128 / 0x80h @@ -86,6 +96,11 @@ class Mcp4461Component final : public Component, public i2c::I2CDevice { /// @param[in] wiper - the wiper to set the value for /// @param[in] initial_value - the initial value in range 0-1.0 as float void set_initial_value(Mcp4461WiperIdx wiper, float initial_value); + /// @brief enable nonvolatile persistence for a volatile wiper (0-3): every level change is + /// mirrored to the corresponding NV wiper register after the given stability delay + /// @param[in] wiper - the (volatile) wiper to persist + /// @param[in] write_delay_ms - stability delay before the NV write (debounce / EEPROM wear) + void set_nonvolatile(Mcp4461WiperIdx wiper, uint32_t write_delay_ms); /// @brief public function used to set disable terminal config /// @param[in] wiper - the wiper to set the value for /// @param[in] terminal - the terminal to disable, one of ['a','b','w','h'] @@ -98,7 +113,10 @@ class Mcp4461Component final : public Component, public i2c::I2CDevice { bool read_16_(uint8_t address, uint16_t *buf); void update_write_protection_status_(); uint8_t get_wiper_address_(uint8_t wiper); - uint16_t read_wiper_level_(uint8_t wiper); + /// Read a wiper register. On I2C failure returns 0 — callers that must distinguish + /// a real 0 from a failed read pass `ok` (added for the NV read-compare paths, where + /// acting on a failed read would skip a required write or drop a pending persist). + uint16_t read_wiper_level_(uint8_t wiper, bool *ok = nullptr); uint8_t get_status_register_(); uint16_t get_wiper_level_(Mcp4461WiperIdx wiper); bool set_wiper_level_(Mcp4461WiperIdx wiper, uint16_t value); @@ -110,6 +128,11 @@ class Mcp4461Component final : public Component, public i2c::I2CDevice { void enable_terminal_(Mcp4461WiperIdx wiper, char terminal); void disable_terminal_(Mcp4461WiperIdx, char terminal); bool is_writing_(); + /// Copy the current volatile level of wiper 0-3 into its NV register (immediate, blocking + /// only for a pending previous EEPROM cycle). Returns false while WP is active or on error. + bool store_level_nonvolatile_(Mcp4461WiperIdx wiper); + /// Deferred NV mirroring driven from loop() — see WiperState::nonvolatile. + void process_nonvolatile_dirty_(); bool is_eeprom_ready_for_writing_(bool wait_if_not_ready); void write_wiper_level_(uint8_t wiper, uint16_t value); bool mcp4461_write_(uint8_t addr, uint16_t data, bool nonvolatile = false); @@ -139,6 +162,9 @@ class Mcp4461Component final : public Component, public i2c::I2CDevice { return LOG_STR("MCP4461 Wiper is locked using WiperLock-technology. All actions on this wiper are prohibited."); case MCP4461_STATUS_OK: return LOG_STR("Status OK"); + case MCP4461_PROHIBITED_FOR_NONVOLATILE: + return LOG_STR( + "Increment/decrement, store, and terminal control are prohibited on the nonvolatile wipers (E-H)."); default: return LOG_STR("Unknown"); } diff --git a/esphome/components/mcp4461/output/__init__.py b/esphome/components/mcp4461/output/__init__.py index 0d145d81d3..1642f6149a 100644 --- a/esphome/components/mcp4461/output/__init__.py +++ b/esphome/components/mcp4461/output/__init__.py @@ -1,3 +1,4 @@ +from esphome import automation import esphome.codegen as cg from esphome.components import output import esphome.config_validation as cv @@ -26,6 +27,43 @@ CHANNEL_OPTIONS = { CONF_TERMINAL_A = "terminal_a" CONF_TERMINAL_B = "terminal_b" CONF_TERMINAL_W = "terminal_w" +CONF_NONVOLATILE = "nonvolatile" +CONF_NONVOLATILE_WRITE_DELAY = "nonvolatile_write_delay" + +# Volatile wiper channels that have a nonvolatile shadow register on the chip +VOLATILE_CHANNELS = ("A", "B", "C", "D") + + +def _validate_nonvolatile(config): + channel = str(config[CONF_CHANNEL]) + + # Channels E-H address the nonvolatile registers directly — the mirroring options only + # make sense for the volatile channels A-D. + if channel not in VOLATILE_CHANNELS: + # Only reject what the user EXPLICITLY asked for and cannot have: enabling the + # mirroring or tuning its delay on E-H. An explicit `nonvolatile: false` is a + # harmless no-op and stays valid; bare configs (no key at all) must keep working. + # NOTE: FINAL_VALIDATE_SCHEMA intentionally mutates `config` in-place (uses setdefault) to apply defaults for callers. + if config.get(CONF_NONVOLATILE) or CONF_NONVOLATILE_WRITE_DELAY in config: + raise cv.Invalid( + f"enabling '{CONF_NONVOLATILE}' or setting '{CONF_NONVOLATILE_WRITE_DELAY}' is only valid for the " + f"volatile channels A-D; channels E-H are the nonvolatile registers themselves" + ) + return config + + config.setdefault(CONF_NONVOLATILE, True) + if config[CONF_NONVOLATILE]: + config.setdefault( + CONF_NONVOLATILE_WRITE_DELAY, + cv.positive_time_period_milliseconds("1s"), + ) + elif CONF_NONVOLATILE_WRITE_DELAY in config: + # Same consistency as the E-H rejection above: never silently ignore user input. + raise cv.Invalid( + f"'{CONF_NONVOLATILE_WRITE_DELAY}' requires '{CONF_NONVOLATILE}: true'" + ) + return config + CONFIG_SCHEMA = output.FLOAT_OUTPUT_SCHEMA.extend( { @@ -36,9 +74,21 @@ CONFIG_SCHEMA = output.FLOAT_OUTPUT_SCHEMA.extend( cv.Optional(CONF_TERMINAL_B, default=True): cv.boolean, cv.Optional(CONF_TERMINAL_W, default=True): cv.boolean, cv.Optional(CONF_INITIAL_VALUE): cv.float_range(min=0.0, max=1.0), + # No schema defaults here: a default would materialize the keys on EVERY channel, + # making existing bare E-H configs fail final validation. The effective defaults + # (nonvolatile: true, delay 1s) are applied for the volatile channels A-D inside + # _validate_nonvolatile instead. Default-on rationale: the chip restores the + # nonvolatile wiper levels at power-on, so persisting every settled level change is + # the least surprising behavior — the pot simply comes back where it was. The write + # is deferred by nonvolatile_write_delay to debounce transitions and protect the + # EEPROM's endurance. + cv.Optional(CONF_NONVOLATILE): cv.boolean, + cv.Optional(CONF_NONVOLATILE_WRITE_DELAY): cv.positive_time_period_milliseconds, } ) +FINAL_VALIDATE_SCHEMA = _validate_nonvolatile + async def to_code(config): parent = await cg.get_variable(config[CONF_MCP4461_ID]) @@ -57,5 +107,71 @@ async def to_code(config): cg.add( parent.set_initial_value(config[CONF_CHANNEL], config[CONF_INITIAL_VALUE]) ) + if str(config[CONF_CHANNEL]) in VOLATILE_CHANNELS and config[CONF_NONVOLATILE]: + cg.add( + parent.set_nonvolatile( + config[CONF_CHANNEL], + config[CONF_NONVOLATILE_WRITE_DELAY], + ) + ) await output.register_output(var, config) await cg.register_parented(var, config[CONF_MCP4461_ID]) + + +# ---- Actions ---- +WiperIncreaseAction = mcp4461_ns.class_("WiperIncreaseAction", automation.Action) +WiperDecreaseAction = mcp4461_ns.class_("WiperDecreaseAction", automation.Action) +WiperStoreNonvolatileAction = mcp4461_ns.class_( + "WiperStoreNonvolatileAction", automation.Action +) +WiperSetTerminalAction = mcp4461_ns.class_("WiperSetTerminalAction", automation.Action) + +WIPER_ACTION_SCHEMA = automation.maybe_simple_id( + {cv.Required(CONF_ID): cv.use_id(Mcp4461Wiper)} +) + +CONF_TERMINAL = "terminal" +CONF_ENABLE = "enable" + +TERMINAL_ACTION_SCHEMA = cv.Schema( + { + cv.Required(CONF_ID): cv.use_id(Mcp4461Wiper), + cv.Required(CONF_TERMINAL): cv.one_of("a", "b", "w", "h", lower=True), + cv.Required(CONF_ENABLE): cv.boolean, + } +) + + +@automation.register_action( + "mcp4461.wiper.increase", WiperIncreaseAction, WIPER_ACTION_SCHEMA, synchronous=True +) +@automation.register_action( + "mcp4461.wiper.decrease", WiperDecreaseAction, WIPER_ACTION_SCHEMA, synchronous=True +) +async def mcp4461_wiper_step_to_code(config, action_id, template_arg, args): + wiper = await cg.get_variable(config[CONF_ID]) + return cg.new_Pvariable(action_id, template_arg, wiper) + + +@automation.register_action( + "mcp4461.wiper.store_nonvolatile", + WiperStoreNonvolatileAction, + WIPER_ACTION_SCHEMA, + synchronous=True, +) +async def mcp4461_wiper_store_to_code(config, action_id, template_arg, args): + wiper = await cg.get_variable(config[CONF_ID]) + return cg.new_Pvariable(action_id, template_arg, wiper) + + +@automation.register_action( + "mcp4461.wiper.set_terminal", + WiperSetTerminalAction, + TERMINAL_ACTION_SCHEMA, + synchronous=True, +) +async def mcp4461_wiper_terminal_to_code(config, action_id, template_arg, args): + wiper = await cg.get_variable(config[CONF_ID]) + return cg.new_Pvariable( + action_id, template_arg, wiper, ord(config[CONF_TERMINAL]), config[CONF_ENABLE] + ) diff --git a/esphome/components/mcp4461/output/automation.h b/esphome/components/mcp4461/output/automation.h new file mode 100644 index 0000000000..4be317b2f8 --- /dev/null +++ b/esphome/components/mcp4461/output/automation.h @@ -0,0 +1,56 @@ +#pragma once + +#include "esphome/core/automation.h" +#include "mcp4461_output.h" + +namespace esphome::mcp4461 { + +template class WiperIncreaseAction : public Action { + public: + explicit WiperIncreaseAction(Mcp4461Wiper *wiper) : wiper_(wiper) {} + void play(Ts... x) override { this->wiper_->increase_wiper(); } + + protected: + Mcp4461Wiper *wiper_; +}; + +template class WiperDecreaseAction : public Action { + public: + explicit WiperDecreaseAction(Mcp4461Wiper *wiper) : wiper_(wiper) {} + void play(Ts... x) override { this->wiper_->decrease_wiper(); } + + protected: + Mcp4461Wiper *wiper_; +}; + +// Persist the current level to the chip's nonvolatile register immediately — useful with +// nonvolatile: false to persist only at deliberate moments (e.g. on a button press), or to +// bypass the stability delay of the automatic mirroring. +template class WiperStoreNonvolatileAction : public Action { + public: + explicit WiperStoreNonvolatileAction(Mcp4461Wiper *wiper) : wiper_(wiper) {} + void play(Ts... x) override { this->wiper_->store_nonvolatile(); } + + protected: + Mcp4461Wiper *wiper_; +}; + +template class WiperSetTerminalAction : public Action { + public: + WiperSetTerminalAction(Mcp4461Wiper *wiper, char terminal, bool enable) + : wiper_(wiper), terminal_(terminal), enable_(enable) {} + void play(Ts... x) override { + if (this->enable_) { + this->wiper_->enable_terminal(this->terminal_); + } else { + this->wiper_->disable_terminal(this->terminal_); + } + } + + protected: + Mcp4461Wiper *wiper_; + char terminal_; + bool enable_; +}; + +} // namespace esphome::mcp4461 diff --git a/esphome/components/mcp4461/output/mcp4461_output.cpp b/esphome/components/mcp4461/output/mcp4461_output.cpp index 3892372cab..5c373ddc7d 100644 --- a/esphome/components/mcp4461/output/mcp4461_output.cpp +++ b/esphome/components/mcp4461/output/mcp4461_output.cpp @@ -66,6 +66,12 @@ void Mcp4461Wiper::decrease_wiper() { } } +void Mcp4461Wiper::store_nonvolatile() { + if (this->parent_->store_level_nonvolatile_(this->wiper_)) { + ESP_LOGV(TAG, "Stored wiper %u level to nonvolatile register", static_cast(this->wiper_)); + } +} + void Mcp4461Wiper::enable_terminal(char terminal) { this->parent_->enable_terminal_(this->wiper_, terminal); } void Mcp4461Wiper::disable_terminal(char terminal) { this->parent_->disable_terminal_(this->wiper_, terminal); } diff --git a/esphome/components/mcp4461/output/mcp4461_output.h b/esphome/components/mcp4461/output/mcp4461_output.h index 20d81d825a..c8d1ef1ec5 100644 --- a/esphome/components/mcp4461/output/mcp4461_output.h +++ b/esphome/components/mcp4461/output/mcp4461_output.h @@ -36,6 +36,9 @@ class Mcp4461Wiper final : public output::FloatOutput, public Parented + #ifdef USE_OTA #include "esphome/components/ota/ota_backend.h" #endif @@ -35,21 +37,34 @@ static const UBaseType_t INFERENCE_TASK_PRIORITY = 3; enum EventGroupBits : uint32_t { COMMAND_STOP = (1 << 0), // Signals the inference task should stop COMMAND_RESET_RING_BUFFER = (1 << 1), // Signals the inference task to discard buffered audio + COMMAND_PAUSE_MODELS = (1 << 2), // Asks the inference task to pause at a safe point so the model lists can be + // mutated from the main loop TASK_STARTING = (1 << 3), TASK_RUNNING = (1 << 4), TASK_STOPPING = (1 << 5), TASK_STOPPED = (1 << 6), + MODELS_PAUSED = (1 << 7), // Inference task acknowledges it is paused and holds no iterators + COMMAND_RESUME_MODELS = (1 << 8), // Main loop signals the inference task it may resume iterating + ERROR_MEMORY = (1 << 9), ERROR_INFERENCE = (1 << 10), WARNING_FULL_RING_BUFFER = (1 << 13), + WARNING_MODELS_RESUME_TIMEOUT = (1 << 14), // The paused inference task gave up waiting to be released ERROR_BITS = ERROR_MEMORY | ERROR_INFERENCE, ALL_BITS = 0xfffff, // 24 total bits available in an event group }; +// How long the main loop waits for the inference task to acknowledge a pause request before giving up. +// The task checks for the command at the top of its loop, which runs at least every DATA_TIMEOUT_MS. +static const uint32_t MODELS_PAUSE_TIMEOUT_MS = 500; +// How long the paused inference task waits to be resumed before rechecking on its own. Only reached if +// the main loop abandoned the handshake (e.g. it timed out first), so recovery just needs to be bounded. +static const uint32_t MODELS_RESUME_TIMEOUT_MS = 1000; + float MicroWakeWord::get_setup_priority() const { return setup_priority::AFTER_CONNECTION; } static const LogString *micro_wake_word_state_to_string(State state) { @@ -176,6 +191,20 @@ void MicroWakeWord::inference_task(void *params) { xEventGroupSetBits(this_mww->event_group_, EventGroupBits::TASK_RUNNING); while (!(xEventGroupGetBits(this_mww->event_group_) & (COMMAND_STOP | ERROR_BITS))) { + if (xEventGroupGetBits(this_mww->event_group_) & EventGroupBits::COMMAND_PAUSE_MODELS) { + // Safe point: no iterators into wake_word_models_ are held here. Acknowledge the pause and wait for the + // main loop to finish mutating the model lists before resuming. + xEventGroupSetBits(this_mww->event_group_, EventGroupBits::MODELS_PAUSED); + EventBits_t resume_bits = xEventGroupWaitBits(this_mww->event_group_, EventGroupBits::COMMAND_RESUME_MODELS, + pdTRUE, pdTRUE, pdMS_TO_TICKS(MODELS_RESUME_TIMEOUT_MS)); + if (!(resume_bits & EventGroupBits::COMMAND_RESUME_MODELS)) { + // Nobody released us, so the main loop abandoned the handshake and did not mutate the lists. + // Rechecking the pause command below is safe, but the wait cost a second of detection, so report it. + xEventGroupSetBits(this_mww->event_group_, EventGroupBits::WARNING_MODELS_RESUME_TIMEOUT); + } + continue; + } + if (xEventGroupGetBits(this_mww->event_group_) & EventGroupBits::COMMAND_RESET_RING_BUFFER) { // Producer asked us to drain; run the consumer-side reset from this thread. audio_source->clear_buffered_data(); @@ -232,6 +261,130 @@ std::vector MicroWakeWord::get_wake_words() { void MicroWakeWord::add_wake_word_model(WakeWordModel *model) { this->wake_word_models_.push_back(model); } +bool MicroWakeWord::try_lock_models_() { + // When the inference task isn't running it holds no iterators into wake_word_models_, so the lists can be + // mutated without a handshake. The main loop is the only caller, so this state cannot change between here + // and the matching unlock_models_() call. + if (!this->inference_task_.is_created() || this->state_ == State::STOPPED) { + return true; + } + + // The task is running and iterates wake_word_models_. Ask it to pause at a safe point before we mutate. + // Clear any stale acknowledgement from an abandoned handshake first. + xEventGroupClearBits(this->event_group_, EventGroupBits::MODELS_PAUSED); + xEventGroupSetBits(this->event_group_, EventGroupBits::COMMAND_PAUSE_MODELS); + + EventBits_t bits = xEventGroupWaitBits(this->event_group_, EventGroupBits::MODELS_PAUSED, pdFALSE, pdTRUE, + pdMS_TO_TICKS(MODELS_PAUSE_TIMEOUT_MS)); + + if (!(bits & EventGroupBits::MODELS_PAUSED)) { + // The task never acknowledged (e.g. it is busy stopping). Withdraw the request and refuse to mutate a + // list it might be iterating. + xEventGroupClearBits(this->event_group_, EventGroupBits::COMMAND_PAUSE_MODELS); + return false; + } + return true; +} + +void MicroWakeWord::unlock_models_() { + if (!this->inference_task_.is_created() || this->state_ == State::STOPPED) { + return; // Nothing was paused + } + xEventGroupClearBits(this->event_group_, EventGroupBits::MODELS_PAUSED | EventGroupBits::COMMAND_PAUSE_MODELS); + xEventGroupSetBits(this->event_group_, EventGroupBits::COMMAND_RESUME_MODELS); +} + +bool MicroWakeWord::add_runtime_model(std::unique_ptr model) { + if (!model) { + ESP_LOGE(TAG, "Cannot add null runtime model"); + return false; + } + + const std::string model_id = model->get_id(); + + // A model without usable data can never load, so keep it out of the lists entirely. Otherwise it would be + // advertised to Home Assistant as selectable and the inference task would silently disable it again every + // time it was enabled. + if (!model->has_model_data()) { + ESP_LOGE(TAG, "Runtime model '%s' has no valid data", model_id.c_str()); + return false; + } + + // Reject a duplicate id against every model (compiled or runtime). The inference task only ever reads + // wake_word_models_, so scanning it here (on the main loop) needs no synchronization. + for (auto *existing : this->wake_word_models_) { + if (existing->get_id() == model_id) { + ESP_LOGW(TAG, "Wake word model '%s' already exists", model_id.c_str()); + return false; + } + } + + if (!this->try_lock_models_()) { + ESP_LOGE(TAG, "Timed out pausing inference task; not adding runtime model '%s'", model_id.c_str()); + return false; + } + + this->wake_word_models_.push_back(model.get()); + this->runtime_models_.push_back(std::move(model)); + + this->unlock_models_(); + ESP_LOGD(TAG, "Added runtime model '%s'", model_id.c_str()); + return true; +} + +bool MicroWakeWord::remove_runtime_model(const std::string &model_id) { + // Only runtime-downloaded models can be removed; compiled-in models never appear in runtime_models_. + auto runtime_it = + std::find_if(this->runtime_models_.begin(), this->runtime_models_.end(), + [&model_id](const std::unique_ptr &m) { return m->get_id() == model_id; }); + if (runtime_it == this->runtime_models_.end()) { + return false; + } + + if (!this->try_lock_models_()) { + ESP_LOGE(TAG, "Timed out pausing inference task; not removing runtime model '%s'", model_id.c_str()); + return false; + } + + WakeWordModel *raw = runtime_it->get(); + auto models_it = std::find(this->wake_word_models_.begin(), this->wake_word_models_.end(), raw); + if (models_it != this->wake_word_models_.end()) { + this->wake_word_models_.erase(models_it); + } + + // Queued detection events hold a pointer into the model being destroyed, so drop them. The inference task + // is parked, so no new events can be queued concurrently. Losing an undelivered detection from another + // model is acceptable for this rare operation. + xQueueReset(this->detection_queue_); + + // Free the interpreter and arenas (safe: the task is parked, not mid-inference), then destroy the model. + // Its ModelData releases the PSRAM model buffer once the last shared_ptr reference drops. + raw->unload_model(); + this->runtime_models_.erase(runtime_it); + + this->unlock_models_(); + ESP_LOGI(TAG, "Removed runtime model '%s'", model_id.c_str()); + return true; +} + +std::vector MicroWakeWord::get_runtime_model_ids() { + std::vector ids; + ids.reserve(this->runtime_models_.size()); + for (const auto &model : this->runtime_models_) { + ids.push_back(model->get_id()); + } + return ids; +} + +WakeWordModel *MicroWakeWord::get_model_by_id(const std::string &model_id) { + for (auto *model : this->wake_word_models_) { + if (model->get_id() == model_id) { + return model; + } + } + return nullptr; +} + #ifdef USE_MICRO_WAKE_WORD_VAD void MicroWakeWord::add_vad_model(const uint8_t *model_start, uint8_t probability_cutoff, size_t sliding_window_size, size_t tensor_arena_size) { @@ -270,6 +423,12 @@ void MicroWakeWord::loop() { "word detection accuracy will temporarily be reduced."); } + if (event_group_bits & EventGroupBits::WARNING_MODELS_RESUME_TIMEOUT) { + xEventGroupClearBits(this->event_group_, EventGroupBits::WARNING_MODELS_RESUME_TIMEOUT); + ESP_LOGW(TAG, "Inference task paused for %" PRIu32 " ms without being released, so it resumed on its own", + MODELS_RESUME_TIMEOUT_MS); + } + if (event_group_bits & EventGroupBits::TASK_STARTING) { ESP_LOGD(TAG, "Inference task has started, attempting to allocate memory for buffers"); xEventGroupClearBits(this->event_group_, EventGroupBits::TASK_STARTING); diff --git a/esphome/components/micro_wake_word/micro_wake_word.h b/esphome/components/micro_wake_word/micro_wake_word.h index aebb5b2595..03f4a86fd4 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.h +++ b/esphome/components/micro_wake_word/micro_wake_word.h @@ -66,6 +66,32 @@ class MicroWakeWord final : public Component void add_wake_word_model(WakeWordModel *model); + /// @brief Adds a runtime-downloaded wake word model. Must be called from the main loop. + /// If the inference task is running it is paused at a safe point before the model lists are mutated, + /// so the task never observes a half-updated vector. + /// Callers should check get_model_by_id() before constructing the model: a WakeWordModel permanently + /// claims a preference backend that is not released when the model is destroyed, so building one only to + /// have it rejected here costs internal RAM that never comes back. + /// @return True if the model was added, false if it has no valid data, on a duplicate id, or if the task + /// could not be paused + bool add_runtime_model(std::unique_ptr model); + + /// @brief Removes a runtime-downloaded wake word model and frees its interpreter, arenas, and model buffer. + /// Must be called from the main loop. If the inference task is running it is paused at a safe point first, + /// and any queued detection events are dropped (they hold pointers into the model being destroyed). + /// @return True if the model was removed, false if the id is not a runtime model or the task could not be paused + bool remove_runtime_model(const std::string &model_id); + + /// @brief Returns the wake word model with the given id, or nullptr if none matches (compiled or runtime). + /// Must be called from the main loop, as the returned pointer is invalidated by remove_runtime_model(). + WakeWordModel *get_model_by_id(const std::string &model_id); + + /// @brief Returns the ids of all runtime-downloaded models. Must be called from the main loop. + std::vector get_runtime_model_ids(); + + /// @brief Returns the feature step size (ms) the frontend is configured for. Runtime models must match it. + uint8_t get_features_step_size() const { return this->features_step_size_; } + #ifdef USE_MICRO_WAKE_WORD_VAD void add_vad_model(const uint8_t *model_start, uint8_t probability_cutoff, size_t sliding_window_size, size_t tensor_arena_size); @@ -85,6 +111,7 @@ class MicroWakeWord final : public Component std::weak_ptr ring_buffer_; std::vector wake_word_models_; + std::vector> runtime_models_; #ifdef USE_MICRO_WAKE_WORD_VAD std::unique_ptr vad_model_; @@ -119,6 +146,13 @@ class MicroWakeWord final : public Component /// @brief Resumes the inference task void resume_task_(); + /// @brief Parks the inference task at a safe point (or verifies it isn't running) so the model lists may be + /// mutated from the main loop. Every successful call must be paired with unlock_models_(). + /// @return True if the lists may be mutated, false if the running task never acknowledged the pause request + bool try_lock_models_(); + /// @brief Releases the inference task parked by a successful try_lock_models_() call + void unlock_models_(); + void set_state_(State state); /// @brief Generates a spectrogram feature from an input buffer of audio samples. The frontend buffers samples diff --git a/esphome/components/micro_wake_word/model_data.cpp b/esphome/components/micro_wake_word/model_data.cpp new file mode 100644 index 0000000000..a7326ab77a --- /dev/null +++ b/esphome/components/micro_wake_word/model_data.cpp @@ -0,0 +1,100 @@ +#include "model_data.h" + +#ifdef USE_ESP32 + +#include +#include "esphome/core/log.h" + +#include +#include + +namespace esphome::micro_wake_word { + +static const char *const TAG = "micro_wake_word"; + +ModelData::~ModelData() { this->deallocate_(); } + +bool ModelData::allocate(size_t size) { + // Reject up front: reallocating to zero frees the buffer and returns null, which would leave data_ pointing at + // freed memory. A zero-length model is never usable anyway. + if (size == 0) { + ESP_LOGE(TAG, "Refusing to allocate a zero-length model"); + return false; + } + + // Already allocated, so reallocate to the new size + if (this->data_) { + uint8_t *new_allocation = this->allocator_.reallocate(this->data_, size); + if (new_allocation == nullptr) { + ESP_LOGE(TAG, "Failed to reallocate %zu bytes", size); + return false; + } + this->data_ = new_allocation; + this->size_ = size; + this->valid_ = false; // Need to revalidate with new data + return true; + } + + // Try to allocate in PSRAM first + this->data_ = this->allocator_.allocate(size); + if (this->data_ == nullptr) { + ESP_LOGE(TAG, "Failed to allocate %zu bytes", size); + return false; + } + + this->size_ = size; + this->valid_ = false; + return true; +} + +void ModelData::deallocate_() { + if (this->data_ != nullptr) { + this->allocator_.deallocate(this->data_, this->size_); + this->data_ = nullptr; + this->size_ = 0; + this->valid_ = false; + } +} + +const uint8_t *ModelData::get_model_pointer() const { return this->valid_ ? this->data_ : nullptr; } + +uint8_t *ModelData::get_write_pointer() { + this->valid_ = false; // Mark invalid while writing + return this->data_; +} + +bool ModelData::validate_and_mark_ready() { + // The magic number lives in bytes 4-7, so we need at least 8 bytes to read it. + if (!this->data_ || this->size_ < 8) { + ESP_LOGE(TAG, "Model data is null or too small"); + return false; + } + + // Check TFLite magic number "TFL3" in bytes 4-7 + if (memcmp(this->data_ + 4, "TFL3", 4) != 0) { + ESP_LOGE(TAG, "Invalid TFLite model magic number"); + return false; + } + + // Bytes 0-3 hold the offset of the root table. tflite::GetModel only adds that offset to the start of the + // buffer, so check it lands inside the buffer before reading through it. + uint32_t root_offset; + memcpy(&root_offset, this->data_, sizeof(root_offset)); + if (root_offset >= this->size_) { + ESP_LOGE(TAG, "TFLite model root offset is out of bounds"); + return false; + } + + const tflite::Model *model = tflite::GetModel(this->data_); + if (model->version() != TFLITE_SCHEMA_VERSION) { + ESP_LOGE(TAG, "TFLite model version mismatch (expected %d, got %d)", TFLITE_SCHEMA_VERSION, model->version()); + return false; + } + + this->valid_ = true; + return true; +} + +} // namespace esphome::micro_wake_word + +#endif // USE_ESP32 diff --git a/esphome/components/micro_wake_word/model_data.h b/esphome/components/micro_wake_word/model_data.h new file mode 100644 index 0000000000..0f0e08c718 --- /dev/null +++ b/esphome/components/micro_wake_word/model_data.h @@ -0,0 +1,60 @@ +#pragma once + +#ifdef USE_ESP32 + +#include +#include +#include "esphome/core/helpers.h" + +namespace esphome::micro_wake_word { + +// Owns the buffer holding a runtime-downloaded TFLite model. The buffer prefers PSRAM but falls back to +// internal RAM, so a device without PSRAM can still hold a single model. It is filled over HTTP, checked +// for integrity by the caller (SHA256) and for a usable TFLite header here, then kept alive for the +// lifetime of the WakeWordModel that uses it. Only ever held behind a std::shared_ptr, so copies and +// moves are disabled. +class ModelData { + public: + ModelData() = default; + ~ModelData(); + + // Non-copyable, non-movable + ModelData(const ModelData &) = delete; + ModelData &operator=(const ModelData &) = delete; + ModelData(ModelData &&) = delete; + ModelData &operator=(ModelData &&) = delete; + + // Allocate memory for model + bool allocate(size_t size); + + // Get stable pointer for TFLite (only valid after validate_and_mark_ready()) + const uint8_t *get_model_pointer() const; + + // Get writable pointer for downloading (invalidates the model) + uint8_t *get_write_pointer(); + + // Validate TFLite model and mark as ready for use + bool validate_and_mark_ready(); + + // Check if model is valid and ready for use + bool is_valid() const { return this->valid_; } + + // Get size of model data + size_t size() const { return this->size_; } + + // Check if memory is allocated + bool is_allocated() const { return this->data_ != nullptr; } + + protected: + // Deallocate memory + void deallocate_(); + + uint8_t *data_{nullptr}; + size_t size_{0}; + bool valid_{false}; + RAMAllocator allocator_{RAMAllocator::NONE}; +}; + +} // namespace esphome::micro_wake_word + +#endif // USE_ESP32 diff --git a/esphome/components/micro_wake_word/streaming_model.cpp b/esphome/components/micro_wake_word/streaming_model.cpp index 1cdc06b352..72984f04fb 100644 --- a/esphome/components/micro_wake_word/streaming_model.cpp +++ b/esphome/components/micro_wake_word/streaming_model.cpp @@ -26,6 +26,11 @@ void VADModel::log_model_config() { } bool StreamingModel::load_model_() { + if (this->model_start_ == nullptr) { + ESP_LOGE(TAG, "Streaming model has no data to load"); + return false; + } + RAMAllocator arena_allocator; if (this->var_arena_ == nullptr) { @@ -188,6 +193,13 @@ void StreamingModel::unload_model() { } bool StreamingModel::perform_streaming_inference(const int8_t features[PREPROCESSOR_FEATURE_SIZE]) { + if (this->model_start_ == nullptr) { + // No usable model data, and that cannot change for this object. Skip the model instead of reporting a + // failure, because a false return here stops the inference task for every other model too. + this->enabled_ = false; + return true; + } + if (this->enabled_ && !this->loaded_) { // Model is enabled but isn't loaded if (!this->load_model_()) { @@ -269,6 +281,41 @@ WakeWordModel::WakeWordModel(const std::string &id, const uint8_t *model_start, } }; +WakeWordModel::WakeWordModel(const std::string &id, std::shared_ptr model_data, + uint8_t default_probability_cutoff, size_t sliding_window_average_size, + const std::string &wake_word, std::vector trained_languages, + size_t tensor_arena_size) { + this->id_ = id; + this->model_data_ = std::move(model_data); + // Callers are expected to pass a validated buffer, so this is normally the stable model pointer. Tolerate a + // null or unvalidated handle rather than dereferencing it blindly: model_start_ stays null and the model is + // never loaded. + this->model_start_ = this->model_data_ ? this->model_data_->get_model_pointer() : nullptr; + if (this->model_start_ == nullptr) { + ESP_LOGE(TAG, "Model '%s' has no valid data and will not be loaded", id.c_str()); + } + this->default_probability_cutoff_ = default_probability_cutoff; + this->probability_cutoff_ = default_probability_cutoff; + this->sliding_window_size_ = sliding_window_average_size; + this->recent_streaming_probabilities_.resize(sliding_window_average_size, 0); + this->wake_word_ = wake_word; + this->trained_languages_ = std::move(trained_languages); + this->tensor_arena_size_ = tensor_arena_size; + this->register_streaming_ops_(this->streaming_op_resolver_); + this->current_stride_step_ = 0; + this->internal_only_ = false; // Runtime models are always exposed to Home Assistant + + this->pref_ = global_preferences->make_preference(fnv1_hash(id)); + bool enabled; + if (this->pref_.load(&enabled)) { + // Use the enabled state loaded from flash + this->enabled_ = enabled; + } else { + // No saved state: stay disabled. The activation flow calls enable() explicitly after adding. + this->enabled_ = false; + } +}; + void WakeWordModel::enable() { this->enabled_ = true; if (!this->internal_only_) { diff --git a/esphome/components/micro_wake_word/streaming_model.h b/esphome/components/micro_wake_word/streaming_model.h index 07ba78d1f4..1cb9d6eba5 100644 --- a/esphome/components/micro_wake_word/streaming_model.h +++ b/esphome/components/micro_wake_word/streaming_model.h @@ -3,9 +3,11 @@ #ifdef USE_ESP32 #include "preprocessor_settings.h" +#include "model_data.h" #include "esphome/core/preferences.h" +#include #include #include #include @@ -27,6 +29,10 @@ struct DetectionEvent { class StreamingModel { public: + // Runtime models are heap owned and destroyed while the device is running, so freeing the arenas cannot + // depend on the owner calling unload_model() first. unload_model() is not virtual and is safe to repeat. + virtual ~StreamingModel() { this->unload_model(); } + virtual void log_model_config() = 0; virtual DetectionEvent determine_detected() = 0; @@ -51,6 +57,9 @@ class StreamingModel { /// @brief Return true if the model is enabled. bool is_enabled() const { return this->enabled_; } + /// @brief Return true if the model has usable data. A model without it can never be loaded or run. + bool has_model_data() const { return this->model_start_ != nullptr; } + bool get_unprocessed_probability_status() const { return this->unprocessed_probability_status_; } // Quantized probability cutoffs mapping 0.0 - 1.0 to 0 - 255 @@ -86,7 +95,7 @@ class StreamingModel { size_t tensor_arena_size_; std::vector recent_streaming_probabilities_; - const uint8_t *model_start_; + const uint8_t *model_start_{nullptr}; uint8_t *tensor_arena_{nullptr}; uint8_t *var_arena_{nullptr}; std::unique_ptr interpreter_; @@ -96,7 +105,7 @@ class StreamingModel { class WakeWordModel final : public StreamingModel { public: - /// @brief Constructs a wake word model object + /// @brief Constructs a wake word model object with compile-time model data /// @param id (std::string) identifier for this model /// @param model_start (const uint8_t *) pointer to the start of the model's TFLite FlatBuffer /// @param default_probability_cutoff (uint8_t) probability cutoff for acceping the wake word has been said @@ -110,6 +119,23 @@ class WakeWordModel final : public StreamingModel { size_t sliding_window_average_size, const std::string &wake_word, size_t tensor_arena_size, bool default_enabled, bool internal_only); + /// @brief Constructs a wake word model object with a runtime-downloaded model + /// @param id (std::string) identifier for this model + /// @param model_data (std::shared_ptr) owning handle to the downloaded model buffer; must be valid + /// @param default_probability_cutoff (uint8_t) probability cutoff for acceping the wake word has been said + /// @param sliding_window_average_size (size_t) the length of the sliding window computing the mean rolling + /// probability + /// @param wake_word (std::string) Friendly name of the wake word + /// @param trained_languages (std::vector) Languages the model was trained on + /// @param tensor_arena_size (size_t) Size in bytes for allocating the tensor arena + WakeWordModel(const std::string &id, std::shared_ptr model_data, uint8_t default_probability_cutoff, + size_t sliding_window_average_size, const std::string &wake_word, + std::vector trained_languages, size_t tensor_arena_size); + + // model_data_ is a member of this class, so it is destroyed before ~StreamingModel() runs. Unload here, while + // the buffer is still alive, so the interpreter is never torn down over freed model data. + ~WakeWordModel() override { this->unload_model(); } + void log_model_config() override; /// @brief Checks for the wake word by comparing the mean probability in the sliding window with the probability @@ -132,6 +158,10 @@ class WakeWordModel final : public StreamingModel { bool get_internal_only() { return this->internal_only_; } protected: + // Kept for runtime-downloaded models so the model buffer stays alive for the model's lifetime. + // Null for compiled-in models (their data lives in flash). + std::shared_ptr model_data_; + std::string id_; std::string wake_word_; std::vector trained_languages_; diff --git a/esphome/components/midea/ac_adapter.cpp b/esphome/components/midea/ac_adapter.cpp index 77bb9bbe86..30771d25ca 100644 --- a/esphome/components/midea/ac_adapter.cpp +++ b/esphome/components/midea/ac_adapter.cpp @@ -1,4 +1,4 @@ -#if defined(USE_ARDUINO) && !defined(USE_RP2) && !defined(USE_LIBRETINY) +#if (defined(USE_ARDUINO) && !defined(USE_RP2) && !defined(USE_LIBRETINY)) || defined(USE_ESP_IDF) #include "esphome/core/log.h" #include "ac_adapter.h" @@ -172,4 +172,4 @@ void Converters::to_climate_traits(ClimateTraits &traits, const dudanov::midea:: } // namespace esphome::midea::ac -#endif // USE_ARDUINO +#endif // USE_ARDUINO || USE_ESP_IDF diff --git a/esphome/components/midea/ac_adapter.h b/esphome/components/midea/ac_adapter.h index 4545743564..4a888ee8ff 100644 --- a/esphome/components/midea/ac_adapter.h +++ b/esphome/components/midea/ac_adapter.h @@ -1,6 +1,6 @@ #pragma once -#if defined(USE_ARDUINO) && !defined(USE_RP2) && !defined(USE_LIBRETINY) +#if (defined(USE_ARDUINO) && !defined(USE_RP2) && !defined(USE_LIBRETINY)) || defined(USE_ESP_IDF) // MideaUART #include @@ -44,4 +44,4 @@ class Converters { } // namespace esphome::midea::ac -#endif // USE_ARDUINO +#endif // USE_ARDUINO || USE_ESP_IDF diff --git a/esphome/components/midea/ac_automations.h b/esphome/components/midea/ac_automations.h index b595a018b3..9572ec6c65 100644 --- a/esphome/components/midea/ac_automations.h +++ b/esphome/components/midea/ac_automations.h @@ -1,6 +1,6 @@ #pragma once -#if defined(USE_ARDUINO) && !defined(USE_RP2) && !defined(USE_LIBRETINY) +#if (defined(USE_ARDUINO) && !defined(USE_RP2) && !defined(USE_LIBRETINY)) || defined(USE_ESP_IDF) #include "esphome/core/automation.h" #include "air_conditioner.h" @@ -63,4 +63,4 @@ template class PowerToggleAction : public MideaActionBase } // namespace esphome::midea::ac -#endif // USE_ARDUINO +#endif // USE_ARDUINO || USE_ESP_IDF diff --git a/esphome/components/midea/air_conditioner.cpp b/esphome/components/midea/air_conditioner.cpp index e55afedd8a..24bbfe76b0 100644 --- a/esphome/components/midea/air_conditioner.cpp +++ b/esphome/components/midea/air_conditioner.cpp @@ -1,4 +1,4 @@ -#if defined(USE_ARDUINO) && !defined(USE_RP2) && !defined(USE_LIBRETINY) +#if (defined(USE_ARDUINO) && !defined(USE_RP2) && !defined(USE_LIBRETINY)) || defined(USE_ESP_IDF) #include "esphome/core/helpers.h" #include "esphome/core/log.h" @@ -197,4 +197,4 @@ void AirConditioner::do_display_toggle() { } // namespace esphome::midea::ac -#endif // USE_ARDUINO +#endif // USE_ARDUINO || USE_ESP_IDF diff --git a/esphome/components/midea/air_conditioner.h b/esphome/components/midea/air_conditioner.h index 089928902e..9977d2088f 100644 --- a/esphome/components/midea/air_conditioner.h +++ b/esphome/components/midea/air_conditioner.h @@ -1,6 +1,6 @@ #pragma once -#if defined(USE_ARDUINO) && !defined(USE_RP2) && !defined(USE_LIBRETINY) +#if (defined(USE_ARDUINO) && !defined(USE_RP2) && !defined(USE_LIBRETINY)) || defined(USE_ESP_IDF) // MideaUART #include @@ -61,4 +61,4 @@ class AirConditioner final : public ApplianceBase #include +#include // Include global defines #include "esphome/core/defines.h" @@ -17,6 +18,13 @@ namespace esphome::midea { +// Mirrors the ARDUINO switch in MideaUART Helpers/Platform.h: these types +// exist in the dudanov namespace exactly when the library is not on Arduino +#ifndef ARDUINO +using dudanov::Stream; +using dudanov::String; +#endif + /* Stream from UART component */ class UARTStream : public Stream { public: @@ -99,4 +107,4 @@ template class ApplianceBase : public Component { } // namespace esphome::midea -#endif // USE_ARDUINO +#endif // USE_ARDUINO || USE_ESP_IDF diff --git a/esphome/components/midea/climate.py b/esphome/components/midea/climate.py index b0c102af6d..0e03bca233 100644 --- a/esphome/components/midea/climate.py +++ b/esphome/components/midea/climate.py @@ -153,7 +153,6 @@ CONFIG_SCHEMA = cv.All( ) .extend(uart.UART_DEVICE_SCHEMA) .extend(cv.COMPONENT_SCHEMA), - cv.only_with_arduino, cv.only_on( [ PLATFORM_ESP32, @@ -305,7 +304,14 @@ async def to_code(config): if CONF_HUMIDITY_SETPOINT in config: sens = await sensor.new_sensor(config[CONF_HUMIDITY_SETPOINT]) cg.add(var.set_humidity_setpoint_sensor(sens)) - # MideaUART library requires WiFi (WiFi auto-enables Network via dependency mapping) - if CORE.is_esp32: + # MideaUART uses the Arduino WiFi API for the network-notify frame + # (WiFi auto-enables Network via dependency mapping). On ESP-IDF the + # library talks to esp_wifi directly, so no library entry is needed. + if CORE.is_esp32 and CORE.using_arduino: cg.add_library("WiFi", None) - cg.add_library("dudanov/MideaUART", "1.1.9") + # Using the repository until a release containing ESP-IDF support is published + cg.add_library( + name="MideaUART", + version=None, + repository="https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815", + ) diff --git a/esphome/components/midea/ir_transmitter.h b/esphome/components/midea/ir_transmitter.h index ecf3fa1c1a..e54df1fd70 100644 --- a/esphome/components/midea/ir_transmitter.h +++ b/esphome/components/midea/ir_transmitter.h @@ -1,6 +1,6 @@ #pragma once -#if defined(USE_ARDUINO) && !defined(USE_RP2) && !defined(USE_LIBRETINY) +#if (defined(USE_ARDUINO) && !defined(USE_RP2) && !defined(USE_LIBRETINY)) || defined(USE_ESP_IDF) #ifdef USE_REMOTE_TRANSMITTER #include "esphome/components/remote_base/midea_protocol.h" @@ -85,5 +85,5 @@ class IrTransmitter { } // namespace esphome::midea -#endif -#endif // USE_ARDUINO +#endif // USE_REMOTE_TRANSMITTER +#endif // USE_ARDUINO || USE_ESP_IDF diff --git a/esphome/components/modbus/__init__.py b/esphome/components/modbus/__init__.py index 9e64540382..bc52263aef 100644 --- a/esphome/components/modbus/__init__.py +++ b/esphome/components/modbus/__init__.py @@ -14,6 +14,12 @@ import esphome.final_validate as fv _LOGGER = logging.getLogger(__name__) DEPENDENCIES = ["uart"] +# Loading the hub makes the modbus_client.* actions available (they are registry entries only; no code is +# generated unless a config uses one). +AUTO_LOAD = ["modbus_client"] + +# Mirrors modbus::MAX_PDU_SIZE in modbus_definitions.h: 256-byte RTU frame minus address and CRC. +MAX_PDU_SIZE = 253 modbus_ns = cg.esphome_ns.namespace("modbus") Modbus = modbus_ns.class_("Modbus", cg.Component, uart.UARTDevice) diff --git a/esphome/components/modbus/helpers.py b/esphome/components/modbus/helpers.py index e3029b2648..e7eaacee0c 100644 --- a/esphome/components/modbus/helpers.py +++ b/esphome/components/modbus/helpers.py @@ -38,7 +38,9 @@ SensorValueType = SensorValueType_ns.enum("SensorValueType") SENSOR_VALUE_TYPE = { "RAW": SensorValueType.RAW, "U_WORD": SensorValueType.U_WORD, + "U_WORD_S": SensorValueType.U_WORD_S, "S_WORD": SensorValueType.S_WORD, + "S_WORD_S": SensorValueType.S_WORD_S, "U_DWORD": SensorValueType.U_DWORD, "U_DWORD_R": SensorValueType.U_DWORD_R, "S_DWORD": SensorValueType.S_DWORD, @@ -54,7 +56,9 @@ SENSOR_VALUE_TYPE = { TYPE_REGISTER_MAP = { "RAW": 1, "U_WORD": 1, + "U_WORD_S": 1, "S_WORD": 1, + "S_WORD_S": 1, "U_DWORD": 2, "U_DWORD_R": 2, "S_DWORD": 2, @@ -70,7 +74,9 @@ TYPE_REGISTER_MAP = { CPP_TYPE_REGISTER_MAP = { "RAW": cg.uint16, "U_WORD": cg.uint16, + "U_WORD_S": cg.uint16, "S_WORD": cg.int16, + "S_WORD_S": cg.int16, "U_DWORD": cg.uint32, "U_DWORD_R": cg.uint32, "S_DWORD": cg.int32, diff --git a/esphome/components/modbus/modbus_helpers.cpp b/esphome/components/modbus/modbus_helpers.cpp index 8428ea27ea..2c87928e9f 100644 --- a/esphome/components/modbus/modbus_helpers.cpp +++ b/esphome/components/modbus/modbus_helpers.cpp @@ -177,7 +177,9 @@ bool is_client_pdu_standard(const uint8_t *pdu, size_t size) { static size_t required_payload_size(SensorValueType sensor_value_type) { switch (sensor_value_type) { case SensorValueType::U_WORD: + case SensorValueType::U_WORD_S: case SensorValueType::S_WORD: + case SensorValueType::S_WORD_S: return 2; case SensorValueType::U_DWORD: case SensorValueType::FP32: @@ -228,6 +230,11 @@ std::optional payload_to_number(const uint8_t *data, size_t size, Senso case SensorValueType::U_WORD: value = mask_and_shift_by_rightbit(get_data(data, offset), bitmask); // default is 0xFFFF ; break; + case SensorValueType::U_WORD_S: { + uint16_t word = byteswap(get_data(data, offset)); + value = mask_and_shift_by_rightbit(word, bitmask); + break; + } case SensorValueType::U_DWORD: case SensorValueType::FP32: value = get_data(data, offset); @@ -242,6 +249,11 @@ std::optional payload_to_number(const uint8_t *data, size_t size, Senso case SensorValueType::S_WORD: value = mask_and_shift_by_rightbit(get_data(data, offset), bitmask); // default is 0xFFFF ; break; + case SensorValueType::S_WORD_S: { + uint16_t word = byteswap(get_data(data, offset)); + value = mask_and_shift_by_rightbit(static_cast(word), bitmask); + break; + } case SensorValueType::S_DWORD: value = mask_and_shift_by_rightbit(get_data(data, offset), bitmask); break; diff --git a/esphome/components/modbus/modbus_helpers.h b/esphome/components/modbus/modbus_helpers.h index 3dd933c4d7..36e3b6c7be 100644 --- a/esphome/components/modbus/modbus_helpers.h +++ b/esphome/components/modbus/modbus_helpers.h @@ -119,13 +119,20 @@ enum class SensorValueType : uint8_t { U_QWORD_R = 0xA, S_QWORD_R = 0xB, FP32 = 0xC, - FP32_R = 0xD + FP32_R = 0xD, + U_WORD_S = 0xE, // 1 Register unsigned, bytes swapped + S_WORD_S = 0xF, // 1 Register signed, bytes swapped }; inline bool value_type_is_float(SensorValueType v) { return v == SensorValueType::FP32 || v == SensorValueType::FP32_R; } +/// Coils and discrete inputs are the bit-addressed entity tables; the other types are 16-bit registers. +inline bool is_entity_type_binary(EntityType type) { + return type == EntityType::COIL || type == EntityType::DISCRETE_INPUT; +} + inline FunctionCode modbus_register_read_function(EntityType reg_type) { switch (reg_type) { case EntityType::COIL: @@ -284,6 +291,10 @@ template void number_to_payload(Container &data, int64_t val case SensorValueType::S_WORD: data.push_back(value & 0xFFFF); break; + case SensorValueType::U_WORD_S: + case SensorValueType::S_WORD_S: + data.push_back(byteswap(static_cast(value & 0xFFFF))); + break; case SensorValueType::U_DWORD: case SensorValueType::S_DWORD: case SensorValueType::FP32: diff --git a/esphome/components/modbus_client/__init__.py b/esphome/components/modbus_client/__init__.py new file mode 100644 index 0000000000..e8a75a1b6c --- /dev/null +++ b/esphome/components/modbus_client/__init__.py @@ -0,0 +1,164 @@ +from esphome import automation +import esphome.codegen as cg +from esphome.components import modbus +import esphome.config_validation as cv +from esphome.const import CONF_ADDRESS, CONF_ON_ERROR, CONF_ON_RESPONSE +from esphome.core import Lambda +from esphome.types import ConfigType, TemplateArgsType + +CODEOWNERS = ["@exciton"] +DEPENDENCIES = ["modbus"] + +CONF_ON_NO_RESPONSE = "on_no_response" +CONF_ON_NOT_SENT = "on_not_sent" +CONF_ON_SENT = "on_sent" +CONF_PDU = "pdu" +CONF_RETRY = "retry" + +modbus_client_ns = cg.esphome_ns.namespace("modbus_client") +ModbusClientSendAction = modbus_client_ns.class_( + "ModbusClientSendAction", automation.Action, modbus.ModbusClientDevice +) + +# The exception code passed to on_error handlers. +ExceptionCode = modbus.modbus_ns.enum("ExceptionCode") + +# Lambda argument types for the reply handlers: the device address the send targeted, and the +# request/response PDUs (function code + data). The spans are only valid for the duration of the handler. +_PDU_SPAN = cg.std_span.template(cg.uint8.operator("const")) + +# The pdu lambda's return type: a stack-allocated StaticVector capped at the Modbus PDU limit +# (modbus.MAX_PDU_SIZE). Lambdas can return a byte list or a modbus::helpers::create_*_pdu() result. +# The list form below is bounded by cv.Length; a lambda cannot be. PduBuffer drops bytes past +# modbus.MAX_PDU_SIZE without reporting it, so an over-long lambda PDU is silently truncated. +_PDU_BUFFER = modbus.modbus_ns.namespace("helpers").class_("PduBuffer") + + +def _synchronous_handler(value: ConfigType) -> ConfigType: + """Reject deferring actions in a handler: its PDU spans point into hub buffers that are reused + once the handler returns, and DelayAction and friends capture the trigger args for later replay.""" + if automation.has_non_synchronous_actions(value): + raise cv.Invalid( + "Deferring actions (delay, wait_until, script.wait, ...) are not allowed in modbus_client " + "handlers: the request/response data is only valid while the handler runs. Copy what you " + "need into globals first, then defer in a separate script or automation." + ) + return value + + +def _handler_schema() -> cv.All: + return cv.All(automation.validate_automation(single=True), _synchronous_handler) + + +# Each action is its own hub device: the modbus hub routes the reply straight back to the action that +# sent it, so the address can even be templatable - the reply is matched by the action's identity, not +# its address. +_ACTION_BASE_SCHEMA = cv.Schema( + { + cv.GenerateID(modbus.CONF_MODBUS_ID): cv.use_id(modbus.ModbusClient), + cv.Required(CONF_ADDRESS): cv.templatable(cv.hex_uint8_t), + # Optional handlers. on_sent fires when the frame reaches the wire; the reply handlers arrive + # later (fire-and-continue), so all run with the request/reply available - not the outer + # automation's variables. + cv.Optional(CONF_ON_SENT): _handler_schema(), + cv.Optional(CONF_ON_ERROR): _handler_schema(), + # on_no_response takes either a returning lambda (`!lambda "return ;"`, gets `request`, + # returns true to have the hub retry the frame) OR a `then:` automation of actions; the automation + # form may also carry an optional `retry:` returning lambda to run actions AND decide the retry. + cv.Optional(CONF_ON_NO_RESPONSE): cv.All( + cv.Any( + cv.returning_lambda, + automation.validate_automation( + {cv.Optional(CONF_RETRY): cv.returning_lambda}, single=True + ), + ), + _synchronous_handler, + ), + cv.Optional(CONF_ON_NOT_SENT): _handler_schema(), + } +) + +MODBUS_CLIENT_SEND_SCHEMA = _ACTION_BASE_SCHEMA.extend( + { + cv.Required(CONF_PDU): cv.templatable( + cv.All( + cv.ensure_list(cv.hex_uint8_t), + cv.Length(min=1, max=modbus.MAX_PDU_SIZE), + ) + ), + cv.Optional(CONF_ON_RESPONSE): _handler_schema(), + } +) + + +async def register_client_action( + var: cg.MockObj, + config: ConfigType, + args: TemplateArgsType, + response_args: TemplateArgsType, +) -> cg.MockObj: + """Wire the shared action plumbing: hub parent, templated device address, outcome triggers. + + response_args are the on_response handler's arguments, which differ per action. + """ + parent = await cg.get_variable(config[modbus.CONF_MODBUS_ID]) + cg.add(var.set_parent(parent)) + cg.add( + var.set_target_address( + await cg.templatable(config[CONF_ADDRESS], args, cg.uint8) + ) + ) + if sent_conf := config.get(CONF_ON_SENT): + await automation.build_automation( + var.get_sent_trigger(), [(_PDU_SPAN, "request")], sent_conf + ) + if response_conf := config.get(CONF_ON_RESPONSE): + await automation.build_automation( + var.get_response_trigger(), response_args, response_conf + ) + if error_conf := config.get(CONF_ON_ERROR): + await automation.build_automation( + var.get_error_trigger(), + [(_PDU_SPAN, "request"), (ExceptionCode, "exception_code")], + error_conf, + ) + if (no_response_conf := config.get(CONF_ON_NO_RESPONSE)) is not None: + # The lambda form IS the retry decision; the automation form runs actions and may carry a nested + # `retry:` lambda. Either way the retry lambda's bool becomes on_no_response()'s return value. + if isinstance(no_response_conf, Lambda): + retry_conf = no_response_conf + else: + await automation.build_automation( + var.get_no_response_trigger(), + [(_PDU_SPAN, "request")], + no_response_conf, + ) + retry_conf = no_response_conf.get(CONF_RETRY) + if retry_conf is not None: + retry_lambda = await cg.process_lambda( + retry_conf, [(_PDU_SPAN, "request")], return_type=cg.bool_ + ) + cg.add(var.set_retry(retry_lambda)) + if not_sent_conf := config.get(CONF_ON_NOT_SENT): + await automation.build_automation( + var.get_not_sent_trigger(), [(_PDU_SPAN, "request")], not_sent_conf + ) + return var + + +@automation.register_action( + "modbus_client.send", + ModbusClientSendAction, + MODBUS_CLIENT_SEND_SCHEMA, + synchronous=True, +) +async def modbus_client_send_to_code(config, action_id, template_arg, args): + var = cg.new_Pvariable(action_id, template_arg) + template_ = await cg.templatable(config[CONF_PDU], args, _PDU_BUFFER) + cg.add(var.set_pdu(template_)) + return await register_client_action( + var, + config, + args, + [(_PDU_SPAN, "request"), (_PDU_SPAN, "response")], + ) diff --git a/esphome/components/modbus_client/modbus_client.h b/esphome/components/modbus_client/modbus_client.h new file mode 100644 index 0000000000..d4b3792a5f --- /dev/null +++ b/esphome/components/modbus_client/modbus_client.h @@ -0,0 +1,99 @@ +#pragma once + +#include "esphome/components/modbus/modbus.h" +#include "esphome/components/modbus/modbus_helpers.h" +#include "esphome/core/automation.h" + +#include + +namespace esphome::modbus_client { + +/// Shared base for the modbus_client actions. Each ACTION INSTANCE is its own modbus::ModbusClientDevice: +/// the hub routes every reply (or its lack) straight back to the action that sent it, so there is no +/// central client object and no request matching. The device address is templatable; it is stamped on the +/// device at play() time; the hub routes each reply by device pointer, so a changed address never +/// mis-routes an earlier reply. (The address is not passed to the reply triggers - under overlapping +/// sends it could misreport, and the handler can recompute the expression it configured.) +template class ClientActionBase : public Action, public modbus::ModbusClientDevice { + public: + TEMPLATABLE_VALUE(uint8_t, target_address) // the modbus device address + + Trigger> *get_sent_trigger() { return &this->sent_trigger_; } + Trigger, modbus::ExceptionCode> *get_error_trigger() { return &this->error_trigger_; } + Trigger> *get_no_response_trigger() { return &this->no_response_trigger_; } + Trigger> *get_not_sent_trigger() { return &this->not_sent_trigger_; } + + /// The retry decision for on_no_response: given the request PDU, return true to have the hub re-queue + /// the frame. Set from the lambda form or a then: automation's nested retry lambda; may coexist with + /// the no_response trigger (actions run, then this decides the retry). + using retry_func_t = bool (*)(std::span); + void set_retry(retry_func_t f) { this->retry_func_ = f; } + + /// The frame was written to the wire: fires once per transmission, before any reply, and never for a + /// send that ended in on_not_sent. request_pdu is the PDU sent (function code + data). + void on_sent(std::span request_pdu) override { this->sent_trigger_.trigger(request_pdu); } + /// Never reached the wire (tx queue full, cleared, or a duplicate write dropped by the hub's dedup). + void on_not_sent(std::span request_pdu) override { this->not_sent_trigger_.trigger(request_pdu); } + /// A Modbus exception reply. Lives here beside its trigger so every action subclass gets the pairing: + /// register_client_action() wires on_error for all of them, so a derived class must not have to + /// remember the override. + void on_error(std::span request_pdu, modbus::ExceptionCode exception_code) override { + this->error_trigger_.trigger(request_pdu, exception_code); + } + /// No reply within send_wait_time. Run the on_no_response actions (empty in the pure-lambda form), + /// then let the retry lambda, if set, decide whether the hub re-queues the frame (true = retry). The + /// two coexist: a then: automation can also carry a retry lambda. No lambda = no retry. + bool on_no_response(std::span request_pdu) override { + this->no_response_trigger_.trigger(request_pdu); + if (this->retry_func_ != nullptr) + return this->retry_func_(request_pdu); + return false; + } + /// Stamp the templated device address before every play(): subclasses cannot forget it, and the hub + /// routes each reply by device pointer, so a changed address never mis-routes earlier replies. + void play_complex(const Ts &...x) override { + this->set_address(this->target_address_.value(x...)); + Action::play_complex(x...); + } + + protected: + Trigger> sent_trigger_; + Trigger, modbus::ExceptionCode> error_trigger_; + Trigger> no_response_trigger_; + Trigger> not_sent_trigger_; + retry_func_t retry_func_{nullptr}; +}; + +/// modbus_client.send: fire a raw PDU (function code + data; the hub adds address and CRC). The reply is +/// delivered raw - on_response(request, response) - deliberately bypassing the typed dispatch, so +/// non-standard/custom transactions pass through untouched. +/// The PDU is a stack-allocated modbus::helpers::PduBuffer, so a pdu lambda can build one with the +/// modbus::helpers::create_*_pdu() builders and return it directly (smaller builder results convert). +/// A PduBuffer drops bytes past modbus::MAX_PDU_SIZE without reporting it (the hub's oversize check +/// cannot fire - that limit is the capacity), so an over-long lambda-built PDU is silently truncated. +template class ModbusClientSendAction : public ClientActionBase { + public: + TEMPLATABLE_VALUE(modbus::helpers::PduBuffer, pdu) + + Trigger, std::span> *get_response_trigger() { + return &this->response_trigger_; + } + + void play(const Ts &...x) override { + auto pdu = this->pdu_.value(x...); + const std::span span(pdu.data(), pdu.size()); + // The hub refuses some sends at the door with no callback (an empty PDU, a duplicate write already + // pending, a full queue). Every send still gets exactly one outcome, so resolve those via on_not_sent. + if (!this->send_pdu(span)) + this->on_not_sent(span); + } + + void on_response(std::span request_pdu, std::span response_pdu) override { + this->response_trigger_.trigger(request_pdu, response_pdu); + } + + protected: + Trigger, std::span> response_trigger_; +}; + +} // namespace esphome::modbus_client diff --git a/esphome/components/modbus_controller/__init__.py b/esphome/components/modbus_controller/__init__.py index 35a5479ecb..1ce1e38d16 100644 --- a/esphome/components/modbus_controller/__init__.py +++ b/esphome/components/modbus_controller/__init__.py @@ -42,22 +42,38 @@ AUTO_LOAD = ["modbus"] MULTI_CONF = True modbus_controller_ns = cg.esphome_ns.namespace("modbus_controller") -ModbusController = modbus_controller_ns.class_( - "ModbusController", cg.PollingComponent, modbus.ModbusClientDevice -) +ModbusController = modbus_controller_ns.class_("ModbusController", cg.PollingComponent) SensorItem = modbus_controller_ns.struct("SensorItem") _LOGGER = logging.getLogger(__name__) +# Remove before 2027.2.0 +_REMOVED_OPTIONS = { + CONF_COMMAND_THROTTLE: "Command spacing is handled by the 'modbus' component - use 'turnaround_time' there instead.", + CONF_ALLOW_DUPLICATE_COMMANDS: "Polling commands are deduplicated by the modbus hub; one-shot commands (writes) are always transmitted.", +} + + +def _warn_removed_options(config: ConfigType) -> ConfigType: + """Warn about options that no longer do anything, but let the config compile.""" + for option, replacement in _REMOVED_OPTIONS.items(): + if option in config: + _LOGGER.warning( + "[modbus_controller] '%s' no longer has any effect and will be removed in 2027.2.0. %s", + option, + replacement, + ) + return config + + CONFIG_SCHEMA = cv.All( cv.Schema( { cv.GenerateID(): cv.declare_id(ModbusController), - cv.Optional(CONF_ALLOW_DUPLICATE_COMMANDS, default=False): cv.boolean, - cv.Optional( - CONF_COMMAND_THROTTLE, default="0ms" - ): cv.positive_time_period_milliseconds, + # Removed options: accepted (and ignored) until 2027.2.0 so existing configs keep building. + cv.Optional(CONF_ALLOW_DUPLICATE_COMMANDS): cv.boolean, + cv.Optional(CONF_COMMAND_THROTTLE): cv.positive_time_period_milliseconds, cv.Optional(CONF_SERVER_COURTESY_RESPONSE): cv.invalid( "This option has been removed. Use modbus_server component instead: https://esphome.io/components/modbus_server/" ), @@ -74,7 +90,8 @@ CONFIG_SCHEMA = cv.All( } ) .extend(cv.polling_component_schema("60s")) - .extend(modbus.modbus_device_schema(0x01)) + .extend(modbus.modbus_device_schema(0x01)), + _warn_removed_options, ) ModbusItemBaseSchema = cv.Schema( @@ -170,10 +187,7 @@ async def add_modbus_base_properties( [ (sensor_type.operator("ptr"), "item"), (lambda_param_type, "x"), - ( - cg.std_vector.template(cg.uint8).operator("const").operator("ref"), - "data", - ), + (cg.std_span.template(cg.uint8.operator("const")), "data"), ], return_type=cg.optional.template(lambda_return_type), ) @@ -201,8 +215,6 @@ _CALLBACK_AUTOMATIONS = ( async def to_code(config): var = cg.new_Pvariable(config[CONF_ID]) - cg.add(var.set_allow_duplicate_commands(config[CONF_ALLOW_DUPLICATE_COMMANDS])) - cg.add(var.set_command_throttle(config[CONF_COMMAND_THROTTLE])) cg.add(var.set_max_cmd_retries(config[CONF_MAX_CMD_RETRIES])) cg.add(var.set_offline_skip_updates(config[CONF_OFFLINE_SKIP_UPDATES])) await register_modbus_device(var, config) diff --git a/esphome/components/modbus_controller/binary_sensor/modbus_binarysensor.cpp b/esphome/components/modbus_controller/binary_sensor/modbus_binarysensor.cpp index d3caaaa3d9..b0c927cf84 100644 --- a/esphome/components/modbus_controller/binary_sensor/modbus_binarysensor.cpp +++ b/esphome/components/modbus_controller/binary_sensor/modbus_binarysensor.cpp @@ -7,17 +7,18 @@ static const char *const TAG = "modbus_controller.binary_sensor"; void ModbusBinarySensor::dump_config() { LOG_BINARY_SENSOR("", "Modbus Controller Binary Sensor", this); } -void ModbusBinarySensor::parse_and_publish(const std::vector &data) { +void ModbusBinarySensor::parse_and_publish(std::span data) { bool value; + // For coils/discrete inputs this is the bit index; for registers it is the byte offset. + const size_t offset = this->offset; switch (this->register_type) { case modbus::EntityType::DISCRETE_INPUT: case modbus::EntityType::COIL: - // offset for coil is the actual number of the coil not the byte offset - value = modbus::helpers::bit_from_packed(this->offset, data); + value = modbus::helpers::bit_from_packed(offset, data); break; default: - value = modbus::helpers::get_data(data, this->offset) & this->bitmask; + value = modbus::helpers::get_data(data.data(), offset) & this->bitmask; break; } // Is there a lambda registered diff --git a/esphome/components/modbus_controller/binary_sensor/modbus_binarysensor.h b/esphome/components/modbus_controller/binary_sensor/modbus_binarysensor.h index f56a32a5ec..62a7fe93d3 100644 --- a/esphome/components/modbus_controller/binary_sensor/modbus_binarysensor.h +++ b/esphome/components/modbus_controller/binary_sensor/modbus_binarysensor.h @@ -4,7 +4,7 @@ #include "esphome/components/modbus_controller/modbus_controller.h" #include "esphome/core/component.h" -#include +#include namespace esphome::modbus_controller { @@ -13,26 +13,26 @@ class ModbusBinarySensor final : public Component, public binary_sensor::BinaryS ModbusBinarySensor(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint32_t bitmask, uint16_t skip_updates, bool force_new_range) { this->register_type = register_type; - this->start_address = start_address; - this->offset = offset; + this->set_address(start_address); + this->set_offset_from_start_address(offset); this->bitmask = bitmask; this->sensor_value_type = SensorValueType::BIT; this->skip_updates = skip_updates; this->force_new_range = force_new_range; - if (register_type == modbus::EntityType::COIL || register_type == modbus::EntityType::DISCRETE_INPUT) { + if (modbus::helpers::is_entity_type_binary(register_type)) { this->register_count = offset + 1; } else { this->register_count = 1; } } - void parse_and_publish(const std::vector &data) override; + void parse_and_publish(std::span data) override; void set_state(bool state) { this->state = state; } void dump_config() override; - using transform_func_t = optional (*)(ModbusBinarySensor *, bool, const std::vector &); + using transform_func_t = optional (*)(ModbusBinarySensor *, bool, std::span); void set_template(transform_func_t f) { this->transform_func_ = f; } protected: diff --git a/esphome/components/modbus_controller/modbus_controller.cpp b/esphome/components/modbus_controller/modbus_controller.cpp index 8822b7b40a..c4161d454f 100644 --- a/esphome/components/modbus_controller/modbus_controller.cpp +++ b/esphome/components/modbus_controller/modbus_controller.cpp @@ -6,308 +6,342 @@ namespace esphome::modbus_controller { static const char *const TAG = "modbus_controller"; -void ModbusController::setup() { this->create_register_ranges_(); } +void ModbusController::setup() { this->create_polling_commands_(); } -/* - To work with the existing modbus class and avoid polling for responses a command queue is used. - send_next_command will submit the command at the top of the queue and set the corresponding callback - to handle the response from the device. - Once the response has been processed it is removed from the queue and the next command is sent -*/ -bool ModbusController::send_next_command_() { - uint32_t last_send = millis() - this->last_command_timestamp_; +ModbusCommandItem::ModbusCommandItem(ModbusController &controller, modbus::ModbusClientHub *parent, uint8_t address, + RegisterRange &&range) + : modbus::ModbusClientDevice(parent, address), + sensors(std::move(range.sensors)), + skip_updates(range.skip_updates), + register_type_(range.register_type), + start_address_(range.start_address), + register_count_(range.register_count), + function_code_(modbus::helpers::modbus_register_read_function(range.register_type)), + controller_(&controller) {} - if ((last_send > this->command_throttle_) && this->ready_for_immediate_send() && !this->command_queue_.empty()) { - auto &command = this->command_queue_.front(); - - // remove from queue if command was sent too often - if (!command->should_retry(this->max_cmd_retries_)) { - if (!this->module_offline_) { - ESP_LOGW(TAG, "Modbus device=%d set offline", this->address_); - - if (this->offline_skip_updates_ > 0) { - // Update skip_updates_counter to stop flooding channel with timeouts - for (auto &r : this->register_ranges_) { - r.skip_updates_counter = this->offline_skip_updates_; - } - } - - this->module_offline_ = true; - this->offline_callback_.call((int) command->function_code, command->register_address); - } - ESP_LOGD(TAG, "Modbus command to device=%d register=0x%02X no response received - removed from send queue", - this->address_, command->register_address); - this->command_queue_.pop_front(); - } else { - ESP_LOGV(TAG, "Sending next modbus command to device %d register 0x%02X count %d", this->address_, - command->register_address, command->register_count); - command->send(); - - this->last_command_timestamp_ = millis(); - - this->command_sent_callback_.call((int) command->function_code, command->register_address); - - // remove from queue if no handler is defined - if (!command->on_data_func) { - this->command_queue_.pop_front(); - } - } - } - return (!this->command_queue_.empty()); +ModbusCommandItem::ModbusCommandItem(ModbusController &controller, modbus::ModbusClientHub *parent, uint8_t address, + SensorItem *sensor) + : modbus::ModbusClientDevice(parent, address), + skip_updates(sensor->skip_updates), + start_address_(sensor->start_address), + register_count_(sensor->register_count), + function_code_(FunctionCode::CUSTOM), + custom_data_(&sensor->custom_data), + controller_(&controller) { + this->sensors.insert(sensor); } -// Queue incoming response -void ModbusController::on_response(std::span request_pdu, std::span response_pdu) { - if (this->command_queue_.empty()) { - ESP_LOGW(TAG, "Received modbus data but command queue is empty"); - return; +// The base deletes copy/move; command items re-provide construction. The moved-from device must not +// unregister the hub slot we just took over, so its parent_ is cleared. The copy constructor exists +// only for callers that pass an lvalue to queue_command() (in-tree callers move); remove it when +// queue_command() is removed. +ModbusCommandItem::ModbusCommandItem(const ModbusCommandItem &other) + : modbus::ModbusClientDevice(other.parent_, other.address_), + sensors(other.sensors), + skip_updates(other.skip_updates), + on_data_func(other.on_data_func), + register_type_(other.register_type_), + start_address_(other.start_address_), + register_count_(other.register_count_), + function_code_(other.function_code_), + custom_data_(other.custom_data_), + controller_(other.controller_) { + // SmallInlineBuffer is move-only, so deep-copy the bytes explicitly. + this->payload.set(other.payload.data(), other.payload.size()); +} + +ModbusCommandItem::ModbusCommandItem(ModbusCommandItem &&other) noexcept + : modbus::ModbusClientDevice(other.parent_, other.address_), + sensors(std::move(other.sensors)), + skip_updates(other.skip_updates), + on_data_func(std::move(other.on_data_func)), + payload(std::move(other.payload)), + register_type_(other.register_type_), + start_address_(other.start_address_), + register_count_(other.register_count_), + function_code_(other.function_code_), + custom_data_(other.custom_data_), + controller_(other.controller_) { + other.parent_ = nullptr; +} + +// A valid response: the device is online. Dispatch the payload to the handler or the range's sensors. +void ModbusCommandItem::on_response(std::span request_pdu, std::span response_pdu) { + if (this->controller_ != nullptr) + this->controller_->set_online(true, static_cast(this->function_code_), this->start_address_); + auto data = modbus::helpers::server_pdu_payload(response_pdu); + if (this->on_data_func) { + this->on_data_func(this->register_type_, this->start_address_, data); + } else if (modbus::helpers::is_function_code_write(static_cast(this->function_code_))) { + // write acknowledgement - nothing to publish + } else { + for (auto *sensor : this->sensors) + sensor->parse_and_publish(data); } - auto ¤t_command = this->command_queue_.front(); - if (current_command != nullptr) { + if (this->controller_ != nullptr) + this->controller_->unqueue_command(this); +} + +// An exception response is still a legitimate reply, so the device is considered online. +void ModbusCommandItem::on_error(std::span request_pdu, modbus::ExceptionCode exception_code) { + const uint8_t function_code = request_pdu.empty() ? 0 : request_pdu[0]; + ESP_LOGW(TAG, "Modbus error function code: 0x%X register 0x%X exception: %d", function_code, this->start_address_, + static_cast(exception_code)); + if (this->controller_ != nullptr) { + this->controller_->set_online(true, function_code, this->start_address_); + this->controller_->unqueue_command(this); + } +} + +// Not being sent says nothing about online/offline status; just drop it from the pending list. +void ModbusCommandItem::on_not_sent(std::span request_pdu) { + // A dropped write is lost while the entity has already published optimistically, so surface it. + if (modbus::helpers::is_function_code_write(static_cast(this->function_code_))) { + ESP_LOGW(TAG, "Write not sent: function 0x%X register 0x%X", static_cast(this->function_code_), + this->start_address_); + } + if (this->controller_ != nullptr) + this->controller_->unqueue_command(this); +} + +// Fired once per wire transmission (including hub re-queues from a retry), so the on_command_sent +// trigger reflects when the frame actually went out, not when it was queued. +void ModbusCommandItem::on_sent(std::span request_pdu) { + if (this->controller_ != nullptr) + this->controller_->command_sent(static_cast(this->function_code_), this->start_address_); +} + +bool ModbusCommandItem::on_no_response(std::span request_pdu) { + if (this->controller_ == nullptr) + return false; + this->controller_->increment_non_response_count(); + if (this->controller_->can_send()) { + // Have the hub re-queue the frame it is holding; on_sent fires again when it goes back out. + return true; + } + this->controller_->set_online(false, static_cast(this->function_code_), this->start_address_); + this->controller_->unqueue_command(this); + return false; +} + +void ModbusController::set_online(bool online, int function_code, int register_address) { + if (online) { + this->cmd_non_responses_ = 0; if (this->module_offline_) { ESP_LOGW(TAG, "Modbus device=%d back online", this->address_); - - if (this->offline_skip_updates_ > 0) { - // Restore skip_updates_counter to restore commands updates - for (auto &r : this->register_ranges_) { - r.skip_updates_counter = 0; - } - } - // Restore module online state this->module_offline_ = false; - this->online_callback_.call((int) current_command->function_code, current_command->register_address); + this->online_callback_.call(function_code, register_address); + } + } else { + // Offline is a property of the physical device, so drop every sender's queued frames for its + // address; retired frames get on_not_sent(), which reclaims one-shots through the normal path. + this->hub_->clear_tx_queue_for_address(this->address_); + if (!this->module_offline_) { + ESP_LOGW(TAG, "Modbus device=%d set offline", this->address_); + this->module_offline_ = true; + this->module_offline_at_ = this->update_counter_; + this->offline_callback_.call(function_code, register_address); } - - // Move the commandItem to the response queue. The span points into the hub's receive buffer, so - // copy the payload into the command for deferred processing in loop(). - auto data = modbus::helpers::server_pdu_payload(response_pdu); - current_command->payload.assign(data.begin(), data.end()); - this->incoming_queue_.push(std::move(current_command)); - ESP_LOGV(TAG, "Modbus response queued"); - this->command_queue_.pop_front(); } } -// Dispatch the response to the registered handler -void ModbusController::process_modbus_data_(const ModbusCommandItem *response) { - ESP_LOGV(TAG, "Process modbus response for address 0x%X size: %zu", response->register_address, - response->payload.size()); - response->on_data_func(response->register_type, response->register_address, response->payload); +void ModbusController::queue_command(ModbusCommandItem command) { + this->sweep_completed_one_shots_(); // reclaim finished one-shots before adding a new one + // Duplicates are the caller's to manage; the controller only holds the item until its terminal callback. + this->one_shot_command_items_.push_back(make_unique(std::move(command))); + // A refused frame gets no terminal callback (see the hub contract), so reclaim the item here. + auto &item = this->one_shot_command_items_.back(); + if (!item->send()) { + // The caller (e.g. a write entity) has usually already published optimistically - surface the loss. + ESP_LOGW(TAG, "Command refused by hub: type=0x%X address=0x%X", static_cast(item->register_type()), + item->register_address()); + item->pending_removal = true; + } } -void ModbusController::on_error(std::span request_pdu, modbus::ExceptionCode exception_code) { - // The request function code (request_pdu[0]) already carries what the log needs; the exception bit only - // ever appears on the response, so no masking is needed here. - const uint8_t function_code = request_pdu.empty() ? 0 : request_pdu[0]; - ESP_LOGE(TAG, "Modbus error function code: 0x%X exception: %d ", function_code, static_cast(exception_code)); - if (this->command_queue_.empty()) { +void ModbusController::unqueue_command(const ModbusCommandItem *command) { + // Called as the last action of the command's own callback, and from send() after send_pdu (which may + // synchronously call on_not_sent). Destroying `command` here would leave send() and the hub touching a + // freed object, so we only FLAG it; sweep_completed_one_shots_() erases it later at a safe point. No-op + // for polling commands (they persist and are not in the one-shot list). + for (auto &item : this->one_shot_command_items_) { + if (item.get() == command) { + item->pending_removal = true; + return; + } + } +} + +void ModbusController::sweep_completed_one_shots_() { + this->one_shot_command_items_.remove_if( + [](const std::unique_ptr &item) { return item->pending_removal; }); +} + +void ModbusController::update_range_(ModbusCommandItem &cmd) { + if (this->update_counter_ % (cmd.skip_updates + 1) != 0) { + ESP_LOGVV(TAG, "Skipping update for range 0x%X", cmd.register_address()); return; } - // Remove pending command waiting for a response - auto ¤t_command = this->command_queue_.front(); - if (current_command != nullptr) { - ESP_LOGE(TAG, - "Modbus error - last command: function code=0x%X register address = 0x%X " - "registers count=%d " - "payload size=%zu", - function_code, current_command->register_address, current_command->register_count, - current_command->payload.size()); - this->command_queue_.pop_front(); - } + // A refusal is already logged by the hub; note the affected range for controller-level diagnostics. + if (!cmd.send()) + ESP_LOGD(TAG, "Poll refused by hub for range 0x%X", cmd.register_address()); } -SensorSet ModbusController::find_sensors_(modbus::EntityType register_type, uint16_t start_address) const { - auto reg_it = std::find_if( - std::begin(this->register_ranges_), std::end(this->register_ranges_), - [=](RegisterRange const &r) { return (r.start_address == start_address && r.register_type == register_type); }); - - if (reg_it == this->register_ranges_.end()) { - ESP_LOGE(TAG, "No matching range for sensor found - start_address : 0x%X", start_address); - } else { - return reg_it->sensors; - } - - // not found - return {}; -} -void ModbusController::on_register_data(modbus::EntityType register_type, uint16_t start_address, - const std::vector &data) { - ESP_LOGV(TAG, "data for register address : 0x%X : ", start_address); - - // loop through all sensors with the same start address - auto sensors = find_sensors_(register_type, start_address); - for (auto *sensor : sensors) { - sensor->parse_and_publish(data); - } -} - -void ModbusController::queue_command(const ModbusCommandItem &command) { - if (!this->allow_duplicate_commands_) { - // check if this command is already qeued. - // not very effective but the queue is never really large - for (auto &item : this->command_queue_) { - if (item->is_equal(command)) { - ESP_LOGW(TAG, "Duplicate modbus command found: type=0x%x address=%u count=%u", - static_cast(command.register_type), command.register_address, command.register_count); - // update the payload of the queued command - // replaces a previous command - item->payload = command.payload; - return; - } - } - } - this->command_queue_.push_back(make_unique(command)); -} - -void ModbusController::update_range_(RegisterRange &r) { - ESP_LOGV(TAG, "Range : %X Size: %x (%d) skip: %d", r.start_address, r.register_count, (int) r.register_type, - r.skip_updates_counter); - if (r.skip_updates_counter == 0) { - // if a custom command is used the user supplied custom_data is only available in the SensorItem. - if (r.register_type == modbus::EntityType::CUSTOM) { - auto sensors = this->find_sensors_(r.register_type, r.start_address); - if (!sensors.empty()) { - auto sensor = sensors.cbegin(); - auto command_item = ModbusCommandItem::create_custom_command( - this, (*sensor)->custom_data, - [this](modbus::EntityType register_type, uint16_t start_address, const std::vector &data) { - this->on_register_data(modbus::EntityType::CUSTOM, start_address, data); - }); - command_item.register_address = (*sensor)->start_address; - command_item.register_count = (*sensor)->register_count; - command_item.function_code = FunctionCode::CUSTOM; - queue_command(command_item); +void ModbusController::update() { + this->sweep_completed_one_shots_(); // reclaim one-shots deferred out of their own callbacks + if (this->module_offline_) { + // Offline probing follows the offline cadence alone; per-range skip_updates resumes once the + // device is back online. Requiring both cadences to coincide would leave phase combinations + // where a probe never goes out. + if (offline_retry_due(this->update_counter_, this->module_offline_at_, this->offline_skip_updates_)) { + ESP_LOGV(TAG, "Module offline - retrying"); + this->cmd_non_responses_ = 0; // allow the probe through can_send() + for (auto &cmd : this->polling_command_items_) { + if (!cmd.send()) + ESP_LOGD(TAG, "Probe refused by hub for range 0x%X", cmd.register_address()); } } else { - queue_command(ModbusCommandItem::create_read_command(this, r.register_type, r.start_address, r.register_count)); + ESP_LOGV(TAG, "Module offline - skipping update"); } - r.skip_updates_counter = r.skip_updates; // reset counter to config value - } else { - r.skip_updates_counter--; - } -} -// -// Queue the modbus requests to be send. -// Once we get a response to the command it is removed from the queue and the next command is send -// -void ModbusController::update() { - if (!this->command_queue_.empty()) { - ESP_LOGV(TAG, "%zu modbus commands already in queue", this->command_queue_.size()); - } else { - ESP_LOGV(TAG, "Updating modbus component"); + this->update_counter_++; + return; } - for (auto &r : this->register_ranges_) { - ESP_LOGVV(TAG, "Updating range 0x%X", r.start_address); - update_range_(r); + if (this->can_send()) { + for (auto &cmd : this->polling_command_items_) { + ESP_LOGVV(TAG, "Updating range 0x%X", cmd.register_address()); + this->update_range_(cmd); + } } + this->update_counter_++; } // walk through the sensors and determine the register ranges to read -size_t ModbusController::create_register_ranges_() { - this->register_ranges_.clear(); +void ModbusController::create_polling_commands_() { if (this->sensorset_.empty()) { ESP_LOGW(TAG, "No sensors registered"); - return 0; + return; } - // iterator is sorted see SensorItemsComparator for details - auto ix = this->sensorset_.begin(); + // Sensors are walked in the sensor set's order (see SensorItemsComparator): register type, then + // force_new_range ahead of the rest, then address - so the walk is not purely address-ordered. + // Each keeps the address it was configured with; what is resolved here is its `offset`, the position + // of its data within the response of whichever range it ends up in. RegisterRange r = {}; - uint8_t buffer_offset = 0; + bool have_range = false; + // Set while the open range belongs to a force_new_range sensor: a range the user asked to keep + // separate must not quietly absorb other sensors. + bool range_forced = false; + // Set once a sensor has joined by sharing the range's start address, which widens the read. Only a + // widened range can absorb a later sensor by coverage: ranges that were kept apart before stay apart, + // so their frames and polling rates are untouched. + bool range_shared = false; + // Bytes the range's registers have consumed so far. An extending sensor starts after them, so a + // register that returns more bytes than its count implies pushes the sensors after it along. + // range_custom_size records whether any of them returns something other than two bytes per register, + // which is what makes a position inside the range impossible to work out from addresses alone. Coils + // count as such: they carry one bit per address, so bit ranges never take the coverage join. + size_t range_bytes = 0; + bool range_custom_size = false; SensorItem *prev = nullptr; - while (ix != this->sensorset_.end()) { - SensorItem *curr = *ix; + for (SensorItem *curr : this->sensorset_) { + ESP_LOGV(TAG, "Register: 0x%X count=%d size=%zu offset=%u skip=%u addr=%p", curr->start_address, + curr->register_count, curr->get_register_size(), curr->offset, curr->skip_updates, curr); - ESP_LOGV(TAG, "Register: 0x%X %d %d %zu offset=%u skip=%u addr=%p", curr->start_address, curr->register_count, - curr->offset, curr->get_register_size(), curr->offset, curr->skip_updates, curr); + const bool custom_size = curr->get_register_size() != static_cast(curr->register_count) * 2; - if (r.register_count == 0) { - // this is the first register in range + bool join = false; + if (have_range && !curr->force_new_range && r.register_type == curr->register_type && + curr->register_type != modbus::EntityType::CUSTOM) { + if (curr->start_address == (r.start_address + r.register_count - prev->register_count) && + prev->start_address + prev->register_count == r.start_address + r.register_count && + curr->register_count == prev->register_count && curr->get_register_size() == prev->get_register_size()) { + // A second sensor on the register(s) the previous one covers: it reads those same bytes, + // starting where that sensor's offset pointed, so a chain configured 0/2/4 resolves to 0/2/6. + // Both address tests matter. The first identifies the previous sensor's register by working back + // from the range's end, which only describes it while it actually sits there - hence the second. + // A sensor that joined mid-range must never anchor this, or the next one inherits its offset. + curr->offset = static_cast(prev->offset + curr->offset_from_start_address); + join = true; + ESP_LOGV(TAG, "Re-use previous register 0x%X", curr->start_address); + } else if (curr->start_address == (r.start_address + r.register_count)) { + // The next contiguous register(s): the data begins after what the range has consumed so far - + // the byte cursor for registers, the distance in bits for coils. + curr->offset = + static_cast((curr->addresses_bits() ? curr->start_address - r.start_address : range_bytes) + + curr->offset_from_start_address); + range_bytes += curr->get_register_size(); + range_custom_size = range_custom_size || custom_size; + r.register_count += curr->register_count; + join = true; + ESP_LOGV(TAG, "Extend range to include 0x%X", curr->start_address); + } else if (range_shared && !range_forced && curr->start_address >= r.start_address && + curr->start_address + curr->register_count <= r.start_address + r.register_count && + !range_custom_size && !custom_size && curr->skip_updates == r.skip_updates) { + // The registers already fall inside a range that a shared-address join widened, so this sensor + // reads its slice of that response instead of adding an overlapping second poll. The guards keep + // it narrow: only a widened range, never a force-isolated one; only where every register in the + // range returns two bytes, so interior positions follow from the addresses; only sensors genuinely + // inside it, which is why the lower bound is needed given the walk is not address-ordered; and + // only where the polling rates already match, since joining runs this sensor through the rate + // merge below and would otherwise change one of them. + const uint16_t addr_delta = curr->start_address - r.start_address; + curr->offset = static_cast((curr->addresses_bits() ? addr_delta : addr_delta * 2) + + curr->offset_from_start_address); + join = true; + ESP_LOGV(TAG, "Register 0x%X already covered by range 0x%X", curr->start_address, r.start_address); + } + } + + // Sensors on the same start address have to share one range: a response is dispatched to a single + // range per (start_address, register_type), so a second range with that key would never receive + // data. This holds for force_new_range and custom entities too. The read widens to cover whichever + // sensor needs the most registers, which also fixes a short read for coils that use offset. + if (!join && have_range && r.register_type == curr->register_type && r.start_address == curr->start_address) { + curr->offset = curr->offset_from_start_address; // shares the range start + r.register_count = std::max(r.register_count, curr->register_count); + range_bytes = std::max(range_bytes, curr->get_register_size()); + range_custom_size = range_custom_size || custom_size; + range_shared = true; + range_forced = range_forced || curr->force_new_range; + join = true; + ESP_LOGV(TAG, "Share range start 0x%X", curr->start_address); + } + + if (!join) { + if (have_range) { + ESP_LOGV(TAG, "Add range 0x%X %d skip:%d", r.start_address, r.register_count, r.skip_updates); + this->create_polling_command_(std::move(r)); + } + r = {}; + range_bytes = curr->get_register_size(); + range_custom_size = custom_size; + range_forced = curr->force_new_range; + range_shared = false; + curr->offset = curr->offset_from_start_address; r.start_address = curr->start_address; r.register_count = curr->register_count; r.register_type = curr->register_type; - r.sensors.insert(curr); r.skip_updates = curr->skip_updates; - r.skip_updates_counter = 0; - buffer_offset = curr->get_register_size(); - - ESP_LOGV(TAG, "Started new range"); - } else { - // this is not the first register in range so it might be possible - // to reuse the last register or extend the current range - if (!curr->force_new_range && r.register_type == curr->register_type && - curr->register_type != modbus::EntityType::CUSTOM) { - if (curr->start_address == (r.start_address + r.register_count - prev->register_count) && - curr->register_count == prev->register_count && curr->get_register_size() == prev->get_register_size()) { - // this register can re-use the data from the previous register - - // remove this sensore because start_address is changed (sort-order) - ix = this->sensorset_.erase(ix); - - curr->start_address = r.start_address; - curr->offset += prev->offset; - - this->sensorset_.insert(curr); - // move iterator backwards because it will be incremented later - ix--; - - ESP_LOGV(TAG, "Re-use previous register - change to register: 0x%X %d offset=%u", curr->start_address, - curr->register_count, curr->offset); - } else if (curr->start_address == (r.start_address + r.register_count)) { - // this register can extend the current range - - // remove this sensore because start_address is changed (sort-order) - ix = this->sensorset_.erase(ix); - - curr->start_address = r.start_address; - curr->offset += buffer_offset; - buffer_offset += curr->get_register_size(); - r.register_count += curr->register_count; - - this->sensorset_.insert(curr); - // move iterator backwards because it will be incremented later - ix--; - - ESP_LOGV(TAG, "Extend range - change to register: 0x%X %d offset=%u", curr->start_address, - curr->register_count, curr->offset); - } - } - } - - if (curr->start_address == r.start_address && curr->register_type == r.register_type) { - // use the lowest non zero value for the whole range - // Because zero is the default value for skip_updates it is excluded from getting the min value. - if (curr->skip_updates != 0) { - if (r.skip_updates != 0) { - r.skip_updates = std::min(r.skip_updates, curr->skip_updates); - } else { - r.skip_updates = curr->skip_updates; - } - } - - // add sensor to this range - r.sensors.insert(curr); - - ix++; - } else { - ESP_LOGV(TAG, "Add range 0x%X %d skip:%d", r.start_address, r.register_count, r.skip_updates); - this->register_ranges_.push_back(r); - r = {}; - buffer_offset = 0; - // do not increment the iterator here because the current sensor has to be re-evaluated + have_range = true; + } else if (curr->skip_updates != 0) { + // use the lowest non-zero skip_updates for the whole range (0 is the default and is excluded) + r.skip_updates = (r.skip_updates != 0) ? std::min(r.skip_updates, curr->skip_updates) : curr->skip_updates; } + // Every member records its range's first register. The resolved offset is relative to it, so the + // two together give the sensor's real position, and the address a write entity targets. + curr->range_start_address = r.start_address; + r.sensors.insert(curr); prev = curr; } - - if (r.register_count > 0) { - // Add the last range + if (have_range) { ESP_LOGV(TAG, "Add last range 0x%X %d skip:%d", r.start_address, r.register_count, r.skip_updates); - this->register_ranges_.push_back(r); + this->create_polling_command_(std::move(r)); } - - return this->register_ranges_.size(); + // Reclaim growth slack; safe here because nothing has registered with the hub yet (see the + // lifetime note on polling_command_items_). + this->polling_command_items_.shrink_to_fit(); } void ModbusController::dump_config() { @@ -326,222 +360,163 @@ void ModbusController::dump_config() { it->get_register_size()); } ESP_LOGCONFIG(TAG, "ranges"); - for (auto &it : this->register_ranges_) { - ESP_LOGCONFIG(TAG, " Range type=%u start=0x%X count=%d skip_updates=%d", static_cast(it.register_type), - it.start_address, it.register_count, it.skip_updates); + for (auto &it : this->polling_command_items_) { + ESP_LOGCONFIG(TAG, " Range type=%u start=0x%X count=%d skip_updates=%d", static_cast(it.register_type()), + it.register_address(), it.register_count(), it.skip_updates); } #endif } -void ModbusController::loop() { - // Incoming data to process? - if (!this->incoming_queue_.empty()) { - auto &message = this->incoming_queue_.front(); - if (message != nullptr) - this->process_modbus_data_(message.get()); - this->incoming_queue_.pop(); - +void ModbusController::on_write_register_response(EntityType register_type, uint16_t start_address, + std::span data) { + // A well-formed write ACK echoes address and value, but a truncated PDU yields a short/empty span. + if (data.size() >= 3) { + ESP_LOGV(TAG, "Command ACK 0x%X %d ", modbus::helpers::get_data(data.data(), 0), + modbus::helpers::get_data(data.data(), 1)); } else { - // all messages processed send pending commands - this->send_next_command_(); - } -} - -void ModbusController::on_write_register_response(modbus::EntityType register_type, uint16_t start_address, - const std::vector &data) { - ESP_LOGV(TAG, "Command ACK 0x%X %d ", modbus::helpers::get_data(data, 0), - modbus::helpers::get_data(data, 1)); -} - -void ModbusController::dump_sensors_() { - ESP_LOGV(TAG, "sensors"); - for (auto &it : this->sensorset_) { - ESP_LOGV(TAG, " Sensor start=0x%X count=%d size=%zu offset=%d", it->start_address, it->register_count, - it->get_register_size(), it->offset); + ESP_LOGV(TAG, "Command ACK (short payload, %zu bytes)", data.size()); } } ModbusCommandItem ModbusCommandItem::create_read_command( - ModbusController *modbusdevice, modbus::EntityType register_type, uint16_t start_address, uint16_t register_count, - std::function &data)> - &&handler) { - ModbusCommandItem cmd; - cmd.modbusdevice = modbusdevice; - cmd.register_type = register_type; - cmd.function_code = modbus::helpers::modbus_register_read_function(register_type); - cmd.register_address = start_address; - cmd.register_count = register_count; + ModbusController *modbusdevice, EntityType register_type, uint16_t start_address, uint16_t register_count, + std::function data)> &&handler) { + ModbusCommandItem cmd(*modbusdevice, modbusdevice->hub(), modbusdevice->device_address()); + cmd.set_command_(modbus::helpers::modbus_register_read_function(register_type), register_type, start_address, + register_count); cmd.on_data_func = std::move(handler); return cmd; } -ModbusCommandItem ModbusCommandItem::create_read_command(ModbusController *modbusdevice, - modbus::EntityType register_type, uint16_t start_address, - uint16_t register_count) { - ModbusCommandItem cmd; - cmd.modbusdevice = modbusdevice; - cmd.register_type = register_type; - cmd.function_code = modbus::helpers::modbus_register_read_function(register_type); - cmd.register_address = start_address; - cmd.register_count = register_count; - cmd.on_data_func = [modbusdevice](modbus::EntityType register_type, uint16_t start_address, - const std::vector &data) { - modbusdevice->on_register_data(register_type, start_address, data); - }; - return cmd; -} - ModbusCommandItem ModbusCommandItem::create_write_multiple_command(ModbusController *modbusdevice, uint16_t start_address, uint16_t register_count, const std::vector &values) { - ModbusCommandItem cmd; - cmd.modbusdevice = modbusdevice; - cmd.register_type = modbus::EntityType::HOLDING; - cmd.function_code = FunctionCode::WRITE_MULTIPLE_REGISTERS; - cmd.register_address = start_address; - cmd.register_count = register_count; - cmd.on_data_func = [modbusdevice, cmd](modbus::EntityType register_type, uint16_t start_address, - const std::vector &data) { - modbusdevice->on_write_register_response(cmd.register_type, start_address, data); + ModbusCommandItem cmd(*modbusdevice, modbusdevice->hub(), modbusdevice->device_address()); + cmd.set_command_(FunctionCode::WRITE_MULTIPLE_REGISTERS, EntityType::HOLDING, start_address, register_count); + cmd.on_data_func = [modbusdevice](EntityType register_type, uint16_t start_address, std::span data) { + modbusdevice->on_write_register_response(register_type, start_address, data); }; + uint8_t *p = cmd.payload.init(values.size() * 2); for (auto v : values) { auto decoded_value = decode_value(v); - cmd.payload.push_back(decoded_value[0]); - cmd.payload.push_back(decoded_value[1]); + *p++ = decoded_value[0]; + *p++ = decoded_value[1]; } return cmd; } ModbusCommandItem ModbusCommandItem::create_write_single_coil(ModbusController *modbusdevice, uint16_t address, bool value) { - ModbusCommandItem cmd; - cmd.modbusdevice = modbusdevice; - cmd.register_type = modbus::EntityType::COIL; - cmd.function_code = FunctionCode::WRITE_SINGLE_COIL; - cmd.register_address = address; - cmd.register_count = 1; - cmd.on_data_func = [modbusdevice, cmd](modbus::EntityType register_type, uint16_t start_address, - const std::vector &data) { - modbusdevice->on_write_register_response(cmd.register_type, start_address, data); + ModbusCommandItem cmd(*modbusdevice, modbusdevice->hub(), modbusdevice->device_address()); + cmd.set_command_(FunctionCode::WRITE_SINGLE_COIL, EntityType::COIL, address, 1); + cmd.on_data_func = [modbusdevice](EntityType register_type, uint16_t start_address, std::span data) { + modbusdevice->on_write_register_response(register_type, start_address, data); }; - cmd.payload.push_back(value ? 0xFF : 0); - cmd.payload.push_back(0); + uint8_t *p = cmd.payload.init(2); + p[0] = value ? 0xFF : 0; + p[1] = 0; return cmd; } ModbusCommandItem ModbusCommandItem::create_write_multiple_coils(ModbusController *modbusdevice, uint16_t start_address, const std::vector &values) { - ModbusCommandItem cmd; - cmd.modbusdevice = modbusdevice; - cmd.register_type = modbus::EntityType::COIL; - cmd.function_code = FunctionCode::WRITE_MULTIPLE_COILS; - cmd.register_address = start_address; - cmd.register_count = values.size(); - cmd.on_data_func = [modbusdevice, cmd](modbus::EntityType register_type, uint16_t start_address, - const std::vector &data) { - modbusdevice->on_write_register_response(cmd.register_type, start_address, data); + ModbusCommandItem cmd(*modbusdevice, modbusdevice->hub(), modbusdevice->device_address()); + cmd.set_command_(FunctionCode::WRITE_MULTIPLE_COILS, EntityType::COIL, start_address, values.size()); + cmd.on_data_func = [modbusdevice](EntityType register_type, uint16_t start_address, std::span data) { + modbusdevice->on_write_register_response(register_type, start_address, data); }; - uint8_t bitmask = 0; - int bitcounter = 0; + uint8_t *p = cmd.payload.init((values.size() + 7) / 8); + memset(p, 0, (values.size() + 7) / 8); + size_t bit = 0; for (auto coil : values) { if (coil) { - bitmask |= (1 << bitcounter); + p[bit / 8] |= (1 << (bit % 8)); } - bitcounter++; - if (bitcounter % 8 == 0) { - cmd.payload.push_back(bitmask); - bitmask = 0; - } - } - // add remaining bits - if (bitcounter % 8) { - cmd.payload.push_back(bitmask); + bit++; } return cmd; } ModbusCommandItem ModbusCommandItem::create_write_single_command(ModbusController *modbusdevice, uint16_t start_address, uint16_t value) { - ModbusCommandItem cmd; - cmd.modbusdevice = modbusdevice; - cmd.register_type = modbus::EntityType::HOLDING; - cmd.function_code = FunctionCode::WRITE_SINGLE_REGISTER; - cmd.register_address = start_address; - cmd.register_count = 1; // not used here anyways - cmd.on_data_func = [modbusdevice, cmd](modbus::EntityType register_type, uint16_t start_address, - const std::vector &data) { - modbusdevice->on_write_register_response(cmd.register_type, start_address, data); + ModbusCommandItem cmd(*modbusdevice, modbusdevice->hub(), modbusdevice->device_address()); + cmd.set_command_(FunctionCode::WRITE_SINGLE_REGISTER, EntityType::HOLDING, start_address, 1); + cmd.on_data_func = [modbusdevice](EntityType register_type, uint16_t start_address, std::span data) { + modbusdevice->on_write_register_response(register_type, start_address, data); }; auto decoded_value = decode_value(value); - cmd.payload.push_back(decoded_value[0]); - cmd.payload.push_back(decoded_value[1]); + uint8_t *p = cmd.payload.init(2); + p[0] = decoded_value[0]; + p[1] = decoded_value[1]; return cmd; } ModbusCommandItem ModbusCommandItem::create_custom_command( ModbusController *modbusdevice, const std::vector &values, - std::function &data)> - &&handler) { - ModbusCommandItem cmd; - cmd.modbusdevice = modbusdevice; - cmd.function_code = FunctionCode::CUSTOM; + std::function data)> &&handler) { + ModbusCommandItem cmd(*modbusdevice, modbusdevice->hub(), modbusdevice->device_address()); + cmd.function_code_ = FunctionCode::CUSTOM; if (handler == nullptr) { - cmd.on_data_func = [](modbus::EntityType register_type, uint16_t start_address, const std::vector &data) { + cmd.on_data_func = [](EntityType register_type, uint16_t start_address, std::span data) { ESP_LOGI(TAG, "Custom Command sent"); }; } else { cmd.on_data_func = handler; } - cmd.payload = values; + cmd.payload.set(values.data(), values.size()); return cmd; } ModbusCommandItem ModbusCommandItem::create_custom_command( ModbusController *modbusdevice, const std::vector &values, - std::function &data)> - &&handler) { - ModbusCommandItem cmd = {}; - cmd.modbusdevice = modbusdevice; - cmd.function_code = FunctionCode::CUSTOM; + std::function data)> &&handler) { + ModbusCommandItem cmd(*modbusdevice, modbusdevice->hub(), modbusdevice->device_address()); + cmd.function_code_ = FunctionCode::CUSTOM; if (handler == nullptr) { - cmd.on_data_func = [](modbus::EntityType register_type, uint16_t start_address, const std::vector &data) { + cmd.on_data_func = [](EntityType register_type, uint16_t start_address, std::span data) { ESP_LOGI(TAG, "Custom Command sent"); }; } else { cmd.on_data_func = handler; } + uint8_t *p = cmd.payload.init(values.size() * 2); for (auto v : values) { - cmd.payload.push_back((v >> 8) & 0xFF); - cmd.payload.push_back(v & 0xFF); + *p++ = (v >> 8) & 0xFF; + *p++ = v & 0xFF; } return cmd; } bool ModbusCommandItem::send() { - if (this->function_code != FunctionCode::CUSTOM) { - modbusdevice->send_pdu( - modbus::helpers::create_client_pdu(this->function_code, this->register_address, this->register_count, - this->payload.empty() ? nullptr : &this->payload[0], this->payload.size())); + bool accepted; + if (this->function_code_ != FunctionCode::CUSTOM) { + accepted = this->send_pdu(modbus::helpers::create_client_pdu( + this->function_code_, this->start_address_, this->register_count_, + this->payload.empty() ? nullptr : this->payload.data(), this->payload.size())); } else { - modbusdevice->send_raw(this->payload); + // Custom command: the bytes are a complete raw frame (address + PDU). Send the PDU to the frame's own + // address (which may differ from this controller's); the hub appends the CRC and routes the response + // back to this item by pointer. (send_raw() is deprecated, so send_pdu() is called with the extracted + // address. Raw-frame semantics are kept here; the custom_pdu migration is a later step.) + std::span frame = + this->custom_data_ != nullptr ? std::span(*this->custom_data_) : this->payload; + if (frame.empty()) { + ESP_LOGW(TAG, "Empty custom command frame, not sent"); + accepted = false; + } else { + accepted = this->parent_->send_pdu(frame[0], frame.subspan(1), this); + } } - this->send_count_++; - ESP_LOGV(TAG, "Command sent %d 0x%X %d send_count: %d", uint8_t(this->function_code), this->register_address, - this->register_count, this->send_count_); - return true; -} - -bool ModbusCommandItem::is_equal(const ModbusCommandItem &other) { - // for custom commands we have to check for identical payloads, since - // address/count/type fields will be set to zero - return this->function_code == FunctionCode::CUSTOM - ? this->payload == other.payload - : other.register_address == this->register_address && other.register_count == this->register_count && - other.register_type == this->register_type && other.function_code == this->function_code; + // The on_command_sent trigger fires from on_sent() when the frame actually reaches the wire. + if (accepted) { + ESP_LOGV(TAG, "Command queued %d 0x%X %d", uint8_t(this->function_code_), this->start_address_, + this->register_count_); + } + return accepted; } } // namespace esphome::modbus_controller diff --git a/esphome/components/modbus_controller/modbus_controller.h b/esphome/components/modbus_controller/modbus_controller.h index 3c789936af..fb0037a0e6 100644 --- a/esphome/components/modbus_controller/modbus_controller.h +++ b/esphome/components/modbus_controller/modbus_controller.h @@ -7,7 +7,6 @@ #include "esphome/core/automation.h" #include -#include #include #include #include @@ -17,6 +16,7 @@ namespace esphome::modbus_controller { class ModbusController; +using modbus::EntityType; using modbus::ExceptionCode; using modbus::FunctionCode; using modbus::helpers::SensorValueType; @@ -72,12 +72,32 @@ T get_data(const std::vector &data, size_t buffer_offset) { return modbus::helpers::get_data(data, buffer_offset); } +// Span overloads of the deprecated helpers below: read lambdas receive their payload as a +// std::span (previously a const std::vector &), and a span does not convert to +// a vector, so existing lambdas calling these by name need an overload that accepts one. These carry +// this release's deprecation window, since the span forms only exist from it. +// payload_to_number() deliberately has no such overload: one of its arguments is a modbus::helpers +// type, so a span call already reaches the helper by argument-dependent lookup, and a forwarder here +// would only make that call ambiguous. +// Remove before 2027.2.0. +template +ESPDEPRECATED("Use modbus::helpers::get_data() instead. Removed in 2027.2.0", "2026.8.0") +T get_data(std::span data, size_t buffer_offset) { + return modbus::helpers::get_data(data.data(), buffer_offset); +} + // Remove before 2027.2.0 (window restarted when the migration target changed to bit_from_packed()) ESPDEPRECATED("Use modbus::helpers::bit_from_packed() instead. Removed in 2027.2.0", "2026.4.0") inline bool coil_from_vector(int coil, const std::vector &data) { return modbus::helpers::bit_from_packed(coil, data); } +// Remove before 2027.2.0 +ESPDEPRECATED("Use modbus::helpers::bit_from_packed() instead. Removed in 2027.2.0", "2026.8.0") +inline bool coil_from_vector(int coil, std::span data) { + return modbus::helpers::bit_from_packed(coil, data); +} + template ESPDEPRECATED("Use modbus::helpers::mask_and_shift_by_rightbit() instead. Removed in 2026.10.0", "2026.4.0") N mask_and_shift_by_rightbit(N data, uint32_t mask) { @@ -107,11 +127,39 @@ class ModbusController; class SensorItem { public: - virtual void parse_and_publish(const std::vector &data) = 0; + /// Parse this sensor's slice out of its range's response and publish it. The span points into the + /// response buffer and is only valid for the duration of the call. Read the sensor's data from + /// `offset` within it. + virtual void parse_and_publish(std::span data) = 0; + + /// Coils and discrete inputs address individual bits; every other type addresses 16-bit registers. + bool addresses_bits() const { return modbus::helpers::is_entity_type_binary(this->register_type); } + + /// Address a write entity (switch/number/select) targets, derived from its resolved position within + /// the range so that a write lands on the register the sensor reads from. + uint16_t write_address() const { + return this->range_start_address + (this->addresses_bits() ? this->offset : this->offset / 2); + } + + /// Records the offset as configured, and seeds the resolved position with it. Building the ranges + /// overwrites `offset` with the position within the range; an item that is never polled keeps this + /// value, which is what its own address arithmetic expects. + void set_offset_from_start_address(uint8_t offset) { + this->offset_from_start_address = offset; + this->offset = offset; + } + + /// Sets the configured address, and points the range base at it. Building the ranges moves the base + /// to the range's first register; an item that is never polled (an output, or a switch with + /// assumed_state) keeps its own address, so write_address() stays correct for it. + void set_address(uint16_t address) { + this->start_address = address; + this->range_start_address = address; + } void set_custom_data(const std::vector &data) { custom_data = data; } size_t virtual get_register_size() const { - if (register_type == modbus::EntityType::COIL || register_type == modbus::EntityType::DISCRETE_INPUT) { + if (this->addresses_bits()) { return 1; } else { // if CONF_RESPONSE_BYTES is used override the default return response_bytes > 0 ? response_bytes : register_count * 2; @@ -123,15 +171,27 @@ class SensorItem { SensorValueType sensor_value_type{SensorValueType::RAW}; uint16_t start_address{0}; uint32_t bitmask{0}; + /// Position of this sensor's data within its range's response - a byte offset for registers, a bit + /// index for coils and discrete inputs. Resolved while the ranges are built, so it already accounts + /// for the registers ahead of it (including wide response_size ones) and for any offset inherited + /// from an earlier sensor sharing the same register. uint8_t offset{0}; uint8_t register_count{0}; uint8_t response_bytes{0}; + /// The offset exactly as configured: measured from this sensor's own start_address, where `offset` + /// is measured from the first register of the range it ends up polled in. Same units as `offset` - + /// bytes for registers, bits for coils and discrete inputs. Kept so the resolution can be recomputed, + /// and so the sort order of the sensor set never depends on the resolved value. + /// Declared before range_start_address so it lands in the padding after response_bytes. + uint8_t offset_from_start_address{0}; + /// First register of the range this sensor is polled in; equals start_address for an unpolled item. + uint16_t range_start_address{0}; uint16_t skip_updates{0}; std::vector custom_data{}; bool force_new_range{false}; }; -// ModbusController::create_register_ranges_ tries to optimize register range +// ModbusController::create_polling_commands_ tries to optimize register range // for this the sensors must be ordered by register_type, start_address and bitmask class SensorItemsComparator { public: @@ -151,9 +211,11 @@ class SensorItemsComparator { return lhs->start_address < rhs->start_address; } - // sort by offset (ensures update of sensors in ascending order) - if (lhs->offset != rhs->offset) { - return lhs->offset < rhs->offset; + // sort by the offset as configured (ensures update of sensors in ascending order). The resolved + // `offset` is deliberately not used: ranges are built while iterating this set and assign it, and + // a sort key that changed under the iteration would corrupt the set's ordering. + if (lhs->offset_from_start_address != rhs->offset_from_start_address) { + return lhs->offset_from_start_address < rhs->offset_from_start_address; } // The pointer to the sensor is used last to ensure that @@ -168,25 +230,64 @@ struct RegisterRange { uint16_t start_address; modbus::EntityType register_type; uint8_t register_count; - uint16_t skip_updates; // the config value - SensorSet sensors; // all sensors of this range - uint16_t skip_updates_counter; // the running value + uint16_t skip_updates; // the config value + SensorSet sensors; // all sensors of this range }; -class ModbusCommandItem { +/// A single modbus command. Each command is its own ModbusClientDevice: it sends its frame to the hub +/// and the hub routes the response back to this object's on_modbus_* callbacks, so the controller no +/// longer has to match responses to a FIFO queue. +class ModbusCommandItem : public modbus::ModbusClientDevice { public: - static const size_t MAX_PAYLOAD_BYTES = 240; - ModbusController *modbusdevice{nullptr}; - uint16_t register_address{0}; - uint16_t register_count{0}; - FunctionCode function_code{FunctionCode::CUSTOM}; - modbus::EntityType register_type{modbus::EntityType::CUSTOM}; - std::function &data)> - on_data_func; - std::vector payload = {}; + /// Empty command with no controller connection (kept for source compatibility with value-type usage). + ModbusCommandItem(ModbusController &controller, modbus::ModbusClientHub *parent, uint8_t address) + : modbus::ModbusClientDevice(parent, address), controller_(&controller) {} + /// Read command built from a range; the read PDU is rebuilt from these fields at send time. + ModbusCommandItem(ModbusController &controller, modbus::ModbusClientHub *parent, uint8_t address, + RegisterRange &&range); + /// Custom polling command: the PDU bytes are referenced from the sensor (not copied); responses are + /// dispatched to that sensor. + ModbusCommandItem(ModbusController &controller, modbus::ModbusClientHub *parent, uint8_t address, SensorItem *sensor); + + // The base deletes copy/move (its destructor unregisters the device from the hub queue), but command + // items are stored in value containers, so copy/move CONSTRUCTION is re-provided (copy only for the + // queue_command() path). Assignment stays deleted: the item's address-in-memory is its hub identity. + ModbusCommandItem(const ModbusCommandItem &other); + ModbusCommandItem(ModbusCommandItem &&other) noexcept; + ModbusCommandItem &operator=(ModbusCommandItem &&) = delete; + + SensorSet sensors; // sensors served by this command (empty for factory/write commands) + uint16_t skip_updates{0}; + std::function data)> on_data_func; + /// Write data bytes for the command (register/coil values), or the raw frame of a one-shot custom + /// command; reads leave it empty. Small-buffer optimized: fixed-size commands (single-register/coil + /// writes) fit in the 8-byte inline buffer with no heap; only large multi-register or custom frames + /// spill to a single one-time heap allocation. This keeps runtime one-shot writes off the heap without + /// reserving a max-size buffer per command item. + SmallInlineBuffer<8> payload; + // Set by unqueue_command() when this one-shot has completed. The controller erases flagged items at a + // safe point (update()/queue_command()), never from inside the command's own callback. + bool pending_removal{false}; + + /// called when a modbus response was parsed without errors + void on_response(std::span request_pdu, std::span response_pdu) override; + /// called when a modbus error (exception) response was received + void on_error(std::span request_pdu, modbus::ExceptionCode exception_code) override; + /// called when the command could not be sent + void on_not_sent(std::span request_pdu) override; + /// called when the command's frame is actually written to the wire; fires the on_command_sent trigger + void on_sent(std::span request_pdu) override; + /// called on timeout; returns true to have the hub re-queue the frame for a retry + bool on_no_response(std::span request_pdu) override; + + uint16_t register_address() const { return this->start_address_; } + uint16_t register_count() const { return this->register_count_; } + EntityType register_type() const { return this->register_type_; } + + /// Queue this command's frame on the hub. Returns false when refused, in which case no callback ever comes. + /// The item is the hub device, so it must stay alive until its terminal callback; a destroyed item's + /// pending frame is silently retired. bool send(); - /// Check if the command should be retried based on the max_retries parameter - bool should_retry(uint8_t max_retries) { return this->send_count_ <= max_retries; }; /// factory methods /** Create modbus read command @@ -199,19 +300,8 @@ class ModbusCommandItem { * @return ModbusCommandItem with the prepared command */ static ModbusCommandItem create_read_command( - ModbusController *modbusdevice, modbus::EntityType register_type, uint16_t start_address, uint16_t register_count, - std::function &data)> - &&handler); - /** Create modbus read command - * Function code 02-04 - * @param modbusdevice pointer to the device to execute the command - * @param function_code modbus function code for the read command - * @param start_address modbus address of the first register to read - * @param register_count number of registers to read - * @return ModbusCommandItem with the prepared command - */ - static ModbusCommandItem create_read_command(ModbusController *modbusdevice, modbus::EntityType register_type, - uint16_t start_address, uint16_t register_count); + ModbusController *modbusdevice, EntityType register_type, uint16_t start_address, uint16_t register_count, + std::function data)> &&handler); /** Create modbus read command * Function code 02-04 * @param modbusdevice pointer to the device to execute the command @@ -260,8 +350,8 @@ class ModbusCommandItem { */ static ModbusCommandItem create_custom_command( ModbusController *modbusdevice, const std::vector &values, - std::function &data)> - &&handler = nullptr); + std::function data)> &&handler = + nullptr); /** Create custom modbus command * @param modbusdevice pointer to the device to execute the command @@ -272,17 +362,33 @@ class ModbusCommandItem { */ static ModbusCommandItem create_custom_command( ModbusController *modbusdevice, const std::vector &values, - std::function &data)> - &&handler = nullptr); - - bool is_equal(const ModbusCommandItem &other); + std::function data)> &&handler = + nullptr); protected: - // wrong commands (esp. custom commands) can block the send queue, limit the number of repeats. - /// How many times this command has been sent - uint8_t send_count_{0}; + void set_command_(FunctionCode function_code, EntityType register_type, uint16_t start_address, + uint16_t register_count) { + this->function_code_ = function_code; + this->register_type_ = register_type; + this->start_address_ = start_address; + this->register_count_ = register_count; + } + EntityType register_type_{EntityType::CUSTOM}; + uint16_t start_address_{0}; + uint16_t register_count_{0}; + FunctionCode function_code_{FunctionCode::CUSTOM}; + /// Custom polling commands reference the PDU bytes owned by their SensorItem instead of copying them. + const std::vector *custom_data_{nullptr}; + ModbusController *controller_{nullptr}; }; +/// Whether an offline probe is due this update cycle: every offline_skip_updates + 1 cycles, +/// anchored at the cycle the device went offline. Pure so the cadence (including update_counter +/// wraparound) can be unit tested; used by ModbusController::update(). +inline bool offline_retry_due(uint16_t update_counter, uint16_t module_offline_at, uint16_t offline_skip_updates) { + return static_cast(update_counter + 1 - module_offline_at) % (offline_skip_updates + 1) == 0; +} + /** Modbus controller class. * Each instance handles the modbus commuinication for all sensors with the same modbus address * @@ -291,48 +397,46 @@ class ModbusCommandItem { * Responses for the commands are dispatched to the modbus sensor items. */ -class ModbusController final : public PollingComponent, public modbus::ModbusClientDevice { +class ModbusController final : public PollingComponent { public: void dump_config() override; - void loop() override; + // No loop() override: the hub owns transmit/receive timing and each command routes its own + // response, so the controller never joins the looping components at all. void setup() override; void update() override; - /// queues a modbus command in the send queue - void queue_command(const ModbusCommandItem &command); - /// Sends a raw payload (address byte + PDU, no CRC) with responses routed back to this controller. - /// The payload carries its own address byte, which may differ from this controller's address. - /// Deliberately shadows the deprecated ModbusClientDevice::send_raw() with identical semantics: - /// controller-level raw sends stay supported until the command machinery is replaced. - void send_raw(const std::vector &payload) { - if (payload.empty()) - return; // refused at the door, like every invalid send; no callback follows - this->parent_->send_pdu(payload[0], std::span(payload).subspan(1), this); - } + // The controller is not itself a modbus device - its commands and writer entities send as their own + // devices. It only owns the hub + address so those senders can be built against them. + void set_parent(modbus::ModbusClientHub *hub) { this->hub_ = hub; } + void set_address(uint8_t address) { this->address_ = address; } + + /// The hub and modbus address this controller talks to. Used to build commands/entities that send as + /// their own device. + modbus::ModbusClientHub *hub() const { return this->hub_; } + uint8_t device_address() const { return this->address_; } + + /// Queues a one-shot modbus command (writes, custom commands); taken by value, so std::move to avoid a copy. + void queue_command(ModbusCommandItem command); + /// Flags a finished one-shot command for removal. Called by the command as the last action of its own + /// callback, so the item is not destroyed here (send() and the hub still touch it) but swept later. + void unqueue_command(const ModbusCommandItem *command); /// Registers a sensor with the controller. Called by esphomes code generator void add_sensor_item(SensorItem *item) { sensorset_.insert(item); } - /// called when a modbus response was parsed without errors - void on_response(std::span request_pdu, std::span response_pdu) override; - /// called when a modbus error response was received - void on_error(std::span request_pdu, modbus::ExceptionCode exception_code) override; - /// default delegate called by process_modbus_data when a response has retrieved from the incoming queue - void on_register_data(modbus::EntityType register_type, uint16_t start_address, const std::vector &data); - /// default delegate called by process_modbus_data when a response for a write response has retrieved from the - /// incoming queue - void on_write_register_response(modbus::EntityType register_type, uint16_t start_address, - const std::vector &data); - /// Allow a duplicate command to be sent - void set_allow_duplicate_commands(bool allow_duplicate_commands) { - this->allow_duplicate_commands_ = allow_duplicate_commands; + /// Handles a write command acknowledgement (used by write command on_data_func handlers). + void on_write_register_response(EntityType register_type, uint16_t start_address, std::span data); + /// Update the online/offline state after a response or a run of timeouts, firing the callbacks. + void set_online(bool online, int function_code, int register_address); + /// Fire the on_command_sent trigger (called when a command's frame reaches the wire). + void command_sent(int function_code, int register_address) { + this->command_sent_callback_.call(function_code, register_address); } - /// get if a duplicate command can be sent - bool get_allow_duplicate_commands() { return this->allow_duplicate_commands_; } - /// called by esphome generated code to set the command_throttle period - void set_command_throttle(uint16_t command_throttle) { this->command_throttle_ = command_throttle; } + /// A command timed out; bump the consecutive-timeout counter used by can_send()/offline detection. + void increment_non_response_count() { this->cmd_non_responses_++; } + /// Whether more retries are allowed before the device is considered offline. Deliberately pooled + /// per device, not per command: online/offline is a property of the physical device. + bool can_send() { return this->cmd_non_responses_ <= this->max_cmd_retries_; } /// called by esphome generated code to set the offline_skip_updates void set_offline_skip_updates(uint16_t offline_skip_updates) { this->offline_skip_updates_ = offline_skip_updates; } - /// get the number of queued modbus commands (should be mostly empty) - size_t get_command_queue_length() { return command_queue_.size(); } /// get if the module is offline, didn't respond the last command bool get_module_offline() { return module_offline_; } /// Set callback for commands @@ -354,33 +458,48 @@ class ModbusController final : public PollingComponent, public modbus::ModbusCli protected: /// parse sensormap_ and create range of sequential addresses - size_t create_register_ranges_(); - // find register in sensormap. Returns iterator with all registers having the same start address - SensorSet find_sensors_(modbus::EntityType register_type, uint16_t start_address) const; - /// submit the read command for the address range to the send queue - void update_range_(RegisterRange &r); - /// parse incoming modbus data - void process_modbus_data_(const ModbusCommandItem *response); - /// send the next modbus command from the send queue - bool send_next_command_(); - /// dump the parsed sensormap for diagnostics - void dump_sensors_(); + /// Group the registered sensors into contiguous ranges and create one polling command per range. + void create_polling_commands_(); + /// build one persistent polling command from a range and add it to polling_command_items_ + void create_polling_command_(RegisterRange &&range) { + // A custom range polls the first sensor's custom_data (a ready-made raw frame); it needs the + // sensor constructor so the command references those bytes and decodes the real function code. + // The response still dispatches to every sensor in the range. + if (range.register_type == EntityType::CUSTOM && !range.sensors.empty()) { + auto &cmd = this->polling_command_items_.emplace_back(*this, this->hub_, this->address_, *range.sensors.begin()); + cmd.sensors = std::move(range.sensors); + cmd.skip_updates = range.skip_updates; // the range's merged rate, not the first sensor's + } else { + this->polling_command_items_.emplace_back(*this, this->hub_, this->address_, std::move(range)); + } + } + /// send a range's polling command if it is due this update + void update_range_(ModbusCommandItem &cmd); + /// The hub this controller's commands/entities send through, and the modbus address they target. + modbus::ModbusClientHub *hub_{nullptr}; + uint8_t address_{0}; /// Collection of all sensors for this component SensorSet sensorset_; - /// Continuous range of modbus registers - std::vector register_ranges_{}; - /// Hold the pending requests to be sent - std::list> command_queue_; - /// modbus response data waiting to get processed - std::queue> incoming_queue_; - /// if duplicate commands can be sent - bool allow_duplicate_commands_{false}; - /// when was the last send operation - uint32_t last_command_timestamp_{0}; - /// min time in ms between sending modbus commands - uint16_t command_throttle_{0}; + /// One persistent command per register range, each its own ModbusClientDevice. Built once in setup() + /// (create_polling_commands_ feeds each range straight in; the vector may reallocate as it grows, which + /// is safe because no command has registered with the hub yet) and never appended to afterward, so the + /// hub's device pointers stay valid once commands start sending. + std::vector polling_command_items_{}; + /// Dynamically queued one-shot commands (writes, custom commands). std::list keeps stable addresses. + std::list> one_shot_command_items_; + /// Erases one-shot commands flagged by unqueue_command(). Safe even when reached from inside a hub + /// callback (via an on_online/on_offline/on_command_sent automation that queues a command): the + /// destructor detaches via clear_tx_queue_for_device(), which the hub allows from callbacks, and the + /// item running its callback is not flagged until that callback returns. + void sweep_completed_one_shots_(); /// if module didn't respond the last command bool module_offline_{false}; + /// update_counter_ value at which the module went offline (for offline_skip_updates timing) + uint16_t module_offline_at_{0}; + /// counts update() cycles; drives skip_updates and offline timing + uint16_t update_counter_{0}; + /// consecutive non-responses; drives can_send() and offline detection + uint8_t cmd_non_responses_{0}; /// how many updates to skip if module is offline uint16_t offline_skip_updates_{0}; /// How many times we will retry a command if we get no response @@ -398,9 +517,8 @@ class ModbusController final : public PollingComponent, public modbus::ModbusCli * @param item SensorItem object * @return float value of data */ -inline float payload_to_float(std::span data, const SensorItem &item) { - int64_t number = - modbus::helpers::payload_to_number(data, item.sensor_value_type, item.offset, item.bitmask).value_or(0); +inline float payload_to_float(std::span data, const SensorItem &item, uint8_t offset) { + int64_t number = modbus::helpers::payload_to_number(data, item.sensor_value_type, offset, item.bitmask).value_or(0); float float_value; if (modbus::helpers::value_type_is_float(item.sensor_value_type)) { @@ -412,4 +530,12 @@ inline float payload_to_float(std::span data, const SensorItem &i return float_value; } +// Remove before 2027.2.0 (window opened when this helper gained an explicit offset). item.offset is +// the item's resolved position within its range's response, so this decodes the same bytes as passing +// that offset explicitly. +ESPDEPRECATED("Pass the offset explicitly: payload_to_float(data, item, item.offset). Removed in 2027.2.0", "2026.8.0") +inline float payload_to_float(std::span data, const SensorItem &item) { + return payload_to_float(data, item, item.offset); +} + } // namespace esphome::modbus_controller diff --git a/esphome/components/modbus_controller/number/modbus_number.cpp b/esphome/components/modbus_controller/number/modbus_number.cpp index 7b18b9e9fc..7903b2e317 100644 --- a/esphome/components/modbus_controller/number/modbus_number.cpp +++ b/esphome/components/modbus_controller/number/modbus_number.cpp @@ -10,8 +10,8 @@ static const char *const TAG = "modbus.number"; // Maximum uint16_t registers to log in verbose hex output static constexpr size_t MODBUS_NUMBER_MAX_LOG_REGISTERS = 32; -void ModbusNumber::parse_and_publish(const std::vector &data) { - float result = payload_to_float(data, *this) / this->multiply_by_; +void ModbusNumber::parse_and_publish(std::span data) { + float result = payload_to_float(data, *this, this->offset) / this->multiply_by_; // Is there a lambda registered // call it with the pre converted value and the raw data array @@ -29,7 +29,7 @@ void ModbusNumber::parse_and_publish(const std::vector &data) { } void ModbusNumber::control(float value) { - ModbusCommandItem write_cmd; + optional write_cmd; std::vector data; float write_value = value; // Is there are lambda configured? @@ -55,11 +55,11 @@ void ModbusNumber::control(float value) { #endif ESP_LOGV(TAG, "Modbus Number write raw: %s", format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); - write_cmd = ModbusCommandItem::create_custom_command( + write_cmd.emplace(ModbusCommandItem::create_custom_command( this->parent_, data, - [this, write_cmd](modbus::EntityType register_type, uint16_t start_address, const std::vector &data) { - this->parent_->on_write_register_response(write_cmd.register_type, this->start_address, data); - }); + [this](modbus::EntityType register_type, uint16_t start_address, std::span data) { + this->parent_->on_write_register_response(register_type, this->start_address, data); + })); } else { std::vector payload; modbus::helpers::float_to_payload(payload, write_value, this->sensor_value_type); @@ -70,22 +70,21 @@ void ModbusNumber::control(float value) { // Create and send the write command if (this->register_count == 1 && !this->use_write_multiple_) { - // since offset is in bytes and a register is 16 bits we get the start by adding offset/2 - write_cmd = ModbusCommandItem::create_write_single_command(this->parent_, this->start_address + this->offset / 2, - payload[0]); + write_cmd.emplace( + ModbusCommandItem::create_write_single_command(this->parent_, this->write_address(), payload[0])); } else { - write_cmd = ModbusCommandItem::create_write_multiple_command( - this->parent_, this->start_address + this->offset / 2, this->register_count, payload); + write_cmd.emplace(ModbusCommandItem::create_write_multiple_command(this->parent_, this->write_address(), + this->register_count, payload)); } // publish new value - write_cmd.on_data_func = [this, write_cmd, value](modbus::EntityType register_type, uint16_t start_address, - const std::vector &data) { + write_cmd->on_data_func = [this, value](modbus::EntityType register_type, uint16_t start_address, + std::span data) { // gets called when the write command is ack'd from the device - this->parent_->on_write_register_response(write_cmd.register_type, start_address, data); + this->parent_->on_write_register_response(register_type, start_address, data); this->publish_state(value); }; } - this->parent_->queue_command(write_cmd); + this->parent_->queue_command(std::move(*write_cmd)); this->publish_state(value); } void ModbusNumber::dump_config() { LOG_NUMBER(TAG, "Modbus Number", this); } diff --git a/esphome/components/modbus_controller/number/modbus_number.h b/esphome/components/modbus_controller/number/modbus_number.h index 582b042caf..1f0d0581eb 100644 --- a/esphome/components/modbus_controller/number/modbus_number.h +++ b/esphome/components/modbus_controller/number/modbus_number.h @@ -15,8 +15,8 @@ class ModbusNumber final : public number::Number, public Component, public Senso ModbusNumber(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint32_t bitmask, SensorValueType value_type, int register_count, uint16_t skip_updates, bool force_new_range) { this->register_type = register_type; - this->start_address = start_address; - this->offset = offset; + this->set_address(start_address); + this->set_offset_from_start_address(offset); this->bitmask = bitmask; this->sensor_value_type = value_type; this->register_count = register_count; @@ -25,12 +25,12 @@ class ModbusNumber final : public number::Number, public Component, public Senso }; void dump_config() override; - void parse_and_publish(const std::vector &data) override; + void parse_and_publish(std::span data) override; float get_setup_priority() const override { return setup_priority::HARDWARE; } void set_parent(ModbusController *parent) { this->parent_ = parent; } void set_write_multiply(float factor) { this->multiply_by_ = factor; } - using transform_func_t = optional (*)(ModbusNumber *, float, const std::vector &); + using transform_func_t = optional (*)(ModbusNumber *, float, std::span); using write_transform_func_t = optional (*)(ModbusNumber *, float, std::vector &); void set_template(transform_func_t f) { this->transform_func_ = f; } void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } diff --git a/esphome/components/modbus_controller/output/modbus_output.cpp b/esphome/components/modbus_controller/output/modbus_output.cpp index 95618a7505..48249f4387 100644 --- a/esphome/components/modbus_controller/output/modbus_output.cpp +++ b/esphome/components/modbus_controller/output/modbus_output.cpp @@ -58,15 +58,15 @@ void ModbusFloatOutput::write_state(float value) { } // Create and send the write command - ModbusCommandItem write_cmd; + optional write_cmd; if (this->register_count == 1 && !this->use_write_multiple_) { - write_cmd = - ModbusCommandItem::create_write_single_command(this->parent_, this->start_address + this->offset, data[0]); + write_cmd.emplace( + ModbusCommandItem::create_write_single_command(this->parent_, this->start_address + this->offset, data[0])); } else { - write_cmd = ModbusCommandItem::create_write_multiple_command(this->parent_, this->start_address + this->offset, - data.size(), data); + write_cmd.emplace(ModbusCommandItem::create_write_multiple_command( + this->parent_, this->start_address + this->offset, data.size(), data)); } - this->parent_->queue_command(write_cmd); + this->parent_->queue_command(std::move(*write_cmd)); } void ModbusFloatOutput::dump_config() { @@ -82,7 +82,7 @@ void ModbusFloatOutput::dump_config() { // ModbusBinaryOutput void ModbusBinaryOutput::write_state(bool state) { // This will be called every time the user requests a state change. - ModbusCommandItem cmd; + optional cmd; std::vector data; // Is there are lambda configured? @@ -105,11 +105,11 @@ void ModbusBinaryOutput::write_state(bool state) { #endif ESP_LOGV(TAG, "Modbus binary output write raw: %s", format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); - cmd = ModbusCommandItem::create_custom_command( + cmd.emplace(ModbusCommandItem::create_custom_command( this->parent_, data, - [this, cmd](modbus::EntityType register_type, uint16_t start_address, const std::vector &data) { - this->parent_->on_write_register_response(cmd.register_type, this->start_address, data); - }); + [this](modbus::EntityType register_type, uint16_t start_address, std::span data) { + this->parent_->on_write_register_response(register_type, this->start_address, data); + })); } else { ESP_LOGV(TAG, "Write new state: value is %s, type is %d address = %X, offset = %x", ONOFF(state), (int) this->register_type, this->start_address, this->offset); @@ -117,12 +117,14 @@ void ModbusBinaryOutput::write_state(bool state) { // offset for coil and discrete inputs is the coil/register number not bytes if (this->use_write_multiple_) { std::vector states{state}; - cmd = ModbusCommandItem::create_write_multiple_coils(this->parent_, this->start_address + this->offset, states); + cmd.emplace( + ModbusCommandItem::create_write_multiple_coils(this->parent_, this->start_address + this->offset, states)); } else { - cmd = ModbusCommandItem::create_write_single_coil(this->parent_, this->start_address + this->offset, state); + cmd.emplace( + ModbusCommandItem::create_write_single_coil(this->parent_, this->start_address + this->offset, state)); } } - this->parent_->queue_command(cmd); + this->parent_->queue_command(std::move(*cmd)); } void ModbusBinaryOutput::dump_config() { diff --git a/esphome/components/modbus_controller/output/modbus_output.h b/esphome/components/modbus_controller/output/modbus_output.h index c9efd42224..17eb8e3a8f 100644 --- a/esphome/components/modbus_controller/output/modbus_output.h +++ b/esphome/components/modbus_controller/output/modbus_output.h @@ -12,21 +12,21 @@ class ModbusFloatOutput final : public output::FloatOutput, public Component, pu public: ModbusFloatOutput(uint16_t start_address, uint8_t offset, SensorValueType value_type, int register_count) { this->register_type = modbus::EntityType::HOLDING; - this->start_address = start_address; - this->offset = offset; + this->set_address(start_address); + this->set_offset_from_start_address(offset); this->bitmask = 0xFFFFFFFF; this->register_count = register_count; this->sensor_value_type = value_type; this->skip_updates = 0; - this->start_address += offset; - this->offset = 0; + this->set_address(this->start_address + offset); + this->set_offset_from_start_address(0); } void dump_config() override; void set_parent(ModbusController *parent) { this->parent_ = parent; } void set_write_multiply(float factor) { this->multiply_by_ = factor; } // Do nothing - void parse_and_publish(const std::vector &data) override{}; + void parse_and_publish(std::span data) override{}; using write_transform_func_t = optional (*)(ModbusFloatOutput *, float, std::vector &); void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } @@ -45,19 +45,19 @@ class ModbusBinaryOutput final : public output::BinaryOutput, public Component, public: ModbusBinaryOutput(uint16_t start_address, uint8_t offset) { this->register_type = modbus::EntityType::COIL; - this->start_address = start_address; + this->set_address(start_address); this->bitmask = 0xFFFFFFFF; this->sensor_value_type = SensorValueType::BIT; this->skip_updates = 0; this->register_count = 1; - this->start_address += offset; - this->offset = 0; + this->set_address(this->start_address + offset); + this->set_offset_from_start_address(0); } void dump_config() override; void set_parent(ModbusController *parent) { this->parent_ = parent; } // Do nothing - void parse_and_publish(const std::vector &data) override{}; + void parse_and_publish(std::span data) override{}; using write_transform_func_t = optional (*)(ModbusBinaryOutput *, bool, std::vector &); void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } diff --git a/esphome/components/modbus_controller/select/__init__.py b/esphome/components/modbus_controller/select/__init__.py index 334a4dfd76..5127360770 100644 --- a/esphome/components/modbus_controller/select/__init__.py +++ b/esphome/components/modbus_controller/select/__init__.py @@ -115,10 +115,7 @@ async def to_code(config): [ (ModbusSelect.operator("const_ptr"), "item"), (cg.int64, "x"), - ( - cg.std_vector.template(cg.uint8).operator("const").operator("ref"), - "data", - ), + (cg.std_span.template(cg.uint8.operator("const")), "data"), ], return_type=cg.optional.template(cg.std_string), ) diff --git a/esphome/components/modbus_controller/select/modbus_select.cpp b/esphome/components/modbus_controller/select/modbus_select.cpp index daa6b10da4..0a9383b1b0 100644 --- a/esphome/components/modbus_controller/select/modbus_select.cpp +++ b/esphome/components/modbus_controller/select/modbus_select.cpp @@ -7,10 +7,9 @@ static const char *const TAG = "modbus_controller.select"; void ModbusSelect::dump_config() { LOG_SELECT(TAG, "Modbus Controller Select", this); } -void ModbusSelect::parse_and_publish(const std::vector &data) { - int64_t value = modbus::helpers::payload_to_number(std::span(data), this->sensor_value_type, - this->offset, this->bitmask) - .value_or(0); +void ModbusSelect::parse_and_publish(std::span data) { + int64_t value = + modbus::helpers::payload_to_number(data, this->sensor_value_type, this->offset, this->bitmask).value_or(0); ESP_LOGD(TAG, "New select value %lld from payload", value); @@ -86,15 +85,16 @@ void ModbusSelect::control(size_t index) { return; } - const uint16_t write_address = this->start_address + this->offset / 2; - ModbusCommandItem write_cmd; + const uint16_t write_address = this->write_address(); + optional write_cmd; if ((this->register_count == 1) && (!this->use_write_multiple_)) { - write_cmd = ModbusCommandItem::create_write_single_command(this->parent_, write_address, data[0]); + write_cmd.emplace(ModbusCommandItem::create_write_single_command(this->parent_, write_address, data[0])); } else { - write_cmd = ModbusCommandItem::create_write_multiple_command(this->parent_, write_address, data.size(), data); + write_cmd.emplace( + ModbusCommandItem::create_write_multiple_command(this->parent_, write_address, data.size(), data)); } - this->parent_->queue_command(write_cmd); + this->parent_->queue_command(std::move(*write_cmd)); if (this->optimistic_) this->publish_state(index); diff --git a/esphome/components/modbus_controller/select/modbus_select.h b/esphome/components/modbus_controller/select/modbus_select.h index b4834ba4c6..e1ae578ddf 100644 --- a/esphome/components/modbus_controller/select/modbus_select.h +++ b/esphome/components/modbus_controller/select/modbus_select.h @@ -15,9 +15,9 @@ class ModbusSelect final : public Component, public select::Select, public Senso bool force_new_range, std::vector mapping) { this->register_type = modbus::EntityType::HOLDING; // not configurable this->sensor_value_type = sensor_value_type; - this->start_address = start_address; - this->offset = 0; // not configurable - this->bitmask = 0xFFFFFFFF; // not configurable + this->set_address(start_address); + this->set_offset_from_start_address(0); // not configurable + this->bitmask = 0xFFFFFFFF; // not configurable this->register_count = register_count; this->response_bytes = 0; // not configurable this->skip_updates = skip_updates; @@ -25,7 +25,7 @@ class ModbusSelect final : public Component, public select::Select, public Senso this->mapping_ = std::move(mapping); } - using transform_func_t = optional (*)(ModbusSelect *const, int64_t, const std::vector &); + using transform_func_t = optional (*)(ModbusSelect *const, int64_t, std::span); using write_transform_func_t = optional (*)(ModbusSelect *const, const std::string &, int64_t, std::vector &); @@ -36,7 +36,7 @@ class ModbusSelect final : public Component, public select::Select, public Senso void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } void dump_config() override; - void parse_and_publish(const std::vector &data) override; + void parse_and_publish(std::span data) override; void control(size_t index) override; protected: diff --git a/esphome/components/modbus_controller/sensor/modbus_sensor.cpp b/esphome/components/modbus_controller/sensor/modbus_sensor.cpp index 559724057a..b2bc2b5fd0 100644 --- a/esphome/components/modbus_controller/sensor/modbus_sensor.cpp +++ b/esphome/components/modbus_controller/sensor/modbus_sensor.cpp @@ -8,8 +8,8 @@ static const char *const TAG = "modbus_controller.sensor"; void ModbusSensor::dump_config() { LOG_SENSOR(TAG, "Modbus Controller Sensor", this); } -void ModbusSensor::parse_and_publish(const std::vector &data) { - float result = payload_to_float(data, *this); +void ModbusSensor::parse_and_publish(std::span data) { + float result = payload_to_float(data, *this, this->offset); // Is there a lambda registered // call it with the pre converted value and the raw data array diff --git a/esphome/components/modbus_controller/sensor/modbus_sensor.h b/esphome/components/modbus_controller/sensor/modbus_sensor.h index 1d11aa4d66..9d66b2afa7 100644 --- a/esphome/components/modbus_controller/sensor/modbus_sensor.h +++ b/esphome/components/modbus_controller/sensor/modbus_sensor.h @@ -4,7 +4,7 @@ #include "esphome/components/sensor/sensor.h" #include "esphome/core/component.h" -#include +#include namespace esphome::modbus_controller { @@ -13,8 +13,8 @@ class ModbusSensor final : public Component, public sensor::Sensor, public Senso ModbusSensor(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint32_t bitmask, SensorValueType value_type, int register_count, uint16_t skip_updates, bool force_new_range) { this->register_type = register_type; - this->start_address = start_address; - this->offset = offset; + this->set_address(start_address); + this->set_offset_from_start_address(offset); this->bitmask = bitmask; this->sensor_value_type = value_type; this->register_count = register_count; @@ -22,9 +22,9 @@ class ModbusSensor final : public Component, public sensor::Sensor, public Senso this->force_new_range = force_new_range; } - void parse_and_publish(const std::vector &data) override; + void parse_and_publish(std::span data) override; void dump_config() override; - using transform_func_t = optional (*)(ModbusSensor *, float, const std::vector &); + using transform_func_t = optional (*)(ModbusSensor *, float, std::span); void set_template(transform_func_t f) { this->transform_func_ = f; } diff --git a/esphome/components/modbus_controller/switch/modbus_switch.cpp b/esphome/components/modbus_controller/switch/modbus_switch.cpp index b8cdbf018d..810d904d85 100644 --- a/esphome/components/modbus_controller/switch/modbus_switch.cpp +++ b/esphome/components/modbus_controller/switch/modbus_switch.cpp @@ -27,16 +27,17 @@ void ModbusSwitch::set_assumed_state(bool assumed_state) { this->assumed_state_ bool ModbusSwitch::assumed_state() { return this->assumed_state_; } -void ModbusSwitch::parse_and_publish(const std::vector &data) { +void ModbusSwitch::parse_and_publish(std::span data) { bool value = false; + // For coils/discrete inputs this is the bit index; for registers it is the byte offset. + const size_t offset = this->offset; switch (this->register_type) { case modbus::EntityType::DISCRETE_INPUT: case modbus::EntityType::COIL: - // offset for coil is the actual number of the coil not the byte offset - value = modbus::helpers::bit_from_packed(this->offset, data); + value = modbus::helpers::bit_from_packed(offset, data); break; default: - value = modbus::helpers::get_data(data, this->offset) & this->bitmask; + value = modbus::helpers::get_data(data.data(), offset) & this->bitmask; break; } @@ -51,14 +52,14 @@ void ModbusSwitch::parse_and_publish(const std::vector &data) { } } - ESP_LOGV(TAG, "Publish '%s': new value = %s type = %d address = %X offset = %x", this->get_name().c_str(), - ONOFF(value), (int) this->register_type, this->start_address, this->offset); + ESP_LOGV(TAG, "Publish '%s': new value = %s type = %d address = %X offset = %zx", this->get_name().c_str(), + ONOFF(value), (int) this->register_type, this->start_address, offset); this->publish_state(value); } void ModbusSwitch::write_state(bool state) { // This will be called every time the user requests a state change. - ModbusCommandItem cmd; + optional cmd; std::vector data; // Is there are lambda configured? if (this->write_transform_func_.has_value()) { @@ -80,11 +81,11 @@ void ModbusSwitch::write_state(bool state) { #endif ESP_LOGV(TAG, "Modbus Switch write raw: %s", format_hex_pretty_to(hex_buf, sizeof(hex_buf), data.data(), data.size())); - cmd = ModbusCommandItem::create_custom_command( + cmd.emplace(ModbusCommandItem::create_custom_command( this->parent_, data, - [this, cmd](modbus::EntityType register_type, uint16_t start_address, const std::vector &data) { - this->parent_->on_write_register_response(cmd.register_type, this->start_address, data); - }); + [this](modbus::EntityType register_type, uint16_t start_address, std::span data) { + this->parent_->on_write_register_response(register_type, this->start_address, data); + })); } else { ESP_LOGV(TAG, "write_state '%s': new value = %s type = %d address = %X offset = %x", this->get_name().c_str(), ONOFF(state), (int) this->register_type, this->start_address, this->offset); @@ -92,23 +93,22 @@ void ModbusSwitch::write_state(bool state) { // offset for coil and discrete inputs is the coil/register number not bytes if (this->use_write_multiple_) { std::vector states{state}; - cmd = ModbusCommandItem::create_write_multiple_coils(this->parent_, this->start_address + this->offset, states); + cmd.emplace(ModbusCommandItem::create_write_multiple_coils(this->parent_, this->write_address(), states)); } else { - cmd = ModbusCommandItem::create_write_single_coil(this->parent_, this->start_address + this->offset, state); + cmd.emplace(ModbusCommandItem::create_write_single_coil(this->parent_, this->write_address(), state)); } } else { - // since offset is in bytes and a register is 16 bits we get the start by adding offset/2 if (this->use_write_multiple_) { std::vector bool_states(1, state ? (0xFFFF & this->bitmask) : 0); - cmd = ModbusCommandItem::create_write_multiple_command(this->parent_, this->start_address + this->offset / 2, 1, - bool_states); + cmd.emplace( + ModbusCommandItem::create_write_multiple_command(this->parent_, this->write_address(), 1, bool_states)); } else { - cmd = ModbusCommandItem::create_write_single_command(this->parent_, this->start_address + this->offset / 2, - state ? 0xFFFF & this->bitmask : 0u); + cmd.emplace(ModbusCommandItem::create_write_single_command(this->parent_, this->write_address(), + state ? 0xFFFF & this->bitmask : 0u)); } } } - this->parent_->queue_command(cmd); + this->parent_->queue_command(std::move(*cmd)); this->publish_state(state); } // ModbusSwitch end diff --git a/esphome/components/modbus_controller/switch/modbus_switch.h b/esphome/components/modbus_controller/switch/modbus_switch.h index 0d5456aa63..e5b8cf5c21 100644 --- a/esphome/components/modbus_controller/switch/modbus_switch.h +++ b/esphome/components/modbus_controller/switch/modbus_switch.h @@ -13,15 +13,15 @@ class ModbusSwitch final : public Component, public switch_::Switch, public Sens ModbusSwitch(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint32_t bitmask, uint16_t skip_updates, bool force_new_range) { this->register_type = register_type; - this->start_address = start_address; - this->offset = offset; + this->set_address(start_address); + this->set_offset_from_start_address(offset); this->bitmask = bitmask; this->sensor_value_type = SensorValueType::BIT; this->skip_updates = skip_updates; this->register_count = 1; if (register_type == modbus::EntityType::HOLDING || register_type == modbus::EntityType::COIL) { - this->start_address += offset; - this->offset = 0; + this->set_address(this->start_address + offset); + this->set_offset_from_start_address(0); } this->force_new_range = force_new_range; }; @@ -30,10 +30,10 @@ class ModbusSwitch final : public Component, public switch_::Switch, public Sens void dump_config() override; void set_assumed_state(bool assumed_state); void set_state(bool state) { this->state = state; } - void parse_and_publish(const std::vector &data) override; + void parse_and_publish(std::span data) override; void set_parent(ModbusController *parent) { this->parent_ = parent; } - using transform_func_t = optional (*)(ModbusSwitch *, bool, const std::vector &); + using transform_func_t = optional (*)(ModbusSwitch *, bool, std::span); using write_transform_func_t = optional (*)(ModbusSwitch *, bool, std::vector &); void set_template(transform_func_t f) { this->publish_transform_func_ = f; } void set_write_template(write_transform_func_t f) { this->write_transform_func_ = f; } diff --git a/esphome/components/modbus_controller/text_sensor/modbus_textsensor.cpp b/esphome/components/modbus_controller/text_sensor/modbus_textsensor.cpp index 5626515638..31b3fb3e55 100644 --- a/esphome/components/modbus_controller/text_sensor/modbus_textsensor.cpp +++ b/esphome/components/modbus_controller/text_sensor/modbus_textsensor.cpp @@ -8,10 +8,11 @@ static const char *const TAG = "modbus_controller.text_sensor"; void ModbusTextSensor::dump_config() { LOG_TEXT_SENSOR("", "Modbus Controller Text Sensor", this); } -void ModbusTextSensor::parse_and_publish(const std::vector &data) { +void ModbusTextSensor::parse_and_publish(std::span data) { std::string output_str{}; uint8_t items_left = this->response_bytes; - uint8_t index = this->offset; + const size_t start_offset = this->offset; + size_t index = start_offset; while ((items_left > 0) && index < data.size()) { uint8_t b = data[index]; switch (this->encode_) { @@ -25,7 +26,7 @@ void ModbusTextSensor::parse_and_publish(const std::vector &data) { case RawEncoding::COMMA: { // max 5: optional ','(1) + uint8(3) + null, for both ",%d" and "%d" char dec_buf[5]; - snprintf(dec_buf, sizeof(dec_buf), index != this->offset ? ",%d" : "%d", b); + snprintf(dec_buf, sizeof(dec_buf), index != start_offset ? ",%d" : "%d", b); output_str += dec_buf; break; } diff --git a/esphome/components/modbus_controller/text_sensor/modbus_textsensor.h b/esphome/components/modbus_controller/text_sensor/modbus_textsensor.h index c7381d7ddd..5bb16eb58a 100644 --- a/esphome/components/modbus_controller/text_sensor/modbus_textsensor.h +++ b/esphome/components/modbus_controller/text_sensor/modbus_textsensor.h @@ -4,7 +4,7 @@ #include "esphome/components/text_sensor/text_sensor.h" #include "esphome/core/component.h" -#include +#include namespace esphome::modbus_controller { @@ -15,8 +15,8 @@ class ModbusTextSensor final : public Component, public text_sensor::TextSensor, ModbusTextSensor(modbus::EntityType register_type, uint16_t start_address, uint8_t offset, uint8_t register_count, uint16_t response_bytes, RawEncoding encode, uint16_t skip_updates, bool force_new_range) { this->register_type = register_type; - this->start_address = start_address; - this->offset = offset; + this->set_address(start_address); + this->set_offset_from_start_address(offset); this->response_bytes = response_bytes; this->register_count = register_count; this->encode_ = encode; @@ -28,8 +28,8 @@ class ModbusTextSensor final : public Component, public text_sensor::TextSensor, void dump_config() override; - void parse_and_publish(const std::vector &data) override; - using transform_func_t = optional (*)(ModbusTextSensor *, std::string, const std::vector &); + void parse_and_publish(std::span data) override; + using transform_func_t = optional (*)(ModbusTextSensor *, std::string, std::span); void set_template(transform_func_t f) { this->transform_func_ = f; } protected: diff --git a/esphome/components/modbus_server/modbus_server.h b/esphome/components/modbus_server/modbus_server.h index 4fddd9854d..f6484d8e6b 100644 --- a/esphome/components/modbus_server/modbus_server.h +++ b/esphome/components/modbus_server/modbus_server.h @@ -61,6 +61,7 @@ class ServerRegister { const char *format_value(int64_t value, char *buf, size_t buf_size) const { switch (this->value_type) { case SensorValueType::U_WORD: + case SensorValueType::U_WORD_S: case SensorValueType::U_DWORD: case SensorValueType::U_DWORD_R: case SensorValueType::U_QWORD: @@ -68,6 +69,7 @@ class ServerRegister { buf_append_printf(buf, buf_size, 0, "%" PRIu64, static_cast(value)); return buf; case SensorValueType::S_WORD: + case SensorValueType::S_WORD_S: case SensorValueType::S_DWORD: case SensorValueType::S_DWORD_R: case SensorValueType::S_QWORD: diff --git a/esphome/components/mqtt/__init__.py b/esphome/components/mqtt/__init__.py index 4a5eacf449..713969ab88 100644 --- a/esphome/components/mqtt/__init__.py +++ b/esphome/components/mqtt/__init__.py @@ -21,6 +21,7 @@ from esphome.const import ( CONF_CLIENT_ID, CONF_COMMAND_RETAIN, CONF_COMMAND_TOPIC, + CONF_DISCOVER_IP, CONF_DISCOVERY, CONF_DISCOVERY_OBJECT_ID_GENERATOR, CONF_DISCOVERY_PREFIX, @@ -62,6 +63,7 @@ from esphome.const import ( PlatformFramework, ) from esphome.core import CORE, CoroPriority, coroutine_with_priority +from esphome.core.entity_helpers import ObjectIdEntity, validate_no_object_id_conflicts from esphome.types import ConfigType DEPENDENCIES = ["network"] @@ -73,7 +75,6 @@ def AUTO_LOAD(): return ["json"] -CONF_DISCOVER_IP = "discover_ip" CONF_IDF_SEND_ASYNC = "idf_send_async" CONF_WAIT_FOR_CONNECTION = "wait_for_connection" @@ -332,6 +333,68 @@ CONFIG_SCHEMA = cv.All( ) +# Platforms whose MQTT components subscribe to an object_id-derived command topic. +# Keep in sync with the platforms extending cv.MQTT_COMMAND_COMPONENT_SCHEMA, plus +# text, whose MQTT component subscribes a command topic that cannot be overridden. +_COMMAND_TOPIC_PLATFORMS = frozenset( + { + "alarm_control_panel", + "button", + "climate", + "cover", + "datetime", + "fan", + "light", + "lock", + "number", + "select", + "switch", + "text", + "update", + "valve", + } +) + + +# Platforms whose MQTT components derive extra sub-topics (position/command, +# mode/command, speed/command, ...) from the object_id, each with its own config +# key; custom state and command topics cannot exempt them from conflicting. +_SUB_TOPIC_PLATFORMS = frozenset({"climate", "cover", "fan", "valve"}) + + +def _topics_conflict(entities: list[ObjectIdEntity], config: ConfigType) -> bool: + """Check whether more than one entity actually uses an object_id-derived topic. + + An empty topic_prefix disables default topics entirely, custom state and + command topics avoid the default topics, and disabling discovery (globally + or per entity) avoids the discovery config topic. + """ + if config[CONF_TOPIC_PREFIX]: + platform = entities[0].platform + if platform in _SUB_TOPIC_PLATFORMS: + return True + if sum(CONF_STATE_TOPIC not in entity.config for entity in entities) > 1: + return True + if ( + platform in _COMMAND_TOPIC_PLATFORMS + and sum(CONF_COMMAND_TOPIC not in entity.config for entity in entities) > 1 + ): + return True + if not config[CONF_DISCOVERY]: + return False + discovery_entities = sum( + entity.config.get(CONF_DISCOVERY, True) for entity in entities + ) + return discovery_entities > 1 + + +FINAL_VALIDATE_SCHEMA = validate_no_object_id_conflicts( + "mqtt builds default topics and discovery topics from the entity object_id, " + "which is the name converted to ASCII", + conflict_filter=_topics_conflict, +) + + def exp_mqtt_message(config): if config is None: return cg.optional(cg.TemplateArguments(MQTTMessage)) diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 4002d1cc04..386fed5412 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -722,11 +722,17 @@ def _addr2line(addr2line: str, elf: Path, addr: str) -> str: return "" +# The PC bound matches the gate in platform_hooks.STACKTRACE_GATES; +# the logger prints both registers with %08x, so a real PC is always +# 8 digits. tests/unit_tests/test_stacktrace.py guards against drift. +STACKTRACE_NRF52_PC_LR_RE = re.compile(r"PC=(0x[0-9a-fA-F]{3,})\s+LR=(0x[0-9a-fA-F]+)") + + def process_stacktrace(config: ConfigType, line: str, backtrace_state: bool) -> bool: if "Last crash:" in line: return True if backtrace_state: - match = re.search(r"PC=(0x[0-9a-fA-F]+)\s+LR=(0x[0-9a-fA-F]+)", line) + match = STACKTRACE_NRF52_PC_LR_RE.search(line) if match: pc = match.group(1) lr = match.group(2) diff --git a/esphome/components/opentherm/output/opentherm_output.h b/esphome/components/opentherm/output/opentherm_output.h index 24d5052076..da2082963d 100644 --- a/esphome/components/opentherm/output/opentherm_output.h +++ b/esphome/components/opentherm/output/opentherm_output.h @@ -14,7 +14,7 @@ class OpenthermOutput final : public output::FloatOutput, public Component, publ float min_value_, max_value_; public: - float state; + float state{0.0f}; void set_id(const char *id) { this->id_ = id; } diff --git a/esphome/components/ota/__init__.py b/esphome/components/ota/__init__.py index 8296410f2f..2d4de52e8f 100644 --- a/esphome/components/ota/__init__.py +++ b/esphome/components/ota/__init__.py @@ -151,7 +151,7 @@ async def final_step(): cg.add_define("USE_OTA_STATE_LISTENER") -FILTER_SOURCE_FILES = filter_source_files_from_platform( +_filter_backend_source_files = filter_source_files_from_platform( { "ota_backend_esp_idf.cpp": { PlatformFramework.ESP32_ARDUINO, @@ -167,3 +167,19 @@ FILTER_SOURCE_FILES = filter_source_files_from_platform( "ota_backend_host.cpp": {PlatformFramework.HOST_NATIVE}, } ) + + +def FILTER_SOURCE_FILES() -> list[str]: + files = _filter_backend_source_files() + # ota_signature_esp_idf.cpp implements multi-key OTA signature verification, + # compiled only when the esp32 component enables it (external RSA signed + # OTA sets USE_OTA_SIGNED_VERIFICATION_MULTI_KEY). The define is set only on + # ESP32/IDF, so this also excludes the file on every other platform. Filter + # it out otherwise so the (otherwise fully #ifdef'd-out) file isn't opened + # and parsed on every build. + if not any( + define.name == "USE_OTA_SIGNED_VERIFICATION_MULTI_KEY" + for define in CORE.defines + ): + files.append("ota_signature_esp_idf.cpp") + return files diff --git a/esphome/components/ota/ota_backend_esp_idf.cpp b/esphome/components/ota/ota_backend_esp_idf.cpp index 8fd21f42bd..108605e4c9 100644 --- a/esphome/components/ota/ota_backend_esp_idf.cpp +++ b/esphome/components/ota/ota_backend_esp_idf.cpp @@ -144,6 +144,9 @@ OTAResponseTypes IDFOTABackend::end() { } } #ifdef USE_OTA_PARTITIONS + // A partition-table update carries an MD5 (checked by IDF), not a Secure Boot + // signature, and only re-points boot at an already-installed app -- so it is + // intentionally not run through the signature verifier below. if (this->ota_type_ == ota::OTA_TYPE_UPDATE_PARTITION_TABLE) { return this->update_partition_table(); } @@ -162,6 +165,16 @@ OTAResponseTypes IDFOTABackend::end() { } #endif if (err == ESP_OK) { +#ifdef USE_OTA_SIGNED_VERIFICATION_MULTI_KEY + // IDF's built-in on-update check is disabled for this scheme (it only + // matches the incoming image's first signature block against the running + // app's first). Verify here against every key the running app trusts, so + // rotation and backup keys are accepted. Leaving the boot partition + // unchanged means a rejected image never boots. + if (!this->verify_signed_image_(this->partition_)) { + return OTA_RESPONSE_ERROR_SIGNATURE_INVALID; + } +#endif #ifdef USE_OTA_DOWNGRADE_PROTECTION // The image is written and (when signing is enabled) signature-verified by // esp_ota_end(), so its embedded project version can be trusted. Reject the diff --git a/esphome/components/ota/ota_backend_esp_idf.h b/esphome/components/ota/ota_backend_esp_idf.h index a49a5e34b3..9dffd5429e 100644 --- a/esphome/components/ota/ota_backend_esp_idf.h +++ b/esphome/components/ota/ota_backend_esp_idf.h @@ -54,6 +54,11 @@ class IDFOTABackend final { #endif private: +#ifdef USE_OTA_SIGNED_VERIFICATION_MULTI_KEY + // Accept an image signed by any key the running app trusts (up to 3 blocks), + // so rotation and backup keys work. Fails closed. Covers app and bootloader. + bool verify_signed_image_(const esp_partition_t *incoming); +#endif // Keep md5_ first since its digest_ is alignas(32) on DMA-SHA variants; md5_set_ stays last so buf_ packs tightly. md5::MD5Digest md5_{}; esp_ota_handle_t update_handle_{0}; diff --git a/esphome/components/ota/ota_bootloader_esp_idf.cpp b/esphome/components/ota/ota_bootloader_esp_idf.cpp index 062e4d0811..264218a3df 100644 --- a/esphome/components/ota/ota_bootloader_esp_idf.cpp +++ b/esphome/components/ota/ota_bootloader_esp_idf.cpp @@ -94,6 +94,18 @@ OTAResponseTypes IDFOTABackend::finalize_bootloader_update_(esp_err_t ota_end_er if (ota_end_err != ESP_OK) { return OTA_RESPONSE_ERROR_BOOTLOADER_VERIFY; } +#ifdef USE_OTA_SIGNED_VERIFICATION_MULTI_KEY + // The new bootloader is staged in partition_. IDF never signature-checks a + // bootloader image in this software-signed config -- esp_image_verify() skips + // it when is_bootloader() is true -- so without this a bootloader OTA would + // install unverified. Require a trusted signature, which means the bootloader + // must be externally signed and 4 KiB-padded, the same as the app. + if (!this->verify_signed_image_(this->partition_)) { + ESP_LOGE(TAG, "Bootloader image is not signed by a trusted key; a bootloader OTA requires an " + "externally-signed, 4 KiB-padded bootloader.bin"); + return OTA_RESPONSE_ERROR_BOOTLOADER_VERIFY; + } +#endif esp_bootloader_desc_t bootloader_desc; esp_err_t desc_err = esp_ota_get_bootloader_description(this->partition_, &bootloader_desc); #ifdef USE_ESP32_SRAM1_AS_IRAM diff --git a/esphome/components/ota/ota_signature_esp_idf.cpp b/esphome/components/ota/ota_signature_esp_idf.cpp new file mode 100644 index 0000000000..edee594bfe --- /dev/null +++ b/esphome/components/ota/ota_signature_esp_idf.cpp @@ -0,0 +1,232 @@ +#ifdef USE_ESP32 +#include "ota_backend_esp_idf.h" + +#ifdef USE_OTA_SIGNED_VERIFICATION_MULTI_KEY +#include "esphome/components/watchdog/watchdog.h" +#include "esphome/core/log.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace esphome::ota { + +static const char *const TAG = "ota.idf"; + +// Route the "Signature check: " prefix (and its per-block form) through one +// shared format string each, so the prefix is pooled once by the linker instead +// of duplicated at every call site. The level macro is forwarded so compile-time +// log-level stripping still applies. +#define OTA_IDF_SIG_LOG(level, msg) level(TAG, "Signature check: %s", msg) +#define OTA_IDF_SIG_LOG_BLOCK(level, i, msg) level(TAG, "Signature check: block %zu: %s", static_cast(i), msg) + +// Secure Boot v2 RSA-3072 signature block, as written by espsecure and stored +// in the 4 KiB sector following the (4 KiB-padded) app image. All bignum +// fields are byte-reversed to little-endian for the RSA accelerator; software +// verification reverses them back. See the espsecure "; +constexpr uint8_t TRUSTED_KEY_DIGESTS[OTA_TRUSTED_KEY_COUNT][SHA256_BYTES] = OTA_TRUSTED_KEY_DIGESTS; + +// A block is structurally valid if the magic, version, and CRC all check out. +// The CRC covers everything before it and uses the same ROM routine the +// bootloader validates the block with, so the check matches byte-for-byte. +bool block_is_valid(const uint8_t *block) { + if (block[0] != SIG_BLOCK_MAGIC || block[1] != SIG_BLOCK_VERSION_RSA) { + return false; + } + uint32_t stored_crc; + memcpy(&stored_crc, block + OFFSET_CRC, sizeof(stored_crc)); + return esp_rom_crc32_le(0, block, OFFSET_CRC) == stored_crc; +} + +bool key_digest_of(const uint8_t *block, KeyDigest &out) { + return mbedtls_sha256(block + OFFSET_KEY, KEY_REGION_LEN, out.data(), /*is224=*/0) == 0; +} + +// The offset of the signature sector: the app length rounded up to 4 KiB. +bool signature_sector_offset(const esp_partition_t *part, size_t &out_offset) { + esp_partition_pos_t pos{.offset = part->address, .size = part->size}; + esp_image_metadata_t meta{}; + if (esp_image_get_metadata(&pos, &meta) != ESP_OK) { + return false; + } + // Bound the image length before rounding up so a crafted header can't + // overflow the addition; the image plus its signature sector must fit. + if (meta.image_len > part->size) { + return false; + } + out_offset = (meta.image_len + SIG_SECTOR_ALIGN - 1) & ~(SIG_SECTOR_ALIGN - 1); + return out_offset + SIG_BLOCK_SIZE <= part->size; +} + +// SHA-256 over the 4 KiB-padded image, i.e. everything the signature covers. +// Returns false on a read or hash error so a hash failure is not later +// misreported as a signature mismatch. +bool image_digest(const esp_partition_t *part, size_t image_padded_len, uint8_t *out) { + mbedtls_sha256_context ctx; + mbedtls_sha256_init(&ctx); + bool ok = mbedtls_sha256_starts(&ctx, /*is224=*/0) == 0; + uint8_t buf[512]; + for (size_t off = 0; ok && off < image_padded_len; off += sizeof(buf)) { + size_t chunk = std::min(sizeof(buf), image_padded_len - off); + if (esp_partition_read(part, off, buf, chunk) != ESP_OK || mbedtls_sha256_update(&ctx, buf, chunk) != 0) { + ok = false; + } + } + if (ok) { + ok = mbedtls_sha256_finish(&ctx, out) == 0; + } + mbedtls_sha256_free(&ctx); + return ok; +} + +// Verify one RSA-PSS-3072-SHA256 signature block over the image digest. The +// block's modulus and signature are stored little-endian; reverse them in place +// -- block is the caller's scratch buffer, overwritten on the next iteration -- +// rather than stacking a second 384-byte copy of each bignum. +bool rsa_pss_verify(uint8_t *block, const uint8_t *digest) { + std::reverse(block + OFFSET_MODULUS, block + OFFSET_MODULUS + RSA_3072_BYTES); + std::reverse(block + OFFSET_SIGNATURE, block + OFFSET_SIGNATURE + RSA_3072_BYTES); + uint32_t exponent_le; + memcpy(&exponent_le, block + OFFSET_EXPONENT, sizeof(exponent_le)); + uint8_t exponent_be[4] = {static_cast(exponent_le >> 24), static_cast(exponent_le >> 16), + static_cast(exponent_le >> 8), static_cast(exponent_le)}; + + mbedtls_rsa_context rsa; + mbedtls_rsa_init(&rsa); + bool key_ok = mbedtls_rsa_import_raw(&rsa, block + OFFSET_MODULUS, RSA_3072_BYTES, nullptr, 0, nullptr, 0, nullptr, 0, + exponent_be, sizeof(exponent_be)) == 0 && + mbedtls_rsa_complete(&rsa) == 0 && + mbedtls_rsa_set_padding(&rsa, MBEDTLS_RSA_PKCS_V21, MBEDTLS_MD_SHA256) == 0; + bool verified = false; + if (!key_ok) { + // A setup/allocation failure (e.g. OOM right after the download) is not a + // signature mismatch -- log it distinctly so it isn't read as "wrong key". + OTA_IDF_SIG_LOG(ESP_LOGE, "RSA key setup failed"); + } else { + verified = + mbedtls_rsa_rsassa_pss_verify(&rsa, MBEDTLS_MD_SHA256, SHA256_BYTES, digest, block + OFFSET_SIGNATURE) == 0; + } + mbedtls_rsa_free(&rsa); + return verified; +} + +} // namespace + +bool IDFOTABackend::verify_signed_image_(const esp_partition_t *incoming) { + // Verification re-hashes the full image (after esp_ota_end already did one + // pass), which can approach the task WDT budget on a large app. Extend it for + // the duration, mirroring the erase budget in begin(). + const uint32_t verify_budget_ms = 15000 + (incoming->size >> 10) * 10; + watchdog::WatchdogManager watchdog(verify_budget_ms); + + size_t incoming_sector; + if (!signature_sector_offset(incoming, incoming_sector)) { + OTA_IDF_SIG_LOG(ESP_LOGE, "cannot locate incoming signature sector"); + return false; + } + uint8_t digest[SHA256_BYTES]; + if (!image_digest(incoming, incoming_sector, digest)) { + OTA_IDF_SIG_LOG(ESP_LOGE, "cannot hash incoming image"); + return false; + } + + // Accept if any incoming block is signed by a compiled-in trusted key AND its + // signature verifies over the image. Iterating all blocks (not just the + // first) is the whole point -- it lets a bridge/backup key in a later block + // be the match. The trust check is against the immutable compiled-in set, so + // extra (self-signed) blocks an attacker appends carry keys we simply ignore. + // Heap-allocate the 1216-byte block for the duration of verification: this + // runs mid-OTA on the loop task, on top of the caller's live 1 KB OTA buffer + // and mbedtls's own ~1 KB verify scratch, so keeping it off the stack widens + // a thin margin. One short-lived allocation right before reboot is not the + // fragmentation pattern the project guards against. nothrow so an OOM here + // fails closed like every other error path, rather than aborting. + std::unique_ptr block(new (std::nothrow) uint8_t[SIG_BLOCK_SIZE]); + if (!block) { + OTA_IDF_SIG_LOG(ESP_LOGE, "out of memory"); + return false; + } + bool any_valid_block = false; + for (size_t i = 0; i < SIG_BLOCK_MAX_COUNT; i++) { + size_t off = incoming_sector + i * SIG_BLOCK_SIZE; + if (off + SIG_BLOCK_SIZE > incoming->size) { + break; // partition has no room for another block; done scanning + } + // A read fault is not "no trusted key" -- fail closed with a distinct error. + if (esp_partition_read(incoming, off, block.get(), SIG_BLOCK_SIZE) != ESP_OK) { + OTA_IDF_SIG_LOG_BLOCK(ESP_LOGE, i, "unreadable"); + return false; + } + if (!block_is_valid(block.get())) { + OTA_IDF_SIG_LOG_BLOCK(ESP_LOGD, i, "absent or malformed"); + continue; + } + any_valid_block = true; + KeyDigest incoming_key; + if (!key_digest_of(block.get(), incoming_key)) { + OTA_IDF_SIG_LOG_BLOCK(ESP_LOGE, i, "key hash failed"); + return false; + } + bool trusted_key = false; + for (const auto &trusted : TRUSTED_KEY_DIGESTS) { + if (memcmp(incoming_key.data(), trusted, SHA256_BYTES) == 0) { + trusted_key = true; + break; + } + } + if (!trusted_key) { + OTA_IDF_SIG_LOG_BLOCK(ESP_LOGW, i, "signed by an untrusted key"); + continue; + } + if (rsa_pss_verify(block.get(), digest)) { + OTA_IDF_SIG_LOG_BLOCK(ESP_LOGD, i, "verified with a trusted key"); + return true; + } + OTA_IDF_SIG_LOG_BLOCK(ESP_LOGW, i, "trusted key failed to verify"); + } + + // Separate "not signed at all" from "signed by an untrusted key" -- the former + // otherwise reads as the latter on a device that only logs at INFO. + if (!any_valid_block) { + OTA_IDF_SIG_LOG(ESP_LOGE, "image has no signature block"); + } else { + OTA_IDF_SIG_LOG(ESP_LOGE, "no trusted key produced a valid signature"); + } + return false; +} + +} // namespace esphome::ota + +#endif // USE_OTA_SIGNED_VERIFICATION_MULTI_KEY +#endif // USE_ESP32 diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 6cb9d5f03a..4d1814ac7a 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -201,6 +201,14 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: if base_path := config.get(CONF_PATH): repo_dir = repo_dir / base_path + # Deferred import: keeps esphome.bundle off the device builder's + # startup path, since packages is loaded on every config parse. + from esphome.bundle import add_secret_scan_dir + + # Register the path-narrowed dir, not repo_root, so example configs + # elsewhere in the repo do not widen the shipped secrets. + add_secret_scan_dir(repo_dir) + for file in config[CONF_FILES]: if isinstance(file, str): files.append({CONF_PATH: file, CONF_VARS: {}}) diff --git a/esphome/components/packet_transport/__init__.py b/esphome/components/packet_transport/__init__.py index 4293dffb15..7beb13ca31 100644 --- a/esphome/components/packet_transport/__init__.py +++ b/esphome/components/packet_transport/__init__.py @@ -4,12 +4,12 @@ import hashlib import logging import esphome.codegen as cg -from esphome.components.api import CONF_ENCRYPTION from esphome.components.binary_sensor import BinarySensor from esphome.components.sensor import Sensor import esphome.config_validation as cv from esphome.const import ( CONF_BINARY_SENSORS, + CONF_ENCRYPTION, CONF_ID, CONF_INTERNAL, CONF_KEY, diff --git a/esphome/components/prometheus/__init__.py b/esphome/components/prometheus/__init__.py index cc1541ce80..0a69160fc1 100644 --- a/esphome/components/prometheus/__init__.py +++ b/esphome/components/prometheus/__init__.py @@ -3,6 +3,7 @@ from esphome.components import web_server_base from esphome.components.web_server_base import CONF_WEB_SERVER_BASE_ID import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_INCLUDE_INTERNAL, CONF_NAME, CONF_RELABEL +from esphome.core.entity_helpers import validate_no_object_id_conflicts from esphome.cpp_types import EntityBase AUTO_LOAD = ["web_server_base"] @@ -35,6 +36,11 @@ CONFIG_SCHEMA = cv.Schema( }, ).extend(cv.COMPONENT_SCHEMA) +FINAL_VALIDATE_SCHEMA = validate_no_object_id_conflicts( + "prometheus builds metric labels from the entity object_id, " + "which is the name converted to ASCII" +) + async def to_code(config): paren = await cg.get_variable(config[CONF_WEB_SERVER_BASE_ID]) diff --git a/esphome/components/radio_frequency/radio_frequency.cpp b/esphome/components/radio_frequency/radio_frequency.cpp index 3e0a905737..fe6c6a9cb5 100644 --- a/esphome/components/radio_frequency/radio_frequency.cpp +++ b/esphome/components/radio_frequency/radio_frequency.cpp @@ -99,12 +99,8 @@ bool RadioFrequency::on_receive(remote_base::RemoteReceiveData data) { // Forward received RF data to API server #if defined(USE_API) && defined(USE_RADIO_FREQUENCY) if (api::global_api_server != nullptr) { -#ifdef USE_DEVICES - uint32_t device_id = this->get_device_id(); -#else - uint32_t device_id = 0; -#endif - api::global_api_server->send_infrared_rf_receive_event(device_id, this->get_object_id_hash(), &data.get_raw_data()); + api::global_api_server->send_infrared_rf_receive_event(this->get_device_id_or_zero(), this->get_entity_key(), + &data.get_raw_data()); } #endif return false; // Don't consume the event, allow other listeners to process it diff --git a/esphome/components/rd03d/sensor.py b/esphome/components/rd03d/sensor.py index 4b4fcfd4e4..953d99c2da 100644 --- a/esphome/components/rd03d/sensor.py +++ b/esphome/components/rd03d/sensor.py @@ -1,5 +1,6 @@ import esphome.codegen as cg from esphome.components import sensor +from esphome.components.const import CONF_TARGET_COUNT import esphome.config_validation as cv from esphome.const import ( CONF_ANGLE, @@ -19,8 +20,6 @@ from . import CONF_RD03D_ID, RD03DComponent DEPENDENCIES = ["rd03d"] -CONF_TARGET_COUNT = "target_count" - MAX_TARGETS = 3 UNIT_MILLIMETER_PER_SECOND = "mm/s" diff --git a/esphome/components/rp2/__init__.py b/esphome/components/rp2/__init__.py index 8bba6bc27d..1bf01e6828 100644 --- a/esphome/components/rp2/__init__.py +++ b/esphome/components/rp2/__init__.py @@ -173,15 +173,10 @@ def get_download_types(storage_json): def _format_framework_arduino_version(ver: cv.Version) -> str: - # The most recent releases have not been uploaded to platformio so grabbing them directly from - # the GitHub release is one path forward for now. + # The framework-arduinopico package is no longer published to the PlatformIO + # registry, so install the framework straight from the GitHub release return f"https://github.com/earlephilhower/arduino-pico/releases/download/{ver}/rp2040-{ver}.zip" - # format the given arduino (https://github.com/earlephilhower/arduino-pico/releases) version to - # a PIO earlephilhower/framework-arduinopico value - # List of package versions: https://api.registry.platformio.org/v3/packages/earlephilhower/tool/framework-arduinopico - # return f"~1.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" - def _parse_platform_version(value): value = cv.string(value) @@ -198,19 +193,20 @@ def _parse_platform_version(value): # The default/recommended arduino framework version # - https://github.com/earlephilhower/arduino-pico/releases -# - https://api.registry.platformio.org/v3/packages/earlephilhower/tool/framework-arduinopico -RECOMMENDED_ARDUINO_FRAMEWORK_VERSION = cv.Version(5, 6, 1) +RECOMMENDED_ARDUINO_FRAMEWORK_VERSION = cv.Version(6, 0, 0) # The raspberrypi platform version to use for arduino frameworks # - https://github.com/maxgerhardt/platform-raspberrypi/tags -RECOMMENDED_ARDUINO_PLATFORM_VERSION = "v1.4.0-gcc14-arduinopico460" +# develop-branch commit carrying the arduino-pico 6.0.0 / pico-quick-toolchain +# 5.0.0 (GCC 16.1) update; replace with a release tag when one is cut +RECOMMENDED_ARDUINO_PLATFORM_VERSION = "9c167c6b8aac4f4cfa6d55a0c4e5b848795150c0" def _arduino_check_versions(value): value = value.copy() lookups = { - "dev": (cv.Version(5, 6, 1), "https://github.com/earlephilhower/arduino-pico"), - "latest": (cv.Version(5, 6, 1), None), + "dev": (cv.Version(6, 0, 0), "https://github.com/earlephilhower/arduino-pico"), + "latest": (cv.Version(6, 0, 0), None), "recommended": (RECOMMENDED_ARDUINO_FRAMEWORK_VERSION, None), } diff --git a/esphome/components/rp2/boards.py b/esphome/components/rp2/boards.py index 94d0ebbb60..d2502b8fb8 100644 --- a/esphome/components/rp2/boards.py +++ b/esphome/components/rp2/boards.py @@ -133,9 +133,7 @@ RP2_BOARD_PINS = { "RX": 1, "SCK": 22, "SCL": 3, - "SCL1": 31, "SDA": 2, - "SDA1": 31, "SS": 13, "TX": 0, }, @@ -146,9 +144,7 @@ RP2_BOARD_PINS = { "RX": 1, "SCK": 22, "SCL": 3, - "SCL1": 31, "SDA": 2, - "SDA1": 31, "SS": 21, "TX": 0, }, @@ -464,9 +460,7 @@ RP2_BOARD_PINS = { "RX": 13, "SCK": 18, "SCL": 21, - "SCL1": 31, "SDA": 20, - "SDA1": 31, "SS": 17, "TX": 12, }, @@ -477,9 +471,7 @@ RP2_BOARD_PINS = { "RX": 13, "SCK": 18, "SCL": 21, - "SCL1": 31, "SDA": 20, - "SDA1": 31, "SS": 17, "TX": 12, }, @@ -509,14 +501,10 @@ RP2_BOARD_PINS = { "LED": 29, "MISO": 20, "MOSI": 19, - "RX": 31, "SCK": 22, "SCL": 17, - "SCL1": 31, "SDA": 16, - "SDA1": 31, "SS": 21, - "TX": 31, }, "cytron_maker_nano_rp2040": { "LED": 2, @@ -708,6 +696,19 @@ RP2_BOARD_PINS = { "SS": 17, "TX": 0, }, + "ilabs_cpico_2350": { + "LED": 25, + "MISO": 16, + "MOSI": 19, + "RX": 1, + "SCK": 18, + "SCL": 5, + "SCL1": 27, + "SDA": 4, + "SDA1": 26, + "SS": 17, + "TX": 0, + }, "ilabs_rpico32": { "MISO": 24, "MOSI": 23, @@ -941,31 +942,15 @@ RP2_BOARD_PINS = { "TX": 0, }, "pimoroni_plasma2040": {"LED": 16, "SCL": 21, "SDA": 20}, - "pimoroni_plasma2350": { - "LED": 16, - "MISO": 31, - "MOSI": 31, - "RX": 31, - "SCK": 31, - "SCL": 21, - "SCL1": 31, - "SDA": 20, - "SDA1": 31, - "SS": 31, - "TX": 31, - }, + "pimoroni_plasma2350": {"LED": 16, "SCL": 21, "SDA": 20}, "pimoroni_plasma2350w": { "LED": 16, "MISO": 24, "MOSI": 24, - "RX": 31, "SCK": 29, "SCL": 21, - "SCL1": 31, "SDA": 20, - "SDA1": 31, "SS": 25, - "TX": 31, }, "pimoroni_servo2040": {"LED": 18, "SCL": 21, "SDA": 20}, "pimoroni_tiny2040": { @@ -1208,9 +1193,7 @@ RP2_BOARD_PINS = { "RX": 19, "SCK": 14, "SCL": 21, - "SCL1": 31, "SDA": 20, - "SDA1": 31, "SS": 13, "TX": 18, }, @@ -1256,9 +1239,7 @@ RP2_BOARD_PINS = { "RX": 1, "SCK": 22, "SCL": 17, - "SCL1": 31, "SDA": 16, - "SDA1": 31, "SS": 21, "TX": 0, }, @@ -1281,9 +1262,7 @@ RP2_BOARD_PINS = { "RX": 1, "SCK": 2, "SCL": 7, - "SCL1": 31, "SDA": 6, - "SDA1": 31, "SS": 9, "TX": 0, }, @@ -1513,6 +1492,19 @@ RP2_BOARD_PINS = { "SS": 17, "TX": 0, }, + "weact_rp2350b": { + "LED": 25, + "MISO": 16, + "MOSI": 19, + "RX": 1, + "SCK": 18, + "SCL": 5, + "SCL1": 27, + "SDA": 4, + "SDA1": 26, + "SS": 17, + "TX": 0, + }, "wiznet_55rp20_evb_pico": { "LED": 19, "MISO": 2, @@ -1595,12 +1587,12 @@ BOARDS = { "adafruit_feather_rp2350_adalogger": { "name": "Adafruit Feather RP2350 Adalogger", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "adafruit_feather_rp2350_hstx": { "name": "Adafruit Feather RP2350 HSTX", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "adafruit_feather_scorpio": { "name": "Adafruit Feather RP2040 SCORPIO", @@ -1770,17 +1762,17 @@ BOARDS = { "challenger_2350_bconnect": { "name": "iLabs Challenger 2350 BConnect", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "challenger_2350_nbiot": { "name": "iLabs Challenger 2350 NB-IoT", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "challenger_2350_wifi6_ble5": { "name": "iLabs Challenger 2350 WiFi/BLE", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "challenger_nb_2040_wifi": { "name": "iLabs Challenger NB 2040 WiFi", @@ -1795,7 +1787,7 @@ BOARDS = { "cytron_iriv_io_controller": { "name": "Cytron IRIV IO Controller", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "cytron_maker_nano_rp2040": { "name": "Cytron Maker Nano RP2040", @@ -1815,7 +1807,7 @@ BOARDS = { "cytron_motion_2350_pro": { "name": "Cytron Motion 2350 Pro", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "datanoisetv_picoadk": { "name": "DatanoiseTV PicoADK", @@ -1825,7 +1817,7 @@ BOARDS = { "datanoisetv_picoadk_v2": { "name": "DatanoiseTV PicoADK v2", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "degz_suibo": { "name": "Degz Robotics Suibo RP2040", @@ -1877,6 +1869,11 @@ BOARDS = { "mcu": "rp2040", "max_pin": 29, }, + "ilabs_cpico_2350": { + "name": "iLabs CPico 2350", + "mcu": "rp2350", + "max_pin": 29, + }, "ilabs_rpico32": { "name": "iLabs RPICO32", "mcu": "rp2040", @@ -1948,12 +1945,12 @@ BOARDS = { "max_pin": 29, }, "pcbcupid_glyph_2040": { - "name": "PCBCupid Glyph 2040", + "name": "Pcbcupid GLYPH 2040", "mcu": "rp2040", "max_pin": 29, }, "pcbcupid_glyph_mini_2040": { - "name": "PCBCupid Glyph Mini 2040", + "name": "Pcbcupid GLYPH MINI 2040", "mcu": "rp2040", "max_pin": 29, }, @@ -1997,12 +1994,12 @@ BOARDS = { "pimoroni_plasma2350": { "name": "Pimoroni Plasma2350", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "pimoroni_plasma2350w": { "name": "Pimoroni Plasma2350W", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, "wifi": True, }, "pimoroni_servo2040": { @@ -2018,7 +2015,7 @@ BOARDS = { "pimoroni_tiny2350": { "name": "Pimoroni Tiny2350", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "pintronix_pinmax": { "name": "Pintronix PinMax", @@ -2048,12 +2045,12 @@ BOARDS = { "rpipico2": { "name": "Raspberry Pi Pico 2", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "rpipico2w": { "name": "Raspberry Pi Pico 2W", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, "wifi": True, "max_virtual_pin": 64, }, @@ -2087,7 +2084,7 @@ BOARDS = { "seeed_xiao_rp2350": { "name": "Seeed XIAO RP2350", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "silicognition_rp2040_shim": { "name": "Silicognition RP2040-Shim", @@ -2113,7 +2110,7 @@ BOARDS = { "solderparty_rp2350_stamp": { "name": "Solder Party RP2350 Stamp", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "solderparty_rp2350_stamp_xl": { "name": "Solder Party RP2350 Stamp XL", @@ -2123,7 +2120,7 @@ BOARDS = { "sparkfun_iotnode_lorawanrp2350": { "name": "SparkFun IoT Node LoRaWAN", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "sparkfun_iotredboard_rp2350": { "name": "SparkFun IoT RedBoard RP2350", @@ -2144,7 +2141,7 @@ BOARDS = { "sparkfun_promicrorp2350": { "name": "SparkFun ProMicro RP2350", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "sparkfun_thingplusrp2040": { "name": "SparkFun Thing Plus RP2040", @@ -2154,7 +2151,7 @@ BOARDS = { "sparkfun_thingplusrp2350": { "name": "SparkFun Thing Plus RP2350", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, "wifi": True, "max_virtual_pin": 64, }, @@ -2235,7 +2232,7 @@ BOARDS = { "waveshare_rp2350_lcd_0_96": { "name": "Waveshare RP2350 LCD 0.96", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "waveshare_rp2350_pizero": { "name": "Waveshare RP2350 PiZero", @@ -2245,12 +2242,12 @@ BOARDS = { "waveshare_rp2350_plus": { "name": "Waveshare RP2350 Plus", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "waveshare_rp2350_zero": { "name": "Waveshare RP2350 Zero", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "waveshare_rp2350b_plus_w": { "name": "Waveshare RP2350B Plus W", @@ -2258,6 +2255,11 @@ BOARDS = { "max_pin": 47, "wifi": True, }, + "weact_rp2350b": { + "name": "WeAct Studio RP2350B Core Board", + "mcu": "rp2350", + "max_pin": 47, + }, "wiznet_5100s_evb_pico": { "name": "WIZnet W5100S-EVB-Pico", "mcu": "rp2040", @@ -2266,7 +2268,7 @@ BOARDS = { "wiznet_5100s_evb_pico2": { "name": "WIZnet W5100S-EVB-Pico2", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "wiznet_5500_evb_pico": { "name": "WIZnet W5500-EVB-Pico", @@ -2276,7 +2278,7 @@ BOARDS = { "wiznet_5500_evb_pico2": { "name": "WIZnet W5500-EVB-Pico2", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "wiznet_55rp20_evb_pico": { "name": "WIZnet W55RP20-EVB-Pico", @@ -2291,7 +2293,7 @@ BOARDS = { "wiznet_6300_evb_pico2": { "name": "WIZnet W6300-EVB-Pico2", "mcu": "rp2350", - "max_pin": 47, + "max_pin": 29, }, "wiznet_wizfi360_evb_pico": { "name": "WIZnet WizFi360-EVB-Pico", diff --git a/esphome/components/rp2/generate_boards.py b/esphome/components/rp2/generate_boards.py index 33eb1b3058..5618287cce 100644 --- a/esphome/components/rp2/generate_boards.py +++ b/esphome/components/rp2/generate_boards.py @@ -34,11 +34,17 @@ CYW43_GPIO_COUNT = 3 # Max GPIO pin per MCU (hardware specs from datasheets) MCU_MAX_PIN = { "rp2040": 29, # GPIO 0-29 - "rp2350": 47, # GPIO 0-47 (RP2350A) + "rp2350": 47, # GPIO 0-47 (RP2350B; A-die boards are narrowed to 29 below) } DEFAULT_MAX_PIN = 29 +# The RP2350 comes in two die variants: RP2350A exposes GPIO 0-29, RP2350B +# GPIO 0-47. Variant headers declare the die via PICO_RP2350A (1 = A, 0 = B). +RP2350A_MAX_PIN = 29 PIN_DEFINE_RE = re.compile(r"#define\s+PIN_(\w+)\s+\((\d+)u\)") +# Accepts the literal forms seen in these headers: 1, (1), 1u, (1u) +RP2350A_DEFINE_RE = re.compile(r"#define\s+PICO_RP2350A\s+(\S+)") +RP2350A_MENU_PLACEHOLDER = "__PICO_RP2350A" def parse_variant_pins(variant_dir: Path) -> dict[str, int]: @@ -56,6 +62,40 @@ def parse_variant_pins(variant_dir: Path) -> dict[str, int]: return pins +def parse_variant_is_rp2350a(variant_dir: Path) -> bool: + """Return True if the variant declares an RP2350A die (GPIO 0-29 only). + + Generic boards leave the die a build-time menu choice (PICO_RP2350A is set + to a __PICO_RP2350A placeholder rather than a literal); those return False + so they keep the permissive B-die pin range. + + A missing or unrecognized define raises: silently treating it as B-die + would widen pin validation back to GPIO 47 on A-die boards, so a framework + bump that changes the header format must fail loudly here instead. + """ + header = variant_dir / "pins_arduino.h" + match = ( + RP2350A_DEFINE_RE.search(header.read_text(encoding="utf-8")) + if header.exists() + else None + ) + if match is None: + raise ValueError( + f"{header}: no PICO_RP2350A define found; cannot classify the " + "RP2350 die (A exposes GPIO 0-29, B exposes GPIO 0-47)" + ) + value = match.group(1) + if value == RP2350A_MENU_PLACEHOLDER: + return False + literal = value.strip("()u") + if not literal.isdigit(): + raise ValueError( + f"{header}: unrecognized PICO_RP2350A value {value!r}; cannot " + "classify the RP2350 die (A exposes GPIO 0-29, B exposes GPIO 0-47)" + ) + return int(literal) == 1 + + def load_boards(arduino_pico_path: Path) -> tuple[dict, dict]: """Load all board definitions and return (board_pins, boards) dicts.""" json_dir = arduino_pico_path / "tools" / "json" @@ -64,6 +104,7 @@ def load_boards(arduino_pico_path: Path) -> tuple[dict, dict]: board_pins = {} boards = {} variant_pins_cache: dict[str, dict[str, int]] = {} + variant_rp2350a_cache: dict[str, bool] = {} for json_file in sorted(json_dir.glob("*.json")): board_name = json_file.stem @@ -81,10 +122,19 @@ def load_boards(arduino_pico_path: Path) -> tuple[dict, dict]: extra_flags = build.get("extra_flags", "") has_wifi = "PICO_CYW43_SUPPORTED=1" in extra_flags + max_pin = MCU_MAX_PIN.get(mcu, DEFAULT_MAX_PIN) + if mcu == "rp2350": + if variant not in variant_rp2350a_cache: + variant_rp2350a_cache[variant] = parse_variant_is_rp2350a( + variants_dir / variant + ) + if variant_rp2350a_cache[variant]: + max_pin = RP2350A_MAX_PIN + board_entry: dict = { "name": display_name, "mcu": mcu, - "max_pin": MCU_MAX_PIN.get(mcu, DEFAULT_MAX_PIN), + "max_pin": max_pin, } if has_wifi: board_entry["wifi"] = True diff --git a/esphome/components/rp2040_ble/__init__.py b/esphome/components/rp2040_ble/__init__.py index ac012b5e85..4baee7e234 100644 --- a/esphome/components/rp2040_ble/__init__.py +++ b/esphome/components/rp2040_ble/__init__.py @@ -17,6 +17,25 @@ CONFIG_SCHEMA = cv.Schema( ).extend(cv.COMPONENT_SCHEMA) +def _validate_board(config: ConfigType) -> ConfigType: + from esphome.components.rp2 import board_has_wifi, get_board + + if not board_has_wifi(): + raise cv.Invalid( + f"Board '{get_board()}' does not have Bluetooth support (no CYW43 wireless " + f"chip). Use a board like 'rpipicow' or 'rpipico2w'." + ) + return config + + +FINAL_VALIDATE_SCHEMA = _validate_board + + +# Once per registered scan listener; sizes the controller's StaticVector +# listener storage. +request_scan_listener_slot = cg.slot_counter("RP2040_BLE_SCAN_LISTENER_COUNT") + + async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) diff --git a/esphome/components/rp2040_ble/rp2040_ble.cpp b/esphome/components/rp2040_ble/rp2040_ble.cpp index e10e85f3c3..8e7c7d6be5 100644 --- a/esphome/components/rp2040_ble/rp2040_ble.cpp +++ b/esphome/components/rp2040_ble/rp2040_ble.cpp @@ -15,29 +15,18 @@ static const char *const TAG = "rp2040_ble"; // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) RP2040BLE *global_ble = nullptr; -// The analyzer cannot see that release() always retains the pointer here: the -// pool's free list is sized SIZE + 1, so its push cannot hit the ring-full -// drop branch for at most SIZE releases. -// NOLINTBEGIN(clang-analyzer-unix.Malloc) void RP2040BLE::setup() { global_ble = this; // Pre-create every pool entry so the packet handler's allocate() is always a - // free-list pop — the IRQ path must never reach malloc() (heap allocation - // after setup is forbidden, and the newlib malloc lock is not IRQ-safe). - // Deliberately unconditional: warming lazily on the first scan would move - // the allocations after setup, and doing it here keeps the pool's RAM cost - // visible at startup instead of appearing once scanning begins. - BLEScanReport *warm[MAX_SCAN_REPORT_QUEUE_SIZE - 1]; - size_t warmed = 0; - while (warmed < MAX_SCAN_REPORT_QUEUE_SIZE - 1 && (warm[warmed] = this->report_pool_.allocate()) != nullptr) - warmed++; - for (size_t i = 0; i < warmed; i++) - this->report_pool_.release(warm[i]); - if (warmed != MAX_SCAN_REPORT_QUEUE_SIZE - 1) { - // An incomplete warm would silently put malloc() back on the IRQ path once - // the free list runs dry; refuse to run instead (the stack is never - // enabled, so the packet handler cannot fire). + // free-list pop — the IRQ path must never reach malloc() (the newlib malloc + // lock is not IRQ-safe). Deliberately + // unconditional: warming lazily on the first scan would move the allocations + // after setup, and doing it here keeps the pool's RAM cost visible at + // startup instead of appearing once scanning begins. On an incomplete warm, + // refuse to run instead (the stack is never enabled, so the packet handler + // cannot fire). + if (!this->report_pool_.warm()) { ESP_LOGE(TAG, "Scan report pool warm-up failed"); this->mark_failed(); return; @@ -49,7 +38,6 @@ void RP2040BLE::setup() { this->state_ = BLEComponentState::DISABLED; } } -// NOLINTEND(clang-analyzer-unix.Malloc) void RP2040BLE::enable() { if (this->state_ == BLEComponentState::ACTIVE || this->state_ == BLEComponentState::ENABLING) { @@ -117,8 +105,10 @@ void RP2040BLE::loop() { if (report == nullptr) return; do { +#ifdef RP2040_BLE_SCAN_LISTENER_COUNT for (auto *listener : this->scan_listeners_) listener->on_scan_report(*report); +#endif this->report_pool_.release(report); } while ((report = this->report_queue_.pop()) != nullptr); @@ -183,6 +173,7 @@ void RP2040BLE::packet_handler(uint8_t type, uint16_t channel, uint8_t *packet, reverse_bd_addr(addr, mac_lsb); // LSB-first, the BLE convention consumers expect global_ble->enqueue_scan_report_(mac_lsb, static_cast(gap_event_advertising_report_get_rssi(packet)), gap_event_advertising_report_get_address_type(packet), + gap_event_advertising_report_get_advertising_event_type(packet), gap_event_advertising_report_get_data(packet), gap_event_advertising_report_get_data_length(packet)); break; @@ -196,8 +187,8 @@ void RP2040BLE::packet_handler(uint8_t type, uint16_t channel, uint8_t *packet, // pool is sized to the queue capacity (SIZE-1), so allocate() returns nullptr // before push() can find the ring full. // NOLINTBEGIN(clang-analyzer-unix.Malloc) -void RP2040BLE::enqueue_scan_report_(const uint8_t *mac_lsb_first, int8_t rssi, uint8_t addr_type, const uint8_t *data, - uint16_t data_len) { +void RP2040BLE::enqueue_scan_report_(const uint8_t *mac_lsb_first, int8_t rssi, uint8_t addr_type, + uint8_t adv_event_type, const uint8_t *data, uint16_t data_len) { BLEScanReport *report = this->report_pool_.allocate(); if (report == nullptr) { // Pool exhausted — the queue is full; count and drop. @@ -207,6 +198,7 @@ void RP2040BLE::enqueue_scan_report_(const uint8_t *mac_lsb_first, int8_t rssi, memcpy(report->mac, mac_lsb_first, 6); report->rssi = rssi; report->addr_type = addr_type; + report->adv_event_type = adv_event_type; report->data_len = (data_len <= sizeof(report->data)) ? static_cast(data_len) : static_cast(sizeof(report->data)); memcpy(report->data, data, report->data_len); @@ -216,7 +208,7 @@ void RP2040BLE::enqueue_scan_report_(const uint8_t *mac_lsb_first, int8_t rssi, void RP2040BLE::get_mac_msb_first(uint8_t out[6]) const { memcpy(out, this->ble_mac_, 6); } -bool RP2040BLE::scan_start(uint16_t interval, uint16_t window) { +bool RP2040BLE::scan_start(uint16_t interval, uint16_t window, bool active) { if (!this->is_active()) { // Power control stays with the user (enable_on_boot or an explicit // enable() call) — auto-enabling here would defeat enable_on_boot: false @@ -226,7 +218,7 @@ bool RP2040BLE::scan_start(uint16_t interval, uint16_t window) { // Serialize with the BTstack background worker (arduino-pico's BluetoothHCI // takes the same lock around its gap_* calls). BluetoothLock lock; - gap_set_scan_params(0 /* passive */, interval, window, 0 /* accept all */); + gap_set_scan_params(active ? 1 : 0, interval, window, 0 /* accept all */); gap_start_scan(); return true; } diff --git a/esphome/components/rp2040_ble/rp2040_ble.h b/esphome/components/rp2040_ble/rp2040_ble.h index 9685b9294e..cc015c0503 100644 --- a/esphome/components/rp2040_ble/rp2040_ble.h +++ b/esphome/components/rp2040_ble/rp2040_ble.h @@ -6,12 +6,12 @@ #include "esphome/core/component.h" #include "esphome/core/event_pool.h" +#include "esphome/core/helpers.h" #include "esphome/core/lock_free_queue.h" #include #include -#include namespace esphome::rp2040_ble { @@ -28,10 +28,13 @@ struct BLEScanReport { uint8_t mac[6]; // LSB-first, as the controller delivers it int8_t rssi; // signed dBm uint8_t addr_type; - uint8_t data_len; // bytes valid in data[] - // Legacy advertisement (31) + scan response (31): passive scans fill at most - // 31 bytes today, but bluetooth_proxy support will flip to active scanning - // in a future PR and the API raw-advertisement contract carries 62. + uint8_t adv_event_type; // GAP advertising event type (ADV_IND .. SCAN_RSP); lets a merger tell the two apart + uint8_t data_len; // bytes valid in data[] + // Legacy advertisement (31) + scan response (31). BTstack delivers the two + // as separate reports, so each report fills at most 31 bytes today; the 62 + // matches the API raw-advertisement contract. adv_event_type is what lets a + // future merge point tell the two frames apart — carrying it beyond this + // struct (RawAdvertisement) is deferred until a consumer needs the merge. uint8_t data[62]; // EventPool contract: nothing is heap-allocated inside a report. @@ -76,17 +79,22 @@ class RP2040BLE final : public Component { /// power-up). void get_mac_msb_first(uint8_t out[6]) const; +#ifdef RP2040_BLE_SCAN_LISTENER_COUNT /// Register a consumer for scan reports (delivered on the main loop via loop()). + /// Storage is codegen-sized: the consumer's codegen requests a slot via + /// request_scan_listener_slot(), which emits RP2040_BLE_SCAN_LISTENER_COUNT. void register_scan_listener(BLEScanListener *listener) { this->scan_listeners_.push_back(listener); } +#endif - /// Start a passive controller scan. Interval/window are in BLE units + /// Start a controller scan; active sends scan requests and receives scan + /// responses as separate reports. Interval/window are in BLE units /// (0.625 ms). Returns false until the stack is ACTIVE (callers retry — the /// tracker's rate-limited retry loop); powering the stack on stays with the /// user (enable_on_boot or an explicit enable() call). The controller keeps /// no scan state: a disable()/enable() power cycle ends the scan, and the /// caller must call scan_start() again once the stack is back to ACTIVE /// (the tracker's loop() reconciliation does exactly that). - bool scan_start(uint16_t interval, uint16_t window); + bool scan_start(uint16_t interval, uint16_t window, bool active); /// Stop the controller scan (no-op when not scanning). void scan_stop(); @@ -95,10 +103,14 @@ class RP2040BLE final : public Component { /// Buffer one controller report (BTstack packet handler, CYW43 async-context /// IRQ — bounded copy into the lock-free queue, nothing else). - void enqueue_scan_report_(const uint8_t *mac_lsb_first, int8_t rssi, uint8_t addr_type, const uint8_t *data, - uint16_t data_len); + void enqueue_scan_report_(const uint8_t *mac_lsb_first, int8_t rssi, uint8_t addr_type, uint8_t adv_event_type, + const uint8_t *data, uint16_t data_len); - std::vector scan_listeners_; +#ifdef RP2040_BLE_SCAN_LISTENER_COUNT + // Codegen-sized: no heap allocation, no std::vector template instantiation — + // the same StaticVector pattern as the tracker's ble_device_base listeners. + StaticVector scan_listeners_; +#endif // Report ring: the BTstack packet handler (async-context IRQ) allocates a // report from the pool, fills it and pushes the pointer; loop() pops, // dispatches and releases. Lock-free SPSC — the esp32_ble/bk72xx_ble pattern. diff --git a/esphome/components/rp2_ble_tracker/__init__.py b/esphome/components/rp2_ble_tracker/__init__.py new file mode 100644 index 0000000000..cfdd78f729 --- /dev/null +++ b/esphome/components/rp2_ble_tracker/__init__.py @@ -0,0 +1,76 @@ +"""BLE scanner for the Raspberry Pi Pico W / Pico 2 W (BLEHub on rp2040_ble). + +Scan modes: + continuous: true — scan runs forever; never stops automatically. + continuous: false — a started scan runs for `duration`, then stops. The first + start is external too; nothing starts a non-continuous + scan on boot. Until start/stop automation actions land + (follow-up PR), starting means a lambda: + `id(my_tracker).start_scan();`. +""" + +import esphome.codegen as cg +from esphome.components import ble_device_base, ota, rp2040_ble +from esphome.components.const import CONF_SCAN_PARAMETERS, CONF_WINDOW +import esphome.config_validation as cv +from esphome.const import ( + CONF_ACTIVE, + CONF_CONTINUOUS, + CONF_DURATION, + CONF_ID, + CONF_INTERVAL, +) +from esphome.types import ConfigType + +CONF_RP2040_BLE_ID = "rp2040_ble_id" + +DEPENDENCIES = ["rp2"] +AUTO_LOAD = ["ble_device_base", "rp2040_ble"] +CODEOWNERS = ["@bdraco"] + +ble_device_base.register_hub_provider("rp2_ble_tracker") + +rp2_ble_tracker_ns = cg.esphome_ns.namespace("rp2_ble_tracker") +RP2BLETracker = rp2_ble_tracker_ns.class_( + "RP2BLETracker", ble_device_base.BLEHub, cg.Component +) + + +# interval defaults to 100 ms with the shared 30 ms window, a 30 % duty cycle — +# the same defaults as bk72xx_ble_tracker, leaving the radio mostly free for +# WiFi on the shared CYW43. Converted to the controller's 0.625 ms BLE units in +# to_code(). `active` defaults on for esp32_ble_tracker parity; it adds scan +# request TX and roughly doubles the reports through the queue, so +# `active: false` is the lighter choice when scan response data is not needed. +SCAN_PARAMETERS_SCHEMA = ble_device_base.scan_parameters_schema( + "100ms", supports_active=True +) + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(): cv.declare_id(RP2BLETracker), + cv.GenerateID(CONF_RP2040_BLE_ID): cv.use_id(rp2040_ble.RP2040BLE), + cv.Optional(CONF_SCAN_PARAMETERS, default={}): SCAN_PARAMETERS_SCHEMA, + } +).extend(cv.COMPONENT_SCHEMA) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + + parent = await cg.get_variable(config[CONF_RP2040_BLE_ID]) + cg.add(var.set_parent(parent)) + # The tracker registers itself as a controller scan listener in setup(); + # request the codegen-sized StaticVector slot for it. + rp2040_ble.request_scan_listener_slot() + + # Get notified when an OTA update starts, to pause scanning (esp32_ble_tracker parity) + ota.request_ota_state_listeners() + + scan = config[CONF_SCAN_PARAMETERS] + cg.add(var.set_scan_interval(ble_device_base.to_ble_units(scan[CONF_INTERVAL]))) + cg.add(var.set_scan_window(ble_device_base.to_ble_units(scan[CONF_WINDOW]))) + cg.add(var.set_scan_duration(scan[CONF_DURATION].total_milliseconds)) + cg.add(var.set_scan_active(scan[CONF_ACTIVE])) + cg.add(var.set_scan_continuous(scan[CONF_CONTINUOUS])) diff --git a/esphome/components/rp2_ble_tracker/rp2_ble_tracker.cpp b/esphome/components/rp2_ble_tracker/rp2_ble_tracker.cpp new file mode 100644 index 0000000000..ed036328ae --- /dev/null +++ b/esphome/components/rp2_ble_tracker/rp2_ble_tracker.cpp @@ -0,0 +1,241 @@ +#ifdef USE_RP2 + +#include "rp2_ble_tracker.h" + +#include + +#include "esphome/core/application.h" +#include "esphome/core/log.h" + +namespace esphome::rp2_ble_tracker { + +static const char *const TAG = "rp2_ble_tracker"; + +// Minimum interval between scan start attempts on an active stack. The +// controller start has no failure mode once HCI is WORKING, so this fires at +// most once per enable cycle today; the floor is insurance against a future +// scan_start() failure being retried every main-loop iteration. +static constexpr uint32_t SCAN_START_RETRY_MS = 1000; + +// One BLE scan unit in milliseconds; the controller programs interval/window in these units. +static constexpr float BLE_SCAN_UNIT_MS = 0.625f; + +void RP2BLETracker::setup() { + // Receive the controller's scan reports; the controller queues them from the + // BTstack packet handler (IRQ) and delivers here on the main loop. + this->parent_->register_scan_listener(this); +#ifdef USE_OTA_STATE_LISTENER + // Pause scanning while an OTA update is in flight — the BLE scan competes with + // the OTA download on the shared CYW43 radio. Mirrors esp32_ble_tracker. + ota::get_global_ota_callback()->add_global_state_listener(this); +#endif + if (!this->scan_continuous_) { + // Nothing to do until an external start_scan(); the loop is re-enabled there. + this->disable_loop(); + } +} + +#ifdef USE_OTA_STATE_LISTENER +void RP2BLETracker::on_ota_global_state(ota::OTAState state, float progress, uint8_t error, ota::OTAComponent *comp) { + if (state == ota::OTA_STARTED) { + this->scan_continuous_before_ota_ = this->scan_continuous_; + // A one-shot scan counts as pending when it is running or still retrying + // its start (loop enabled); captured before stop_scan() disables the loop. + this->scan_pending_before_ota_ = !this->scan_continuous_ && (this->scan_running_ || this->is_in_loop_state()); + this->stop_scan(); + } else if (state == ota::OTA_ERROR || state == ota::OTA_ABORT) { + // On success the device reboots, so restore only on a failed/aborted update; + // loop()'s retry branch restarts the scan on its next iteration. + if (this->scan_continuous_before_ota_) { + this->scan_continuous_before_ota_ = false; + this->scan_continuous_ = true; + this->enable_loop(); + } + // A one-shot scan interrupted by the OTA resumes for a fresh duration + // rather than silently staying idle — an OTA failure does not reboot, so + // nothing external would restart it. + if (this->scan_pending_before_ota_) { + this->scan_pending_before_ota_ = false; + this->enable_loop(); + } + } +} +#endif // USE_OTA_STATE_LISTENER + +void RP2BLETracker::loop() { + const uint32_t now = App.get_loop_component_start_time(); + if (this->scan_running_ && !this->parent_->is_active()) { + // The controller was disabled underneath us (e.g. a lambda calling + // rp2040_ble's disable()); the scan died with the stack. Reconcile so the + // retry branch below takes over once the user re-enables the stack. + this->scan_running_ = false; + this->fire_scan_end_(); + } + if (!this->scan_running_) { + // A scan should be running but is not: continuous mode is always in this + // state until the start succeeds, and non-continuous mode only reaches + // here between start_scan() and a successful controller start, because + // stop_scan_() disables the loop otherwise. + if (!this->parent_->is_active()) { + // Stack not up (still booting, or the user called disable()) — + // scan_start() cannot succeed, so there is nothing to attempt; scanning + // starts on the first iteration after HCI reaches WORKING. + return; + } + if (now - this->last_scan_start_attempt_ >= SCAN_START_RETRY_MS) { + this->start_scan_(); + } + return; + } + + if (this->scan_continuous_) { + // Period timer: fire on_scan_end() once per scan_duration_ window, mirroring + // esp32_ble_tracker::cleanup_scan_state_(). + if (now - this->scan_period_start_ >= this->scan_duration_) { + this->fire_scan_end_(); + this->scan_period_start_ = now; + } + return; + } + + // Non-continuous mode: run for scan_duration_ ms, then stop and fire on_scan_end. + // Restart is driven externally (e.g. api: on_client_connected:). + if (now - this->scan_period_start_ >= this->scan_duration_) { + this->stop_scan_(); + } +} + +void RP2BLETracker::dump_config() { + ESP_LOGCONFIG(TAG, + "RP2 BLE Tracker:\n" + " Scan Duration: %" PRIu32 " s\n" + " Scan Interval: %.0f ms (%" PRIu32 " BLE units)\n" + " Scan Window: %.0f ms (%" PRIu32 " BLE units)\n" + " Scan Type: %s\n" + " Continuous Scanning: %s", + this->scan_duration_ / 1000, this->scan_interval_ * BLE_SCAN_UNIT_MS, this->scan_interval_, + this->scan_window_ * BLE_SCAN_UNIT_MS, this->scan_window_, + this->scan_active_ ? LOG_STR_LITERAL("ACTIVE") : LOG_STR_LITERAL("PASSIVE"), + YESNO(this->scan_continuous_)); +} + +void RP2BLETracker::on_scan_report(const rp2040_ble::BLEScanReport &report) { + // Raw callback (the raw-advertisement path). + if (this->raw_advertisement_callback_.is_set()) { + const ble_device_base::RawAdvertisement adv{.mac = report.mac, + .data = report.data, + .data_len = report.data_len, + .rssi = report.rssi, + .addr_type = report.addr_type}; + this->raw_advertisement_callback_.invoke(adv); + } + +#ifdef ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT + ble_device_base::ESPBTDevice device; + device.from_scan_result(report.mac, report.rssi, report.addr_type, report.data, report.data_len); + bool found = false; + for (auto *listener : this->listeners_) { + if (listener->parse_device(device)) + found = true; + } + // Mirror esp32_ble_tracker: log a newly-seen device only when nothing claimed + // it and the scan is one-shot (continuous scans would spam). + if (!found && !this->scan_continuous_) + this->discovered_log_.log_device(TAG, device); +#endif // ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT +} + +void RP2BLETracker::start_scan() { + // Mirrors esp32_ble_tracker::start_scan(): caller sets scan_continuous_ via + // set_scan_continuous() first, then calls start_scan() to begin scanning. + this->enable_loop(); + this->start_scan_(); +} + +bool RP2BLETracker::request_scan_mode(bool active) { + if (this->scan_active_ == active) + return true; + this->scan_active_ = active; + ESP_LOGD(TAG, "Scan mode %s", active ? "active" : "passive"); + // Apply to a running scan by restarting the CONTROLLER scan with the new + // mode, bypassing the tracker's stop/start bookkeeping: no on_scan_end (the + // scan logically continues, only the request mode changes), no period reset. + // An idle scanner picks the mode up on its next start. + if (this->scan_running_) { + this->parent_->scan_stop(); + if (!this->controller_scan_start_()) { + // The controller really stopped: behave exactly like loop()'s + // reconciliation branch - notify listeners and let its retry recover. + this->scan_running_ = false; + this->fire_scan_end_(); + } + } + return true; +} + +void RP2BLETracker::stop_scan() { + this->scan_continuous_ = false; + this->stop_scan_(); + // stop_scan_() early-returns when no scan is running, so disable the loop + // here too: a scan that never came up (stack still powering on at OTA start) + // must not keep attempting scan_start() from the loop's retry branch. + this->disable_loop(); +} + +// Stamp-and-start for every controller scan attempt: the stamp keeps the +// SCAN_START_RETRY_MS floor covering all callers, not only loop()'s retry. +bool RP2BLETracker::controller_scan_start_() { + this->last_scan_start_attempt_ = App.get_loop_component_start_time(); + return this->parent_->scan_start(static_cast(this->scan_interval_), + static_cast(this->scan_window_), this->scan_active_); +} + +void RP2BLETracker::start_scan_() { + if (this->scan_running_) + return; + + if (!this->controller_scan_start_()) + return; + + this->scan_running_ = true; + // Log every explicit start at DEBUG — stop_scan_() logs every stop at DEBUG, and + // in non-continuous mode each period is an explicit start, so asymmetric logging + // would read as the scanner failing to come back up. + ESP_LOGD(TAG, "Scan started (%s, window=%.0fms, interval=%.0fms)", + this->scan_active_ ? LOG_STR_LITERAL("active") : LOG_STR_LITERAL("passive"), + this->scan_window_ * BLE_SCAN_UNIT_MS, this->scan_interval_ * BLE_SCAN_UNIT_MS); + // Re-anchor the scan period to every successful start — first start (so the + // period counts from the scan, not from boot) and every restart after a stop (so + // resuming after longer than scan_duration, e.g. a failed OTA restoring continuous + // mode 10 minutes later, does not fire on_scan_end before an advertisement can + // arrive). Same clock as loop()'s `now`: a fresh millis() here would be ahead of + // the cached loop time and make the same-iteration period check underflow. + this->scan_period_start_ = App.get_loop_component_start_time(); +} + +void RP2BLETracker::stop_scan_() { + if (!this->scan_running_) + return; + this->parent_->scan_stop(); + this->scan_running_ = false; + ESP_LOGD(TAG, "Scan stopped"); + this->fire_scan_end_(); + // Reset the period clock so on_scan_end does not double-fire; same clock as loop(). + this->scan_period_start_ = App.get_loop_component_start_time(); + if (!this->scan_continuous_) { + // Nothing left to time; start_scan() re-enables the loop. + this->disable_loop(); + } +} + +void RP2BLETracker::fire_scan_end_() { +#ifdef ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT + for (auto *listener : this->listeners_) + listener->on_scan_end(); + this->discovered_log_.clear(); // reset per-scan "Found device" dedup (esp32_ble_tracker parity) +#endif +} + +} // namespace esphome::rp2_ble_tracker + +#endif // USE_RP2 diff --git a/esphome/components/rp2_ble_tracker/rp2_ble_tracker.h b/esphome/components/rp2_ble_tracker/rp2_ble_tracker.h new file mode 100644 index 0000000000..70ececb528 --- /dev/null +++ b/esphome/components/rp2_ble_tracker/rp2_ble_tracker.h @@ -0,0 +1,116 @@ +#pragma once + +#ifdef USE_RP2 + +#include "esphome/components/ble_device_base/ble_device.h" +#include "esphome/components/ble_device_base/ble_hub.h" +#include "esphome/components/rp2040_ble/rp2040_ble.h" +#include "esphome/core/component.h" +#include "esphome/core/helpers.h" + +#include + +#ifdef USE_OTA_STATE_LISTENER +#include "esphome/components/ota/ota_backend.h" +#endif + +namespace esphome::rp2_ble_tracker { + +class RP2BLETracker : public Component, + public ble_device_base::BLEHub, + public rp2040_ble::BLEScanListener, + public Parented +#ifdef USE_OTA_STATE_LISTENER + , + public ota::OTAGlobalStateListener +#endif +{ + public: + // ---- ESPHome Component ---- + void setup() override; + void loop() override; + void dump_config() override; + float get_setup_priority() const override { return setup_priority::AFTER_WIFI; } + +#ifdef USE_OTA_STATE_LISTENER + // Pause scanning while an OTA update runs (the BLE scan competes with the OTA + // download on the shared CYW43 radio); mirrors esp32_ble_tracker. + void on_ota_global_state(ota::OTAState state, float progress, uint8_t error, ota::OTAComponent *comp) override; +#endif + + // ---- YAML configuration setters ---- + void set_scan_interval(uint32_t scan_interval) { this->scan_interval_ = scan_interval; } + void set_scan_window(uint32_t scan_window) { this->scan_window_ = scan_window; } + void set_scan_duration(uint32_t scan_duration) { this->scan_duration_ = scan_duration; } + void set_scan_active(bool scan_active) { this->scan_active_ = scan_active; } + void set_scan_continuous(bool scan_continuous) { this->scan_continuous_ = scan_continuous; } + + // ---- Public scan control ---- + // Mirrors esp32_ble_tracker: set_scan_continuous() + start_scan() / stop_scan(). + void start_scan(); + void stop_scan(); + + // ---- ble_device_base::BLEHub contract ---- + void register_listener(ble_device_base::ESPBTDeviceListener *listener) override { +#ifdef ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT + this->listeners_.push_back(listener); +#endif + } + void set_raw_advertisement_callback(ble_device_base::RawAdvertisementCallback callback) override { + this->raw_advertisement_callback_ = callback; + } + ble_device_base::HubCapabilities get_capabilities() const override { + // BTstack delivers scan responses as separate advertisement reports rather + // than merging them into the advertisement — consumers relying on + // scan-response fields (device names) get them only where the receiver + // merges per address (Home Assistant does). No GATT path yet. + return {.active_scan = true, .merges_scan_response = false, .gatt = false, .scan_mode_switch = true}; + } + // The controller stores the address in printable (MSB-first) order, which is + // exactly what the contract wants. + void get_adapter_mac(uint8_t out[6]) override { this->parent_->get_mac_msb_first(out); } + bool scan_running() override { return this->scan_running_; } + bool scan_active() override { return this->scan_active_; } + bool request_scan_mode(bool active) override; + + // ---- rp2040_ble::BLEScanListener ---- + // Delivered by the controller's loop() on the ESPHome main loop — the + // IRQ → main-loop handoff already happened in the controller's queue. + void on_scan_report(const rp2040_ble::BLEScanReport &report) override; + + protected: + void start_scan_(); + bool controller_scan_start_(); + void stop_scan_(); + void fire_scan_end_(); + + // Defaults: 30 % duty cycle (interval 100 ms / window 30 ms), in 0.625 ms + // BLE units — same defaults as bk72xx_ble_tracker. + uint32_t scan_interval_{160}; // 160 × 0.625 ms = 100 ms + uint32_t scan_window_{48}; // 48 × 0.625 ms = 30 ms (30/100 = 30 %) + uint32_t scan_duration_{300000}; + uint32_t last_scan_start_attempt_{0}; // loop time of last start_scan_() attempt; rate-limits retries + uint32_t scan_period_start_{0}; // loop time at start of current scan period; rate-limits on_scan_end() + bool scan_running_{false}; + bool scan_active_{true}; + bool scan_continuous_{true}; +#ifdef USE_OTA_STATE_LISTENER + bool scan_continuous_before_ota_{false}; // continuous mode saved at OTA start, restored on OTA failure + bool scan_pending_before_ota_{false}; // one-shot scan in flight at OTA start, resumed on OTA failure +#endif + + ble_device_base::RawAdvertisementCallback raw_advertisement_callback_{}; +#ifdef ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT + // Parsed-advertisement consumers registered through ble_device_base. + // Codegen-sized: no heap allocation, no std::vector template instantiations. + StaticVector listeners_; + // Per-period "Found device" DEBUG log with MAC dedup — shared implementation + // in ble_device_base, identical output on every tracker backend. Guarded like + // its only writer so a no-listener build does not carry an unused vector. + ble_device_base::DiscoveredDeviceLog discovered_log_{}; +#endif +}; + +} // namespace esphome::rp2_ble_tracker + +#endif // USE_RP2 diff --git a/esphome/components/rtl87xx/__init__.py b/esphome/components/rtl87xx/__init__.py index a3b1dba4f2..a8eabae9a0 100644 --- a/esphome/components/rtl87xx/__init__.py +++ b/esphome/components/rtl87xx/__init__.py @@ -51,7 +51,11 @@ def _set_core_data(config): return config -CONFIG_SCHEMA = libretiny.BASE_SCHEMA +# extend({}) makes this platform's own schema instance: BASE_SCHEMA is shared +# by every LibreTiny platform, and prepending this platform's _set_core_data +# onto the shared object would run it for every platform's validation once two +# platform modules are imported in one process (device-builder, tests). +CONFIG_SCHEMA = libretiny.BASE_SCHEMA.extend({}) PIN_SCHEMA = libretiny.gpio.BASE_PIN_SCHEMA diff --git a/esphome/components/script/script.h b/esphome/components/script/script.h index 790ac107c5..63d0ff7cb3 100644 --- a/esphome/components/script/script.h +++ b/esphome/components/script/script.h @@ -142,6 +142,9 @@ template class QueueingScript : public Script, public Com // Use std::make_unique to replace the unique_ptr this->var_queue_[write_pos] = std::make_unique>(x...); this->num_queued_++; + // Enable loop now that there is something to dequeue - don't call loop() + // synchronously! Let the event loop call it to avoid reentrancy issues + this->enable_loop(); return; } @@ -168,6 +171,15 @@ template class QueueingScript : public Script, public Com this->queue_front_ = (this->queue_front_ + 1) % queue_capacity; this->trigger_tuple_(*tuple_ptr, std::make_index_sequence{}); } + if (this->num_queued_ == 0 && !this->is_idle()) { + // Queue is now empty - disable loop until the next execute() queues an + // instance. The inline is_idle() check skips the out-of-line call when + // the loop is already disabled (execute() calls loop() synchronously). + // This can run before this component's setup() (execute() from on_boot), + // which leaves the state machine in LOOP_DONE and skips call_setup(); + // this class therefore must not rely on a setup() override. + this->disable_loop(); + } } void set_max_runs(int max_runs) { max_runs_ = max_runs; } diff --git a/esphome/components/substitutions/__init__.py b/esphome/components/substitutions/__init__.py index ea79054c88..b4fcf36c9e 100644 --- a/esphome/components/substitutions/__init__.py +++ b/esphome/components/substitutions/__init__.py @@ -1,5 +1,7 @@ from collections import ChainMap +from itertools import product import logging +import re from typing import Any import esphome @@ -7,6 +9,7 @@ from esphome import core from esphome.config_helpers import Extend, Remove, merge_config, merge_dicts_ordered import esphome.config_validation as cv from esphome.const import CONF_SUBSTITUTIONS, VALID_SUBSTITUTIONS_CHARACTERS +from esphome.expression import JINJA_PROG from esphome.types import ConfigType from esphome.util import OrderedDict from esphome.yaml_util import ( @@ -27,6 +30,14 @@ _LOGGER = logging.getLogger(__name__) ContextVars = ChainMap[str, Any] ErrList = list[tuple[UndefinedError, DocumentPath, Any]] +# Candidate-pattern shaping for include_candidate_patterns. +_ADJACENT_WILDCARDS_RE = re.compile(r"\*+") +# Dots are included so a variant like `../*` counts as fully dynamic too; +# it would otherwise glob everything in the parent directory. +_WILDCARDS_ONLY_RE = re.compile(r"[*./\\]+") +_GLOB_META_RE = re.compile(r"[?\[]") +_STRING_LITERAL_RE = re.compile(r"'([^']*)'|\"([^\"]*)\"") + # Module-level instance is safe: context_vars is passed per-call, and context_trace # is stack-saved/restored within expand(). Not thread-safe — only use from one thread. jinja = Jinja() @@ -360,9 +371,7 @@ def resolve_include( ) substituted = filename != original_str if substituted: - include = IncludeFile( - include.parent_file, filename, include.vars, include.yaml_loader - ) + include = include.with_file(filename) try: return include.load() except esphome.core.EsphomeError as err: @@ -374,6 +383,45 @@ def resolve_include( ) from err +def include_candidate_patterns(value: str) -> list[str]: + """Expand a substitution/Jinja-templated path into glob-style candidate patterns. + + Mirrors the two phases of :func:`_expand_substitutions` without variable + values: ``$var`` / ``${var}`` references become ``*`` and each remaining + Jinja expression contributes one pattern per quoted string literal it + holds (``*`` when it holds none), so every conditional branch is a + candidate — deliberately over-inclusive. Emitted wildcard patterns are + glob-safe: adjacent wildcards collapse (no recursive ``**``), ``[`` / + ``?`` from the filename text are escaped, and variants reduced to + nothing but wildcards, dots and separators are dropped so a fully + dynamic filename never expands to "everything in the directory", + including via a ``../*`` parent traversal. + """ + # Replacing $var / ${var} first also keeps JINJA_PROG's first-} span + # matching correct for references nested inside string literals, the + # same ordering _expand_substitutions relies on. + value = cv.VARIABLE_PROG.sub("*", value) + options = [ + [a or b for a, b in _STRING_LITERAL_RE.findall(expr)] or ["*"] + for expr in JINJA_PROG.findall(value) + ] + + variants: list[str] = [] + for combination in product(*options): + replacements = iter(combination) + spliced = JINJA_PROG.sub(lambda _, _next=replacements: next(_next), value) + variants.append(_ADJACENT_WILDCARDS_RE.sub("*", spliced)) + + patterns: list[str] = [] + for variant in dict.fromkeys(variants): + if not variant or _WILDCARDS_ONLY_RE.fullmatch(variant): + continue + if "*" in variant: + variant = _GLOB_META_RE.sub(r"[\g<0>]", variant) + patterns.append(variant) + return patterns + + def _substitute_include( include: IncludeFile, path: DocumentPath, diff --git a/esphome/components/template/text/template_text.cpp b/esphome/components/template/text/template_text.cpp index af134e6ed4..ffe11cf229 100644 --- a/esphome/components/template/text/template_text.cpp +++ b/esphome/components/template/text/template_text.cpp @@ -20,18 +20,14 @@ void TemplateText::setup() { // Need std::string for pref_->setup() to fill from flash std::string value{this->initial_value_ != nullptr ? this->initial_value_ : ""}; - // For future hash migration: use migrate_entity_preference_() with: - // old_key = get_preference_hash() + extra - // new_key = get_preference_hash_v2() + extra - // See: https://github.com/esphome/backlog/issues/85 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - uint32_t key = this->get_preference_hash(); -#pragma GCC diagnostic pop - key += this->traits.get_min_length() << 2; - key += this->traits.get_max_length() << 4; - key += fnv1_hash(this->traits.get_pattern_c_str()) << 6; - this->pref_->setup(key, value); + uint32_t extra = 0; + extra += this->traits.get_min_length() << 2; + extra += this->traits.get_max_length() << 4; + extra += fnv1_hash(this->traits.get_pattern_c_str()) << 6; + // TextSaver::setup() picks the key for the platform and migrates old data once + uint32_t key = this->preference_key_base_() + extra; + uint32_t old_key = this->old_preference_key_base_() + extra; + this->pref_->setup(key, old_key, value); if (!value.empty()) this->publish_state(value); } diff --git a/esphome/components/template/text/template_text.h b/esphome/components/template/text/template_text.h index 229a61d9b8..beeea4396a 100644 --- a/esphome/components/template/text/template_text.h +++ b/esphome/components/template/text/template_text.h @@ -14,7 +14,9 @@ class TemplateTextSaverBase { public: virtual bool save(const std::string &value) { return true; } - virtual void setup(uint32_t id, std::string &value) {} + /// old_id is the pre-2026.8.0 preference key; data stored under it is moved to id once. + /// See: https://github.com/esphome/backlog/issues/85 + virtual void setup(uint32_t id, uint32_t old_id, std::string &value) {} protected: ESPPreferenceObject pref_; @@ -45,11 +47,16 @@ template class TextSaver : public TemplateTextSaverBase { // Make the preference object. Fill the provided location with the saved data // If it is available, else leave it alone - void setup(uint32_t id, std::string &value) override { - this->pref_ = global_preferences->make_preference(id); - + void setup(uint32_t id, uint32_t old_id, std::string &value) override { char temp[SZ + 1]; +#ifdef USE_PREFERENCE_KEY_LOOKUP + this->pref_ = global_preferences->make_preference(id); + bool hasdata = migrate_preference(this->pref_, reinterpret_cast(temp), SZ + 1, old_id, id); +#else + // Slot-based backends keep the old key; it is only a validity tag on a positional slot + this->pref_ = global_preferences->make_preference(old_id); bool hasdata = this->pref_.load(&temp); +#endif if (hasdata) { size_t len = static_cast(temp[0]); diff --git a/esphome/components/thermopro_ble/thermopro_ble.cpp b/esphome/components/thermopro_ble/thermopro_ble.cpp index 2a950d3664..72e398f774 100644 --- a/esphome/components/thermopro_ble/thermopro_ble.cpp +++ b/esphome/components/thermopro_ble/thermopro_ble.cpp @@ -92,7 +92,7 @@ bool ThermoProBLE::parse_device(const esp32_ble_tracker::ESPBTDevice &device) { return success; } -void ThermoProBLE::update_device_type_(const std::string &device_name) { +void ThermoProBLE::update_device_type_(StringRef device_name) { // check for changed device name (should only happen on initial call) if (this->device_name_ == device_name) { return; diff --git a/esphome/components/thermopro_ble/thermopro_ble.h b/esphome/components/thermopro_ble/thermopro_ble.h index 2d7523e07a..ca04fbea39 100644 --- a/esphome/components/thermopro_ble/thermopro_ble.h +++ b/esphome/components/thermopro_ble/thermopro_ble.h @@ -41,7 +41,7 @@ class ThermoProBLE final : public Component, public esp32_ble_tracker::ESPBTDevi sensor::Sensor *humidity_{nullptr}; sensor::Sensor *battery_level_{nullptr}; - void update_device_type_(const std::string &device_name); + void update_device_type_(StringRef device_name); }; } // namespace esphome::thermopro_ble diff --git a/esphome/components/udp/packet_transport/__init__.py b/esphome/components/udp/packet_transport/__init__.py index b6957a372b..e725276717 100644 --- a/esphome/components/udp/packet_transport/__init__.py +++ b/esphome/components/udp/packet_transport/__init__.py @@ -1,12 +1,11 @@ import esphome.codegen as cg -from esphome.components.api import CONF_ENCRYPTION from esphome.components.packet_transport import ( CONF_PING_PONG_ENABLE, PacketTransport, new_packet_transport, transport_schema, ) -from esphome.const import CONF_BINARY_SENSORS, CONF_SENSORS +from esphome.const import CONF_BINARY_SENSORS, CONF_ENCRYPTION, CONF_SENSORS from esphome.cpp_types import PollingComponent from .. import UDP_SCHEMA, register_udp_client, udp_ns diff --git a/esphome/components/zephyr/__init__.py b/esphome/components/zephyr/__init__.py index 9f755a6eea..338d1986ea 100644 --- a/esphome/components/zephyr/__init__.py +++ b/esphome/components/zephyr/__init__.py @@ -158,6 +158,8 @@ def add_extra_script(stage: str, filename: str, path: Path) -> None: def zephyr_to_code(config: ConfigType) -> None: cg.add_build_flag("-DUSE_ZEPHYR") cg.add_define("USE_NATIVE_64BIT_TIME") + # The settings subsystem finds stored preferences by key, so key migration is possible + cg.add_define("USE_PREFERENCE_KEY_LOOKUP") cg.set_cpp_standard("gnu++20") # c++ support zephyr_add_prj_conf("FPU", True) diff --git a/esphome/components/zephyr/library.py b/esphome/components/zephyr/library.py index 7654e63700..0e6551ccf1 100644 --- a/esphome/components/zephyr/library.py +++ b/esphome/components/zephyr/library.py @@ -65,10 +65,16 @@ def generate_cmakelists_txt(component: ConvertedLibrary) -> str: """ build = component.data.get("build", {}) + # The library's own files live in source_path (the user's directory for a + # local library, the downloaded dir otherwise); the generated zephyr/ files + # go under component.path. Sources are already emitted as absolute paths, so + # they resolve correctly wherever source_path points. + read_path = component.source_dir + build_src_dir = build.get("srcDir") if not build_src_dir: for d in ["src", "Src", "."]: - if (component.path / Path(d)).is_dir(): + if (read_path / Path(d)).is_dir(): build_src_dir = d break @@ -77,7 +83,7 @@ def generate_cmakelists_txt(component: ConvertedLibrary) -> str: build_flags = ensure_list(build.get("flags", DEFAULT_BUILD_FLAGS)) src_files = collect_filtered_files( - component.path / Path(build_src_dir), build_src_filter + read_path / Path(build_src_dir), build_src_filter ) src_files = sorted( str(Path(p).resolve()) @@ -91,15 +97,19 @@ def generate_cmakelists_txt(component: ConvertedLibrary) -> str: link_directories, build_flags = split_list_by_condition( build_flags, lambda a: a[2:].strip() if a.startswith("-L") else None ) + # The zephyr/CMakeLists lives in a subdir, so a relative -L would resolve + # from there rather than the library root; make link dirs absolute against + # the library's own directory (source_dir), matching src/include handling. + link_directories = [str((read_path / Path(d)).resolve()) for d in link_directories] link_libraries, build_flags = split_list_by_condition( build_flags, lambda a: a[2:].strip() if a.startswith("-l") else None ) include_dirs = [build_include_dir, build_src_dir, *include_dir_flags] include_dirs = [ - str((component.path / Path(d)).resolve()) + str((read_path / Path(d)).resolve()) for d in include_dirs - if (component.path / Path(d)).is_dir() + if (read_path / Path(d)).is_dir() ] lines = [f"zephyr_library_named({component.get_require_name()})"] diff --git a/esphome/components/zephyr/preferences.cpp b/esphome/components/zephyr/preferences.cpp index c26a1d6d53..ed22613625 100644 --- a/esphome/components/zephyr/preferences.cpp +++ b/esphome/components/zephyr/preferences.cpp @@ -58,12 +58,19 @@ void ZephyrPreferences::open() { ESP_LOGD(TAG, "Loaded %zu settings.", this->backends_.size()); } -ESPPreferenceObject ZephyrPreferences::make_preference(size_t length, uint32_t type) { +ZephyrPreferenceBackend *ZephyrPreferences::find_backend_(uint32_t type) { for (auto *backend : this->backends_) { if (backend->get_type() == type) { - return ESPPreferenceObject(backend); + return backend; } } + return nullptr; +} + +ESPPreferenceObject ZephyrPreferences::make_preference(size_t length, uint32_t type) { + if (auto *backend = this->find_backend_(type)) { + return ESPPreferenceObject(backend); + } auto *pref = new ZephyrPreferenceBackend(type); // NOLINT(cppcoreguidelines-owning-memory) char key_buf[KEY_BUFFER_SIZE]; pref->format_key(key_buf, sizeof(key_buf)); @@ -72,6 +79,13 @@ ESPPreferenceObject ZephyrPreferences::make_preference(size_t length, uint32_t t return ESPPreferenceObject(pref); } +bool ZephyrPreferences::load_from_key(uint32_t type, uint8_t *data, size_t len) { + // Stored settings are preloaded into backends_ at boot by settings_load_subtree(), + // so a key with no registered backend has no stored data. + auto *backend = this->find_backend_(type); + return backend != nullptr && backend->load(data, len); +} + bool ZephyrPreferences::sync() { ESP_LOGD(TAG, "Save settings"); int err = settings_save(); diff --git a/esphome/components/zephyr/preferences.h b/esphome/components/zephyr/preferences.h index 9e2555f910..b1ad95fd74 100644 --- a/esphome/components/zephyr/preferences.h +++ b/esphome/components/zephyr/preferences.h @@ -16,10 +16,13 @@ class ZephyrPreferences final : public PreferencesMixin { return this->make_preference(length, type); } ESPPreferenceObject make_preference(size_t length, uint32_t type); + /// One-shot read of a stored preference by key, without allocating or registering a backend + bool load_from_key(uint32_t type, uint8_t *data, size_t len); bool sync(); bool reset(); protected: + ZephyrPreferenceBackend *find_backend_(uint32_t type); std::vector backends_; static int load_setting(const char *name, size_t len, settings_read_cb read_cb, void *cb_arg); diff --git a/esphome/components/zigbee/__init__.py b/esphome/components/zigbee/__init__.py index 775fb35140..47913b34d7 100644 --- a/esphome/components/zigbee/__init__.py +++ b/esphome/components/zigbee/__init__.py @@ -13,7 +13,7 @@ from esphome.components.esp32.const import ( VARIANT_ESP32S31, ) import esphome.config_validation as cv -from esphome.const import CONF_ID, CONF_INTERNAL, CONF_MODEL, CONF_NAME +from esphome.const import CONF_ID, CONF_INTERNAL, CONF_MODEL, CONF_NAME, CONF_ON_START from esphome.core import CORE, CoroPriority, coroutine_with_priority from esphome.types import ConfigType @@ -108,6 +108,7 @@ CONFIG_SCHEMA = cv.All( ), cv.Optional(CONF_ROUTER, default=False): cv.boolean, cv.Optional(CONF_ON_JOIN): automation.validate_automation({}), + cv.Optional(CONF_ON_START): automation.validate_automation({}), cv.OnlyWith(CONF_WIPE_ON_BOOT, "nrf52", default=False): cv.All( cv.Any( cv.boolean, @@ -175,6 +176,7 @@ FINAL_VALIDATE_SCHEMA = cv.All( _CALLBACK_AUTOMATIONS = [ automation.CallbackAutomation(CONF_ON_JOIN, "add_on_join_callback", [(bool, "x")]), + automation.CallbackAutomation(CONF_ON_START, "add_on_start_callback", []), ] diff --git a/esphome/components/zigbee/zigbee_attribute_esp32.cpp b/esphome/components/zigbee/zigbee_attribute_esp32.cpp index d7176e6ca5..1fb8d1abe4 100644 --- a/esphome/components/zigbee/zigbee_attribute_esp32.cpp +++ b/esphome/components/zigbee/zigbee_attribute_esp32.cpp @@ -9,16 +9,18 @@ namespace esphome::zigbee { static const char *const TAG = "zigbee.attribute"; void ZigbeeAttribute::set_attr_() { - if (!this->zb_->is_connected()) { + if (!this->zb_->is_started()) { return; } if (esp_zigbee_lock_acquire(10 / portTICK_PERIOD_MS)) { ezb_zcl_status_t state = ezb_zcl_set_attr_value(this->endpoint_id_, this->cluster_id_, this->role_, this->attr_id_, EZB_ZCL_STD_MANUF_CODE, this->value_p_, false); + // cleared before report_() so it can disable the loop + // when the report has to wait for join + this->set_attr_requested_ = false; if (this->force_report_) { this->report_(true); } - this->set_attr_requested_ = false; // Check for error if (state != EZB_ZCL_STATUS_SUCCESS) { ESP_LOGE(TAG, "Setting attribute failed, ZCL status: %u", static_cast(state)); @@ -28,7 +30,14 @@ void ZigbeeAttribute::set_attr_() { } void ZigbeeAttribute::report_(bool has_lock) { - if (!this->zb_->is_connected() || !this->report_enabled) { + if (!this->report_enabled) { + return; + } + if (!this->zb_->is_joined()) { + this->report_requested_ = true; + if (!this->set_attr_requested_) { + this->disable_loop(); + } return; } if (has_lock or esp_zigbee_lock_acquire(10 / portTICK_PERIOD_MS)) { @@ -44,6 +53,7 @@ void ZigbeeAttribute::report_(bool has_lock) { cmd.payload.attr_id = this->attr_id_; ezb_zcl_report_attr_cmd_req(&cmd); + this->report_requested_ = false; if (!has_lock) { esp_zigbee_lock_release(); } @@ -55,6 +65,11 @@ void ZigbeeAttribute::set_report(ZigbeeReportT report) { if (report == ZigbeeReportT::ZIGBEE_REPORT_FORCE) { this->force_report_ = true; } + this->zb_->add_on_join_callback([this](bool) { + if (this->report_requested_) { + this->enable_loop(); + } + }); } void ZigbeeAttribute::loop() { @@ -62,7 +77,11 @@ void ZigbeeAttribute::loop() { this->set_attr_(); } - if (!this->set_attr_requested_) { + if (this->report_requested_) { + this->report_(false); + } + + if (!this->report_requested_ && !this->set_attr_requested_) { this->disable_loop(); } } diff --git a/esphome/components/zigbee/zigbee_attribute_esp32.h b/esphome/components/zigbee/zigbee_attribute_esp32.h index e5f8c8b1cf..fc229b4e95 100644 --- a/esphome/components/zigbee/zigbee_attribute_esp32.h +++ b/esphome/components/zigbee/zigbee_attribute_esp32.h @@ -66,6 +66,7 @@ class ZigbeeAttribute final : public Component { float scale_; void *value_p_{nullptr}; bool set_attr_requested_{false}; + bool report_requested_{false}; bool force_report_{false}; }; diff --git a/esphome/components/zigbee/zigbee_ep_esp32.py b/esphome/components/zigbee/zigbee_ep_esp32.py index 2ed3dddb67..c2001c66d6 100644 --- a/esphome/components/zigbee/zigbee_ep_esp32.py +++ b/esphome/components/zigbee/zigbee_ep_esp32.py @@ -83,7 +83,7 @@ ep_configs: dict[str, dict[str, Any]] = { } -def get_next_ep_num(eps: list[int]) -> int: +def _get_next_ep_num(eps: list[int]) -> int: try: ep_num = [i for i in range(1, CONF_MAX_EP_NUMBER + 1) if i not in eps][0] eps.append(ep_num) @@ -94,7 +94,7 @@ def get_next_ep_num(eps: list[int]) -> int: return ep_num -def compare_clusters( +def _compare_clusters( existing_ep: dict[str, Any], ep: dict[str, Any], ) -> tuple[str | int, str] | None: @@ -105,12 +105,12 @@ def compare_clusters( return None -def merge_endpoints( +def _merge_endpoints( existing_ep: dict[str, Any], ep: dict[str, Any], use_type: bool | None, ) -> bool: - if compare_clusters(existing_ep, ep): + if _compare_clusters(existing_ep, ep): return False if ( ep.get(DEVICE_TYPE) @@ -134,7 +134,12 @@ def merge_endpoints( return True -def validate_endpoints(ep_dict: dict[int, dict]) -> None: +def _validate_endpoints(ep_dict: dict[int, dict]) -> None: + """Validate endpoint device type selection before endpoint creation. + + This resolves any deferred device type selections stored in CONF_USE_DEVICE_TYPE, + ensuring each endpoint has at most one active device type. + """ for num, ep in ep_dict.items(): types_dict = ep.get(CONF_USE_DEVICE_TYPE) if not types_dict: @@ -157,10 +162,18 @@ def validate_endpoints(ep_dict: dict[int, dict]) -> None: def create_ep(router: bool) -> None: + """Finalize Zigbee endpoint creation and normalize endpoint storage. + + Validate endpoints, merge endpoints, and assign numbers to endpoints without an explicit number. + This is called from final_validate. + + Args: + router: Whether the device is acting as a Zigbee router. + """ zb_data = CORE.data.setdefault(KEY_ZIGBEE, {}) ep_dict: dict[int, dict] = zb_data.setdefault(KEY_ZIGBEE_EP, {}) ep_list: list[dict] = zb_data.setdefault(KEY_ZIGBEE_EP_NO_NUM, []) - validate_endpoints(ep_dict) + _validate_endpoints(ep_dict) # create dummy endpoint if list is empty if not ep_dict and not ep_list: ep_type = "CUSTOM_ATTR" @@ -173,7 +186,7 @@ def create_ep(router: bool) -> None: for ep in ep_list: added = False for existing_ep in ep_list_new: - if merge_endpoints(existing_ep, ep, ep.get(CONF_USE_DEVICE_TYPE)): + if _merge_endpoints(existing_ep, ep, ep.get(CONF_USE_DEVICE_TYPE)): added = True break if not added: @@ -182,7 +195,7 @@ def create_ep(router: bool) -> None: # Add endpoints with no number to the endpoint dict with a new number eps = list(ep_dict.keys()) for ep in ep_list_new: - ep_num = get_next_ep_num(eps) + ep_num = _get_next_ep_num(eps) ep_dict[ep_num] = ep # clear list so that it is not processed again @@ -195,6 +208,14 @@ def create_ep(router: bool) -> None: def add_ep(ep: dict[str, Any], ep_num: int | None, use_type: bool | None) -> None: + """Add a Zigbee endpoint configuration to CORE.data. + + Args: + ep: Endpoint configuration dictionary. + ep_num: Optional explicit endpoint number. + use_type: Optional boolean indicating whether this component's device type should be + used for the endpoint (True claims it, False drops it, None leaves it as a candidate). + """ zb_data = CORE.data.setdefault(KEY_ZIGBEE, {}) if use_type is False: ep.pop(DEVICE_TYPE, None) @@ -208,7 +229,7 @@ def add_ep(ep: dict[str, Any], ep_num: int | None, use_type: bool | None) -> Non if ep_num in ep_dict: # check if the existing endpoint has same clusters existing_ep = ep_dict[ep_num] - if cl := compare_clusters( + if cl := _compare_clusters( existing_ep, ep, ): diff --git a/esphome/components/zigbee/zigbee_esp32.cpp b/esphome/components/zigbee/zigbee_esp32.cpp index 3e0f6cd745..482995e2c5 100644 --- a/esphome/components/zigbee/zigbee_esp32.cpp +++ b/esphome/components/zigbee/zigbee_esp32.cpp @@ -36,6 +36,17 @@ uint8_t *get_zcl_string(const char *str, uint8_t max_size, bool use_max_size) { return zcl_str; } +void ZigbeeComponent::factory_reset() { + esp_zigbee_lock_acquire(portMAX_DELAY); + if (this->joined_) { + // send leave request and trigger EZB_ZDO_SIGNAL_LEAVE + ezb_bdb_reset_via_local_action(); + } else { + esp_zigbee_factory_reset(); // triggers a reboot + } + esp_zigbee_lock_release(); +} + void ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(ezb_bdb_comm_mode_mask_t mode) { if (!esp_zigbee_lock_acquire(10 / portTICK_PERIOD_MS)) { global_zigbee->set_timeout("zb_init", 10, [mode]() { ZigbeeComponent::esp_zigbee_alarm_bdb_commissioning(mode); }); @@ -53,6 +64,8 @@ bool ZigbeeComponent::app_signal_handler(const ezb_app_signal_t *app_signal) { switch (signal_type) { case EZB_ZDO_SIGNAL_SKIP_STARTUP: ESP_LOGD(TAG, "Zigbee stack initialized"); + global_zigbee->started_ = true; + global_zigbee->enable_loop_soon_any_context(); ezb_bdb_start_top_level_commissioning(EZB_BDB_MODE_INITIALIZATION); break; case EZB_BDB_SIGNAL_DEVICE_FIRST_START: @@ -60,14 +73,14 @@ bool ZigbeeComponent::app_signal_handler(const ezb_app_signal_t *app_signal) { ezb_bdb_comm_status_t status = *((ezb_bdb_comm_status_t *) ezb_app_signal_get_params(app_signal)); if (status == EZB_BDB_STATUS_SUCCESS) { ESP_LOGD(TAG, "Device started up in %sfactory-reset mode", ezb_bdb_is_factory_new() ? "" : "non "); - global_zigbee->started = true; if (ezb_bdb_is_factory_new()) { - global_zigbee->factory_new = true; + global_zigbee->factory_new_ = true; ESP_LOGD(TAG, "Start network steering"); ezb_bdb_start_top_level_commissioning(EZB_BDB_MODE_NETWORK_STEERING); } else { ESP_LOGD(TAG, "Device rebooted"); - global_zigbee->joined = true; + global_zigbee->joined_ = true; + global_zigbee->join_pending_ = true; global_zigbee->enable_loop_soon_any_context(); } } else { @@ -85,7 +98,8 @@ bool ZigbeeComponent::app_signal_handler(const ezb_app_signal_t *app_signal) { ezb_nwk_get_extended_panid(&extended_pan_id); ESP_LOGD(TAG, "Joined network successfully: PAN ID(0x%04hx, EXT: 0x%llx), Channel(%d), Short Address(0x%04hx)", ezb_nwk_get_panid(), extended_pan_id.u64, ezb_nwk_get_current_channel(), ezb_nwk_get_short_address()); - global_zigbee->joined = true; + global_zigbee->joined_ = true; + global_zigbee->join_pending_ = true; global_zigbee->enable_loop_soon_any_context(); } else { ESP_LOGD(TAG, "Failed to join network with status(0x%02x)", status); @@ -105,7 +119,29 @@ bool ZigbeeComponent::app_signal_handler(const ezb_app_signal_t *app_signal) { const ezb_zdo_signal_leave_params_t *leave_params = (const ezb_zdo_signal_leave_params_t *) ezb_app_signal_get_params(app_signal); if (leave_params->leave_type == EZB_ZDO_LEAVE_TYPE_RESET) { - esp_zigbee_factory_reset(); + esp_zigbee_factory_reset(); // triggers a reboot + } + global_zigbee->joined_ = false; + } break; + case EZB_NWK_SIGNAL_NETWORK_STATUS: { + const ezb_nwk_signal_network_status_params_t *network_status_params = + (const ezb_nwk_signal_network_status_params_t *) ezb_app_signal_get_params(app_signal); + if (network_status_params->status == EZB_NWK_NETWORK_STATUS_PARENT_LINK_FAILURE) { + global_zigbee->joined_ = false; + ESP_LOGW(TAG, "Parent link failure, attempting rejoin"); + ezb_zdo_nwk_mgmt_leave_req_t leave_req = { + .dst_nwk_addr = ezb_nwk_get_short_address(), + .field = + { + .remove_children = false, + .rejoin = true, + }, + }; + // Send leave request to the network to rejoin + // triggers EZB_ZDO_SIGNAL_LEAVE signal first, then EZB_BDB_SIGNAL_DEVICE_REBOOT + ezb_zdo_nwk_mgmt_leave_req(&leave_req); + } else { + ESP_LOGD(TAG, "Zigbee APP Signal NETWORK_STATUS: 0x%02x", network_status_params->status); } } break; default: @@ -303,9 +339,13 @@ void ZigbeeComponent::setup() { } void ZigbeeComponent::loop() { - if (this->joined.exchange(false)) { - this->connected_ = true; - this->join_cb_.call(this->factory_new); + if (!this->start_reported_ && this->started_) { + this->start_cb_.call(); + this->start_reported_ = true; + } + if (this->join_pending_.exchange(false)) { + this->join_cb_.call(this->factory_new_); + this->factory_new_ = false; } this->disable_loop(); } diff --git a/esphome/components/zigbee/zigbee_esp32.h b/esphome/components/zigbee/zigbee_esp32.h index f4bafac294..c19fc3ad63 100644 --- a/esphome/components/zigbee/zigbee_esp32.h +++ b/esphome/components/zigbee/zigbee_esp32.h @@ -54,20 +54,19 @@ class ZigbeeComponent final : public Component { static bool app_signal_handler(const ezb_app_signal_t *app_signal); static void esp_zigbee_alarm_bdb_commissioning(ezb_bdb_comm_mode_mask_t mode); - void factory_reset() { - esp_zigbee_lock_acquire(portMAX_DELAY); - esp_zigbee_factory_reset(); // triggers a reboot - esp_zigbee_lock_release(); - } + void factory_reset(); template void add_on_join_callback(F &&cb) { this->join_cb_.add(std::forward(cb)); } + template void add_on_start_callback(F &&cb) { this->start_cb_.add(std::forward(cb)); } bool is_battery_powered() { return this->basic_cluster_data_.power_source == EZB_ZCL_BASIC_POWER_SOURCE_BATTERY; } - bool is_started() { return this->started; } - bool is_connected() { return this->connected_; } - std::atomic started = false; - std::atomic joined = false; - std::atomic factory_new = false; + + // True after the Zigbee stack has been initialized and the device has started up. Is set before the stack started + // network commissioning or has joined a network and won't be reset until the device is rebooted. + bool is_started() { return this->started_; } + + // True if the device has joined a network and is ready to send and receive messages. + bool is_joined() { return this->joined_; } protected: struct { @@ -76,7 +75,6 @@ class ZigbeeComponent final : public Component { uint8_t *date; uint8_t power_source; } basic_cluster_data_; - bool connected_ = false; #ifdef CONFIG_ZB_ZED ezb_nwk_device_type_t device_role_ = EZB_NWK_DEVICE_TYPE_END_DEVICE; #else @@ -92,6 +90,12 @@ class ZigbeeComponent final : public Component { std::map, ZigbeeAttribute *> attributes_; ezb_af_device_desc_t dev_desc_; CallbackManager join_cb_{}; + LazyCallbackManager start_cb_{}; + bool start_reported_{false}; + std::atomic started_ = false; + std::atomic joined_ = false; + std::atomic join_pending_ = false; + std::atomic factory_new_ = false; }; template diff --git a/esphome/components/zigbee/zigbee_esp32.py b/esphome/components/zigbee/zigbee_esp32.py index 116dce8cc5..8e63c09e67 100644 --- a/esphome/components/zigbee/zigbee_esp32.py +++ b/esphome/components/zigbee/zigbee_esp32.py @@ -13,6 +13,7 @@ from esphome.components.esp32 import ( ) import esphome.config_validation as cv from esphome.const import ( + CONF_ACCURACY_DECIMALS, CONF_AP, CONF_DEVICE, CONF_DEVICE_CLASS, @@ -185,6 +186,7 @@ def validate_sensor_esp32(config: ConfigType) -> ConfigType: unit = config.get(CONF_UNIT_OF_MEASUREMENT) apptype = ANALOG_INPUT_APPTYPE.get((dev_class, unit)) bacunit = BACNET_UNITS.get(unit, BACNET_UNIT_NO_UNITS) + accuracy = config.get(CONF_ACCURACY_DECIMALS) if apptype is not None: ep[CONF_CLUSTERS][0][CONF_ATTRIBUTES].append( { @@ -200,6 +202,15 @@ def validate_sensor_esp32(config: ConfigType) -> ConfigType: CONF_TYPE: "ENUM16", }, ) + if accuracy is not None: + # Analog Input Resolution (0x006A): smallest reportable change + ep[CONF_CLUSTERS][0][CONF_ATTRIBUTES].append( + { + CONF_ATTRIBUTE_ID: 0x6A, + CONF_VALUE: 10**-accuracy, + CONF_TYPE: "SINGLE", + }, + ) setup_attributes(config, ep[CONF_CLUSTERS]) add_ep(ep, config.get(CONF_ENDPOINT), config.get(CONF_USE_DEVICE_TYPE)) return config diff --git a/esphome/components/zigbee/zigbee_zephyr.cpp b/esphome/components/zigbee/zigbee_zephyr.cpp index fedcb4a9c2..b8bb0a2036 100644 --- a/esphome/components/zigbee/zigbee_zephyr.cpp +++ b/esphome/components/zigbee/zigbee_zephyr.cpp @@ -30,6 +30,9 @@ void ZigbeeComponent::zboss_signal_handler_esphome(zb_bufid_t bufid) { switch (sig) { case ZB_ZDO_SIGNAL_SKIP_STARTUP: ESP_LOGD(TAG, "ZB_ZDO_SIGNAL_SKIP_STARTUP, status: %d", status); + if (status == RET_OK) { + on_start_(); + } break; case ZB_ZDO_SIGNAL_PRODUCTION_CONFIG_READY: ESP_LOGD(TAG, "ZB_ZDO_SIGNAL_PRODUCTION_CONFIG_READY, status: %d", status); @@ -137,6 +140,13 @@ void ZigbeeComponent::on_join_(bool factory_new) { }); } +void ZigbeeComponent::on_start_() { + this->defer([this]() { + ESP_LOGD(TAG, "Started zigbee stack"); + this->start_cb_.call(); + }); +} + #ifdef USE_ZIGBEE_WIPE_ON_BOOT void ZigbeeComponent::erase_flash_(int area) { const struct flash_area *fap; diff --git a/esphome/components/zigbee/zigbee_zephyr.h b/esphome/components/zigbee/zigbee_zephyr.h index 8528aebff8..cd6deb0e95 100644 --- a/esphome/components/zigbee/zigbee_zephyr.h +++ b/esphome/components/zigbee/zigbee_zephyr.h @@ -75,6 +75,7 @@ class ZigbeeComponent final : public Component { this->callbacks_[endpoint - 1] = std::move(cb); } template void add_on_join_callback(F &&cb) { this->join_cb_.add(std::forward(cb)); } + template void add_on_start_callback(F &&cb) { this->start_cb_.add(std::forward(cb)); } void zboss_signal_handler_esphome(zb_bufid_t bufid); void after_reporting_info(zb_zcl_configure_reporting_req_t *config_rep_req, zb_zcl_attr_addr_info_t *attr_addr_info); void factory_reset(); @@ -86,12 +87,14 @@ class ZigbeeComponent final : public Component { protected: static void zcl_device_cb(zb_bufid_t bufid); void on_join_(bool factory_new); + void on_start_(); #ifdef USE_ZIGBEE_WIPE_ON_BOOT void erase_flash_(int area); #endif void dump_reporting_(); std::array, ZIGBEE_ENDPOINTS_COUNT> callbacks_{}; CallbackManager join_cb_; + LazyCallbackManager start_cb_; bool force_report_{false}; uint32_t sleep_time_{}; uint32_t sleep_remainder_{}; diff --git a/esphome/config.py b/esphome/config.py index 976faed447..b747c69b3a 100644 --- a/esphome/config.py +++ b/esphome/config.py @@ -1108,6 +1108,7 @@ def validate_config( config: dict[str, Any], command_line_substitutions: dict[str, Any] | None, skip_external_update: bool = False, + snapshot_user_config: bool = False, ) -> Config: result = Config() @@ -1218,11 +1219,13 @@ def validate_config( # Snapshot the user's config before any schema validation defaults are # applied. preload_core_config and later validation steps rewrite entries # in-place with defaulted values; deep-copying here preserves the - # user-supplied keys for `esphome config --no-defaults`. - result.user_config = copy.deepcopy(config) - if substitutions is not None: - result.user_config[CONF_SUBSTITUTIONS] = copy.deepcopy(substitutions) - result.user_config.move_to_end(CONF_SUBSTITUTIONS, last=False) + # user-supplied keys for `esphome config --no-defaults`. The deep copy is + # expensive, so it is only taken when that command actually asked for it. + if snapshot_user_config: + result.user_config = copy.deepcopy(config) + if substitutions is not None: + result.user_config[CONF_SUBSTITUTIONS] = copy.deepcopy(substitutions) + result.user_config.move_to_end(CONF_SUBSTITUTIONS, last=False) # 2. Load partial core config import esphome.core.config as core_config @@ -1335,7 +1338,9 @@ class InvalidYAMLError(EsphomeError): def _load_config( - command_line_substitutions: dict[str, Any], skip_external_update: bool = False + command_line_substitutions: dict[str, Any], + skip_external_update: bool = False, + snapshot_user_config: bool = False, ) -> Config: """Load the configuration file.""" try: @@ -1344,7 +1349,12 @@ def _load_config( raise InvalidYAMLError(e) from e try: - return validate_config(config, command_line_substitutions, skip_external_update) + return validate_config( + config, + command_line_substitutions, + skip_external_update=skip_external_update, + snapshot_user_config=snapshot_user_config, + ) except EsphomeError: raise except Exception: @@ -1353,10 +1363,16 @@ def _load_config( def load_config( - command_line_substitutions: dict[str, Any], skip_external_update: bool = False + command_line_substitutions: dict[str, Any], + skip_external_update: bool = False, + snapshot_user_config: bool = False, ) -> Config: try: - return _load_config(command_line_substitutions, skip_external_update) + return _load_config( + command_line_substitutions, + skip_external_update=skip_external_update, + snapshot_user_config=snapshot_user_config, + ) except vol.Invalid as err: raise EsphomeError(f"Error while parsing config: {err}") from err @@ -1497,11 +1513,17 @@ def strip_default_ids(config): def read_config( - command_line_substitutions: dict[str, Any], skip_external_update: bool = False + command_line_substitutions: dict[str, Any], + skip_external_update: bool = False, + snapshot_user_config: bool = False, ) -> Config | None: _LOGGER.info("Reading configuration %s...", CORE.config_path) try: - res = load_config(command_line_substitutions, skip_external_update) + res = load_config( + command_line_substitutions, + skip_external_update=skip_external_update, + snapshot_user_config=snapshot_user_config, + ) except EsphomeError as err: _LOGGER.error("Error while reading config: %s", err) return None diff --git a/esphome/config_validation.py b/esphome/config_validation.py index ff9170813c..c04d43bbee 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -4,7 +4,6 @@ from __future__ import annotations from collections.abc import Callable from contextlib import contextmanager, suppress -from dataclasses import dataclass from datetime import datetime from ipaddress import ( AddressValueError, @@ -88,6 +87,7 @@ from esphome.core import ( TimePeriodMinutes, TimePeriodNanoseconds, TimePeriodSeconds, + Version, ) from esphome.enum import StrEnum from esphome.expression import SUBSTITUTION_VARIABLE_PROG as VARIABLE_PROG @@ -99,7 +99,6 @@ from esphome.schema_extractors import ( schema_extractor_registry, schema_extractor_typed, ) -from esphome.util import parse_esphome_version from esphome.voluptuous_schema import _Schema from esphome.yaml_util import SensitiveStr, make_data_base @@ -408,42 +407,6 @@ class FinalExternalInvalid(Invalid): """Represents an invalid value in the final validation phase where the path should not be prepended.""" -@dataclass(frozen=True, order=True) -class Version: - major: int - minor: int - patch: int - extra: str = "" - - def __str__(self): - if self.extra: - return f"{self.major}.{self.minor}.{self.patch}-{self.extra}" - return f"{self.major}.{self.minor}.{self.patch}" - - @classmethod - def parse(cls, value: str) -> Version: - # The patch component is optional and defaults to 0, so "6.0" and - # "6.0-rc1" parse as 6.0.0 and 6.0.0-rc1. - match = re.match(r"^(\d+)\.(\d+)(?:\.(\d+))?[-.]?(\w*)$", value) - if match is None: - raise ValueError(f"Not a valid version number {value}") - major = int(match[1]) - minor = int(match[2]) - patch = int(match[3] or 0) - extra = match[4] or "" - return Version(major=major, minor=minor, patch=patch, extra=extra) - - @property - def is_beta(self) -> bool: - """Check if this version is a beta version.""" - return self.extra.startswith("b") - - @property - def is_dev(self) -> bool: - """Check if this version is a development version.""" - return self.extra.startswith("dev") - - def check_not_templatable(value): if isinstance(value, Lambda): raise Invalid("This option is not templatable!") @@ -2648,13 +2611,30 @@ def require_framework_version( return validator -def require_esphome_version(year, month, patch): +def require_esphome_version( + year: Version | int, month: int | None = None, patch: int | None = None +): + """Validator requiring at least the given ESPHome version. + + Accepts a single ``Version`` like the sibling + ``require_framework_version``, or the legacy ``(year, month, patch)`` + ints external components already pass. + """ + if isinstance(year, Version): + required = year + elif month is None or patch is None: + raise ValueError( + "require_esphome_version needs a Version or (year, month, patch)" + ) + else: + required = Version(year, month, patch) + def validator(value): - esphome_version = parse_esphome_version() - if esphome_version < (year, month, patch): - requires_version = f"{year}.{month}.{patch}" + # A dev or beta build of the required version still satisfies it, + # matching the old tuple comparison that dropped the suffix. + if Version.parse(ESPHOME_VERSION) < required: raise Invalid( - f"This component requires at least ESPHome version {requires_version}" + f"This component requires at least ESPHome version {required}" ) return value diff --git a/esphome/const.py b/esphome/const.py index a7306edeb0..a3e9f47909 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -121,6 +121,7 @@ PLATFORM_RP2040 = Platform.RP2040 PLATFORM_RTL87XX = Platform.RTL87XX +BUNDLE_EXTENSION = ".esphomebundle.tar.gz" SOURCE_FILE_EXTENSIONS = {".cpp", ".hpp", ".h", ".c", ".tcc", ".ino"} HEADER_FILE_EXTENSIONS = {".h", ".hpp", ".tcc"} SECRETS_FILES = ("secrets.yaml", "secrets.yml") @@ -347,6 +348,7 @@ CONF_DISABLE_CRC = "disable_crc" CONF_DISABLED = "disabled" CONF_DISABLED_BY_DEFAULT = "disabled_by_default" CONF_DISCONNECT_DELAY = "disconnect_delay" +CONF_DISCOVER_IP = "discover_ip" CONF_DISCOVERY = "discovery" CONF_DISCOVERY_OBJECT_ID_GENERATOR = "discovery_object_id_generator" CONF_DISCOVERY_PREFIX = "discovery_prefix" @@ -384,6 +386,7 @@ CONF_ENABLE_PIN = "enable_pin" CONF_ENABLE_PRIVATE_NETWORK_ACCESS = "enable_private_network_access" CONF_ENABLE_RRM = "enable_rrm" CONF_ENABLE_TIME = "enable_time" +CONF_ENCRYPTION = "encryption" CONF_ENERGY = "energy" CONF_ENTITY_CATEGORY = "entity_category" CONF_ENTITY_ID = "entity_id" @@ -1421,6 +1424,14 @@ KEY_FRAMEWORK_VERSION = "framework_version" KEY_NAME = "name" KEY_VARIANT = "variant" KEY_PAST_SAFE_MODE = "past_safe_mode" +# esp32 storage keys; defined here so the upload/logs fast path +# (storage_json.apply_to_core, espidf.toolchain) can use them without +# importing the esp32 component package. +KEY_ESP32 = "esp32" +# Also used by esp8266 to index its BOARDS metadata dicts, whose +# entries in boards.py spell the literal; do not change the value. +KEY_FLASH_SIZE = "flash_size" +KEY_IDF_VERSION = "idf_version" # Entity categories ENTITY_CATEGORY_NONE = "" diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index bf637d4c1f..e5b3ebb84d 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -1,5 +1,6 @@ from collections import defaultdict from contextlib import contextmanager +from dataclasses import dataclass import logging import math import os @@ -279,6 +280,42 @@ class TimePeriodMinutes(TimePeriod): pass +@dataclass(frozen=True, order=True) +class Version: + major: int + minor: int + patch: int + extra: str = "" + + def __str__(self): + if self.extra: + return f"{self.major}.{self.minor}.{self.patch}-{self.extra}" + return f"{self.major}.{self.minor}.{self.patch}" + + @classmethod + def parse(cls, value: str) -> "Version": + # The patch component is optional and defaults to 0, so "6.0" and + # "6.0-rc1" parse as 6.0.0 and 6.0.0-rc1. + match = re.match(r"^(\d+)\.(\d+)(?:\.(\d+))?[-.]?(\w*)$", value) + if match is None: + raise ValueError(f"Not a valid version number {value}") + major = int(match[1]) + minor = int(match[2]) + patch = int(match[3] or 0) + extra = match[4] or "" + return Version(major=major, minor=minor, patch=patch, extra=extra) + + @property + def is_beta(self) -> bool: + """Check if this version is a beta version.""" + return self.extra.startswith("b") + + @property + def is_dev(self) -> bool: + """Check if this version is a development version.""" + return self.extra.startswith("dev") + + LAMBDA_PROG = re.compile(r"\bid\(\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\)(\.?)") @@ -621,8 +658,8 @@ class EsphomeCore: # Key: platform name (e.g. "sensor", "binary_sensor"), Value: count self.platform_counts: defaultdict[str, int] = defaultdict(int) # Track entity unique IDs to handle duplicates - # Dict mapping (device_id, platform, sanitized_name) -> entity metadata - self.unique_ids: dict[tuple[str, str, str], EntityMetadata] = {} + # Dict mapping (device_id, platform, name_hash) -> entity metadata + self.unique_ids: dict[tuple[str, str, int], EntityMetadata] = {} # Whether ESPHome was started in verbose mode self.verbose = False # Whether ESPHome was started in quiet mode diff --git a/esphome/core/application.h b/esphome/core/application.h index a12cdc4ac8..a18a6b31c8 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -120,8 +120,8 @@ class Application { // NOLINTBEGIN(bugprone-macro-parentheses) #define ENTITY_TYPE_(type, singular, plural, count, upper) \ void register_##singular(type *obj) { this->plural##_.push_back(obj); } \ - void register_##singular(type *obj, const char *name, uint32_t object_id_hash, uint32_t entity_fields) { \ - obj->configure_entity_(name, object_id_hash, entity_fields); \ + void register_##singular(type *obj, const char *name, uint32_t entity_key, uint32_t entity_fields) { \ + obj->configure_entity_(name, entity_key, entity_fields); \ this->plural##_.push_back(obj); \ } #define ENTITY_CONTROLLER_TYPE_(type, singular, plural, count, upper, callback) \ @@ -329,7 +329,7 @@ class Application { #define GET_ENTITY_METHOD(entity_type, entity_name, entities_member) \ entity_type *get_##entity_name##_by_key(uint32_t key, uint32_t device_id, bool include_internal = false) { \ for (auto *obj : this->entities_member##_) { \ - if (obj->get_object_id_hash() == key && obj->get_device_id() == device_id && \ + if (obj->get_entity_key() == key && obj->get_device_id() == device_id && \ (include_internal || !obj->is_internal())) \ return obj; \ } \ @@ -340,7 +340,7 @@ class Application { #define GET_ENTITY_METHOD(entity_type, entity_name, entities_member) \ entity_type *get_##entity_name##_by_key(uint32_t key, bool include_internal = false) { \ for (auto *obj : this->entities_member##_) { \ - if (obj->get_object_id_hash() == key && (include_internal || !obj->is_internal())) \ + if (obj->get_entity_key() == key && (include_internal || !obj->is_internal())) \ return obj; \ } \ return nullptr; \ diff --git a/esphome/core/config.py b/esphome/core/config.py index 6b24a55487..1095a4886e 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -542,8 +542,10 @@ def _add_library_str(lib: str) -> None: if "@" in lib: name, vers = lib.split("@", 1) cg.add_library(name, vers) - elif "://" in lib: - # Repository... + elif "://" in lib or lib.split("=", 1)[-1].startswith("file:"): + # A repository or URL source. Also catch a ``file:`` source spelled with + # fewer than two slashes (e.g. ``file:lib_dev``) so it reaches the + # file:// handling and its clear error, rather than a registry lookup. if "=" in lib: name, repo = lib.split("=", 1) cg.add_library(name, None, repo) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 24358c9fcb..d621ee4071 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -157,6 +157,9 @@ #define USE_OUTPUT_FLOAT_POWER_SCALING #define USE_POWER_SUPPLY #define USE_PREFERENCES_SYNC_EVERY_LOOP +// Only defined by key-lookup preference backends (esp32, libretiny, host, zephyr); +// slot-based platforms (esp8266, rp2040) never set it in generated builds +#define USE_PREFERENCE_KEY_LOOKUP #define USE_PROVISIONING #define USE_QR_CODE #define USE_SAFE_MODE_BOOT_IS_GOOD_ON_SHUTDOWN @@ -241,6 +244,27 @@ #define USE_NATIVE_64BIT_TIME #endif +// bluetooth_proxy runs on any platform with a BLE hub (advertisement-only off +// esp32). Declared here per analysis ENVIRONMENT, not per hub platform — +// USE_LIBRETINY also covers chips with no hub, e.g. rtl87xx (the authoritative +// gate is _HUB_PLATFORMS in bluetooth_proxy/__init__.py) — so the neutral +// declarations in bluetooth_proxy.h are parsed under LibreTiny static analysis +// (the header is included by api_connection.cpp, which the tidy filter selects; +// the proxy's own .cpp is not a selected translation unit). Not declared for +// platforms whose API/network types the proxy header cannot assume. +#if defined(USE_ESP32) || defined(USE_LIBRETINY) || defined(USE_RP2) +#define USE_BLUETOOTH_PROXY +// Mirror the codegen values per platform: _to_code_esp32() emits the connection +// count (default 3), _to_code_ble_hub() emits 0 — so static analysis checks the +// same std::array instantiation a real build produces. +#ifdef USE_ESP32 +#define BLUETOOTH_PROXY_MAX_CONNECTIONS 3 +#else +#define BLUETOOTH_PROXY_MAX_CONNECTIONS 0 +#endif +#define BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE 16 +#endif + // ESP32-specific feature flags #ifdef USE_ESP32 #define USE_ESP32_CRASH_HANDLER @@ -249,6 +273,13 @@ #define ESPHOME_TASK_LOG_BUFFER_SIZE 768 #define USE_OTA_ROLLBACK #define USE_OTA_SIGNED_VERIFICATION +#define USE_OTA_SIGNED_VERIFICATION_MULTI_KEY +// Stub values for tooling; a real build's codegen emits these from verification_keys. +#define OTA_TRUSTED_KEY_COUNT 1 +#define OTA_TRUSTED_KEY_DIGESTS \ + { \ + { 0 } \ + } #define USE_OTA_DOWNGRADE_PROTECTION #define USE_ESP32_MIN_CHIP_REVISION_SET #define USE_ESP32_RTC_PREFERENCES @@ -256,9 +287,6 @@ #define USE_ESPNOW #define USE_ESPNOW_MAX_PAYLOAD_SIZE 1470 -#define USE_BLUETOOTH_PROXY -#define BLUETOOTH_PROXY_MAX_CONNECTIONS 3 -#define BLUETOOTH_PROXY_ADVERTISEMENT_BATCH_SIZE 16 #define USE_CAPTIVE_PORTAL #define USE_WIFI_SCAN_RESULTS_LOCK #define USE_ESP32_BLE @@ -277,6 +305,7 @@ #define USE_ESP32_BLE_SERVER_ON_DISCONNECT #define ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT 1 #define ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT 1 +#define ESPHOME_ESP32_BLE_TRACKER_SCANNER_STATE_LISTENER_COUNT 1 #define ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT 1 #define ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT 2 #define ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT 1 @@ -428,13 +457,17 @@ // rp2/__init__.py codegen also defines USE_RP2040 as a back-compat alias // for external custom components that may still test for it. #ifdef USE_RP2 -#define USE_ARDUINO_VERSION_CODE VERSION_CODE(3, 3, 0) +#define USE_ARDUINO_VERSION_CODE VERSION_CODE(6, 0, 0) #define USE_RP2_CRASH_HANDLER #define USE_HTTP_REQUEST_RESPONSE #define USE_I2C #define USE_LOGGER_USB_CDC #define USE_SOCKET_IMPL_LWIP_TCP #define USE_RP2040_BLE +#define RP2040_BLE_SCAN_LISTENER_COUNT 1 +#define ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT 1 +#define USE_BLE_GATT_CLIENT +#define ESPHOME_BLE_GATT_CLIENT_COUNT 1 #define USE_RP2040_VARIANT_RP2040 #define USE_SPI #ifndef USE_ETHERNET @@ -452,7 +485,10 @@ #ifdef USE_LIBRETINY #define USE_BK72XX_BLE +#define BK72XX_BLE_SCAN_LISTENER_COUNT 1 #define USE_LN882H_BLE +#define LN882H_BLE_SCAN_LISTENER_COUNT 1 +#define ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT 1 #define USE_CAPTIVE_PORTAL #define USE_WIFI_SCAN_RESULTS_LOCK #define USE_SOCKET_IMPL_LWIP_SOCKETS diff --git a/esphome/core/entity_base.cpp b/esphome/core/entity_base.cpp index 32135860bb..328de05302 100644 --- a/esphome/core/entity_base.cpp +++ b/esphome/core/entity_base.cpp @@ -8,7 +8,7 @@ namespace esphome { static const char *const TAG = "entity_base"; -void EntityBase::configure_entity_(const char *name, uint32_t object_id_hash, uint32_t entity_fields) { +void EntityBase::configure_entity_(const char *name, uint32_t entity_key, uint32_t entity_fields) { this->name_ = StringRef(name); if (this->name_.empty()) { #ifdef USE_DEVICES @@ -30,15 +30,15 @@ void EntityBase::configure_entity_(const char *name, uint32_t object_id_hash, ui } } this->flags_.has_own_name = false; - // Dynamic name - must calculate hash at runtime - this->calc_object_id_(); + // Dynamic name - must calculate key at runtime + this->calc_entity_key_(); } else { this->flags_.has_own_name = true; - // Static name - use pre-computed hash if provided - if (object_id_hash != 0) { - this->object_id_hash_ = object_id_hash; + // Static name - use pre-computed key if provided + if (entity_key != 0) { + this->entity_key_ = entity_key; } else { - this->calc_object_id_(); + this->calc_entity_key_(); } } // Unpack entity string table indices and flags from entity_fields. @@ -147,9 +147,15 @@ std::string EntityBase::get_icon() const { } #endif // !USE_ESP8266 -// Calculate Object ID Hash directly from name using snake_case + sanitize -void EntityBase::calc_object_id_() { - this->object_id_hash_ = fnv1_hash_object_id(this->name_.c_str(), this->name_.size()); +// Calculate the entity key directly from the raw name (no transformations) +void EntityBase::calc_entity_key_() { this->entity_key_ = fnv1_hash_bytes(this->name_.c_str(), this->name_.size()); } + +// Reconstruct the OLD (pre-2026.8.0) object_id-based hash for preference key compatibility. +// Named entities historically used the hash pre-computed by Python code generation, which +// sanitized per UTF-8 code point; entities without their own name computed the hash at +// runtime per byte. See https://github.com/esphome/backlog/issues/85 +uint32_t EntityBase::calc_old_object_id_hash_() const { + return fnv1_hash_object_id(this->name_.c_str(), this->name_.size(), this->flags_.has_own_name); } size_t EntityBase::write_object_id_to(char *buf, size_t buf_size) const { @@ -166,46 +172,23 @@ StringRef EntityBase::get_object_id_to(std::span buf) c return StringRef(buf.data(), len); } -// Migrate preference data from old_key to new_key if they differ. -// This helper is exposed so callers with custom key computation (like TextPrefs) -// can use it for manual migration. See: https://github.com/esphome/backlog/issues/85 -// -// FUTURE IMPLEMENTATION: -// This will require raw load/save methods on ESPPreferenceObject that take uint8_t* and size. -// void EntityBase::migrate_entity_preference_(size_t size, uint32_t old_key, uint32_t new_key) { -// if (old_key == new_key) -// return; -// auto old_pref = global_preferences->make_preference(size, old_key); -// auto new_pref = global_preferences->make_preference(size, new_key); -// SmallBufferWithHeapFallback<64> buffer(size); -// if (old_pref.load(buffer.data(), size)) { -// new_pref.save(buffer.data(), size); -// } -// } - ESPPreferenceObject EntityBase::make_entity_preference_(size_t size, uint32_t version) { - // This helper centralizes preference creation to enable fixing hash collisions. + // The old key hashed the sanitized object_id, so multiple entity names could collide on + // one key and overwrite each other's stored preferences; the new key hashes the raw name. // See: https://github.com/esphome/backlog/issues/85 - // - // COLLISION PROBLEM: get_preference_hash() uses fnv1_hash on sanitized object_id. - // Multiple entity names can sanitize to the same object_id: - // - "Living Room" and "living_room" both become "living_room" - // - UTF-8 names like "温度" and "湿度" both become "__" (underscores) - // This causes entities to overwrite each other's stored preferences. - // - // FUTURE MIGRATION: When implementing get_preference_hash_v2() that hashes - // the original entity name (not sanitized object_id): - // - // uint32_t old_key = this->get_preference_hash() ^ version; - // uint32_t new_key = this->get_preference_hash_v2() ^ version; - // this->migrate_entity_preference_(size, old_key, new_key); - // return global_preferences->make_preference(size, new_key); - // -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - uint32_t key = this->get_preference_hash() ^ version; -#pragma GCC diagnostic pop - return global_preferences->make_preference(size, key); + uint32_t old_key = this->old_preference_key_base_() ^ version; +#ifdef USE_PREFERENCE_KEY_LOOKUP + uint32_t new_key = this->preference_key_base_() ^ version; + auto pref = global_preferences->make_preference(size, new_key); + // All in-tree entity preferences fit the stack buffer, so migration never hits the heap + SmallBufferWithHeapFallback<64> buffer(size); + migrate_preference(pref, buffer.get(), size, old_key, new_key); + return pref; +#else + // Slot-based backends keep the old key: it is only a validity tag on a positional slot, + // so collisions cannot corrupt data there and keeping it preserves stored state. + return global_preferences->make_preference(size, old_key); +#endif } #ifdef USE_ENTITY_ICON diff --git a/esphome/core/entity_base.h b/esphome/core/entity_base.h index 4f708209d4..7f8e5f2630 100644 --- a/esphome/core/entity_base.h +++ b/esphome/core/entity_base.h @@ -73,8 +73,17 @@ class EntityBase { // Get whether this Entity has its own name or it should use the device friendly_name. bool has_own_name() const { return this->flags_.has_own_name; } - // Get the unique Object ID of this Entity - uint32_t get_object_id_hash() const { return this->object_id_hash_; } + // Get the unique key of this Entity: FNV-1 hash of the raw entity name. + // This is the key sent to API clients and used to route entity state. + uint32_t get_entity_key() const { return this->entity_key_; } + + /// Returns the LEGACY object_id hash, unchanged from previous releases, so existing + /// callers keep getting stable values (for example preference keys). This is no longer + /// the key sent to API clients; that is get_entity_key(). + ESPDEPRECATED("Use get_entity_key() for the entity key sent to API clients, or " + "make_entity_preference() for preference storage. Will be removed in 2027.1.0.", + "2026.8.0") + uint32_t get_object_id_hash() const { return this->calc_old_object_id_hash_(); } /// Get object_id with zero heap allocation /// For static case: returns StringRef to internal storage (buffer unused) @@ -181,40 +190,24 @@ class EntityBase { // Set has_state - for components that need to manually set this void set_has_state(bool state) { this->flags_.has_state = state; } - /** - * @brief Get a unique hash for storing preferences/settings for this entity. - * - * This method returns a hash that uniquely identifies the entity for the purpose of - * storing preferences (such as calibration, state, etc.). Unlike get_object_id_hash(), - * this hash also incorporates the device_id (if devices are enabled), ensuring uniqueness - * across multiple devices that may have entities with the same object_id. - * - * Use this method when storing or retrieving preferences/settings that should be unique - * per device-entity pair. Use get_object_id_hash() when you need a hash that identifies - * the entity regardless of the device it belongs to. - * - * For backward compatibility, if device_id is 0 (the main device), the hash is unchanged - * from previous versions, so existing single-device configurations will continue to work. - * - * @return uint32_t The unique hash for preferences, including device_id if available. - * @deprecated Use make_entity_preference() instead, or preferences won't be migrated. - * See https://github.com/esphome/backlog/issues/85 - */ - ESPDEPRECATED("Use make_entity_preference() instead, or preferences won't be migrated. " - "See https://github.com/esphome/backlog/issues/85. Will be removed in 2027.1.0.", - "2026.7.0") - uint32_t get_preference_hash() { + /// Get this entity's device id, or 0 when devices are not compiled in (main device). + uint32_t get_device_id_or_zero() const { #ifdef USE_DEVICES - // Combine object_id_hash with device_id to ensure uniqueness across devices - // Note: device_id is 0 for the main device, so XORing with 0 preserves the original hash - // This ensures backward compatibility for existing single-device configurations - return this->get_object_id_hash() ^ this->get_device_id(); + return this->get_device_id(); #else - // Without devices, just use object_id_hash as before - return this->get_object_id_hash(); + return 0; #endif } + /// Get the LEGACY preference key: FNV-1 hash of the sanitized object_id, XOR device_id. + /// Intentionally keeps the old algorithm so external callers that store preferences under + /// this key keep stable keys; make_entity_preference() migrates to the new raw-name key, + /// this method never will. + ESPDEPRECATED("Use make_entity_preference() instead, or preferences won't be migrated. " + "See https://github.com/esphome/backlog/issues/85. Will be removed in 2027.1.0.", + "2026.8.0") + uint32_t get_preference_hash() { return this->old_preference_key_base_(); } + /// Create a preference object for storing this entity's state/settings. /// @tparam T The type of data to store (must be trivially copyable) /// @param version Optional version hash XORed with preference key (change when struct layout changes) @@ -230,9 +223,9 @@ class EntityBase { // before push_back, so codegen can emit a single combined call per entity. friend class Application; - /// Combined entity setup from codegen: set name, object_id hash, entity string indices, and flags. + /// Combined entity setup from codegen: set name, entity key, entity string indices, and flags. /// Bit layout of entity_fields is defined by the ENTITY_FIELD_*_SHIFT constants above. - void configure_entity_(const char *name, uint32_t object_id_hash, uint32_t entity_fields); + void configure_entity_(const char *name, uint32_t entity_key, uint32_t entity_fields); #ifdef USE_DEVICES // Codegen-only setter — only accessible from setup() via friend declaration. @@ -240,13 +233,24 @@ class EntityBase { #endif /// Non-template helper for make_entity_preference() to avoid code bloat. - /// When preference hash algorithm changes, migration logic goes here. + /// Migrates preferences from the old sanitized-object_id key to the raw-name key + /// on key-lookup platforms. See: https://github.com/esphome/backlog/issues/85 ESPPreferenceObject make_entity_preference_(size_t size, uint32_t version); - void calc_object_id_(); + void calc_entity_key_(); + + /// Reconstruct the OLD (pre-2026.8.0) sanitized-object_id hash for preference keys. + uint32_t calc_old_object_id_hash_() const; + + /// Preference key base for this entity: raw-name entity key XOR device_id. + uint32_t preference_key_base_() const { return this->entity_key_ ^ this->get_device_id_or_zero(); } + + /// Legacy preference key base: sanitized-object_id hash XOR device_id. + /// Note: device_id is 0 for the main device, so XORing with 0 preserves the original hash. + uint32_t old_preference_key_base_() const { return this->calc_old_object_id_hash_() ^ this->get_device_id_or_zero(); } StringRef name_; - uint32_t object_id_hash_{}; + uint32_t entity_key_{}; #ifdef USE_DEVICES Device *device_{}; #endif diff --git a/esphome/core/entity_helpers.py b/esphome/core/entity_helpers.py index 38c7f3ca43..5060e32a2d 100644 --- a/esphome/core/entity_helpers.py +++ b/esphome/core/entity_helpers.py @@ -25,19 +25,86 @@ from esphome.core.config import ( from esphome.cpp_generator import MockObj, RawStatement, add, get_variable from esphome.cpp_types import App import esphome.final_validate as fv -from esphome.helpers import cpp_string_escape, fnv1_hash_object_id, sanitize, snake_case +from esphome.helpers import cpp_string_escape, fnv1_hash_name, sanitize, snake_case from esphome.types import ConfigType, EntityMetadata _LOGGER = logging.getLogger(__name__) DOMAIN = "entity_string_pool" +_OBJECT_ID_DOMAIN = "entity_object_ids" + + +@dataclass +class ObjectIdEntity: + """An entity tracked by the sanitized object_id its name resolves to.""" + + name: str + platform: str + config: ConfigType + + +def _get_object_id_registry() -> dict[tuple[str, str, str], list[ObjectIdEntity]]: + """(device_id, platform, sanitized object_id) -> entities resolving to it.""" + return CORE.data.setdefault(_OBJECT_ID_DOMAIN, {}) + + +def validate_no_object_id_conflicts( + reason: str, + conflict_filter: Callable[[list[ObjectIdEntity], ConfigType], bool] | None = None, +) -> Callable[[ConfigType], ConfigType]: + """Create a final-validate step that rejects entities with colliding object_ids. + + Entity keys are hashed from the raw name, so names that only differ in characters + lost during sanitizing (for example two UTF-8 names) validate fine in general. + Components that still address entities by the sanitized object_id string must + reject those configs until they are migrated to raw names. + + Args: + reason: One sentence stating what the component builds from the object_id, + e.g. "mqtt builds default topics from the entity object_id" + conflict_filter: Optional predicate receiving the colliding entities and the + component config; return False when the component is not affected + + Returns: + A validator function for use as (or within) FINAL_VALIDATE_SCHEMA + """ + + def validator(config: ConfigType) -> ConfigType: + # Skip in testing_mode, which is used for grouped component testing + if CORE.testing_mode: + return config + conflicts = { + key: entities + for key, entities in _get_object_id_registry().items() + if len(entities) > 1 + and (conflict_filter is None or conflict_filter(entities, config)) + } + if not conflicts: + return config + lines = [f"{reason}, so these entities would conflict:"] + lines.extend( + f" - {platform} entities " + + ", ".join(f"'{e.name}'" for e in entities) + + (f" on device '{device_id}'" if device_id else "") + + f" share the object_id '{object_id}'" + for (device_id, platform, object_id), entities in conflicts.items() + ) + lines.append( + "To fix: Add unique ASCII characters (e.g., '1', '2', or 'A', 'B') " + "to distinguish the names" + ) + raise cv.Invalid("\n".join(lines)) + + return validator + + # Private config keys for storing registered string indices _KEY_DC_IDX = "_entity_dc_idx" _KEY_UOM_IDX = "_entity_uom_idx" _KEY_ICON_IDX = "_entity_icon_idx" _KEY_ENTITY_NAME = "_entity_name" -_KEY_OBJECT_ID_HASH = "_entity_object_id_hash" +_KEY_ENTITY_KEY = "_entity_key" # Bit layout for entity_fields in configure_entity_(). # Keep in sync with ENTITY_FIELD_*_SHIFT constants in esphome/core/entity_base.h @@ -300,7 +367,7 @@ def finalize_entity_strings(var: MockObj, config: ConfigType) -> None: standalone ``var->configure_entity_(name, hash, packed)``. """ entity_name = config[_KEY_ENTITY_NAME] - object_id_hash = config[_KEY_OBJECT_ID_HASH] + entity_key = config[_KEY_ENTITY_KEY] dc_idx = config.get(_KEY_DC_IDX, 0) uom_idx = config.get(_KEY_UOM_IDX, 0) icon_idx = config.get(_KEY_ICON_IDX, 0) @@ -320,57 +387,30 @@ def finalize_entity_strings(var: MockObj, config: ConfigType) -> None: register_method = config.get(_KEY_REGISTER_METHOD) if register_method is not None: expr = getattr(App, f"register_{register_method}")( - var, entity_name, object_id_hash, packed + var, entity_name, entity_key, packed ) else: - expr = var.configure_entity_(entity_name, object_id_hash, packed) + expr = var.configure_entity_(entity_name, entity_key, packed) if comment: add(RawStatement(f"{expr}; // {comment}")) else: add(expr) -def get_base_entity_object_id( +def get_base_entity_name( name: str, friendly_name: str | None, device_name: str | None = None ) -> str: - """Calculate the base object ID for an entity that will be set via set_object_id(). + """Return the base name whose hash becomes this entity's key on the device. - This function calculates what object_id_c_str_ should be set to in C++. + Follows the name selection in C++ EntityBase::configure_entity_() (entity_base.cpp): + entity name, then sub-device name, then friendly name, then the device name. - The C++ EntityBase::write_object_id_to() (entity_base.cpp) works as: - - If !has_own_name && is_name_add_mac_suffix_enabled(): - return str_sanitize(str_snake_case(App.get_friendly_name())) // Dynamic - - Else: - return object_id_c_str_ ?? "" // What we set via set_object_id() - - Since we're calculating what to pass to set_object_id(), we always need to - generate the object_id the same way, regardless of name_add_mac_suffix setting. - - Args: - name: The entity name (empty string if no name) - friendly_name: The friendly name from CORE.friendly_name - device_name: The device name if entity is on a sub-device - - Returns: - The base object ID to use for duplicate checking and to pass to set_object_id() + This is a config-time approximation for duplicate checking: when + name_add_mac_suffix is enabled the device appends the MAC suffix at runtime, + which is unknown here and identical for every entity on the device, so + ignoring it cannot change whether two entities collide with each other. """ - - if name: - # Entity has its own name (has_own_name will be true) - base_str = name - elif device_name: - # Entity has empty name and is on a sub-device - # C++ EntityBase::set_name() uses device->get_name() when device is set - base_str = device_name - elif friendly_name: - # Entity has empty name (has_own_name will be false) - # C++ uses App.get_friendly_name() which returns friendly_name or device name - base_str = friendly_name - else: - # Fallback to device name - base_str = CORE.name - - return sanitize(snake_case(base_str)) + return name or device_name or friendly_name or CORE.name def setup_entity(var_or_platform, config=None, platform=None): @@ -429,15 +469,15 @@ async def _setup_entity_impl(var: MockObj, config: ConfigType, platform: str) -> device: MockObj = await get_variable(device_id_obj) add(var.set_device_(device)) - # Pre-compute entity name and object_id hash for configure_entity_() + # Pre-compute entity name and entity key for configure_entity_() # which is emitted later by finalize_entity_strings(). - # For named entities: pre-compute hash from entity name - # For empty-name entities: pass 0, C++ calculates hash at runtime from - # device name, friendly_name, or app name (bug-for-bug compatibility) + # For named entities: pre-compute the key from the raw entity name + # For empty-name entities: pass 0, C++ calculates the key at runtime from + # device name, friendly_name, or app name entity_name = config[CONF_NAME] - object_id_hash = fnv1_hash_object_id(entity_name) if entity_name else 0 + entity_key = fnv1_hash_name(entity_name) if entity_name else 0 config[_KEY_ENTITY_NAME] = entity_name - config[_KEY_OBJECT_ID_HASH] = object_id_hash + config[_KEY_ENTITY_KEY] = entity_key # Store flags for packing into configure_entity_() config[_KEY_DISABLED_BY_DEFAULT] = int(config[CONF_DISABLED_BY_DEFAULT]) if CONF_INTERNAL in config: @@ -550,14 +590,14 @@ def entity_duplicate_validator(platform: str) -> Callable[[ConfigType], ConfigTy # Use the device ID string directly for uniqueness device_id = device_id_obj.id - # Calculate what object_id will actually be used - # This handles empty names correctly by using device/friendly names - name_key = get_base_entity_object_id( - entity_name, CORE.friendly_name, device_name - ) + # Hash the same raw name the device hashes into the entity key at runtime. + # This handles empty names correctly by using device/friendly names. + base_name = get_base_entity_name(entity_name, CORE.friendly_name, device_name) + name_hash = fnv1_hash_name(base_name) - # Check for duplicates - unique_key = (device_id, platform, name_key) + # Check for duplicates: two entities on the same device and platform must not + # share an entity key, since the key is what routes state to API clients + unique_key = (device_id, platform, name_hash) if unique_key in CORE.unique_ids: # Get the existing entity metadata existing = CORE.unique_ids[unique_key] @@ -581,14 +621,13 @@ def entity_duplicate_validator(platform: str) -> Callable[[ConfigType], ConfigTy if existing_component != "unknown": conflict_msg += f" from component '{existing_component}'" - # Show both original names and their ASCII-only versions if they differ - sanitized_msg = "" + # Different names can only clash here through a genuine hash collision + collision_msg = "" if entity_name != existing_name: - sanitized_msg = ( - f"\n Original names: '{entity_name}' and '{existing_name}'" - f"\n Both convert to ASCII ID: '{name_key}'" - "\n To fix: Add unique ASCII characters (e.g., '1', '2', or 'A', 'B')" - "\n to distinguish them" + collision_msg = ( + f"\n The names '{entity_name}' and '{existing_name}' produce the" + f"\n same entity key hash ({name_hash:#010x})." + "\n To fix: Rename one of the entities" ) # Skip duplicate entity name validation when testing_mode is enabled @@ -598,9 +637,22 @@ def entity_duplicate_validator(platform: str) -> Callable[[ConfigType], ConfigTy f"Duplicate {platform} entity with name '{entity_name}' found{device_prefix}. " f"{conflict_msg}. " "Each entity on a device must have a unique name within its platform." - f"{sanitized_msg}" + f"{collision_msg}" ) + # Components that still address entities by the sanitized object_id reject + # colliding names in final validation via validate_no_object_id_conflicts(), + # so track every entity by the object_id its name resolves to. Scoped per + # device and platform to match the strictness configs had before entity keys + # moved to raw names: same-named entities on different sub-devices were + # already accepted then, internal entities were already skipped (above), and + # overlaps between platforms that share an MQTT component type (sensor and + # text_sensor both publish under "sensor") were already possible. + object_id = sanitize(snake_case(base_name)) + _get_object_id_registry().setdefault( + (device_id, platform, object_id), [] + ).append(ObjectIdEntity(base_name, platform, config)) + # Store metadata about this entity entity_metadata: EntityMetadata = { "name": entity_name, diff --git a/esphome/core/event_pool.h b/esphome/core/event_pool.h index fe207d04bf..b53b8064a3 100644 --- a/esphome/core/event_pool.h +++ b/esphome/core/event_pool.h @@ -10,7 +10,8 @@ namespace esphome { // Event Pool - On-demand pool of objects to avoid heap fragmentation -// Events are allocated on first use and reused thereafter, growing to peak usage +// Events are allocated on first use and reused thereafter, growing to peak +// usage; warm() pre-creates every entry up front for malloc-free producers // @tparam T The type of objects managed by the pool (must have a release() method) // @tparam SIZE The maximum number of objects in the pool (1-254, limited by uint8_t and the +1 free-list slot) // @@ -53,26 +54,8 @@ template class EventPool { T *event = this->free_list_.pop(); if (event != nullptr) return event; - // Need to create a new event - if (this->total_created_ >= SIZE) { - // Pool is at capacity - return nullptr; - } - - // Use internal RAM for better performance - RAMAllocator allocator(RAMAllocator::ALLOC_INTERNAL); - event = allocator.allocate(1); - - if (event == nullptr) { - // Memory allocation failed - return nullptr; - } - - // Placement new to construct the object - new (event) T(); - this->total_created_++; - return event; + return this->create_(); } // Return an event to the pool for reuse @@ -84,7 +67,45 @@ template class EventPool { } } + // Pre-create every pool entry so allocate() is always a free-list pop + // (for producers that must never malloc, e.g. IRQ-context handlers). + // Call from setup(); on false the heap could not supply every entry and + // the caller should mark_failed() — an incomplete warm puts malloc() + // back on the producer path. Tops the pool up from any quiescent state + // (entries that already exist are counted, not re-created); must not run + // concurrently with allocate()/release(). + bool warm() { + // NOLINTNEXTLINE(clang-analyzer-unix.Malloc) -- ownership transfers to the free list + while (this->total_created_ < SIZE) { + T *event = this->create_(); + if (event == nullptr) + return false; + this->free_list_.push(event); + } + return true; + } + private: + // Create and count one new object (shared by allocate() and warm()). + // Returns nullptr at capacity or when the heap is exhausted. + T *create_() { + if (this->total_created_ >= SIZE) { + // Pool is at capacity + return nullptr; + } + // Use internal RAM for better performance + RAMAllocator allocator(RAMAllocator::ALLOC_INTERNAL); + T *event = allocator.allocate(1); + if (event == nullptr) { + // Memory allocation failed + return nullptr; + } + // Placement new to construct the object + new (event) T(); + this->total_created_++; + return event; + } + // SIZE + 1 slots so all SIZE objects fit when the pool is fully drained // (the ring reserves one slot); otherwise the last release() of a // completely returned pool would drop, permanently orphaning one object. diff --git a/esphome/core/helpers.h b/esphome/core/helpers.h index 7940df8780..d883ce146e 100644 --- a/esphome/core/helpers.h +++ b/esphome/core/helpers.h @@ -184,6 +184,11 @@ template class SmallInlineBuffer { SmallInlineBuffer(const SmallInlineBuffer &) = delete; SmallInlineBuffer &operator=(const SmallInlineBuffer &) = delete; + bool empty() const { return this->len_ == 0; } + + // Conversion to std::span for compatibility with span-based APIs + operator std::span() const { return std::span(this->data(), this->len_); } + /// Resize to `size` bytes of (uninitialized) storage and return a writable pointer to fill. /// Allocates heap only when `size` exceeds the inline capacity. Use this when the contents are /// built in place (e.g. assembling a frame and appending a checksum) to avoid a staging copy. @@ -251,6 +256,11 @@ template class StaticVector { } } + // Converting constructor from a smaller StaticVector of the same element type + template StaticVector(const StaticVector &other) : StaticVector(other.begin(), other.end()) { + static_assert(M <= N, "Source StaticVector cannot be larger than the destination"); + } + // Minimal vector-compatible interface - only what we actually use void push_back(const T &value) { if (count_ < N) { @@ -799,6 +809,19 @@ constexpr uint32_t FNV1_OFFSET_BASIS = 2166136261UL; /// FNV-1 32-bit prime constexpr uint32_t FNV1_PRIME = 16777619UL; +/// Calculate a FNV-1 hash over raw bytes with an explicit length. Unlike fnv1_hash(const char *), +/// each byte is hashed as an unsigned value, so results are platform-independent for bytes >= 0x80. +/// IMPORTANT: Must match Python fnv1_hash_name() in esphome/helpers.py, which hashes the UTF-8 +/// encoded bytes of the name. Used to compute entity keys from raw names. +inline uint32_t fnv1_hash_bytes(const char *str, size_t len) { + uint32_t hash = FNV1_OFFSET_BASIS; + for (size_t i = 0; i < len; i++) { + hash *= FNV1_PRIME; + hash ^= static_cast(str[i]); + } + return hash; +} + /// Extend a FNV-1 hash with an integer (hashes each byte). template constexpr uint32_t fnv1_hash_extend(uint32_t hash, T value) { using UnsignedT = std::make_unsigned_t; @@ -1003,12 +1026,20 @@ template inline char *str_sanitize_to(char (&buffer)[N], const char *s // str_sanitize moved to alloc_helpers.h - remove this comment before 2026.11.0 /// Calculate FNV-1 hash of a string while applying snake_case + sanitize transformations. -/// This computes object_id hashes directly from names without creating an intermediate buffer. -/// IMPORTANT: Must match Python fnv1_hash_object_id() in esphome/helpers.py. -/// If you modify this function, update the Python version and tests in both places. -inline uint32_t fnv1_hash_object_id(const char *str, size_t len) { +/// This is the LEGACY entity hash, kept only to reconstruct preference keys that existing +/// devices already have stored; see https://github.com/esphome/backlog/issues/85. +/// With per_code_point set, UTF-8 continuation bytes are skipped so each multi-byte character +/// contributes one underscore — this matches Python fnv1_hash_object_id() in esphome/helpers.py, +/// which produced the hash for named entities. The per-byte form (default) matches the old +/// runtime hash for entities without their own name. Do not change either behavior. +/// Known limitation: Python's lower() is Unicode aware, so the rare code points it maps to a +/// different number of characters or to ASCII (e.g. 'İ', the Kelvin sign) reconstruct wrong; +/// such names skip migration once and fall back to their defaults. +inline uint32_t fnv1_hash_object_id(const char *str, size_t len, bool per_code_point = false) { uint32_t hash = FNV1_OFFSET_BASIS; for (size_t i = 0; i < len; i++) { + if (per_code_point && (static_cast(str[i]) & 0xC0) == 0x80) + continue; // UTF-8 continuation byte, already counted via its lead byte hash *= FNV1_PRIME; // Apply snake_case (space->underscore, uppercase->lowercase) then sanitize hash ^= static_cast(to_sanitized_char(to_snake_case_char(str[i]))); diff --git a/esphome/core/preference_backend.h b/esphome/core/preference_backend.h index 34bf84409d..b9bb9a0252 100644 --- a/esphome/core/preference_backend.h +++ b/esphome/core/preference_backend.h @@ -22,6 +22,12 @@ #include "esphome/components/zephyr/preference_backend.h" #endif +// Key-lookup preference backends find stored data by key; their platforms add the +// USE_PREFERENCE_KEY_LOOKUP define from Python codegen, which enables preference key +// migration. Slot-based backends (ESP8266, RP2040) instead allocate a storage slot for +// every make_preference() call and use the key only as a validity tag on that slot; +// migration is not possible there, and key collisions cannot corrupt data. + namespace esphome { #if !defined(USE_ESP32) && !defined(USE_ESP8266) && !defined(USE_RP2) && !defined(USE_LIBRETINY) && \ @@ -40,16 +46,22 @@ class ESPPreferenceObject { ESPPreferenceObject() = default; explicit ESPPreferenceObject(PreferenceBackend *backend) : backend_(backend) {} - template bool save(const T *src) { + template bool save(const T *src) { return this->save(reinterpret_cast(src), sizeof(T)); } + + template bool load(T *dest) { return this->load(reinterpret_cast(dest), sizeof(T)); } + + /// Raw save with explicit length, for callers that only know the size at runtime. + bool save(const uint8_t *src, size_t len) { if (this->backend_ == nullptr) return false; - return this->backend_->save(reinterpret_cast(src), sizeof(T)); + return this->backend_->save(src, len); } - template bool load(T *dest) { + /// Raw load with explicit length, for callers that only know the size at runtime. + bool load(uint8_t *dest, size_t len) { if (this->backend_ == nullptr) return false; - return this->backend_->load(reinterpret_cast(dest), sizeof(T)); + return this->backend_->load(dest, len); } protected: diff --git a/esphome/core/preferences.cpp b/esphome/core/preferences.cpp new file mode 100644 index 0000000000..8508647255 --- /dev/null +++ b/esphome/core/preferences.cpp @@ -0,0 +1,25 @@ +#include "esphome/core/preferences.h" +#include "esphome/core/log.h" +#include + +namespace esphome { + +#ifdef USE_PREFERENCE_KEY_LOOKUP +static const char *const TAG = "preferences"; + +bool migrate_preference(ESPPreferenceObject &new_pref, uint8_t *scratch, size_t size, uint32_t old_key, + uint32_t new_key) { + if (new_pref.load(scratch, size)) + return true; // Current data present - never overwrite newer data with the old copy + // One-shot read by key: no backend is allocated for the old key, so boots with + // nothing to migrate (for example fresh installs) cost no heap + if (old_key == new_key || !global_preferences->load_from_key(old_key, scratch, size)) + return false; // No data stored under the old key, nothing to migrate + if (!new_pref.save(scratch, size)) { + ESP_LOGW(TAG, "Pref migration %" PRIx32 " -> %" PRIx32 " failed", old_key, new_key); + } + return true; +} +#endif // USE_PREFERENCE_KEY_LOOKUP + +} // namespace esphome diff --git a/esphome/core/preferences.h b/esphome/core/preferences.h index 1efce5af51..d24d51164a 100644 --- a/esphome/core/preferences.h +++ b/esphome/core/preferences.h @@ -23,6 +23,7 @@ struct Preferences : public PreferencesMixin { using PreferencesMixin::make_preference; ESPPreferenceObject make_preference(size_t, uint32_t, bool) { return {}; } ESPPreferenceObject make_preference(size_t, uint32_t) { return {}; } + bool load_from_key(uint32_t, uint8_t *, size_t) { return false; } /** * Commit pending writes to flash. @@ -43,3 +44,19 @@ using ESPPreferences = Preferences; extern ESPPreferences *global_preferences; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) } // namespace esphome #endif + +#ifdef USE_PREFERENCE_KEY_LOOKUP +namespace esphome { +/// Copy preference data stored under old_key into new_pref (created for new_key) if the keys +/// differ and new_pref has no data yet. scratch must hold at least size bytes. +/// Returns true when scratch holds the entity's current data (loaded or just migrated). +/// The old entry is intentionally left in place so a firmware downgrade still finds its data. +/// If saving under the new key fails, callers that consume scratch (like TextSaver) still get +/// valid data for this boot, callers that reload from the preference fall back to their +/// defaults, and the migration simply runs again on the next boot. +/// Only available on key-lookup preference backends; slot-based backends keep their old +/// keys instead. See: https://github.com/esphome/backlog/issues/85 +bool migrate_preference(ESPPreferenceObject &new_pref, uint8_t *scratch, size_t size, uint32_t old_key, + uint32_t new_key); +} // namespace esphome +#endif // USE_PREFERENCE_KEY_LOOKUP diff --git a/esphome/core/string_ref.h b/esphome/core/string_ref.h index 34ba2474b2..33459f48af 100644 --- a/esphome/core/string_ref.h +++ b/esphome/core/string_ref.h @@ -76,6 +76,13 @@ class StringRef { constexpr bool empty() const { return len_ == 0; } constexpr const_reference operator[](size_type pos) const { return *(base_ + pos); } + /// True if the view begins with the given prefix (std::string::starts_with-like) + bool starts_with(const StringRef &prefix) const { + return len_ >= prefix.len_ && std::memcmp(base_, prefix.base_, prefix.len_) == 0; + } + bool starts_with(const char *prefix) const { return this->starts_with(StringRef(prefix)); } + bool starts_with(const std::string &prefix) const { return this->starts_with(StringRef(prefix)); } + /// Copy characters to destination buffer (std::string::copy-like, but returns 0 instead of throwing on out-of-range) size_type copy(char *dest, size_type count, size_type pos = 0) const { if (pos >= len_) diff --git a/esphome/cpp_helpers.py b/esphome/cpp_helpers.py index b2338e5bc1..53b59cb124 100644 --- a/esphome/cpp_helpers.py +++ b/esphome/cpp_helpers.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from dataclasses import dataclass, field import logging @@ -136,6 +137,67 @@ async def _generate_component_source_table() -> None: ) +_SLOT_COUNTER_DOMAIN = "slot_counter" + + +@dataclass +class _SlotCounterState: + """Per-run slot counter state: requested counts and already-emitted defines.""" + + counts: dict[str, int] = field(default_factory=dict) + emitted: set[str] = field(default_factory=set) + + +def _get_slot_counter_state() -> _SlotCounterState: + """Get or create the slot counter state from CORE.data.""" + if _SLOT_COUNTER_DOMAIN not in CORE.data: + CORE.data[_SLOT_COUNTER_DOMAIN] = _SlotCounterState() + return CORE.data[_SLOT_COUNTER_DOMAIN] + + +def get_slot_count(define: str) -> int: + """Number of slots requested so far for `define`.""" + return _get_slot_counter_state().counts.get(define, 0) + + +def slot_counter(define: str) -> Callable[[], None]: + """Create a request_slot function for codegen-sized storage. + + The pattern behind a StaticVector listener array: a consumer's to_code + calls the returned function once per slot it will occupy at runtime, and + at FINAL priority — after every consumer's to_code has run — `define` is + emitted with the requested count. No requests, no define: the guarded + storage and its registration method compile out entirely. + + The counts live in a table under CORE.data, which clears between runs. + A request arriving after the define was already emitted raises instead of + silently undercounting: the define would keep the stale smaller value and + StaticVector::push_back would drop the extra listener at runtime. + """ + + @coroutine_with_priority(CoroPriority.FINAL) + async def emit_job() -> None: + state = _get_slot_counter_state() + state.emitted.add(define) + # Scheduled only by the first request, so the count is always >= 1 here. + add_define(define, state.counts[define]) + + def request_slot() -> None: + state = _get_slot_counter_state() + if define in state.emitted: + raise ValueError( + f"slot_counter('{define}'): slot requested after the count " + f"define was emitted; request slots from to_code, not from a " + f"job running after FINAL emission" + ) + counts = state.counts + counts[define] = (count := counts.get(define, 0) + 1) + if count == 1: + CORE.add_job(emit_job) + + return request_slot + + async def gpio_pin_expression(conf): """Generate an expression for the given pin option. diff --git a/esphome/espidf/__init__.py b/esphome/espidf/__init__.py index e69de29bb2..079eede1f1 100644 --- a/esphome/espidf/__init__.py +++ b/esphome/espidf/__init__.py @@ -0,0 +1,11 @@ +"""ESP-IDF direct build support. + +Deliberately light: the upload fast path imports submodules of this +package without the esp32 component package, so nothing here may pull +in codegen or validation. +""" + + +def variant_to_idf_target(variant: str) -> str: + """Map an esp32 variant name (e.g. "ESP32S3") to its ESP-IDF target name.""" + return variant.lower().replace("-", "") diff --git a/esphome/espidf/clang_tidy.py b/esphome/espidf/clang_tidy.py index 88ecda60b9..c91db775a3 100644 --- a/esphome/espidf/clang_tidy.py +++ b/esphome/espidf/clang_tidy.py @@ -141,10 +141,15 @@ idf_component_register( def _setup_core(work_dir: Path, settings: _Settings) -> None: """Point CORE at the tidy project + IDF version, without any YAML config.""" - from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT - import esphome.config_validation as cv - from esphome.const import KEY_CORE, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM - from esphome.core import CORE + from esphome.const import ( + KEY_CORE, + KEY_ESP32, + KEY_IDF_VERSION, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, + KEY_VARIANT, + ) + from esphome.core import CORE, Version CORE.name = TIDY_PROJECT_NAME # config_path's parent is the data dir root for per-run artifacts (idedata, @@ -153,7 +158,7 @@ def _setup_core(work_dir: Path, settings: _Settings) -> None: CORE.config_path = work_dir.parent / "tidy.yaml" CORE.build_path = work_dir esp32 = CORE.data.setdefault(KEY_ESP32, {}) - esp32[KEY_IDF_VERSION] = cv.Version.parse(settings.idf_version) + esp32[KEY_IDF_VERSION] = Version.parse(settings.idf_version) esp32[KEY_VARIANT] = settings.variant # The target framework drives the PlatformIO-library -> IDF-component # converter and ESPHome's CORE.using_arduino / using_esp_idf helpers. diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index cad5bbf665..aa6f10c261 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -12,6 +12,7 @@ import os from pathlib import Path from esphome.core import CORE, Library +from esphome.espidf import variant_to_idf_target from esphome.helpers import write_file_if_changed from esphome.platformio.library import ( DEFAULT_BUILD_FLAGS, @@ -46,21 +47,21 @@ def _apply_extra_script(component: IDFComponent) -> None: extra_script = component.data.get("build", {}).get("extraScript") if not extra_script: return - # Resolve and confine to the component dir so a malicious library.json - # can't escape (e.g. ``"extraScript": "../../etc/passwd"``). - library_root = component.path.resolve() - script_path = (component.path / extra_script).resolve() + # Resolve and confine to the library's source dir so a malicious + # library.json can't escape (e.g. ``"extraScript": "../../etc/passwd"``). + source_path = component.source_dir + library_root = source_path.resolve() + script_path = (source_path / extra_script).resolve() if not script_path.is_relative_to(library_root) or not script_path.is_file(): return from esphome.components.esp32 import get_esp32_variant - from esphome.components.esp32.const import variant_to_idf_target from esphome.espidf.extra_script import captured_as_build_flags, run_extra_script idf_target = variant_to_idf_target(get_esp32_variant()) result = run_extra_script( - script_path, library_dir=component.path, idf_target=idf_target + script_path, library_dir=source_path, idf_target=idf_target ) - extra_flags = captured_as_build_flags(result, library_dir=component.path) + extra_flags = captured_as_build_flags(result, library_dir=source_path) if not extra_flags: return flags = component.data.setdefault("build", {}).setdefault("flags", []) @@ -101,11 +102,17 @@ def generate_cmakelists_txt(component: IDFComponent) -> str: # which Windows accepts too, so the generated CMakeLists is portable. return f'"{str(p).replace(os.sep, "/")}"' + # The library's own files live in source_path (the user's directory for a + # local library, the downloaded dir otherwise). When it differs from the + # component dir the CMakeLists must reference sources by absolute path. + read_path = component.source_dir + external = read_path.resolve() != component.path.resolve() + # Extract the values build_src_dir = component.data.get("build", {}).get("srcDir", None) if not build_src_dir: for d in ["src", "Src", "."]: - if (component.path / Path(d)).is_dir(): + if (read_path / Path(d)).is_dir(): build_src_dir = d break @@ -138,7 +145,7 @@ def generate_cmakelists_txt(component: IDFComponent) -> str: # List all sources files build_src_files = collect_filtered_files( - component.path / Path(build_src_dir), build_src_filter + read_path / Path(build_src_dir), build_src_filter ) # Only bake library.json-declared deps here. Project-managed and @@ -150,8 +157,12 @@ def generate_cmakelists_txt(component: IDFComponent) -> str: dependency.get_require_name() for dependency in component.dependencies } - # Only keep sources - build_src_files = [os.path.relpath(p, component.path) for p in build_src_files] + # Only keep sources. Reference them absolutely when they live outside the + # component dir (a local library), relative otherwise. + if external: + build_src_files = [str(Path(p).resolve()) for p in build_src_files] + else: + build_src_files = [os.path.relpath(p, component.path) for p in build_src_files] build_src_files = [ f for f in build_src_files if Path(f).suffix in SRC_FILE_EXTENSIONS ] @@ -166,13 +177,24 @@ def generate_cmakelists_txt(component: IDFComponent) -> str: link_libraries, build_flags = split_list_by_condition( build_flags, lambda a: a[2:].strip() if a.startswith("-l") else None ) + # A local library's relative -L paths are relative to its own directory; + # resolve them against it so they still work from the component cache dir. + # (read_path / d yields d unchanged when d is already absolute.) + if external: + link_directories = [ + str((read_path / Path(d)).resolve()) for d in link_directories + ] # Split include directories from build_flags # Only keep an include directory if it exists build_include_dirs = [build_include_dir, build_src_dir] + include_dir_flags build_include_dirs = [ - d for d in build_include_dirs if (component.path / Path(d)).is_dir() + d for d in build_include_dirs if (read_path / Path(d)).is_dir() ] + if external: + build_include_dirs = [ + str((read_path / Path(d)).resolve()) for d in build_include_dirs + ] # Split build_flags list into private and public lists private_build_flags, public_build_flags = split_list_by_condition( diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 0ca7a9d14b..39bf0465d5 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -13,8 +13,7 @@ from typing import Any, NoReturn import platformdirs -from esphome.config_validation import Version -from esphome.core import CORE +from esphome.core import CORE, Version from esphome.framework_helpers import ( PathType, archive_extract_all, diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index fd95805c6c..e1688f4170 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -9,20 +9,18 @@ import re import shutil import subprocess -from esphome.components.esp32.const import ( - KEY_ESP32, - KEY_FLASH_SIZE, - KEY_IDF_VERSION, - KEY_VARIANT, - variant_to_idf_target, -) from esphome.const import ( CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, CONF_FRAMEWORK, CONF_SOURCE, + KEY_ESP32, + KEY_FLASH_SIZE, + KEY_IDF_VERSION, + KEY_VARIANT, ) from esphome.core import CORE, EsphomeError +from esphome.espidf import variant_to_idf_target from esphome.espidf.framework import check_esp_idf_install, get_framework_env from esphome.espidf.size_summary import print_summary from esphome.helpers import add_git_ceiling_directory diff --git a/esphome/expression.py b/esphome/expression.py index d425d822a4..13da3b6a06 100644 --- a/esphome/expression.py +++ b/esphome/expression.py @@ -1,4 +1,4 @@ -"""Helpers for detecting substitution variables and Jinja expressions.""" +"""Helpers for detecting and matching substitution variables and Jinja expressions.""" import re @@ -8,7 +8,7 @@ SUBSTITUTION_VARIABLE_PROG = re.compile( rf"\$([{VALID_SUBSTITUTIONS_CHARACTERS}]+|\{{[{VALID_SUBSTITUTIONS_CHARACTERS}]*\}})" ) -_JINJA_RE = re.compile( +JINJA_PROG = re.compile( r"<%.+?%>" # Block: <% ... %> r"|\$\{[^}]+\}", # Braced: ${ ... } flags=re.MULTILINE, @@ -17,7 +17,7 @@ _JINJA_RE = re.compile( def has_jinja(value: str) -> bool: """Check if a string contains Jinja expressions.""" - return _JINJA_RE.search(value) is not None + return JINJA_PROG.search(value) is not None def has_substitution_or_expression(value: str) -> bool: diff --git a/esphome/helpers.py b/esphome/helpers.py index 683aaedcf5..15d9797ce1 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import MutableMapping +from collections.abc import Iterable, MutableMapping from contextlib import suppress import ipaddress import logging @@ -8,12 +8,9 @@ import os from pathlib import Path import platform import re -import shutil import stat import sys -import tempfile from typing import TYPE_CHECKING, TextIO -from urllib.parse import urlparse from esphome.const import __version__ as ESPHOME_VERSION @@ -56,15 +53,20 @@ def ensure_unique_string(preferred_string, current_strings): return test_string -def fnv1_hash(string: str) -> int: - """FNV-1 32-bit hash function (multiply then XOR).""" +def _fnv1_hash(values: Iterable[int]) -> int: + """FNV-1 32-bit hash (multiply then XOR) over a sequence of integer values.""" hash_value = FNV1_OFFSET_BASIS - for char in string: + for value in values: hash_value = (hash_value * FNV1_PRIME) & 0xFFFFFFFF - hash_value ^= ord(char) + hash_value ^= value return hash_value +def fnv1_hash(string: str) -> int: + """FNV-1 32-bit hash function (multiply then XOR) over code points.""" + return _fnv1_hash(map(ord, string)) + + def fnv1a_32bit_hash(string: str) -> int: """FNV-1a 32-bit hash function (XOR then multiply). @@ -89,12 +91,27 @@ def fnv1a_32bit_hash(string: str) -> int: def fnv1_hash_object_id(name: str) -> int: """Compute FNV-1 hash of name with snake_case + sanitize transformations. - IMPORTANT: Must produce same result as C++ fnv1_hash_object_id() in helpers.h. - Used for pre-computing entity object_id hashes at code generation time. + IMPORTANT: Must produce same result as C++ fnv1_hash_object_id() in helpers.h + with per_code_point set. This is the OLD entity hash; it computes preference + keys that existing devices already have stored (see + https://github.com/esphome/backlog/issues/85) and is also still used for live + keys derived from config IDs (see the motion component's calibration key). + Note: lower() here is Unicode aware while the C++ reconstruction is not; see + the known limitation note on the C++ function. """ return fnv1_hash(sanitize(snake_case(name))) +def fnv1_hash_name(name: str) -> int: + """Compute FNV-1 hash of the raw entity name (UTF-8 bytes, no transformations). + + IMPORTANT: Must produce same result as C++ fnv1_hash_bytes() in helpers.h, + which hashes the name bytes as stored on the device. + Used for pre-computing entity keys at code generation time. + """ + return _fnv1_hash(name.encode("utf-8")) + + def strip_accents(value: str) -> str: """Remove accents from a string.""" import unicodedata @@ -261,6 +278,9 @@ def resolve_ip_address( hosts = host else: if not is_ip_address(host): + # Deferred: upload/logs with an IP target never parse a URL. + from urllib.parse import urlparse + url = urlparse(host) if url.scheme != "": host = url.hostname @@ -412,6 +432,8 @@ def rmtree(path: Path | str) -> None: read-only flag and retrying. """ + import shutil + def _onexc(func, path, exc): if os.access(path, os.W_OK): raise exc @@ -449,6 +471,11 @@ def _write_file( Automatically creates all parent directories. """ + # Deferred: a cache-hit upload/logs run never writes a file; keep the + # tempfile/shutil chain (bz2, lzma, random) off that path. + import shutil + import tempfile + data = text if isinstance(text, str): data = text.encode() @@ -524,6 +551,8 @@ def copy_file_if_changed(src: Path, dst: Path) -> bool: Returns True if file was copied, False if files already matched. """ + import shutil + if file_compare(src, dst): return False dst.parent.mkdir(parents=True, exist_ok=True) @@ -627,7 +656,7 @@ def add_class_to_obj(value, cls): raise -def snake_case(value): +def snake_case(value: str) -> str: """Same behaviour as `helpers.cpp` method `str_snake_case`.""" return value.replace(" ", "_").lower() @@ -635,7 +664,7 @@ def snake_case(value): _DISALLOWED_CHARS = re.compile(r"[^a-zA-Z0-9-_]") -def sanitize(value): +def sanitize(value: str) -> str: """Same behaviour as `helpers.cpp` method `str_sanitize`.""" return _DISALLOWED_CHARS.sub("_", value) @@ -685,7 +714,7 @@ class ProgressBar: def docs_url(path: str) -> str: """Return the URL to the documentation for a given path.""" # Local import to avoid circular import - from esphome.config_validation import Version + from esphome.core import Version version = Version.parse(ESPHOME_VERSION) if version.is_beta: diff --git a/esphome/platform_hooks.py b/esphome/platform_hooks.py new file mode 100644 index 0000000000..10515723de --- /dev/null +++ b/esphome/platform_hooks.py @@ -0,0 +1,175 @@ +"""Registry of platform packages that provide optional CLI hooks. + +The logs/upload fast path must know whether a target platform overrides +``show_logs``/``upload_program`` or provides ``process_stacktrace`` +without importing the platform package to find out; importing one pulls +in the whole validation stack (config_validation, voluptuous, boards), +which costs seconds on slow hardware. Keep the mapping in sync with the +hook definitions in ``esphome/components/*/__init__.py``; a unit test +imports each platform package and fails when they drift. + +The compile-path ``run_compile`` hook is deliberately not registered: +compiling imports the platform package regardless, so its probe in +``__main__.py`` stays eager. Both log paths resolve +``process_stacktrace`` through ``esphome.stacktrace.LogLineProcessor``, +which uses get_stacktrace_handler below. +""" + +from __future__ import annotations + +from collections.abc import Callable +from importlib import import_module +import logging +from typing import Any, Final + +from esphome.const import ( + PLATFORM_ESP32, + PLATFORM_ESP8266, + PLATFORM_NRF52, + PLATFORM_RP2, + Platform, +) + +_LOGGER = logging.getLogger(__name__) + +# Hooks whose loss only degrades diagnostics; skipping one of these is +# logged at debug, while skipping a hook that changes what the CLI does +# (upload method, log transport) warns. A new hook is loud by default. +COSMETIC_HOOKS: Final = frozenset({"process_stacktrace"}) + +# Per-platform trigger languages for lazy stacktrace decoding: a +# matching line is what imports the platform package, so false triggers +# (8-digit uptime counters, ESP-IDF decimal timestamps) must stay out. +# Declaring a gate registers the process_stacktrace hook, and each gate +# must stay a superset of its decoder patterns' trigger language; both +# are enforced by tests/unit_tests/test_stacktrace.py. Stored as +# strings so a log session compiles only its own platform's gate. +STACKTRACE_GATES: Final[dict[str, str]] = { + PLATFORM_ESP32: ( + r"0x[0-9a-fA-F]{3,}\b" + r"|(?:PC|RA|MEPC|MTVAL|EXCVADDR|call)\s*[:=]\s*(?:0x)?4[0-9a-fA-F]{7}" + r"|CRASH DETECTED ON PREVIOUS BOOT" + ), + PLATFORM_ESP8266: ( + r"0x[0-9a-fA-F]{3,}\b" + r"|\b(?![0-9]{8}\b)[0-9a-fA-F]{8}\b" + r"|(?:PC|EXCVADDR|call)\s*[:=]\s*(?:0x)?4[0-9a-fA-F]{7}" + r"|[eE]xception \(\d+\):" + r"|>>>stack>>>" + r"|CRASH DETECTED ON PREVIOUS BOOT" + ), + PLATFORM_RP2: r"0x[0-9a-fA-F]{3,}\b|CRASH DETECTED ON PREVIOUS BOOT", + PLATFORM_NRF52: r"0x[0-9a-fA-F]{3,}\b|Last crash:", +} + +PLATFORM_HOOKS: Final[dict[str, frozenset[str]]] = { + "show_logs": frozenset({PLATFORM_NRF52}), + "upload_program": frozenset({PLATFORM_NRF52}), + "process_stacktrace": frozenset(STACKTRACE_GATES), +} + + +# The registry only speaks for in-tree platforms; a target platform +# supplied via external_components is normally not in Platform and falls +# back to probing the imported package, as the CLI did before the +# registry. Deliberate trade: an external component that shadows an +# in-tree platform name (the meta finder allows it) is treated as the +# in-tree platform here, so its own hooks are not probed. +_IN_TREE_PLATFORMS: Final = frozenset(Platform) + + +def has_registered_hook(platform: str, hook: str) -> bool: + """True when *platform* declares *hook* in ``PLATFORM_HOOKS``. + + Callers that defer imports key off this: a registered hook is known + to exist, so ``get_platform_hook`` can wait until it is needed; + anything else must be probed up front so availability is reported + at session start. Keeping the predicate here keeps the resolution + rule in one module. + """ + return platform in PLATFORM_HOOKS[hook] + + +def get_platform_hook(platform: str, hook: str) -> Callable[..., Any] | None: + """Return ``esphome.components..`` or None. + + In-tree platforms not registered for the hook return None without + being imported. A registered platform that no longer defines the + hook also returns None, so a stale registry degrades to the generic + path instead of raising. + """ + registered = has_registered_hook(platform, hook) + if not registered and platform in _IN_TREE_PLATFORMS: + return None + # For external platforms this probes the imported package like the + # CLI used to; the package can be missing entirely on the warm-cache + # path, where the external_components meta finder never registered. + # Degrade to the generic path then, but let a failure deeper in the + # package (missing dependency) surface. + module_name = f"esphome.components.{platform}" + try: + module = import_module(module_name) + except ModuleNotFoundError as err: + if registered or err.name != module_name: + raise + if hook in COSMETIC_HOOKS: + _LOGGER.debug( + "External platform %s is not importable; using the generic %s path", + platform, + hook, + ) + else: + # Deliberately loud even though the warm-cache path makes + # this expected: the user's platform hooks are not in effect + # for this run, and a silently substituted upload method is + # worse than a routine warning. + _LOGGER.warning( + "External platform %s is not importable; using the generic %s path", + platform, + hook, + ) + return None + handler = getattr(module, hook, None) + if handler is None: + if registered: + _LOGGER.warning( + "%s is registered for %s but no longer exposes it; using the generic path", + platform, + hook, + ) + else: + # The common case for external platforms; debug so a typoed + # hook name is still diagnosable without being noisy. + _LOGGER.debug( + "External platform %s does not expose %s; using the generic path", + platform, + hook, + ) + return handler + + +def get_stacktrace_handler(platform: str) -> Callable[..., Any] | None: + """Resolve ``process_stacktrace`` for *platform*, degrading with a log. + + Stacktrace decoding is a diagnostic nicety. This only distinguishes + an import failure from an ordinary capability gap so the message is + accurate; it returns None for both, and callers own any further + containment. Shared so the user-facing message lives in one place. + """ + try: + handler = get_platform_hook(platform, "process_stacktrace") + except ImportError as err: + # A real breakage, not an ordinary capability gap; say so louder. + _LOGGER.debug("Stacktrace analyzer import failed", exc_info=True) + _LOGGER.warning( + 'Stacktrace analysis is unavailable: analyzer for target platform "%s" failed to import: %s', + platform, + err, + ) + return None + if handler is None: + _LOGGER.info( + 'Stacktrace analysis is unavailable: no compatible analyzer found for target platform "%s".', + platform, + ) + return handler diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 1a523ce0ab..ee0a758a31 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -21,14 +21,15 @@ import itertools import json import logging import os -from pathlib import Path +from pathlib import Path, PurePosixPath import re import tempfile from typing import Any from urllib.parse import urlsplit, urlunsplit +from urllib.request import url2pathname from esphome import git -from esphome.core import CORE, Library +from esphome.core import CORE, EsphomeError, Library from esphome.framework_helpers import archive_extract_all, download_from_mirrors, rmdir _LOGGER = logging.getLogger(__name__) @@ -73,6 +74,14 @@ class Source: ) -> Path: raise NotImplementedError + def source_root(self, build_path: Path) -> Path: + """Directory holding the library's own files (manifest + sources). + + Defaults to the downloaded build directory; a source that references its + files in place (:class:`LocalSource`) overrides this to point elsewhere. + """ + return build_path + class URLSource(Source): def __init__(self, url: str): @@ -143,6 +152,53 @@ class GitSource(Source): return f"{self.url}#{self.ref}" if self.ref else self.url +class LocalSource(Source): + """A library that already exists as a directory on the local filesystem. + + Referenced with a ``file://`` URL (PlatformIO's spelling for a local library + folder). Nothing is copied: the backend generates its build files into an + otherwise empty cache directory and references the library's own sources in + place by absolute path (via :meth:`source_root`). So the user's source tree + stays untouched and edits are picked up on the next build without syncing. + """ + + def __init__(self, path: str): + self.local_path = path + + def download( + self, dir_suffix: str, force: bool = False, salt: str = "", namespace: str = "" + ) -> Path: + src = Path(self.local_path) + if not src.is_dir(): + # EsphomeError (not InvalidLibrary) so the CLI prints a clean message + # instead of a traceback -- pointing a file:// at a missing folder is + # the most common first mistake with a local library. + raise EsphomeError( + f"Local library directory does not exist: {self.local_path}" + ) + base_dir = Path(CORE.data_dir) / DOMAIN + if namespace: + base_dir = base_dir / namespace + h = hashlib.new("sha256") + h.update(str(src.resolve()).encode()) + if salt: + h.update(salt.encode()) + # Only the generated build files live here; the library's own sources + # are referenced in place from source_root(). + path = base_dir / h.hexdigest()[:8] / dir_suffix + path.mkdir(parents=True, exist_ok=True) + return path + + def source_root(self, build_path: Path) -> Path: + return Path(self.local_path) + + def __str__(self): + path = Path(self.local_path) + # as_uri() needs an absolute path; _node_key rejects relative file:// + # URLs, but guard anyway so a diagnostic can't itself raise. + return path.as_uri() if path.is_absolute() else f"file://{self.local_path}" + + class InvalidLibrary(Exception): pass @@ -162,6 +218,9 @@ class ConvertedLibrary: self.data = {} self.dependencies: list[ConvertedLibrary] = [] self._path: Path | None = None + # Where the library's own files live (manifest + sources). Set by + # download(); equals path for registry/git, the user's dir for local. + self.source_path: Path | None = None def __str__(self): return f"{self.name}@{self.version}={self.source}" @@ -176,6 +235,16 @@ class ConvertedLibrary: def path(self, value: Path) -> None: self._path = value + @property + def source_dir(self) -> Path: + """Directory the library's own files (manifest + sources) are read from. + + The build dir for a registry/git source; the user's directory for a + local library. Backends read sources from here and emit their build + files into ``path``. + """ + return self.source_path or self.path + def get_sanitized_name(self): return re.sub(r"[^a-zA-Z0-9_.\-/]", "_", self.name) @@ -193,6 +262,7 @@ class ConvertedLibrary: self.path = self.source.download( self.get_sanitized_name(), force=force, salt=salt, namespace=namespace ) + self.source_path = self.source.source_root(self.path) @dataclass @@ -515,11 +585,14 @@ class _LibNode: key: str is_git: bool + is_local: bool = False + is_registry: bool = False owner: str | None = None pkgname: str | None = None requirements: set[str] = field(default_factory=set) url: str | None = None ref: str | None = None + local_path: str | None = None edges: set[str] = field(default_factory=set) @@ -536,40 +609,83 @@ def _url_or_none(value: Any) -> str | None: def _node_key( name: str | None, version: str | None, repository: str | None -) -> tuple[str, bool, tuple[str | None, str | None]]: - """Return ``(key, is_git, locator)`` for a library or dependency spec. +) -> tuple[str, str, tuple[str | None, str | None]]: + """Return ``(key, kind, locator)`` for a library or dependency spec. - The key is derived from the *input* spec (the registry name as written, or - the git URL path), not the resolved canonical name. So a package referenced - inconsistently -- bare ``name`` vs ``owner/name``, or git vs registry -- maps - to distinct keys and isn't deduplicated; ``convert_libraries`` warns about - that after resolution rather than merging the nodes. + ``kind`` is one of: - PlatformIO's Library Manager also accepted a git URL in the *name* - position (``add_library("https://github.com/x/y", None)``), including the - ``git+`` VCS prefix and the ``CustomName=URL`` form; recognize those here - so such specs resolve as git sources instead of failing a registry lookup. + - ``"registry"`` -- ``locator`` is ``(owner, pkgname)``. + - ``"git"`` -- ``locator`` is ``(url, ref)``. + - ``"local"`` -- a ``file://`` directory; ``locator`` is ``(path, None)``. + + The key is derived from the *input* spec (the registry name as written, the + git URL path, or the custom name / directory name for a local folder), not + the resolved canonical name. So a package referenced inconsistently -- bare + ``name`` vs ``owner/name``, or git vs registry -- maps to distinct keys and + isn't deduplicated; ``convert_libraries`` warns about that after resolution + rather than merging the nodes. + + PlatformIO's Library Manager also accepted a URL in the *name* position + (``add_library("https://github.com/x/y", None)``), including the ``git+`` + VCS prefix and the ``CustomName=URL`` form; recognize those here so such + specs resolve as git (or local) sources instead of failing a registry + lookup. A plain ``file://`` URL is PlatformIO's spelling for a local library + folder, so it resolves as a local directory; ``git+file://`` stays a git + source. """ if not repository and name and "://" in name: - # Try the whole name first so a bare URL whose query contains ``=`` - # stays intact; fall back to the ``CustomName=URL`` form, where the - # key derives from the URL path and the custom name is irrelevant. - repository = _url_or_none(name) or _url_or_none(name.split("=", 1)[-1]) - if repository is None: + # Split a ``CustomName=URL`` name, but only when the whole string isn't + # itself a valid URL (a bare URL whose query contains ``=`` must stay + # intact). + custom_name, candidate = None, name + if "=" in name and _url_or_none(name) is None: + custom_name, candidate = name.split("=", 1) + try: + scheme = urlsplit(candidate).scheme + except ValueError: + scheme = "" + if scheme == "file" or _url_or_none(candidate): + name, repository = custom_name, candidate + else: # Anything with ``://`` was meant to be a URL; failing it fast # beats a confusing registry "package not found" error. raise RuntimeError(f"Invalid PIO library URL: {name}") if repository: + is_git_prefixed = repository.startswith("git+") split_result = urlsplit(repository.removeprefix("git+")) + if split_result.scheme == "file" and not is_git_prefixed: + # A plain file:// URL points at a local library directory. A local + # file URL is written file:///absolute/path (empty host) or, less + # commonly, file://localhost/path. Anything else -- a real host, or + # a relative path whose first segment parses as the host -- is + # rejected rather than silently resolved to the wrong directory. + if split_result.netloc not in ("", "localhost"): + raise RuntimeError( + f"Unsupported host in file:// library URL '{repository}'; " + "use an absolute path, e.g. file:///path/to/lib" + ) + # Validate the URL path itself (always POSIX-style, leading slash), + # not the OS path: on Windows a "/foo" path is not is_absolute() + # without a drive, which would wrongly reject a valid file:/// URL. + # Reject a relative path (``file:lib_dev``) or a bare root + # (``file:///``, which has no final segment). + url_path = split_result.path + if not url_path.startswith("/") or not PurePosixPath(url_path).name: + raise RuntimeError( + f"file:// library URL '{repository}' must be an absolute " + "directory path, e.g. file:///path/to/lib" + ) + path = url2pathname(url_path) + return (name or PurePosixPath(url_path).name), "local", (path, None) key = str(split_result.path).strip("/").removesuffix(".git") ref = split_result.fragment.strip() or None url = urlunsplit(split_result._replace(fragment="")) - return key, True, (url, ref) + return key, "git", (url, ref) if name and "/" in name: owner, pkgname = name.split("/", 1) else: owner, pkgname = None, name - return name, False, (owner, pkgname) + return name, "registry", (owner, pkgname) def convert_libraries( @@ -618,13 +734,45 @@ def convert_libraries( return name.split("/")[-1].lower() in lib_ignore def add_spec(name: str | None, version: str | None, repository: str | None) -> str: - key, is_git, locator = _node_key(name, version, repository) - node = nodes.get(key) or _LibNode(key=key, is_git=is_git) + key, kind, locator = _node_key(name, version, repository) + node = nodes.get(key) or _LibNode(key=key, is_git=kind == "git") nodes[key] = node - if is_git: + # The same key requested from two different kinds of source (or two + # different local paths) is a config mistake: one silently wins. Warn so + # it isn't a surprise. (git-vs-registry is reported separately below.) + if kind == "git": + if node.is_local: + _LOGGER.warning( + "Library %s is requested as both a local directory and a git " + "source; using the git source.", + key, + ) node.is_git = True node.url, node.ref = locator + elif kind == "local": + new_path = locator[0] + if node.is_git: + # git wins (checked first when building the source); leave the + # node as a git source. + _LOGGER.warning( + "Library %s is requested as both a local directory and a git " + "source; using the git source.", + key, + ) + else: + if node.is_local and node.local_path != new_path: + _LOGGER.warning( + "Library %s is requested from two local directories (%s " + "and %s); using %s.", + key, + node.local_path, + new_path, + new_path, + ) + node.is_local = True + node.local_path = new_path else: + node.is_registry = True node.owner, node.pkgname = locator if version: node.requirements.add(version) @@ -658,6 +806,8 @@ def convert_libraries( if node.is_git: component = ConvertedLibrary(key, "*", GitSource(node.url, node.ref)) + elif node.is_local: + component = ConvertedLibrary(key, "*", LocalSource(node.local_path)) else: owner, name, version, url = _resolve_registry_version( node.owner, node.pkgname, node.requirements @@ -667,20 +817,22 @@ def convert_libraries( ) component.download(salt=salt, namespace=backend.cache_key) - library_json_path = component.path / "library.json" - library_properties_path = component.path / "library.properties" + source_dir = component.source_dir + library_json_path = source_dir / "library.json" + library_properties_path = source_dir / "library.properties" has_json = library_json_path.is_file() has_properties = library_properties_path.is_file() - if not has_json and not has_properties: + if not has_json and not has_properties and not node.is_local: # The shared cache can hold a broken copy (e.g. a clone or an # extraction interrupted by a killed process). Force one # re-download so a bad cache entry self-heals instead of failing - # every build until the user runs a full clean. + # every build until the user runs a full clean. A local source is + # read in place, so there is nothing to re-download. _LOGGER.warning( "Library %s at %s is missing library.json and library.properties; " "re-downloading", key, - component.path, + source_dir, ) component.download(force=True, salt=salt, namespace=backend.cache_key) has_json = library_json_path.is_file() @@ -690,9 +842,14 @@ def convert_libraries( elif has_properties: component.data = _parse_library_properties(library_properties_path) else: - raise RuntimeError( + # For a local library a missing manifest is user input, so raise + # EsphomeError (clean CLI message) like the missing-directory case; + # for registry/git a missing manifest means a corrupt cache, which + # is not user error, so keep RuntimeError. + error_cls = EsphomeError if node.is_local else RuntimeError + raise error_cls( f"Invalid PIO library {key}: missing library.json and " - f"library.properties in {component.path}" + f"library.properties in {source_dir}" ) try: @@ -735,17 +892,26 @@ def convert_libraries( node.edges.add(dep_key) worklist.append(dep_key) - # A git source wins over any registry version requested for the same - # component. That's intentional, but warn so a dropped registry pin isn't a - # silent surprise. + # A git or local source wins over the same component requested from the + # registry. That's intentional, but warn so the dropped registry spec isn't + # a silent surprise -- including when it carried no version pin (a bare + # cg.add_library("Foo"), which is how most components add libraries). for node in nodes.values(): - if node.is_git and node.requirements: + if (node.is_git or node.is_local) and (node.is_registry or node.requirements): + source = "git" if node.is_git else "local" + registry = ( + f"registry version(s) {sorted(node.requirements)}" + if node.requirements + else "a registry package" + ) _LOGGER.warning( - "Library %s is requested both from a git source (%s) and as " - "registry version(s) %s; using the git source.", + "Library %s is requested both from a %s source (%s) and as %s; " + "using the %s source.", node.key, - node.url, - sorted(node.requirements), + source, + node.url if node.is_git else node.local_path, + registry, + source, ) # Two graph nodes that resolve to the same component name (e.g. a package diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 0959cbfffb..32e30290ac 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -6,7 +6,7 @@ from pathlib import Path import re import shutil import sys -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import platformdirs @@ -364,18 +364,24 @@ def run_compile(config, verbose): def _run_idedata(config): args = ["-t", "idedata"] stdout = run_platformio_cli_run(config, False, *args, capture_stdout=True) + if not isinstance(stdout, str): + # run_external_process returns 1 instead of captured output when + # launching platformio raised; see the error it logged above. + raise EsphomeError("Could not launch platformio to get idedata") match = re.search(r'{\s*".*}', stdout) if match is None: - _LOGGER.error("Could not match idedata, please report this error") + # A run that launches but fails emits its build error instead of + # idedata; the logged stdout is the useful part, not a bug report. + _LOGGER.error("Could not find idedata in the platformio output") _LOGGER.error("Stdout: %s", stdout) - raise EsphomeError + raise EsphomeError("PlatformIO did not report idedata") try: return json.loads(match.group()) - except ValueError: + except ValueError as err: _LOGGER.exception("Could not parse idedata") _LOGGER.error("Stdout: %s", stdout) - raise + raise EsphomeError("Could not parse idedata from platformio") from err def _load_idedata(config): @@ -419,9 +425,27 @@ class IDEData: def __init__(self, raw): self.raw = raw + def _require(self, *keys: str) -> Any: + """Read a nested key, classifying a miss as an environment error. + + A stale or truncated cached idedata JSON is the user's build + tree, not a bug; recompiling regenerates it. The message names + the key so a platformio schema change stays diagnosable. + """ + value = self.raw + # TypeError covers a key that is null instead of absent. + try: + for key in keys: + value = value[key] + except (KeyError, TypeError) as err: + raise EsphomeError( + f"Cached idedata is incomplete (missing {'.'.join(keys)})" + ) from err + return value + @property def firmware_elf_path(self) -> Path: - return Path(self.raw["prog_path"]) + return Path(self._require("prog_path")) @property def firmware_bin_path(self) -> Path: @@ -429,15 +453,22 @@ class IDEData: @property def extra_flash_images(self) -> list[FlashImage]: - return [ - FlashImage(path=Path(entry["path"]), offset=entry["offset"]) - for entry in self.raw["extra"]["flash_images"] - ] + try: + return [ + FlashImage(path=Path(entry["path"]), offset=entry["offset"]) + for entry in self._require("extra", "flash_images") + ] + except (KeyError, TypeError) as err: + # Covers entries missing path/offset and a null or non-list + # flash_images value alike. + raise EsphomeError( + "Cached idedata is incomplete (malformed extra.flash_images)" + ) from err @property def cc_path(self) -> str: # For example /Users//.platformio/packages/toolchain-xtensa32/bin/xtensa-esp32-elf-gcc - return self.raw["cc_path"] + return self._require("cc_path") @property def addr2line_path(self) -> str: diff --git a/esphome/resolver.py b/esphome/resolver.py index f80a910afe..68bf37eecd 100644 --- a/esphome/resolver.py +++ b/esphome/resolver.py @@ -8,7 +8,7 @@ import os from aioesphomeapi.core import ResolveAPIError, ResolveTimeoutAPIError import aioesphomeapi.host_resolver as hr -from esphome.async_thread import AsyncThreadRunner +from esphome.async_thread import AsyncDispatchTimeout, run_async from esphome.core import EsphomeError _LOGGER = logging.getLogger(__name__) @@ -31,9 +31,9 @@ class AsyncResolver: This resolver uses aioesphomeapi's async_resolve_host to handle DNS resolution, including proper .local domain fallback. Running in a thread - (via :class:`AsyncThreadRunner`) allows us to get the result immediately - without waiting for ``asyncio.run()`` to complete its cleanup cycle, which - can take significant time. + (via :func:`run_async`) allows us to get the result immediately without + waiting for ``asyncio.run()`` to complete its cleanup cycle, which can + take significant time. """ def __init__(self, hosts: list[str], port: int) -> None: @@ -48,21 +48,13 @@ class AsyncResolver: ) def resolve(self) -> list[hr.AddrInfo]: - """Start the thread and wait for the result.""" - runner: AsyncThreadRunner[list[hr.AddrInfo]] = AsyncThreadRunner(self._resolve) - runner.start() - - if not runner.event.wait( - timeout=RESOLVE_TIMEOUT + 1.0 - ): # Give it 1 second more than the resolver timeout - raise EsphomeError("Timeout resolving IP address") - - if exc := runner.exception: - if isinstance(exc, ResolveTimeoutAPIError): - raise EsphomeError(f"Timeout resolving IP address: {exc}") from exc - if isinstance(exc, ResolveAPIError): - raise EsphomeError(f"Error resolving IP address: {exc}") from exc - raise exc - - assert runner.result is not None # guaranteed when event set and no exception - return runner.result + """Resolve and wait for the result.""" + try: + # Give it 1 second more than the resolver timeout + return run_async(self._resolve, timeout=RESOLVE_TIMEOUT + 1.0) + except ResolveTimeoutAPIError as exc: + raise EsphomeError(f"Timeout resolving IP address: {exc}") from exc + except ResolveAPIError as exc: + raise EsphomeError(f"Error resolving IP address: {exc}") from exc + except AsyncDispatchTimeout as exc: + raise EsphomeError("Timeout resolving IP address") from exc diff --git a/esphome/stacktrace.py b/esphome/stacktrace.py new file mode 100644 index 0000000000..0adbbf6f2b --- /dev/null +++ b/esphome/stacktrace.py @@ -0,0 +1,124 @@ +"""Lazy stack-trace decoding for streamed device log lines. + +Shared by the serial (run_miniterm) and network (api_client) log paths. +Deliberately light: importing this module must not pull in aioesphomeapi +or any platform package. +""" + +from __future__ import annotations + +import logging +import re +from typing import TYPE_CHECKING + +from esphome import platform_hooks +from esphome.core import EsphomeError +from esphome.types import ConfigType + +if TYPE_CHECKING: + from collections.abc import Callable + + # The contract every platform's process_stacktrace implements. + StacktraceHandler = Callable[[ConfigType, str, bool], bool] + +_LOGGER = logging.getLogger(__name__) + + +class LogLineProcessor: + """Feeds incoming log lines to the stack-trace decoder. + + Three responsibilities beyond just calling the decoder: + 1. Resolve the platform decoder lazily: registered platforms import + nothing until a line matches their gate, registry misses report + at session start without importing, and external platforms + resolve eagerly since their import is unavoidable and belongs + off the streaming callback. + 2. Catch everything the decoder can raise; decoding is a diagnostic + nicety and an escaping exception would log a traceback per dump + line, burying the dump the user is trying to read. + 3. Disable decoding for the rest of the session after a failure. + Retrying means re-running a failing toolchain subprocess on the + stream, and nothing a decode failure depends on heals by itself; + the warning names the fix and a fresh run picks it up. Working + at all requires catching every failure, which is why 2 is not + narrowed to EsphomeError. + """ + + def __init__(self, config: ConfigType, platform: str) -> None: + self._config = config + self._platform = platform + self._platform_handler: StacktraceHandler | None = None + self._decode_enabled = True + # None only for platforms resolved eagerly below; a registered + # platform always declares a gate. + gate = platform_hooks.STACKTRACE_GATES.get(platform) + self._gate: re.Pattern[str] | None = None if gate is None else re.compile(gate) + self.backtrace_state = False + if not platform_hooks.has_registered_hook(platform, "process_stacktrace"): + self._resolve_handler() + + def process_line(self, raw_line: str) -> None: + if not self._decode_enabled: + return + if self._platform_handler is None: + if not self._gate.search(raw_line): + return + # Deliberate trade: the platform import blocks the stream + # here, once per session, instead of at every startup. + if not self._resolve_handler(): + return + _LOGGER.debug( + "Stacktrace gate fired for %s; decoder resolved", self._platform + ) + self._feed(raw_line) + + def _resolve_handler(self) -> bool: + try: + handler = platform_hooks.get_stacktrace_handler(self._platform) + except Exception as exc: # noqa: BLE001 # pylint: disable=broad-except + # Containment includes resolution: a broken platform package + # must not kill the session or retry per line. + _LOGGER.debug("Stacktrace analyzer resolution failed", exc_info=True) + _LOGGER.warning( + 'Stacktrace analysis is unavailable: analyzer for target platform "%s" could not be loaded: %s', + self._platform, + f"{type(exc).__name__}: {exc}", + ) + handler = None + if handler is None: + self._decode_enabled = False + return False + self._platform_handler = handler + return True + + def _feed(self, raw_line: str) -> None: + try: + self.backtrace_state = self._platform_handler( + self._config, raw_line, self.backtrace_state + ) + except Exception as exc: # noqa: BLE001 # pylint: disable=broad-except + self._decode_enabled = False + self.backtrace_state = False + _LOGGER.debug("Stack-trace decoding failed", exc_info=True) + if isinstance(exc, (EsphomeError, OSError)): + # Environment failures (idedata, build tree) get the + # remediation hint; the fallback string keeps a bare + # EsphomeError from rendering as empty parens. + _LOGGER.warning( + "Crash trace decoding unavailable: %s. " + "Run 'esphome compile' for this device to enable PC decoding.", + str(exc) or "build artifacts not found locally", + ) + else: + # A decoder bug is ESPHome's problem, not the user's; + # don't send them to recompile a healthy build. Name the + # type so a bare KeyError message reads as an exception. + detail = type(exc).__name__ + if msg := str(exc): + detail = f"{detail}: {msg}" + _LOGGER.warning( + 'Crash trace decoding disabled: decoder for "%s" raised %s ' + "(this is a bug; run with -v for the traceback)", + self._platform, + detail, + ) diff --git a/esphome/storage_json.py b/esphome/storage_json.py index 6376e573c4..a90a36b848 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -1,26 +1,32 @@ from __future__ import annotations import binascii -from datetime import datetime import json import logging import os from pathlib import Path +from typing import TYPE_CHECKING from esphome import const from esphome.const import ( CONF_DISABLED, CONF_MDNS, KEY_CORE, + KEY_ESP32, KEY_FRAMEWORK_VERSION, + KEY_IDF_VERSION, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM, + KEY_VARIANT, Toolchain, ) -from esphome.core import CORE, EsphomeError +from esphome.core import CORE, EsphomeError, Version from esphome.helpers import write_file_if_changed from esphome.types import CoreType +if TYPE_CHECKING: + from datetime import datetime + _LOGGER = logging.getLogger(__name__) @@ -69,6 +75,17 @@ def _to_path_if_not_none(value: str | None) -> Path | None: return Path(value) if value is not None else None +def _parse_framework_version(framework_version: str) -> Version: + try: + return Version.parse(framework_version) + except ValueError as err: + raise EsphomeError( + f"Could not parse the framework version " + f"{framework_version!r} from {storage_path()}. " + f"Please clean the build files and recompile." + ) from err + + class StorageJSON: """Persisted device metadata sidecar. @@ -315,41 +332,20 @@ class StorageJSON: } # The compile pipeline populates CORE.data[KEY_ESP32] when esp32's # validator runs; on the cache fast path that validator is skipped, - # so populate the variant upload_using_esptool reads via - # esp32.get_esp32_variant(). target_platform on disk is the variant - # (e.g. "ESP32S3"); core_platform is the family (e.g. "esp32"). + # so populate the variant upload_using_esptool reads from + # CORE.data[KEY_ESP32][KEY_VARIANT]. target_platform on disk is the + # variant (e.g. "ESP32S3"); core_platform is the family (e.g. "esp32"). if target_platform == const.PLATFORM_ESP32: - from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION - from esphome.const import KEY_VARIANT - esp32_data = {KEY_VARIANT: self.target_platform} if self.framework_version: - import esphome.config_validation as cv - - try: - esp32_data[KEY_IDF_VERSION] = cv.Version.parse( - self.framework_version - ) - except ValueError as err: - raise EsphomeError( - f"Could not parse the framework version " - f"{self.framework_version!r} from {storage_path()}. " - f"Please clean the build files and recompile." - ) from err - CORE.data[KEY_ESP32] = esp32_data - elif target_platform == const.PLATFORM_NRF52 and self.framework_version: - import esphome.config_validation as cv - - try: - CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] = cv.Version.parse( + esp32_data[KEY_IDF_VERSION] = _parse_framework_version( self.framework_version ) - except ValueError as err: - raise EsphomeError( - f"Could not parse the framework version " - f"{self.framework_version!r} from {storage_path()}. " - f"Please clean the build files and recompile." - ) from err + CORE.data[KEY_ESP32] = esp32_data + elif target_platform == const.PLATFORM_NRF52 and self.framework_version: + CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] = _parse_framework_version( + self.framework_version + ) def __eq__(self, o) -> bool: return isinstance(o, StorageJSON) and self.as_dict() == o.as_dict() @@ -379,6 +375,10 @@ class EsphomeStorageJSON: @property def last_update_check(self) -> datetime | None: + # Deferred: this module is on the upload/logs fast path; only the + # dashboard's update check touches these accessors. + from datetime import datetime + try: # Stored format is naive ISO without %z; preserved for backward compat. return datetime.strptime( # noqa: DTZ007 diff --git a/esphome/util.py b/esphome/util.py index f7d33bd2a9..136d6362f2 100644 --- a/esphome/util.py +++ b/esphome/util.py @@ -1,11 +1,10 @@ import collections -from collections.abc import Callable +from collections.abc import Callable, Iterable from dataclasses import dataclass import io import logging from pathlib import Path import re -import subprocess import sys from typing import TYPE_CHECKING, Any @@ -289,6 +288,9 @@ def run_external_command( def run_external_process(*cmd: str, **kwargs: Any) -> int | str: + # Deferred: an OTA upload/logs run never spawns an external process. + import subprocess + full_cmd = " ".join(shlex_quote(x) for x in cmd) _LOGGER.debug("Running: %s", full_cmd) filter_lines = kwargs.get("filter_lines") @@ -329,13 +331,6 @@ def is_dev_esphome_version(): return "dev" in const.__version__ -def parse_esphome_version() -> tuple[int, int, int]: - match = re.match(r"^(\d+).(\d+).(\d+)(-dev\d*|b\d*)?$", const.__version__) - if match is None: - raise ValueError(f"Failed to parse ESPHome version '{const.__version__}'") - return int(match.group(1)), int(match.group(2)), int(match.group(3)) - - # Custom OrderedDict with nicer repr method for debugging class OrderedDict(collections.OrderedDict): def __repr__(self): @@ -356,7 +351,7 @@ def list_yaml_files(configs: list[str | Path]) -> list[Path]: return sorted(files) -def filter_yaml_files(files: list[Path]) -> list[Path]: +def filter_yaml_files(files: Iterable[Path]) -> list[Path]: return [ f for f in files @@ -450,6 +445,8 @@ def detect_rp2040_bootsel(picotool_path: str | Path) -> BootselResult: Returns a BootselResult with the number of devices found (by counting 'type:' lines in output), and whether a permission error was detected. """ + import subprocess + try: result = subprocess.run( [str(picotool_path), "info", "-d"], diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index c2db9b97ed..981e508d5d 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Callable, Generator +from collections.abc import Callable, Generator, Iterator from contextlib import contextmanager, suppress from dataclasses import dataclass, field import functools @@ -253,7 +253,7 @@ class IncludeFile: if self._content is not _UNSET: return self._content if self.has_unresolved_expressions(): - from esphome.config_validation import Invalid + from voluptuous import Invalid raise Invalid( f"Cannot load include with unresolved substitutions: {self.file}" @@ -266,12 +266,135 @@ class IncludeFile: """Check if the filename contains substitution variables or Jinja expressions.""" return has_substitution_or_expression(str(self.file)) + def with_file(self, file: Path | str) -> IncludeFile: + """Clone this include with *file* as the filename.""" + return IncludeFile(self.parent_file, file, self.vars, self.yaml_loader) + + +def _is_visible_path(rel: Path) -> bool: + """Report whether no component of *rel* is hidden (``..`` stays valid).""" + return all(part == ".." or _is_file_valid(part) for part in rel.parts) + + +def _glob_include_candidates(parent_dir: Path, pattern: str) -> list[Path]: + """ + Expand a candidate glob under *parent_dir*, keeping hidden files out. + + An un-globbable pattern (absolute, or one the filesystem rejects) is + skipped instead of crashing discovery. + """ + try: + found_paths = parent_dir.glob(pattern) + return [ + rel + for found in found_paths + if _is_visible_path(rel := found.relative_to(parent_dir)) + ] + except (NotImplementedError, ValueError) as err: + _LOGGER.debug("Cannot glob include pattern %r: %s", pattern, err) + return [] + except OSError as err: + _LOGGER.warning("I/O error globbing include pattern %r: %s", pattern, err) + return [] + + +def _candidate_include_paths(include: IncludeFile) -> list[Path]: + """Enumerate resolved files an expression-templated ``!include`` could select. + + Wildcard patterns from ``substitutions.include_candidate_patterns`` glob + under the including file's directory with hidden files excluded (like + ``!include_dir_*``); literal branch patterns are tried verbatim. Matches + still carrying expression markers or pointing back at the including file + are skipped. + """ + # Deferred import — the substitutions component imports this module. + from esphome.components.substitutions import include_candidate_patterns + + parent_dir = include.parent_file.parent + parent_resolved = include.parent_file.resolve() + candidates: list[Path] = [] + for pattern in include_candidate_patterns(str(include.file)): + if "*" in pattern: + matches = sorted(_glob_include_candidates(parent_dir, pattern)) + else: + matches = [Path(pattern)] + for match in matches: + if has_substitution_or_expression(str(match)): + continue + candidate = parent_dir / match + if not candidate.is_file(): + continue + resolved = candidate.resolve() + if resolved == parent_resolved: + continue + candidates.append(resolved) + return candidates + + +def _load_include_candidates( + include: IncludeFile, + *, + warn_on_unresolved: bool, + seen: set[int], + expanded_paths: set[Path], + keepalive: list[Any], +) -> None: + """Load every filesystem candidate for an unresolved ``IncludeFile``.""" + from voluptuous import Invalid + + log = _LOGGER.warning if warn_on_unresolved else _LOGGER.debug + candidates = _candidate_include_paths(include) + if not candidates: + log( + "Cannot resolve !include %s (referenced from %s) with substitutions in path", + include.file, + include.parent_file, + ) + return + _LOGGER.debug( + "Expanding !include %s (referenced from %s) to %d candidate file(s)", + include.file, + include.parent_file, + len(candidates), + ) + for candidate in candidates: + if candidate in expanded_paths: + continue + expanded_paths.add(candidate) + try: + loaded = include.with_file(candidate).load() + except (EsphomeError, Invalid) as err: + # Unlike an unresolved pattern (expected during the discovery + # re-parse), a matched on-disk candidate that fails to load is a + # genuine user error; warn in every mode. The file itself is + # still tracked (the load listener fires before parsing), only + # its nested includes go undiscovered. + _LOGGER.warning( + "Failed to load candidate %s for !include %s: %s", + candidate, + include.file, + err, + ) + continue + # The throwaway IncludeFile is this tree's only owner; keep the tree + # alive so ids recorded in ``seen`` stay unique for the traversal. + keepalive.append(loaded) + force_load_include_files( + loaded, + warn_on_unresolved=warn_on_unresolved, + _seen=seen, + _expanded_paths=expanded_paths, + _keepalive=keepalive, + ) + def force_load_include_files( obj: Any, *, warn_on_unresolved: bool = True, _seen: set[int] | None = None, + _expanded_paths: set[Path] | None = None, + _keepalive: list[Any] | None = None, ) -> None: """Recursively resolve any deferred ``IncludeFile`` instances in a YAML tree. @@ -282,29 +405,43 @@ def force_load_include_files( loader fires and records every reachable file. ``IncludeFile`` instances whose path contains unresolved substitution - variables cannot be loaded. By default a warning is logged for each one; - pass ``warn_on_unresolved=False`` (used by discovery paths that run on a - fresh re-parse where substitutions haven't been applied yet) to demote it - to a debug log. + variables or Jinja expressions are expanded against the filesystem and + every existing candidate file is loaded, so bundles ship all branches the + expression could select. By default a warning is logged when no candidate + exists; pass ``warn_on_unresolved=False`` (used by discovery paths that + run on a fresh re-parse where substitutions haven't been applied yet) to + demote it to a debug log. """ + from voluptuous import Invalid + if _seen is None: _seen = set() + if _expanded_paths is None: + _expanded_paths = set() + if _keepalive is None: + # ``_seen`` tracks ids, which is only safe while every traversed + # object stays alive; candidate trees are otherwise freed between + # loop iterations and CPython recycles their addresses, making a + # fresh tree look already seen. Discovery is a one-shot operation, + # so holding the parsed trees costs nothing. + _keepalive = [] if isinstance(obj, IncludeFile): if id(obj) in _seen: return _seen.add(id(obj)) if obj.has_unresolved_expressions(): - log = _LOGGER.warning if warn_on_unresolved else _LOGGER.debug - log( - "Cannot resolve !include %s (referenced from %s) with substitutions in path", - obj.file, - obj.parent_file, + _load_include_candidates( + obj, + warn_on_unresolved=warn_on_unresolved, + seen=_seen, + expanded_paths=_expanded_paths, + keepalive=_keepalive, ) return try: loaded = obj.load() - except EsphomeError as err: + except (EsphomeError, Invalid) as err: _LOGGER.warning( "Failed to load !include %s (referenced from %s): %s", obj.file, @@ -313,7 +450,11 @@ def force_load_include_files( ) return force_load_include_files( - loaded, warn_on_unresolved=warn_on_unresolved, _seen=_seen + loaded, + warn_on_unresolved=warn_on_unresolved, + _seen=_seen, + _expanded_paths=_expanded_paths, + _keepalive=_keepalive, ) elif isinstance(obj, dict): if id(obj) in _seen: @@ -321,7 +462,11 @@ def force_load_include_files( _seen.add(id(obj)) for value in obj.values(): force_load_include_files( - value, warn_on_unresolved=warn_on_unresolved, _seen=_seen + value, + warn_on_unresolved=warn_on_unresolved, + _seen=_seen, + _expanded_paths=_expanded_paths, + _keepalive=_keepalive, ) elif isinstance(obj, (list, tuple)): if id(obj) in _seen: @@ -329,7 +474,11 @@ def force_load_include_files( _seen.add(id(obj)) for item in obj: force_load_include_files( - item, warn_on_unresolved=warn_on_unresolved, _seen=_seen + item, + warn_on_unresolved=warn_on_unresolved, + _seen=_seen, + _expanded_paths=_expanded_paths, + _keepalive=_keepalive, ) @@ -402,6 +551,13 @@ def _add_data_ref(fn): # Let generator finish for _ in generator: pass + # Fast mode keeps this per-node attribute check instead of a second + # constructor table: measured, fast mode already parses within ~8% + # of a raw CSafeLoader, so a parallel table isn't worth the + # duplication (and undecorated constructors return generators with + # different resolution ordering). + if not loader.track_document_range: + return res res = make_data_base(res) if isinstance(res, ESPHomeDataBase): res.from_node(node) @@ -441,14 +597,19 @@ def _resolve_merge_include(value: Any, node: yaml.Node, value_node: yaml.Node) - class ESPHomeLoaderMixin: - """Loader class that keeps track of line numbers.""" + """Loader that tracks line numbers unless track_document_range is off.""" def __init__( - self, name: Path, yaml_loader: Callable[[Path], dict[str, Any]] + self, + name: Path, + yaml_loader: Callable[[Path], dict[str, Any]], + *, + track_document_range: bool, ) -> None: - """Initialize the loader.""" + """Initialize the loader. See load_yaml for track_document_range.""" self.name = name self.yaml_loader = yaml_loader + self.track_document_range = track_document_range @_add_data_ref def construct_yaml_int(self, node): @@ -511,8 +672,10 @@ class ESPHomeLoaderMixin: f'Invalid key "{key}" (not hashable)', key_node.start_mark ) from None - key = make_data_base(str(key)) - key.from_node(key_node) + key = str(key) + if self.track_document_range: + key = make_data_base(key) + key.from_node(key_node) # Check if it is a duplicate key if key in seen_keys: @@ -647,12 +810,12 @@ class ESPHomeLoaderMixin: @_add_data_ref def construct_include_dir_list(self, node: yaml.Node) -> list[dict[str, Any]]: - files = filter_yaml_files(_find_files(self._rel_path(node.value), "*.yaml")) + files = filter_yaml_files(find_files(self._rel_path(node.value), "*.yaml")) return [self.yaml_loader(f) for f in files] @_add_data_ref def construct_include_dir_merge_list(self, node: yaml.Node) -> list[dict[str, Any]]: - files = filter_yaml_files(_find_files(self._rel_path(node.value), "*.yaml")) + files = filter_yaml_files(find_files(self._rel_path(node.value), "*.yaml")) merged_list = [] for fname in files: loaded_yaml = self.yaml_loader(fname) @@ -664,7 +827,7 @@ class ESPHomeLoaderMixin: def construct_include_dir_named( self, node: yaml.Node ) -> OrderedDict[str, dict[str, Any]]: - files = filter_yaml_files(_find_files(self._rel_path(node.value), "*.yaml")) + files = filter_yaml_files(find_files(self._rel_path(node.value), "*.yaml")) mapping = OrderedDict() for fname in files: filename = fname.stem @@ -675,7 +838,7 @@ class ESPHomeLoaderMixin: def construct_include_dir_merge_named( self, node: yaml.Node ) -> OrderedDict[str, dict[str, Any]]: - files = filter_yaml_files(_find_files(self._rel_path(node.value), "*.yaml")) + files = filter_yaml_files(find_files(self._rel_path(node.value), "*.yaml")) mapping = OrderedDict() for fname in files: loaded_yaml = self.yaml_loader(fname) @@ -708,29 +871,37 @@ class ESPHomeLoaderMixin: class ESPHomeLoader(ESPHomeLoaderMixin, FastestAvailableSafeLoader): - """Loader class that keeps track of line numbers.""" + """C-accelerated loader; see ESPHomeLoaderMixin.""" def __init__( self, stream: TextIOBase | BytesIO, name: Path, yaml_loader: Callable[[Path], dict[str, Any]], + *, + track_document_range: bool, ) -> None: FastestAvailableSafeLoader.__init__(self, stream) - ESPHomeLoaderMixin.__init__(self, name, yaml_loader) + ESPHomeLoaderMixin.__init__( + self, name, yaml_loader, track_document_range=track_document_range + ) class ESPHomePurePythonLoader(ESPHomeLoaderMixin, PurePythonLoader): - """Loader class that keeps track of line numbers.""" + """Pure-Python loader with readable errors; see ESPHomeLoaderMixin.""" def __init__( self, stream: TextIOBase | BytesIO, name: Path, yaml_loader: Callable[[Path], dict[str, Any]], + *, + track_document_range: bool, ) -> None: PurePythonLoader.__init__(self, stream) - ESPHomeLoaderMixin.__init__(self, name, yaml_loader) + ESPHomeLoaderMixin.__init__( + self, name, yaml_loader, track_document_range=track_document_range + ) for _loader in (ESPHomeLoader, ESPHomePurePythonLoader): @@ -758,20 +929,31 @@ for _loader in (ESPHomeLoader, ESPHomePurePythonLoader): _loader.add_constructor("!remove", _loader.construct_remove) -def load_yaml(fname: Path, clear_secrets: bool = True) -> Any: +def load_yaml( + fname: Path, clear_secrets: bool = True, *, track_document_range: bool = True +) -> Any: + """Load a YAML file. + + track_document_range=False skips wrapping every node in an + ESPHomeDataBase subclass carrying its source range. That metadata + serves validation error messages and lambda source locations in + generated code; callers that neither validate nor generate code (the + upload/logs fast path re-reading the validated config cache) can skip + it, roughly halving parse time. + """ if clear_secrets: _SECRET_VALUES.clear() _SECRET_CACHE.clear() - return _load_yaml_internal(fname) + return _load_yaml_internal(fname, track_document_range=track_document_range) -def _load_yaml_internal(fname: Path) -> Any: +def _load_yaml_internal(fname: Path, *, track_document_range: bool = True) -> Any: """Load a YAML file.""" for listener in _load_listeners: listener(fname) try: with fname.open(encoding="utf-8") as f_handle: - res = parse_yaml(fname, f_handle) + res = parse_yaml(fname, f_handle, track_document_range=track_document_range) except (UnicodeDecodeError, OSError) as err: raise EsphomeError(f"Error reading file {fname}: {err}") from err # Top-level !include returns a deferred IncludeFile; resolve it so @@ -781,13 +963,32 @@ def _load_yaml_internal(fname: Path) -> Any: return res -def parse_yaml(file_name: Path, file_handle: TextIOWrapper, yaml_loader=None) -> Any: +_FAST_YAML_LOADER = functools.partial(_load_yaml_internal, track_document_range=False) + + +def parse_yaml( + file_name: Path, + file_handle: TextIOWrapper, + yaml_loader=None, + *, + track_document_range: bool = True, +) -> Any: """Parse a YAML file.""" if yaml_loader is None: - yaml_loader = _load_yaml_internal + # Nested loads (!include, !secret, !include_dir_*) inherit the + # same tracking mode. + yaml_loader = _load_yaml_internal if track_document_range else _FAST_YAML_LOADER + elif not track_document_range: + # A caller-supplied loader would silently revert nested loads to + # tracked mode; reject the combination instead of half-applying it. + raise ValueError("track_document_range=False requires the default yaml_loader") try: return _load_yaml_internal_with_type( - ESPHomeLoader, file_name, file_handle, yaml_loader + ESPHomeLoader, + file_name, + file_handle, + yaml_loader, + track_document_range=track_document_range, ) except EsphomeError: # Loading failed, so we now load with the Python loader which has more @@ -795,7 +996,11 @@ def parse_yaml(file_name: Path, file_handle: TextIOWrapper, yaml_loader=None) -> # Rewind the stream so we can try again file_handle.seek(0, 0) return _load_yaml_internal_with_type( - ESPHomePurePythonLoader, file_name, file_handle, yaml_loader + ESPHomePurePythonLoader, + file_name, + file_handle, + yaml_loader, + track_document_range=track_document_range, ) @@ -804,6 +1009,8 @@ def _load_yaml_internal_with_type( fname: Path, content: TextIOWrapper, yaml_loader: Callable[[Path], dict[str, Any]], + *, + track_document_range: bool, ) -> Any: """Load a YAML file. @@ -814,7 +1021,9 @@ def _load_yaml_internal_with_type( configuration. Frontmatter is ignored by config validation and code generation. """ - loader = loader_type(content, fname, yaml_loader) + loader = loader_type( + content, fname, yaml_loader, track_document_range=track_document_range + ) try: documents: list[Any] = [] while loader.check_data(): @@ -871,8 +1080,8 @@ def _is_file_valid(name: str) -> bool: return not name.startswith(".") -def _find_files(directory: Path, pattern): - """Recursively load files in a directory.""" +def find_files(directory: Path, pattern: str) -> Iterator[Path]: + """Recursively find files in a directory matching *pattern*, skipping hidden entries.""" for root, dirs, files in os.walk(directory): dirs[:] = [d for d in dirs if _is_file_valid(d)] for f in files: diff --git a/platformio.ini b/platformio.ini index 9f2ac74ac0..bf3b0685f8 100644 --- a/platformio.ini +++ b/platformio.ini @@ -44,6 +44,7 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} + https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea esphome/noise-c@0.1.11 ; api improv/Improv@1.2.6 ; improv_serial / esp32_improv kikuchan98/pngle@1.1.0 ; online_image @@ -80,7 +81,6 @@ lib_deps = Wire ; i2c (Arduino built-int) heman/AsyncMqttClient-esphome@1.0.0 ; mqtt freekode/TM1651@1.0.1 ; tm1651 - dudanov/MideaUART@1.1.9 ; midea tonia/HeatpumpIR@1.0.42 ; heatpumpir build_flags = ${common.build_flags} @@ -203,10 +203,11 @@ extra_scripts = extends = common:arduino board_build.filesystem_size = 0.5m -platform = https://github.com/maxgerhardt/platform-raspberrypi.git#v1.4.0-gcc14-arduinopico460 +platform = https://github.com/maxgerhardt/platform-raspberrypi.git#9c167c6b8aac4f4cfa6d55a0c4e5b848795150c0 platform_packages = - ; earlephilhower/framework-arduinopico@~1.20602.0 ; Cannot use the platformio package until old releases stop getting deleted - earlephilhower/framework-arduinopico@https://github.com/earlephilhower/arduino-pico/releases/download/5.6.1/rp2040-5.6.1.zip + ; The framework-arduinopico package is no longer published to the PlatformIO + ; registry, so install the framework straight from the GitHub release + earlephilhower/framework-arduinopico@https://github.com/earlephilhower/arduino-pico/releases/download/6.0.0/rp2040-6.0.0.zip framework = arduino lib_deps = diff --git a/requirements.txt b/requirements.txt index afdb921f7e..d56a8daec1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import esphome-glyphsets==0.2.0 pillow==12.3.0 -resvg-py==0.3.3 +resvg-py==0.3.4 freetype-py==2.5.1 jinja2==3.1.6 bleak==2.1.1 diff --git a/requirements_test.txt b/requirements_test.txt index 389bf6dbf0..b5753066ba 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,7 @@ pylint==4.0.6 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating ruff==0.16.1 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating -pre-commit +prek==0.4.12 # also change in .github/workflows/ci.yml when updating # Unit tests pytest==9.1.1 diff --git a/script/ci-custom.py b/script/ci-custom.py index 90748a13b9..2d2da20995 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -557,7 +557,7 @@ def lint_constants_usage(): # Maximum allowed CONF_ constants in esphome/const.py. # This file is frozen — new constants go in esphome/components/const/__init__.py. # Decrease this number when constants are moved out of const.py. -CONST_PY_MAX_CONF = 1015 +CONST_PY_MAX_CONF = 1017 @lint_content_check(include=["esphome/const.py"]) diff --git a/script/clang-tidy b/script/clang-tidy index 4f1bc6021c..ad6c99d637 100755 --- a/script/clang-tidy +++ b/script/clang-tidy @@ -5,11 +5,13 @@ import os from pathlib import Path import queue import re +import shlex import shutil import subprocess import sys import tempfile import threading +from typing import Any import click import colorama @@ -29,6 +31,36 @@ from helpers import ( ) +def gcc_multilib_directory(idedata: dict[str, Any]) -> str | None: + """The toolchain's active multilib subdirectory (e.g. "thumb"), if any. + + PlatformIO's idedata lists the generic toolchain include directories; GCC + resolves the active multilib subdirectory internally while searching them. + Toolchains without a default multilib (pico-quick-toolchain 5.0.0+) ship + the libstdc++ target config (bits/c++config.h) only inside the multilib + subdirectories, so clang needs the resolved directory spelled out. + """ + machine_flags = [f for f in idedata["cxx_flags"] if f.startswith("-m")] + cmd = [idedata["cxx_path"], *machine_flags, "-print-multi-directory"] + try: + multilib = subprocess.run( + cmd, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (OSError, subprocess.CalledProcessError) as err: + # Without the multilib dir, toolchains lacking a default multilib fail + # later with "bits/c++config.h not found"; point at the probe instead. + stderr = getattr(err, "stderr", "") or "" + print( + f"WARNING: multilib probe failed ({shlex.join(cmd)}): {err} {stderr}".strip(), + file=sys.stderr, + ) + return None + return None if multilib in ("", ".") else multilib + + def clang_options(idedata, environment): cmd = [] @@ -203,9 +235,12 @@ def clang_options(idedata, environment): # toolchain include directories, using -isystem to suppress their errors # idedata contains include directories for all toolchains of this platform, only use those from the one in use toolchain_dir = os.path.normpath(f"{idedata['cxx_path']}/../../") + multilib = gcc_multilib_directory(idedata) toolchain_includes = [] for directory in idedata["includes"]["toolchain"]: if directory.startswith(toolchain_dir) and "picolibc" not in directory: + if multilib and (multilib_dir := Path(directory) / multilib).is_dir(): + toolchain_includes.extend(["-isystem", str(multilib_dir)]) toolchain_includes.extend(["-isystem", directory]) # library include directories, using -isystem to suppress their errors diff --git a/script/clang_tidy_hash.py b/script/clang_tidy_hash.py index 57ca90711c..f4fd5a4dff 100644 --- a/script/clang_tidy_hash.py +++ b/script/clang_tidy_hash.py @@ -18,6 +18,7 @@ from pathlib import Path # Root-relative paths whose contents affect clang-tidy results. CLANG_TIDY_GLOBAL_FILES = ( ".clang-tidy", + "script/clang-tidy", "platformio.ini", "requirements_dev.txt", "esphome/idf_component.yml", diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 061485c76c..e4d002975c 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -23,7 +23,7 @@ what files have changed. It outputs JSON with the following structure: } The CI workflow uses this information to: -- Gate the unconditional jobs (ci-custom, pytest, pre-commit-ci-lite) via core_ci; +- Gate the unconditional jobs (ci-custom, pytest, lint-format) via core_ci; false when a pull_request only touches CI-irrelevant meta paths (other workflow files, .github/actions/build-image/*, .yamllint, .github/dependabot.yml, docker/**) so workflow-only PRs satisfy the required CI Status check without running the @@ -63,6 +63,7 @@ from helpers import ( CPP_FILE_EXTENSIONS, ESPHOME_TESTS_COMPONENTS_PATH, PYTHON_FILE_EXTENSIONS, + base_python_changed, changed_files, core_changed, filter_component_and_test_cpp_files, @@ -657,16 +658,20 @@ BENCHMARK_INFRASTRUCTURE_FILES = frozenset( def should_run_benchmarks(branch: str | None = None) -> bool: - """Determine if C++ benchmarks should run based on changed files. + """Determine if benchmarks (C++ and Python) should run based on changed files. Benchmarks run when any of the following conditions are met: - 1. Core C++ files changed (esphome/core/*) - 2. The host platform changed (esphome/components/host/*) — benchmarks + 1. Core files changed (esphome/core/*, C++ or Python) + 2. Top-level Python files changed (esphome/*.py and esphome/*.pyi) — + the Python benchmarks exercise config loading (config.py, + yaml_util.py, ...), so a slowdown there is invisible unless the + benchmarks job runs + 3. The host platform changed (esphome/components/host/*) — benchmarks are built and run on the host platform, so its implementations of ``millis()``/``micros()``/etc. affect every benchmark - 3. A directly changed component has benchmark files (no dependency expansion) - 4. Benchmark infrastructure changed (tests/benchmarks/*, script/cpp_benchmark.py, + 4. A directly changed component has benchmark files (no dependency expansion) + 5. Benchmark infrastructure changed (tests/benchmarks/*, script/cpp_benchmark.py, script/build_helpers.py, script/setup_codspeed_lib.py) Unlike unit tests, benchmarks do NOT expand to dependent components. @@ -683,6 +688,11 @@ def should_run_benchmarks(branch: str | None = None) -> bool: if core_changed(files): return True + # Top-level esphome/*.py modules are what the Python benchmarks in + # tests/benchmarks/python/ exercise + if base_python_changed(files): + return True + # Host platform supplies the runtime that benchmarks execute on if any(f.startswith("esphome/components/host/") for f in files): return True @@ -708,7 +718,7 @@ def should_run_benchmarks(branch: str | None = None) -> bool: # Files / path patterns whose changes alone don't warrant running the -# unconditional CI jobs (`ci-custom`, `pytest`, `pre-commit-ci-lite`). +# unconditional CI jobs (`ci-custom`, `pytest`, `lint-format`). # Single source of truth for what we treat as "CI-irrelevant" on # pull_request events; ci.yml used to encode this in its own # `pull_request.paths` filter, but that hid the required `CI Status` @@ -752,7 +762,7 @@ def _is_ci_irrelevant_path(path: str) -> bool: def should_run_core_ci(branch: str | None = None) -> bool: - """Determine if the unconditional CI jobs (ci-custom/pytest/pre-commit-ci-lite) should run. + """Determine if the unconditional CI jobs (ci-custom/pytest/lint-format) should run. Returns False only when every changed file is in the CI-irrelevant set above (see ``_is_ci_irrelevant_path``). Empty diffs return True so we @@ -1177,7 +1187,7 @@ def main() -> None: # Determine what should run # core_ci gates the unconditional jobs in ci.yml (ci-custom, pytest, - # pre-commit-ci-lite). Non-pull_request events (push to dev/beta/release + # lint-format). Non-pull_request events (push to dev/beta/release # and merge_group) always run them so behavior like venv-cache saves on # push to dev is preserved. event_name = os.environ.get("GITHUB_EVENT_NAME", "") diff --git a/script/helpers.py b/script/helpers.py index 6ba093b413..7cc001d92f 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -1380,6 +1380,27 @@ def core_changed(files: list[str]) -> bool: ) +def base_python_changed(files: list[str]) -> bool: + """Check if any Python file directly in esphome/ has changed. + + Matches top-level modules and stubs (.py and .pyi) like esphome/config.py + and esphome/yaml_util.py but not files in subdirectories such as + esphome/components/ or esphome/dashboard/. + + Args: + files: List of file paths to check + + Returns: + True if any top-level esphome Python file has changed + """ + return any( + f.startswith("esphome/") + and f.endswith(PYTHON_FILE_EXTENSIONS) + and "/" not in f.removeprefix("esphome/") + for f in files + ) + + def get_cpp_changed_components(files: list[str]) -> list[str]: """Get components that have changed C++ files or tests. diff --git a/script/setup b/script/setup index 709eaee0f3..5dfc0efe5d 100755 --- a/script/setup +++ b/script/setup @@ -25,7 +25,11 @@ fi uv pip install setuptools wheel uv pip install -e ".[dev,test]" --config-settings editable_mode=compat -pre-commit install +# --overwrite replaces any hook already in place. Without it, prek finds a +# previously installed pre-commit hook, moves it aside to +# .git/hooks/pre-commit.legacy and keeps calling it, so every commit would +# run both tools. +prek install --overwrite mkdir -p .temp diff --git a/script/setup.bat b/script/setup.bat index 003ea31b36..809d05ae93 100644 --- a/script/setup.bat +++ b/script/setup.bat @@ -17,7 +17,11 @@ pip3 install -r requirements.txt -r requirements_test.txt -r requirements_dev.tx pip3 install setuptools wheel pip3 install -e ".[dev,test]" --config-settings editable_mode=compat -pre-commit install +rem --overwrite replaces any hook already in place. Without it, prek finds a +rem previously installed pre-commit hook, moves it aside to +rem .git/hooks/pre-commit.legacy and keeps calling it, so every commit would +rem run both tools. +prek install --overwrite echo . echo . diff --git a/tests/unit_tests/components/api/__init__.py b/tests/component_tests/bk72xx_ble_tracker/__init__.py similarity index 100% rename from tests/unit_tests/components/api/__init__.py rename to tests/component_tests/bk72xx_ble_tracker/__init__.py diff --git a/tests/component_tests/bk72xx_ble_tracker/config/test_automations.yaml b/tests/component_tests/bk72xx_ble_tracker/config/test_automations.yaml new file mode 100644 index 0000000000..994855b782 --- /dev/null +++ b/tests/component_tests/bk72xx_ble_tracker/config/test_automations.yaml @@ -0,0 +1,44 @@ +esphome: + name: bk-trigger-codegen + on_boot: + then: + - bk72xx_ble_tracker.start_scan: + continuous: true + # Bare form: restores the configured scan_parameters mode — no + # set_continuous emitted (asserted in the codegen test). + - bk72xx_ble_tracker.start_scan: + - bk72xx_ble_tracker.stop_scan + +bk72xx: + board: cb2s + +bk72xx_ble_tracker: + scan_parameters: + continuous: false + on_ble_advertise: + - mac_address: + - AC:37:43:77:5F:4C + - 11:22:33:44:55:66 + then: + - lambda: 'char addr[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; ESP_LOGD("t", "%s", x.address_str_to(addr));' + on_ble_service_data_advertise: + - service_uuid: ABCDABCD-ABCD-ABCD-ABCD-ABCDABCDABCD + mac_address: AC:37:43:77:5F:4C + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + - service_uuid: ABCDABCD + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + on_ble_manufacturer_data_advertise: + - manufacturer_id: ABCD + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + - manufacturer_id: ABCDABCD + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + - manufacturer_id: ABCDABCD-ABCD-ABCD-ABCD-ABCDABCDABCD + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + on_scan_end: + - then: + - lambda: 'ESP_LOGD("t", "end");' diff --git a/tests/component_tests/bk72xx_ble_tracker/test_automations_codegen.py b/tests/component_tests/bk72xx_ble_tracker/test_automations_codegen.py new file mode 100644 index 0000000000..3a03f98adf --- /dev/null +++ b/tests/component_tests/bk72xx_ble_tracker/test_automations_codegen.py @@ -0,0 +1,60 @@ +"""Codegen tests for the tracker automations. + +The shared trigger classes (ble_device_base/automation.h) are compiled by every +esp32 BLE compile test via AUTO_LOAD, but the BK-specific side — automation.h's +action templates and restart_scan_duration() — compiles on no CI board (the +bk72xx base board generic-bk7252 is BLE 4.2 and cannot build the tracker), and +validate fixtures never run to_code. The generated main is therefore the only +automated check on the setter spellings and the listener accounting.""" + +from collections.abc import Callable +from pathlib import Path +import re + +from esphome.components import ble_device_base +from tests.component_tests.helpers import get_define_value + + +def test_trigger_codegen( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("test_automations.yaml")) + + # on_ble_advertise: multi-mac filter (two addresses in one initializer list) + assert "set_addresses({0xAC3743775F4CULL, 0x112233445566ULL})" in main_cpp + # 128-bit service uuid goes out reversed (BLE wire order); single-mac filter + assert ( + "set_service_uuid128((uint8_t*)(const uint8_t[16]){0xCD,0xAB,0xCD,0xAB," + "0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB})" in main_cpp + ) + assert "set_address(0xAC3743775F4CULL)" in main_cpp + # 32-bit middle branch of the width dispatch + assert "set_service_uuid32(0xABCDABCDULL)" in main_cpp + # All three manufacturer widths: getattr() builds these names as strings, + # so a misspelling only ever fails here. + assert "set_manufacturer_uuid16(0xABCDULL)" in main_cpp + assert "set_manufacturer_uuid32(0xABCDABCDULL)" in main_cpp + assert ( + "set_manufacturer_uuid128((uint8_t*)(const uint8_t[16]){0xCD,0xAB,0xCD,0xAB," + "0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB})" in main_cpp + ) + # scan-control actions: templatable continuous lambda + parented actions. + # Exactly one set_continuous: the bare start_scan emits none, pinning the + # restore-configured-mode divergence from esp32 against a future default=. + assert main_cpp.count("->set_continuous(") == 1 + assert "startscanaction_id->set_continuous(" in main_cpp + assert "stopscanaction_id->set_parent(" in main_cpp + # scan_parameters continuous: false reaches the YAML-mode setter, not the + # runtime override. + assert "->set_configured_continuous(false)" in main_cpp + # Constructor call, not just the declaration: the parent argument is what + # registers the trigger as a listener. + assert re.search( + r"new\(\w+\) ble_device_base::BLEEndOfScanTrigger\(\w+\)", main_cpp + ) + + # Seven triggers register as listeners; an undercount silently drops the + # last trigger at runtime (StaticVector::push_back past capacity), so the + # define is the assertion that matters most. + assert get_define_value(ble_device_base.LISTENER_COUNT_DEFINE) == "7" diff --git a/tests/component_tests/ble_client/__init__.py b/tests/component_tests/ble_client/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/ble_client/test_validation.py b/tests/component_tests/ble_client/test_validation.py new file mode 100644 index 0000000000..1865812b74 --- /dev/null +++ b/tests/component_tests/ble_client/test_validation.py @@ -0,0 +1,86 @@ +"""Tests for ble_client config validation.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components.ble_client import ( + CONF_DESCRIPTOR_UUID, + CONF_ON_NOTIFY, + notify_from_on_notify, + validate_descriptor_not_notify, +) +from esphome.components.ble_client.sensor import CONFIG_SCHEMA as SENSOR_SCHEMA +from esphome.components.ble_client.text_sensor import ( + CONFIG_SCHEMA as TEXT_SENSOR_SCHEMA, +) +from esphome.const import ( + CONF_CHARACTERISTIC_UUID, + CONF_NAME, + CONF_NOTIFY, + CONF_SERVICE_UUID, + CONF_TYPE, +) +from esphome.types import ConfigType + +DESCRIPTOR_CONFIG: ConfigType = { + CONF_NAME: "test", + CONF_SERVICE_UUID: "6E400001-B5A3-F393-E0A9-E50E24DCCA9E", + CONF_CHARACTERISTIC_UUID: "6E400003-B5A3-F393-E0A9-E50E24DCCA9E", + CONF_DESCRIPTOR_UUID: "2902", +} + + +def test_notify_with_descriptor_uuid_rejected() -> None: + config: ConfigType = {CONF_NOTIFY: True, CONF_DESCRIPTOR_UUID: "2902"} + with pytest.raises(cv.Invalid, match="cannot send notifications"): + validate_descriptor_not_notify(config) + + +def test_on_notify_with_descriptor_uuid_rejected() -> None: + config: ConfigType = { + CONF_NOTIFY: False, + CONF_ON_NOTIFY: [{}], + CONF_DESCRIPTOR_UUID: "2902", + } + with pytest.raises(cv.Invalid, match="cannot send notifications"): + validate_descriptor_not_notify(config) + + +def test_descriptor_uuid_without_notify_allowed() -> None: + config: ConfigType = {CONF_NOTIFY: False, CONF_DESCRIPTOR_UUID: "2902"} + assert validate_descriptor_not_notify(config) is config + + +def test_notify_without_descriptor_uuid_allowed() -> None: + config: ConfigType = {CONF_NOTIFY: True} + assert validate_descriptor_not_notify(config) is config + + +def test_sensor_schema_rejects_notify_with_descriptor() -> None: + config = {**DESCRIPTOR_CONFIG, CONF_TYPE: "characteristic", CONF_NOTIFY: True} + with pytest.raises(cv.Invalid, match="cannot send notifications"): + SENSOR_SCHEMA(config) + + +def test_text_sensor_schema_rejects_notify_with_descriptor() -> None: + config = {**DESCRIPTOR_CONFIG, CONF_NOTIFY: True} + with pytest.raises(cv.Invalid, match="cannot send notifications"): + TEXT_SENSOR_SCHEMA(config) + + +def test_sensor_schema_allows_descriptor_polling() -> None: + assert SENSOR_SCHEMA({**DESCRIPTOR_CONFIG, CONF_TYPE: "characteristic"}) + + +def test_text_sensor_schema_allows_descriptor_polling() -> None: + assert TEXT_SENSOR_SCHEMA(dict(DESCRIPTOR_CONFIG)) + + +def test_on_notify_implies_notify() -> None: + config: ConfigType = {CONF_NOTIFY: False, CONF_ON_NOTIFY: [{}]} + assert notify_from_on_notify(config)[CONF_NOTIFY] is True + + +def test_notify_unchanged_without_on_notify() -> None: + config: ConfigType = {CONF_NOTIFY: False} + assert notify_from_on_notify(config)[CONF_NOTIFY] is False diff --git a/tests/component_tests/ble_device_base/config/bk72xx_controller_only.yaml b/tests/component_tests/ble_device_base/config/bk72xx_controller_only.yaml new file mode 100644 index 0000000000..4d4dab0198 --- /dev/null +++ b/tests/component_tests/ble_device_base/config/bk72xx_controller_only.yaml @@ -0,0 +1,7 @@ +esphome: + name: slotcount-controller + +bk72xx: + board: generic-bk7252 + +bk72xx_ble: diff --git a/tests/component_tests/ble_device_base/config/bk72xx_tracker.yaml b/tests/component_tests/ble_device_base/config/bk72xx_tracker.yaml new file mode 100644 index 0000000000..79e9644006 --- /dev/null +++ b/tests/component_tests/ble_device_base/config/bk72xx_tracker.yaml @@ -0,0 +1,7 @@ +esphome: + name: slotcount-tracker + +bk72xx: + board: generic-bk7252 + +bk72xx_ble_tracker: diff --git a/tests/component_tests/ble_device_base/config/esp32_bluetooth_proxy.yaml b/tests/component_tests/ble_device_base/config/esp32_bluetooth_proxy.yaml new file mode 100644 index 0000000000..7500f2133b --- /dev/null +++ b/tests/component_tests/ble_device_base/config/esp32_bluetooth_proxy.yaml @@ -0,0 +1,16 @@ +esphome: + name: slotcount-esp32-proxy + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: MySSID + password: password1 + +api: + +bluetooth_proxy: + active: true diff --git a/tests/component_tests/ble_device_base/config/esp32_tracker_only.yaml b/tests/component_tests/ble_device_base/config/esp32_tracker_only.yaml new file mode 100644 index 0000000000..46a76cfec8 --- /dev/null +++ b/tests/component_tests/ble_device_base/config/esp32_tracker_only.yaml @@ -0,0 +1,9 @@ +esphome: + name: slotcount-esp32-tracker + +esp32: + board: esp32dev + framework: + type: esp-idf + +esp32_ble_tracker: diff --git a/tests/component_tests/ble_device_base/config/rp2_controller_only.yaml b/tests/component_tests/ble_device_base/config/rp2_controller_only.yaml new file mode 100644 index 0000000000..e64b328051 --- /dev/null +++ b/tests/component_tests/ble_device_base/config/rp2_controller_only.yaml @@ -0,0 +1,7 @@ +esphome: + name: slotcount-rp2-controller + +rp2: + board: rpipicow + +rp2040_ble: diff --git a/tests/component_tests/ble_device_base/config/rp2_tracker.yaml b/tests/component_tests/ble_device_base/config/rp2_tracker.yaml new file mode 100644 index 0000000000..31686dd236 --- /dev/null +++ b/tests/component_tests/ble_device_base/config/rp2_tracker.yaml @@ -0,0 +1,7 @@ +esphome: + name: slotcount-rp2-tracker + +rp2: + board: rpipicow + +rp2_ble_tracker: diff --git a/tests/component_tests/ble_device_base/test_hub_binding.py b/tests/component_tests/ble_device_base/test_hub_binding.py new file mode 100644 index 0000000000..3dc71daf58 --- /dev/null +++ b/tests/component_tests/ble_device_base/test_hub_binding.py @@ -0,0 +1,204 @@ +"""Tests for the BLE hub provider registry and the missing-hub diagnostics.""" + +from collections.abc import Generator +from importlib import import_module +from pathlib import Path + +import pytest + +import esphome.codegen as cg +from esphome.components import ble_device_base +import esphome.config_validation as cv +from esphome.const import KEY_TARGET_PLATFORM, Platform +from esphome.core import CORE, ID, KEY_CORE +from esphome.cpp_generator import MockObjClass + +COMPONENTS_DIR = Path(ble_device_base.__file__).parent.parent + + +@pytest.fixture +def hub_registry() -> Generator[set[str]]: + """Save/restore _HUB_PROVIDERS — a module global with no reset hook. + + CORE state needs no bookkeeping here: conftest's autouse reset_core + fixture reassigns it after every test. + """ + saved = set(ble_device_base._HUB_PROVIDERS) + yield ble_device_base._HUB_PROVIDERS + ble_device_base._HUB_PROVIDERS.clear() + ble_device_base._HUB_PROVIDERS.update(saved) + + +def _generated_id() -> ID: + """An ID as cv.GenerateID leaves it before the ID-assignment pass.""" + return ID(None, is_declaration=False, type="ble_device_base::BLEHub") + + +def _set_platform(platform: str | None) -> None: + core_data = CORE.data.setdefault(KEY_CORE, {}) + if platform is None: + core_data.pop(KEY_TARGET_PLATFORM, None) + else: + core_data[KEY_TARGET_PLATFORM] = platform + + +# The missing-hub diagnostics: one test per path so a regression in one +# scenario cannot mask the others. The hub binding must fail with a +# tracker-naming message, not use_id's C++-class error, regardless of +# config-step ordering internals. + + +def test_empty_registry_names_every_in_tree_tracker(hub_registry: set[str]) -> None: + # The common failure: a fresh CLI process where the tracker was simply + # forgotten, so no tracker module was ever imported and the registry is + # empty. The error must still name the in-tree trackers. + hub_registry.clear() + CORE.loaded_integrations.clear() + _set_platform(None) + with pytest.raises( + cv.Invalid, + match="add one of: bk72xx_ble_tracker, esp32_ble_tracker, ln882h_ble_tracker, rp2_ble_tracker", + ): + ble_device_base._require_hub(_generated_id()) + + +def test_platform_filters_the_suggested_trackers(hub_registry: set[str]) -> None: + hub_registry.clear() + CORE.loaded_integrations.clear() + _set_platform("esp32") + with pytest.raises(cv.Invalid, match="add one of: esp32_ble_tracker$"): + ble_device_base._require_hub(_generated_id()) + + +def test_ble_less_platform_is_not_misdirected(hub_registry: set[str]) -> None: + # A known platform with no in-tree hub must not be pointed at other + # platforms' trackers; out-of-tree BLE hubs are not supported. + hub_registry.clear() + CORE.loaded_integrations.clear() + _set_platform("esp8266") + with pytest.raises( + cv.Invalid, + match="No BLE tracker exists for esp8266; BLE components are not supported", + ): + ble_device_base._require_hub(_generated_id()) + + +def test_explicit_id_bypasses_the_registry(hub_registry: set[str]) -> None: + # Explicit ble_hub_id: is the multi-hub disambiguation case; the ID pass + # owns that diagnosis and its error names the missing id. + hub_registry.clear() + CORE.loaded_integrations.clear() + explicit = ID("my_hub", is_declaration=False, type="ble_device_base::BLEHub") + assert ble_device_base._require_hub(explicit) is explicit + + +def test_registered_and_loaded_provider_passes(hub_registry: set[str]) -> None: + hub_registry.add("esp32_ble_tracker") + CORE.loaded_integrations.add("esp32_ble_tracker") + generated = _generated_id() + assert ble_device_base._require_hub(generated) is generated + + +def _module_name(path: Path) -> str: + """Dotted module name for a file under esphome/components.""" + rel = path.relative_to(COMPONENTS_DIR.parent) + parts = rel.with_suffix("").parts + if parts[-1] == "__init__": + parts = parts[:-1] + return "esphome." + ".".join(parts) + + +def _hub_component_modules() -> list[str]: + """Components whose codegen class inherits ble_device_base.BLEHub. + + The source-text pass only selects import candidates (importing all ~900 + component packages is too slow); membership is decided by the class + hierarchy via MockObjClass.inherits_from on every module whose source + matched — nested declaring modules included — so a comment mentioning + BLEHub in a consumer cannot produce a false positive. + """ + hub_modules = [] + for pkg in sorted(COMPONENTS_DIR.iterdir()): + if pkg.name == "ble_device_base" or not (pkg / "__init__.py").is_file(): + continue + matched = [ + path + for path in pkg.rglob("*.py") + if "BLEHub" in path.read_text(encoding="utf-8") + ] + if not matched: + continue + for path in matched: + mod = import_module(_module_name(path)) + if any( + isinstance(attr, MockObjClass) + and attr is not ble_device_base.BLEHub + and attr.inherits_from(ble_device_base.BLEHub) + for attr in vars(mod).values() + ): + hub_modules.append(pkg.name) + break + return hub_modules + + +def test_every_in_tree_hub_registers_as_provider() -> None: + """A BLEHub subclass that forgets register_hub_provider() makes _require_hub + reject valid configs for that platform — fail CI instead of the user.""" + hub_modules = _hub_component_modules() + assert hub_modules, "hub discovery found no BLEHub subclasses — scan stale?" + for name in hub_modules: + assert name in ble_device_base._HUB_PROVIDERS, ( + f"{name} subclasses ble_device_base.BLEHub but never calls " + "register_hub_provider(); a valid config using it would be rejected" + ) + # The per-platform error table must know every in-tree hub, keyed by real + # platform names — a typo'd key would silently route that platform into + # the no-in-tree-tracker branch. + assert set(ble_device_base._IN_TREE_HUB_PROVIDERS.values()) == set(hub_modules) + platforms = {platform.value for platform in Platform} + assert set(ble_device_base._IN_TREE_HUB_PROVIDERS) <= platforms + + +def test_ble_device_schema_declares_the_binding_key(hub_registry: set[str]) -> None: + """Extending BLE_DEVICE_SCHEMA keeps ble_hub_id a declared key on a strict + schema, for both the generated and the explicit form, and the missing-hub + rejection surfaces through the schema itself.""" + schema = cv.Schema({}).extend(ble_device_base.BLE_DEVICE_SCHEMA) + hub_registry.clear() + CORE.loaded_integrations.discard("esp32_ble_tracker") + with pytest.raises(cv.Invalid, match="No BLE tracker configured"): + schema({}) + hub_registry.add("esp32_ble_tracker") + CORE.loaded_integrations.add("esp32_ble_tracker") + generated = schema({})[ble_device_base.CONF_BLE_HUB_ID] + assert isinstance(generated, ID) and generated.id is None + explicit = schema({"ble_hub_id": "my_hub"})[ble_device_base.CONF_BLE_HUB_ID] + assert explicit.id == "my_hub" + + +def test_rename_legacy_hub_id_migrates_the_old_key() -> None: + validator = ble_device_base.rename_legacy_hub_id("my_sensor") + migrated = validator({"esp32_ble_id": "tracker1"}) + assert migrated == {ble_device_base.CONF_BLE_HUB_ID: "tracker1"} + untouched = validator({"name": "x"}) + assert untouched == {"name": "x"} + + +def test_add_service_uuid_dispatches_by_width(monkeypatch: pytest.MonkeyPatch) -> None: + emitted: list[str] = [] + monkeypatch.setattr( + "esphome.components.ble_device_base.cg.add", lambda e: emitted.append(str(e)) + ) + var = cg.MockObj("trig") + ble_device_base.add_service_uuid(var, "11AA") + ble_device_base.add_service_uuid(var, "11223344") + ble_device_base.add_service_uuid(var, "11223344-5566-7788-99aa-bbccddeeff00") + assert "set_service_uuid16" in emitted[0] + assert "set_service_uuid32" in emitted[1] + assert "set_service_uuid128" in emitted[2] + # BLE wire order: the 128-bit array must be byte-reversed — as_hex_array + # in its place would still emit the right setter name and silently never + # match on-air. + assert "0x00,0xff,0xee,0xdd" in emitted[2] + with pytest.raises(ValueError, match="Unsupported UUID format"): + ble_device_base.add_service_uuid(var, "123") diff --git a/tests/component_tests/ble_device_base/test_scan_parameter_validation.py b/tests/component_tests/ble_device_base/test_scan_parameter_validation.py index 363c129f4b..bd41a9476a 100644 --- a/tests/component_tests/ble_device_base/test_scan_parameter_validation.py +++ b/tests/component_tests/ble_device_base/test_scan_parameter_validation.py @@ -10,6 +10,10 @@ from esphome.components.bk72xx_ble_tracker import ( ) from esphome.components.ble_device_base import to_ble_units from esphome.components.esp32_ble_tracker import SCAN_PARAMETERS_SCHEMA as ESP32_SCHEMA +from esphome.components.ln882h_ble_tracker import ( + SCAN_PARAMETERS_SCHEMA as LN882H_SCHEMA, +) +from esphome.components.rp2_ble_tracker import SCAN_PARAMETERS_SCHEMA as RP2_SCHEMA def _validate(**kwargs: str) -> dict: @@ -59,6 +63,24 @@ def test_esp32_defaults_are_valid() -> None: assert config["active"] is True +def test_rp2_defaults_are_valid() -> None: + """rp2 pins 100 ms interval / 30 ms window — a 30 % duty cycle leaving the + shared CYW43 radio mostly free for WiFi — and exposes active (default on).""" + config = RP2_SCHEMA({}) + assert to_ble_units(config["interval"]) == 160 + assert to_ble_units(config["window"]) == 48 + assert config["active"] is True + + +def test_ln882h_defaults_are_valid() -> None: + """ln882h pins the LN SDK reference rate — 100 ms interval / 50 ms window + (50 % duty) — and exposes active (default on).""" + config = LN882H_SCHEMA({}) + assert to_ble_units(config["interval"]) == 160 + assert to_ble_units(config["window"]) == 80 + assert config["active"] is True + + def test_esp32_active_can_disable() -> None: config = ESP32_SCHEMA({"active": False}) assert config["active"] is False @@ -102,6 +124,11 @@ def test_window_equal_to_interval_accepted() -> None: assert to_ble_units(config["interval"]) == to_ble_units(config["window"]) +def test_duration_equal_to_three_intervals_accepted() -> None: + """The three-interval floor is inclusive, mirroring the ceilings above.""" + _validate(duration="3s", interval="1s", window="500ms") + + # --- rejected configurations --- diff --git a/tests/component_tests/ble_device_base/test_slot_counter.py b/tests/component_tests/ble_device_base/test_slot_counter.py new file mode 100644 index 0000000000..0fa5577a0b --- /dev/null +++ b/tests/component_tests/ble_device_base/test_slot_counter.py @@ -0,0 +1,135 @@ +"""Tests for the shared slot_counter codegen factory. + +The factory is exercised end to end through the real controllers: a tracker +config must emit the platform's scan listener count define, and a +controller-only config must emit nothing so the guarded StaticVector storage +compiles out. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.core import CORE + +from ..helpers import get_define_value + + +@pytest.mark.parametrize( + ("config", "define"), + [ + ("bk72xx_tracker.yaml", "BK72XX_BLE_SCAN_LISTENER_COUNT"), + ("rp2_tracker.yaml", "RP2040_BLE_SCAN_LISTENER_COUNT"), + ], +) +def test_tracker_requests_one_slot( + config: str, + define: str, + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """The tracker's to_code requests a slot; the FINAL job emits the count. + + The neutral listener count must stay absent from the same build: no BLE + consumer registered through register_ble_device(). + """ + generate_main(component_config_path(config)) + assert get_define_value(define) == "1" + assert get_define_value("ESPHOME_BLE_DEVICE_BASE_LISTENER_COUNT") is None + + +@pytest.mark.parametrize( + ("config", "define"), + [ + ("bk72xx_controller_only.yaml", "BK72XX_BLE_SCAN_LISTENER_COUNT"), + ("rp2_controller_only.yaml", "RP2040_BLE_SCAN_LISTENER_COUNT"), + ], +) +def test_controller_only_emits_no_count( + config: str, + define: str, + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """No consumer, no define — the guarded listener storage compiles out.""" + generate_main(component_config_path(config)) + assert get_define_value(define) is None + + +def test_neutral_listener_count_emitted_when_requested() -> None: + """Registering through register_ble_device() emits the neutral count. + + No in-tree sensor registers through ble_device_base.register_ble_device() + yet (consumer migration is a follow-up), so the coroutine is driven with a + mock hub instead of a config; every tracker's #ifdef-guarded listener + storage keys on this define, and a broken emit path would compile the + storage out silently. + """ + import esphome.codegen as cg + from esphome.components import ble_device_base + from esphome.core import ID + + hub_id = ID("hub", type=ble_device_base.BLEHub) + CORE.register_variable(hub_id, cg.MockObj("hub")) + CORE.add_job( + ble_device_base.register_ble_device, + cg.MockObj("listener"), + {ble_device_base.CONF_BLE_HUB_ID: hub_id}, + ) + CORE.flush_tasks() + assert get_define_value(ble_device_base.LISTENER_COUNT_DEFINE) == "1" + + +def test_esp32_tracker_handler_counts( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """A bare tracker registers its four esp32_ble handlers and nothing else.""" + generate_main(component_config_path("esp32_tracker_only.yaml")) + assert get_define_value("ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT") == "1" + assert get_define_value("ESPHOME_ESP32_BLE_GAP_SCAN_EVENT_HANDLER_COUNT") == "1" + assert get_define_value("ESPHOME_ESP32_BLE_GATTC_EVENT_HANDLER_COUNT") == "1" + assert get_define_value("ESPHOME_ESP32_BLE_BLE_STATUS_EVENT_HANDLER_COUNT") == "1" + assert get_define_value("ESPHOME_ESP32_BLE_GATTS_EVENT_HANDLER_COUNT") is None + # No consumer subscribed to scanner state, so the storage compiles out. + assert ( + get_define_value("ESPHOME_ESP32_BLE_TRACKER_SCANNER_STATE_LISTENER_COUNT") + is None + ) + assert get_define_value("ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT") is None + assert get_define_value("ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT") is None + + +def test_esp32_bluetooth_proxy_requests_scanner_state_slot( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """The proxy requests one scanner state slot, one raw listener slot and a + client slot per connection (three by default with active: true).""" + generate_main(component_config_path("esp32_bluetooth_proxy.yaml")) + assert ( + get_define_value("ESPHOME_ESP32_BLE_TRACKER_SCANNER_STATE_LISTENER_COUNT") + == "1" + ) + assert get_define_value("ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT") == "1" + assert get_define_value("ESPHOME_ESP32_BLE_TRACKER_CLIENT_COUNT") == "3" + + +def test_counts_reset_between_compiles( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """A second compile in the same process starts from zero. + + The module level counters this change removes leaked across compiles in a + long lived host process (dashboard, device-builder), growing the handler + counts by one per compile and oversizing the StaticCallbackManager storage. + """ + generate_main(component_config_path("esp32_tracker_only.yaml")) + assert get_define_value("ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT") == "1" + CORE.reset() + generate_main(component_config_path("esp32_tracker_only.yaml")) + assert get_define_value("ESPHOME_ESP32_BLE_GAP_EVENT_HANDLER_COUNT") == "1" diff --git a/tests/component_tests/bluetooth_proxy/__init__.py b/tests/component_tests/bluetooth_proxy/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/bluetooth_proxy/test_idf_max_connections_mirror.py b/tests/component_tests/bluetooth_proxy/test_idf_max_connections_mirror.py new file mode 100644 index 0000000000..3042c91f84 --- /dev/null +++ b/tests/component_tests/bluetooth_proxy/test_idf_max_connections_mirror.py @@ -0,0 +1,19 @@ +"""bluetooth_proxy mirrors esp32_ble.IDF_MAX_CONNECTIONS; pin them together. + +The mirror exists so the statically walkable CONFIG_SCHEMA can express the +connection_slots range without importing the esp32 BLE stack (that import +registers esp32-only automations on every platform). The runtime check in +_esp32_config_schema() only fires while validating an esp32 config, so this +test is what actually catches drift when the upstream constant changes. +""" + +from esphome.components import esp32_ble +from esphome.components.bluetooth_proxy import _IDF_MAX_CONNECTIONS + + +def test_mirror_matches_esp32_ble() -> None: + assert _IDF_MAX_CONNECTIONS == esp32_ble.IDF_MAX_CONNECTIONS, ( + "bluetooth_proxy._IDF_MAX_CONNECTIONS is out of sync with " + "esp32_ble.IDF_MAX_CONNECTIONS; update the mirror in " + "esphome/components/bluetooth_proxy/__init__.py" + ) diff --git a/tests/component_tests/bluetooth_proxy/test_outer_schema_mirror.py b/tests/component_tests/bluetooth_proxy/test_outer_schema_mirror.py new file mode 100644 index 0000000000..17a05a67b9 --- /dev/null +++ b/tests/component_tests/bluetooth_proxy/test_outer_schema_mirror.py @@ -0,0 +1,65 @@ +"""The outer CONFIG_SCHEMA re-declares the esp32 scalar keys so tooling can walk +them without importing the esp32 BLE stack; pin the two declarations together. + +The outer schema carries no defaults (the per-platform schema applies them), so +drift cannot surface in validation output — a key renamed or removed in +_esp32_config_schema() but not here would silently vanish from the dashboard's +field extractor. This test is what catches that; validator bounds are pinned +separately only for connection_slots (test_idf_max_connections_mirror). +""" + +import voluptuous as vol + +from esphome import config_validation as cv +from esphome.components.bluetooth_proxy import CONFIG_SCHEMA, _esp32_config_schema + +# esp32-schema keys with no place in the outer schema: COMPONENT_SCHEMA +# plumbing (derived, so a future core key does not fail this component's test), +# generated IDs (not user-walkable options), and connections (must validate +# exactly once — see the comment above CONFIG_SCHEMA). +_NOT_MIRRORED = {str(key.schema) for key in cv.COMPONENT_SCHEMA.schema} | { + "connections" +} + + +def _schema_of(validator: cv.All) -> vol.Schema: + """The vol.Schema stage of a cv.All chain, found by type rather than by + position so reordering the chain cannot silently break these tests.""" + schemas = [v for v in validator.validators if isinstance(v, vol.Schema)] + assert len(schemas) == 1, f"expected exactly one vol.Schema stage, got {schemas}" + return schemas[0] + + +def _keys(schema: vol.Schema) -> dict[str, object]: + return {str(key.schema): key for key in schema.schema} + + +def test_outer_scalar_keys_exist_in_esp32_schema() -> None: + outer = _keys(_schema_of(CONFIG_SCHEMA)) + esp32 = _keys(_schema_of(_esp32_config_schema())) + missing = set(outer) - set(esp32) + assert not missing, ( + f"outer CONFIG_SCHEMA declares {sorted(missing)} which the esp32 schema " + "does not; update one of them in " + "esphome/components/bluetooth_proxy/__init__.py" + ) + + +def test_esp32_scalars_all_walkable() -> None: + """Every non-generated esp32 scalar option must appear in the outer schema + (connections is deliberately excluded — it must validate exactly once).""" + outer = _keys(_schema_of(CONFIG_SCHEMA)) + esp32 = _keys(_schema_of(_esp32_config_schema())) + scalar = { + name + for name, key in esp32.items() + if isinstance(key, vol.Optional) + and not isinstance(key, cv.GenerateID) + and name not in _NOT_MIRRORED + } + missing = scalar - set(outer) + assert not missing, ( + f"esp32 scalar options {sorted(missing)} are missing from the outer " + "CONFIG_SCHEMA and invisible to schema tooling; update " + "esphome/components/bluetooth_proxy/__init__.py" + ) diff --git a/tests/component_tests/bluetooth_proxy/test_platform_gates.py b/tests/component_tests/bluetooth_proxy/test_platform_gates.py new file mode 100644 index 0000000000..4d7997fbce --- /dev/null +++ b/tests/component_tests/bluetooth_proxy/test_platform_gates.py @@ -0,0 +1,48 @@ +"""The three platform-gate branches: BLE-less platforms are rejected with the +real reason, hub platforms reject GATT-only options by name, and the +advertisement-only arm applies its own defaults.""" + +import pytest + +from esphome import config_validation as cv +from esphome.components import bluetooth_proxy +from esphome.const import CONF_ACTIVE, KEY_TARGET_PLATFORM +from esphome.core import CORE, KEY_CORE + + +def _set_platform(platform: str) -> None: + CORE.data.setdefault(KEY_CORE, {})[KEY_TARGET_PLATFORM] = platform + + +def test_ble_less_platform_gets_the_real_reason() -> None: + _set_platform("esp8266") + with pytest.raises(cv.Invalid, match="not supported on esp8266"): + bluetooth_proxy.CONFIG_SCHEMA({}) + + +def test_ble_less_platform_connection_keys_fall_through() -> None: + # The key-level rejection must not fire here — it would imply an + # advertisement-only proxy exists on this platform. + _set_platform("esp8266") + with pytest.raises(cv.Invalid, match="not supported on esp8266"): + bluetooth_proxy.CONFIG_SCHEMA({"connection_slots": 2}) + + +def test_hub_platform_rejects_active() -> None: + _set_platform("ln882x") + with pytest.raises(cv.Invalid, match="Active connections are not supported"): + bluetooth_proxy.CONFIG_SCHEMA({"active": True}) + + +def test_hub_platform_rejects_connection_keys_by_name() -> None: + _set_platform("ln882x") + with pytest.raises(cv.Invalid, match="'connection_slots' requires active"): + bluetooth_proxy.CONFIG_SCHEMA({"connection_slots": 2}) + with pytest.raises(cv.Invalid, match="'cache_services' requires active"): + bluetooth_proxy.CONFIG_SCHEMA({"cache_services": True}) + + +def test_hub_platform_accepts_the_advertisement_only_shape() -> None: + _set_platform("ln882x") + validated = bluetooth_proxy.CONFIG_SCHEMA({}) + assert validated[CONF_ACTIVE] is False diff --git a/tests/component_tests/esp32/config/signed_ota_ecdsa256_c6.yaml b/tests/component_tests/esp32/config/signed_ota_ecdsa256_c6.yaml new file mode 100644 index 0000000000..0d504e36a2 --- /dev/null +++ b/tests/component_tests/esp32/config/signed_ota_ecdsa256_c6.yaml @@ -0,0 +1,10 @@ +esphome: + name: test + +esp32: + variant: esp32c6 + framework: + type: esp-idf + advanced: + signed_ota_verification: + signing_scheme: ecdsa256 diff --git a/tests/component_tests/esp32/config/signed_ota_ecdsa_v1.yaml b/tests/component_tests/esp32/config/signed_ota_ecdsa_v1.yaml new file mode 100644 index 0000000000..8d0dad947a --- /dev/null +++ b/tests/component_tests/esp32/config/signed_ota_ecdsa_v1.yaml @@ -0,0 +1,11 @@ +esphome: + name: test + +esp32: + variant: esp32 + framework: + type: esp-idf + advanced: + signed_ota_verification: + signing_scheme: ecdsa_v1 + verification_key: ../../../components/esp32/dummy_signing_key_v1_ecdsa.pem diff --git a/tests/component_tests/esp32/config/signed_ota_external_rsa_s3.yaml b/tests/component_tests/esp32/config/signed_ota_external_rsa_s3.yaml new file mode 100644 index 0000000000..f63f3ab690 --- /dev/null +++ b/tests/component_tests/esp32/config/signed_ota_external_rsa_s3.yaml @@ -0,0 +1,10 @@ +esphome: + name: test + +esp32: + variant: esp32s3 + framework: + type: esp-idf + advanced: + signed_ota_verification: + signing_scheme: rsa3072 diff --git a/tests/component_tests/esp32/config/signed_ota_signing_key_s3.yaml b/tests/component_tests/esp32/config/signed_ota_signing_key_s3.yaml new file mode 100644 index 0000000000..1a42301e9e --- /dev/null +++ b/tests/component_tests/esp32/config/signed_ota_signing_key_s3.yaml @@ -0,0 +1,11 @@ +esphome: + name: test + +esp32: + variant: esp32s3 + framework: + type: esp-idf + advanced: + signed_ota_verification: + signing_scheme: rsa3072 + signing_key: ../../../components/esp32/dummy_signing_key.pem diff --git a/tests/component_tests/esp32/config/signed_ota_verification_keys_s3.yaml b/tests/component_tests/esp32/config/signed_ota_verification_keys_s3.yaml new file mode 100644 index 0000000000..28966eba40 --- /dev/null +++ b/tests/component_tests/esp32/config/signed_ota_verification_keys_s3.yaml @@ -0,0 +1,12 @@ +esphome: + name: test + +esp32: + variant: esp32s3 + framework: + type: esp-idf + advanced: + signed_ota_verification: + signing_scheme: rsa3072 + verification_keys: + - ../../../components/esp32/dummy_signing_key.pem diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index c374e7c964..5620a220f8 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -270,6 +270,53 @@ def test_nvs_encryption_sdkconfig( assert "PERMANENT and IRREVERSIBLE" in caplog.text +@pytest.mark.parametrize( + ("fixture", "multi_key", "idf_on_update"), + [ + # Externally-signed RSA with a declared trusted-key list hands + # verification to ESPHome's multi-key verifier, so IDF's single-block + # on-update check must be OFF. It defaults ON under + # SECURE_SIGNED_APPS_NO_SECURE_BOOT, so it has to be set to False + # explicitly -- not merely omitted. + ("signed_ota_verification_keys_s3.yaml", True, False), + # Externally-signed RSA without a trusted-key list has no trust anchor, + # so it falls back to IDF's built-in check. + ("signed_ota_external_rsa_s3.yaml", False, True), + # Build-time signing and the other schemes keep IDF's check. + ("signed_ota_signing_key_s3.yaml", False, True), + ("signed_ota_ecdsa256_c6.yaml", False, True), + ("signed_ota_ecdsa_v1.yaml", False, True), + ], +) +def test_signed_ota_verification_sdkconfig( + fixture: str, + multi_key: bool, + idf_on_update: bool, + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Only external RSA disables IDF's on-update check and uses ESPHome's verifier.""" + generate_main(component_config_path(fixture)) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + # The padded, externally-signable image is always produced. + assert sdkconfig.get("CONFIG_SECURE_SIGNED_APPS_NO_SECURE_BOOT") is True + # Explicit value (never left to the Kconfig default) decides who verifies. + assert ( + sdkconfig.get("CONFIG_SECURE_SIGNED_ON_UPDATE_NO_SECURE_BOOT") is idf_on_update + ) + defines = {define.name for define in CORE.defines} + assert ("USE_OTA_SIGNED_VERIFICATION_MULTI_KEY" in defines) is multi_key + if multi_key: + # The padding / reserved signature sector the verifier depends on keys + # off the RSA scheme symbol, not the hidden CONFIG_SECURE_SIGNED_APPS + # (which the explicit `n` above drives to n). Pin the real dependency. + assert sdkconfig.get("CONFIG_SECURE_SIGNED_APPS_RSA_SCHEME") is True + # The compiled-in trust anchor: the fixture lists one key. + define_values = {define.name: str(define.value) for define in CORE.defines} + assert define_values["OTA_TRUSTED_KEY_COUNT"] == "1" + assert "OTA_TRUSTED_KEY_DIGESTS" in define_values + + @pytest.mark.parametrize( ("fixture", "expect_warning"), [ @@ -472,26 +519,18 @@ def test_flash_mode_unset_leaves_defaults( ), pytest.param( PlatformFramework.ESP32_IDF, - NetworkSdkconfigData( - wifi=True, bluetooth=True, ble_42=True, software_coexistence=True - ), + NetworkSdkconfigData(wifi=True, bluetooth=True, software_coexistence=True), {}, { "CONFIG_BT_ENABLED": True, "CONFIG_BT_BLE_42_FEATURES_SUPPORTED": True, + "CONFIG_BT_BLE_50_FEATURES_SUPPORTED": False, "CONFIG_SW_COEXIST_ENABLE": True, "CONFIG_ESP_WIFI_SOFTAP_SUPPORT": False, "CONFIG_LWIP_DHCPS": False, }, id="idf_wifi_ble_tracker_coexistence", ), - pytest.param( - PlatformFramework.ESP32_IDF, - NetworkSdkconfigData(bluetooth=True), - {}, - {"CONFIG_BT_ENABLED": True}, - id="idf_ble_server_only_no_ble42", - ), # --- IDF: user sdkconfig_options always win --- pytest.param( PlatformFramework.ESP32_IDF, @@ -612,6 +651,7 @@ def test_network_wifi_ble_coexistence_reconciles_end_to_end( sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] assert sdkconfig.get("CONFIG_BT_ENABLED") is True assert sdkconfig.get("CONFIG_BT_BLE_42_FEATURES_SUPPORTED") is True + assert sdkconfig.get("CONFIG_BT_BLE_50_FEATURES_SUPPORTED") is False assert sdkconfig.get("CONFIG_SW_COEXIST_ENABLE") is True assert sdkconfig.get("CONFIG_ESP_WIFI_SOFTAP_SUPPORT") is False assert sdkconfig.get("CONFIG_LWIP_DHCPS") is False @@ -714,6 +754,9 @@ def test_downgrade_protection_reports_all_unmet_requirements() -> None: # V1 ECDSA: exactly one of signing key / verification key. {"signing_scheme": "ecdsa_v1", "signing_key": "key.pem"}, {"signing_scheme": "ecdsa_v1", "verification_key": "key.bin"}, + # External RSA with a compiled-in trusted-key list (digests). + {"signing_scheme": "rsa3072", "verification_keys": ["ab" * 32]}, + {"signing_scheme": "rsa3072", "verification_keys": ["ab" * 32, "cd" * 32]}, ], ) def test_signed_ota_keys_valid_combinations(config: dict) -> None: @@ -768,6 +811,34 @@ def test_signed_ota_bare_block_selects_v2_external_signing(value: dict | None) - }, "not both", ), + # A trusted-key list only applies to external RSA. + ( + {"signing_scheme": "ecdsa256", "verification_keys": ["ab" * 32]}, + "only used with signing scheme 'rsa3072'", + ), + # Can't both auto-sign and verify against a fixed trusted set. + ( + { + "signing_scheme": "rsa3072", + "signing_key": "key.pem", + "verification_keys": ["ab" * 32], + }, + "cannot be combined with", + ), + # The singular V1 key and the RSA trusted-key list are mutually exclusive. + ( + { + "signing_scheme": "rsa3072", + "verification_key": "key.bin", + "verification_keys": ["ab" * 32], + }, + "at most one", + ), + # Duplicate trusted keys are rejected. + ( + {"signing_scheme": "rsa3072", "verification_keys": ["ab" * 32, "ab" * 32]}, + "must be unique", + ), ], ) def test_signed_ota_keys_invalid_combinations(config: dict, match: str) -> None: @@ -777,6 +848,48 @@ def test_signed_ota_keys_invalid_combinations(config: dict, match: str) -> None: _validate_signed_ota_keys(config) +def test_sbv2_rsa_key_digest_known_answer() -> None: + """The compiled-in trust anchor is the block-format digest the device + computes per signature block; pin it to espsecure's known output for the + shipped dummy key so a future change to the derivation can't drift silently. + """ + from esphome.components.esp32 import _sbv2_rsa_key_digest + + key = ( + Path(__file__).parent.parent.parent + / "components" + / "esp32" + / "dummy_signing_key.pem" + ) + assert ( + _sbv2_rsa_key_digest(key).hex() + == "957671f5ec1b55b3fb1d32c5525a68d3b8c33847922daddb4feefe64cd679f65" + ) + + +def test_validate_trusted_key_hex_forms() -> None: + """The digest-input branch: the same key as an uppercase 64-hex digest + normalizes to the PEM-derived value (the two forms are interchangeable), and + a mangled digest fails clearly instead of as a missing file. + """ + from esphome.components.esp32 import _sbv2_rsa_key_digest, _validate_trusted_key + + key = ( + Path(__file__).parent.parent.parent + / "components" + / "esp32" + / "dummy_signing_key.pem" + ) + pem_digest = _sbv2_rsa_key_digest(key).hex() + assert _validate_trusted_key(pem_digest.upper()) == pem_digest + for bad in (pem_digest[:-1], "0x" + pem_digest): + with pytest.raises(cv.Invalid, match="64 hex"): + _validate_trusted_key(bad) + # An unquoted 0x.../all-digit digest reaches the validator as a YAML int. + with pytest.raises(cv.Invalid, match="Quote the digest"): + _validate_trusted_key(0x957671F5EC1B55B3) + + @pytest.mark.parametrize( ("value", "expected"), [ diff --git a/tests/component_tests/helpers.py b/tests/component_tests/helpers.py index 2eb588c0ca..3b5e5bbd6e 100644 --- a/tests/component_tests/helpers.py +++ b/tests/component_tests/helpers.py @@ -27,3 +27,19 @@ def extract_packed_value(main_cpp: str, var_name: str) -> int: match = re.search(combined_pattern, main_cpp) or re.search(legacy_pattern, main_cpp) assert match, f"configure call not found for {var_name}" return int(match.group(1)) + + +def get_define_value(name: str) -> str | None: + """Rendered value of a CORE define, or None when absent. + + Values are codegen expressions (IntLiteral); they are compared rendered. + A value-less define (e.g. USE_BK72XX_BLE) is present but renders as the + string "None", while an absent define returns the None object — easy to + conflate in assertions, so use this helper for valued defines only. + """ + from esphome.core import CORE + + for define in CORE.defines: + if define.name == name: + return str(define.value) + return None diff --git a/tests/component_tests/ln882h_ble_tracker/__init__.py b/tests/component_tests/ln882h_ble_tracker/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/component_tests/ln882h_ble_tracker/config/test_automations.yaml b/tests/component_tests/ln882h_ble_tracker/config/test_automations.yaml new file mode 100644 index 0000000000..883d20b7ce --- /dev/null +++ b/tests/component_tests/ln882h_ble_tracker/config/test_automations.yaml @@ -0,0 +1,44 @@ +esphome: + name: ln-trigger-codegen + on_boot: + then: + - ln882h_ble_tracker.start_scan: + continuous: true + # Bare form: restores the configured scan_parameters mode — no + # set_continuous emitted (asserted in the codegen test). + - ln882h_ble_tracker.start_scan: + - ln882h_ble_tracker.stop_scan: + +ln882x: + board: generic-ln882h + +ln882h_ble_tracker: + scan_parameters: + continuous: false + on_ble_advertise: + - mac_address: + - AC:37:43:77:5F:4C + - 11:22:33:44:55:66 + then: + - lambda: 'char addr[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; ESP_LOGD("t", "%s", x.address_str_to(addr));' + on_ble_service_data_advertise: + - service_uuid: ABCDABCD-ABCD-ABCD-ABCD-ABCDABCDABCD + mac_address: AC:37:43:77:5F:4C + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + - service_uuid: ABCDABCD + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + on_ble_manufacturer_data_advertise: + - manufacturer_id: ABCD + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + - manufacturer_id: ABCDABCD + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + - manufacturer_id: ABCDABCD-ABCD-ABCD-ABCD-ABCDABCDABCD + then: + - lambda: 'ESP_LOGD("t", "%zu", x.size());' + on_scan_end: + - then: + - lambda: 'ESP_LOGD("t", "end");' diff --git a/tests/component_tests/ln882h_ble_tracker/test_automations_codegen.py b/tests/component_tests/ln882h_ble_tracker/test_automations_codegen.py new file mode 100644 index 0000000000..608a4c6694 --- /dev/null +++ b/tests/component_tests/ln882h_ble_tracker/test_automations_codegen.py @@ -0,0 +1,55 @@ +"""Codegen tests for the tracker automations: the generated main is the +automated check on the setter calls and the listener accounting (the +test.ln882x-ard.yaml compile fixture proves linkage, not codegen shape).""" + +from collections.abc import Callable +from pathlib import Path +import re + +from esphome.components import ble_device_base +from tests.component_tests.helpers import get_define_value + + +def test_trigger_codegen( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + main_cpp = generate_main(component_config_path("test_automations.yaml")) + + # on_ble_advertise: multi-mac filter (two addresses in one initializer list) + assert "set_addresses({0xAC3743775F4CULL, 0x112233445566ULL})" in main_cpp + # 128-bit service uuid goes out reversed (BLE wire order); single-mac filter + assert ( + "set_service_uuid128((uint8_t*)(const uint8_t[16]){0xCD,0xAB,0xCD,0xAB," + "0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB})" in main_cpp + ) + assert "set_address(0xAC3743775F4CULL)" in main_cpp + # 32-bit middle branch of the width dispatch + assert "set_service_uuid32(0xABCDABCDULL)" in main_cpp + # All three manufacturer widths: getattr() builds these names as strings, + # so a misspelling only ever fails here. + assert "set_manufacturer_uuid16(0xABCDULL)" in main_cpp + assert "set_manufacturer_uuid32(0xABCDABCDULL)" in main_cpp + assert ( + "set_manufacturer_uuid128((uint8_t*)(const uint8_t[16]){0xCD,0xAB,0xCD,0xAB," + "0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB,0xCD,0xAB})" in main_cpp + ) + # scan-control actions: templatable continuous lambda + parented actions. + # Exactly one set_continuous: the bare start_scan emits none, pinning the + # restore-configured-mode divergence from esp32 against a future default=. + assert main_cpp.count("->set_continuous(") == 1 + assert "startscanaction_id->set_continuous(" in main_cpp + assert "stopscanaction_id->set_parent(" in main_cpp + # scan_parameters continuous: false reaches the YAML-mode setter, not the + # runtime override. + assert "->set_configured_continuous(false)" in main_cpp + # Constructor call, not just the declaration: the parent argument is what + # registers the trigger as a listener. + assert re.search( + r"new\(\w+\) ble_device_base::BLEEndOfScanTrigger\(\w+\)", main_cpp + ) + + # Seven triggers register as listeners; an undercount silently drops the + # last trigger at runtime (StaticVector::push_back past capacity), so the + # define is the assertion that matters most. + assert get_define_value(ble_device_base.LISTENER_COUNT_DEFINE) == "7" diff --git a/tests/component_tests/lvgl/config/set_z_index_test.yaml b/tests/component_tests/lvgl/config/set_z_index_test.yaml new file mode 100644 index 0000000000..61a248ff99 --- /dev/null +++ b/tests/component_tests/lvgl/config/set_z_index_test.yaml @@ -0,0 +1,60 @@ +esphome: + name: test-set-z-index + +esp32: + board: esp32dev + framework: + type: esp-idf + +spi: + - id: spi_bus + clk_pin: GPIO18 + mosi_pin: GPIO23 + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: tft_display + dimensions: + width: 240 + height: 320 + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + invert_colors: false + update_interval: never + +lvgl: + displays: tft_display + widgets: + - label: + id: label_a + text: "A" + - label: + id: label_b + text: "B" + - button: + id: trigger_btn + on_click: + - lvgl.widget.set_z_index: + id: label_a + position: top + - lvgl.widget.set_z_index: + id: label_a + position: bottom + - lvgl.widget.set_z_index: + id: label_a + position: up + - lvgl.widget.set_z_index: + id: label_a + position: down + - lvgl.widget.set_z_index: + id: label_a + position: 3 + - lvgl.widget.set_z_index: + id: label_a + position: -2 + - lvgl.widget.set_z_index: + id: [label_a, label_b] + position: up diff --git a/tests/component_tests/lvgl/test_animation.py b/tests/component_tests/lvgl/test_animation.py index 1a2cde632c..ce9a162d99 100644 --- a/tests/component_tests/lvgl/test_animation.py +++ b/tests/component_tests/lvgl/test_animation.py @@ -169,14 +169,27 @@ class TestTimingSchema: def test_round_trip_string(self) -> None: assert TIMING_SCHEMA("round_trip")["type"] == "round_trip" + def test_round_trip_default_pause(self) -> None: + # Back-compat default: no pause, matching the pre-existing round_trip behavior. + assert TIMING_SCHEMA("round_trip")["pause"] == pytest.approx(0.0) + + def test_round_trip_pause_percentage_string(self) -> None: + result = TIMING_SCHEMA({"type": "round_trip", "pause": "50%"}) + assert result["pause"] == pytest.approx(0.5) + + def test_round_trip_pause_rejects_one(self) -> None: + # pause == 1.0 would make moving_length_ zero and divide by zero in map_progress. + with pytest.raises((Invalid, MultipleInvalid)): + TIMING_SCHEMA({"type": "round_trip", "pause": 1.0}) + def test_ease_in_out_default_weight(self) -> None: result = TIMING_SCHEMA("ease_in_out") assert result["type"] == "ease_in_out" - assert result["weight"] == pytest.approx(2.0) + assert result["weight"] == pytest.approx(1.0) def test_ease_in_out_custom_weight(self) -> None: - result = TIMING_SCHEMA({"type": "ease_in_out", "weight": 3}) - assert result["weight"] == pytest.approx(3.0) + result = TIMING_SCHEMA({"type": "ease_in_out", "weight": 0.5}) + assert result["weight"] == pytest.approx(0.5) def test_gravity_defaults(self) -> None: result = TIMING_SCHEMA("gravity") diff --git a/tests/component_tests/lvgl/test_set_z_index.py b/tests/component_tests/lvgl/test_set_z_index.py new file mode 100644 index 0000000000..cf2a072ca0 --- /dev/null +++ b/tests/component_tests/lvgl/test_set_z_index.py @@ -0,0 +1,128 @@ +"""Tests for the ``lvgl.widget.set_z_index`` action: schema validation and +code generation. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from esphome.components.lvgl.automation import SET_Z_INDEX_SCHEMA +from esphome.config_validation import Invalid + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +class TestSetZIndexSchemaValidation: + """Test that SET_Z_INDEX_SCHEMA accepts the documented forms and rejects + everything else. + """ + + @pytest.mark.parametrize("position", ["top", "bottom", "up", "down"]) + def test_keyword_position_accepted(self, position: str) -> None: + config = SET_Z_INDEX_SCHEMA({"id": "my_widget", "position": position}) + assert config["position"] == position.upper() + + @pytest.mark.parametrize("position", ["Top", "BOTTOM", "Up", "dOwN"]) + def test_keyword_position_case_insensitive(self, position: str) -> None: + config = SET_Z_INDEX_SCHEMA({"id": "my_widget", "position": position}) + assert config["position"] == position.upper() + + @pytest.mark.parametrize("position", [0, 1, 5, -1, -5]) + def test_integer_position_accepted(self, position: int) -> None: + config = SET_Z_INDEX_SCHEMA({"id": "my_widget", "position": position}) + assert config["position"] == position + + def test_unknown_keyword_rejected(self) -> None: + with pytest.raises(Invalid): + SET_Z_INDEX_SCHEMA({"id": "my_widget", "position": "sideways"}) + + def test_float_position_rejected(self) -> None: + with pytest.raises(Invalid): + SET_Z_INDEX_SCHEMA({"id": "my_widget", "position": 1.5}) + + def test_missing_id_rejected(self) -> None: + with pytest.raises(Invalid): + SET_Z_INDEX_SCHEMA({"position": "top"}) + + def test_missing_position_rejected(self) -> None: + with pytest.raises(Invalid): + SET_Z_INDEX_SCHEMA({"id": "my_widget"}) + + def test_single_id_is_wrapped_in_list(self) -> None: + config = SET_Z_INDEX_SCHEMA({"id": "my_widget", "position": "top"}) + assert len(config["id"]) == 1 + assert config["id"][0]["id"].id == "my_widget" + + def test_list_of_ids_accepted(self) -> None: + config = SET_Z_INDEX_SCHEMA({"id": ["widget_a", "widget_b"], "position": "top"}) + assert [entry["id"].id for entry in config["id"]] == ["widget_a", "widget_b"] + + +# --------------------------------------------------------------------------- +# Code generation +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def main_cpp(request: pytest.FixtureRequest) -> str: + """Generate the C++ output for the shared set_z_index YAML config once + per module. See ``test_widget_state.py`` for why this is module-scoped + and self-contained rather than using the function-scoped ``generate_main`` + fixture from ``conftest.py``. + """ + from esphome.__main__ import generate_cpp_contents + from esphome.config import read_config + from esphome.core import CORE + + config_path = Path(request.fspath).parent / "config" / "set_z_index_test.yaml" + original_path = CORE.config_path + try: + CORE.config_path = config_path + CORE.config = read_config({}) + generate_cpp_contents(CORE.config) + return CORE.cpp_global_section + CORE.cpp_main_section + finally: + CORE.config_path = original_path + CORE.reset() + + +def test_top_emits_move_foreground(main_cpp: str) -> None: + assert "lv_obj_move_foreground(label_a);" in main_cpp + + +def test_bottom_emits_move_background(main_cpp: str) -> None: + assert "lv_obj_move_background(label_a);" in main_cpp + + +def test_up_emits_unguarded_index_increment(main_cpp: str) -> None: + assert "lv_obj_move_to_index(label_a, lv_obj_get_index(label_a) + 1);" in main_cpp + + +def test_down_emits_guarded_index_decrement(main_cpp: str) -> None: + """``down`` must be guarded so that a widget already at index 0 isn't + reinterpreted by LVGL as "move to the top" (LVGL treats a negative + index as "count from the back"). + """ + assert "if (lv_obj_get_index(label_a) > 0) {" in main_cpp + assert "lv_obj_move_to_index(label_a, lv_obj_get_index(label_a) - 1);" in main_cpp + + +def test_positive_integer_emits_direct_index(main_cpp: str) -> None: + assert "lv_obj_move_to_index(label_a, 3);" in main_cpp + + +def test_negative_integer_emits_direct_index(main_cpp: str) -> None: + assert "lv_obj_move_to_index(label_a, -2);" in main_cpp + + +def test_list_of_ids_applies_to_each_widget(main_cpp: str) -> None: + """``id: [label_a, label_b]`` must emit the move call once per widget.""" + assert ( + main_cpp.count("lv_obj_move_to_index(label_a, lv_obj_get_index(label_a) + 1);") + == 2 + ) + assert "lv_obj_move_to_index(label_b, lv_obj_get_index(label_b) + 1);" in main_cpp diff --git a/tests/component_tests/modbus_client/test_modbus_client.py b/tests/component_tests/modbus_client/test_modbus_client.py new file mode 100644 index 0000000000..10f9bc588e --- /dev/null +++ b/tests/component_tests/modbus_client/test_modbus_client.py @@ -0,0 +1,116 @@ +"""Tests for modbus_client configuration validation. + +Handler PDU spans point into hub buffers reused once the handler returns, so the deferring-actions +guard is a safety property: these tests pin it to every handler slot. +""" + +import pytest + +from esphome import config_validation as cv +from esphome.components.modbus_client import ( + CONF_ON_NO_RESPONSE, + CONF_ON_NOT_SENT, + CONF_ON_SENT, + CONF_PDU, + MODBUS_CLIENT_SEND_SCHEMA, +) +from esphome.const import CONF_ADDRESS, CONF_ON_ERROR, CONF_ON_RESPONSE +from esphome.core import Lambda +from esphome.types import ConfigType + +# Every handler slot on modbus_client.send. All five must reject deferring actions. +HANDLER_KEYS = [ + CONF_ON_SENT, + CONF_ON_RESPONSE, + CONF_ON_ERROR, + CONF_ON_NO_RESPONSE, + CONF_ON_NOT_SENT, +] + +# A deferring action (registered synchronous=False) and a synchronous one, for contrast. +DEFERRING_ACTION = {"delay": "1s"} +SYNCHRONOUS_ACTION = {"lambda": Lambda('ESP_LOGD("test", "ran");')} +TRUE_CONDITION = {"lambda": Lambda("return true;")} + +# The same deferring action buried inside nested control flow, which the guard must still find. +NESTED_ACTIONS = [ + pytest.param( + [{"if": {"condition": TRUE_CONDITION, "then": [DEFERRING_ACTION]}}], + id="if", + ), + pytest.param([{"repeat": {"count": 2, "then": [DEFERRING_ACTION]}}], id="repeat"), + pytest.param( + [ + { + "repeat": { + "count": 2, + "then": [ + { + "if": { + "condition": TRUE_CONDITION, + "then": [DEFERRING_ACTION], + } + } + ], + } + } + ], + id="repeat_if", + ), +] + +DEFER_MESSAGE = "Deferring actions" + + +def _config(handler_key: str, actions: list) -> ConfigType: + """A minimal valid modbus_client.send config with one handler populated.""" + return { + CONF_ADDRESS: 0x01, + CONF_PDU: [0x03, 0x00, 0x10, 0x00, 0x01], + handler_key: {"then": actions}, + } + + +@pytest.mark.parametrize("handler_key", HANDLER_KEYS) +def test_synchronous_handler_accepted(handler_key: str) -> None: + # The guard must not get in the way of an ordinary inline handler. + MODBUS_CLIENT_SEND_SCHEMA(_config(handler_key, [SYNCHRONOUS_ACTION])) + + +@pytest.mark.parametrize("handler_key", HANDLER_KEYS) +def test_deferring_action_rejected(handler_key: str) -> None: + with pytest.raises(cv.Invalid, match=DEFER_MESSAGE): + MODBUS_CLIENT_SEND_SCHEMA(_config(handler_key, [DEFERRING_ACTION])) + + +@pytest.mark.parametrize("handler_key", HANDLER_KEYS) +@pytest.mark.parametrize("actions", NESTED_ACTIONS) +def test_nested_deferring_action_rejected(handler_key: str, actions: list) -> None: + # has_non_synchronous_actions recurses, so a delay buried in if:/repeat: is still caught. + with pytest.raises(cv.Invalid, match=DEFER_MESSAGE): + MODBUS_CLIENT_SEND_SCHEMA(_config(handler_key, actions)) + + +def test_on_no_response_lambda_form_accepted() -> None: + # The returning-lambda form has no action list; the guard is a no-op on it. + MODBUS_CLIENT_SEND_SCHEMA( + { + CONF_ADDRESS: 0x01, + CONF_PDU: [0x03, 0x00, 0x10, 0x00, 0x01], + CONF_ON_NO_RESPONSE: Lambda("return false;"), + } + ) + + +def test_on_no_response_retry_lambda_accepted() -> None: + # The automation form may also carry a nested retry: lambda. + MODBUS_CLIENT_SEND_SCHEMA( + { + CONF_ADDRESS: 0x01, + CONF_PDU: [0x03, 0x00, 0x10, 0x00, 0x01], + CONF_ON_NO_RESPONSE: { + "then": [SYNCHRONOUS_ACTION], + "retry": Lambda("return true;"), + }, + } + ) diff --git a/tests/component_tests/modbus_server/test_modbus_server.py b/tests/component_tests/modbus_server/test_modbus_server.py index 7c978a5cd5..3e041c6d4a 100644 --- a/tests/component_tests/modbus_server/test_modbus_server.py +++ b/tests/component_tests/modbus_server/test_modbus_server.py @@ -82,3 +82,5 @@ def test_raw_value_type_rejected() -> None: with pytest.raises(cv.Invalid): validator("RAW") assert validator("U_WORD") == "U_WORD" + assert validator("U_WORD_S") == "U_WORD_S" + assert validator("S_WORD_S") == "S_WORD_S" diff --git a/tests/component_tests/packages/test_packages.py b/tests/component_tests/packages/test_packages.py index 39bffd31b7..418cab5ea1 100644 --- a/tests/component_tests/packages/test_packages.py +++ b/tests/component_tests/packages/test_packages.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import pytest +from esphome import bundle from esphome.components.packages import ( CONFIG_SCHEMA, _substitute_package_definition, @@ -1694,3 +1695,34 @@ def test_resolve_packages_does_not_apply_extend_remove() -> None: # over the package value during merge), and the marker is not # resolved by this wrapper. assert isinstance(result[CONF_WIFI], Remove) + + +@patch("esphome.git.clone_or_update") +def test_remote_package_registers_checkout_for_secret_scan( + mock_clone_or_update, tmp_path: Path +) -> None: + """Loading a remote package registers its path-narrowed checkout dir + as a bundle secret-scan dir (issue 18023).""" + repo_root = tmp_path / "repo" + package_dir = repo_root / "packages" + package_dir.mkdir(parents=True) + (package_dir / "base.yml").write_text( + f"sensor:\n - platform: {TEST_SENSOR_PLATFORM_1}\n name: {TEST_SENSOR_NAME_1}\n" + ) + mock_clone_or_update.return_value = (repo_root, None) + + config = { + CONF_PACKAGES: { + "package1": { + CONF_URL: "https://github.com/esphome/non-existant-repo", + CONF_REF: "main", + CONF_PATH: "packages", + CONF_FILES: ["base.yml"], + CONF_REFRESH: "1d", + } + } + } + packages_pass(config) + + assert package_dir in bundle._get_data().secret_scan_dirs + assert repo_root not in bundle._get_data().secret_scan_dirs diff --git a/tests/components/bk72xx_ble_tracker/validate-automations.bk72xx-ard.yaml b/tests/components/bk72xx_ble_tracker/validate-automations.bk72xx-ard.yaml new file mode 100644 index 0000000000..e110369b0b --- /dev/null +++ b/tests/components/bk72xx_ble_tracker/validate-automations.bk72xx-ard.yaml @@ -0,0 +1,54 @@ +packages: + bk72xx_ble_tracker: !include common.yaml + +esphome: + on_boot: + then: + - bk72xx_ble_tracker.start_scan + - bk72xx_ble_tracker.start_scan: + continuous: true + - bk72xx_ble_tracker.stop_scan + - bk72xx_ble_tracker.stop_scan: ble_tracker + +bk72xx_ble_tracker: + on_ble_advertise: + - mac_address: AC:37:43:77:5F:4C + then: + - lambda: |- + char addr[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; + ESP_LOGD("main", "The device address is %s", x.address_str_to(addr)); + - mac_address: + - AC:37:43:77:5F:4C + - AC:37:43:77:5F:4D + then: + - lambda: |- + char addr[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; + ESP_LOGD("main", "The device address is %s", x.address_str_to(addr)); + on_ble_service_data_advertise: + - service_uuid: ABCD + # mac_address exercises the UUID triggers' set_address() codegen branch. + mac_address: AC:37:43:77:5F:4C + then: + - lambda: |- + ESP_LOGD("main", "Length of service data is %zu", x.size()); + - service_uuid: ABCDABCD + then: + - lambda: |- + ESP_LOGD("main", "32-bit service data is %zu", x.size()); + - service_uuid: ABCDABCD-ABCD-ABCD-ABCD-ABCDABCDABCD + then: + - lambda: |- + ESP_LOGD("main", "128-bit service data is %zu", x.size()); + on_ble_manufacturer_data_advertise: + - manufacturer_id: ABCD + then: + - lambda: |- + ESP_LOGD("main", "Length of manufacturer data is %zu", x.size()); + - manufacturer_id: ABCDABCD-ABCD-ABCD-ABCD-ABCDABCDABCD + then: + - lambda: |- + ESP_LOGD("main", "128-bit manufacturer data is %zu", x.size()); + on_scan_end: + - then: + - lambda: |- + ESP_LOGD("main", "Scan ended"); diff --git a/tests/components/ble_device_base/test_address.cpp b/tests/components/ble_device_base/test_address.cpp index 9e4bca4c57..7ff318b66b 100644 --- a/tests/components/ble_device_base/test_address.cpp +++ b/tests/components/ble_device_base/test_address.cpp @@ -8,7 +8,7 @@ namespace esphome::ble_device_base::testing { // from_scan_result() ingests BLE controller order (LSB-first); the public // accessors must expose the historical esp32 semantics: address() in printable -// (MSB-first) order, address_uint64() with byte 0 in the LSB, address_str() +// (MSB-first) order, address_uint64() with byte 0 in the LSB, address_str_to() // printed MSB-first. namespace { // Device AA:BB:CC:DD:EE:FF — controller order delivers FF first. @@ -25,7 +25,14 @@ TEST(BleDeviceAddress, AccessorsMatchEsp32Semantics) { EXPECT_EQ(device.address_uint64(), 0xAABBCCDDEEFFULL); + char buf[ESPBTDevice::MAC_ADDRESS_PRETTY_BUFFER_SIZE]; + EXPECT_STREQ(device.address_str_to(buf), "AA:BB:CC:DD:EE:FF"); + + // The deprecated wrapper must keep returning the same string until its 2027.2.0 removal. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" EXPECT_EQ(device.address_str(), "AA:BB:CC:DD:EE:FF"); +#pragma GCC diagnostic pop } // mac_lsb_first_to_uint64() packs the controller-order bytes a raw-advertisement @@ -43,4 +50,18 @@ TEST(BleDeviceAddress, MacLsbFirstToUint64AgreesWithParsedDevice) { EXPECT_EQ(mac_lsb_first_to_uint64(MAC_LSB_FIRST), device.address_uint64()); } +// uint64_to_mac_msb_first() is the inverse: unpacking the wire value yields +// printable (MSB-first) order, and round-tripping through the LSB-first +// packer restores the original value. +TEST(BleDeviceAddress, Uint64ToMacMsbFirstRoundTrip) { + uint8_t msb_first[6]; + uint64_to_mac_msb_first(0xAABBCCDDEEFFULL, msb_first); + EXPECT_EQ(msb_first[0], 0xaa); + EXPECT_EQ(msb_first[5], 0xff); + uint8_t lsb_first[6]; + for (int i = 0; i < 6; i++) + lsb_first[i] = msb_first[5 - i]; + EXPECT_EQ(mac_lsb_first_to_uint64(lsb_first), 0xAABBCCDDEEFFULL); +} + } // namespace esphome::ble_device_base::testing diff --git a/tests/components/ble_device_base/test_adv_name.cpp b/tests/components/ble_device_base/test_adv_name.cpp new file mode 100644 index 0000000000..44fb3525ea --- /dev/null +++ b/tests/components/ble_device_base/test_adv_name.cpp @@ -0,0 +1,91 @@ +#include "esphome/components/ble_device_base/ble_device.h" + +#include + +#include +#include +#include + +namespace esphome::ble_device_base { +namespace { + +// AD types under test +constexpr uint8_t AD_SHORT_NAME = 0x08; +constexpr uint8_t AD_COMPLETE_NAME = 0x09; + +void append_name(std::vector &adv, uint8_t ad_type, const char *name) { + size_t len = strlen(name); + adv.push_back(static_cast(len + 1)); + adv.push_back(ad_type); + adv.insert(adv.end(), name, name + len); +} + +ESPBTDevice device_from(const std::vector &adv) { + const uint8_t mac[6] = {0x11, 0x22, 0x33, 0x44, 0x55, 0x66}; + ESPBTDevice device; + device.from_scan_result(mac, -59, 0, adv.data(), static_cast(adv.size())); + return device; +} + +} // namespace + +TEST(BleAdvName, ParsesACompleteName) { + std::vector adv; + append_name(adv, AD_COMPLETE_NAME, "TP96"); + ESPBTDevice device = device_from(adv); + EXPECT_EQ(device.get_name(), "TP96"); + // The backing buffer is NUL-terminated so c_str() is usable directly. + EXPECT_STREQ(device.get_name().c_str(), "TP96"); +} + +TEST(BleAdvName, LongestNameWinsShortenedThenComplete) { + // A merged adv + scan-response frame can carry both forms; the shortened + // one must never replace the complete one. + std::vector adv; + append_name(adv, AD_SHORT_NAME, "Radon"); + append_name(adv, AD_COMPLETE_NAME, "RadonEye"); + EXPECT_EQ(device_from(adv).get_name(), "RadonEye"); +} + +TEST(BleAdvName, LongestNameWinsCompleteThenShortened) { + std::vector adv; + append_name(adv, AD_COMPLETE_NAME, "RadonEye"); + append_name(adv, AD_SHORT_NAME, "Radon"); + EXPECT_EQ(device_from(adv).get_name(), "RadonEye"); +} + +TEST(BleAdvName, MaxLengthNameFitsAndTerminates) { + // 29 bytes is the largest name a legacy AD element can carry and exactly + // fills the fixed buffer. + std::string max_name(29, 'a'); + std::vector adv; + append_name(adv, AD_COMPLETE_NAME, max_name.c_str()); + ESPBTDevice device = device_from(adv); + EXPECT_EQ(device.get_name().size(), 29u); + EXPECT_EQ(device.get_name(), max_name); + EXPECT_STREQ(device.get_name().c_str(), max_name.c_str()); +} + +TEST(BleAdvName, ReparseResetsThePreviousName) { + const uint8_t mac[6] = {0x11, 0x22, 0x33, 0x44, 0x55, 0x66}; + std::vector first; + append_name(first, AD_COMPLETE_NAME, "RadonEye"); + std::vector second; + append_name(second, AD_COMPLETE_NAME, "TP96"); + + ESPBTDevice device; + device.from_scan_result(mac, -59, 0, first.data(), static_cast(first.size())); + ASSERT_EQ(device.get_name(), "RadonEye"); + // A shorter name from a fresh report must fully replace the longer one: + // the longest-name rule applies within one report, not across reports. + device.from_scan_result(mac, -59, 0, second.data(), static_cast(second.size())); + EXPECT_EQ(device.get_name(), "TP96"); + EXPECT_STREQ(device.get_name().c_str(), "TP96"); +} + +TEST(BleAdvName, NoNamePresentIsEmpty) { + std::vector adv = {0x02, 0x0A, 0x00}; // TX power only + EXPECT_TRUE(device_from(adv).get_name().empty()); +} + +} // namespace esphome::ble_device_base diff --git a/tests/components/ble_device_base/test_ble_uuid.cpp b/tests/components/ble_device_base/test_ble_uuid.cpp index 45abd48479..7e99ce8a95 100644 --- a/tests/components/ble_device_base/test_ble_uuid.cpp +++ b/tests/components/ble_device_base/test_ble_uuid.cpp @@ -27,6 +27,60 @@ TEST(BleDeviceUuid, ThirtyTwoBitMatchesEquivalentLongForm) { EXPECT_TRUE(u32 == u128); } +// A default-constructed UUID is UNSET, the historical "not configured" sentinel +// (len 0 through the esp32 get_uuid() adapter). +TEST(BleDeviceUuid, DefaultConstructedIsUnset) { + const ESPBTUUID unset; + EXPECT_EQ(unset.type(), ESPBTUUID::Type::UNSET); + EXPECT_TRUE(unset == ESPBTUUID()); + EXPECT_FALSE(unset == ESPBTUUID::from_uint16(0x1234)); + EXPECT_FALSE(unset.contains(0x00, 0x00)); +} + +// Every factory yields a non-UNSET UUID, even for 0x0000: only default construction and a +// failed text parse are unset, keeping type() != UNSET equivalent to the old len > 0 check. +TEST(BleDeviceUuid, AllFactoriesProduceSetUuids) { + const uint8_t raw[16] = {0xFB, 0x34, 0x9B, 0x5F, 0x80, 0x00, 0x00, 0x80, + 0x00, 0x10, 0x00, 0x00, 0x34, 0x12, 0x00, 0x00}; + EXPECT_NE(ESPBTUUID::from_uint16(0x0000).type(), ESPBTUUID::Type::UNSET); + EXPECT_NE(ESPBTUUID::from_uint32(0).type(), ESPBTUUID::Type::UNSET); + EXPECT_NE(ESPBTUUID::from_raw(raw).type(), ESPBTUUID::Type::UNSET); + EXPECT_NE(ESPBTUUID::from_raw_reversed(raw).type(), ESPBTUUID::Type::UNSET); + EXPECT_NE(ESPBTUUID::from_raw("180F", 4).type(), ESPBTUUID::Type::UNSET); + EXPECT_NE(ESPBTUUID::from_raw("0000180F", 8).type(), ESPBTUUID::Type::UNSET); + EXPECT_NE(ESPBTUUID::from_raw(reinterpret_cast(raw), 16).type(), ESPBTUUID::Type::UNSET); + EXPECT_NE(ESPBTUUID::from_raw("6E400001-B5A3-F393-E0A9-E50E24DCCA9E").type(), ESPBTUUID::Type::UNSET); +} + +// 0x0000 is a valid short UUID on real devices (esphome/aioesphomeapi#1742); an unset +// UUID must never compare equal to it. Unset equals only unset. +TEST(BleDeviceUuid, UnsetIsNotEqualToZeroUuid) { + EXPECT_FALSE(ESPBTUUID() == ESPBTUUID::from_uint16(0x0000)); + EXPECT_FALSE(ESPBTUUID::from_uint16(0x0000) == ESPBTUUID()); + const uint8_t base[16] = {0xFB, 0x34, 0x9B, 0x5F, 0x80, 0x00, 0x00, 0x80, + 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + EXPECT_FALSE(ESPBTUUID() == ESPBTUUID::from_raw(base)); + EXPECT_TRUE(ESPBTUUID() == ESPBTUUID()); + EXPECT_FALSE(ESPBTUUID().as_128bit().is_set()); // widening preserves the unset state + // A configured 0x0000 still matches its own 128-bit base UUID expansion. + EXPECT_TRUE(ESPBTUUID::from_uint16(0x0000) == ESPBTUUID::from_raw(base)); +} + +// is_set() is the sentinel check; an unset UUID prints as "None" instead of a +// valid-looking all-zero 128-bit UUID. +TEST(BleDeviceUuid, IsSetAndUnsetToStr) { + char buf[UUID_STR_LEN]; + EXPECT_FALSE(ESPBTUUID().is_set()); + EXPECT_STREQ(ESPBTUUID().to_str(buf), "None"); + EXPECT_TRUE(ESPBTUUID::from_uint16(0x0000).is_set()); + EXPECT_STREQ(ESPBTUUID::from_uint16(0x0000).to_str(buf), "0x0000"); +} + +// Text parsing of an invalid length historically produced a len-0 (unset) UUID. +TEST(BleDeviceUuid, InvalidTextFormParsesToUnset) { + EXPECT_EQ(ESPBTUUID::from_raw("nope", 3).type(), ESPBTUUID::Type::UNSET); +} + TEST(BleDeviceUuid, DifferentUuidsDoNotMatch) { EXPECT_FALSE(ESPBTUUID::from_uint16(0x1234) == ESPBTUUID::from_uint16(0x1235)); const uint8_t raw128[16] = {0xFB, 0x34, 0x9B, 0x5F, 0x80, 0x00, 0x00, 0x80, diff --git a/tests/components/ble_device_base/test_gatt_client_contract.cpp b/tests/components/ble_device_base/test_gatt_client_contract.cpp new file mode 100644 index 0000000000..fb743b2699 --- /dev/null +++ b/tests/components/ble_device_base/test_gatt_client_contract.cpp @@ -0,0 +1,66 @@ +// The GATT client contract compiles in no real build until a hub backend is +// configured; this TU pins it on the host so the header cannot rot unseen. +#define USE_BLE_GATT_CLIENT + +#include "esphome/components/ble_device_base/ble_gatt_client.h" + +#include + +namespace esphome::ble_device_base::testing { + +class RecordingListener : public GattClientEventListener { + public: + void on_connection_state(bool connected, uint16_t mtu, int error) override { this->connected_ = connected; } + void on_service_discovery_done(int error) override { this->discovery_error_ = error; } + void on_read_result(uint16_t handle, const uint8_t *data, uint16_t len, int error) override {} + void on_write_result(uint16_t handle, int error) override {} + void on_notify_state(uint16_t handle, bool enabled, int error) override {} + void on_notify_data(uint16_t handle, const uint8_t *data, uint16_t len) override {} + bool connected_{false}; + int discovery_error_{0}; +}; + +class MinimalConnection : public BLEGattConnection { + public: + int connect(uint64_t address, uint8_t addr_type) override { + if (this->listener_ != nullptr) + this->listener_->on_connection_state(true, 517, 0); + return 0; + } + int disconnect() override { return 0; } + int discover_services() override { + if (this->listener_ != nullptr) + this->listener_->on_service_discovery_done(0); + return 0; + } + int read_characteristic(uint16_t handle) override { return GATT_ERR_NOT_CONNECTED; } + int write_characteristic(uint16_t handle, const uint8_t *data, uint16_t len, bool response) override { return 0; } + int read_descriptor(uint16_t handle) override { return 0; } + int write_descriptor(uint16_t handle, const uint8_t *data, uint16_t len) override { return 0; } + int notify_characteristic(uint16_t handle, bool enable) override { return 0; } + int update_connection_params(uint16_t min_interval, uint16_t max_interval, uint16_t latency, + uint16_t timeout) override { + return 0; + } + GattServiceTable get_service_table() override { return {}; } + void release_services() override {} +}; + +TEST(BleGattClientContract, MinimalImplementerCompilesAndRoutesEvents) { + MinimalConnection connection; + RecordingListener listener; + connection.set_listener(&listener); + EXPECT_EQ(connection.connect(0xAABBCCDDEEFFULL, 0), 0); + EXPECT_TRUE(listener.connected_); + EXPECT_EQ(connection.discover_services(), 0); + EXPECT_EQ(listener.discovery_error_, 0); + EXPECT_EQ(connection.read_characteristic(1), GATT_ERR_NOT_CONNECTED); + + // A default table is empty and safe to walk. + GattServiceTable table = connection.get_service_table(); + EXPECT_EQ(table.service_count, 0); + EXPECT_EQ(table.characteristic_count, 0); + EXPECT_EQ(table.descriptor_count, 0); +} + +} // namespace esphome::ble_device_base::testing diff --git a/tests/components/ble_device_base/test_ibeacon.cpp b/tests/components/ble_device_base/test_ibeacon.cpp new file mode 100644 index 0000000000..b154742ee2 --- /dev/null +++ b/tests/components/ble_device_base/test_ibeacon.cpp @@ -0,0 +1,158 @@ +#include + +#include +#include + +#include "esphome/components/ble_device_base/ble_device.h" + +namespace esphome::ble_device_base::testing { + +// from_manufacturer_data() accepts exactly the iBeacon frame: Apple company ID, +// 23 payload bytes, and the 0x02/0x15 sub-type/length prefix. The prefix check +// is stricter than the legacy esp32 parser (which surfaced any 23-byte Apple +// payload as a beacon) — a declared behavior change; these tests pin the +// accept/reject boundary. +namespace { + +ServiceData make_apple_payload(uint8_t sub_type, uint8_t length, size_t size = 23) { + ServiceData data; + data.uuid = ESPBTUUID::from_uint16(0x004C); // Apple company ID + data.data.assign(size, 0); + if (size >= 2) { + data.data[0] = sub_type; + data.data[1] = length; + } + // BeaconData layout: sub_type[0], length[1], proximity_uuid[2..17], + // major[18..19], minor[20..21], signal_power[22] — all wire values big-endian. + if (size >= 23) { + data.data[18] = 0x12; // major 0x1234 + data.data[19] = 0x34; + data.data[20] = 0x56; // minor 0x5678 + data.data[21] = 0x78; + data.data[22] = 0xC5; // signal power -59 dBm + } + return data; +} + +} // namespace + +TEST(BleIBeacon, AcceptsWellFormedFrame) { + auto beacon = ESPBLEiBeacon::from_manufacturer_data(make_apple_payload(0x02, 0x15)); + ASSERT_TRUE(beacon.has_value()); + // Explicit guard: clang-tidy's unchecked-optional-access models neither + // gtest's ASSERT_TRUE nor value() as a check. + if (beacon.has_value()) { + // Pins every scalar accessor's offset and the on-wire big-endian order. + EXPECT_EQ(beacon->get_major(), 0x1234); + EXPECT_EQ(beacon->get_minor(), 0x5678); + EXPECT_EQ(beacon->get_signal_power(), -59); + } +} + +TEST(BleIBeacon, RejectsWrongSubType) { + // Apple "nearby" and other frames of coincidental length must not parse. + EXPECT_FALSE(ESPBLEiBeacon::from_manufacturer_data(make_apple_payload(0x10, 0x15)).has_value()); +} + +TEST(BleIBeacon, RejectsWrongLengthByte) { + EXPECT_FALSE(ESPBLEiBeacon::from_manufacturer_data(make_apple_payload(0x02, 0x14)).has_value()); +} + +TEST(BleIBeacon, RejectsWrongPayloadSize) { + EXPECT_FALSE(ESPBLEiBeacon::from_manufacturer_data(make_apple_payload(0x02, 0x15, 22)).has_value()); + EXPECT_FALSE(ESPBLEiBeacon::from_manufacturer_data(make_apple_payload(0x02, 0x15, 24)).has_value()); +} + +TEST(BleIBeacon, RejectsNonAppleCompany) { + auto data = make_apple_payload(0x02, 0x15); + data.uuid = ESPBTUUID::from_uint16(0x0059); // Nordic + EXPECT_FALSE(ESPBLEiBeacon::from_manufacturer_data(data).has_value()); +} + +TEST(BleIBeacon, PrefixRejectedFlagsOnlyTheSubTypeCase) { + // The out-param drives the get_ibeacon() diagnostic for frames the legacy + // parser accepted: exactly the 23-byte Apple payload with a wrong prefix. + // Wrong size and non-Apple frames were never accepted and must stay silent. + bool flagged = false; + EXPECT_FALSE(ESPBLEiBeacon::from_manufacturer_data(make_apple_payload(0x10, 0x15), &flagged).has_value()); + EXPECT_TRUE(flagged); + + flagged = false; + ESPBLEiBeacon::from_manufacturer_data(make_apple_payload(0x02, 0x15), &flagged); + EXPECT_FALSE(flagged); + + flagged = false; + ESPBLEiBeacon::from_manufacturer_data(make_apple_payload(0x10, 0x15, 22), &flagged); + EXPECT_FALSE(flagged); + + flagged = false; + auto nordic = make_apple_payload(0x10, 0x15); + nordic.uuid = ESPBTUUID::from_uint16(0x0059); + ESPBLEiBeacon::from_manufacturer_data(nordic, &flagged); + EXPECT_FALSE(flagged); +} + +namespace { + +// One AD manufacturer-data record: [len][0xFF][company LE][payload...]. +void append_mfr_record(std::vector &adv, uint16_t company, const std::vector &payload) { + adv.push_back(static_cast(1 + 2 + payload.size())); + adv.push_back(0xFF); + adv.push_back(static_cast(company & 0xFF)); + adv.push_back(static_cast(company >> 8)); + adv.insert(adv.end(), payload.begin(), payload.end()); +} + +std::vector beacon_payload(uint8_t sub_type, uint8_t length) { + std::vector p(23, 0); + p[0] = sub_type; + p[1] = length; + p[18] = 0x12; + p[19] = 0x34; + p[20] = 0x56; + p[21] = 0x78; + p[22] = 0xC5; + return p; +} + +ESPBTDevice device_from(const std::vector &adv) { + const uint8_t mac[6] = {0x11, 0x22, 0x33, 0x44, 0x55, 0x66}; + ESPBTDevice device; + device.from_scan_result(mac, -59, 0, adv.data(), static_cast(adv.size())); + return device; +} + +} // namespace + +// get_ibeacon() wraps the parser with first-rejection capture and the log +// gate; pin its short circuits so a regression there needs a code change, not +// a review, to surface. +TEST(BleIBeacon, GetIbeaconReturnsBeaconDespitePrecedingRejectedFrame) { + std::vector adv; + append_mfr_record(adv, 0x004C, beacon_payload(0x10, 0x15)); // rejected prefix + append_mfr_record(adv, 0x004C, beacon_payload(0x02, 0x15)); // real iBeacon + auto device = device_from(adv); + auto beacon = device.get_ibeacon(); + ASSERT_TRUE(beacon.has_value()); + if (beacon.has_value()) { + EXPECT_EQ(beacon->get_major(), 0x1234); + } +} + +TEST(BleIBeacon, GetIbeaconEmptyWhenOnlyRejectedFrames) { + std::vector adv; + append_mfr_record(adv, 0x004C, beacon_payload(0x10, 0x15)); + auto device = device_from(adv); + EXPECT_FALSE(device.get_ibeacon().has_value()); +} + +TEST(BleIBeacon, GetIbeaconEmptyWithoutManufacturerData) { + std::vector adv; + adv.push_back(0x02); // flags record only + adv.push_back(0x01); + adv.push_back(0x06); + auto device = device_from(adv); + EXPECT_FALSE(device.get_ibeacon().has_value()); +} + +} // namespace esphome::ble_device_base::testing diff --git a/tests/components/ble_device_base/test_scan_mode_request.cpp b/tests/components/ble_device_base/test_scan_mode_request.cpp new file mode 100644 index 0000000000..9125eb6f2f --- /dev/null +++ b/tests/components/ble_device_base/test_scan_mode_request.cpp @@ -0,0 +1,76 @@ +#include + +#include + +#include "esphome/components/ble_device_base/ble_hub.h" + +namespace esphome::ble_device_base::testing { + +// Pins the request_scan_mode() contract: the base default refuses (so hubs +// without a mode switch — and out-of-tree trackers — keep building and +// callers report the real state), while an overriding hub both honors the +// request and applies it. +namespace { + +class DefaultHub : public BLEHub { + public: + void register_listener(ESPBTDeviceListener *listener) override {} + void set_raw_advertisement_callback(RawAdvertisementCallback callback) override {} + HubCapabilities get_capabilities() const override { return {false, false, false}; } + void get_adapter_mac(uint8_t out[6]) override {} + bool scan_running() override { return false; } + // Backed by real state so "changes nothing" is observable: a base default + // that silently mutated the hub would flip this and fail the assertion. + bool scan_active() override { return this->active_; } + + protected: + bool active_{true}; +}; + +class SwitchingHub : public DefaultHub { + public: + HubCapabilities get_capabilities() const override { return {true, false, false, /* scan_mode_switch = */ true}; } + bool request_scan_mode(bool active) override { + this->active_ = active; + return true; + } +}; + +// The esp32 shape: the controller supports active scanning but the hub keeps +// the refusing default (mode is driven through its own tracker API). +class CapableRefusingHub : public DefaultHub { + public: + HubCapabilities get_capabilities() const override { return {true, false, false}; } +}; + +} // namespace + +TEST(BLEHubScanModeRequest, DefaultRefusesAndChangesNothing) { + DefaultHub hub; + EXPECT_TRUE(hub.scan_active()); + EXPECT_FALSE(hub.request_scan_mode(false)); + // Refused, not applied-and-reported-false: the state is untouched. + EXPECT_TRUE(hub.scan_active()); + EXPECT_FALSE(hub.request_scan_mode(true)); + EXPECT_TRUE(hub.scan_active()); +} + +TEST(BLEHubScanModeRequest, OverrideHonorsAndApplies) { + SwitchingHub hub; + EXPECT_TRUE(hub.get_capabilities().scan_mode_switch); + EXPECT_TRUE(hub.request_scan_mode(true)); + EXPECT_TRUE(hub.scan_active()); + EXPECT_TRUE(hub.request_scan_mode(false)); + EXPECT_FALSE(hub.scan_active()); +} + +TEST(BLEHubScanModeRequest, CapabilityAndSwitchAreIndependent) { + CapableRefusingHub hub; + EXPECT_TRUE(hub.get_capabilities().active_scan); + // The esp32 shape advertises no runtime switch, and the request refuses. + EXPECT_FALSE(hub.get_capabilities().scan_mode_switch); + EXPECT_FALSE(hub.request_scan_mode(false)); + EXPECT_TRUE(hub.scan_active()); +} + +} // namespace esphome::ble_device_base::testing diff --git a/tests/components/bluetooth_proxy/test.ln882x-ard.yaml b/tests/components/bluetooth_proxy/test.ln882x-ard.yaml new file mode 100644 index 0000000000..ae1aed7c22 --- /dev/null +++ b/tests/components/bluetooth_proxy/test.ln882x-ard.yaml @@ -0,0 +1,10 @@ +# Advertisement-only proxy on the ln882x BLE hub (active-scan-capable, in-tree +# since #16691) — a target CI fully compiles. Same bare-hub arrangement as +# test.rp2040-ard.yaml: no explicit ble_hub_id so a grouped build cannot +# collide with ln882h_ble_tracker's own fixture id. +packages: + common: !include common.yaml + +ln882h_ble_tracker: + +bluetooth_proxy: diff --git a/tests/components/bluetooth_proxy/test.rp2040-ard.yaml b/tests/components/bluetooth_proxy/test.rp2040-ard.yaml new file mode 100644 index 0000000000..fd327bcc78 --- /dev/null +++ b/tests/components/bluetooth_proxy/test.rp2040-ard.yaml @@ -0,0 +1,13 @@ +# Advertisement-only proxy on the rp2 BLE hub — the one non-esp32 platform the +# proxy admits today (active-scan-capable), and a target CI fully compiles. +# No explicit ble_hub_id: the generated binding resolves the single declared +# hub, and an inline id here would collide with rp2_ble_tracker's own fixture +# once CI merges both components into one grouped rp2040-ard build (grouped +# component dicts collapse; only one id survives). The explicit-key form is +# covered by validate.rp2040-ard.yaml, which never participates in grouping. +packages: + common: !include common.yaml + +rp2_ble_tracker: + +bluetooth_proxy: diff --git a/tests/components/bluetooth_proxy/validate.esp32-idf.yaml b/tests/components/bluetooth_proxy/validate.esp32-idf.yaml new file mode 100644 index 0000000000..a92716ebeb --- /dev/null +++ b/tests/components/bluetooth_proxy/validate.esp32-idf.yaml @@ -0,0 +1,13 @@ +# Connections given as a bare list, with no explicit per-entry id. The ids are +# generated during validation, so this config breaks if the schema validates the +# connections list more than once. +packages: + common: !include common.yaml + +esp32_ble_tracker: + +bluetooth_proxy: + active: true + connections: + - {} + - {} diff --git a/tests/components/bluetooth_proxy/validate.rp2040-ard.yaml b/tests/components/bluetooth_proxy/validate.rp2040-ard.yaml new file mode 100644 index 0000000000..fa385dd5bc --- /dev/null +++ b/tests/components/bluetooth_proxy/validate.rp2040-ard.yaml @@ -0,0 +1,11 @@ +# Explicit ble_hub_id on the rp2 hub — the documented disambiguator once a +# platform has more than one tracker. Validate-only: never merged into grouped +# builds, so the inline id cannot collide with rp2_ble_tracker's own fixture. +packages: + common: !include common.yaml + +rp2_ble_tracker: + id: ble_hub + +bluetooth_proxy: + ble_hub_id: ble_hub diff --git a/tests/components/core/helpers_test.cpp b/tests/components/core/helpers_test.cpp index 468185787f..a9a940392f 100644 --- a/tests/components/core/helpers_test.cpp +++ b/tests/components/core/helpers_test.cpp @@ -55,4 +55,32 @@ TEST(HelpersTest, Ilog10RoundTripMatchesLog10) { } } +TEST(StaticVectorTest, ConvertingConstructorFromSmaller) { + StaticVector small{0x03, 0x00, 0x10, 0x00, 0x01}; + StaticVector big = small; + ASSERT_EQ(big.size(), small.size()); + for (size_t i = 0; i < small.size(); i++) { + EXPECT_EQ(big[i], small[i]) << "mismatch at index " << i; + } +} + +TEST(StaticVectorTest, ConvertingConstructorPartiallyFilledAndEmpty) { + StaticVector partial{0xAA, 0xBB}; + StaticVector from_partial = partial; + ASSERT_EQ(from_partial.size(), 2u); + EXPECT_EQ(from_partial[0], 0xAA); + EXPECT_EQ(from_partial[1], 0xBB); + + StaticVector empty; + StaticVector from_empty = empty; + EXPECT_TRUE(from_empty.empty()); +} + +TEST(StaticVectorTest, ConvertingConstructorSameSize) { + StaticVector src{1, 2, 3}; + StaticVector dst = src; + ASSERT_EQ(dst.size(), 3u); + EXPECT_EQ(dst[2], 3); +} + } // namespace esphome diff --git a/tests/components/core/test_event_pool.cpp b/tests/components/core/test_event_pool.cpp index af54ac3e14..da13924c65 100644 --- a/tests/components/core/test_event_pool.cpp +++ b/tests/components/core/test_event_pool.cpp @@ -70,4 +70,67 @@ TEST(EventPool, ReleaseNullptrIsSafe) { EXPECT_NE(pool.allocate(), nullptr); } +TEST(EventPool, WarmFullyPopulatesThePool) { + // warm()'s guarantee is invisible at runtime: no later allocate() may touch + // malloc(). Fully populated means SIZE allocations succeed from the free + // list and the SIZE + 1-th refuses. + esphome::EventPool pool; + ASSERT_TRUE(pool.warm()); + PoolItem *items[4]; + for (auto *&item : items) { + item = pool.allocate(); + ASSERT_NE(item, nullptr); + } + EXPECT_EQ(pool.allocate(), nullptr); +} + +TEST(EventPool, WarmIsIdempotent) { + esphome::EventPool pool; + ASSERT_TRUE(pool.warm()); + ASSERT_TRUE(pool.warm()); + // Still exactly SIZE objects: no growth past capacity. + PoolItem *items[3]; + for (auto *&item : items) { + item = pool.allocate(); + ASSERT_NE(item, nullptr); + } + EXPECT_EQ(pool.allocate(), nullptr); +} + +TEST(EventPool, AllocateAfterWarmRecyclesTheWarmedObjects) { + // The objects handed out after warm() are the ones warm() created, + // recycled rather than re-created. + esphome::EventPool pool; + ASSERT_TRUE(pool.warm()); + std::set first_round; + PoolItem *items[4]; + for (auto *&item : items) { + item = pool.allocate(); + first_round.insert(item); + } + for (auto *item : items) + pool.release(item); + for (int i = 0; i < 4; i++) { + PoolItem *item = pool.allocate(); + ASSERT_NE(item, nullptr); + EXPECT_TRUE(first_round.count(item) == 1); + } +} + +TEST(EventPool, WarmTopsUpWithEntriesOutstanding) { + // warm() counts existing entries (free or checked out) instead of failing + // when some are outstanding: it tops the pool up from any state. + esphome::EventPool pool; + PoolItem *held = pool.allocate(); + ASSERT_NE(held, nullptr); + ASSERT_TRUE(pool.warm()); + // The held object plus three more accounts for all SIZE entries. + PoolItem *items[3]; + for (auto *&item : items) { + item = pool.allocate(); + ASSERT_NE(item, nullptr); + } + EXPECT_EQ(pool.allocate(), nullptr); +} + } // namespace esphome::core::testing diff --git a/tests/components/core/test_string_ref.cpp b/tests/components/core/test_string_ref.cpp new file mode 100644 index 0000000000..bcbd0aa0d4 --- /dev/null +++ b/tests/components/core/test_string_ref.cpp @@ -0,0 +1,62 @@ +#include + +#include "esphome/core/string_ref.h" + +namespace esphome::core::testing { + +TEST(StringRefStartsWith, ProperPrefixMatches) { + StringRef ref("FR:R20:12345", 12); + EXPECT_TRUE(ref.starts_with("FR:")); +} + +TEST(StringRefStartsWith, WholeStringIsAPrefixOfItself) { + StringRef ref("TP96", 4); + EXPECT_TRUE(ref.starts_with("TP96")); +} + +TEST(StringRefStartsWith, PrefixLongerThanViewFails) { + StringRef ref("TP", 2); + EXPECT_FALSE(ref.starts_with("TP96")); +} + +TEST(StringRefStartsWith, DifferentContentFails) { + StringRef ref("TP96", 4); + EXPECT_FALSE(ref.starts_with("FR:")); +} + +TEST(StringRefStartsWith, EmptyPrefixAlwaysMatches) { + StringRef ref("abc", 3); + EXPECT_TRUE(ref.starts_with("")); + StringRef empty; + EXPECT_TRUE(empty.starts_with("")); +} + +TEST(StringRefStartsWith, EmptyViewOnlyMatchesEmptyPrefix) { + StringRef empty; + EXPECT_FALSE(empty.starts_with("a")); +} + +TEST(StringRefStartsWith, WorksOnANonTerminatedBuffer) { + // The reason the helper exists: a bounded view over a buffer with no + // terminator anywhere near the viewed bytes. + const char raw[] = {'R', 'a', 'd', 'o', 'n', 'X'}; + StringRef ref(raw, 5); + EXPECT_TRUE(ref.starts_with("Radon")); + EXPECT_FALSE(ref.starts_with("RadonEye")); + EXPECT_FALSE(ref.starts_with("adon")); +} + +TEST(StringRefStartsWith, StdStringOverload) { + StringRef ref("TP96", 4); + EXPECT_TRUE(ref.starts_with(std::string("TP"))); + EXPECT_FALSE(ref.starts_with(std::string("96"))); +} + +TEST(StringRefStartsWith, RefOverloadComparesOnlyTheViewedLength) { + // The prefix is a bounded view: bytes past its length must not be compared. + StringRef ref("FR:123", 6); + StringRef prefix("FR:xyz", 3); + EXPECT_TRUE(ref.starts_with(prefix)); +} + +} // namespace esphome::core::testing diff --git a/tests/components/epaper_spi/common.h b/tests/components/epaper_spi/common.h new file mode 100644 index 0000000000..5ac12afa6a --- /dev/null +++ b/tests/components/epaper_spi/common.h @@ -0,0 +1,50 @@ +#pragma once + +#include + +#include "esphome/components/spi/spi.h" +#include "esphome/core/hal.h" + +namespace esphome::epaper_spi::testing { + +/// SPI delegate that records transaction boundaries and burns wall-clock time on each +/// row write, so a transfer can be driven past its MAX_TRANSFER_TIME yield deadline. +class TimedSPIDelegate : public spi::SPIDelegate { + public: + explicit TimedSPIDelegate(uint32_t row_transfer_ms) : row_transfer_ms_(row_transfer_ms) {} + + uint8_t transfer(uint8_t data) override { return 0; } + + void write_array(const uint8_t *ptr, size_t length) override { + // A row of pixel data is one "slow" write; single-byte writes are commands. + if (length > 1) { + const uint32_t until = millis() + this->row_transfer_ms_; + while (millis() < until) { + } + } + } + + void begin_transaction() override { this->begin_count++; } + void end_transaction() override { this->end_count++; } + + int begin_count{0}; + int end_count{0}; + + protected: + uint32_t row_transfer_ms_; +}; + +/// GPIO pin that just remembers the last level written to it. +class RecordingPin : public GPIOPin { + public: + void setup() override {} + void pin_mode(gpio::Flags flags) override {} + gpio::Flags get_flags() const override { return gpio::Flags::FLAG_NONE; } + bool digital_read() override { return false; } + void digital_write(bool value) override { this->level = value; } + size_t dump_summary(char *buffer, size_t len) const override { return snprintf(buffer, len, "recording"); } + + bool level{true}; +}; + +} // namespace esphome::epaper_spi::testing diff --git a/tests/components/epaper_spi/display/test_t133a01_transfer.cpp b/tests/components/epaper_spi/display/test_t133a01_transfer.cpp new file mode 100644 index 0000000000..5c7abdc022 --- /dev/null +++ b/tests/components/epaper_spi/display/test_t133a01_transfer.cpp @@ -0,0 +1,77 @@ +#include + +#include "../common.h" +#include "esphome/components/epaper_spi/epaper_spi_t133a01.h" + +namespace esphome::epaper_spi::testing { + +/// Exposes the protected transfer machinery so the yield behaviour can be driven directly. +class TestableT133A01 : public EPaperT133A01 { + public: + TestableT133A01(uint16_t width, uint16_t height) : EPaperT133A01("test", width, height, nullptr, 0) {} + + void install(spi::SPIDelegate *delegate) { + this->delegate_ = delegate; + this->set_dc_pin(&this->dc); + this->set_cs_pins(&this->cs, &this->cs1); + ASSERT_TRUE(this->init_buffer_(this->buffer_length_)); + } + + using EPaperT133A01::transfer_data; + + RecordingPin dc, cs, cs1; +}; + +/// Regression test for the T133A01 transfer deadlock (issue #17668). +/// +/// `transfer_data()` evaluates its yield deadline *after* incrementing the row counter, so the +/// deadline can expire on a phase's final row. The phase is then complete but the function +/// reports "not done"; on the next call the phase guard is false, so the `disable()` / +/// CS-deassert epilogue is skipped permanently. The SPI transaction is never closed and the +/// next `enable()` blocks forever, tripping the task watchdog. +/// +/// Here the CS phase is two rows and every row write overruns the deadline, so the second call +/// completes the phase exactly as the deadline expires, which is the failing alignment. The completed +/// phase must still run its epilogue: end the transaction and deassert CS. +TEST(EPaperT133A01, CompletedPhaseRunsEpilogueWhenDeadlineExpiresOnFinalRow) { + // width 8 -> 4 bytes per row, 2 per half-row; height 2 -> a two-row CS phase + TestableT133A01 display(8, 2); + TimedSPIDelegate delegate(MAX_TRANSFER_TIME + 5); + display.install(&delegate); + + // First call performs the one-off CCSET setup (which opens and closes a transaction of its + // own) and then writes row 0 of the CS phase before yielding on the deadline. + ASSERT_FALSE(display.transfer_data()) << "transfer should have yielded after the first row"; + ASSERT_FALSE(display.cs.level) << "CS must stay asserted across a yield mid-phase"; + const int closed_after_setup = delegate.end_count; + + // Second call writes the final row of the CS phase; the deadline expires as it lands. + display.transfer_data(); + + EXPECT_EQ(delegate.end_count, closed_after_setup + 1) + << "completed CS phase skipped disable() -- SPI transaction left open"; + EXPECT_TRUE(display.cs.level) << "completed CS phase left CS asserted"; +} + +/// The CS1 phase has the same off-by-one, but fails worse: after the skipped epilogue the +/// function falls through to `return true`, reporting the transfer complete while the SPI +/// transaction is still open and CS1 is still asserted. The next command's `enable()` then +/// blocks forever. A transfer that reports done must have released the bus. +TEST(EPaperT133A01, TransferReportsDoneOnlyAfterReleasingTheBus) { + TestableT133A01 display(8, 2); + TimedSPIDelegate delegate(MAX_TRANSFER_TIME + 5); + display.install(&delegate); + + // Both phases are two rows each and every row overruns the deadline, so the transfer needs + // one call per row plus the setup call. Bound the loop so a regression fails rather than hangs. + int calls = 0; + while (!display.transfer_data()) { + ASSERT_LT(++calls, 10) << "transfer never reported completion"; + } + + EXPECT_TRUE(display.cs1.level) << "transfer reported done with CS1 still asserted"; + EXPECT_EQ(delegate.begin_count, delegate.end_count) + << "transfer reported done with an SPI transaction still open -- the next enable() would deadlock"; +} + +} // namespace esphome::epaper_spi::testing diff --git a/tests/components/esp32/test-signed_ota_external.esp32-s3-idf.yaml b/tests/components/esp32/test-signed_ota_external.esp32-s3-idf.yaml new file mode 100644 index 0000000000..fab1d922fc --- /dev/null +++ b/tests/components/esp32/test-signed_ota_external.esp32-s3-idf.yaml @@ -0,0 +1,23 @@ +# External RSA signing mode with a declared trusted-key list enables ESPHome's +# own multi-key OTA signature verifier (USE_OTA_SIGNED_VERIFICATION_MULTI_KEY), +# which accepts an image whose signature block matches one of the compiled-in +# trusted keys. wifi + ota pull in the ota component so CI actually compiles that +# verifier; allow_partition_access exercises the bootloader-update path too. +esp32: + variant: esp32s3 + framework: + type: esp-idf + advanced: + signed_ota_verification: + verification_keys: + - ../../components/esp32/dummy_signing_key.pem + +wifi: + ssid: MySSID + password: password1 + +ota: + - platform: esphome + allow_partition_access: true + +<<: !include common.yaml diff --git a/tests/components/esp32/validate-signed_ota_external.esp32-s3-idf.yaml b/tests/components/esp32/validate-signed_ota_external.esp32-s3-idf.yaml deleted file mode 100644 index 5b57993e87..0000000000 --- a/tests/components/esp32/validate-signed_ota_external.esp32-s3-idf.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Secure Boot V2 schemes carry the public key inside each image's signature -# block, so verifying externally-signed binaries needs no key in the config: -# a bare block enables verification with the default rsa3072 scheme. -esp32: - variant: esp32s3 - framework: - type: esp-idf - advanced: - signed_ota_verification: - -<<: !include common.yaml diff --git a/tests/components/esp32_ble_tracker/common.yaml b/tests/components/esp32_ble_tracker/common.yaml index 564cf1f6ea..9c880dbf1a 100644 --- a/tests/components/esp32_ble_tracker/common.yaml +++ b/tests/components/esp32_ble_tracker/common.yaml @@ -12,18 +12,21 @@ esp32_ble_tracker: then: # yamllint disable rule:line-length - lambda: !lambda |- - ESP_LOGD("main", "The device address (%s) exists in list", x.address_str().c_str()); + char addr[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; + ESP_LOGD("main", "The device address (%s) exists in list", x.address_str_to(addr)); # yamllint enable rule:line-length - mac_address: AC:37:43:77:5F:4C then: # yamllint disable rule:line-length - lambda: !lambda |- - ESP_LOGD("main", "The device address is %s", x.address_str().c_str()); + char addr[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; + ESP_LOGD("main", "The device address is %s", x.address_str_to(addr)); # yamllint enable rule:line-length - then: # yamllint disable rule:line-length - lambda: !lambda |- - ESP_LOGD("main", "The device address is %s", x.address_str().c_str()); + char addr[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; + ESP_LOGD("main", "The device address is %s", x.address_str_to(addr)); # yamllint enable rule:line-length on_ble_service_data_advertise: - service_uuid: ABCD diff --git a/tests/components/esp32_rmt_led_strip/common.yaml b/tests/components/esp32_rmt_led_strip/common.yaml index f3ee86bcce..701e513ebd 100644 --- a/tests/components/esp32_rmt_led_strip/common.yaml +++ b/tests/components/esp32_rmt_led_strip/common.yaml @@ -9,7 +9,7 @@ light: id: led_strip2 pin: ${pin2} num_leds: 60 - rgb_order: RGB + rgbw_order: RWGB bit0_high: 100us bit0_low: 100us bit1_high: 100us diff --git a/tests/components/espnow/common-wifi.yaml b/tests/components/espnow/common-wifi.yaml new file mode 100644 index 0000000000..5ffa9dd44b --- /dev/null +++ b/tests/components/espnow/common-wifi.yaml @@ -0,0 +1,9 @@ +wifi: + ssid: MySSID + password: password1 + +espnow: + id: espnow_component + auto_add_peer: true + peers: + - 11:22:33:44:55:66 diff --git a/tests/components/espnow/test-wifi.esp32-idf.yaml b/tests/components/espnow/test-wifi.esp32-idf.yaml new file mode 100644 index 0000000000..c45547cd53 --- /dev/null +++ b/tests/components/espnow/test-wifi.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + espnow: !include common-wifi.yaml diff --git a/tests/components/ld6002b/common.yaml b/tests/components/ld6002b/common.yaml new file mode 100644 index 0000000000..15ab06c394 --- /dev/null +++ b/tests/components/ld6002b/common.yaml @@ -0,0 +1,50 @@ +ld6002b: + id: ld6002b_radar + wakeup_pin: GPIO14 + +sensor: + - platform: ld6002b + ld6002b_id: ld6002b_radar + target_count: + name: Target Count + target_1: + x: + name: Target-1 X + y: + name: Target-1 Y + z: + name: Target-1 Z + doppler_index: + name: Target-1 Dop + cluster_id: + name: Target-1 Cluster + target_2: + x: + name: Target-2 X + y: + name: Target-2 Y + z: + name: Target-2 Z + doppler_index: + name: Target-2 Dop + cluster_id: + name: Target-2 Cluster + target_3: + x: + name: Target-3 X + y: + name: Target-3 Y + z: + name: Target-3 Z + doppler_index: + name: Target-3 Dop + cluster_id: + name: Target-3 Cluster + +binary_sensor: + - platform: ld6002b + ld6002b_id: ld6002b_radar + target: + name: Presence + target_1: + name: Target-1 Presence diff --git a/tests/components/ld6002b/test.esp32-idf.yaml b/tests/components/ld6002b/test.esp32-idf.yaml new file mode 100644 index 0000000000..d26ef5c348 --- /dev/null +++ b/tests/components/ld6002b/test.esp32-idf.yaml @@ -0,0 +1,3 @@ +packages: + uart_115200: !include ../../test_build_components/common/uart_115200/esp32-idf.yaml + ld6002b: !include common.yaml diff --git a/tests/components/ld6002b/test.esp8266-ard.yaml b/tests/components/ld6002b/test.esp8266-ard.yaml new file mode 100644 index 0000000000..8846b4ab50 --- /dev/null +++ b/tests/components/ld6002b/test.esp8266-ard.yaml @@ -0,0 +1,3 @@ +packages: + uart_115200: !include ../../test_build_components/common/uart_115200/esp8266-ard.yaml + ld6002b: !include common.yaml diff --git a/tests/components/ld6002b/test.rp2040-ard.yaml b/tests/components/ld6002b/test.rp2040-ard.yaml new file mode 100644 index 0000000000..4edcb6965e --- /dev/null +++ b/tests/components/ld6002b/test.rp2040-ard.yaml @@ -0,0 +1,3 @@ +packages: + uart_115200: !include ../../test_build_components/common/uart_115200/rp2040-ard.yaml + ld6002b: !include common.yaml diff --git a/tests/components/ln882h_ble_tracker/common-boundary.yaml b/tests/components/ln882h_ble_tracker/common-boundary.yaml new file mode 100644 index 0000000000..b6df6f6f39 --- /dev/null +++ b/tests/components/ln882h_ble_tracker/common-boundary.yaml @@ -0,0 +1,12 @@ +ln882h_ble_tracker: + id: ble_tracker + scan_parameters: + # Boundary coverage: the documented 2.5 ms floor on window (expressible only + # via the microsecond-accurate validation), a non-round interval exercising the + # 0.625 ms unit conversion without collapsing onto the window's unit count, + # and the non-continuous config path. + interval: 5000us + window: 2500us + duration: 5min + active: false + continuous: false diff --git a/tests/components/ln882h_ble_tracker/common.yaml b/tests/components/ln882h_ble_tracker/common.yaml new file mode 100644 index 0000000000..aba02147c8 --- /dev/null +++ b/tests/components/ln882h_ble_tracker/common.yaml @@ -0,0 +1,16 @@ +ln882h_ble_tracker: + id: ble_tracker + scan_parameters: + interval: 100ms + window: 50ms + duration: 5min + continuous: true + +# Pulls in USE_OTA_STATE_LISTENER so the OTA scan-pause path compiles in CI +# (same coverage arrangement as the rp2_ble_tracker tests). +wifi: + ssid: MySSID + password: password1 + +ota: + - platform: esphome diff --git a/tests/components/ln882h_ble_tracker/test-automations.ln882x-ard.yaml b/tests/components/ln882h_ble_tracker/test-automations.ln882x-ard.yaml new file mode 100644 index 0000000000..3cd3ce28b2 --- /dev/null +++ b/tests/components/ln882h_ble_tracker/test-automations.ln882x-ard.yaml @@ -0,0 +1,34 @@ +packages: + ln882h_ble_tracker: !include common.yaml + +esphome: + on_boot: + then: + - ln882h_ble_tracker.start_scan + - ln882h_ble_tracker.start_scan: + continuous: true + - ln882h_ble_tracker.start_scan: + continuous: !lambda return false; + - ln882h_ble_tracker.stop_scan + +ln882h_ble_tracker: + on_ble_advertise: + - mac_address: AC:37:43:77:5F:4C + then: + - lambda: |- + char addr[MAC_ADDRESS_PRETTY_BUFFER_SIZE]; + ESP_LOGD("main", "The device address is %s", x.address_str_to(addr)); + on_ble_service_data_advertise: + - service_uuid: ABCD + then: + - lambda: |- + ESP_LOGD("main", "Length of service data is %zu", x.size()); + on_ble_manufacturer_data_advertise: + - manufacturer_id: ABCD + then: + - lambda: |- + ESP_LOGD("main", "Length of manufacturer data is %zu", x.size()); + on_scan_end: + - then: + - lambda: |- + ESP_LOGD("main", "Scan ended"); diff --git a/tests/components/ln882h_ble_tracker/test.ln882x-ard.yaml b/tests/components/ln882h_ble_tracker/test.ln882x-ard.yaml new file mode 100644 index 0000000000..6a9efad314 --- /dev/null +++ b/tests/components/ln882h_ble_tracker/test.ln882x-ard.yaml @@ -0,0 +1,2 @@ +packages: + ln882h_ble_tracker: !include common.yaml diff --git a/tests/components/ln882h_ble_tracker/validate-boundary.ln882x-ard.yaml b/tests/components/ln882h_ble_tracker/validate-boundary.ln882x-ard.yaml new file mode 100644 index 0000000000..fc5790e3b2 --- /dev/null +++ b/tests/components/ln882h_ble_tracker/validate-boundary.ln882x-ard.yaml @@ -0,0 +1,2 @@ +packages: + ln882h_ble_tracker: !include common-boundary.yaml diff --git a/tests/components/logger/test-uart0_no_logging.esp8266-ard.yaml b/tests/components/logger/test-uart0_no_logging.esp8266-ard.yaml new file mode 100644 index 0000000000..76444a2e89 --- /dev/null +++ b/tests/components/logger/test-uart0_no_logging.esp8266-ard.yaml @@ -0,0 +1 @@ +<<: !include common-uart0_no_logging.yaml diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index ef8ba13e42..46c1fd362a 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -174,7 +174,8 @@ lvgl: - id: anim_color duration: 2s timing: - - round_trip + - type: round_trip + pause: 0.5 - type: gravity bounce: 0.3 acceleration: 0.8 @@ -756,7 +757,25 @@ lvgl: on_defocus: lvgl.widget.hide: hello_label on_focus: - logger.log: Button clicked + - logger.log: Button clicked + - lvgl.widget.set_z_index: + id: hello_label + position: top + - lvgl.widget.set_z_index: + id: hello_label + position: bottom + - lvgl.widget.set_z_index: + id: hello_label + position: up + - lvgl.widget.set_z_index: + id: hello_label + position: down + - lvgl.widget.set_z_index: + id: hello_label + position: 1 + - lvgl.widget.set_z_index: + id: hello_label + position: -1 on_scroll: logger.log: Button clicked on_scroll_end: @@ -1539,17 +1558,20 @@ font: image: - id: cat_image + platform: file resize: 256x48 file: $component_dir/logo-text.svg type: RGB565 transparency: alpha_channel - id: dog_image + platform: file file: $component_dir/logo-text.svg resize: 256x48 type: BINARY transparency: chroma_key - id: alert + platform: file file: $component_dir/logo-text.svg type: grayscale resize: 100x100 diff --git a/tests/components/lvgl/test.host.yaml b/tests/components/lvgl/test.host.yaml index 90cbb3c0a5..3fa54fa3d6 100644 --- a/tests/components/lvgl/test.host.yaml +++ b/tests/components/lvgl/test.host.yaml @@ -39,7 +39,7 @@ lvgl: timing: - round_trip - type: ease_in_out - weight: 3 + weight: 0.5 on_start: - logger.log: anim started on_stop: diff --git a/tests/components/mcp4461/common.yaml b/tests/components/mcp4461/common.yaml index 71e2528aa4..8accc2ea54 100644 --- a/tests/components/mcp4461/common.yaml +++ b/tests/components/mcp4461/common.yaml @@ -3,30 +3,61 @@ mcp4461: i2c_id: i2c_bus output: + # All-terminals-off coverage lives here (folded from a former second channel-A + # output — one output per channel keeps the reg_ state deterministic). - platform: mcp4461 id: digipot_wiper_1 mcp4461_id: mcp4461_digipot_01 channel: A - - - platform: mcp4461 - id: digipot_wiper_2 - mcp4461_id: mcp4461_digipot_01 - channel: B - - - platform: mcp4461 - id: digipot_wiper_3 - mcp4461_id: mcp4461_digipot_01 - channel: C - - - platform: mcp4461 - id: digipot_wiper_4 - mcp4461_id: mcp4461_digipot_01 - channel: D - - - platform: mcp4461 - id: digipot_wiper_5 - mcp4461_id: mcp4461_digipot_01 - channel: A terminal_a: false terminal_b: false terminal_w: false + + - platform: mcp4461 + id: digipot_wiper_2 + mcp4461_id: mcp4461_digipot_01 + channel: B + nonvolatile: false + + - platform: mcp4461 + id: digipot_wiper_3 + mcp4461_id: mcp4461_digipot_01 + channel: C + nonvolatile_write_delay: 5s + initial_value: 0.5 + + # TCON1 coverage: terminal flags on a channel D output exercise the + # calc_terminal_connector_byte_() write path for wipers 2/3. + - platform: mcp4461 + id: digipot_wiper_4 + mcp4461_id: mcp4461_digipot_01 + channel: D + terminal_a: false + terminal_w: false + + # Bare NV-channel output — the pre-existing persistence workaround; must + # keep validating without any nonvolatile key (regression: schema default + # used to materialize the key on every channel and fail final validation). + - platform: mcp4461 + id: digipot_nv_wiper_1 + mcp4461_id: mcp4461_digipot_01 + channel: E + + # Explicit opt-out on an NV channel is a harmless no-op and stays valid. + - platform: mcp4461 + id: digipot_nv_wiper_2 + mcp4461_id: mcp4461_digipot_01 + channel: F + nonvolatile: false + +button: + - platform: template + name: "Digipot test actions" + on_press: + - mcp4461.wiper.increase: digipot_wiper_1 + - mcp4461.wiper.decrease: digipot_wiper_1 + - mcp4461.wiper.store_nonvolatile: digipot_wiper_2 + - mcp4461.wiper.set_terminal: + id: digipot_wiper_1 + terminal: a + enable: false diff --git a/tests/components/micro_wake_word/validate.esp32-idf.yaml b/tests/components/micro_wake_word/validate.esp32-idf.yaml new file mode 100644 index 0000000000..d87b19bdcf --- /dev/null +++ b/tests/components/micro_wake_word/validate.esp32-idf.yaml @@ -0,0 +1,21 @@ +# Config-only test: micro_wake_word without any compiled-in models. Covers the optional models +# schema, which validates without a model list. Wake word models are added at runtime instead, +# which voice_assistant wires up. +substitutions: + mic_din_pin: GPIO36 + +packages: + i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-idf.yaml + +microphone: + - platform: i2s_audio + id: echo_microphone + i2s_audio_id: i2s_audio_bus + i2s_din_pin: ${mic_din_pin} + adc_type: external + pdm: true + bits_per_sample: 16bit + +micro_wake_word: + microphone: echo_microphone + # models is omitted entirely, so the default empty list applies diff --git a/tests/components/midea/common.yaml b/tests/components/midea/common.yaml index c7b18a6701..25fc2debcd 100644 --- a/tests/components/midea/common.yaml +++ b/tests/components/midea/common.yaml @@ -1,7 +1,3 @@ -wifi: - ssid: MySSID - password: password1 - climate: - platform: midea id: midea_unit diff --git a/tests/components/midea/test.esp32-ard.yaml b/tests/components/midea/test.esp32-ard.yaml index 1e3fe0ff51..17ced80477 100644 --- a/tests/components/midea/test.esp32-ard.yaml +++ b/tests/components/midea/test.esp32-ard.yaml @@ -1,5 +1,8 @@ packages: remote_transmitter: !include ../../test_build_components/common/remote_transmitter/esp32-ard.yaml uart: !include ../../test_build_components/common/uart/esp32-ard.yaml + midea: !include common.yaml -<<: !include common.yaml +wifi: + ssid: MySSID + password: password1 diff --git a/tests/components/midea/test.esp32-h2-idf.yaml b/tests/components/midea/test.esp32-h2-idf.yaml new file mode 100644 index 0000000000..45b73dc6c7 --- /dev/null +++ b/tests/components/midea/test.esp32-h2-idf.yaml @@ -0,0 +1,5 @@ +# ESP32-H2 has no WiFi PHY; this verifies the component builds without wifi +packages: + remote_transmitter: !include ../../test_build_components/common/remote_transmitter/esp32-idf.yaml + uart: !include ../../test_build_components/common/uart/esp32-h2-idf.yaml + midea: !include common.yaml diff --git a/tests/components/midea/test.esp32-idf.yaml b/tests/components/midea/test.esp32-idf.yaml new file mode 100644 index 0000000000..5ad22b5b93 --- /dev/null +++ b/tests/components/midea/test.esp32-idf.yaml @@ -0,0 +1,15 @@ +packages: + remote_transmitter: !include ../../test_build_components/common/remote_transmitter/esp32-idf.yaml + uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + midea: !include common.yaml + +wifi: + ssid: MySSID + password: password1 + +# Regression test for https://github.com/esphome/esphome/issues/18054: the +# MideaUART ESP-IDF shims must not make unqualified millis() ambiguous +interval: + - interval: 10s + then: + - lambda: ESP_LOGD("test", "%u", millis()); diff --git a/tests/components/midea/test.esp8266-ard.yaml b/tests/components/midea/test.esp8266-ard.yaml index 9825ff85a1..70a0b00105 100644 --- a/tests/components/midea/test.esp8266-ard.yaml +++ b/tests/components/midea/test.esp8266-ard.yaml @@ -1,5 +1,8 @@ packages: remote_transmitter: !include ../../test_build_components/common/remote_transmitter/esp8266-ard.yaml uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml + midea: !include common.yaml -<<: !include common.yaml +wifi: + ssid: MySSID + password: password1 diff --git a/tests/components/modbus/modbus_helpers_test.cpp b/tests/components/modbus/modbus_helpers_test.cpp index 49de4f9d14..553ec163b2 100644 --- a/tests/components/modbus/modbus_helpers_test.cpp +++ b/tests/components/modbus/modbus_helpers_test.cpp @@ -331,6 +331,7 @@ TEST(ModbusCreateClientPdu, WriteCoilsUseTheCoilLimitNotTheRegisterLimit) { EXPECT_TRUE(create_client_pdu(FC::WRITE_MULTIPLE_COILS, 0x0000, 1969, big.data(), big.size()).empty()); } +// --- payload_to_number ----------------------------------------------------- TEST(ModbusHelpersTest, PayloadToNumberRejectsOffsetAtEndOfBuffer) { const std::vector data{0x12, 0x34}; EXPECT_FALSE(payload_to_number(std::span(data), SensorValueType::U_WORD, 2, 0xFFFFFFFF).has_value()); @@ -346,6 +347,28 @@ TEST(ModbusHelpersTest, PayloadToNumberDecodesValidWord) { EXPECT_EQ(payload_to_number(std::span(data), SensorValueType::U_WORD, 0, 0xFFFFFFFF), 0x1234); } +TEST(ModbusHelpersTest, PayloadToNumberDecodesSwappedUnsignedWord) { + const std::vector data{0x34, 0x12}; + EXPECT_EQ(payload_to_number(std::span(data), SensorValueType::U_WORD_S, 0, 0xFFFFFFFF), 0x1234); +} + +TEST(ModbusHelpersTest, PayloadToNumberDecodesSwappedSignedWord) { + const std::vector data{0xFE, 0xFF}; + EXPECT_EQ(payload_to_number(std::span(data), SensorValueType::S_WORD_S, 0, 0xFFFFFFFF), -2); +} + +TEST(ModbusHelpersTest, PayloadToNumberAppliesBitmaskAfterSwap) { + // Bytes {0x34,0x12} decode as U_WORD_S to 0x1234; mask 0xFF00 then right-shift by bit 8 -> 0x12 + const std::vector data{0x34, 0x12}; + EXPECT_EQ(payload_to_number(std::span(data), SensorValueType::U_WORD_S, 0, 0xFF00), 0x12); +} + +TEST(ModbusHelpersTest, PayloadToNumberAppliesBitmaskAfterSwapSigned) { + // Bytes {0x34,0xFE} decode as S_WORD_S to 0xFE34 (negative); mask 0x00F0 then right-shift by bit 4 -> 0x3 + const std::vector data{0x34, 0xFE}; + EXPECT_EQ(payload_to_number(std::span(data), SensorValueType::S_WORD_S, 0, 0x00F0), 0x3); +} + // --- registers_to_number --------------------------------------------------- // Register words are host byte order; results must match the byte-based payload_to_number. @@ -354,6 +377,16 @@ TEST(ModbusHelpersTest, RegistersToNumberDecodesWord) { EXPECT_EQ(registers_to_number(registers, 1, SensorValueType::U_WORD), 0x1234); } +TEST(ModbusHelpersTest, RegistersToNumberDecodesSwappedUnsignedWord) { + const uint16_t registers[] = {0x3412}; + EXPECT_EQ(registers_to_number(registers, 1, SensorValueType::U_WORD_S), 0x1234); +} + +TEST(ModbusHelpersTest, RegistersToNumberDecodesSwappedSignedWord) { + const uint16_t registers[] = {0xFEFF}; + EXPECT_EQ(registers_to_number(registers, 1, SensorValueType::S_WORD_S), -2); +} + TEST(ModbusHelpersTest, RegistersToNumberDecodesDwordHighWordFirst) { const uint16_t registers[] = {0x1234, 0x5678}; EXPECT_EQ(registers_to_number(registers, 2, SensorValueType::U_DWORD), 0x12345678); @@ -434,6 +467,24 @@ TEST(ModbusTypedBuilders, FloatToPayloadAppendsToExistingContent) { EXPECT_EQ(data[1], 0x0001); } +// --- number_to_payload ----------------------------------------------------- + +TEST(ModbusHelpersTest, NumberToPayloadRoundTripsSwappedUnsignedWord) { + std::vector regs; + number_to_payload(regs, 0x1234, SensorValueType::U_WORD_S); + ASSERT_EQ(regs.size(), 1u); + EXPECT_EQ(regs[0], 0x3412); + EXPECT_EQ(registers_to_number(regs.data(), regs.size(), SensorValueType::U_WORD_S), 0x1234); +} + +TEST(ModbusHelpersTest, NumberToPayloadRoundTripsSwappedSignedWord) { + std::vector regs; + number_to_payload(regs, -2, SensorValueType::S_WORD_S); + ASSERT_EQ(regs.size(), 1u); + EXPECT_EQ(regs[0], 0xFEFF); + EXPECT_EQ(registers_to_number(regs.data(), regs.size(), SensorValueType::S_WORD_S), -2); +} + TEST(ModbusCreateClientPdu, ExceptionFlaggedWriteCodesRejected) { // is_function_code_write() masks the exception bit; the builder must not. const uint8_t values[] = {0x00, 0x0B, 0x00, 0x16}; diff --git a/tests/components/modbus_client/common.yaml b/tests/components/modbus_client/common.yaml new file mode 100644 index 0000000000..19e81f4e48 --- /dev/null +++ b/tests/components/modbus_client/common.yaml @@ -0,0 +1,53 @@ +# The modbus_client actions are self-contained hub devices: each takes the hub (auto-resolved when there +# is a single modbus client hub) and a templatable device address; no component block is needed. The +# address is not passed back to reply handlers - recompute the configured expression if needed. +# The hub does not bound retries, so a retry lambda must (here: a counter capped at 3), or a dead +# device is retried forever. Reset the counter before the send or on a terminal outcome (on_response) +# so the cap is per transaction, not per device lifetime. Never reset in on_sent: it fires again on +# every retry, so the cap would never be reached. +globals: + - id: read_retries + type: int + initial_value: "0" + - id: combined_retries + type: int + initial_value: "0" + +button: + - platform: template + name: "Send Read" + on_press: + - lambda: "id(read_retries) = 0;" + - modbus_client.send: + address: 0x01 + pdu: [0x03, 0x00, 0x10, 0x00, 0x01] + # on_no_response lambda form: return true to retry. `request` is the timed-out PDU. + on_no_response: !lambda "return !request.empty() && request[0] == 0x03 && id(read_retries)++ < 3;" + # Per-send inline reply handlers (fire-and-continue): they run when this send's outcome is known; + # the targeted address is not passed back - recompute the configured expression if needed. + # A pdu lambda can hand-assemble bytes or return a modbus::helpers::create_*_pdu() builder result. + - modbus_client.send: + address: 0x01 + pdu: !lambda "return modbus::helpers::create_read_pdu(modbus::FunctionCode::READ_HOLDING_REGISTERS, 0x0010, 1);" + - modbus_client.send: + address: !lambda "return 1;" + pdu: !lambda "return {0x03, 0x00, 0x10, 0x00, 0x01};" + on_sent: + then: + - lambda: 'ESP_LOGI("modbus_client.test", "sent fc 0x%X", request.empty() ? 0 : request[0]);' + on_response: + then: + - lambda: |- + id(combined_retries) = 0; + ESP_LOGI("modbus_client.test", "got %d bytes", (int) response.size()); + on_error: + then: + - lambda: 'ESP_LOGW("modbus_client.test", "fc 0x%X exception %d", request.empty() ? 0 : request[0], (int) exception_code);' + # on_no_response combined form: run actions on timeout AND decide the retry via nested retry:. + on_no_response: + then: + - lambda: 'ESP_LOGW("modbus_client.test", "no reply for fc 0x%X", request.empty() ? 0 : request[0]);' + retry: !lambda "return !request.empty() && request[0] == 0x03 && id(combined_retries)++ < 3;" + on_not_sent: + then: + - lambda: 'ESP_LOGW("modbus_client.test", "not sent fc 0x%X", request.empty() ? 0 : request[0]);' diff --git a/tests/components/modbus_client/test.esp32-idf.yaml b/tests/components/modbus_client/test.esp32-idf.yaml new file mode 100644 index 0000000000..b5882e90d8 --- /dev/null +++ b/tests/components/modbus_client/test.esp32-idf.yaml @@ -0,0 +1,3 @@ +packages: + modbus: !include ../../test_build_components/common/modbus/esp32-idf.yaml + modbus_client: !include common.yaml diff --git a/tests/components/modbus_client/test.esp8266-ard.yaml b/tests/components/modbus_client/test.esp8266-ard.yaml new file mode 100644 index 0000000000..151922b0d5 --- /dev/null +++ b/tests/components/modbus_client/test.esp8266-ard.yaml @@ -0,0 +1,3 @@ +packages: + modbus: !include ../../test_build_components/common/modbus/esp8266-ard.yaml + modbus_client: !include common.yaml diff --git a/tests/components/modbus_client/test.rp2040-ard.yaml b/tests/components/modbus_client/test.rp2040-ard.yaml new file mode 100644 index 0000000000..aaf115ae45 --- /dev/null +++ b/tests/components/modbus_client/test.rp2040-ard.yaml @@ -0,0 +1,3 @@ +packages: + modbus: !include ../../test_build_components/common/modbus/rp2040-ard.yaml + modbus_client: !include common.yaml diff --git a/tests/components/modbus_controller/common.yaml b/tests/components/modbus_controller/common.yaml index aa2855c2b0..67b022cdf5 100644 --- a/tests/components/modbus_controller/common.yaml +++ b/tests/components/modbus_controller/common.yaml @@ -41,6 +41,13 @@ number: return x * 2.0; write_lambda: |- return x / 2.0; + # Covers Python value-type maps + read/write path for byte-swapped words + - platform: modbus_controller + modbus_controller_id: modbus_controller1 + id: modbus_number3 + name: Test Number Swapped Word + address: 0x9003 + value_type: U_WORD_S output: - platform: modbus_controller @@ -118,6 +125,61 @@ sensor: value_type: U_WORD lambda: |- return x / 10.0; + # Non-mergeable sensor sharing the start address of modbus_sensor1 (different register_count): + # must join the same range, never open a second range keyed on the same (address, type). + - platform: modbus_controller + modbus_controller_id: modbus_controller1 + id: modbus_sensor_shared_addr + name: Test Sensor Shared Address + register_type: holding + address: 0x9001 + value_type: U_DWORD + # Sensors sharing one start address with distinct byte offsets (mixed register counts, so they take + # the shared-start path: each resolves to exactly its configured offset, no accumulation). + - platform: modbus_controller + modbus_controller_id: modbus_controller1 + id: modbus_sensor_shared_offs0 + name: Test Sensor Shared Offset Base + register_type: holding + address: 0x9020 + value_type: U_DWORD + - platform: modbus_controller + modbus_controller_id: modbus_controller1 + id: modbus_sensor_shared_offs1 + name: Test Sensor Shared Offset Low Word + register_type: holding + address: 0x9020 + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller1 + id: modbus_sensor_shared_offs2 + name: Test Sensor Shared Offset High Word + register_type: holding + address: 0x9020 + value_type: U_WORD + offset: 2 + # Raw-decode lambda kept on the deprecated get_data() helper on purpose: `data` is a span now, so this + # pins that the compatibility overload still accepts one. The deprecation warning it raises is the + # point - it is what a user on the old helper sees. `item->offset` locates this sensor's data. + - platform: modbus_controller + modbus_controller_id: modbus_controller1 + id: modbus_sensor_raw_lambda + name: Test Sensor Raw Lambda + register_type: holding + address: 0x9050 + value_type: U_WORD + lambda: |- + return modbus_controller::get_data(data, item->offset) * 0.1f; + # force_new_range sensors sort before plain ones, so this high-address forced sensor is grouped + # first and the lower-address plain sensors above must still get their own ranges. + - platform: modbus_controller + modbus_controller_id: modbus_controller1 + id: modbus_sensor_forced_high + name: Test Sensor Forced High Address + register_type: holding + address: 0x9040 + value_type: U_WORD + force_new_range: true switch: - platform: modbus_controller @@ -158,3 +220,22 @@ text_sensor: response_size: 4 lambda: |- return "Modified: " + x; + # A register reporting FEWER bytes than 2*register_count (response_size: 3 for 2 registers), followed + # by a contiguous sensor: the follower's byte position must track the actual 3 bytes, not underflow. + - platform: modbus_controller + modbus_controller_id: modbus_controller1 + id: modbus_text_sensor_narrow + name: Test Text Sensor Narrow Response + register_type: holding + address: 0x9030 + register_count: 2 + response_size: 3 + raw_encode: HEXBYTES + - platform: modbus_controller + modbus_controller_id: modbus_controller1 + id: modbus_text_sensor_after_narrow + name: Test Text Sensor After Narrow + register_type: holding + address: 0x9032 + register_count: 1 + raw_encode: HEXBYTES diff --git a/tests/components/modbus_controller/offline_cadence_test.cpp b/tests/components/modbus_controller/offline_cadence_test.cpp new file mode 100644 index 0000000000..1dade5dd98 --- /dev/null +++ b/tests/components/modbus_controller/offline_cadence_test.cpp @@ -0,0 +1,56 @@ +#include + +#include + +#include "esphome/components/modbus_controller/modbus_controller.h" + +namespace esphome::modbus_controller::testing { + +// A probe must come due exactly once per offline_skip_updates + 1 cycles from the trip point, +// for every phase between the trip cycle and the update counter. Pins the regression where a +// probe additionally required a range's skip_updates cadence to coincide, which some phase +// combinations never satisfy - the device then never polled again. +TEST(OfflineRetryCadence, DueOncePerWindowForEveryPhase) { + for (uint16_t skip = 0; skip <= 5; skip++) { + const uint16_t period = skip + 1; + for (uint16_t offline_at = 0; offline_at <= 7; offline_at++) { + uint16_t due_count = 0; + for (uint32_t counter = offline_at; counter < offline_at + 4u * period; counter++) { + if (offline_retry_due(static_cast(counter), offline_at, skip)) + due_count++; + } + EXPECT_EQ(due_count, 4) << "skip=" << skip << " offline_at=" << offline_at; + } + } +} + +// The first probe goes out within one window of going offline: after at most skip skipped cycles. +TEST(OfflineRetryCadence, FirstProbeWithinOneWindow) { + for (uint16_t skip = 0; skip <= 5; skip++) { + for (uint16_t offline_at = 0; offline_at <= 7; offline_at++) { + uint16_t counter = offline_at; + uint16_t skipped = 0; + while (!offline_retry_due(counter, offline_at, skip)) { + counter++; + skipped++; + ASSERT_LE(skipped, skip) << "skip=" << skip << " offline_at=" << offline_at; + } + } + } +} + +// The cadence neither stretches nor collapses when update_counter_ wraps past 65535. +TEST(OfflineRetryCadence, SurvivesCounterWraparound) { + const uint16_t skip = 2; // period 3 + const uint16_t offline_at = 65530; + uint16_t counter = offline_at; + uint16_t due_count = 0; + for (int i = 0; i < 30; i++) { // crosses the wrap mid-run + if (offline_retry_due(counter, offline_at, skip)) + due_count++; + counter++; + } + EXPECT_EQ(due_count, 10); +} + +} // namespace esphome::modbus_controller::testing diff --git a/tests/components/modbus_controller/sensor_item_position_test.cpp b/tests/components/modbus_controller/sensor_item_position_test.cpp new file mode 100644 index 0000000000..2fb679ee07 --- /dev/null +++ b/tests/components/modbus_controller/sensor_item_position_test.cpp @@ -0,0 +1,89 @@ +#include + +#include +#include + +#include "esphome/components/modbus_controller/modbus_controller.h" + +namespace esphome::modbus_controller::testing { + +namespace { + +// Minimal concrete SensorItem so the position/address accessors can be exercised directly. +class TestSensorItem : public SensorItem { + public: + void parse_and_publish(std::span /*data*/) override {} +}; + +// Builds an item the way a platform constructor does, before ranges are built. +TestSensorItem make_item(modbus::EntityType type, uint16_t address, uint8_t offset) { + TestSensorItem item; + item.register_type = type; + item.set_address(address); + item.set_offset_from_start_address(offset); + return item; +} + +} // namespace + +// A freshly constructed item is already usable: its resolved position is the offset as configured and +// its range base is its own address, which is what an item that never gets polled relies on. +TEST(SensorItemPosition, ConstructionSeedsResolvedPositionAndRangeBase) { + auto item = make_item(modbus::EntityType::HOLDING, 0x9001, 4); + EXPECT_EQ(item.offset_from_start_address, 4); + EXPECT_EQ(item.offset, 4); + EXPECT_EQ(item.range_start_address, 0x9001); +} + +// A write lands on the register the sensor reads from. The resolved position is relative to the range's +// first register, which may be earlier than the sensor's own address, so both are needed to get there. +TEST(SensorItemPosition, WriteAddressForRegisters) { + auto item = make_item(modbus::EntityType::HOLDING, 0x9003, 0); + item.range_start_address = 0x9001; + item.offset = 4; + EXPECT_EQ(item.write_address(), 0x9003); +} + +// Coils index bits, so the resolved offset is a bit count and is added to the range base directly. +TEST(SensorItemPosition, WriteAddressForCoils) { + auto item = make_item(modbus::EntityType::COIL, 0x15, 0); + item.range_start_address = 0x10; + item.offset = 5; + EXPECT_EQ(item.write_address(), 0x15); + EXPECT_TRUE(item.addresses_bits()); +} + +// An item that is never polled keeps the range base its constructor set, so its write address is still +// its own address plus its configured offset - a switch with assumed_state, or an output. +TEST(SensorItemPosition, WriteAddressWithoutAGroupedRange) { + auto item = make_item(modbus::EntityType::HOLDING, 0x9010, 2); + EXPECT_EQ(item.write_address(), 0x9011); +} + +// A sensor re-using a register after one with a non-zero offset resolves past that offset, and its +// write address follows the same position - the behaviour releases before the range rework had. +TEST(SensorItemPosition, ReUseChainWriteAddressFollowsResolvedPosition) { + auto item = make_item(modbus::EntityType::HOLDING, 0x9001, 4); + item.range_start_address = 0x9001; + item.offset = 6; // 4 configured, plus the 2 the previous sensor on this register resolved to + EXPECT_EQ(item.write_address(), 0x9004); +} + +// Registers address 16-bit words; only coils and discrete inputs address bits. +TEST(SensorItemPosition, AddressesBitsOnlyForCoilAndDiscreteInput) { + EXPECT_FALSE(make_item(modbus::EntityType::HOLDING, 0, 0).addresses_bits()); + EXPECT_FALSE(make_item(modbus::EntityType::INPUT_REGISTER, 0, 0).addresses_bits()); + EXPECT_TRUE(make_item(modbus::EntityType::COIL, 0, 0).addresses_bits()); + EXPECT_TRUE(make_item(modbus::EntityType::DISCRETE_INPUT, 0, 0).addresses_bits()); +} + +// A span payload reaches payload_to_number() unqualified from inside this namespace: SensorValueType +// lives in modbus::helpers, so argument-dependent lookup finds the helper. Declaring a same-signature +// forwarder here would make the call ambiguous rather than convenient, which is why none exists. +TEST(SensorItemPosition, UnqualifiedPayloadToNumberResolvesToTheHelper) { + const uint8_t bytes[] = {0x01, 0x02}; + auto value = payload_to_number(std::span(bytes), SensorValueType::U_WORD, 0, 0xFFFFFFFF); + EXPECT_EQ(value, 0x0102); +} + +} // namespace esphome::modbus_controller::testing diff --git a/tests/components/modbus_server/common.yaml b/tests/components/modbus_server/common.yaml index 8b2316b6e3..1f3a8f551b 100644 --- a/tests/components/modbus_server/common.yaml +++ b/tests/components/modbus_server/common.yaml @@ -40,3 +40,11 @@ modbus_server: value_type: U_WORD read_lambda: |- return (random_uint32() % 100); + # Covers CPP_TYPE_REGISTER_MAP / signed byte-swapped codegen + - address: 0x6 + value_type: S_WORD_S + read_lambda: |- + return -2; + write_lambda: |- + printf("address=%d, value=%d\n", (int) address, (int) x); + return true; diff --git a/tests/components/modbus_server/modbus_server_test.cpp b/tests/components/modbus_server/modbus_server_test.cpp index 8c2e1d16d9..2137a77f3d 100644 --- a/tests/components/modbus_server/modbus_server_test.cpp +++ b/tests/components/modbus_server/modbus_server_test.cpp @@ -34,6 +34,21 @@ TEST(ModbusServerWrite, SingleWordSucceeds) { EXPECT_EQ(written, 0x1234); } +TEST(ModbusServerWrite, SwappedWordSucceeds) { + ModbusServer server; + int64_t written = -1; + ServerRegister reg(0x0000, SensorValueType::U_WORD_S, 1); + reg.write_lambda = [&written](int64_t value) { + written = value; + return true; + }; + server.add_server_register(®); + + auto status = server.on_write_registers(0x0000, make_registers({0x3412})); + EXPECT_FALSE(status.has_value()); + EXPECT_EQ(written, 0x1234); +} + // A multi-register value is decoded high word first and applied as a single number. TEST(ModbusServerWrite, DwordSucceeds) { ModbusServer server; @@ -136,6 +151,19 @@ TEST(ModbusServerRead, SingleWordSucceeds) { EXPECT_EQ(out[0], 0x1234); } +TEST(ModbusServerRead, SwappedWordReturnsByteSwappedRegister) { + ModbusServer server; + ServerRegister reg(0x0000, SensorValueType::U_WORD_S, 1); + reg.read_lambda = []() -> int64_t { return 0x1234; }; + server.add_server_register(®); + + RegisterValues out; + auto status = server.on_read_registers(0x0000, 1, out); + EXPECT_FALSE(status.has_value()); + ASSERT_EQ(out.size(), 1u); + EXPECT_EQ(out[0], 0x3412); +} + TEST(ModbusServerRead, DwordReturnsTwoWordsHighFirst) { ModbusServer server; ServerRegister reg(0x0000, SensorValueType::U_DWORD, 2); diff --git a/tests/components/rp2040_ble/test-scan.rp2040-ard.yaml b/tests/components/rp2040_ble/test-scan.rp2040-ard.yaml index 251c83a92f..401a18c0de 100644 --- a/tests/components/rp2040_ble/test-scan.rp2040-ard.yaml +++ b/tests/components/rp2040_ble/test-scan.rp2040-ard.yaml @@ -1,12 +1,16 @@ -# Exercises the controller scan API from a lambda: passive scan start with -# interval/window in 0.625 ms BLE units, stop, and the adapter MAC accessor. +# Exercises the controller scan API from a lambda: scan start with +# interval/window in 0.625 ms BLE units and the active flag, stop, and the +# adapter MAC accessor. esphome: on_boot: then: - lambda: |- uint8_t mac[6]; id(ble).get_mac_msb_first(mac); - if (id(ble).scan_start(160, 48)) { + if (id(ble).scan_start(160, 48, false)) { + id(ble).scan_stop(); + } + if (id(ble).scan_start(160, 48, true)) { id(ble).scan_stop(); } diff --git a/tests/components/rp2_ble_tracker/common-boundary.yaml b/tests/components/rp2_ble_tracker/common-boundary.yaml new file mode 100644 index 0000000000..91e010b121 --- /dev/null +++ b/tests/components/rp2_ble_tracker/common-boundary.yaml @@ -0,0 +1,12 @@ +rp2_ble_tracker: + id: ble_tracker + scan_parameters: + # Boundary coverage: the documented 2.5 ms floor on window (expressible only + # via the microsecond-accurate validation), a non-round interval exercising the + # 0.625 ms unit conversion without collapsing onto the window's unit count, + # and the non-continuous config path. + interval: 5000us + window: 2500us + duration: 5min + active: false + continuous: false diff --git a/tests/components/rp2_ble_tracker/common.yaml b/tests/components/rp2_ble_tracker/common.yaml new file mode 100644 index 0000000000..633a4e1d0f --- /dev/null +++ b/tests/components/rp2_ble_tracker/common.yaml @@ -0,0 +1,17 @@ +rp2_ble_tracker: + id: ble_tracker + scan_parameters: + interval: 100ms + window: 30ms + duration: 5min + active: true + continuous: true + +# Pulls in USE_OTA_STATE_LISTENER so the OTA scan-pause path compiles in CI +# (same coverage arrangement as the esp32_ble_tracker tests). +wifi: + ssid: MySSID + password: password1 + +ota: + - platform: esphome diff --git a/tests/components/rp2_ble_tracker/test.rp2040-ard.yaml b/tests/components/rp2_ble_tracker/test.rp2040-ard.yaml new file mode 100644 index 0000000000..8b94f3cade --- /dev/null +++ b/tests/components/rp2_ble_tracker/test.rp2040-ard.yaml @@ -0,0 +1,2 @@ +packages: + rp2_ble_tracker: !include common.yaml diff --git a/tests/components/rp2_ble_tracker/validate-boundary.rp2040-ard.yaml b/tests/components/rp2_ble_tracker/validate-boundary.rp2040-ard.yaml new file mode 100644 index 0000000000..b62a401320 --- /dev/null +++ b/tests/components/rp2_ble_tracker/validate-boundary.rp2040-ard.yaml @@ -0,0 +1,2 @@ +packages: + rp2_ble_tracker: !include common-boundary.yaml diff --git a/tests/components/zigbee/common.yaml b/tests/components/zigbee/common.yaml index c689d07f6b..cc0d28ea61 100644 --- a/tests/components/zigbee/common.yaml +++ b/tests/components/zigbee/common.yaml @@ -13,6 +13,7 @@ sensor: - platform: template name: "Analog 1" lambda: return 10.0; + accuracy_decimals: 0 - platform: template name: "Analog 2" lambda: return 11.0; diff --git a/tests/components/zigbee/common_esp32.yaml b/tests/components/zigbee/common_esp32.yaml index 6cac9c9e2a..ac25fb8faf 100644 --- a/tests/components/zigbee/common_esp32.yaml +++ b/tests/components/zigbee/common_esp32.yaml @@ -27,3 +27,6 @@ zigbee: on_join: then: - logger.log: "Joined network" + on_start: + then: + - logger.log: "Started zigbee stack" diff --git a/tests/components/zigbee/common_nrf52.yaml b/tests/components/zigbee/common_nrf52.yaml index bc39b371f5..c05c4053a5 100644 --- a/tests/components/zigbee/common_nrf52.yaml +++ b/tests/components/zigbee/common_nrf52.yaml @@ -7,6 +7,9 @@ zigbee: on_join: then: - logger.log: "Joined network" + on_start: + then: + - logger.log: "Started zigbee stack" time: - platform: zigbee diff --git a/tests/integration/entity_utils.py b/tests/integration/entity_utils.py index 7596983ee2..95f6a0321e 100644 --- a/tests/integration/entity_utils.py +++ b/tests/integration/entity_utils.py @@ -8,7 +8,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -from esphome.helpers import fnv1_hash_object_id, sanitize, snake_case +from esphome.helpers import fnv1_hash_name, sanitize, snake_case if TYPE_CHECKING: from aioesphomeapi import DeviceInfo, EntityInfo @@ -25,15 +25,16 @@ def infer_name_add_mac_suffix(device_info: DeviceInfo) -> bool: return device_info.name.endswith(f"-{mac_suffix}") -def _get_name_for_object_id( +def _resolve_entity_name( entity: EntityInfo, device_info: DeviceInfo, device_id_to_name: dict[int, str], ) -> str: - """Get the name used for object_id computation. + """Resolve the effective name for an entity. This is the algorithm that aioesphomeapi will use to determine which - name to use for computing object_id client-side from API data. + name to use for computing object_id client-side from API data; the same + name is what the device hashes into the entity key. Args: entity: The entity to get name for @@ -72,27 +73,27 @@ def compute_entity_object_id( Returns: The computed object_id string """ - name_for_id = _get_name_for_object_id(entity, device_info, device_id_to_name) - return compute_object_id(name_for_id) + name = _resolve_entity_name(entity, device_info, device_id_to_name) + return compute_object_id(name) -def compute_entity_hash( +def compute_entity_key( entity: EntityInfo, device_info: DeviceInfo, device_id_to_name: dict[int, str], ) -> int: - """Compute expected object_id hash for an entity. + """Compute expected entity key for an entity. Args: - entity: The entity to compute hash for + entity: The entity to compute the key for device_info: Device info from the API device_id_to_name: Mapping of device_id to device name for sub-devices Returns: - The computed FNV-1 hash + The computed FNV-1 hash of the raw name """ - name_for_id = _get_name_for_object_id(entity, device_info, device_id_to_name) - return fnv1_hash_object_id(name_for_id) + name = _resolve_entity_name(entity, device_info, device_id_to_name) + return fnv1_hash_name(name) def verify_entity_object_id( @@ -118,7 +119,7 @@ def verify_entity_object_id( f"expected '{expected_object_id}', got '{entity.object_id}'" ) - expected_hash = compute_entity_hash(entity, device_info, device_id_to_name) + expected_hash = compute_entity_key(entity, device_info, device_id_to_name) assert entity.key == expected_hash, ( f"hash mismatch for entity '{entity.name}': " f"expected {expected_hash:#x}, got {entity.key:#x}" diff --git a/tests/integration/fixtures/fnv1_hash_object_id.yaml b/tests/integration/fixtures/fnv1_hash_object_id.yaml index 2097b2fbf9..d4511bb8c6 100644 --- a/tests/integration/fixtures/fnv1_hash_object_id.yaml +++ b/tests/integration/fixtures/fnv1_hash_object_id.yaml @@ -71,6 +71,38 @@ esphome: ESP_LOGE("FNV1_OID", "empty FAILED: 0x%08x != 0x811c9dc5", hash_empty); } + // Raw name hash: matches Python fnv1_hash_name("My Sensor Name") + uint32_t hash_raw = esphome::fnv1_hash_bytes("My Sensor Name", 14); + if (hash_raw == 0x8cec6fb0) { + ESP_LOGI("FNV1_OID", "raw PASSED"); + } else { + ESP_LOGE("FNV1_OID", "raw FAILED: 0x%08x != 0x8cec6fb0", hash_raw); + } + + // Raw name hash over UTF-8 bytes: matches Python fnv1_hash_name("Température") + uint32_t hash_raw_utf8 = esphome::fnv1_hash_bytes("Temp\xc3\xa9rature", 12); + if (hash_raw_utf8 == 0x531a74aa) { + ESP_LOGI("FNV1_OID", "raw_utf8 PASSED"); + } else { + ESP_LOGE("FNV1_OID", "raw_utf8 FAILED: 0x%08x != 0x531a74aa", hash_raw_utf8); + } + + // Old-key UTF-8 variant: matches Python fnv1_hash_object_id("Température") + uint32_t hash_old_utf8 = esphome::fnv1_hash_object_id("Temp\xc3\xa9rature", 12, true); + if (hash_old_utf8 == 0x965698f3) { + ESP_LOGI("FNV1_OID", "old_utf8 PASSED"); + } else { + ESP_LOGE("FNV1_OID", "old_utf8 FAILED: 0x%08x != 0x965698f3", hash_old_utf8); + } + + // Old-key UTF-8 variant with multi-byte only name: Python fnv1_hash_object_id("温度") + uint32_t hash_old_cjk = esphome::fnv1_hash_object_id("\xe6\xb8\xa9\xe5\xba\xa6", 6, true); + if (hash_old_cjk == 0x3276cb9f) { + ESP_LOGI("FNV1_OID", "old_cjk PASSED"); + } else { + ESP_LOGE("FNV1_OID", "old_cjk FAILED: 0x%08x != 0x3276cb9f", hash_old_cjk); + } + host: api: logger: diff --git a/tests/integration/fixtures/multi_device_preferences.yaml b/tests/integration/fixtures/multi_device_preferences.yaml index 01e4394559..582add90a8 100644 --- a/tests/integration/fixtures/multi_device_preferences.yaml +++ b/tests/integration/fixtures/multi_device_preferences.yaml @@ -156,10 +156,17 @@ button: ESP_LOGI("test", "Device A Mode: %s", id(mode_device_a).current_option().c_str()); ESP_LOGI("test", "Device B Mode: %s", id(mode_device_b).current_option().c_str()); ESP_LOGI("test", "Main Mode: %s", id(mode_main).current_option().c_str()); - // Log preference hashes for entities that actually store preferences - ESP_LOGI("test", "Device A Switch Pref Hash: %u", id(light_device_a).get_preference_hash()); - ESP_LOGI("test", "Device B Switch Pref Hash: %u", id(light_device_b).get_preference_hash()); - ESP_LOGI("test", "Main Switch Pref Hash: %u", id(light_main).get_preference_hash()); - ESP_LOGI("test", "Device A Number Pref Hash: %u", id(setpoint_device_a).get_preference_hash()); - ESP_LOGI("test", "Device B Number Pref Hash: %u", id(setpoint_device_b).get_preference_hash()); - ESP_LOGI("test", "Main Number Pref Hash: %u", id(setpoint_main).get_preference_hash()); + // Log preference key bases for entities that actually store preferences. + // This is the key base make_entity_preference() uses: entity key XOR device id. + ESP_LOGI("test", "Device A Switch Pref Hash: %u", + id(light_device_a).get_entity_key() ^ id(light_device_a).get_device_id_or_zero()); + ESP_LOGI("test", "Device B Switch Pref Hash: %u", + id(light_device_b).get_entity_key() ^ id(light_device_b).get_device_id_or_zero()); + ESP_LOGI("test", "Main Switch Pref Hash: %u", + id(light_main).get_entity_key() ^ id(light_main).get_device_id_or_zero()); + ESP_LOGI("test", "Device A Number Pref Hash: %u", + id(setpoint_device_a).get_entity_key() ^ id(setpoint_device_a).get_device_id_or_zero()); + ESP_LOGI("test", "Device B Number Pref Hash: %u", + id(setpoint_device_b).get_entity_key() ^ id(setpoint_device_b).get_device_id_or_zero()); + ESP_LOGI("test", "Main Number Pref Hash: %u", + id(setpoint_main).get_entity_key() ^ id(setpoint_main).get_device_id_or_zero()); diff --git a/tests/integration/fixtures/preference_key_migration.yaml b/tests/integration/fixtures/preference_key_migration.yaml new file mode 100644 index 0000000000..a9b01fc2d2 --- /dev/null +++ b/tests/integration/fixtures/preference_key_migration.yaml @@ -0,0 +1,35 @@ +esphome: + name: host-pref-key-migration + +host: +api: +logger: + +switch: + - platform: template + id: test_switch_restore + name: Test Switch + optimistic: true + restore_mode: RESTORE_DEFAULT_OFF + +number: + - platform: template + id: test_number_restore + name: Test Number + optimistic: true + restore_value: true + initial_value: 1.0 + min_value: 0 + max_value: 100 + step: 0.5 + +text: + - platform: template + id: test_text_restore + name: Test Text + mode: text + optimistic: true + restore_value: true + initial_value: fallback + min_length: 0 + max_length: 20 diff --git a/tests/integration/fixtures/script_queued.yaml b/tests/integration/fixtures/script_queued.yaml index 996dd6436f..c8c56113db 100644 --- a/tests/integration/fixtures/script_queued.yaml +++ b/tests/integration/fixtures/script_queued.yaml @@ -1,5 +1,17 @@ esphome: name: test-script-queued + on_boot: + # Default priority (600.0) runs before the script component is set up + # This tests that an instance queued during boot still gets dequeued + # once the main loop starts (the idle-loop disabling must not eat it) + then: + - logger.log: "=== BOOT: Executing queued script twice ===" + - script.execute: + id: boot_script + tag: 1 + - script.execute: + id: boot_script + tag: 2 host: api: @@ -98,6 +110,15 @@ api: - script.execute: no_params_script - script.execute: no_params_script + # Test 6: Re-execute after stop() cleared the queue + # (the idle loop must re-enable on demand) + - action: test_after_stop + then: + - logger.log: "=== TEST 6: Re-execute after stop ===" + - script.execute: + id: stop_script + num: 9 + logger: level: DEBUG @@ -168,3 +189,18 @@ script: - logger.log: "No params: START" - delay: 50ms - logger.log: "No params: END" + + # Boot script: executed twice from on_boot before setup() + - id: boot_script + mode: queued + max_runs: 3 + parameters: + tag: int + then: + - logger.log: + format: "Boot queued: START %d" + args: ['tag'] + - delay: 50ms + - logger.log: + format: "Boot queued: END %d" + args: ['tag'] diff --git a/tests/integration/fixtures/script_queued_idle_loop.yaml b/tests/integration/fixtures/script_queued_idle_loop.yaml new file mode 100644 index 0000000000..7d5d3cb86f --- /dev/null +++ b/tests/integration/fixtures/script_queued_idle_loop.yaml @@ -0,0 +1,25 @@ +esphome: + name: test-script-queued-idle + +host: +api: + actions: + # Execute twice: the first runs immediately, the second gets queued, + # which must re-enable the loop; draining must disable it again + - action: run_twice + then: + - script.execute: idle_script + - script.execute: idle_script + +# VERY_VERBOSE exposes the component framework's "loop disabled" and +# "loop enabled" messages that this test asserts on +logger: + level: VERY_VERBOSE + +script: + - id: idle_script + mode: queued + then: + - logger.log: "idle_script: START" + - delay: 50ms + - logger.log: "idle_script: END" diff --git a/tests/integration/fixtures/uart_mock_modbus_client_inline.yaml b/tests/integration/fixtures/uart_mock_modbus_client_inline.yaml new file mode 100644 index 0000000000..f85206107f --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_client_inline.yaml @@ -0,0 +1,108 @@ +esphome: + name: uart-mock-modbus-client-inline + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_server + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_client + data: !lambda return data; + - id: virtual_uart_client + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + +modbus: + - uart_id: virtual_uart_server + id: virtual_modbus_server + role: server + - uart_id: virtual_uart_client + id: virtual_modbus_client + role: client + turnaround_time: 10ms + # Short wait so the no-reply cases (address 2 below) time out well within the test window. + send_wait_time: 500ms + +modbus_server: + - address: 1 + modbus_id: virtual_modbus_server + id: modbus_server_1 + registers: + - address: 0x10 + value_type: U_WORD + read_lambda: return 1234; + +sensor: + - platform: template + name: "inline_value" + id: inline_value + - platform: template + name: "timeout_flag" + id: timeout_flag + - platform: template + name: "skipped_flag" + id: skipped_flag + +# The same write action fired twice while its first frame is still awaiting a reply: the hub drops the +# duplicate write (writes are never merged) and the second firing resolves via its own on_not_sent. +# mode: parallel so the second run starts while the first send is pending. +script: + - id: dup_write + mode: parallel + then: + - modbus_client.send: + address: 2 + pdu: [0x06, 0x00, 0x10, 0x01, 0x02] + on_not_sent: + then: + - lambda: "id(skipped_flag).publish_state(1);" + +# Each action is its own hub device: address 1 is served by the mock server, address 2 answers nothing. +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + on_press: + # Per-send inline on_response: decode this reply where the send was fired (fire-and-continue). + - modbus_client.send: + address: 1 + pdu: [0x03, 0x00, 0x10, 0x00, 0x01] + on_response: + then: + - lambda: |- + if (response.size() >= 4) + id(inline_value).publish_state((response[2] << 8) | response[3]); + # No server answers address 2, so this resolves via on_no_response. + - modbus_client.send: + address: 2 + pdu: [0x03, 0x00, 0x10, 0x00, 0x01] + on_no_response: + then: + - lambda: "id(timeout_flag).publish_state(1);" + - script.execute: dup_write + - script.execute: dup_write diff --git a/tests/integration/fixtures/uart_mock_modbus_custom_command.yaml b/tests/integration/fixtures/uart_mock_modbus_custom_command.yaml new file mode 100644 index 0000000000..738e691110 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_custom_command.yaml @@ -0,0 +1,87 @@ +esphome: + name: uart-mock-modbus-custom-command + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_server + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_controller + data: !lambda return data; + - id: virtual_uart_controller + baud_rate: 9600 + auto_start: true + debug: + on_tx: + - then: + - uart_mock.inject_rx: + id: virtual_uart_server + data: !lambda return data; + +modbus: + - uart_id: virtual_uart_server + id: virtual_modbus_server + role: server + - uart_id: virtual_uart_controller + id: virtual_modbus_controller + role: client + turnaround_time: 10ms + +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_controller + id: modbus_controller_1 + update_interval: 1s + +modbus_server: + - address: 1 + modbus_id: virtual_modbus_server + id: modbus_server_1 + registers: + - address: 0x01 + value_type: U_WORD + read_lambda: return 259; + +sensor: + # Plain read to confirm the controller <-> server link is up. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "plain_read" + address: 0x01 + register_type: holding + value_type: U_WORD + # Custom command: a raw frame {device address, function code, address hi, address lo, + # count hi, count lo}; the CRC is appended by the hub. Reads holding register 0x0001, + # count 1; the lambda parses the response payload (the register value, big-endian). + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "custom_read" + custom_command: [0x01, 0x03, 0x00, 0x01, 0x00, 0x01] + lambda: |- + if (data.size() < 2) return {}; + return (float) ((data[0] << 8) | data[1]); + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + # This test does not have anything to start (mock is autostart) diff --git a/tests/integration/fixtures/uart_mock_modbus_fairness.yaml b/tests/integration/fixtures/uart_mock_modbus_fairness.yaml new file mode 100644 index 0000000000..e2918c82dd --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_fairness.yaml @@ -0,0 +1,125 @@ +esphome: + name: uart-mock-modbus-fairness + +host: +api: +logger: + # DEBUG (not VERBOSE) keeps the log volume manageable while both controllers + # hammer the bus at a high rate. + level: DEBUG + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +# Counters for the number of requests seen on the bus for each device address. +globals: + - id: req_count_1 + type: int + initial_value: "0" + - id: req_count_2 + type: int + initial_value: "0" + +uart_mock: + - id: virtual_uart + baud_rate: 9600 + # auto_start so the mock is ready to deliver injected responses. Polling + # itself is gated by the Start button below. + auto_start: true + debug: + on_tx: + - then: + # Count each outgoing request by device address (byte 0 of the frame). + - lambda: |- + if (data.empty()) + return; + if (data[0] == 0x01) { + id(req_count_1) += 1; + id(requests_1).publish_state(id(req_count_1)); + } else if (data[0] == 0x02) { + id(req_count_2) += 1; + id(requests_2).publish_state(id(req_count_2)); + } + # Reply directly with a canned, CRC-correct "read holding register" + # response for whichever device was addressed (both controllers only + # ever issue this one fixed request, so the responses are constant). + - uart_mock.inject_rx: + id: virtual_uart + data: !lambda |- + if (!data.empty() && data[0] == 0x01) + return {0x01, 0x03, 0x02, 0x00, 0x6F, 0xF8, 0x68}; // value 111 + if (!data.empty() && data[0] == 0x02) + return {0x02, 0x03, 0x02, 0x00, 0xDE, 0x7C, 0x1C}; // value 222 + return {}; + +modbus: + - uart_id: virtual_uart + id: virtual_modbus_client + role: client + turnaround_time: 15ms #This is longer than the polling interval to cause contention + +# Two controllers sharing one client bus, each polling a different device. +# Polling is started by the test (update_interval: never until then) so counting +# only begins once the API client has subscribed. +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_client + id: modbus_controller_1 + update_interval: never + - address: 2 + modbus_id: virtual_modbus_client + id: modbus_controller_2 + update_interval: never + +sensor: + # These sensors define the register range each controller polls (and so drive + # the requests). Their values are not checked by the test. + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: reg_1 + address: 0x01 + register_type: holding + value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_2 + name: reg_2 + address: 0x01 + register_type: holding + value_type: U_WORD + # Request counters exposed to the test. Updated manually from the on_tx hook. + - platform: template + name: requests_1 + id: requests_1 + update_interval: never + - platform: template + name: requests_2 + id: requests_2 + update_interval: never + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + on_press: + - lambda: |- + // Poll much faster than the bus can service so both controllers always + // have a request pending and must contend for the bus. + id(modbus_controller_1).set_update_interval(10); + id(modbus_controller_1).start_poller(); + id(modbus_controller_2).set_update_interval(10); + id(modbus_controller_2).start_poller(); + - platform: template + name: "Stop Scenario" + id: stop_scenario_btn + on_press: + - lambda: |- + id(modbus_controller_1).stop_poller(); + id(modbus_controller_2).stop_poller(); diff --git a/tests/integration/fixtures/uart_mock_modbus_grouping.yaml b/tests/integration/fixtures/uart_mock_modbus_grouping.yaml new file mode 100644 index 0000000000..a5394f1d05 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_grouping.yaml @@ -0,0 +1,234 @@ +esphome: + name: uart-mock-modbus-group + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_dev + baud_rate: 9600 + rx_full_threshold: 120 + rx_timeout: 2 + auto_start: false + debug: + responses: + # One entry per range the controller polls. A frame the controller does not send goes unanswered, + # so these also pin the grouping: an extra or differently shaped read fails the test. + - expect_tx: [0x01, 0x01, 0x00, 0x10, 0x00, 0x02, 0xBC, 0x0E] # coils 0x10 count 2 + inject_rx: [0x01, 0x01, 0x01, 0x01, 0x90, 0x48] # bit0 set, bit1 clear + - expect_tx: [0x01, 0x03, 0x01, 0x60, 0x00, 0x01, 0x85, 0xE8] # holding 0x160 count 1 + inject_rx: [0x01, 0x03, 0x02, 0x01, 0x60, 0xB9, 0xFC] # 352 + - expect_tx: [0x01, 0x03, 0x01, 0x00, 0x00, 0x01, 0x85, 0xF6] # holding 0x100 count 1 + inject_rx: [0x01, 0x03, 0x04, 0x01, 0x11, 0x02, 0x22, 0x2A, 0xB3] # 4 bytes: 273 then 546 + - expect_tx: [0x01, 0x03, 0x01, 0x20, 0x00, 0x04, 0x44, 0x3F] # holding 0x120 count 4 + inject_rx: [0x01, 0x03, 0x08, 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0, 0x7A, 0x25] + - expect_tx: [0x01, 0x03, 0x01, 0x30, 0x00, 0x02, 0xC5, 0xF8] # holding 0x130 count 2 + inject_rx: [0x01, 0x03, 0x06, 0x0A, 0xAA, 0xFF, 0xFF, 0x0B, 0xBB, 0x7E, 0xA0] # 6 bytes + - expect_tx: [0x01, 0x03, 0x01, 0x40, 0x00, 0x01, 0x84, 0x22] # holding 0x140 count 1 + inject_rx: [0x01, 0x03, 0x02, 0x01, 0x40, 0xB8, 0x24] # 320 + - expect_tx: [0x01, 0x03, 0x01, 0x45, 0x00, 0x01, 0x94, 0x23] # holding 0x145 count 1 + inject_rx: [0x01, 0x03, 0x02, 0x01, 0x45, 0x78, 0x27] # 325 + - expect_tx: [0x01, 0x03, 0x01, 0x50, 0x00, 0x02, 0xC5, 0xE6] # holding 0x150 count 2 + inject_rx: [0x01, 0x03, 0x04, 0x01, 0x50, 0x01, 0x51, 0x3B, 0xB2] # 336, 337 + - expect_tx: [0x01, 0x03, 0x01, 0x80, 0x00, 0x02, 0xC4, 0x1F] # holding 0x180 count 2 + inject_rx: [0x01, 0x03, 0x06, 0x11, 0x11, 0x22, 0x22, 0x33, 0x33, 0x20, 0xA0] # 6 bytes + # 0x181 answers with the same value whether it is read on its own or as part of the block above, + # so the sensor there is pinned to one value regardless of which range it lands in. + - expect_tx: [0x01, 0x03, 0x01, 0x81, 0x00, 0x01, 0xD5, 0xDE] # holding 0x181 count 1 + inject_rx: [0x01, 0x03, 0x02, 0x33, 0x33, 0xEC, 0xA1] # 13107 + - expect_tx: [0x01, 0x03, 0x01, 0x70, 0x00, 0x03, 0x05, 0xEC] # holding 0x170 count 3 + inject_rx: [0x01, 0x03, 0x06, 0x00, 0x2A, 0x1B, 0x2C, 0x03, 0x0D, 0x3E, 0xAB] # 6 bytes + - expect_tx: [0x01, 0x03, 0x01, 0x61, 0x00, 0x01, 0xD4, 0x28] # holding 0x161 count 1 + inject_rx: [0x01, 0x03, 0x02, 0x01, 0x61, 0x78, 0x3C] # 353 + +modbus: + uart_id: virtual_uart_dev + send_wait_time: 200ms + turnaround_time: 10ms + +modbus_controller: + - address: 1 + id: modbus_controller_ok + max_cmd_retries: 2 + update_interval: never + +# Each block below is a distinct address range exercising one grouping relationship. The blocks are far +# enough apart that they never merge into each other. +sensor: + # A - two sensors on one register that returns more bytes than its count implies (response_size), + # reading different halves of it. + - platform: modbus_controller + name: "reuse_lo" + address: 0x100 + register_type: holding + value_type: U_WORD + response_size: 4 + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "reuse_hi" + address: 0x100 + register_type: holding + value_type: U_WORD + offset: 2 + response_size: 4 + modbus_controller_id: modbus_controller_ok + + # C - plain contiguous registers of differing widths. + - platform: modbus_controller + name: "ext_word" + address: 0x120 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "ext_next" + address: 0x121 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "ext_dword" + address: 0x122 + register_type: holding + value_type: U_DWORD + modbus_controller_id: modbus_controller_ok + + # D - a wide (response_size) register followed by a contiguous one: the follower must start after the + # bytes the wide register actually returned, not after 2 * register_count. + - platform: modbus_controller + name: "wide_first" + address: 0x130 + register_type: holding + value_type: U_WORD + response_size: 4 + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "wide_next" + address: 0x131 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + + # E - a gap: these must never share a range. + - platform: modbus_controller + name: "gap_low" + address: 0x140 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "gap_high" + address: 0x145 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + + # F - contiguous registers where the second asks for a slower rate. + - platform: modbus_controller + name: "rate_first" + address: 0x150 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "rate_slow" + address: 0x151 + register_type: holding + value_type: U_WORD + skip_updates: 5 + modbus_controller_id: modbus_controller_ok + + # B - a wide value and one of its halves share a start address, with a contiguous sensor after them. + # The differing offsets give these a defined order, unlike two sensors that differ only in width. + - platform: modbus_controller + name: "shared_dword" + address: 0x170 + register_type: holding + value_type: U_DWORD + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "shared_high" + address: 0x170 + register_type: holding + value_type: U_WORD + offset: 2 + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "shared_after" + address: 0x172 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + + # I - a register that returns more bytes than its count implies, sharing its address with a plain + # wider sensor. Whether the sensor after them is read as part of that block or on its own, it must + # decode 0x181 - never the bytes that lie two into the block, which is where the widened register + # count alone would put it. + - platform: modbus_controller + name: "masked_wide" + address: 0x180 + register_type: holding + value_type: U_WORD + response_size: 4 + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "masked_pair" + address: 0x180 + register_type: holding + value_type: U_DWORD + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "masked_after" + address: 0x181 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + + # H - a sensor pinned to its own range, followed by a contiguous one. + - platform: modbus_controller + name: "forced_first" + address: 0x160 + register_type: holding + value_type: U_WORD + force_new_range: true + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "forced_next" + address: 0x161 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + +binary_sensor: + # G - contiguous coils, addressed by bit. + - platform: modbus_controller + name: "coil_first" + address: 0x10 + register_type: coil + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "coil_next" + address: 0x11 + register_type: coil + modbus_controller_id: modbus_controller_ok + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + on_press: + - lambda: |- + id(virtual_uart_dev).start_scenario(); + id(modbus_controller_ok).set_update_interval(1000); + id(modbus_controller_ok).start_poller(); diff --git a/tests/integration/fixtures/uart_mock_modbus_offline.yaml b/tests/integration/fixtures/uart_mock_modbus_offline.yaml new file mode 100644 index 0000000000..e4d2dfa294 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_offline.yaml @@ -0,0 +1,95 @@ +esphome: + name: uart-mock-modbus-offline + +host: +api: +logger: + level: DEBUG + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +# Whether the mock device answers requests. Starts false so the controller +# runs through its retries and goes offline; the test flips it via the +# "Serve" button to exercise the offline retry/recovery path. +globals: + - id: serve + type: bool + initial_value: "false" + +uart_mock: + - id: virtual_uart + baud_rate: 9600 + auto_start: true + debug: + on_tx: + # While serve is false every request times out; once true, answer the + # (only) request - read holding register 3 on device 1 - with value 259. + - uart_mock.inject_rx: + id: virtual_uart + data: !lambda |- + if (!id(serve)) + return {}; + return {0x01, 0x03, 0x02, 0x01, 0x03, 0xF9, 0xD5}; + +modbus: + - uart_id: virtual_uart + id: virtual_modbus_client + send_wait_time: 100ms + turnaround_time: 10ms + +modbus_controller: + - address: 1 + modbus_id: virtual_modbus_client + id: ctl + max_cmd_retries: 1 + # offline_skip_updates and the sensor's skip_updates deliberately share a period: offline + # probing must follow the offline cadence alone, or phase combinations like this one can + # leave the device never probing again. + offline_skip_updates: 1 + update_interval: never + on_offline: + then: + - lambda: id(link_state).publish_state(0); + on_online: + then: + - lambda: id(link_state).publish_state(1); + +sensor: + - platform: modbus_controller + modbus_controller_id: ctl + name: reg + id: reg + address: 0x03 + register_type: holding + value_type: U_WORD + skip_updates: 1 + # Mirrors the controller's online state so the test can await the transitions. + - platform: template + name: link_state + id: link_state + update_interval: never + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + on_press: + - lambda: |- + id(ctl).set_update_interval(200); + id(ctl).start_poller(); + - platform: template + name: "Serve" + id: serve_btn + on_press: + - globals.set: + id: serve + value: "true" diff --git a/tests/integration/fixtures/uart_mock_modbus_server_controller.yaml b/tests/integration/fixtures/uart_mock_modbus_server_controller.yaml index 20306bd73a..4a5d280a2f 100644 --- a/tests/integration/fixtures/uart_mock_modbus_server_controller.yaml +++ b/tests/integration/fixtures/uart_mock_modbus_server_controller.yaml @@ -64,9 +64,15 @@ modbus_server: - address: 0x01 value_type: U_WORD read_lambda: return 99; + - address: 0x02 + value_type: U_WORD_S + read_lambda: return 4660; - address: 0x03 value_type: S_WORD read_lambda: return -99; + - address: 0x04 + value_type: S_WORD_S + read_lambda: return -2; - address: 0x05 value_type: U_DWORD read_lambda: return 16909060; @@ -105,12 +111,30 @@ sensor: address: 0x01 register_type: holding value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_u_word_s" + address: 0x02 + register_type: holding + value_type: U_WORD_S + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_u_word_s_raw" + address: 0x02 + register_type: holding + value_type: U_WORD - platform: modbus_controller modbus_controller_id: modbus_controller_1 name: "reg_s_word" address: 0x03 register_type: holding value_type: S_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_s_word_s" + address: 0x04 + register_type: holding + value_type: S_WORD_S - platform: modbus_controller modbus_controller_id: modbus_controller_1 name: "reg_u_dword" diff --git a/tests/integration/fixtures/uart_mock_modbus_server_controller_write.yaml b/tests/integration/fixtures/uart_mock_modbus_server_controller_write.yaml index b3b5e76e31..5ade49bd48 100644 --- a/tests/integration/fixtures/uart_mock_modbus_server_controller_write.yaml +++ b/tests/integration/fixtures/uart_mock_modbus_server_controller_write.yaml @@ -45,9 +45,15 @@ globals: - id: stored_u_word type: uint16_t initial_value: "11" + - id: stored_u_word_s + type: uint16_t + initial_value: "4660" - id: stored_s_word type: int16_t initial_value: "-11" + - id: stored_s_word_s + type: int16_t + initial_value: "-2" - id: stored_u_dword type: uint32_t initial_value: "1001" @@ -103,10 +109,18 @@ modbus_server: value_type: U_WORD read_lambda: return id(stored_u_word); write_lambda: id(stored_u_word) = x; return true; + - address: 0x02 + value_type: U_WORD_S + read_lambda: return id(stored_u_word_s); + write_lambda: id(stored_u_word_s) = x; return true; - address: 0x03 value_type: S_WORD read_lambda: return id(stored_s_word); write_lambda: id(stored_s_word) = x; return true; + - address: 0x04 + value_type: S_WORD_S + read_lambda: return id(stored_s_word_s); + write_lambda: id(stored_s_word_s) = x; return true; - address: 0x05 value_type: U_DWORD read_lambda: return id(stored_u_dword); @@ -155,12 +169,24 @@ sensor: address: 0x01 register_type: holding value_type: U_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_u_word_s" + address: 0x02 + register_type: holding + value_type: U_WORD_S - platform: modbus_controller modbus_controller_id: modbus_controller_1 name: "reg_s_word" address: 0x03 register_type: holding value_type: S_WORD + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "reg_s_word_s" + address: 0x04 + register_type: holding + value_type: S_WORD_S - platform: modbus_controller modbus_controller_id: modbus_controller_1 name: "reg_u_dword" @@ -231,6 +257,14 @@ number: value_type: U_WORD min_value: 0 max_value: 65535 + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "write_u_word_s" + address: 0x02 + register_type: holding + value_type: U_WORD_S + min_value: 0 + max_value: 65535 - platform: modbus_controller modbus_controller_id: modbus_controller_1 name: "write_s_word" @@ -239,6 +273,14 @@ number: value_type: S_WORD min_value: -16777215 max_value: 16777215 + - platform: modbus_controller + modbus_controller_id: modbus_controller_1 + name: "write_s_word_s" + address: 0x04 + register_type: holding + value_type: S_WORD_S + min_value: -16777215 + max_value: 16777215 - platform: modbus_controller modbus_controller_id: modbus_controller_1 name: "write_u_dword" diff --git a/tests/integration/fixtures/uart_mock_modbus_shared_address.yaml b/tests/integration/fixtures/uart_mock_modbus_shared_address.yaml new file mode 100644 index 0000000000..25574d0c42 --- /dev/null +++ b/tests/integration/fixtures/uart_mock_modbus_shared_address.yaml @@ -0,0 +1,160 @@ +esphome: + name: uart-mock-modbus-shared + +host: +api: +logger: + level: VERBOSE + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + +# Dummy uart entry to satisfy modbus's DEPENDENCIES = ["uart"] +# The actual UART bus used is the uart_mock component below +uart: + baud_rate: 115200 + port: /dev/null + +uart_mock: + - id: virtual_uart_dev + baud_rate: 9600 + rx_full_threshold: 120 + rx_timeout: 2 + auto_start: false + debug: + responses: + # Three sensors, one frame. At 0x9001 a U_WORD (1 register) and a U_DWORD (2 registers) share the + # start address but cannot merge, so the range widens to count 2. A third sensor at 0x9002 falls + # inside the widened range and must read its slice of the same response rather than splitting into + # a second overlapping poll. The single expect_tx pins the "one frame on the wire" contract - any + # duplicate or overlapping range would put an extra frame on the bus and fail to match. + - expect_tx: [0x01, 0x03, 0x90, 0x01, 0x00, 0x02, 0xB8, 0xCB] # Read holding 0x9001 count 2 on device 1 + inject_rx: [0x01, 0x03, 0x04, 0x03, 0x97, 0x02, 0x91, 0x8B, 0x57] # 0x9001=0x0397, 0x9002=0x0291 + # A force_new_range sensor at a HIGH address (0x30) sorts before the plain sensor at a LOW address + # (0x10). The two must poll as separate ranges: the covered branch's lower-bound check prevents the + # 0x10 sensor from being absorbed into the forced 0x30 range with a wrapped byte offset. + - expect_tx: [0x01, 0x03, 0x00, 0x30, 0x00, 0x01, 0x84, 0x05] # Read holding 0x30 count 1 (forced range) + inject_rx: [0x01, 0x03, 0x02, 0x01, 0x11, 0x79, 0xD8] # 0x30 = 0x0111 = 273 + - expect_tx: [0x01, 0x03, 0x00, 0x10, 0x00, 0x01, 0x85, 0xCF] # Read holding 0x10 count 1 (own range) + inject_rx: [0x01, 0x03, 0x02, 0x02, 0x22, 0x39, 0x3D] # 0x10 = 0x0222 = 546 + # A wide sensor (U_QWORD at 0x100, 4 registers) followed by plain sensors at 0x101 and 0x103. + # None of them merge, so all three poll separately - exactly as before the range refactor. The + # 0x103 sensor sits at the wide range's tail address, so it must not anchor a re-use join on a + # mid-range predecessor and inherit its byte offset. + - expect_tx: [0x01, 0x03, 0x01, 0x00, 0x00, 0x04, 0x45, 0xF5] # Read holding 0x100 count 4 + inject_rx: [0x01, 0x03, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x94, 0x3C] # = 100 + - expect_tx: [0x01, 0x03, 0x01, 0x01, 0x00, 0x01, 0xD4, 0x36] # Read holding 0x101 count 1 + inject_rx: [0x01, 0x03, 0x02, 0x01, 0x41, 0x79, 0xE4] # 0x101 = 0x0141 = 321 + - expect_tx: [0x01, 0x03, 0x01, 0x03, 0x00, 0x01, 0x75, 0xF6] # Read holding 0x103 count 1 + inject_rx: [0x01, 0x03, 0x02, 0x01, 0xA5, 0x79, 0xAF] # 0x103 = 0x01A5 = 421 + # A widened shared-address range at 0x200 plus a sensor at 0x201 carrying its own skip_updates. + # The sensor must keep its own range so the polling rates stay independent; if it were folded into + # the widened range it would decode 0x201 from THAT response (2, not 777) and drag the range's + # rate down to its own. + - expect_tx: [0x01, 0x03, 0x02, 0x00, 0x00, 0x02, 0xC5, 0xB3] # Read holding 0x200 count 2 + inject_rx: [0x01, 0x03, 0x04, 0x01, 0x41, 0x00, 0x02, 0x2A, 0x1A] # 0x200=0x0141, 0x201=0x0002 + - expect_tx: [0x01, 0x03, 0x02, 0x01, 0x00, 0x01, 0xD4, 0x72] # Read holding 0x201 count 1 + inject_rx: [0x01, 0x03, 0x02, 0x03, 0x09, 0x78, 0xB2] # 0x201 = 0x0309 = 777 + +modbus: + uart_id: virtual_uart_dev + send_wait_time: 200ms + turnaround_time: 10ms + +modbus_controller: + - address: 1 + id: modbus_controller_ok + max_cmd_retries: 2 + update_interval: never + +sensor: + # Word sensor at 0x9001 (1 register) + - platform: modbus_controller + name: "shared_word" + address: 0x9001 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + # Dword sensor at the SAME address 0x9001 (2 registers) - non-mergeable, shares the range start + - platform: modbus_controller + name: "shared_dword" + address: 0x9001 + register_type: holding + value_type: U_DWORD + modbus_controller_id: modbus_controller_ok + # Word sensor at 0x9002 - inside the widened range, reads bytes 2-3 of the same response + - platform: modbus_controller + name: "covered_word" + address: 0x9002 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + # Forced sensor at a high address: sorts first, opens its own isolated range + - platform: modbus_controller + name: "forced_high" + address: 0x30 + register_type: holding + value_type: U_WORD + force_new_range: true + modbus_controller_id: modbus_controller_ok + # Plain sensor at a lower address: must get its own range, never absorbed into the forced one + - platform: modbus_controller + name: "plain_low" + address: 0x10 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + # Wide sensor spanning 0x100-0x103; the two sensors below sit inside its span but do not merge + - platform: modbus_controller + name: "wide_qword" + address: 0x100 + register_type: holding + value_type: U_QWORD + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "inside_wide" + address: 0x101 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + # At the wide range's tail address: must decode its own poll, not inherit a mid-range byte offset + - platform: modbus_controller + name: "tail_of_wide" + address: 0x103 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + # Shared address 0x200: the dword widens the range the word opened (or vice versa) + - platform: modbus_controller + name: "rate_word" + address: 0x200 + register_type: holding + value_type: U_WORD + modbus_controller_id: modbus_controller_ok + - platform: modbus_controller + name: "rate_dword" + address: 0x200 + register_type: holding + value_type: U_DWORD + modbus_controller_id: modbus_controller_ok + # Inside the widened range but with its own skip_updates: must NOT be folded in, or the two sensors + # above would silently drop to this sensor's polling rate + - platform: modbus_controller + name: "own_rate" + address: 0x201 + register_type: holding + value_type: U_WORD + skip_updates: 100 + modbus_controller_id: modbus_controller_ok + +button: + - platform: template + name: "Start Scenario" + id: start_scenario_btn + on_press: + - lambda: |- + id(virtual_uart_dev).start_scenario(); + id(modbus_controller_ok).set_update_interval(1000); + id(modbus_controller_ok).start_poller(); diff --git a/tests/integration/host_prefs.py b/tests/integration/host_prefs.py index f835bee3bc..c7f21d8a01 100644 --- a/tests/integration/host_prefs.py +++ b/tests/integration/host_prefs.py @@ -25,15 +25,25 @@ def clear_host_prefs(device_name: str) -> None: host_prefs_path(device_name).unlink(missing_ok=True) +def write_host_prefs(device_name: str, entries: dict[int, bytes]) -> Path: + """Write preference entries, replacing the file's contents. + + Returns the path that was written. + """ + payload = b"" + for key, data in entries.items(): + if len(data) > 255: + raise ValueError(f"Preference data too long: {len(data)} bytes (max 255)") + payload += struct.pack(" Path: """Write a single preference entry, replacing the file's contents. Returns the path that was written. """ - if len(data) > 255: - raise ValueError(f"Preference data too long: {len(data)} bytes (max 255)") - path = host_prefs_path(device_name) - path.parent.mkdir(parents=True, exist_ok=True) - payload = struct.pack(" None: diff --git a/tests/integration/test_object_id_api_verification.py b/tests/integration/test_object_id_api_verification.py index c8603e0682..8dafb37c64 100644 --- a/tests/integration/test_object_id_api_verification.py +++ b/tests/integration/test_object_id_api_verification.py @@ -2,8 +2,8 @@ This test verifies a three-way match between: 1. C++ object_id generation (get_object_id_to using to_sanitized_char/to_snake_case_char) -2. C++ hash generation (fnv1_hash_object_id in helpers.h) -3. Python computation (sanitize/snake_case in helpers.py, fnv1_hash_object_id) +2. C++ entity key generation (fnv1_hash of the raw name in helpers.h) +3. Python computation (sanitize/snake_case and fnv1_hash_name in helpers.py) The API response contains C++ computed values, so verifying API == Python implicitly verifies C++ == Python == API for both object_id and hash. @@ -25,7 +25,7 @@ from __future__ import annotations import pytest -from esphome.helpers import fnv1_hash_object_id +from esphome.helpers import fnv1_hash_name from .entity_utils import compute_object_id, verify_all_entities from .types import APIClientConnectedFactory, RunCompiledFunction @@ -123,7 +123,7 @@ async def test_object_id_api_verification( ) # Verify hash can be computed from the name - hash_from_name = fnv1_hash_object_id(entity_name) + hash_from_name = fnv1_hash_name(entity_name) assert hash_from_name == entity.key, ( f"Entity '{entity_name}': hash mismatch. " f"Python hash {hash_from_name:#x}, API key {entity.key:#x}" @@ -164,7 +164,7 @@ async def test_object_id_api_verification( ) # Verify hash matches - expected_hash = fnv1_hash_object_id(expected_name) + expected_hash = fnv1_hash_name(expected_name) assert entity.key == expected_hash, ( f"Empty-name entity (device_id={entity.device_id}): hash mismatch. " f"API key: {entity.key:#x}, expected: {expected_hash:#x}" diff --git a/tests/integration/test_object_id_friendly_name_no_mac_suffix.py b/tests/integration/test_object_id_friendly_name_no_mac_suffix.py index 7199a2b371..b58593f2ef 100644 --- a/tests/integration/test_object_id_friendly_name_no_mac_suffix.py +++ b/tests/integration/test_object_id_friendly_name_no_mac_suffix.py @@ -11,7 +11,7 @@ from __future__ import annotations import pytest -from esphome.helpers import fnv1_hash_object_id +from esphome.helpers import fnv1_hash_name from .entity_utils import ( compute_object_id, @@ -62,7 +62,7 @@ async def test_object_id_friendly_name_no_mac_suffix( ) # Hash should match friendly_name - expected_hash = fnv1_hash_object_id("My Friendly Device") + expected_hash = fnv1_hash_name("My Friendly Device") assert entity.key == expected_hash, ( f"Expected hash {expected_hash:#x}, got {entity.key:#x}" ) diff --git a/tests/integration/test_object_id_no_friendly_name.py b/tests/integration/test_object_id_no_friendly_name.py index b548f02fde..45b5f730a6 100644 --- a/tests/integration/test_object_id_no_friendly_name.py +++ b/tests/integration/test_object_id_no_friendly_name.py @@ -17,7 +17,7 @@ from __future__ import annotations import pytest -from esphome.helpers import fnv1_hash_object_id +from esphome.helpers import fnv1_hash_name from .entity_utils import compute_object_id, verify_all_entities from .types import APIClientConnectedFactory, RunCompiledFunction @@ -96,7 +96,7 @@ async def test_object_id_no_friendly_name_no_mac_suffix( OLD behavior: - is_object_id_dynamic_() returned false (mac suffix not enabled) - Used object_id_c_str_ which was pre-computed in Python - - Python used get_base_entity_object_id() with fallback to CORE.name + - Python used get_base_entity_name() with fallback to CORE.name Result: object_id = sanitize(snake_case(device_name)) """ @@ -126,7 +126,7 @@ async def test_object_id_no_friendly_name_no_mac_suffix( ) # Hash should match device name - expected_hash = fnv1_hash_object_id("test-device") + expected_hash = fnv1_hash_name("test-device") assert entity.key == expected_hash, ( f"Expected hash {expected_hash:#x}, got {entity.key:#x}" ) diff --git a/tests/integration/test_preference_key_migration.py b/tests/integration/test_preference_key_migration.py new file mode 100644 index 0000000000..e7f699bb12 --- /dev/null +++ b/tests/integration/test_preference_key_migration.py @@ -0,0 +1,165 @@ +"""Integration test for entity preference key migration. + +Entity keys are now the FNV-1 hash of the raw name instead of the sanitized +object_id (https://github.com/esphome/backlog/issues/85). On key-lookup +preference backends, make_entity_preference() must move data stored under the +old key to the new key, so devices keep their restored state after upgrading. + +This test seeds the host preferences file the way a pre-migration firmware +would have written it and verifies: +1. Data stored under the OLD key is restored (migration happened, no data loss) +2. Data already stored under the NEW key is never overwritten by old data +""" + +from __future__ import annotations + +import socket +import struct + +from aioesphomeapi import ( + NumberInfo, + NumberState, + SwitchInfo, + SwitchState, + TextInfo, + TextState, +) +import pytest + +from esphome.helpers import fnv1_hash, fnv1_hash_name, fnv1_hash_object_id + +from .conftest import run_binary_and_wait_for_port, wait_and_connect_api_client +from .host_prefs import clear_host_prefs, write_host_prefs +from .state_utils import InitialStateHelper, require_entity +from .types import CompileFunction, ConfigWriter + +DEVICE_NAME = "host-pref-key-migration" + +# The pre-migration preference key was the sanitized object_id hash; the new +# key is the raw-name hash. All entities are on the main device (device_id 0) +# and their preferences use no version salt, so the key is just the hash. +SWITCH_OLD_KEY = fnv1_hash_object_id("Test Switch") +SWITCH_NEW_KEY = fnv1_hash_name("Test Switch") +NUMBER_OLD_KEY = fnv1_hash_object_id("Test Number") +NUMBER_NEW_KEY = fnv1_hash_name("Test Number") + +# template_text salts its key with the length limits and pattern hash; this must +# match TemplateText::setup() in template_text.cpp (min_length 0, max_length 20, +# no pattern configured) +TEXT_KEY_EXTRA = (0 << 2) + (20 << 4) + (fnv1_hash("") << 6) +TEXT_OLD_KEY = (fnv1_hash_object_id("Test Text") + TEXT_KEY_EXTRA) & 0xFFFFFFFF +TEXT_NEW_KEY = (fnv1_hash_name("Test Text") + TEXT_KEY_EXTRA) & 0xFFFFFFFF + +# TextSaver<20> stores a length-prefixed buffer of max_length + 1 bytes +TEXT_MAX_LENGTH = 20 + + +def text_pref_payload(value: str) -> bytes: + """Build the length-prefixed buffer TextSaver stores for a value.""" + data = value.encode("utf-8") + assert len(data) <= TEXT_MAX_LENGTH + return bytes([len(data)]) + data + b"\x00" * (TEXT_MAX_LENGTH - len(data)) + + +@pytest.mark.asyncio +async def test_preference_key_migration( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> None: + """Test that preferences stored under the old key survive the upgrade.""" + port, port_socket = reserved_tcp_port + + assert SWITCH_OLD_KEY != SWITCH_NEW_KEY + assert NUMBER_OLD_KEY != NUMBER_NEW_KEY + assert TEXT_OLD_KEY != TEXT_NEW_KEY + + # Write and compile once + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + + # Release the reserved port so the binary can bind to it + port_socket.close() + + async def boot_and_get_initial_states() -> tuple[ + SwitchState, NumberState, TextState + ]: + """Boot the binary and return the restored entity states.""" + async with ( + run_binary_and_wait_for_port(binary_path, "127.0.0.1", port), + wait_and_connect_api_client(port=port) as client, + ): + device_info = await client.device_info() + assert device_info.name == DEVICE_NAME + + entities, _ = await client.list_entities_services() + switch_entity = require_entity( + entities, "test_switch", SwitchInfo, "Test Switch" + ) + number_entity = require_entity( + entities, "test_number", NumberInfo, "Test Number" + ) + text_entity = require_entity(entities, "test_text", TextInfo, "Test Text") + + initial_state_helper = InitialStateHelper(entities) + client.subscribe_states( + initial_state_helper.on_state_wrapper(lambda s: None) + ) + await initial_state_helper.wait_for_initial_states() + + switch_state = initial_state_helper.initial_states[switch_entity.key] + number_state = initial_state_helper.initial_states[number_entity.key] + text_state = initial_state_helper.initial_states[text_entity.key] + assert isinstance(switch_state, SwitchState) + assert isinstance(number_state, NumberState) + assert isinstance(text_state, TextState) + return switch_state, number_state, text_state + + try: + # --- Run 1: only OLD keys present, as written by pre-migration firmware. + # The restored states prove the data was migrated to the new keys. + write_host_prefs( + DEVICE_NAME, + { + SWITCH_OLD_KEY: b"\x01", # bool: switch was ON + NUMBER_OLD_KEY: struct.pack(" None: """Check log output for all test messages.""" @@ -122,11 +132,24 @@ async def test_script_queued( # Test 5: No params if no_params_end.search(line): test_results["no_params"]["executions"] += 1 - if ( - test_results["no_params"]["executions"] == 3 - and not test5_complete.done() - ): - test5_complete.set_result(True) + executions = test_results["no_params"]["executions"] + for count, future in ((3, test5_complete), (6, test5_again_complete)): + if executions == count and not future.done(): + future.set_result(True) + + # Boot script (queued from on_boot before setup) + if match := boot_end.search(line): + test_results["boot"]["ended"].append(int(match.group(1))) + if len(test_results["boot"]["ended"]) == 2 and not boot_complete.done(): + boot_complete.set_result(True) + + # Test 6: Re-execute after stop + if ( + (match := after_stop_end.search(line)) + and int(match.group(1)) == 9 + and not test6_complete.done() + ): + test6_complete.set_result(True) async with ( run_compiled(yaml_config, line_callback=check_output), @@ -135,6 +158,13 @@ async def test_script_queued( # Get services _, services = await client.list_entities_services() + # Boot: both executions from on_boot must complete, including the one + # that was queued before QueueingScript::setup() ran + await asyncio.wait_for(boot_complete, timeout=2.0) + assert sorted(test_results["boot"]["ended"]) == [1, 2], ( + f"Boot: Expected both on_boot executions to complete, got {sorted(test_results['boot']['ended'])}" + ) + # Test 1: Queue depth limit test_service = next((s for s in services if s.name == "test_queue_depth"), None) assert test_service is not None, "test_queue_depth service not found" @@ -203,3 +233,20 @@ async def test_script_queued( assert test_results["no_params"]["executions"] == 3, ( f"Test 5: Expected 3 executions, got {test_results['no_params']['executions']}" ) + + # Test 5 again: after the queue fully drained (loop disabled while + # idle), executing again must still work + test_service = next((s for s in services if s.name == "test_no_params"), None) + assert test_service is not None, "test_no_params service not found" + await client.execute_service(test_service, {}) + await asyncio.wait_for(test5_again_complete, timeout=2.0) + assert test_results["no_params"]["executions"] == 6, ( + f"Test 5 again: Expected 6 executions total, got {test_results['no_params']['executions']}" + ) + + # Test 6: a stopped script (queue cleared, loop disabled) must run + # again on the next execute; the future resolves only on "END 9" + test_service = next((s for s in services if s.name == "test_after_stop"), None) + assert test_service is not None, "test_after_stop service not found" + await client.execute_service(test_service, {}) + await asyncio.wait_for(test6_complete, timeout=2.0) diff --git a/tests/integration/test_script_queued_idle_loop.py b/tests/integration/test_script_queued_idle_loop.py new file mode 100644 index 0000000000..44f0ab7ec6 --- /dev/null +++ b/tests/integration/test_script_queued_idle_loop.py @@ -0,0 +1,85 @@ +"""Test that an idle queued script disables its loop and re-enables on demand.""" + +from __future__ import annotations + +import asyncio +import re + +import pytest + +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_script_queued_idle_loop( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Assert the loop state transitions of a queued script via VV logs. + + Expected sequence: the idle script disables its loop on the first + iteration after boot, re-enables it when an instance gets queued, + and disables it again once the queue drains. + """ + loop = asyncio.get_running_loop() + + loop_state = re.compile(r"\bscript loop (disabled|enabled)\b") + script_end = re.compile(r"idle_script: END") + + transitions: list[str] = [] + end_count = 0 + + boot_disabled = loop.create_future() + enabled_after_queue = loop.create_future() + disabled_after_drain = loop.create_future() + runs_complete = loop.create_future() + + def check_output(line: str) -> None: + nonlocal end_count + if match := loop_state.search(line): + transitions.append(match.group(1)) + if transitions == ["disabled"] and not boot_disabled.done(): + boot_disabled.set_result(True) + elif ( + transitions == ["disabled", "enabled"] + and not enabled_after_queue.done() + ): + enabled_after_queue.set_result(True) + elif ( + transitions + == [ + "disabled", + "enabled", + "disabled", + ] + and not disabled_after_drain.done() + ): + disabled_after_drain.set_result(True) + + if script_end.search(line): + end_count += 1 + if end_count == 2 and not runs_complete.done(): + runs_complete.set_result(True) + + async with ( + run_compiled(yaml_config, line_callback=check_output), + api_client_connected() as client, + ): + # The idle script must disable its loop on the first iteration + await asyncio.wait_for(boot_disabled, timeout=5.0) + + _, services = await client.list_entities_services() + run_twice = next((s for s in services if s.name == "run_twice"), None) + assert run_twice is not None, "run_twice service not found" + await client.execute_service(run_twice, {}) + + # Queueing the second instance must re-enable the loop + await asyncio.wait_for(enabled_after_queue, timeout=2.0) + # Both runs must complete and the drained queue must disable it again + await asyncio.wait_for(runs_complete, timeout=2.0) + await asyncio.wait_for(disabled_after_drain, timeout=2.0) + + assert transitions == ["disabled", "enabled", "disabled"], ( + f"Unexpected loop state sequence: {transitions}" + ) diff --git a/tests/integration/test_uart_mock_modbus.py b/tests/integration/test_uart_mock_modbus.py index 2c437341c6..bf163665f2 100644 --- a/tests/integration/test_uart_mock_modbus.py +++ b/tests/integration/test_uart_mock_modbus.py @@ -9,6 +9,10 @@ test_uart_mock_modbus_no_threshold : Test modbus with no rx_full_threshold set (simulating USB UART / non-hardware UART). Verifies the 50ms fallback timeout handles chunked data with USB packet gaps. +test_uart_mock_modbus_fairness : + Two controllers sharing one client bus, both polling far faster than the bus + can service. Verifies the hub schedules them fairly (request counts within 1). + """ from __future__ import annotations @@ -17,7 +21,7 @@ import asyncio from collections.abc import Callable from dataclasses import dataclass -from aioesphomeapi import NumberInfo +from aioesphomeapi import ButtonInfo, NumberInfo import pytest from .state_utils import SensorTracker, find_entity @@ -211,7 +215,10 @@ async def test_uart_mock_modbus_server_controller( expected_values = { "reg_u_word": 99, + "reg_u_word_s": 4660, + "reg_u_word_s_raw": 13330, "reg_s_word": -99, + "reg_s_word_s": -2, "reg_u_dword": 16909060, "reg_s_dword": -16909060, "reg_u_dword_r": pytest.approx(67305985), @@ -245,14 +252,16 @@ async def test_uart_mock_modbus_server_controller_write( Verifies that writing to modbus server registers via the controller updates the server's stored values, which are then read back correctly on the next poll. - All 12 value types are tested: U/S_WORD, U/S_DWORD(_R), U/S_QWORD(_R), FP32(_R). + All 14 value types are tested: U/S_WORD, U/S_WORD_S, U/S_DWORD(_R), U/S_QWORD(_R), FP32(_R). """ line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback() register_test_cases: dict[str, RegisterTestCase] = { "reg_u_word": RegisterTestCase(11, "write_u_word", 42, 42), + "reg_u_word_s": RegisterTestCase(4660, "write_u_word_s", 17185, 17185), "reg_s_word": RegisterTestCase(-11, "write_s_word", -42, -42), + "reg_s_word_s": RegisterTestCase(-2, "write_s_word_s", -257, -257), "reg_u_dword": RegisterTestCase(1001, "write_u_dword", 2002, 2002), "reg_s_dword": RegisterTestCase(-1001, "write_s_dword", -2002, -2002), "reg_u_dword_r": RegisterTestCase(3003, "write_u_dword_r", 4004, 4004), @@ -330,3 +339,275 @@ async def test_uart_mock_modbus_server_controller_multiple( await tracker.setup_and_start_scenario(client) await tracker.await_all(futures) _assert_no_modbus_errors(error_log_lines, warning_log_lines) + + +@pytest.mark.asyncio +async def test_uart_mock_modbus_client_inline( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test modbus_client.send actions: each action is its own hub device. + + Start Scenario fires: a read of served address 1 decoded in its inline on_response -> inline_value; a + read of address 2, which no server answers, resolving via on_no_response -> timeout_flag. A parallel + script fires the same write action twice while its first frame is pending; the hub drops the duplicate + write, and the second firing resolves via its own on_not_sent -> skipped_flag. This exercises + per-action reply routing, the no-reply path, and the one-outcome guarantee under the hub's write + dedup. + """ + + tracker = SensorTracker(["inline_value", "timeout_flag", "skipped_flag"]) + futures = tracker.expect_all( + {"inline_value": 1234, "timeout_flag": 1, "skipped_flag": 1} + ) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + await tracker.setup_and_start_scenario(client) + await tracker.await_all(futures, timeout=5.0) + + +@pytest.mark.asyncio +async def test_uart_mock_modbus_grouping( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Pins how sensors are grouped into polled ranges across the combinations that matter. + + Each block in the fixture covers one relationship between neighbouring sensors - sharing a wide + register, contiguous, separated by a gap, differing polling rates, coils, and a pinned range - so + that the frames on the wire and the byte each sensor decodes from are locked down. + """ + + line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback() + + # Values are those the component produced before the range rework, captured from it directly. + expected_values = { + # one register returning 4 bytes, read as two halves + "reuse_lo": 273, + "reuse_hi": 546, + # contiguous registers, mixed widths + "ext_word": 4660, + "ext_next": 22136, + "ext_dword": pytest.approx(2596069120), + # a wide register pushes its neighbour past the bytes it actually returned + "wide_first": 2730, + "wide_next": 3003, + # a gap keeps them apart + "gap_low": 320, + "gap_high": 325, + # contiguous, second one polling more slowly + "rate_first": 336, + "rate_slow": 337, + # a wide value, one of its halves, and the register after it + "shared_dword": pytest.approx(2759468), + "shared_high": 6956, + "shared_after": 781, + # a wide register hidden behind a wider plain sibling, and the sensor after them + "masked_wide": 4369, + "masked_pair": pytest.approx(286335522), + "masked_after": 13107, + # pinned range, and the contiguous sensor after it + "forced_first": 352, + "forced_next": 353, + } + tracker = SensorTracker(list(expected_values.keys())) + futures = tracker.expect_all(expected_values) + + async with ( + run_compiled(yaml_config, line_callback=line_callback), + api_client_connected() as client, + ): + await tracker.setup_and_start_scenario(client) + await tracker.await_all(futures) + # Every frame sent must match one the mock answers, so an unexpected read (a range that split, + # merged or changed length) shows up here as an unanswered request. This is what pins the coil + # grouping too, since binary sensors carry no numeric state to compare. + _assert_no_modbus_errors(error_log_lines, warning_log_lines) + + +@pytest.mark.asyncio +async def test_uart_mock_modbus_shared_address( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Sensors sharing and overlapping one register range must all decode from a single read. + + A U_WORD and a U_DWORD share start address 0x9001 (non-mergeable, so the range widens to 2 + registers) and a third U_WORD at 0x9002 falls inside the widened range. A regression guard for the + range-grouping rewrite: without the same-address fallback the shared sensors land in duplicate + ranges and one never publishes; without the in-range join the 0x9002 sensor splits into a second + overlapping frame that the mock (which expects exactly one read) never answers. + + A force_new_range sensor at 0x30 plus a plain sensor at 0x10 pin the covered branch's lower-bound + check: the forced sensor sorts first, and without the bound the lower-address sensor is absorbed + into the forced range with a wrapped byte offset and never polls its own register. + + A U_QWORD at 0x100 with plain sensors at 0x101 and 0x103 pins that non-merging sensors inside a + wide sensor's span keep polling separately, and that the sensor at the span's tail address does not + anchor a re-use join on a mid-range predecessor (which would make it decode that sensor's bytes). + + A sensor at 0x201 carrying skip_updates sits inside a widened shared-address range at 0x200 but + keeps its own range, so polling rates stay independent; folding it in would also make it decode + 0x201 out of the shared response (2) instead of its own poll (777). + """ + + line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback() + + # 0x9001 = 0x0397 (919); 0x9001..0x9002 = 0x03970291 (60228241, approx: not exact in float32); + # 0x9002 = 0x0291 (657); 0x30 = 0x0111 (273); 0x10 = 0x0222 (546) + expected_values = { + "shared_word": 919, + "shared_dword": pytest.approx(60228241), + "covered_word": 657, + "forced_high": 273, + "plain_low": 546, + "wide_qword": 100, + "inside_wide": 321, + "tail_of_wide": 421, + "rate_word": 321, + "rate_dword": pytest.approx(21037058), + "own_rate": 777, + } + tracker = SensorTracker(list(expected_values.keys())) + futures = tracker.expect_all(expected_values) + + async with ( + run_compiled(yaml_config, line_callback=line_callback), + api_client_connected() as client, + ): + await tracker.setup_and_start_scenario(client) + await tracker.await_all(futures) + _assert_no_modbus_errors(error_log_lines, warning_log_lines) + + +@pytest.mark.asyncio +async def test_uart_mock_modbus_custom_command( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Test a custom_command sensor polling a register served by the mock server. + + The custom_command is a raw frame (device address + PDU); the hub appends the CRC and + routes the response back to the polling command, whose sensor lambda parses the payload. + Guards the custom polling wiring: the command must reference the sensor's custom_data and + decode the real function code, or nothing is ever transmitted. A plain read on the same + register anchors the bus. + """ + + line_callback, error_log_lines, warning_log_lines = _make_modbus_line_callback() + + expected_values = {"plain_read": 259, "custom_read": 259} + tracker = SensorTracker(list(expected_values.keys())) + futures = tracker.expect_all(expected_values) + + async with ( + run_compiled(yaml_config, line_callback=line_callback), + api_client_connected() as client, + ): + await tracker.setup_and_start_scenario(client) + await tracker.await_all(futures) + _assert_no_modbus_errors(error_log_lines, warning_log_lines) + + +@pytest.mark.asyncio +async def test_uart_mock_modbus_offline( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """A silent device drives the controller offline; answering again recovers it. + + The mock answers nothing at first, so the controller burns through max_cmd_retries + (1 retry after the first timeout) and fires on_offline. While offline it keeps + retrying every offline_skip_updates+1 cycles. The test then flips the mock to + answering; the next retry gets a response, on_online fires, and the register value + publishes. This pins the pooled non-response counter, can_send() gating, the + offline retry cadence, and recovery - none of which the responding-path tests touch. + + The fixture gives offline_skip_updates and the sensor's skip_updates the same period + on purpose: offline probing must follow the offline cadence alone, since requiring + both cadences to coincide leaves phase combinations where no probe ever goes out. + """ + + tracker = SensorTracker(["link_state", "reg"]) + offline_future = tracker.expect("link_state", 0) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + entities = await tracker.setup_and_start_scenario(client) + + # The unanswered poll and its retry each time out (~100ms), then on_offline fires. + await tracker.await_change(offline_future, "link_state", timeout=5.0) + + # Register the recovery expectations before waking the device so no update is missed. + online_future = tracker.expect("link_state", 1) + value_future = tracker.expect("reg", 259) + serve_btn = find_entity(entities, "serve", ButtonInfo) + assert serve_btn is not None, "Serve button not found" + client.button_command(serve_btn.key) + + # The next offline-cadence retry gets an answer: back online, value published. + await tracker.await_change(online_future, "link_state", timeout=5.0) + await tracker.await_change(value_future, "reg", timeout=5.0) + + +@pytest.mark.asyncio +async def test_uart_mock_modbus_fairness( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Two controllers sharing one bus should get a fair share of it. + + Both controllers poll different devices (addresses 1 and 2) on the same + client hub, far faster than the bus can service, so they continually + contend for it. The on_tx hook in the fixture counts the requests issued + for each address. With fair scheduling in the modbus hub, neither + controller should starve the other: the two request counts must end up + within 1 of each other. + """ + + tracker = SensorTracker(["requests_1", "requests_2"]) + + async with ( + run_compiled(yaml_config), + api_client_connected() as client, + ): + entities = await tracker.setup_and_start_scenario(client) + + # Let both controllers hammer the bus for a while. + await asyncio.sleep(2.0) + + # Stop polling so the counters settle to a final, stable value (state + # coalescing means intermediate values may be skipped, but the final + # value is always delivered once changes stop). + stop_btn = find_entity(entities, "stop_scenario", ButtonInfo) + assert stop_btn is not None, "Stop Scenario button not found" + client.button_command(stop_btn.key) + await asyncio.sleep(0.5) + + assert tracker.sensor_states["requests_1"], "controller 1 issued no requests" + assert tracker.sensor_states["requests_2"], "controller 2 issued no requests" + count_1 = tracker.sensor_states["requests_1"][-1] + count_2 = tracker.sensor_states["requests_2"][-1] + + # Both must have polled repeatedly, otherwise "fairness" is meaningless. + assert count_1 >= 5 and count_2 >= 5, ( + f"expected both controllers to poll repeatedly, " + f"got controller 1={count_1}, controller 2={count_2}" + ) + # Fair scheduling: the bus alternates between the two pending requests, + # so the counts can differ by at most one in-flight request. + assert abs(count_1 - count_2) <= 1, ( + f"controllers did not get a fair share of the bus: " + f"controller 1 issued {count_1}, controller 2 issued {count_2}" + ) diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index a05b683a5f..80f572d9fe 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -2475,6 +2475,35 @@ def test_should_run_benchmarks_core_header_change() -> None: assert determine_jobs.should_run_benchmarks() is True +def test_should_run_benchmarks_top_level_python_change() -> None: + """Test benchmarks trigger on top-level esphome Python module changes. + + The Python benchmarks exercise config loading, so changes to modules + like config.py and yaml_util.py must run them; a regression in #16718 + went unnoticed because these files matched no trigger. + """ + for py_file in [ + "esphome/config.py", + "esphome/yaml_util.py", + "esphome/__main__.py", + "esphome/helpers.py", + ]: + with patch.object(determine_jobs, "changed_files", return_value=[py_file]): + assert determine_jobs.should_run_benchmarks() is True, ( + f"Expected benchmarks to run for {py_file}" + ) + + +def test_should_run_benchmarks_nested_python_change() -> None: + """Test benchmarks do NOT trigger for nested non-core Python changes.""" + with patch.object( + determine_jobs, + "changed_files", + return_value=["esphome/dashboard/web_server.py"], + ): + assert determine_jobs.should_run_benchmarks() is False + + def test_should_run_benchmarks_host_platform_change() -> None: """Test benchmarks trigger on host platform changes. diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 43c4445dcf..077b6ef23e 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -1851,3 +1851,24 @@ def test_get_component_test_files_component_without_tests( ) def test_is_validate_only_file(filename: str, expected: bool, tmp_path: Path) -> None: assert helpers.is_validate_only_file(tmp_path / filename) is expected + + +@pytest.mark.parametrize( + ("files", "expected"), + [ + (["esphome/config.py"], True), + (["esphome/yaml_util.py"], True), + (["esphome/__main__.py"], True), + (["esphome/const.pyi"], True), + (["README.md", "esphome/helpers.py"], True), + (["esphome/core/config.py"], False), + (["esphome/components/sensor/__init__.py"], False), + (["esphome/dashboard/web_server.py"], False), + (["esphome/idf_component.yml"], False), + (["tests/unit_tests/test_config.py"], False), + ([], False), + ], +) +def test_base_python_changed(files: list[str], expected: bool) -> None: + """Only Python modules directly in esphome/ count as base Python changes.""" + assert helpers.base_python_changed(files) is expected diff --git a/tests/test_build_components/common/uart/esp32-h2-idf.yaml b/tests/test_build_components/common/uart/esp32-h2-idf.yaml new file mode 100644 index 0000000000..51d45fe6d5 --- /dev/null +++ b/tests/test_build_components/common/uart/esp32-h2-idf.yaml @@ -0,0 +1,13 @@ +# Common UART configuration for ESP32-H2 IDF tests +# Provides a shared UART bus that components can use +# Components will auto-use this bus if they don't specify uart_id + +substitutions: + tx_pin: GPIO12 + rx_pin: GPIO13 + +uart: + - id: uart_bus + tx_pin: ${tx_pin} + rx_pin: ${rx_pin} + baud_rate: 9600 diff --git a/tests/unit_tests/analyze_memory/test_build_artifacts.py b/tests/unit_tests/analyze_memory/test_build_artifacts.py index ad2f8c2210..d97ee94e1c 100644 --- a/tests/unit_tests/analyze_memory/test_build_artifacts.py +++ b/tests/unit_tests/analyze_memory/test_build_artifacts.py @@ -143,8 +143,8 @@ def test_cc_path_from_cxx(cxx_path: str, expected: str) -> None: def test_native_idedata_resolves_toolchain_tools() -> None: """The binutils paths are derived from the native ESP-IDF cc_path. - Without cc_path, IDEData.objdump_path raises KeyError and the memory - analysis silently degrades to no component or symbol detail. + Without cc_path, IDEData.objdump_path raises EsphomeError and the + memory analysis silently degrades to no component or symbol detail. """ idedata = IDEData( { diff --git a/tests/unit_tests/components/api/test_client.py b/tests/unit_tests/components/api/test_client.py deleted file mode 100644 index 4ebcecbfff..0000000000 --- a/tests/unit_tests/components/api/test_client.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Tests for esphome.components.api.client.""" - -from __future__ import annotations - -from unittest.mock import AsyncMock, patch - -import pytest - -from esphome.components import esp32 -from esphome.components.api import client as api_client -from esphome.const import CONF_PORT, KEY_CORE, KEY_TARGET_PLATFORM -from esphome.core import CORE, EsphomeError - - -def test_decoder_swallows_esphome_error() -> None: - """A failing stack-trace decode must not propagate. - - aioesphomeapi isolates exceptions raised by log handlers, so an - escaping one logs a full traceback for every line it fires on rather - than being reported once as an unavailable decoder. - """ - config = {"esphome": {"name": "test"}} - - with patch.object( - esp32, "process_stacktrace", side_effect=EsphomeError("no idedata") - ) as mock_process: - processor = api_client._LogLineProcessor(config, esp32.process_stacktrace) - processor.process_line("PC: 0x4010496e") - - assert mock_process.called - assert processor.backtrace_state is False - - -def test_decoder_swallows_platform_handler_error() -> None: - """The same protection must apply to the platform-specific handler.""" - config = {"esphome": {"name": "test"}} - - def platform_handler(_config, _line, _state): - raise EsphomeError("no idedata") - - processor = api_client._LogLineProcessor(config, platform_handler) - processor.process_line("PC: 0x4010496e") - - assert processor.backtrace_state is False - - -def test_decoder_swallows_non_esphome_error() -> None: - """Decoding failures that aren't EsphomeError must be contained too. - - A missing build directory surfaces as FileNotFoundError from the toolchain - subprocess. aioesphomeapi isolates it, so the session survives, but it logs - a traceback for every PC/BT line and decoding is never disabled, which - buries the crash dump the user is trying to read. - """ - config = {"esphome": {"name": "test"}} - - with patch.object( - esp32, - "process_stacktrace", - side_effect=FileNotFoundError( - 2, "No such file or directory", "/build/ol/build" - ), - ) as mock_process: - processor = api_client._LogLineProcessor(config, esp32.process_stacktrace) - processor.process_line("PC: 0x4010496e") - processor.process_line("BT0: 0x4010496e") - - # Disabled after the first failure rather than retried per backtrace line. - assert mock_process.call_count == 1 - assert processor.backtrace_state is False - - -def test_decoder_warning_uses_fallback_for_empty_error(caplog) -> None: - """_run_idedata raises EsphomeError with no message; the warning - must show a useful explanation rather than empty parens. - """ - config = {"esphome": {"name": "test"}} - - with patch.object(esp32, "process_stacktrace", side_effect=EsphomeError()): - processor = api_client._LogLineProcessor(config, esp32.process_stacktrace) - processor.process_line("PC: 0x4010496e") - - warnings = [r.message for r in caplog.records if r.levelname == "WARNING"] - assert any("build artifacts not found locally" in m for m in warnings) - assert not any("()" in m for m in warnings) - - -def test_decoder_short_circuits_after_failure() -> None: - """After one failure, subsequent lines must not retry the decoder. - - _decode_pc shells out to the toolchain; a crash dump can contain many - PC/BT lines and retrying the failing subprocess for each one would - stall log streaming. - """ - config = {"esphome": {"name": "test"}} - - with patch.object( - esp32, "process_stacktrace", side_effect=EsphomeError("no idedata") - ) as mock_process: - processor = api_client._LogLineProcessor(config, esp32.process_stacktrace) - processor.process_line("PC: 0x4010496e") - processor.process_line("BT0: 0x4010496e") - processor.process_line("BT1: 0x401049aa") - - assert mock_process.call_count == 1 - - -def test_decoder_threads_backtrace_state() -> None: - """When decoding succeeds, backtrace_state is threaded across calls.""" - config = {"esphome": {"name": "test"}} - - with patch.object( - esp32, "process_stacktrace", side_effect=[True, False] - ) as mock_process: - processor = api_client._LogLineProcessor(config, esp32.process_stacktrace) - processor.process_line(">>>stack>>>") - assert processor.backtrace_state is True - processor.process_line("<< None: - """The platform handler is preferred over the generic one.""" - config = {"esphome": {"name": "test"}} - calls: list[tuple[object, str, bool]] = [] - - def platform_handler(cfg, line, state): - calls.append((cfg, line, state)) - return True - - processor = api_client._LogLineProcessor(config, platform_handler) - - with patch.object(esp32, "process_stacktrace") as mock_generic: - processor.process_line("BT0: 0x4010496e") - - assert calls == [(config, "BT0: 0x4010496e", False)] - assert mock_generic.called is False - assert processor.backtrace_state is True - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("extra_config", "expected_deep_sleep"), - [({"deep_sleep": {}}, True), ({}, False)], -) -async def test_async_run_logs_passes_deep_sleep( - extra_config: dict, expected_deep_sleep: bool -) -> None: - """async_run_logs tells async_run whether the device deep sleeps, from the config.""" - CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} - config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}, **extra_config} - # async_run blocks forever after connecting; raise to unwind async_run_logs - # once we have captured how it was called. - sentinel = RuntimeError("stop the wait") - - with ( - patch.object( - api_client, "async_run", AsyncMock(side_effect=sentinel) - ) as mock_run, - patch.object(api_client, "APIClient"), - pytest.raises(RuntimeError, match="stop the wait"), - ): - await api_client.async_run_logs(config, ["1.2.3.4"]) - - assert mock_run.call_args.kwargs["deep_sleep"] is expected_deep_sleep diff --git a/tests/unit_tests/components/mqtt/__init__.py b/tests/unit_tests/components/mqtt/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/components/mqtt/test_object_id_conflicts.py b/tests/unit_tests/components/mqtt/test_object_id_conflicts.py new file mode 100644 index 0000000000..0ac61b4dad --- /dev/null +++ b/tests/unit_tests/components/mqtt/test_object_id_conflicts.py @@ -0,0 +1,239 @@ +"""Tests for the MQTT object_id conflict filter. + +MQTT still builds default topics and discovery topics from the sanitized +object_id, so entity names that only differ in characters lost during +sanitizing conflict there; _topics_conflict() exempts entities that never +use an object_id-derived topic. See https://github.com/esphome/backlog/issues/85 +""" + +from pathlib import Path + +import pytest + +from esphome.components.mqtt import ( + _COMMAND_TOPIC_PLATFORMS, + _SUB_TOPIC_PLATFORMS, + _topics_conflict, +) +from esphome.config_validation import Invalid +from esphome.const import ( + CONF_COMMAND_TOPIC, + CONF_DISCOVERY, + CONF_NAME, + CONF_STATE_TOPIC, + CONF_TOPIC_PREFIX, +) +from esphome.core import CORE +from esphome.core.entity_helpers import ( + entity_duplicate_validator, + validate_no_object_id_conflicts, +) + +COMPONENTS_DIR = Path(__file__).parents[4] / "esphome" / "components" + +REASON = "mqtt builds default topics from the entity object_id" + + +# MQTT infrastructure sources, not entity components +_NON_ENTITY_MQTT_SOURCES = {"mqtt_client", "mqtt_component"} +# The date, time and datetime MQTT components all belong to the datetime platform +_DATETIME_STEMS = {"date", "time", "datetime"} + + +def test_command_topic_platforms_in_sync() -> None: + """Verify _COMMAND_TOPIC_PLATFORMS matches the MQTT components that subscribe. + + Drift silently reintroduces shared subscribe topics, so this derives the set + from the C++ components that actually call subscribe(); that also catches + platforms like text that subscribe a command topic without exposing a + command_topic key in their schema. + """ + expected: set[str] = set() + for path in (COMPONENTS_DIR / "mqtt").glob("mqtt_*.cpp"): + if path.stem in _NON_ENTITY_MQTT_SOURCES: + continue + if "this->subscribe" not in path.read_text(encoding="utf-8"): + continue + stem = path.stem.removeprefix("mqtt_") + expected.add("datetime" if stem in _DATETIME_STEMS else stem) + assert expected == _COMMAND_TOPIC_PLATFORMS + + +def test_sub_topic_platforms_in_sync() -> None: + """Verify _SUB_TOPIC_PLATFORMS matches the MQTT components with sub-topics. + + Platforms whose MQTT headers use MQTT_COMPONENT_CUSTOM_TOPIC derive extra + topics such as position/command from the object_id. + """ + expected = { + path.stem.removeprefix("mqtt_") + for path in (COMPONENTS_DIR / "mqtt").glob("mqtt_*.h") + if path.stem != "mqtt_component" + and "MQTT_COMPONENT_CUSTOM_TOPIC" in path.read_text(encoding="utf-8") + } + assert expected == _SUB_TOPIC_PLATFORMS + + +def test_conflict_filter_exempts_custom_topics() -> None: + """Test that custom state topics with discovery off avoid the conflict.""" + validator = entity_duplicate_validator("sensor") + # Both entities have custom state topics and discovery disabled per entity, + # so no object_id-derived MQTT topic is used + validator( + { + CONF_NAME: "Датчик открытия", + CONF_STATE_TOPIC: "custom/topic/a", + CONF_DISCOVERY: False, + } + ) + validator( + { + CONF_NAME: "Датчик закрытия", + CONF_STATE_TOPIC: "custom/topic/b", + CONF_DISCOVERY: False, + } + ) + + component_validator = validate_no_object_id_conflicts( + REASON, conflict_filter=_topics_conflict + ) + config: dict = {CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: "test-device"} + assert component_validator(config) is config + + # Without the filter the same conflicts are fatal + with pytest.raises(Invalid, match=r"mqtt builds default topics"): + validate_no_object_id_conflicts(REASON)({}) + + +def test_conflict_on_default_command_topic() -> None: + """Test that commandable platforms conflict through their default command topic. + + Custom state topics with discovery off are not enough for platforms that also + subscribe to an object_id-derived command topic. + """ + validator = entity_duplicate_validator("switch") + validator( + { + CONF_NAME: "Датчик открытия", + CONF_STATE_TOPIC: "custom/topic/a", + CONF_DISCOVERY: False, + } + ) + validator( + { + CONF_NAME: "Датчик закрытия", + CONF_STATE_TOPIC: "custom/topic/b", + CONF_DISCOVERY: False, + } + ) + + component_validator = validate_no_object_id_conflicts( + REASON, conflict_filter=_topics_conflict + ) + mqtt_config: dict = {CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: "test-device"} + # Both switches share the default command topic: rejected + with pytest.raises(Invalid, match=r"mqtt builds default topics"): + component_validator(mqtt_config) + + # With custom command topics as well, nothing derives from the object_id + CORE.reset() + validator = entity_duplicate_validator("switch") + validator( + { + CONF_NAME: "Датчик открытия", + CONF_STATE_TOPIC: "custom/topic/a", + CONF_COMMAND_TOPIC: "custom/cmd/a", + CONF_DISCOVERY: False, + } + ) + validator( + { + CONF_NAME: "Датчик закрытия", + CONF_STATE_TOPIC: "custom/topic/b", + CONF_COMMAND_TOPIC: "custom/cmd/b", + CONF_DISCOVERY: False, + } + ) + assert component_validator(mqtt_config) is mqtt_config + + +def test_conflict_on_sub_topic_platforms() -> None: + """Test that platforms with extra object_id sub-topics always conflict. + + Covers derive topics like position/command from the object_id through their + own config keys, so custom state and command topics cannot exempt them. + """ + validator = entity_duplicate_validator("cover") + validator( + { + CONF_NAME: "Датчик открытия", + CONF_STATE_TOPIC: "custom/topic/a", + CONF_COMMAND_TOPIC: "custom/cmd/a", + CONF_DISCOVERY: False, + } + ) + validator( + { + CONF_NAME: "Датчик закрытия", + CONF_STATE_TOPIC: "custom/topic/b", + CONF_COMMAND_TOPIC: "custom/cmd/b", + CONF_DISCOVERY: False, + } + ) + + component_validator = validate_no_object_id_conflicts( + REASON, conflict_filter=_topics_conflict + ) + with pytest.raises(Invalid, match=r"mqtt builds default topics"): + component_validator({CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: "test-device"}) + + +def test_no_conflict_on_disjoint_default_topics() -> None: + """Test that entities whose default topics are disjoint do not conflict. + + One entity uses only the default command topic and the other only the default + state topic, so they never share a topic. + """ + validator = entity_duplicate_validator("switch") + validator( + { + CONF_NAME: "Датчик открытия", + CONF_STATE_TOPIC: "custom/topic/a", + CONF_DISCOVERY: False, + } + ) + validator( + { + CONF_NAME: "Датчик закрытия", + CONF_COMMAND_TOPIC: "custom/cmd/b", + CONF_DISCOVERY: False, + } + ) + + component_validator = validate_no_object_id_conflicts( + REASON, conflict_filter=_topics_conflict + ) + config: dict = {CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: "test-device"} + assert component_validator(config) is config + + +def test_no_conflict_on_empty_topic_prefix() -> None: + """Test that an empty topic_prefix disables the default topic conflict. + + With topic_prefix set to null no default topics exist at runtime, so entities + without custom state topics cannot conflict; only discovery still matters. + """ + validator = entity_duplicate_validator("sensor") + validator({CONF_NAME: "Датчик открытия"}) + validator({CONF_NAME: "Датчик закрытия"}) + + component_validator = validate_no_object_id_conflicts( + REASON, conflict_filter=_topics_conflict + ) + # No default topics and no discovery: valid + config: dict = {CONF_DISCOVERY: False, CONF_TOPIC_PREFIX: ""} + assert component_validator(config) is config + + # Discovery still uses object_id-derived config topics: rejected + with pytest.raises(Invalid, match=r"mqtt builds default topics"): + component_validator({CONF_DISCOVERY: True, CONF_TOPIC_PREFIX: ""}) diff --git a/tests/unit_tests/components/test_esp32_rmt_led_strip.py b/tests/unit_tests/components/test_esp32_rmt_led_strip.py new file mode 100644 index 0000000000..e2cb513e3b --- /dev/null +++ b/tests/unit_tests/components/test_esp32_rmt_led_strip.py @@ -0,0 +1,57 @@ +import pytest + +from esphome.components.esp32_rmt_led_strip.light import ( + CONF_IS_WRGB, + CONF_RGBW_ORDER, + _split_rgbw_order, + _validate_rgbw_order, + _validate_rgbw_order_exclusivity, +) +import esphome.config_validation as cv +from esphome.const import CONF_IS_RGBW + + +def test_validate_rgbw_order() -> None: + assert _validate_rgbw_order("rwgb") == "RWGB" + + +@pytest.mark.parametrize("rgbw_order", ["RGB", "RRGB", "RGBWW"]) +def test_validate_rgbw_order_rejects_invalid_order(rgbw_order: str) -> None: + with pytest.raises(cv.Invalid, match="permutation of RGBW"): + _validate_rgbw_order(rgbw_order) + + +@pytest.mark.parametrize( + ("rgbw_order", "expected"), + [ + ("WRGB", ("RGB", 0)), + ("RWGB", ("RGB", 1)), + ("GWRB", ("GRB", 1)), + ("RGBW", ("RGB", 3)), + ], +) +def test_split_rgbw_order(rgbw_order: str, expected: tuple[str, int]) -> None: + assert _split_rgbw_order(rgbw_order) == expected + + +@pytest.mark.parametrize("conflict", [CONF_IS_RGBW, CONF_IS_WRGB]) +def test_rgbw_order_is_mutually_exclusive(conflict: str) -> None: + with pytest.raises(cv.Invalid, match="cannot be used with"): + _validate_rgbw_order_exclusivity( + { + CONF_RGBW_ORDER: "RGBW", + CONF_IS_RGBW: conflict == CONF_IS_RGBW, + CONF_IS_WRGB: conflict == CONF_IS_WRGB, + } + ) + + +@pytest.mark.parametrize("legacy_option", [CONF_IS_RGBW, CONF_IS_WRGB]) +def test_rgbw_order_allows_disabled_legacy_options(legacy_option: str) -> None: + config = { + CONF_RGBW_ORDER: "RGBW", + CONF_IS_RGBW: False, + CONF_IS_WRGB: False, + } + config[legacy_option] = False + assert _validate_rgbw_order_exclusivity(config) is config diff --git a/tests/unit_tests/components/test_esp_stacktrace.py b/tests/unit_tests/components/test_esp_stacktrace.py index eb7e63fc4d..ed1b12029e 100644 --- a/tests/unit_tests/components/test_esp_stacktrace.py +++ b/tests/unit_tests/components/test_esp_stacktrace.py @@ -179,3 +179,35 @@ def test_process_stacktrace_esp32_crash_handler( state = process_stacktrace(config, line_mtval_data, False) mock_esp32_decode_pc.assert_not_called() assert state is False + + +def test_process_stacktrace_esp32_foreign_crash( + setup_core: Path, mock_esp32_decode_pc: Mock +) -> None: + """Crash records from a different firmware build must not be decoded.""" + from esphome.components.esp32 import process_stacktrace + + config = {"name": "test"} + + line_note = ( + "[E][esp32.crash:390]: Captured by a different firmware build; " + "addresses belong to that build's ELF" + ) + state = process_stacktrace(config, line_note, False) + mock_esp32_decode_pc.assert_not_called() + assert state is False + + # Lowercase labels are deliberately not matched by any decoder regex, + # since symbols would come from the wrong ELF + lines_addrs = [ + "[E][esp32.crash:391]: pc: 0x400D1234", + "[E][esp32.crash:392]: excvaddr: 0x400D5678", + "[E][esp32.crash:392]: mtval: 0x42001234", + "[E][esp32.crash:393]: bt0: 0x400F19A6", + "[E][esp32.crash:394]: other core (0):", + "[E][esp32.crash:395]: bt15: 0x42005ABC", + ] + for line in lines_addrs: + state = process_stacktrace(config, line, False) + mock_esp32_decode_pc.assert_not_called() + assert state is False diff --git a/tests/unit_tests/components/test_libretiny.py b/tests/unit_tests/components/test_libretiny.py index ee00bdc180..de54fcc6ca 100644 --- a/tests/unit_tests/components/test_libretiny.py +++ b/tests/unit_tests/components/test_libretiny.py @@ -2,7 +2,8 @@ import pytest -from esphome.components.libretiny import _detect_variant +from esphome.components import bk72xx, ln882x, rtl87xx +from esphome.components.libretiny import BASE_SCHEMA, _detect_variant from esphome.components.libretiny.const import ( FAMILY_LN882H, KEY_COMPONENT_DATA, @@ -11,7 +12,7 @@ from esphome.components.libretiny.const import ( from esphome.components.ln882x import COMPONENT_DATA import esphome.config_validation as cv from esphome.const import CONF_BOARD, CONF_FAMILY -from esphome.core import CORE +from esphome.core import CORE, KEY_CORE @pytest.fixture @@ -50,3 +51,36 @@ def test_detect_variant_unknown_board_still_raises(ln882x_core_data: None) -> No """Ids outside the rename map keep the family-override error.""" with pytest.raises(cv.Invalid, match="This board is unknown"): _detect_variant({CONF_BOARD: "not-a-real-board"}) + + +def test_platform_schemas_are_isolated_instances() -> None: + """Each LibreTiny platform must own its CONFIG_SCHEMA instance. + + BASE_SCHEMA is shared; every platform prepends its own _set_core_data + extra. On the shared object, importing two platform modules in one process + made either platform's validation run both extras, so the wrong platform's + component data won and known boards failed to resolve. + """ + platforms = (bk72xx, ln882x, rtl87xx) + schemas = [platform.CONFIG_SCHEMA for platform in platforms] + assert len({id(schema) for schema in (BASE_SCHEMA, *schemas)}) == 4 + # The shared base must not have accumulated any platform's extra. + # prepend_extra wraps validators in _Schema, so unwrap before comparing. + base_extras = [extra.schema for extra in BASE_SCHEMA._extra_schemas] + for platform in platforms: + assert platform._set_core_data not in base_extras + + +def test_each_platform_resolves_its_own_boards() -> None: + """Validating one platform's config must leave that platform's component + data in CORE.data. On the shared schema, the last-imported platform's + _set_core_data won for every platform, so known boards failed to resolve + with "This board is unknown".""" + CORE.data[KEY_CORE] = {} # written by the schema's _update_core_data extra + for platform, board in ( + (ln882x, "generic-ln882h"), + (bk72xx, "generic-bk7252"), + (rtl87xx, "generic-rtl8720cf-2mb-896k"), + ): + platform.CONFIG_SCHEMA({CONF_BOARD: board}) + assert CORE.data[KEY_LIBRETINY][KEY_COMPONENT_DATA] is platform.COMPONENT_DATA diff --git a/tests/unit_tests/components/test_rp2_generate_boards.py b/tests/unit_tests/components/test_rp2_generate_boards.py index 68bbada59b..c5d2214695 100644 --- a/tests/unit_tests/components/test_rp2_generate_boards.py +++ b/tests/unit_tests/components/test_rp2_generate_boards.py @@ -158,19 +158,122 @@ def test_load_basic_board(arduino_pico: Path) -> None: def test_load_rp2350_board(arduino_pico: Path) -> None: + """The Pico 2 uses the RP2350A die, which only exposes GPIO 0-29.""" _add_board( arduino_pico, "rpipico2", mcu="rp2350", vendor="Raspberry Pi", name="Pico 2", - pins_header=PICO_PINS_HEADER, + pins_header="#define PICO_RP2350A 1\n" + PICO_PINS_HEADER, ) _, boards = load_boards(arduino_pico) assert boards["rpipico2"]["mcu"] == "rp2350" - assert boards["rpipico2"]["max_pin"] == 47 + assert boards["rpipico2"]["max_pin"] == 29 + + +def test_rp2350_missing_die_define_raises(arduino_pico: Path) -> None: + """A variant without PICO_RP2350A cannot be classified; fail loudly.""" + _add_board( + arduino_pico, + "no_die_define", + mcu="rp2350", + pins_header=PICO_PINS_HEADER, + ) + + with pytest.raises(ValueError, match="no PICO_RP2350A define"): + load_boards(arduino_pico) + + +def test_rp2350_unrecognized_die_define_raises(arduino_pico: Path) -> None: + """An unparseable PICO_RP2350A value must not silently widen to B-die.""" + _add_board( + arduino_pico, + "hex_die_define", + mcu="rp2350", + pins_header="#define PICO_RP2350A 0x1\n" + PICO_PINS_HEADER, + ) + + with pytest.raises(ValueError, match="unrecognized PICO_RP2350A value"): + load_boards(arduino_pico) + + +def test_rp2350a_parenthesized_die_define(arduino_pico: Path) -> None: + """Literal forms like (1u) classify the same as bare 1.""" + _add_board( + arduino_pico, + "paren_die", + mcu="rp2350", + pins_header="#define PICO_RP2350A (1u)\n" + PICO_PINS_HEADER, + ) + + _, boards = load_boards(arduino_pico) + + assert boards["paren_die"]["max_pin"] == 29 + + +def test_rp2350b_board_keeps_max_pin_47(arduino_pico: Path) -> None: + """A variant declaring the RP2350B die keeps the full GPIO 0-47 range. + + The define uses extra whitespace, matching real variant headers. + """ + _add_board( + arduino_pico, + "weact_rp2350b", + mcu="rp2350", + pins_header="#define PICO_RP2350A 0 // RP2350B\n" + PICO_PINS_HEADER, + ) + + _, boards = load_boards(arduino_pico) + + assert boards["weact_rp2350b"]["max_pin"] == 47 + + +def test_rp2350_menu_selectable_die_keeps_max_pin_47(arduino_pico: Path) -> None: + """Generic boards leave the die a build-time choice; stay permissive.""" + _add_board( + arduino_pico, + "generic_rp2350", + mcu="rp2350", + pins_header="#define PICO_RP2350A __PICO_RP2350A\n" + PICO_PINS_HEADER, + ) + + _, boards = load_boards(arduino_pico) + + assert boards["generic_rp2350"]["max_pin"] == 47 + + +def test_rp2350a_pins_above_29_filtered(arduino_pico: Path) -> None: + """Pin defines beyond the A-die range are dropped from the pin map.""" + header = textwrap.dedent("""\ + #define PICO_RP2350A 1 + #define PIN_LED (25u) + #define PIN_SPI0_MISO (40u) + """) + _add_board(arduino_pico, "a_die", mcu="rp2350", pins_header=header) + + board_pins, _ = load_boards(arduino_pico) + + assert board_pins["a_die"]["LED"] == 25 + assert "MISO" not in board_pins["a_die"] + + +def test_rp2350a_board_keeps_cyw43_virtual_pins(arduino_pico: Path) -> None: + """A-die narrowing must not filter CYW43 virtual pins (64-66).""" + _add_board( + arduino_pico, + "rpipico2w", + mcu="rp2350", + pins_header="#define PICO_RP2350A 1\n" + PICOW_PINS_HEADER, + ) + + board_pins, boards = load_boards(arduino_pico) + + assert boards["rpipico2w"]["max_pin"] == 29 + assert boards["rpipico2w"]["max_virtual_pin"] == 64 + assert board_pins["rpipico2w"]["LED"] == 64 def test_cyw43_board_has_max_virtual_pin(arduino_pico: Path) -> None: diff --git a/tests/unit_tests/core/common.py b/tests/unit_tests/core/common.py index daa429dc96..96fcc5b1c6 100644 --- a/tests/unit_tests/core/common.py +++ b/tests/unit_tests/core/common.py @@ -29,5 +29,5 @@ def load_config_from_fixture( ) -> Config | None: """Load configuration from a fixture file.""" fixture_path = fixtures_dir / fixture_name - yaml_content = fixture_path.read_text() + yaml_content = fixture_path.read_text(encoding="utf-8") return load_config_from_yaml(yaml_file, yaml_content) diff --git a/tests/unit_tests/core/conftest.py b/tests/unit_tests/core/conftest.py index 42e59c15e6..9ef31a82b9 100644 --- a/tests/unit_tests/core/conftest.py +++ b/tests/unit_tests/core/conftest.py @@ -12,7 +12,7 @@ def yaml_file(tmp_path: Path) -> Callable[[str], Path]: def _yaml_file(content: str) -> Path: yaml_path = tmp_path / "test.yaml" - yaml_path.write_text(content) + yaml_path.write_text(content, encoding="utf-8") return yaml_path return _yaml_file diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index 0362c40bce..e09edd7f26 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -1242,6 +1242,15 @@ def test_make_app_name_cpp_special_chars_escaped() -> None: None, "https://github.com/esphome/noise-c.git", ), + # A local file:// source is routed to the repository, not a registry name + # -- including the fewer-than-two-slashes spelling. + ( + "TeslaBLE=file:///config/esphome/lib_dev", + "TeslaBLE", + None, + "file:///config/esphome/lib_dev", + ), + ("MyLib=file:lib_dev", "MyLib", None, "file:lib_dev"), ], ) def test_add_library_str( diff --git a/tests/unit_tests/core/test_entity_helpers.py b/tests/unit_tests/core/test_entity_helpers.py index 3ac4ce27af..64400c4fd4 100644 --- a/tests/unit_tests/core/test_entity_helpers.py +++ b/tests/unit_tests/core/test_entity_helpers.py @@ -1,4 +1,4 @@ -"""Test get_base_entity_object_id function matches C++ behavior.""" +"""Tests for entity helpers: name selection, entity key hashing, duplicate checks.""" from collections.abc import Callable, Generator from pathlib import Path @@ -25,16 +25,17 @@ from esphome.core.entity_helpers import ( _setup_entity_impl, entity_duplicate_validator, finalize_entity_strings, - get_base_entity_object_id, + get_base_entity_name, register_device_class, register_icon, register_unit_of_measurement, setup_device_class, setup_entity, setup_unit_of_measurement, + validate_no_object_id_conflicts, ) from esphome.cpp_generator import MockObj -from esphome.helpers import sanitize, snake_case +from esphome.helpers import fnv1_hash_name, sanitize, snake_case from .common import load_config_from_fixture @@ -57,206 +58,26 @@ def restore_core_state() -> Generator[None, None, None]: CORE.friendly_name = original_friendly_name -def test_with_entity_name() -> None: - """Test when entity has its own name - should use entity name.""" - # Simple name - assert get_base_entity_object_id("Temperature Sensor", None) == "temperature_sensor" - assert ( - get_base_entity_object_id("Temperature Sensor", "Device Name") - == "temperature_sensor" - ) - # Even with device name, entity name takes precedence - assert ( - get_base_entity_object_id("Temperature Sensor", "Device Name", "Sub Device") - == "temperature_sensor" - ) - - # Name with special characters - assert ( - get_base_entity_object_id("Temp!@#$%^&*()Sensor", None) - == "temp__________sensor" - ) - assert get_base_entity_object_id("Temp-Sensor_123", None) == "temp-sensor_123" - - # Already snake_case - assert get_base_entity_object_id("temperature_sensor", None) == "temperature_sensor" - - # Mixed case - assert get_base_entity_object_id("TemperatureSensor", None) == "temperaturesensor" - assert get_base_entity_object_id("TEMPERATURE SENSOR", None) == "temperature_sensor" - - -def test_empty_name_with_device_name() -> None: - """Test when entity has empty name and is on a sub-device - should use device name.""" - # C++ behavior: when has_own_name is false and device is set, uses device->get_name() - assert ( - get_base_entity_object_id("", "Friendly Device", "Sub Device 1") - == "sub_device_1" - ) - assert ( - get_base_entity_object_id("", "Kitchen Controller", "controller_1") - == "controller_1" - ) - assert get_base_entity_object_id("", None, "Test-Device_123") == "test-device_123" - - -def test_empty_name_with_friendly_name() -> None: - """Test when entity has empty name and no device - should use friendly name.""" - # C++ behavior: when has_own_name is false, uses App.get_friendly_name() - assert get_base_entity_object_id("", "Friendly Device") == "friendly_device" - assert get_base_entity_object_id("", "Kitchen Controller") == "kitchen_controller" - assert get_base_entity_object_id("", "Test-Device_123") == "test-device_123" - - # Special characters in friendly name - assert get_base_entity_object_id("", "Device!@#$%") == "device_____" - - -def test_empty_name_no_friendly_name() -> None: - """Test when entity has empty name and no friendly name - should use device name.""" - # Test with CORE.name set - CORE.name = "device-name" - assert get_base_entity_object_id("", None) == "device-name" - - CORE.name = "Test Device" - assert get_base_entity_object_id("", None) == "test_device" - - -def test_edge_cases() -> None: - """Test edge cases.""" - # Only spaces - assert get_base_entity_object_id(" ", None) == "___" - - # Unicode characters (should be replaced) - assert get_base_entity_object_id("Température", None) == "temp_rature" - assert get_base_entity_object_id("测试", None) == "__" - - # Empty string with empty friendly name (empty friendly name is treated as None) - # Falls back to CORE.name - CORE.name = "device" - assert get_base_entity_object_id("", "") == "device" - - # Very long name (should work fine) - long_name = "a" * 100 + " " + "b" * 100 - expected = "a" * 100 + "_" + "b" * 100 - assert get_base_entity_object_id(long_name, None) == expected - - -@pytest.mark.parametrize( - ("name", "expected"), - [ - ("Temperature Sensor", "temperature_sensor"), - ("Living Room Light", "living_room_light"), - ("Test-Device_123", "test-device_123"), - ("Special!@#Chars", "special___chars"), - ("UPPERCASE NAME", "uppercase_name"), - ("lowercase name", "lowercase_name"), - ("Mixed Case Name", "mixed_case_name"), - (" Spaces ", "___spaces___"), - ], -) -def test_matches_cpp_helpers(name: str, expected: str) -> None: - """Test that the logic matches using snake_case and sanitize directly.""" - # For non-empty names, verify our function produces same result as direct snake_case + sanitize - assert get_base_entity_object_id(name, None) == sanitize(snake_case(name)) - assert get_base_entity_object_id(name, None) == expected - - -def test_empty_name_fallback() -> None: - """Test empty name handling which falls back to friendly_name or CORE.name.""" - # Empty name is handled specially - it doesn't just use sanitize(snake_case("")) - # Instead it falls back to friendly_name or CORE.name - assert sanitize(snake_case("")) == "" # Direct conversion gives empty string - # But our function returns a fallback - CORE.name = "device" - assert get_base_entity_object_id("", None) == "device" # Uses device name - - -def test_name_add_mac_suffix_behavior() -> None: - """Test behavior related to name_add_mac_suffix. - - In C++, an entity's object_id is computed from its name_ via - write_object_id_to() (sanitized snake_case). When an entity has no name, - configure_entity_() sets name_ from the friendly name, with the MAC suffix - appended when name_add_mac_suffix is enabled. Our function always returns - the same result since we're calculating the base for duplicate tracking. - """ - # The function should always return the same result regardless of - # name_add_mac_suffix setting, as we're calculating the base object_id - assert get_base_entity_object_id("", "Test Device") == "test_device" - assert get_base_entity_object_id("Entity Name", "Test Device") == "entity_name" - - -def test_priority_order() -> None: +def test_get_base_entity_name_priority_order() -> None: """Test the priority order: entity name > device name > friendly name > CORE.name.""" CORE.name = "core-device" - # 1. Entity name has highest priority + # 1. Entity name has highest priority and is used as-is, no transformations assert ( - get_base_entity_object_id("Entity Name", "Friendly Name", "Device Name") - == "entity_name" + get_base_entity_name("Entity Name", "Friendly Name", "Device Name") + == "Entity Name" ) + assert get_base_entity_name("Température", None) == "Température" # 2. Device name is next priority (when entity name is empty) - assert ( - get_base_entity_object_id("", "Friendly Name", "Device Name") == "device_name" - ) + assert get_base_entity_name("", "Friendly Name", "Device Name") == "Device Name" # 3. Friendly name is next (when entity and device names are empty) - assert get_base_entity_object_id("", "Friendly Name", None) == "friendly_name" + assert get_base_entity_name("", "Friendly Name", None) == "Friendly Name" - # 4. CORE.name is last resort - assert get_base_entity_object_id("", None, None) == "core-device" - - -@pytest.mark.parametrize( - ("name", "friendly_name", "device_name", "expected"), - [ - # name, friendly_name, device_name, expected - ("Living Room Light", None, None, "living_room_light"), - ("", "Kitchen Controller", None, "kitchen_controller"), - ( - "", - "ESP32 Device", - "controller_1", - "controller_1", - ), # Device name takes precedence - ("GPIO2 Button", None, None, "gpio2_button"), - ("WiFi Signal", "My Device", None, "wifi_signal"), - ("", None, "esp32_node", "esp32_node"), - ("Front Door Sensor", "Home Assistant", "door_controller", "front_door_sensor"), - ], -) -def test_real_world_examples( - name: str, friendly_name: str | None, device_name: str | None, expected: str -) -> None: - """Test real-world entity naming scenarios.""" - result = get_base_entity_object_id(name, friendly_name, device_name) - assert result == expected - - -def test_issue_6953_scenarios() -> None: - """Test specific scenarios from issue #6953.""" - # Scenario 1: Multiple empty names on main device with name_add_mac_suffix - # The Python code calculates the base, C++ might append MAC suffix dynamically - CORE.name = "device-name" - CORE.friendly_name = "Friendly Device" - - # All empty names should resolve to same base - assert get_base_entity_object_id("", CORE.friendly_name) == "friendly_device" - assert get_base_entity_object_id("", CORE.friendly_name) == "friendly_device" - assert get_base_entity_object_id("", CORE.friendly_name) == "friendly_device" - - # Scenario 2: Empty names on sub-devices - assert ( - get_base_entity_object_id("", "Main Device", "controller_1") == "controller_1" - ) - assert ( - get_base_entity_object_id("", "Main Device", "controller_2") == "controller_2" - ) - - # Scenario 3: xyz duplicates - assert get_base_entity_object_id("xyz", None) == "xyz" - assert get_base_entity_object_id("xyz", "Device") == "xyz" + # 4. CORE.name is last resort; an empty friendly name falls through to it + assert get_base_entity_name("", None, None) == "core-device" + assert get_base_entity_name("", "") == "core-device" # Tests for setup_entity function @@ -515,9 +336,10 @@ def test_entity_duplicate_validator() -> None: config1 = {CONF_NAME: "Temperature"} validated1 = validator(config1) assert validated1 == config1 - assert ("", "sensor", "temperature") in CORE.unique_ids + temperature_key = ("", "sensor", fnv1_hash_name("Temperature")) + assert temperature_key in CORE.unique_ids # Check metadata was stored - metadata = CORE.unique_ids[("", "sensor", "temperature")] + metadata = CORE.unique_ids[temperature_key] assert metadata["name"] == "Temperature" assert metadata["platform"] == "sensor" @@ -525,8 +347,9 @@ def test_entity_duplicate_validator() -> None: config2 = {CONF_NAME: "Humidity"} validated2 = validator(config2) assert validated2 == config2 - assert ("", "sensor", "humidity") in CORE.unique_ids - metadata2 = CORE.unique_ids[("", "sensor", "humidity")] + humidity_key = ("", "sensor", fnv1_hash_name("Humidity")) + assert humidity_key in CORE.unique_ids + metadata2 = CORE.unique_ids[humidity_key] assert metadata2["name"] == "Humidity" # Duplicate entity should fail @@ -547,18 +370,19 @@ def test_entity_duplicate_validator_with_devices() -> None: device2 = ID("device2", type="Device") # Same name on different devices should pass + name_hash = fnv1_hash_name("Temperature") config1 = {CONF_NAME: "Temperature", CONF_DEVICE_ID: device1} validated1 = validator(config1) assert validated1 == config1 - assert ("device1", "sensor", "temperature") in CORE.unique_ids - metadata1 = CORE.unique_ids[("device1", "sensor", "temperature")] + assert ("device1", "sensor", name_hash) in CORE.unique_ids + metadata1 = CORE.unique_ids[("device1", "sensor", name_hash)] assert metadata1["device_id"] == "device1" config2 = {CONF_NAME: "Temperature", CONF_DEVICE_ID: device2} validated2 = validator(config2) assert validated2 == config2 - assert ("device2", "sensor", "temperature") in CORE.unique_ids - metadata2 = CORE.unique_ids[("device2", "sensor", "temperature")] + assert ("device2", "sensor", name_hash) in CORE.unique_ids + metadata2 = CORE.unique_ids[("device2", "sensor", name_hash)] assert metadata2["device_id"] == "device2" # Duplicate on same device should fail @@ -610,6 +434,33 @@ def test_entity_different_platforms_yaml_validation( assert result is not None +def test_object_id_conflict_mqtt_yaml_validation( + yaml_file: Callable[[str], str], capsys: pytest.CaptureFixture[str] +) -> None: + """Test that names sanitizing to the same object_id fail when mqtt is configured.""" + result = load_config_from_fixture( + yaml_file, "object_id_conflict_mqtt.yaml", FIXTURES_DIR + ) + assert result is None + + captured = capsys.readouterr() + assert ( + "mqtt builds default topics and discovery topics from the entity object_id" + in captured.out + ) + + +def test_object_id_conflict_without_mqtt_yaml_validation( + yaml_file: Callable[[str], str], +) -> None: + """Test that names sanitizing to the same object_id pass without mqtt/prometheus.""" + result = load_config_from_fixture( + yaml_file, "object_id_conflict_no_mqtt.yaml", FIXTURES_DIR + ) + # This should succeed + assert result is not None + + def test_entity_duplicate_validator_error_message() -> None: """Test that duplicate entity error messages include helpful metadata.""" # Create validator for sensor platform @@ -668,7 +519,8 @@ def test_entity_duplicate_validator_internal_entities() -> None: validated1 = validator(config1) assert validated1 == config1 # New format includes device_id (empty string for main device) - assert ("", "sensor", "temperature") in CORE.unique_ids + temperature_key = ("", "sensor", fnv1_hash_name("Temperature")) + assert temperature_key in CORE.unique_ids # Internal entity with same name should pass (not added to unique_ids) config2 = {CONF_NAME: "Temperature", CONF_INTERNAL: True} @@ -676,7 +528,7 @@ def test_entity_duplicate_validator_internal_entities() -> None: assert validated2 == config2 # Internal entity should not be added to unique_ids # Count how many times the key appears (should still be 1) - count = sum(1 for k in CORE.unique_ids if k == ("", "sensor", "temperature")) + count = sum(1 for k in CORE.unique_ids if k == temperature_key) assert count == 1 # Another internal entity with same name should also pass @@ -684,7 +536,7 @@ def test_entity_duplicate_validator_internal_entities() -> None: validated3 = validator(config3) assert validated3 == config3 # Still only one entry in unique_ids (from the non-internal entity) - count = sum(1 for k in CORE.unique_ids if k == ("", "sensor", "temperature")) + count = sum(1 for k in CORE.unique_ids if k == temperature_key) assert count == 1 # Non-internal entity with same name should fail @@ -712,30 +564,148 @@ def test_empty_or_null_device_id_on_entity() -> None: def test_entity_duplicate_validator_non_ascii_names() -> None: - """Test that non-ASCII names show helpful error messages.""" + """Test that distinct non-ASCII names no longer collide. + + These names used to be rejected because both sanitize to only underscores; + the entity key now hashes the raw name so they stay distinct. + """ # Create validator for binary_sensor platform validator = entity_duplicate_validator("binary_sensor") - # First Russian sensor should pass + # Both Russian sensors should pass even though they sanitize identically config1 = {CONF_NAME: "Датчик открытия основного крана"} validated1 = validator(config1) assert validated1 == config1 - # Second Russian sensor with different text but same ASCII conversion should fail config2 = {CONF_NAME: "Датчик закрытия основного крана"} + validated2 = validator(config2) + assert validated2 == config2 + + # An exact duplicate still fails + config3 = {CONF_NAME: "Датчик открытия основного крана"} + with pytest.raises( + Invalid, + match=r"Duplicate binary_sensor entity with name 'Датчик открытия основного крана' found", + ): + validator(config3) + + +def test_entity_duplicate_validator_hash_collision() -> None: + """Test that two different names with the same FNV-1 hash are rejected.""" + # Brute-forced FNV-1 32-bit collision pair; both hash to 0x0ee5ff7b + name_a = "Sensor m2CZ" + name_b = "Sensor qCaa" + assert name_a != name_b + assert fnv1_hash_name(name_a) == fnv1_hash_name(name_b) + + validator = entity_duplicate_validator("sensor") + + config1 = {CONF_NAME: name_a} + validated1 = validator(config1) + assert validated1 == config1 + + config2 = {CONF_NAME: name_b} with pytest.raises( Invalid, match=re.compile( - r"Duplicate binary_sensor entity with name 'Датчик закрытия основного крана' found.*" - r"Original names: 'Датчик закрытия основного крана' and 'Датчик открытия основного крана'.*" - r"Both convert to ASCII ID: '_______________________________'.*" - r"To fix: Add unique ASCII characters \(e\.g\., '1', '2', or 'A', 'B'\)", + rf"Duplicate sensor entity with name '{name_b}' found.*" + rf"The names '{name_b}' and '{name_a}' produce the.*" + r"same entity key hash \(0x0ee5ff7b\).*" + r"To fix: Rename one of the entities", re.DOTALL, ), ): validator(config2) +def test_object_id_conflicts_rejected_by_component_validator() -> None: + """Test that object_id conflicts pass entity validation but fail for mqtt/prometheus.""" + validator = entity_duplicate_validator("sensor") + + # Both names validate fine in general (distinct raw names, distinct keys) + validator({CONF_NAME: "Датчик открытия"}) + validator({CONF_NAME: "Датчик закрытия"}) + + # A component that addresses entities by object_id must reject the config + component_validator = validate_no_object_id_conflicts( + "mqtt builds default topics from the entity object_id" + ) + with pytest.raises( + Invalid, + match=re.compile( + r"mqtt builds default topics from the entity object_id.*" + r"sensor entities 'Датчик открытия', 'Датчик закрытия' " + r"share the object_id '_______________'.*" + r"To fix: Add unique ASCII characters", + re.DOTALL, + ), + ): + component_validator({}) + + +def test_object_id_conflicts_skipped_in_testing_mode() -> None: + """Test that testing_mode skips the conflict check, as used for grouped testing.""" + validator = entity_duplicate_validator("sensor") + validator({CONF_NAME: "Датчик открытия"}) + validator({CONF_NAME: "Датчик закрытия"}) + + component_validator = validate_no_object_id_conflicts( + "mqtt builds default topics from the entity object_id" + ) + CORE.testing_mode = True + try: + config: dict = {} + assert component_validator(config) is config + finally: + CORE.testing_mode = False + + +def test_object_id_conflicts_none_recorded() -> None: + """Test that distinct object_ids produce no conflicts.""" + validator = entity_duplicate_validator("sensor") + validator({CONF_NAME: "Temperature"}) + validator({CONF_NAME: "Humidity"}) + + component_validator = validate_no_object_id_conflicts( + "mqtt builds default topics from the entity object_id" + ) + config: dict = {} + assert component_validator(config) is config + + +def test_object_id_conflicts_device_scoped() -> None: + """Test that the object_id conflict check is scoped per device. + + Same-named entities on different sub-devices were accepted before entity keys + moved to raw names, so the check keeps that scope; conflicts within one device + are still reported with the device named in the message. + """ + validator = entity_duplicate_validator("sensor") + validator({CONF_NAME: "Temperature", CONF_DEVICE_ID: ID("device1", type="Device")}) + validator({CONF_NAME: "Temperature", CONF_DEVICE_ID: ID("device2", type="Device")}) + + component_validator = validate_no_object_id_conflicts( + "prometheus builds metric labels from the entity object_id" + ) + config: dict = {} + assert component_validator(config) is config + + # Two names sanitizing identically on the same sub-device still conflict + validator( + {CONF_NAME: "Датчик открытия", CONF_DEVICE_ID: ID("device1", type="Device")} + ) + validator( + {CONF_NAME: "Датчик закрытия", CONF_DEVICE_ID: ID("device1", type="Device")} + ) + with pytest.raises( + Invalid, + match=re.compile( + r"prometheus builds metric labels.*on device 'device1'", re.DOTALL + ), + ): + component_validator({}) + + def test_entity_duplicate_validator_same_name_no_enhanced_message() -> None: """Test that identical names don't show the enhanced message.""" # Create validator for sensor platform @@ -793,7 +763,7 @@ async def test_setup_entity_empty_name_with_device( # For empty-name entities, Python stores hash 0 - C++ calculates hash at runtime assert config.get("_entity_name") == "" - assert config.get("_entity_object_id_hash") == 0 + assert config.get("_entity_key") == 0 @pytest.mark.asyncio @@ -822,7 +792,7 @@ async def test_setup_entity_empty_name_with_mac_suffix( # For empty-name entities, Python stores hash 0 - C++ calculates hash at runtime assert config.get("_entity_name") == "" - assert config.get("_entity_object_id_hash") == 0 + assert config.get("_entity_key") == 0 @pytest.mark.asyncio @@ -852,7 +822,7 @@ async def test_setup_entity_empty_name_with_mac_suffix_no_friendly_name( # For empty-name entities, Python stores hash 0 - C++ calculates hash at runtime assert config.get("_entity_name") == "" - assert config.get("_entity_object_id_hash") == 0 + assert config.get("_entity_key") == 0 @pytest.mark.asyncio @@ -883,7 +853,7 @@ async def test_setup_entity_empty_name_no_mac_suffix_no_friendly_name( # For empty-name entities, Python stores hash 0 - C++ calculates hash at runtime assert config.get("_entity_name") == "" - assert config.get("_entity_object_id_hash") == 0 + assert config.get("_entity_key") == 0 def test_register_string_overflow() -> None: diff --git a/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_mqtt.yaml b/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_mqtt.yaml new file mode 100644 index 0000000000..4a6f56f473 --- /dev/null +++ b/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_mqtt.yaml @@ -0,0 +1,22 @@ +esphome: + name: test-object-id-conflict + +esp32: + board: esp32dev + +wifi: + ssid: MySSID + password: password1 + +mqtt: + broker: test.mosquitto.org + +sensor: + # Distinct raw names are fine in general, but both sanitize to the same + # object_id, which MQTT still uses to build default topics - should fail + - platform: template + name: "Датчик открытия" + lambda: return 21.0; + - platform: template + name: "Датчик закрытия" + lambda: return 22.0; diff --git a/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_no_mqtt.yaml b/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_no_mqtt.yaml new file mode 100644 index 0000000000..c0fbd5cbba --- /dev/null +++ b/tests/unit_tests/fixtures/core/entity_helpers/object_id_conflict_no_mqtt.yaml @@ -0,0 +1,15 @@ +esphome: + name: test-object-id-ok + +esp32: + board: esp32dev + +sensor: + # Distinct raw names that sanitize to the same object_id are allowed when no + # component addresses entities by object_id (no mqtt or prometheus configured) + - platform: template + name: "Датчик открытия" + lambda: return 21.0; + - platform: template + name: "Датчик закрытия" + lambda: return 22.0; diff --git a/tests/unit_tests/fixtures/lazy_imports/_leak_report.py b/tests/unit_tests/fixtures/lazy_imports/_leak_report.py new file mode 100644 index 0000000000..00d387cd04 --- /dev/null +++ b/tests/unit_tests/fixtures/lazy_imports/_leak_report.py @@ -0,0 +1,19 @@ +"""Shared tail for the lazy-import fixture scripts.""" + +import sys + + +def print_leaked_modules() -> None: + """Report argv-listed heavy modules (plus any component package) loaded. + + Any component package counts as a leak, not just the ones on the + watch list: executing one drags in codegen/validation machinery by + design. + """ + leaked = [module for module in sys.argv[1:] if module in sys.modules] + leaked += [ + module + for module in sys.modules + if module.startswith("esphome.components.") and module not in leaked + ] + print(",".join(leaked)) diff --git a/tests/unit_tests/fixtures/lazy_imports/_storage.py b/tests/unit_tests/fixtures/lazy_imports/_storage.py new file mode 100644 index 0000000000..969528304b --- /dev/null +++ b/tests/unit_tests/fixtures/lazy_imports/_storage.py @@ -0,0 +1,27 @@ +"""Shared storage-sidecar factory for the lazy-import fixture scripts.""" + +from esphome.storage_json import StorageJSON + + +def make_storage() -> StorageJSON: + """A minimal post-compile esp32 sidecar the upload/logs fast path accepts.""" + return StorageJSON( + storage_version=1, + name="test", + friendly_name="Test", + comment=None, + esphome_version="2026.1.0", + src_version=1, + address="1.2.3.4", + web_port=None, + target_platform="ESP32S3", + build_path=None, + firmware_bin_path=None, + loaded_integrations=set(), + loaded_platforms=set(), + no_mdns=False, + framework="esp-idf", + core_platform="esp32", + area=None, + framework_version="5.3.1", + ) diff --git a/tests/unit_tests/fixtures/lazy_imports/esptool_upload_fast_path.py b/tests/unit_tests/fixtures/lazy_imports/esptool_upload_fast_path.py new file mode 100644 index 0000000000..e622948aed --- /dev/null +++ b/tests/unit_tests/fixtures/lazy_imports/esptool_upload_fast_path.py @@ -0,0 +1,45 @@ +"""Run the esptool serial-upload path and report which heavy modules loaded. + +Executed as a subprocess by test_lazy_imports.py: heavy module names come +in on argv, the ones found in sys.modules afterwards go out on stdout. +The variant reaches the esptool command line from CORE.data directly; if +someone re-adds the esp32 package import for it, this reports the leak. +""" + +import os +import sys +from unittest.mock import patch + +from _leak_report import print_leaked_modules + +from esphome.__main__ import upload_using_esptool +from esphome.const import ( + CONF_ESPHOME, + KEY_CORE, + KEY_ESP32, + KEY_TARGET_PLATFORM, + KEY_VARIANT, +) +from esphome.core import CORE + +# An ambient ESPHOME_USE_SUBPROCESS would route past the patched +# run_external_command into run_external_process and confuse the checks. +os.environ.pop("ESPHOME_USE_SUBPROCESS", None) + +CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} +CORE.data[KEY_ESP32] = {KEY_VARIANT: "ESP32S3"} + +with patch("esphome.__main__.run_external_command", return_value=0) as mock_run: + rc = upload_using_esptool( + {CONF_ESPHOME: {"platformio_options": {}}}, "/dev/ttyUSB0", "firmware.bin", None + ) + +# Fail loudly if the upload path stopped doing its work; otherwise an +# empty leak list could just mean nothing ran. +if rc != 0: + sys.exit(f"upload_using_esptool returned {rc}") +cmd = list(mock_run.call_args[0][1:]) +if cmd[cmd.index("--chip") + 1] != "esp32s3": + sys.exit(f"variant did not reach the esptool command line: {cmd}") + +print_leaked_modules() diff --git a/tests/unit_tests/fixtures/lazy_imports/storage_json_fast_path.py b/tests/unit_tests/fixtures/lazy_imports/storage_json_fast_path.py new file mode 100644 index 0000000000..1e34bc90a1 --- /dev/null +++ b/tests/unit_tests/fixtures/lazy_imports/storage_json_fast_path.py @@ -0,0 +1,26 @@ +"""Run the esp32 storage fast path and report which heavy modules loaded. + +Executed as a subprocess by test_lazy_imports.py: heavy module names come +in on argv, the ones found in sys.modules afterwards go out on stdout. +""" + +import sys + +from _leak_report import print_leaked_modules +from _storage import make_storage + +from esphome.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT +from esphome.core import CORE, Version + +make_storage().apply_to_core() + +# Fail loudly if the esp32 fast path stopped doing its work; otherwise an +# empty leak list could just mean nothing ran. Explicit exits rather than +# asserts so PYTHONOPTIMIZE in the ambient environment can't strip them. +esp32_data = CORE.data.get(KEY_ESP32, {}) +if esp32_data.get(KEY_VARIANT) != "ESP32S3": + sys.exit(f"apply_to_core did not record the variant: {esp32_data!r}") +if esp32_data.get(KEY_IDF_VERSION) != Version(5, 3, 1): + sys.exit(f"apply_to_core did not parse the framework version: {esp32_data!r}") + +print_leaked_modules() diff --git a/tests/unit_tests/fixtures/lazy_imports/upload_command_fast_path.py b/tests/unit_tests/fixtures/lazy_imports/upload_command_fast_path.py new file mode 100644 index 0000000000..f0df08aa4e --- /dev/null +++ b/tests/unit_tests/fixtures/lazy_imports/upload_command_fast_path.py @@ -0,0 +1,82 @@ +"""Run the upload command dispatch path and report which heavy modules loaded. + +Executed as a subprocess by test_lazy_imports.py: heavy module names come +in on argv, the ones found in sys.modules afterwards go out on stdout. +Covers both fast-path claims: the bundle suffix check in run_esphome reads +BUNDLE_EXTENSION from esphome.const without importing esphome.bundle, and +the real validated-config cache parse, include resolution included, stays +voluptuous free. +""" + +import os +from pathlib import Path +import sys +import tempfile +from unittest.mock import patch + +from _leak_report import print_leaked_modules +from _storage import make_storage +import yaml + +# Everything imported past this point is the code under test; the pop +# below must only drop what the setup itself preloaded, or it would +# hide modules the dispatch chain pulls in (tarfile has no other guard). +_FIXTURE_PRELOADED = frozenset(sys.modules) + +from esphome import __main__ as main_mod # noqa: E402 + +CONFIG_TEXT = "esphome:\n name: t\n" + +# An ambient data-dir override would relocate the storage tree away +# from the tmp config dir this fixture builds. +os.environ.pop("ESPHOME_DATA_DIR", None) +os.environ.pop("ESPHOME_IS_HA_ADDON", None) + +with tempfile.TemporaryDirectory() as _td: + tmp = Path(_td) + conf_path = tmp / "test.yaml" + conf_path.write_text(CONFIG_TEXT) + + storage_dir = tmp / ".esphome" / "storage" + storage_dir.mkdir(parents=True) + # The cache is a top-level !include so loading it resolves an + # IncludeFile for real on the fast path. The sidecar is written to the + # layout ext_storage_path resolves once run_esphome sets + # CORE.config_path; going through CORE here would be circular. + (storage_dir / "inc.yaml").write_text(CONFIG_TEXT) + cache_path = storage_dir / "test.yaml.validated.yaml" + cache_path.write_text("!include inc.yaml\n") + os.utime(cache_path) # keep the cache at least as fresh as the source + make_storage().save(storage_dir / "test.yaml.json") + + dispatched = {} + + def fake_upload(args, config): + dispatched["config"] = config + return 0 + + # This setup pre-imports some watched stdlib modules (tempfile above, + # write_file inside make_storage().save(), unittest.mock -> asyncio -> + # subprocess). Drop exactly those so only a genuine dispatch-time + # re-import is reported; live objects keep their references, so + # cleanup still works. Module-level re-imports are out of reach here + # (esphome.__main__ is already loaded) — the bare-import check in + # test_lazy_imports owns that contract. + for module in sys.argv[1:]: + if module in _FIXTURE_PRELOADED: + sys.modules.pop(module, None) + + with patch.dict(main_mod.POST_CONFIG_ACTIONS, {"upload": fake_upload}): + exit_code = main_mod.run_esphome( + ["esphome", "upload", str(conf_path), "--device", "192.0.2.1"] + ) + + # Fail loudly if the fast path didn't do its work; otherwise an empty + # leak list could just mean nothing ran. Explicit exits rather than + # asserts so PYTHONOPTIMIZE in the ambient environment can't strip them. + if exit_code != 0: + sys.exit(f"run_esphome exited {exit_code} before dispatching upload") + if dispatched.get("config") != yaml.safe_load(CONFIG_TEXT): + sys.exit(f"cache include did not resolve through the fast path: {dispatched!r}") + + print_leaked_modules() diff --git a/tests/unit_tests/test_api_client.py b/tests/unit_tests/test_api_client.py new file mode 100644 index 0000000000..19ed83abe1 --- /dev/null +++ b/tests/unit_tests/test_api_client.py @@ -0,0 +1,165 @@ +"""Tests for esphome.api_client.""" + +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, Mock, patch + +import pytest + +from esphome import api_client +from esphome.const import ( + CONF_ENCRYPTION, + CONF_KEY, + CONF_PORT, + KEY_CORE, + KEY_TARGET_PLATFORM, +) +from esphome.core import CORE + + +def test_component_shim_reexports_runtime_client() -> None: + """The old import paths must keep working for external code.""" + from esphome.components import api + from esphome.components.api import client as shim + + assert shim.run_logs is api_client.run_logs + assert shim.async_run_logs is api_client.async_run_logs + assert api.CONF_ENCRYPTION is CONF_ENCRYPTION + + +@pytest.mark.asyncio +async def test_async_run_logs_full_flow(caplog) -> None: + """Drive async_run_logs end to end with a fake connection. + + Covers the encryption key extraction, the multi-address banner, the + registry-miss unavailable notice at session start, the on_log + handler, and the stop() cleanup in the finally block. + """ + caplog.set_level("INFO", logger="esphome.api_client") + caplog.set_level("INFO", logger="esphome.platform_hooks") + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "host"} + config = { + "esphome": {"name": "test"}, + "api": {CONF_PORT: 6053, CONF_ENCRYPTION: {CONF_KEY: "psk123"}}, + } + + stop = AsyncMock() + run_started = asyncio.Event() + + async def fake_async_run(*args, **kwargs): + run_started.set() + return stop + + mock_run = AsyncMock(side_effect=fake_async_run) + printed: list[str] = [] + + with ( + patch.object(api_client, "async_run", mock_run), + patch.object(api_client, "APIClient") as mock_client, + patch.object(api_client, "safe_print", printed.append), + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4", "5.6.7.8"]) + ) + # Let the task run up to the forever-wait; the timeout fails the + # test instead of hanging it if the task dies early. + async with asyncio.timeout(1): + await run_started.wait() + on_log = mock_run.call_args.args[1] + on_log(Mock(message=b"[I][main:001] hello world\nPC: 0x40104960")) + # Cancellation is the real termination path; stop() must still run. + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + # Both addresses reach APIClient, along with the noise key. + assert mock_client.call_args.kwargs["noise_psk"] == "psk123" + assert mock_client.call_args.kwargs["addresses"] == ["1.2.3.4", "5.6.7.8"] + assert "1.2.3.4 or 5.6.7.8" in caplog.text + # host has no stacktrace analyzer; the notice fires at session start. + assert "Stacktrace analysis is unavailable" in caplog.text + # The log message was printed with a timestamp prefix. + assert any("hello world" in line for line in printed) + # stop() ran in the finally block despite the cancellation. + stop.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_async_run_logs_never_resolves_without_crash_lines() -> None: + """The headline claim: an ordinary session imports no platform code.""" + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}} + + stop = AsyncMock() + run_started = asyncio.Event() + + async def fake_async_run(*args, **kwargs): + run_started.set() + return stop + + mock_run = AsyncMock(side_effect=fake_async_run) + + with ( + patch.object(api_client, "async_run", mock_run), + patch.object(api_client, "APIClient"), + patch.object(api_client, "safe_print"), + patch("esphome.platform_hooks.get_stacktrace_handler") as mock_resolve, + ): + task = asyncio.get_running_loop().create_task( + api_client.async_run_logs(config, ["1.2.3.4"]) + ) + async with asyncio.timeout(1): + await run_started.wait() + on_log = mock_run.call_args.args[1] + on_log(Mock(message=b"[I][app:100] hello\n[C][wifi:200] connected")) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + mock_resolve.assert_not_called() + + +def test_run_logs_suppresses_keyboard_interrupt() -> None: + """Ctrl-C during log streaming exits cleanly instead of tracebacking.""" + with patch.object( + api_client, + "async_run_logs", + AsyncMock(side_effect=KeyboardInterrupt), + ) as mock_run: + api_client.run_logs( + {"esphome": {"name": "test"}}, ["1.2.3.4"], subscribe_states=False + ) + + assert mock_run.call_args.kwargs["subscribe_states"] is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("extra_config", "expected_deep_sleep"), + [({"deep_sleep": {}}, True), ({}, False)], +) +async def test_async_run_logs_passes_deep_sleep( + extra_config: dict, expected_deep_sleep: bool +) -> None: + """async_run_logs tells async_run whether the device deep sleeps. + + That flag is the only thing capping reconnect backoff for a device + that is only briefly awake; dropping it means missed wake windows. + """ + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: "esp32"} + config = {"esphome": {"name": "test"}, "api": {CONF_PORT: 6053}, **extra_config} + # async_run blocks forever after connecting; raise to unwind + # async_run_logs once we have captured how it was called. + sentinel = RuntimeError("stop the wait") + + with ( + patch.object( + api_client, "async_run", AsyncMock(side_effect=sentinel) + ) as mock_run, + patch.object(api_client, "APIClient"), + pytest.raises(RuntimeError, match="stop the wait"), + ): + await api_client.async_run_logs(config, ["1.2.3.4"]) + + assert mock_run.call_args.kwargs["deep_sleep"] is expected_deep_sleep diff --git a/tests/unit_tests/test_async_thread.py b/tests/unit_tests/test_async_thread.py new file mode 100644 index 0000000000..a64be2f7c7 --- /dev/null +++ b/tests/unit_tests/test_async_thread.py @@ -0,0 +1,316 @@ +"""Tests for the async thread helpers.""" + +from __future__ import annotations + +import asyncio +import threading +from typing import Any +from unittest.mock import patch + +import pytest + +from esphome.async_thread import AsyncDispatchTimeout, AsyncThreadRunner, run_async + + +def _cleanup_threads() -> set[threading.Thread]: + """Return the currently live orphan-cleanup threads.""" + return {t for t in threading.enumerate() if t.name == "async-orphan-cleanup"} + + +def _join_new_cleanup_threads(before: set[threading.Thread]) -> None: + """Wait for cleanup threads spawned since ``before`` to finish.""" + for thread in _cleanup_threads() - before: + thread.join(5) + assert not thread.is_alive() + + +def test_run_async_returns_result() -> None: + """The coroutine's result is returned to the sync caller.""" + + async def coro() -> int: + await asyncio.sleep(0) + return 42 + + assert run_async(coro) == 42 + + +def test_run_async_propagates_exception() -> None: + """Exceptions raised by the coroutine surface in the caller.""" + + async def coro() -> None: + raise ValueError("boom") + + with pytest.raises(ValueError, match="boom"): + run_async(coro) + + +def test_run_async_propagates_base_exception() -> None: + """A BaseException from the coroutine surfaces instead of a None result.""" + + class Boom(BaseException): + pass + + async def coro() -> None: + raise Boom + + with pytest.raises(Boom): + run_async(coro) + + +def test_run_async_timeout() -> None: + """A coroutine that does not finish in time raises TimeoutError.""" + release = threading.Event() + + async def coro() -> None: + await asyncio.get_running_loop().run_in_executor(None, release.wait) + + before = _cleanup_threads() + with pytest.raises(TimeoutError): + run_async(coro, timeout=0.05) + + # Unblock the abandoned runner so its cleanup thread exits promptly. + release.set() + _join_new_cleanup_threads(before) + + +def test_run_async_surfaces_loop_startup_failure() -> None: + """A failure before the coroutine runs raises instead of hanging.""" + + def failing_run(main: Any) -> None: + # Close the never-awaited coroutine so the test does not leave a + # RuntimeWarning attributed to whatever module GC runs in later. + main.close() + raise OSError("no fds for the event loop") + + with ( + patch("esphome.async_thread.asyncio.run", side_effect=failing_run), + pytest.raises(OSError, match="no fds"), + ): + run_async(lambda: asyncio.sleep(0), timeout=5) + + +def test_run_preserves_result_when_cleanup_fails( + caplog: pytest.LogCaptureFixture, +) -> None: + """A loop-cleanup failure after success is logged, not raised.""" + + async def coro() -> str: + return "ok" + + runner: AsyncThreadRunner[str] = AsyncThreadRunner(coro) + + def fake_run(main: Any) -> None: + main.close() + # Emulate _runner delivering the result before cleanup raised. A + # None result must count as delivered too, hence the completed flag. + runner.result = "ok" + runner.completed = True + raise KeyboardInterrupt + + with ( + caplog.at_level("DEBUG", logger="esphome.async_thread"), + patch("esphome.async_thread.asyncio.run", side_effect=fake_run), + ): + runner.run() + + assert runner.event.is_set() + assert runner.exception is None + assert runner.result == "ok" + assert "teardown failed after outcome recorded" in caplog.text + + +def test_run_async_none_result_is_success() -> None: + """A coroutine legitimately returning None is not treated as a failure.""" + + async def coro() -> None: + return None + + assert run_async(coro) is None + + +def test_run_async_on_orphan_skips_none_result() -> None: + """A late None result completes cleanly without invoking on_orphan.""" + orphaned: list[Any] = [] + finished = threading.Event() + release = threading.Event() + + async def coro() -> None: + await asyncio.get_running_loop().run_in_executor(None, release.wait) + finished.set() + + before = _cleanup_threads() + with pytest.raises(TimeoutError): + run_async(coro, timeout=0.01, on_orphan=orphaned.append) + + release.set() + assert finished.wait(5) + _join_new_cleanup_threads(before) + assert not orphaned + + +def test_late_failure_without_on_orphan_is_logged( + caplog: pytest.LogCaptureFixture, +) -> None: + """An abandoned thread's real error leaves a visible trace.""" + release = threading.Event() + + async def coro() -> str: + await asyncio.get_running_loop().run_in_executor(None, release.wait) + raise ValueError("the real cause") + + before = _cleanup_threads() + with caplog.at_level("DEBUG", logger="esphome.async_thread"): + with pytest.raises(TimeoutError): + run_async(coro, timeout=0.01) + + release.set() + _join_new_cleanup_threads(before) + assert "Abandoned async operation failed" in caplog.text + assert "the real cause" in caplog.text + + +def test_run_async_on_orphan_failure_is_contained( + caplog: pytest.LogCaptureFixture, +) -> None: + """An on_orphan callback that raises is logged, not propagated.""" + released = threading.Event() + release = threading.Event() + + def on_orphan(result: str) -> None: + released.set() + raise OSError("close failed") + + async def coro() -> str: + await asyncio.get_running_loop().run_in_executor(None, release.wait) + return "late result" + + before = _cleanup_threads() + with caplog.at_level("DEBUG", logger="esphome.async_thread"): + with pytest.raises(TimeoutError): + run_async(coro, timeout=0.01, on_orphan=on_orphan) + + release.set() + assert released.wait(5) + _join_new_cleanup_threads(before) + assert "Error releasing orphaned result" in caplog.text + + +def test_run_async_on_orphan_releases_late_result() -> None: + """A result produced after the timeout is handed to on_orphan.""" + orphaned: list[Any] = [] + delivered = threading.Event() + release = threading.Event() + + def on_orphan(result: str) -> None: + orphaned.append(result) + delivered.set() + + async def coro() -> str: + # Block until the test has observed the timeout, so the result is + # guaranteed to arrive late no matter how slowly the runner is + # scheduled. + await asyncio.get_running_loop().run_in_executor(None, release.wait) + return "late result" + + before = _cleanup_threads() + with pytest.raises(TimeoutError): + run_async(coro, timeout=0.01, on_orphan=on_orphan) + + release.set() + assert delivered.wait(5) + _join_new_cleanup_threads(before) + assert orphaned == ["late result"] + + +def test_run_async_on_orphan_skips_late_failure() -> None: + """A late failure after the timeout is not handed to on_orphan.""" + orphaned: list[Any] = [] + failed = threading.Event() + release = threading.Event() + + async def coro() -> str: + # Block until the test has observed the timeout, so the failure is + # guaranteed to arrive late. + await asyncio.get_running_loop().run_in_executor(None, release.wait) + failed.set() + raise ValueError("late failure") + + before = _cleanup_threads() + with pytest.raises(TimeoutError): + run_async(coro, timeout=0.01, on_orphan=orphaned.append) + + release.set() + assert failed.wait(5) + _join_new_cleanup_threads(before) + assert not orphaned + + +def test_run_async_detects_missing_outcome() -> None: + """A run that records neither result nor exception raises loudly.""" + + def fake_run(main: Any) -> None: + # Simulate a loop that silently dropped the coroutine. + main.close() + + with ( + patch("esphome.async_thread.asyncio.run", side_effect=fake_run), + pytest.raises(RuntimeError, match="without a result"), + ): + run_async(lambda: asyncio.sleep(0), timeout=5) + + +def test_run_async_raises_distinguishable_timeout() -> None: + """The dispatcher's own expiry is a distinct TimeoutError subclass.""" + release = threading.Event() + + async def coro() -> None: + await asyncio.get_running_loop().run_in_executor(None, release.wait) + + before = _cleanup_threads() + with pytest.raises(AsyncDispatchTimeout): + run_async(coro, timeout=0.01) + release.set() + _join_new_cleanup_threads(before) + + +def test_orphan_watcher_gives_up_on_a_hung_coroutine( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The watcher exits after its bound instead of parking forever.""" + from esphome import async_thread + + monkeypatch.setattr(async_thread, "ORPHAN_WAIT_TIMEOUT", 0.01) + release = threading.Event() + orphaned: list[Any] = [] + + async def coro() -> str: + await asyncio.get_running_loop().run_in_executor(None, release.wait) + return "too late" + + before = _cleanup_threads() + with pytest.raises(TimeoutError): + run_async(coro, timeout=0.01, on_orphan=orphaned.append) + + _join_new_cleanup_threads(before) + assert not orphaned + release.set() + + +def test_late_real_result_without_handler_is_logged( + caplog: pytest.LogCaptureFixture, +) -> None: + """A genuinely dropped late result leaves the discard trace.""" + release = threading.Event() + + async def coro() -> str: + await asyncio.get_running_loop().run_in_executor(None, release.wait) + return "dropped" + + before = _cleanup_threads() + with caplog.at_level("DEBUG", logger="esphome.async_thread"): + with pytest.raises(TimeoutError): + run_async(coro, timeout=0.01) + + release.set() + _join_new_cleanup_threads(before) + assert "Discarding late result" in caplog.text diff --git a/tests/unit_tests/test_bundle.py b/tests/unit_tests/test_bundle.py index f0abcc74c6..29e917fe44 100644 --- a/tests/unit_tests/test_bundle.py +++ b/tests/unit_tests/test_bundle.py @@ -23,8 +23,8 @@ from esphome.bundle import ( _default_target_dir, _find_used_secret_keys, add_bundle_file, + add_secret_scan_dir, extract_bundle, - is_bundle_path, prepare_bundle_for_compile, read_bundle_manifest, remap_bundle_path, @@ -98,26 +98,6 @@ def _setup_config_dir( return config_dir -# --------------------------------------------------------------------------- -# is_bundle_path -# --------------------------------------------------------------------------- - - -@pytest.mark.parametrize( - ("filename", "expected"), - [ - (f"my_device{BUNDLE_EXTENSION}", True), - (f"MY_DEVICE{BUNDLE_EXTENSION.upper()}", True), - ("my_device.yaml", False), - ("my_device.tar.gz", False), - ("my_device.zip", False), - ("", False), - ], -) -def test_is_bundle_path(filename: str, expected: bool) -> None: - assert is_bundle_path(Path(filename)) is expected - - # --------------------------------------------------------------------------- # _default_target_dir # --------------------------------------------------------------------------- @@ -1248,7 +1228,8 @@ def test_discover_files_deeply_nested_include(tmp_path: Path) -> None: def test_discover_files_nested_include_unresolved_substitution( tmp_path: Path, ) -> None: - """!include with substitution vars in path cannot be resolved; skipped gracefully.""" + """!include with substitution vars in path but no candidate files on disk + (the glob's only match is the config itself) is skipped gracefully.""" config_dir = _setup_config_dir(tmp_path) (config_dir / "test.yaml").write_text( "esphome:\n name: test\nwifi: !include ${platform}.yaml\n" @@ -1262,6 +1243,62 @@ def test_discover_files_nested_include_unresolved_substitution( assert "test.yaml" in paths +def test_discover_files_bundles_all_include_candidates(tmp_path: Path) -> None: + """The issue-17650 layout: templated package includes chain through a glob + candidate into a Jinja conditional whose ``../`` branch is bundled.""" + config_dir = _setup_config_dir( + tmp_path, + files={ + "includes/esp-basics.yaml": ( + "packages:\n" + " - !include boards/${board}.yaml\n" + " - !include keys/${system_name}.yaml\n" + ), + "includes/boards/wemos-d1-mini.yaml": ( + 'packages:\n - !include ${ "NO BT.yaml" if bt else "../empty.yaml" }\n' + ), + "includes/keys/device-a.yaml": "api:\n", + "includes/keys/device-b.yaml": "api:\n", + "includes/empty.yaml": "{}\n", + }, + ) + (config_dir / "test.yaml").write_text( + "esphome:\n name: test\npackages:\n - !include includes/esp-basics.yaml\n" + ) + + creator = ConfigBundleCreator({}) + files = creator.discover_files() + + paths = [f.path for f in files] + assert "includes/esp-basics.yaml" in paths + assert "includes/boards/wemos-d1-mini.yaml" in paths + assert "includes/keys/device-a.yaml" in paths + assert "includes/keys/device-b.yaml" in paths + assert "includes/empty.yaml" in paths + + +def test_discover_files_candidate_outside_config_dir_skipped( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A candidate branch resolving above the config dir is not bundled.""" + config_dir = _setup_config_dir(tmp_path) + (tmp_path / "outside.yaml").write_text("api:\n") + (config_dir / "test.yaml").write_text( + "esphome:\n name: test\n" + 'wifi: !include ${ "a.yaml" if x else "../outside.yaml" }\n' + ) + + creator = ConfigBundleCreator({}) + files = creator.discover_files() + + paths = [f.path for f in files] + assert not any("outside" in p for p in paths) + assert any( + "outside config directory" in r.message and "outside.yaml" in r.message + for r in caplog.records + ) + + def test_discover_files_nested_include_load_failure( tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: @@ -1594,6 +1631,44 @@ def test_create_bundle_filters_secrets_quoted(tmp_path: Path) -> None: assert "unused" not in secrets_data +def test_create_bundle_scans_remote_package_files_for_secrets(tmp_path: Path) -> None: + """Secrets referenced only by git-fetched package files must be shipped + in the filtered secrets.yaml (regression test for issue 18023).""" + config_dir = _setup_config_dir(tmp_path) + + secrets = config_dir / "secrets.yaml" + secrets.write_text("ota_password: hunter2\nunused: should_not_appear\n") + + # Simulate a git-fetched package checkout referencing a secret + repo_dir = config_dir / ".esphome" / "packages" / "6bcd6aa8" + package_dir = repo_dir / "packages" + package_dir.mkdir(parents=True) + (package_dir / "base.yml").write_text( + "ota:\n - platform: esphome\n password: !secret ota_password\n" + ) + # References inside hidden directories such as .git must not be scanned + hidden_dir = repo_dir / ".git" + hidden_dir.mkdir() + (hidden_dir / "leak.yaml").write_text("password: !secret unused\n") + add_secret_scan_dir(repo_dir) + + creator = ConfigBundleCreator({}) + result = creator.create_bundle() + + assert result.manifest[ManifestKey.HAS_SECRETS] is True + + buf = io.BytesIO(result.data) + with tarfile.open(fileobj=buf, mode="r:gz") as tar: + secrets_data = tar.extractfile("secrets.yaml").read().decode() + names = tar.getnames() + + assert "ota_password" in secrets_data + assert "hunter2" in secrets_data + assert "unused" not in secrets_data + # The package checkout itself must not be bundled + assert not any("base.yml" in name for name in names) + + def test_create_bundle_no_secrets(tmp_path: Path) -> None: _setup_config_dir(tmp_path) diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index e17271e2b4..b852d2d596 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -20,11 +20,13 @@ from esphome.const import ( CONF_ESPHOME, CONF_NAME, KEY_CORE, + KEY_ESP32, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM, KEY_VARIANT, ) from esphome.core import CORE +from esphome.yaml_util import ESPHomeDataBase _VALIDATED_CONFIG_YAML = """\ esphome: @@ -124,21 +126,21 @@ def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: assert config[CONF_API]["encryption"]["key"] == "6dGhpcyBpcyBhIHRlc3Q=" assert config["ota"][0]["password"] == "secret" + # The fast path loads without per-node source ranges (the full + # contract lives in test_yaml_util; this checks the flag is wired up). + assert not isinstance(config[CONF_ESPHOME][CONF_NAME], ESPHomeDataBase) + # apply_to_core populated exactly what upload/logs read off CORE. assert CORE.name == "lite_test" assert CORE.build_path == Path("/build/lite_test") assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32" assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino" # upload_using_esptool reads get_esp32_variant() off CORE.data[KEY_ESP32]. - from esphome.components.esp32.const import KEY_ESP32 - assert CORE.data[KEY_ESP32][KEY_VARIANT] == "ESP32" def test_load_compiled_config_populates_esp32_variant(tmp_path: Path) -> None: """ESP32 variants survive the cache fast path so esptool gets the right --chip.""" - from esphome.components.esp32.const import KEY_ESP32 - yaml_path = tmp_path / "lite_test.yaml" yaml_path.write_text("esphome:\n name: lite_test\n") CORE.config_path = yaml_path @@ -156,8 +158,6 @@ def test_load_compiled_config_skips_esp32_block_for_other_platforms( tmp_path: Path, ) -> None: """Non-esp32 targets shouldn't fabricate an esp32 data block.""" - from esphome.components.esp32.const import KEY_ESP32 - yaml_path = tmp_path / "lite_test.yaml" yaml_path.write_text("esphome:\n name: lite_test\n") CORE.config_path = yaml_path diff --git a/tests/unit_tests/test_config_validation.py b/tests/unit_tests/test_config_validation.py index 79bfc303b7..4a4e37e5c4 100644 --- a/tests/unit_tests/test_config_validation.py +++ b/tests/unit_tests/test_config_validation.py @@ -2,6 +2,7 @@ import json import logging from pathlib import Path import string +from unittest.mock import patch from hypothesis import example, given, settings from hypothesis.strategies import builds, integers, ip_addresses, one_of, text @@ -2926,11 +2927,46 @@ def test_require_esphome_version_ok() -> None: assert cv.require_esphome_version(1, 0, 0)("test") == "test" +def test_require_esphome_version_accepts_version_object() -> None: + """The Version form matches require_framework_version's style.""" + assert cv.require_esphome_version(cv.Version(1, 0, 0))("test") == "test" + with pytest.raises(Invalid, match="at least ESPHome version 9999.0.0"): + cv.require_esphome_version(cv.Version(9999, 0, 0))("test") + + +def test_require_esphome_version_partial_ints_fail_at_call_site() -> None: + """Missing ints raise immediately instead of a TypeError inside the validator.""" + with pytest.raises(ValueError, match="needs a Version or"): + cv.require_esphome_version(2026, 8) + with pytest.raises(ValueError, match="needs a Version or"): + cv.require_esphome_version(2026) + + def test_require_esphome_version_too_old() -> None: with pytest.raises(Invalid, match="at least ESPHome version 9999.0.0"): cv.require_esphome_version(9999, 0, 0)("test") +@pytest.mark.parametrize("current", ["2026.8.0", "2026.8.0b1", "2026.8.0-dev20260801"]) +def test_require_esphome_version_prerelease_of_required_passes(current: str) -> None: + """A dev or beta build of the required version satisfies it. + + Pins the behavior of the old tuple comparison that dropped the + suffix, now expressed through Version ordering where the extra field + only breaks ties upward. + """ + with patch.object(cv, "ESPHOME_VERSION", current): + assert cv.require_esphome_version(2026, 8, 0)("test") == "test" + + +def test_require_esphome_version_older_prerelease_fails() -> None: + with ( + patch.object(cv, "ESPHOME_VERSION", "2026.7.0-dev20260701"), + pytest.raises(Invalid, match="at least ESPHome version 2026.8.0"), + ): + cv.require_esphome_version(2026, 8, 0)("test") + + # --------------------------------------------------------------------------- # suppress_invalid / validate_source_shorthand / rename_key # --------------------------------------------------------------------------- diff --git a/tests/unit_tests/test_cpp_helpers.py b/tests/unit_tests/test_cpp_helpers.py index e389b56ada..1c0e0d0a93 100644 --- a/tests/unit_tests/test_cpp_helpers.py +++ b/tests/unit_tests/test_cpp_helpers.py @@ -4,6 +4,7 @@ from unittest.mock import Mock import pytest from esphome import const, cpp_helpers as ch +from esphome.core import CoroPriority, coroutine_with_priority from esphome.cpp_helpers import ComponentSourcePool, register_component_source @@ -167,3 +168,53 @@ def test_register_component_source_overflow_suppressed_in_testing_mode( idx = register_component_source("overflow_component") assert idx == 0 assert "Too many unique component source names" not in caplog.text + + +def _define_value(name: str) -> str | None: + for define in ch.CORE.defines: + if define.name == name: + # Values are codegen expressions (IntLiteral); compare rendered. + return str(define.value) + return None + + +def test_slot_counter_emits_requested_count() -> None: + """Each request bumps the count; the self-scheduled FINAL job emits it.""" + request = ch.slot_counter("TEST_SLOT_COUNT") + request() + request() + ch.CORE.flush_tasks() + assert _define_value("TEST_SLOT_COUNT") == "2" + + +def test_slot_counter_without_requests_emits_nothing() -> None: + """No requests, no job, no define — the guarded storage compiles out.""" + ch.slot_counter("TEST_SLOT_COUNT_UNUSED") + ch.CORE.flush_tasks() + assert _define_value("TEST_SLOT_COUNT_UNUSED") is None + + +def test_slot_counter_request_from_final_job_still_emits() -> None: + """The FIRST request for a define may come from a FINAL job: its emit job + is scheduled mid-drain and flush_tasks() loops until the heap is empty. + Later requests do not get this guarantee — see the companion test.""" + request = ch.slot_counter("TEST_SLOT_COUNT_LATE") + + @coroutine_with_priority(CoroPriority.FINAL) + async def late_requester() -> None: + request() + + ch.CORE.add_job(late_requester) + ch.CORE.flush_tasks() + assert _define_value("TEST_SLOT_COUNT_LATE") == "1" + + +def test_slot_counter_request_after_emit_raises() -> None: + """The boundary of FINAL-time requests: once the define was emitted, a + further request would silently undersize the storage, so it fails loudly.""" + request = ch.slot_counter("TEST_SLOT_COUNT_TOO_LATE") + request() + ch.CORE.flush_tasks() + assert _define_value("TEST_SLOT_COUNT_TOO_LATE") == "1" + with pytest.raises(ValueError, match="TEST_SLOT_COUNT_TOO_LATE"): + request() diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index 879d98c0a7..f9e048f6f4 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -155,6 +155,62 @@ def test_generate_cmakelists_txt_basic(tmp_component): assert "main.c" in content +def test_generate_cmakelists_txt_external_source_uses_absolute_paths( + tmp_component, tmp_path +): + # A local library's sources live outside the component dir (source_path), + # so SRCS and INCLUDE_DIRS must be emitted as absolute paths into it. + source = tmp_path / "user_lib" + (source / "src").mkdir(parents=True) + (source / "include").mkdir() + (source / "src" / "thing.cpp").write_text("int t;") + tmp_component.source_path = source + tmp_component.data = {} + + content = generate_cmakelists_txt(tmp_component) + + abs_src = str((source / "src" / "thing.cpp").resolve()).replace("\\", "/") + abs_inc = str((source / "include").resolve()).replace("\\", "/") + assert abs_src in content + assert abs_inc in content + # Nothing was copied into the component dir. + assert not (tmp_component.path / "src").exists() + + +def test_generate_cmakelists_txt_external_source_absolutises_link_dirs( + tmp_component, tmp_path +): + # A local library's relative -L path must be made absolute against its own + # directory so it resolves from the component cache dir. + source = tmp_path / "user_lib" + (source / "src").mkdir(parents=True) + (source / "src" / "thing.cpp").write_text("int t;") + (source / "libs").mkdir() + tmp_component.source_path = source + tmp_component.data = {"build": {"flags": ["-Llibs"]}} + + content = generate_cmakelists_txt(tmp_component) + + abs_lib = str((source / "libs").resolve()).replace("\\", "/") + assert "target_link_directories" in content + assert abs_lib in content + + +def test_generate_cmakelists_txt_external_source_root_srcdir(tmp_component, tmp_path): + # An external source with files at its root (no src/ or include/ dir): + # the src-dir search falls through to "." and the missing include dirs are + # filtered out. + source = tmp_path / "flat_lib" + source.mkdir() + (source / "thing.cpp").write_text("int t;") + tmp_component.source_path = source + tmp_component.data = {} + + content = generate_cmakelists_txt(tmp_component) + + assert str((source / "thing.cpp").resolve()).replace("\\", "/") in content + + def test_generate_cmakelists_txt_with_flags(tmp_component, tmp_path): src_dir = tmp_component.path / "src" src_dir.mkdir() @@ -462,70 +518,66 @@ empty= def test_node_key_git_with_ref(): - key, is_git, locator = _node_key( + key, kind, locator = _node_key( "name", None, "https://github.com/foo/bar.git#v1.2.3" ) assert key == "foo/bar" - assert is_git is True + assert kind == "git" assert locator == ("https://github.com/foo/bar.git", "v1.2.3") def test_node_key_git_branch_ref(): - key, is_git, locator = _node_key( + key, kind, locator = _node_key( "name", None, "https://github.com/foo/bar.git#some-branch" ) - assert (key, is_git, locator[1]) == ("foo/bar", True, "some-branch") + assert (key, kind, locator[1]) == ("foo/bar", "git", "some-branch") def test_node_key_git_no_ref(): - _key, is_git, locator = _node_key("name", None, "https://github.com/foo/bar.git") - assert is_git is True + _key, kind, locator = _node_key("name", None, "https://github.com/foo/bar.git") + assert kind == "git" assert locator == ("https://github.com/foo/bar.git", None) def test_node_key_url_in_name_is_git(): # add_library("https://github.com/x/y", None): PlatformIO accepted a bare # git URL as the library name, so the converter must too. - key, is_git, locator = _node_key( - "https://github.com/pstolarz/OneWireNg", None, None - ) + key, kind, locator = _node_key("https://github.com/pstolarz/OneWireNg", None, None) assert key == "pstolarz/OneWireNg" - assert is_git is True + assert kind == "git" assert locator == ("https://github.com/pstolarz/OneWireNg", None) def test_node_key_url_in_name_with_ref(): - key, is_git, locator = _node_key( - "https://github.com/foo/bar.git#v1.2.3", None, None - ) - assert (key, is_git, locator) == ( + key, kind, locator = _node_key("https://github.com/foo/bar.git#v1.2.3", None, None) + assert (key, kind, locator) == ( "foo/bar", - True, + "git", ("https://github.com/foo/bar.git", "v1.2.3"), ) def test_node_key_url_in_name_git_plus_prefix(): - key, is_git, locator = _node_key("git+https://github.com/foo/bar", None, None) - assert (key, is_git, locator) == ( + key, kind, locator = _node_key("git+https://github.com/foo/bar", None, None) + assert (key, kind, locator) == ( "foo/bar", - True, + "git", ("https://github.com/foo/bar", None), ) def test_node_key_git_plus_prefix_in_repository(): - _key, is_git, locator = _node_key("name", None, "git+https://github.com/foo/bar") - assert (is_git, locator) == (True, ("https://github.com/foo/bar", None)) + _key, kind, locator = _node_key("name", None, "git+https://github.com/foo/bar") + assert (kind, locator) == ("git", ("https://github.com/foo/bar", None)) def test_node_key_custom_name_equals_url_is_git(): - key, is_git, locator = _node_key( + key, kind, locator = _node_key( "OneWireNg=https://github.com/pstolarz/OneWireNg", None, None ) - assert (key, is_git, locator) == ( + assert (key, kind, locator) == ( "pstolarz/OneWireNg", - True, + "git", ("https://github.com/pstolarz/OneWireNg", None), ) @@ -533,14 +585,70 @@ def test_node_key_custom_name_equals_url_is_git(): def test_node_key_url_in_name_with_query_containing_equals(): # A bare URL whose query string contains ``=`` must not be split by the # CustomName=URL handling. - key, is_git, locator = _node_key("https://host/x/y.git?ref=main", None, None) - assert (key, is_git, locator) == ( + key, kind, locator = _node_key("https://host/x/y.git?ref=main", None, None) + assert (key, kind, locator) == ( "x/y", - True, + "git", ("https://host/x/y.git?ref=main", None), ) +def test_node_key_file_url_in_repository_is_local(): + # A plain file:// entry (PlatformIO's spelling for a local library folder) + # resolves as a local directory, keeping the custom name as the key. The + # path is the OS-native form of the URL (backslashes on Windows). + key, kind, (path, ref) = _node_key( + "TeslaBLE", None, "file:///config/esphome/lib_dev" + ) + assert (key, kind, ref) == ("TeslaBLE", "local", None) + assert Path(path) == Path("/config/esphome/lib_dev") + + +def test_node_key_bare_file_url_is_local_named_for_dir(): + # Without a custom name the directory's own name becomes the key. + key, kind, (path, ref) = _node_key(None, None, "file:///opt/mylib") + assert (key, kind, ref) == ("mylib", "local", None) + assert Path(path) == Path("/opt/mylib") + + +def test_node_key_custom_name_equals_file_url_is_local(): + key, kind, (path, ref) = _node_key("Foo=file:///opt/mylib", None, None) + assert (key, kind, ref) == ("Foo", "local", None) + assert Path(path) == Path("/opt/mylib") + + +def test_node_key_file_url_localhost_host_is_local(): + # A localhost host is ignored; only the path identifies the directory. + key, kind, (path, ref) = _node_key(None, None, "file://localhost/opt/mylib") + assert (key, kind, ref) == ("mylib", "local", None) + assert Path(path) == Path("/opt/mylib") + + +@pytest.mark.parametrize( + "url", ["file://server/share/lib", "file://lib_dev", "file://../mylib"] +) +def test_node_key_file_url_with_host_rejected(url: str) -> None: + # A real host, or a relative path whose first segment parses as the host, + # is rejected rather than silently resolved to the wrong directory. + with pytest.raises(RuntimeError, match="Unsupported host in file://"): + _node_key(None, None, url) + + +@pytest.mark.parametrize("url", ["file:lib_dev", "file:./lib", "file:///"]) +def test_node_key_file_url_must_be_absolute(url: str) -> None: + # A relative path (no host, e.g. file:lib_dev) or a bare root (file:///) + # is rejected rather than resolved against the cwd or yielding an empty name. + with pytest.raises(RuntimeError, match="must be an absolute"): + _node_key(None, None, url) + + +def test_node_key_git_plus_file_url_stays_git(): + # git+file:// is an explicit local git repo, not a plain directory. + _key, kind, locator = _node_key("X", None, "git+file:///srv/foo.git") + assert kind == "git" + assert locator == ("file:///srv/foo.git", None) + + @pytest.mark.parametrize("name", ["http://[::1", "CustomName=http://[::1"]) def test_node_key_malformed_url_in_name_raises(name: str) -> None: # A name that was clearly meant to be a URL but does not parse must fail @@ -550,25 +658,25 @@ def test_node_key_malformed_url_in_name_raises(name: str) -> None: def test_node_key_name_with_equals_but_no_url_is_registry(): - key, is_git, locator = _node_key("FOO=BAR", "1.0", None) - assert (key, is_git, locator) == ("FOO=BAR", False, (None, "FOO=BAR")) + key, kind, locator = _node_key("FOO=BAR", "1.0", None) + assert (key, kind, locator) == ("FOO=BAR", "registry", (None, "FOO=BAR")) def test_node_key_version_url_still_ignored_when_name_plain(): # A version that is a URL is handled by the dependency walk, not here; # a plain name must stay a registry spec regardless of version shape. - key, is_git, _locator = _node_key("bar", "https://github.com/foo/bar", None) - assert (key, is_git) == ("bar", False) + key, kind, _locator = _node_key("bar", "https://github.com/foo/bar", None) + assert (key, kind) == ("bar", "registry") def test_node_key_registry_owner_name(): - key, is_git, locator = _node_key("foo/bar", "^1.0.0", None) - assert (key, is_git, locator) == ("foo/bar", False, ("foo", "bar")) + key, kind, locator = _node_key("foo/bar", "^1.0.0", None) + assert (key, kind, locator) == ("foo/bar", "registry", ("foo", "bar")) def test_node_key_registry_bare_name(): - key, is_git, locator = _node_key("bar", "1.0", None) - assert (key, is_git, locator) == ("bar", False, (None, "bar")) + key, kind, locator = _node_key("bar", "1.0", None) + assert (key, kind, locator) == ("bar", "registry", (None, "bar")) def test_normalize_dependencies_none(): diff --git a/tests/unit_tests/test_lazy_imports.py b/tests/unit_tests/test_lazy_imports.py index ee570a84f6..8358f4b781 100644 --- a/tests/unit_tests/test_lazy_imports.py +++ b/tests/unit_tests/test_lazy_imports.py @@ -14,6 +14,9 @@ test pins down *which* heavy modules must stay out entirely. from __future__ import annotations +import importlib.util +import os +from pathlib import Path import subprocess import sys @@ -30,12 +33,47 @@ HEAVY_MODULES = ( "voluptuous", ) +# Everything the storage fast path must keep out of sys.modules; the +# existence guard and the leak check must watch the same list. +FAST_PATH_HEAVY_MODULES = HEAVY_MODULES + ("esphome.components.esp32",) -def test_main_module_does_not_import_heavy_modules() -> None: - """A bare ``import esphome.__main__`` must not drag in validation/codegen.""" +# Heavy only for modules that must not know about the API transport; +# in the existence guard so a rename can't silently no-op its check. +API_HEAVY_MODULES = ("aioesphomeapi",) + +# Heavy only for the single-config dispatch path: the bundle suffix +# check reads BUNDLE_EXTENSION from esphome.const so an ordinary run +# never pays for the bundle machinery and its tarfile chain. +BUNDLE_HEAVY_MODULES = ("esphome.bundle", "tarfile") + +# Stdlib modules deferred out of the dispatch fast path: a cache-hit +# upload/logs run never writes a file (tempfile), spawns a process +# (subprocess), parses a URL (urllib.parse), or prints a serial +# permission hint (getpass). shutil is deferred too but unwatchable: +# argparse imports it from every add_argument on py3.14. urllib.parse +# is only watchable on 3.13+ where pathlib stopped importing it. +STDLIB_FAST_PATH_MODULES = ( + "tempfile", + "subprocess", + "getpass", + # Pins the module-level contract only: PyYAML's constructor loads + # datetime during the cache parse until the JSON cache lands. + "datetime", + *(("urllib.parse",) if sys.version_info >= (3, 13) else ()), +) + + +def _leaked_heavy_modules(module: str, extra: tuple[str, ...] = ()) -> str: + """Import ``module`` in a subprocess and report the heavy modules it pulled. + + Any ``esphome.components.*`` package counts as heavy: executing a + component package drags in codegen/validation machinery by design. + ``extra`` adds modules that are heavy for this caller specifically. + """ check = ( - "import sys; import esphome.__main__; " - f"leaked = [m for m in {HEAVY_MODULES!r} if m in sys.modules]; " + f"import sys; import {module}; " + f"leaked = [m for m in {HEAVY_MODULES + extra!r} if m in sys.modules]; " + "leaked += [m for m in sys.modules if m.startswith('esphome.components.')]; " "print(','.join(leaked))" ) result = subprocess.run( @@ -44,10 +82,201 @@ def test_main_module_does_not_import_heavy_modules() -> None: text=True, check=True, ) - leaked = result.stdout.strip() + return result.stdout.strip() + + +def test_main_module_does_not_import_heavy_modules() -> None: + """A bare ``import esphome.__main__`` must not drag in validation/codegen. + + The stdlib watch list rides along here because this check runs in a + clean subprocess: a module-level re-import anywhere on the chain is + caught, which the dispatch fixture (whose setup pre-imports them and + pops before dispatch) structurally cannot do. + """ + leaked = _leaked_heavy_modules("esphome.__main__", extra=STDLIB_FAST_PATH_MODULES) assert not leaked, ( f"esphome.__main__ imports heavy modules at top level: {leaked}. " "Import them lazily inside the command that needs them instead; " "every esphome invocation (including each parallel dashboard " "upload subprocess) pays for top-level imports." ) + + +def test_watched_heavy_modules_exist() -> None: + """A renamed heavy module would silently disable the leak checks.""" + for module in ( + FAST_PATH_HEAVY_MODULES + + API_HEAVY_MODULES + + BUNDLE_HEAVY_MODULES + + STDLIB_FAST_PATH_MODULES + ): + assert importlib.util.find_spec(module) is not None, ( + f"{module} no longer resolves; update the heavy-module lists" + ) + + +def _leaked_from_fixture( + fixture_path: Path, script_name: str, extra: tuple[str, ...] = () +) -> str: + """Run a fixture script with the watched modules on argv. + + Running a script file drops the cwd from sys.path, so prepend the + repo root for the child; a non-zero exit surfaces the child's stderr. + """ + script = fixture_path / "lazy_imports" / script_name + python_path = str(Path(__file__).parents[2]) + if ambient := os.environ.get("PYTHONPATH"): + python_path = os.pathsep.join((python_path, ambient)) + env = os.environ | {"PYTHONPATH": python_path} + result = subprocess.run( + [sys.executable, str(script), *FAST_PATH_HEAVY_MODULES, *extra], + capture_output=True, + text=True, + env=env, + check=False, + ) + assert result.returncode == 0, result.stderr + return result.stdout.strip() + + +def test_storage_json_fast_path_does_not_import_heavy_modules( + fixture_path: Path, +) -> None: + """``apply_to_core`` runs on the upload/logs fast path for every + platform; parsing the stored framework version must not drag in the + validation stack or the esp32 component package. + """ + leaked = _leaked_from_fixture(fixture_path, "storage_json_fast_path.py") + assert not leaked, ( + f"storage_json.apply_to_core pulls in heavy modules: {leaked}. " + "The upload/logs fast path skips validation; importing the " + "validation stack anyway defeats the validated-config cache." + ) + + +def test_esptool_upload_fast_path_does_not_import_heavy_modules( + fixture_path: Path, +) -> None: + """The esptool serial upload reads the esp32 variant from CORE.data; + resolving it must not drag in the esp32 component package or the + validation stack. + """ + leaked = _leaked_from_fixture(fixture_path, "esptool_upload_fast_path.py") + assert not leaked, ( + f"upload_using_esptool pulls in heavy modules: {leaked}. " + "The upload fast path skips validation; importing the validation " + "stack anyway defeats the validated-config cache." + ) + + +def test_api_client_does_not_import_heavy_modules() -> None: + """``esphome.api_client`` is on the logs fast path and must stay light. + + Importing it must not execute any component package (the api package + pulls the whole validation stack: logger, esp32, writer, config, + jinja2, voluptuous). + """ + leaked = _leaked_heavy_modules("esphome.api_client") + assert not leaked, ( + f"esphome.api_client imports heavy modules at top level: {leaked}. " + "The logs fast path skips validation; importing the validation " + "stack anyway defeats the validated-config cache." + ) + + +def test_stacktrace_does_not_import_heavy_modules() -> None: + """``esphome.stacktrace`` guards its own docstring's contract. + + Both log paths construct a LogLineProcessor before streaming + starts; importing the module must not pull in aioesphomeapi or + any platform package. + """ + leaked = _leaked_heavy_modules("esphome.stacktrace", extra=API_HEAVY_MODULES) + assert not leaked, ( + f"esphome.stacktrace imports heavy modules at top level: {leaked}. " + "The logs fast path skips validation; importing the validation " + "stack anyway defeats the validated-config cache." + ) + + +def test_espidf_toolchain_does_not_import_heavy_modules() -> None: + """The esp-idf upload path must not pull the esp32 package back in. + + upload_using_esptool reaches espidf.toolchain for esp-idf builds; + its keys and the variant mapping live in esphome.const and + esphome.espidf precisely so this import stays light. + """ + leaked = _leaked_heavy_modules("esphome.espidf.toolchain") + assert not leaked, ( + f"esphome.espidf.toolchain imports heavy modules: {leaked}. " + "The upload fast path skips validation; importing the validation " + "stack anyway defeats the validated-config cache." + ) + + +def test_has_mqtt_ip_lookup_does_not_import_mqtt() -> None: + """``has_mqtt_ip_lookup`` runs on the upload/logs fast path for mqtt + configs; reading ``CONF_DISCOVER_IP`` must not drag in the mqtt + component and, with it, the validation stack. + + Runs in a subprocess because this session's other tests import the + mqtt component; the fast path itself must not. + """ + check = ( + "import sys; from esphome.__main__ import has_mqtt_ip_lookup; " + "from esphome.core import CORE; from esphome.const import CONF_MQTT; " + "CORE.config = {CONF_MQTT: {}}; " + "assert has_mqtt_ip_lookup() is True, 'mqtt IP lookup default broke'; " + f"leaked = [m for m in {HEAVY_MODULES!r} if m in sys.modules]; " + "leaked += [m for m in sys.modules if m.startswith('esphome.components.')]; " + "print(','.join(leaked))" + ) + # check=False keeps the child's stderr (its assertion message or an + # import traceback) visible on failure. + result = subprocess.run( + [sys.executable, "-c", check], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + leaked = result.stdout.strip() + assert not leaked, ( + f"has_mqtt_ip_lookup pulls in heavy modules: {leaked}. " + "The upload/logs fast path skips validation; importing the " + "validation stack anyway defeats the validated-config cache." + ) + + +def test_yaml_util_does_not_import_heavy_modules() -> None: + """``esphome.yaml_util`` parses the validated-config cache on the + upload/logs fast path; importing it must not pull in voluptuous. + """ + leaked = _leaked_heavy_modules("esphome.yaml_util") + assert not leaked, ( + f"esphome.yaml_util imports heavy modules at top level: {leaked}. " + "The upload/logs fast path skips validation; importing the " + "validation stack anyway defeats the validated-config cache." + ) + + +def test_upload_command_path_does_not_import_heavy_modules( + fixture_path: Path, +) -> None: + """The single-config dispatch path checks the bundle suffix on every + run; reading it from esphome.const must not drag in esphome.bundle + and its tarfile chain. + """ + leaked = _leaked_from_fixture( + fixture_path, + "upload_command_fast_path.py", + extra=BUNDLE_HEAVY_MODULES + STDLIB_FAST_PATH_MODULES, + ) + assert not leaked, ( + f"the upload dispatch path pulls in heavy modules: {leaked}. " + "An ordinary run only needs the bundle suffix constant, and the " + "cache parse must not resolve voluptuous; keep the esphome.bundle " + "import inside the branch that extracts one, the Invalid import " + "inside the branch that raises it, and the deferred stdlib " + "imports inside the write/spawn/serial helpers that use them." + ) diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index e575934870..6c13cd5f12 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -27,6 +27,7 @@ from esphome.__main__ import ( _unresolved_default_error, _validate_bootloader_binary, _validate_partition_table_binary, + check_permissions, choose_upload_log_host, command_analyze_memory, command_bundle, @@ -63,8 +64,13 @@ from esphome.__main__ import ( ) from esphome.address_cache import AddressCache from esphome.bundle import BUNDLE_EXTENSION, BundleFile, BundleResult -from esphome.components import esp32 -from esphome.components.esp32 import KEY_ESP32, KEY_VARIANT, VARIANT_ESP32 +from esphome.components import esp32, esp8266 +from esphome.components.esp32 import ( + KEY_ESP32, + KEY_VARIANT, + VARIANT_ESP32, + get_esp32_variant, +) from esphome.const import ( CONF_API, CONF_AUTH, @@ -94,6 +100,7 @@ from esphome.const import ( PLATFORM_BK72XX, PLATFORM_ESP32, PLATFORM_ESP8266, + PLATFORM_NRF52, PLATFORM_RP2, Toolchain, ) @@ -1621,6 +1628,12 @@ def test_upload_using_esptool_path_conversion( assert isinstance(partitions_path, str) assert partitions_path.endswith("partitions.bin") + # The chip argument must track get_esp32_variant: upload_using_esptool + # reads CORE.data directly to avoid the esp32 package import, and the + # two resolutions must not drift. + chip = cmd_list[cmd_list.index("--chip") + 1] + assert chip == get_esp32_variant().lower() + def test_upload_using_esptool_skips_missing_extra_flash_images( tmp_path: Path, @@ -2862,18 +2875,17 @@ def test_upload_program_ota_with_mqtt_empty_broker( assert "MQTT IP discovery failed" in caplog.text -@patch("esphome.__main__.importlib.import_module") +@patch("esphome.platform_hooks.get_platform_hook") def test_upload_program_platform_specific_handler( - mock_import: Mock, + mock_get_hook: Mock, mock_get_port_type: Mock, ) -> None: """Test upload_program with platform-specific upload handler.""" - setup_core(platform="custom_platform") + setup_core(platform=PLATFORM_NRF52) mock_get_port_type.return_value = "CUSTOM" - mock_module = MagicMock() - mock_module.upload_program.return_value = True - mock_import.return_value = mock_module + platform_upload = MagicMock(return_value=True) + mock_get_hook.return_value = platform_upload config = {} args = MockArgs() @@ -2883,8 +2895,8 @@ def test_upload_program_platform_specific_handler( assert exit_code == 0 assert host == "custom_device" - mock_import.assert_called_once_with("esphome.components.custom_platform") - mock_module.upload_program.assert_called_once_with(config, args, "custom_device") + mock_get_hook.assert_called_once_with(PLATFORM_NRF52, "upload_program") + platform_upload.assert_called_once_with(config, args, "custom_device") def test_show_logs_serial( @@ -2918,7 +2930,7 @@ def test_show_logs_no_logger() -> None: show_logs(CORE.config, args, devices) -@patch("esphome.components.api.client.run_logs") +@patch("esphome.api_client.run_logs") def test_show_logs_api( mock_run_logs: Mock, ) -> None: @@ -2944,7 +2956,7 @@ def test_show_logs_api( ) -@patch("esphome.components.api.client.run_logs") +@patch("esphome.api_client.run_logs") def test_show_logs_api_no_states( mock_run_logs: Mock, ) -> None: @@ -2971,7 +2983,7 @@ def test_show_logs_api_no_states( ) -@patch("esphome.components.api.client.run_logs") +@patch("esphome.api_client.run_logs") def test_show_logs_api_with_fqdn_mdns_disabled( mock_run_logs: Mock, ) -> None: @@ -2998,7 +3010,7 @@ def test_show_logs_api_with_fqdn_mdns_disabled( ) -@patch("esphome.components.api.client.run_logs") +@patch("esphome.api_client.run_logs") def test_show_logs_api_with_mqtt_fallback( mock_run_logs: Mock, mock_mqtt_get_ip: Mock, @@ -3108,16 +3120,15 @@ def test_show_logs_no_method_configured() -> None: show_logs(CORE.config, args, devices) -@patch("esphome.__main__.importlib.import_module") +@patch("esphome.platform_hooks.get_platform_hook") def test_show_logs_platform_specific_handler( - mock_import: Mock, + mock_get_hook: Mock, ) -> None: """Test show_logs with platform-specific logs handler.""" - setup_core(platform="custom_platform", config={"logger": {}}) + setup_core(platform=PLATFORM_NRF52, config={"logger": {}}) - mock_module = MagicMock() - mock_module.show_logs.return_value = True - mock_import.return_value = mock_module + platform_show_logs = MagicMock(return_value=True) + mock_get_hook.return_value = platform_show_logs config = {"logger": {}} args = MockArgs() @@ -3126,8 +3137,8 @@ def test_show_logs_platform_specific_handler( result = show_logs(config, args, devices) assert result == 0 - mock_import.assert_called_once_with("esphome.components.custom_platform") - mock_module.show_logs.assert_called_once_with(config, args, devices) + mock_get_hook.assert_called_once_with(PLATFORM_NRF52, "show_logs") + platform_show_logs.assert_called_once_with(config, args, devices) def test_has_mqtt_logging_no_log_topic() -> None: @@ -3247,6 +3258,14 @@ def test_get_port_type() -> None: assert get_port_type("BOOTSEL") == "BOOTSEL" +def test_mqtt_reexports_discover_ip() -> None: + """The old import path must keep working for external code.""" + from esphome.components import mqtt + from esphome.const import CONF_DISCOVER_IP + + assert mqtt.CONF_DISCOVER_IP is CONF_DISCOVER_IP + + def test_has_mqtt_ip_lookup() -> None: """Test has_mqtt_ip_lookup function.""" @@ -4974,7 +4993,7 @@ def test_upload_program_ota_mqttip_deduplication( assert "192.168.1.100" in call_args[0] -@patch("esphome.components.api.client.run_logs") +@patch("esphome.api_client.run_logs") def test_show_logs_api_static_ip_with_mqttip( mock_run_logs: Mock, mock_mqtt_get_ip: Mock, @@ -5013,7 +5032,7 @@ def test_show_logs_api_static_ip_with_mqttip( ) -@patch("esphome.components.api.client.run_logs") +@patch("esphome.api_client.run_logs") def test_show_logs_api_multiple_mqttip_resolves_once( mock_run_logs: Mock, mock_mqtt_get_ip: Mock, @@ -5096,7 +5115,7 @@ def test_upload_program_ota_mqtt_timeout_fallback( ) -@patch("esphome.components.api.client.run_logs") +@patch("esphome.api_client.run_logs") def test_show_logs_api_mqtt_timeout_fallback( mock_run_logs: Mock, mock_mqtt_get_ip: Mock, @@ -5717,6 +5736,65 @@ def test_run_miniterm_batches_lines_with_same_timestamp( ) +def test_run_miniterm_analyzer_import_failure_keeps_streaming( + caplog: pytest.LogCaptureFixture, +) -> None: + """A broken platform import must not stop serial log streaming. + + The decoder resolves lazily, so a crash-shaped line has to arrive + before the import is attempted at all. + """ + mock_serial = MockSerial([b"PC: 0x40104960\r\n", MOCK_SERIAL_END]) + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_ESP32} + config = { + CONF_LOGGER: { + CONF_BAUD_RATE: 115200, + "deassert_rts_dtr": False, + } + } + args = MockArgs() + + with ( + caplog.at_level("INFO", logger="esphome.platform_hooks"), + patch("serial.Serial", return_value=mock_serial), + patch( + "esphome.platform_hooks.get_platform_hook", + side_effect=ImportError("broken platform package"), + ), + ): + result = run_miniterm(config, "/dev/ttyUSB0", args) + + assert result == 0 + # A broken package is distinguishable from a plain capability gap. + assert "failed to import: broken platform package" in caplog.text + + +def test_run_miniterm_no_stacktrace_analyzer( + caplog: pytest.LogCaptureFixture, +) -> None: + """Platforms without a stacktrace analyzer log an info and stream anyway.""" + mock_serial = MockSerial([b"[I][app:100]: Line 1\r\n", MOCK_SERIAL_END]) + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_BK72XX} + config = { + CONF_LOGGER: { + CONF_BAUD_RATE: 115200, + "deassert_rts_dtr": False, + } + } + args = MockArgs() + + with ( + caplog.at_level("INFO", logger="esphome.platform_hooks"), + patch("serial.Serial", return_value=mock_serial), + ): + result = run_miniterm(config, "/dev/ttyUSB0", args) + + assert result == 0 + assert "Stacktrace analysis is unavailable" in caplog.text + + def test_run_miniterm_different_chunks_different_timestamps( capfd: CaptureFixture[str], ) -> None: @@ -5797,7 +5875,9 @@ def test_run_miniterm_backtrace_state_maintained() -> None: mock_serial = MockSerial([backtrace_chunk, MOCK_SERIAL_END]) - CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_ESP32} + # An esp8266 dump on an esp8266 session; the platform-scoped gate + # would rightly never resolve esp32's decoder for these lines. + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_ESP8266} config = { CONF_LOGGER: { CONF_BAUD_RATE: 115200, @@ -5823,7 +5903,7 @@ def test_run_miniterm_backtrace_state_maintained() -> None: with ( patch("serial.Serial", return_value=mock_serial), patch.object( - esp32, + esp8266, "process_stacktrace", side_effect=track_backtrace_state, ), @@ -5850,6 +5930,38 @@ def test_run_miniterm_backtrace_state_maintained() -> None: assert backtrace_states[3][1] is True +def test_run_miniterm_decoder_failure_keeps_streaming( + caplog: pytest.LogCaptureFixture, +) -> None: + """A decoder exception must not kill serial streaming. + + This is the serial path's gain from sharing LogLineProcessor: before + the lift a decoder exception propagated out of the read loop. + """ + chunk = b"PC: 0x4010496e\r\nBT0: 0x4010496e\r\nstill streaming\r\n" + mock_serial = MockSerial([chunk, MOCK_SERIAL_END]) + + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: PLATFORM_ESP32} + config = { + CONF_LOGGER: { + CONF_BAUD_RATE: 115200, + "deassert_rts_dtr": False, + } + } + args = MockArgs() + + decoder = Mock(side_effect=EsphomeError("no idedata")) + with ( + patch("serial.Serial", return_value=mock_serial), + patch.object(esp32, "process_stacktrace", decoder), + ): + run_miniterm(config, "/dev/ttyUSB0", args) + + # The failure is contained and latched; streaming continued to EOF. + assert decoder.call_count == 1 + assert "Crash trace decoding unavailable" in caplog.text + + def test_run_miniterm_handles_empty_reads( capfd: CaptureFixture[str], ) -> None: @@ -6196,7 +6308,6 @@ def test_run_esphome_bundle_detection(tmp_path: Path) -> None: extracted_yaml = tmp_path / "extracted" / "device.yaml" with ( - patch("esphome.bundle.is_bundle_path", return_value=True) as mock_is_bundle, patch( "esphome.bundle.prepare_bundle_for_compile", return_value=extracted_yaml, @@ -6205,7 +6316,6 @@ def test_run_esphome_bundle_detection(tmp_path: Path) -> None: ): result = run_esphome(["esphome", "compile", str(bundle_path)]) - mock_is_bundle.assert_called_once() mock_prepare.assert_called_once_with(bundle_path) # read_config returns None → exit code 2 assert result == 2 @@ -6217,13 +6327,11 @@ def test_run_esphome_non_bundle_skips_extraction(tmp_path: Path) -> None: yaml_file.write_text("esphome:\n name: test\n") with ( - patch("esphome.bundle.is_bundle_path", return_value=False) as mock_is_bundle, patch("esphome.bundle.prepare_bundle_for_compile") as mock_prepare, patch("esphome.config.read_config", return_value=None), ): result = run_esphome(["esphome", "compile", str(yaml_file)]) - mock_is_bundle.assert_called_once() mock_prepare.assert_not_called() assert result == 2 @@ -6254,6 +6362,26 @@ def test_run_esphome_skip_external_update_per_command( assert mock_read.call_args.kwargs["skip_external_update"] is expected_skip +@pytest.mark.parametrize( + ("argv_extra", "expected"), + [(["--no-defaults"], True), ([], False)], +) +def test_run_esphome_snapshot_user_config_only_for_no_defaults( + tmp_path: Path, argv_extra: list[str], expected: bool +) -> None: + """read_config is invoked with snapshot_user_config=True only when the + config command is run with --no-defaults; otherwise the expensive deep + copy is skipped.""" + yaml_file = tmp_path / "device.yaml" + yaml_file.write_text("esphome:\n name: test\n") + + with patch("esphome.config.read_config", return_value=None) as mock_read: + run_esphome(["esphome", "config", str(yaml_file), *argv_extra]) + + mock_read.assert_called_once() + assert mock_read.call_args.kwargs["snapshot_user_config"] is expected + + def test_get_configured_xtal_freq_reads_sdkconfig(tmp_path: Path) -> None: """Test reading XTAL_FREQ from sdkconfig.""" CORE.name = "test-device" @@ -6468,7 +6596,7 @@ def test_should_subscribe_states_no_flag_overrides_env() -> None: assert _should_subscribe_states(args) is False -@patch("esphome.components.api.client.run_logs") +@patch("esphome.api_client.run_logs") def test_command_run_passes_no_states_to_show_logs( mock_run_logs: Mock, ) -> None: @@ -6506,7 +6634,7 @@ def test_command_run_passes_no_states_to_show_logs( ) -@patch("esphome.components.api.client.run_logs") +@patch("esphome.api_client.run_logs") def test_command_run_defaults_subscribe_states_true( mock_run_logs: Mock, ) -> None: @@ -6608,3 +6736,27 @@ def test_command_idedata_esp_idf_no_build_errors() -> None: result = command_idedata(MagicMock(), CORE.config) assert result == 1 + + +@pytest.mark.skipif( + os.name != "posix", reason="serial permission checks are posix-only" +) +def test_check_permissions_missing_port() -> None: + """A nonexistent serial port raises the does-not-exist guidance.""" + with ( + patch("os.access", return_value=False), + pytest.raises(EsphomeError, match="serial port does not exist"), + ): + check_permissions("/dev/ttyUSB99") + + +@pytest.mark.skipif( + os.name != "posix", reason="serial permission checks are posix-only" +) +def test_check_permissions_unreadable_port() -> None: + """An existing but unreadable serial port raises the dialout guidance.""" + with ( + patch("os.access", side_effect=lambda _path, mode: mode == os.F_OK), + pytest.raises(EsphomeError, match="read or write permission"), + ): + check_permissions("/dev/ttyUSB99") diff --git a/tests/unit_tests/test_platform_hooks.py b/tests/unit_tests/test_platform_hooks.py new file mode 100644 index 0000000000..97b25e7c0f --- /dev/null +++ b/tests/unit_tests/test_platform_hooks.py @@ -0,0 +1,186 @@ +"""Guard the platform CLI-hook registry in ``esphome.platform_hooks``. + +The registry lets the logs/upload fast path skip importing platform +packages that don't provide a hook; these tests fail when a platform +gains or loses a hook without the registry being updated, and pin down +that the fast path really avoids the import. +""" + +from __future__ import annotations + +import importlib +import logging +from unittest.mock import Mock + +import pytest + +from esphome import platform_hooks +from esphome.const import PLATFORM_BK72XX, PLATFORM_ESP32, Platform + + +def test_no_unregistered_platform_exposes_a_hook() -> None: + """Every platform hook the packages expose must be registered. + + Behavioural on purpose: a hook added as a re-export, an assignment, + or an ``async def`` is invisible to source scanning but very visible + to ``hasattr``, and an unregistered hook is silently never called. + The registered direction is covered by + test_every_registered_pair_resolves below. + """ + for platform in frozenset(Platform): + module = importlib.import_module(f"esphome.components.{platform}") + for hook, registered in platform_hooks.PLATFORM_HOOKS.items(): + if hasattr(module, hook): + assert platform in registered, ( + f"{platform} exposes {hook} but is not registered for it. " + "Update esphome/platform_hooks.py." + ) + + +def test_registered_platform_resolves_hook() -> None: + hook = platform_hooks.get_platform_hook(PLATFORM_ESP32, "process_stacktrace") + from esphome.components import esp32 + + assert hook is esp32.process_stacktrace + + +def test_every_registered_pair_resolves() -> None: + """Each registered platform must actually expose the hook at runtime. + + Text scanning can miss re-exports or decorated definitions; this is + the behavioural check for the direction that matters when the CLI + runs. + """ + for hook, platforms in platform_hooks.PLATFORM_HOOKS.items(): + for platform in platforms: + assert callable(platform_hooks.get_platform_hook(platform, hook)), ( + f"{platform} is registered for {hook} but does not expose it" + ) + + +def test_external_platform_falls_back_to_probe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Out-of-tree target platforms keep working via the dynamic probe.""" + module = type("FakePlatform", (), {"show_logs": staticmethod(lambda *a: True)}) + imported: list[str] = [] + + def fake_import(name: str): + imported.append(name) + return module + + monkeypatch.setattr(platform_hooks, "import_module", fake_import) + hook = platform_hooks.get_platform_hook("my_external_chip", "show_logs") + assert hook is module.show_logs + assert imported == ["esphome.components.my_external_chip"] + + +def test_external_platform_missing_module_degrades( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """A warm-cache run may not have the external package importable. + + Skipping a behavior-changing hook is visible at warning; losing + stacktrace decoding is cosmetic and stays at debug. + """ + monkeypatch.setattr( + platform_hooks, + "import_module", + Mock( + side_effect=ModuleNotFoundError( + "not found", name="esphome.components.my_external_chip" + ) + ), + ) + assert platform_hooks.get_platform_hook("my_external_chip", "show_logs") is None + assert "not importable" in caplog.text + assert any(r.levelname == "WARNING" for r in caplog.records) + + caplog.clear() + assert ( + platform_hooks.get_platform_hook("my_external_chip", "process_stacktrace") + is None + ) + assert not any(r.levelname == "WARNING" for r in caplog.records) + + +def test_external_platform_without_hook_logs_debug( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """The common no-hook case stays quiet but diagnosable.""" + caplog.set_level("DEBUG", logger="esphome.platform_hooks") + module = type("ExternalPlatform", (), {}) # imports fine, no hook + monkeypatch.setattr(platform_hooks, "import_module", Mock(return_value=module)) + assert platform_hooks.get_platform_hook("my_external_chip", "show_logs") is None + assert "does not expose" in caplog.text + assert not any(r.levelname == "WARNING" for r in caplog.records) + + +def test_stale_registry_entry_warns( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """A vendored tree where a registered hook vanished must say so.""" + module = type("StalePlatform", (), {}) # registered but no hook + monkeypatch.setattr(platform_hooks, "import_module", Mock(return_value=module)) + assert platform_hooks.get_platform_hook("nrf52", "show_logs") is None + assert "no longer exposes it" in caplog.text + + +def test_external_platform_broken_dependency_raises( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A missing dependency inside the external package must surface.""" + monkeypatch.setattr( + platform_hooks, + "import_module", + Mock(side_effect=ModuleNotFoundError("not found", name="some_missing_dep")), + ) + with pytest.raises(ModuleNotFoundError, match="not found"): + platform_hooks.get_platform_hook("my_external_chip", "show_logs") + + +def test_lookup_miss_does_not_import_platform_package( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The whole point: probing a platform without hooks must not import it.""" + monkeypatch.setattr( + platform_hooks, + "import_module", + Mock(side_effect=AssertionError("platform package imported on registry miss")), + ) + assert platform_hooks.get_platform_hook(PLATFORM_ESP32, "show_logs") is None + + +def test_get_stacktrace_handler_resolves_registered_platform() -> None: + hook = platform_hooks.get_stacktrace_handler(PLATFORM_ESP32) + from esphome.components import esp32 + + assert hook is esp32.process_stacktrace + + +def test_get_stacktrace_handler_reports_missing_analyzer( + caplog: pytest.LogCaptureFixture, +) -> None: + caplog.set_level("INFO", logger="esphome.platform_hooks") + assert platform_hooks.get_stacktrace_handler(PLATFORM_BK72XX) is None + assert "no compatible analyzer" in caplog.text + # A capability gap is ordinary; it must not warn. + assert not any(r.levelno >= logging.WARNING for r in caplog.records) + + +def test_get_stacktrace_handler_reports_import_failure( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setattr( + platform_hooks, + "import_module", + Mock(side_effect=ImportError("broken install")), + ) + assert platform_hooks.get_stacktrace_handler(PLATFORM_ESP32) is None + assert "failed to import: broken install" in caplog.text + # A broken install is a real breakage; it must warn, not inform. + assert any(r.levelno == logging.WARNING for r in caplog.records) diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index c0a0c678db..0eede78656 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -10,13 +10,14 @@ from pathlib import Path import pytest -from esphome.core import Library +from esphome.core import EsphomeError, Library import esphome.platformio.library as lib from esphome.platformio.library import ( ConvertedLibrary, GitSource, InvalidLibrary, LibraryBackend, + LocalSource, Source, URLSource, _resolve_registry_version, @@ -87,6 +88,68 @@ def test_gitsource_str_includes_ref_when_present(): assert str(GitSource("http://git/repo.git", None)) == "http://git/repo.git" +def test_source_root_defaults_to_build_dir() -> None: + # Registry/git sources are read from where they were downloaded. + build = Path("/some/build/dir") + assert URLSource("http://x/y.tar.gz").source_root(build) == build + assert GitSource("http://x/y.git", None).source_root(build) == build + + +def test_converted_library_source_dir_defaults_to_path() -> None: + c = ConvertedLibrary("x", "1.0", source=None) + c.path = Path("/build") + assert c.source_dir == Path("/build") # no source_path set -> build dir + c.source_path = Path("/user/lib") + assert c.source_dir == Path("/user/lib") + + +def test_convert_libraries_local_missing_manifest_is_esphome_error( + setup_core: Path, +) -> None: + # A local directory that has no library.json/library.properties is user + # input, so it must surface as a clean EsphomeError (named at the user's dir). + src = setup_core / "not_a_lib" + src.mkdir() # exists, but no manifest + # match= is a regex; a Windows path has backslashes, so match a literal + # fragment and check the directory is named separately. + with pytest.raises(EsphomeError, match="missing library.json") as excinfo: + convert_libraries([Library("Foo", None, src.as_uri())], _backend()) + assert str(src) in str(excinfo.value) + + +def test_localsource_download_missing_dir_raises(tmp_path: Path) -> None: + # EsphomeError so the CLI prints it cleanly instead of a traceback. + with pytest.raises(EsphomeError, match="does not exist"): + LocalSource(str(tmp_path / "nope")).download("mylib") + + +def test_localsource_str() -> None: + assert str(LocalSource("/tmp/lib")) == "file:///tmp/lib" + # A relative path can't form a file:// URI; fall back rather than raise. + assert str(LocalSource("rel/lib")) == "file://rel/lib" + + +def test_localsource_download_returns_empty_build_dir(setup_core: Path) -> None: + # Nothing is copied: download() returns an empty build dir (for generated + # files), and source_root() points back at the user's directory. + src = setup_core / "lib_dev" + (src / "src").mkdir(parents=True) + (src / "library.json").write_text("{}") + (src / "src" / "a.cpp").write_text("int a;") + + source = LocalSource(str(src)) + out = source.download("mylib", salt="s", namespace="ns") + + assert out.is_dir() + assert list(out.iterdir()) == [] # no sources copied in + assert out != src + assert source.source_root(out) == src + + # salt/namespace change the cache path. + plain = LocalSource(str(src)).download("mylib") + assert plain != out + + def test_urlsource_download_extracts_then_reuses_marker(setup_core, monkeypatch): monkeypatch.setattr(lib, "rmdir", lambda path, msg="": None) dl_calls: list[list[str]] = [] @@ -317,6 +380,140 @@ def test_convert_libraries_url_in_name_resolves_as_git( assert source.ref is None +def test_convert_libraries_file_url_resolves_as_local( + setup_core: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # A "Name=file://" library points at an on-disk folder: it resolves as a + # local source read in place (no copy), and the registry is never consulted. + src = setup_core / "lib_dev" + (src / "src").mkdir(parents=True) + (src / "library.json").write_text(json.dumps({"name": "TeslaBLE"})) + (src / "src" / "tesla.cpp").write_text("int foo() { return 1; }") + + def fail_registry(owner: str, pkgname: str, requirements: set[str]) -> None: + raise AssertionError(f"registry consulted for {owner}/{pkgname}") + + monkeypatch.setattr(lib, "_resolve_registry_version", fail_registry) + + # as_uri() produces a valid file:// URL on every platform (file:///tmp/... on + # POSIX, file:///C:/... on Windows). + top = convert_libraries([Library("TeslaBLE", None, src.as_uri())], _backend()) + + assert [c.name for c in top] == ["TeslaBLE"] + assert top[0].data["name"] == "TeslaBLE" + assert isinstance(top[0].source, LocalSource) + # Sources are read in place from the user's dir; the build dir stays separate + # and holds no copied sources. + assert top[0].source_path == src + assert top[0].path != src + assert not (top[0].path / "src").exists() + + +def test_convert_libraries_local_overrides_registry_version( + setup_core: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + # The same library requested both from the registry (with a version) and as + # a local directory resolves to the local source, with a warning that the + # registry version was dropped. + src = setup_core / "lib_dev" + (src / "src").mkdir(parents=True) + (src / "library.json").write_text(json.dumps({"name": "TeslaBLE"})) + + def fail_registry(owner: str, pkgname: str, requirements: set[str]) -> None: + raise AssertionError(f"registry consulted for {owner}/{pkgname}") + + monkeypatch.setattr(lib, "_resolve_registry_version", fail_registry) + + with caplog.at_level(logging.WARNING, logger="esphome.platformio.library"): + top = convert_libraries( + [ + Library("TeslaBLE", "1.0.0", None), + Library("TeslaBLE", None, src.as_uri()), + ], + _backend(), + ) + + assert isinstance(top[0].source, LocalSource) + assert "local source" in caplog.text + + +def test_convert_libraries_versionless_registry_and_local_warns( + setup_core: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + # A bare cg.add_library("Foo") (versionless registry, the common case) that + # collides with a local directory of the same key must still warn -- the + # registry spec is dropped and the local folder silently takes over. + src = setup_core / "foo" + src.mkdir() + (src / "library.json").write_text(json.dumps({"name": "Foo"})) + + def fail_registry(owner: str, pkgname: str, requirements: set[str]) -> None: + raise AssertionError(f"registry consulted for {owner}/{pkgname}") + + monkeypatch.setattr(lib, "_resolve_registry_version", fail_registry) + + with caplog.at_level(logging.WARNING, logger="esphome.platformio.library"): + top = convert_libraries( + [Library("Foo", None, None), Library("Foo", None, src.as_uri())], + _backend(), + ) + + assert isinstance(top[0].source, LocalSource) + assert "a registry package" in caplog.text + + +def test_convert_libraries_two_local_dirs_warns( + setup_core: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + # The same key pointed at two local directories warns and uses the last one. + dir_a = setup_core / "a" + dir_b = setup_core / "b" + for d in (dir_a, dir_b): + d.mkdir() + (d / "library.json").write_text(json.dumps({"name": "Foo"})) + + with caplog.at_level(logging.WARNING, logger="esphome.platformio.library"): + top = convert_libraries( + [ + Library("Foo", None, dir_a.as_uri()), + Library("Foo", None, dir_b.as_uri()), + ], + _backend(), + ) + + assert isinstance(top[0].source, LocalSource) + assert top[0].source_path == dir_b # the last one wins + assert "two local directories" in caplog.text + + +@pytest.mark.parametrize("local_first", [True, False]) +def test_convert_libraries_git_and_local_same_key_warns( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + local_first: bool, +) -> None: + # A key requested as both a git source and a local directory warns and uses + # git, whichever order they appear in. The git URL basename matches the local + # custom name so both map to the key "Foo". + _patch_download_with_manifests(monkeypatch, tmp_path, {"Foo": {"name": "Foo"}}) + git = Library("X", None, "https://host/Foo") + local = Library("Foo", None, "file:///abs/foo") + libs = [local, git] if local_first else [git, local] + + with caplog.at_level(logging.WARNING, logger="esphome.platformio.library"): + top = convert_libraries(libs, _backend()) + + assert isinstance(top[0].source, GitSource) + assert "using the git source" in caplog.text + + def test_convert_libraries_skips_incompatible_dependency(tmp_path, monkeypatch): # A dependency that declares an incompatible platform is skipped (the # top-level library still builds). diff --git a/tests/unit_tests/test_platformio_toolchain.py b/tests/unit_tests/test_platformio_toolchain.py index 723646fbd6..9450e8e0e1 100644 --- a/tests/unit_tests/test_platformio_toolchain.py +++ b/tests/unit_tests/test_platformio_toolchain.py @@ -278,15 +278,56 @@ def test_run_idedata_raises_on_no_json( def test_run_idedata_raises_on_invalid_json( setup_core: Path, mock_run_platformio_cli_run: Mock ) -> None: - """Test _run_idedata raises on malformed JSON.""" + """Malformed JSON is the environment (garbage stdout), so it must + surface as EsphomeError and get the recompile hint downstream. + """ config = {"name": "test"} mock_run_platformio_cli_run.return_value = '{"invalid": json"}' - # The ValueError from json.loads is re-raised - with pytest.raises(ValueError): + with pytest.raises(EsphomeError): toolchain._run_idedata(config) +def test_run_idedata_raises_on_launch_failure( + setup_core: Path, mock_run_platformio_cli_run: Mock +) -> None: + """A failed platformio launch returns its exit code as an int; that + must surface as EsphomeError, not a TypeError from re.search. + """ + config = {"name": "test"} + mock_run_platformio_cli_run.return_value = 1 + + with pytest.raises(EsphomeError): + toolchain._run_idedata(config) + + +def test_idedata_missing_prog_path_raises_esphome_error(setup_core: Path) -> None: + """A stale cached idedata JSON without prog_path is the build tree's + fault; it must surface as EsphomeError, not a KeyError. + """ + with pytest.raises(EsphomeError): + _ = toolchain.IDEData({}).firmware_elf_path + + +def test_idedata_missing_flash_image_field_raises_esphome_error( + setup_core: Path, +) -> None: + """A cached idedata whose flash image entries lost a field must + classify as an environment error too, not a raw KeyError. + """ + idedata = toolchain.IDEData({"extra": {"flash_images": [{"offset": "0x1000"}]}}) + with pytest.raises(EsphomeError): + _ = idedata.extra_flash_images + + +def test_idedata_null_section_raises_esphome_error(setup_core: Path) -> None: + """A section that is null instead of absent must classify the same + as a missing key instead of escaping as TypeError. + """ + with pytest.raises(EsphomeError): + _ = toolchain.IDEData({"extra": None}).extra_flash_images + + def test_run_platformio_cli_sets_environment_variables( setup_core: Path, mock_run_external_process: Mock ) -> None: diff --git a/tests/unit_tests/test_preference_hash_stability.py b/tests/unit_tests/test_preference_hash_stability.py new file mode 100644 index 0000000000..d3e5fac36a --- /dev/null +++ b/tests/unit_tests/test_preference_hash_stability.py @@ -0,0 +1,239 @@ +"""Tests to verify preference and entity key hash values remain stable. + +These tests ensure the hash algorithms do NOT change, as any change would cause +users to lose stored preferences (calibration values, restore states, etc.) on +firmware upgrades, or break entity state routing to API clients. + +Two algorithms are locked here (see https://github.com/esphome/backlog/issues/85): +1. `fnv1_hash_object_id(name)` - the LEGACY hash (snake_case + sanitize, then FNV-1). + Existing devices have preferences stored under keys derived from it; slot-based + backends (ESP8266, RP2040) keep using it, and key-lookup backends migrate FROM it. +2. `fnv1_hash_name(name)` - the entity key (FNV-1 over the raw UTF-8 name bytes). + Sent to API clients and used as the preference key base on key-lookup backends. + +DO NOT CHANGE THE EXPECTED VALUES - if tests fail after modifying a hash algorithm, +the change breaks backward compatibility and will cause data loss. +""" + +import pytest + +from esphome.helpers import ( + FNV1_OFFSET_BASIS, + FNV1_PRIME, + fnv1_hash_name, + fnv1_hash_object_id, +) + +# ============================================================================= +# Test: fnv1_hash_object_id produces stable hashes for entity names +# ============================================================================= + + +@pytest.mark.parametrize( + ("entity_name", "expected_object_id_hash"), + [ + # ===================================================================== + # Core entity types - these names appear in many ESPHome configurations + # ===================================================================== + # Basic single-word names + ("Light", 0x735CF023), + ("Switch", 0xBEDF78E5), + ("Sensor", 0x75E61B1B), + ("Fan", 0x468F6780), + ("Climate", 0xAA22FD4A), + ("Cover", 0xA630D0A2), + ("Lock", 0x1D2FD708), + ("Valve", 0x25ED5F65), + ("Button", 0x3A42C455), + ("Number", 0xB900E22A), + ("Select", 0x556391B5), + ("Text", 0xB12BFA38), + # Multi-word names (spaces become underscores, lowercase) + ("Living Room Light", 0xC6F81EC9), + ("Kitchen Switch", 0xC63C0F6E), + ("Temperature Sensor", 0x16AF55B6), + ("Garage Door Cover", 0x685E5281), + ("Bedroom Fan", 0x21AB1DED), + ("Front Door Lock", 0xB9BEF8E1), + # Already snake_case names (should hash same as space-separated) + ("living_room_light", 0xC6F81EC9), # Same as "Living Room Light" + ("kitchen_switch", 0xC63C0F6E), # Same as "Kitchen Switch" + # Names with numbers + ("Sensor 1", 0x99828E4B), + ("Relay 2", 0x6FFEF2FB), + ("Zone 10", 0xFD83AA95), + # Names with special characters (become underscores) + ("AC Unit", 0x336C6886), + ("WiFi Signal", 0x2FA52175), + ("CO2 Level", 0x31049870), + # Mixed case handling + ("mySwitch", 0x9AA10553), + ("MySwitch", 0x9AA10553), # Same as lowercase + ("MYSWITCH", 0x9AA10553), # Same as lowercase + # ===================================================================== + # Edge cases + # ===================================================================== + # Empty name (hashes to the FNV-1 offset basis since no chars processed) + ("", 0x811C9DC5), + # Single character + ("a", 0x050C5D7E), + ("A", 0x050C5D7E), # Same after lowercase + ("1", 0x050C5D2E), + ("_", 0x050C5D40), + # Names that differ only in case (should hash identically) + ("test", 0xBC2C0BE9), + ("Test", 0xBC2C0BE9), + ("TEST", 0xBC2C0BE9), + # Names that differ only in spaces vs underscores (should hash identically) + ("foo bar", 0x3AE35AA1), + ("foo_bar", 0x3AE35AA1), + ("Foo Bar", 0x3AE35AA1), + ("FOO_BAR", 0x3AE35AA1), + # Non-ASCII names (sanitized per code point, one underscore per character) + ("äöü", 0x10028B12), + ("温度", 0x3276CB9F), + ("Température", 0x965698F3), + # ===================================================================== + # Real-world component entity names from ESPHome codebase + # ===================================================================== + # From fan.cpp - FanRestoreState + ("Ceiling Fan", 0x640DEF00), + # From climate.cpp - ClimateRestoreState + ("HVAC", 0xDD68438B), + ("Thermostat", 0x30A5B7C6), + # From light/light_state.cpp + ("LED Strip", 0x2A068423), + ("Dimmable Light", 0xD70393F3), + # From cover/cover.cpp + ("Garage Door", 0x53987A5D), + ("Window Blind", 0x851291A5), + # From switch/switch.cpp + ("Relay", 0xD3A92FE4), + ("Power Switch", 0x5C4A47B3), + # From number/automation.cpp + ("Brightness", 0xF46E252C), + ("Volume", 0x8FFEBE43), + # From template datetime entities + ("Wake Time", 0xEE612B53), + ("Schedule Date", 0xF538C8DD), + ], +) +def test_entity_object_id_hash_stability( + entity_name: str, expected_object_id_hash: int +) -> None: + """Verify fnv1_hash_object_id produces stable hashes for entity names. + + CRITICAL: These expected values MUST NOT CHANGE. Existing devices have + preferences stored under keys derived from this legacy hash; changing it + breaks the old-to-new key migration and loses stored preferences. + """ + actual = fnv1_hash_object_id(entity_name) + assert actual == expected_object_id_hash, ( + f"Hash for '{entity_name}' changed from {expected_object_id_hash:#010x} to {actual:#010x}. " + f"This will cause users to lose stored preferences!" + ) + + +# ============================================================================= +# Test: Legacy preference key computation formula +# ============================================================================= + + +def compute_legacy_preference_key( + entity_name: str, version: int = 0, device_id: int = 0 +) -> int: + """Compute the legacy preference key: (object_id_hash ^ device_id) ^ version. + + This is the key existing devices have data stored under. Slot-based backends + (ESP8266, RP2040) still use it directly; key-lookup backends compute it as the + migration source in EntityBase::make_entity_preference_() (entity_base.cpp). + """ + object_id_hash = fnv1_hash_object_id(entity_name) + preference_hash = object_id_hash ^ device_id + key = preference_hash ^ version + return key & 0xFFFFFFFF + + +# Restore state version constants from ESPHome components +# These MUST match the RESTORE_STATE_VERSION values in the C++ code +FAN_RESTORE_STATE_VERSION = 0x71700ABA # From fan/fan.cpp +CLIMATE_RESTORE_STATE_VERSION = 0x848EA6AD # From climate/climate.cpp + + +@pytest.mark.parametrize( + ("entity_name", "version", "device_id", "expected_key"), + [ + # No version, main device (key equals the plain object_id hash) + ("Test Sensor", 0, 0, 0x5D74FA46), + ("Light", 0, 0, 0x735CF023), + # Restore state versions on the main device + ("Ceiling Fan", FAN_RESTORE_STATE_VERSION, 0, 0x157DE5BA), + ("HVAC", CLIMATE_RESTORE_STATE_VERSION, 0, 0x59E6E526), + # Sub-devices: same entity name on different devices gets different keys + ("Light", 0, 1, 0x735CF022), + ("Fan", FAN_RESTORE_STATE_VERSION, 0xABCD, 0x37FFC6F7), + ], +) +def test_legacy_preference_key_computation( + entity_name: str, version: int, device_id: int, expected_key: int +) -> None: + """Verify legacy preference key computation matches expected values. + + This test ensures the formula doesn't change, which would break both slot-based + preference storage and the migration source keys on key-lookup backends. + """ + actual_key = compute_legacy_preference_key(entity_name, version, device_id) + + assert actual_key == expected_key, ( + f"Preference key for '{entity_name}' (version={version:#x}, device_id={device_id}) " + f"changed from {expected_key:#010x} to {actual_key:#010x}. " + f"This will cause users to lose stored preferences!" + ) + + +# ============================================================================= +# Test: fnv1_hash_name produces stable entity keys (raw name, UTF-8 bytes) +# ============================================================================= + + +@pytest.mark.parametrize( + ("entity_name", "expected_key"), + [ + # ASCII names + ("Temperature Sensor", 0x801C3665), + ("LED Strip", 0xD5C7B082), + ("Garage Door", 0x2D70E086), + ("Relay", 0x565177C4), + # Raw names are case and space sensitive, unlike the old object_id hash + ("temperature sensor", 0xF9F431E5), + # Non-ASCII names hash their UTF-8 bytes and stay distinct + ("Датчик открытия", 0x001861C1), + ("温度", 0x8EDF61C9), + ("Température", 0x531A74AA), + # Empty name hashes to the FNV-1 offset basis + ("", 0x811C9DC5), + ], +) +def test_entity_key_hash_stability(entity_name: str, expected_key: int) -> None: + """Verify fnv1_hash_name produces stable entity keys. + + CRITICAL: These expected values MUST NOT CHANGE. The entity key is sent to + API clients and is the new preference key base; changing the algorithm + would break state routing and lose stored preferences. + Must match C++ fnv1_hash_bytes() in esphome/core/helpers.h. + """ + actual = fnv1_hash_name(entity_name) + assert actual == expected_key, ( + f"Entity key for '{entity_name}' changed from {expected_key:#010x} to {actual:#010x}. " + f"This breaks state routing and stored preferences!" + ) + + +def test_fnv1_hash_name_matches_utf8_byte_hash() -> None: + """Verify fnv1_hash_name hashes the UTF-8 encoded bytes of the name.""" + name = "Température 温度" + hash_value = FNV1_OFFSET_BASIS + for byte in name.encode("utf-8"): + hash_value = (hash_value * FNV1_PRIME) & 0xFFFFFFFF + hash_value ^= byte + assert fnv1_hash_name(name) == hash_value diff --git a/tests/unit_tests/test_resolver.py b/tests/unit_tests/test_resolver.py index 7862c268ca..16294a3813 100644 --- a/tests/unit_tests/test_resolver.py +++ b/tests/unit_tests/test_resolver.py @@ -4,12 +4,13 @@ from __future__ import annotations import re import socket -from unittest.mock import MagicMock, patch +from unittest.mock import ANY, patch from aioesphomeapi.core import ResolveAPIError, ResolveTimeoutAPIError from aioesphomeapi.host_resolver import AddrInfo, IPv4Sockaddr, IPv6Sockaddr import pytest +from esphome.async_thread import AsyncDispatchTimeout from esphome.core import EsphomeError from esphome.resolver import RESOLVE_TIMEOUT, AsyncResolver @@ -116,20 +117,17 @@ def test_async_resolver_generic_exception() -> None: def test_async_resolver_thread_timeout() -> None: """Test timeout when the runner thread doesn't complete in time.""" - # Patch AsyncThreadRunner inside esphome.resolver so we never actually - # start a thread and can control the wait return value directly. - fake_runner = MagicMock() - fake_runner.start = MagicMock() - fake_runner.event.wait.return_value = False # simulate timeout - + # Patch run_async inside esphome.resolver so we never actually start a + # thread and can simulate the wait timing out. with ( - patch("esphome.resolver.AsyncThreadRunner", return_value=fake_runner), - patch("esphome.resolver.hr.async_resolve_host"), + patch( + "esphome.resolver.run_async", side_effect=AsyncDispatchTimeout + ) as mock_run, pytest.raises(EsphomeError, match=re.escape("Timeout resolving IP address")), ): AsyncResolver(["test.local"], 6053).resolve() - fake_runner.start.assert_called_once() + mock_run.assert_called_once_with(ANY, timeout=RESOLVE_TIMEOUT + 1.0) def test_async_resolver_ip_addresses(mock_addr_info_ipv4: AddrInfo) -> None: diff --git a/tests/unit_tests/test_stacktrace.py b/tests/unit_tests/test_stacktrace.py new file mode 100644 index 0000000000..0b11ac3f83 --- /dev/null +++ b/tests/unit_tests/test_stacktrace.py @@ -0,0 +1,565 @@ +"""Tests for esphome.stacktrace.""" + +from __future__ import annotations + +import importlib +import inspect +from pathlib import Path +import re +from unittest.mock import Mock, patch + +from hypothesis import given, settings +from hypothesis.strategies import data as st_data, from_regex +import pytest + +from esphome import stacktrace +from esphome.const import ( + PLATFORM_BK72XX, + PLATFORM_ESP32, + PLATFORM_ESP8266, + PLATFORM_NRF52, + PLATFORM_RP2, +) +from esphome.core import EsphomeError + +CONFIG = {"esphome": {"name": "test"}} + +# Real dump lines per registered platform; the gate must fire on each. +# "addresses" are decoder-consumed dump lines, "state_markers" open a +# decoder's dump region, and "extra_triggers" fire the gate without a +# decoder pattern (the stored-dump banner). A new decoder declares its +# lines here so drift fails in CI instead of in the field. +CRASH_SAMPLES: dict[str, dict[str, list[str]]] = { + PLATFORM_ESP32: { + "state_markers": [], + "extra_triggers": ["*** CRASH DETECTED ON PREVIOUS BOOT ***"], + "addresses": [ + "Backtrace: 0x400d1a2c:0x3ffb1f60 0x400d2a3c:0x3ffb1f80", + "PC : 0x400d1a2c PS : 0x00060330", + "EXCVADDR: 0x40001234", + "MEPC : 0x40380abc RA : 0x40380def", + "MTVAL : 0x40000123", + "last failed alloc call: 40201234(512)", + "BT0: 0x40104960", + ], + }, + PLATFORM_ESP8266: { + "state_markers": [">>>stack>>>"], + "extra_triggers": ["*** CRASH DETECTED ON PREVIOUS BOOT ***"], + "addresses": [ + "epc1=0x40201234 epc2=0x00000000 excvaddr=0x40001234", + "3ffffe10: 40201234 3ffe8410 00000000 40201000", + "PC : 40201234", + "EXCVADDR: 0x40001234", + "BT0: 0x40201234", + "last failed alloc call: 40201234(512)", + "Exception (28):", + ], + }, + PLATFORM_RP2: { + "state_markers": ["CRASH DETECTED ON PREVIOUS BOOT"], + "addresses": ["PC: 0x10001234 (fault location)"], + }, + PLATFORM_NRF52: { + "state_markers": ["Last crash:"], + "addresses": [ + # %08x zero-pads even a vector-table PC past the {3,} bound. + "PC=0x00000050 LR=0x00000000", + # Synthetic short form; pins the bound's lower edge. + "PC=0x27a1c LR=0x1e33", + ], + }, +} + +BENIGN_LINES = [ + "[I][app:100] hello world", + "[C][wifi:400] BSSID: AA:BB:CC:DD:EE:FF", + "[19:26:11.966][I][main:151]: version 2026.7.0-dev", + "[I][app:102]: Uptime: 12345678 ms", + "[I][app:102]: Uptime: 41234567 ms", + "[V][esp-idf:000]: I (40219876) wifi: connected", + "[D][api:102]: Client connected (40123456)", + "[D][sensor:093]: 'Water meter': Sending state 12345678.00000 L", + # No internal word boundary; the bare-8-hex branch must not fire. + "[I][ota:117]: MD5 of binary: d41d8cd98f00b204e9800998ecf8427e", + # Short 0x tokens (BLE handles); the 3-digit minimum keeps them out. + "[D][ble:200]: Connection handle 0x1F, MTU 23", + "[C][network:600]: IPv6: fe80::1a2b:3c4d:5e6f:7a8b", + "[C][ota:097]: Version: 2026.7.0", +] + +GATE_PARAMS = [ + pytest.param(platform, line, True, id=f"{platform}-{kind}-{n}") + for platform, samples in CRASH_SAMPLES.items() + for kind in ("addresses", "state_markers", "extra_triggers") + for n, line in enumerate(samples.get(kind, [])) +] + [ + pytest.param(platform, line, False, id=f"benign-{platform}-{n}") + for platform in CRASH_SAMPLES + for n, line in enumerate(BENIGN_LINES) +] + + +@pytest.mark.parametrize(("platform", "line", "should_fire"), GATE_PARAMS) +def test_platform_gate(platform: str, line: str, should_fire: bool) -> None: + gate = re.compile(stacktrace.platform_hooks.STACKTRACE_GATES[platform]) + assert bool(gate.search(line)) is should_fire + + +def test_gates_are_platform_scoped() -> None: + """Another platform's markers must not fire an esp32 session's gate.""" + esp32_gate = re.compile(stacktrace.platform_hooks.STACKTRACE_GATES[PLATFORM_ESP32]) + for line in ( + ">>>stack>>>", + "Last crash:", + "Exception (28):", + "3ffffe10: 40201234 3ffe8410 00000000 40201000", + ): + assert not esp32_gate.search(line) + + +def _top_level_branches(pattern: str) -> list[str]: + """Split a regex source on alternations outside groups and classes.""" + branches: list[str] = [] + depth = 0 + in_class = False + esc = False + start = 0 + for i, ch in enumerate(pattern): + if esc: + esc = False + elif ch == "\\": + esc = True + elif in_class: + in_class = ch != "]" + elif ch == "[": + in_class = True + elif ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + elif ch == "|" and depth == 0: + branches.append(pattern[start:i]) + start = i + 1 + branches.append(pattern[start:]) + return branches + + +@pytest.mark.parametrize("platform", sorted(CRASH_SAMPLES)) +def test_every_gate_branch_is_exercised(platform: str) -> None: + """Every gate branch must be hit by a sample; the superset checks + stay green when a typoed alternation matches nothing. + """ + samples = CRASH_SAMPLES[platform] + lines = [line for kind in samples for line in samples[kind]] + branches = _top_level_branches(stacktrace.platform_hooks.STACKTRACE_GATES[platform]) + assert len(branches) > 1 + for branch in branches: + assert any(re.search(branch, line) for line in lines), ( + f"no {platform} sample exercises gate branch {branch!r}; add one " + "or drop the dead branch" + ) + + +# In-tree sources that print each marker literal the gates key on; +# esp8266's >>>stack>>> comes from the Arduino core, outside this tree. +FIRMWARE_MARKER_SOURCES = { + "CRASH DETECTED ON PREVIOUS BOOT": ( + "esphome/components/esp32/crash_handler.cpp", + "esphome/components/esp8266/crash_handler.cpp", + "esphome/components/rp2/crash_handler.cpp", + ), + "Last crash:": ("esphome/components/logger/logger_zephyr.cpp",), +} + + +def test_gate_markers_match_firmware_output() -> None: + """A reworded firmware banner must fail here, not in the field; + every regex-level guard stays green when the C++ side drifts. + """ + root = Path(__file__).parents[2] + for marker, sources in FIRMWARE_MARKER_SOURCES.items(): + for source in sources: + text = (root / source).read_text(encoding="utf-8") + assert marker in text, ( + f"{source} no longer prints {marker!r}; update the gates and " + "samples to the new banner" + ) + + +def test_crash_samples_cover_registry() -> None: + """A newly registered decoder must come with a non-empty gate sample.""" + assert set(CRASH_SAMPLES) == set(stacktrace.platform_hooks.STACKTRACE_GATES) + assert set(stacktrace.platform_hooks.STACKTRACE_GATES) == set( + stacktrace.platform_hooks.PLATFORM_HOOKS["process_stacktrace"] + ) + assert all(samples["addresses"] for samples in CRASH_SAMPLES.values()) + + +# The stacktrace pattern constants each decoder module exports. The +# samples and these patterns must cover each other, so an edit on either +# side fails the guards below instead of quietly widening the gap +# between the gate and the decoders. +DECODER_PATTERNS: dict[str, list[str]] = { + PLATFORM_ESP32: [ + "STACKTRACE_ESP32_PC_RE", + "STACKTRACE_ESP32_EXCVADDR_RE", + "STACKTRACE_ESP32_C3_PC_RE", + "STACKTRACE_ESP32_C3_RA_RE", + "STACKTRACE_ESP32_C3_MTVAL_RE", + "STACKTRACE_BAD_ALLOC_RE", + "STACKTRACE_ESP32_BACKTRACE_RE", + "STACKTRACE_ESP32_BACKTRACE_PC_RE", + "STACKTRACE_ESP32_CRASH_BT_RE", + ], + PLATFORM_ESP8266: [ + "STACKTRACE_ESP8266_EXCEPTION_TYPE_RE", + "STACKTRACE_ESP8266_PC_RE", + "STACKTRACE_ESP8266_EXCVADDR_RE", + "STACKTRACE_ESP8266_CRASH_PC_RE", + "STACKTRACE_ESP8266_CRASH_EXCVADDR_RE", + "STACKTRACE_ESP8266_CRASH_BT_RE", + "STACKTRACE_BAD_ALLOC_RE", + "STACKTRACE_ESP8266_BACKTRACE_PC_RE", + ], + PLATFORM_RP2: ["_CRASH_RE", "_CRASH_ADDR_RE"], + PLATFORM_NRF52: ["STACKTRACE_NRF52_PC_LR_RE"], +} + +# Declared decoder patterns whose language the gate deliberately does +# not cover: bare stack-dump words, where the gate keys on the dump +# line's 3ff... stack address instead and a lone letter-free word never +# appears outside a dump region whose other lines already fired. +GATE_EXEMPT_PATTERNS = { + "STACKTRACE_ESP32_BACKTRACE_PC_RE", + "STACKTRACE_ESP8266_BACKTRACE_PC_RE", +} + + +@pytest.mark.parametrize("platform", sorted(CRASH_SAMPLES)) +def test_platform_declarations_match_decoder(platform: str) -> None: + r"""Samples, declared patterns, and the decoder must agree. + + Checks: declared patterns exist, samples and patterns cover each + other, no stacktrace pattern is undeclared, markers open the dump + region, and a state-setting decoder declares a marker. + + Known blind spots: the catch-all backtrace patterns can satisfy the + sample direction alone; the undeclared sweep keys off naming; the + state-gating check is a textual heuristic (pinned against + respelling by the declared-markers direction); a second opening + marker beside a declared one passes unnoticed; and the generative + guard draws full matches, so trailing word characters defeating the + pointer branch's ``\b`` are invisible to it. + """ + module = importlib.import_module(f"esphome.components.{platform}") + patterns: dict[str, re.Pattern] = {} + for name in DECODER_PATTERNS[platform]: + pattern = getattr(module, name, None) + if pattern is None: + pytest.fail( + f"{platform} no longer defines {name}; update DECODER_PATTERNS " + "and CRASH_SAMPLES together" + ) + patterns[name] = pattern + + lines = ( + CRASH_SAMPLES[platform]["state_markers"] + CRASH_SAMPLES[platform]["addresses"] + ) + for line in CRASH_SAMPLES[platform]["addresses"]: + assert any(p.search(line) for p in patterns.values()), ( + f"{line!r} no longer matches any {platform} decoder pattern; " + "update CRASH_SAMPLES and re-derive the gate" + ) + for name, pattern in patterns.items(): + assert any(pattern.search(line) for line in lines), ( + f"no sample exercises {platform}.{name}; add one so the gate " + "provably covers it" + ) + undeclared = [ + name + for name, value in vars(module).items() + if isinstance(value, re.Pattern) + and ("STACKTRACE" in name or name.startswith("_CRASH")) + and name not in DECODER_PATTERNS[platform] + ] + assert not undeclared, ( + f"{platform} gained stacktrace patterns {undeclared}; declare them in " + "DECODER_PATTERNS with samples" + ) + + for marker in CRASH_SAMPLES[platform]["state_markers"]: + assert module.process_stacktrace(CONFIG, marker, False) is True, ( + f"{marker!r} no longer opens {platform}'s dump region; update " + "state_markers to the line the decoder actually keys on" + ) + # Textual heuristic, deliberately one-directional: a state-gated + # decoder must declare a marker. The reverse (a stateless decoder + # declaring none) is not asserted; an unrelated "return True" added + # to a decoder would turn it into a false failure. + source = inspect.getsource(module.process_stacktrace) + sets_state = "return True" in source or "backtrace_state = True" in source + if CRASH_SAMPLES[platform]["state_markers"]: + # The heuristic fails open on a respelling (return bool(...)); + # pinning it against the decoders known to be state-gated today + # turns a silent disarm into a failure that names the fix. + assert sets_state, ( + f"{platform}.process_stacktrace declares state_markers but the " + "state-gating heuristic no longer recognises it; update the " + "spelling list in this test" + ) + if sets_state: + assert CRASH_SAMPLES[platform]["state_markers"], ( + f"{platform}.process_stacktrace is state-gated but declares no " + "state_markers; the gate cannot promise to open its dump region" + ) + + +@pytest.mark.parametrize( + ("platform", "name"), + [ + (platform, name) + for platform, names in DECODER_PATTERNS.items() + for name in names + if name not in GATE_EXEMPT_PATTERNS + ], +) +@given(data=st_data()) +@settings(max_examples=25, deadline=None) +def test_address_gate_covers_decoder_pattern_languages( + platform: str, name: str, data +) -> None: + """Each platform's gate must be a superset of its decoder patterns; + generated inputs catch a widened decoder the finite samples miss. + """ + pattern = getattr(importlib.import_module(f"esphome.components.{platform}"), name) + example = data.draw(from_regex(pattern, fullmatch=True)) + gate = re.compile(stacktrace.platform_hooks.STACKTRACE_GATES[platform]) + assert gate.search(example), ( + f"{platform}.{name} accepts {example!r} but the {platform} gate does " + "not fire; decoding would silently never start on that form" + ) + + +def _run( + handler, + platform: str = PLATFORM_ESP32, + lines: tuple[str, ...] = ("PC: 0x4010496e",), +) -> stacktrace.LogLineProcessor: + """Processor with the resolver stubbed, fed the given lines.""" + with patch.object( + stacktrace.platform_hooks, "get_stacktrace_handler", return_value=handler + ): + processor = stacktrace.LogLineProcessor(CONFIG, platform) + for line in lines: + processor.process_line(line) + return processor + + +def _fed(handler) -> list[str]: + return [call.args[1] for call in handler.call_args_list] + + +def _warnings(caplog: pytest.LogCaptureFixture) -> list[str]: + return [r.message for r in caplog.records if r.levelname == "WARNING"] + + +def test_decoder_contains_failures_and_short_circuits() -> None: + """One decode failure is contained and never retried; a retry per + backtrace line would stall streaming on a failing subprocess. + """ + handler = Mock(side_effect=EsphomeError("no idedata")) + processor = _run( + handler, lines=("PC: 0x4010496e", "BT0: 0x4010496e", "BT1: 0x401049aa") + ) + + assert handler.call_count == 1 + assert processor.backtrace_state is False + + +def test_decoder_swallows_os_error_with_remediation_hint( + caplog: pytest.LogCaptureFixture, +) -> None: + """An OSError (missing build tree) is the user's environment, not a + decoder bug; it must keep the recompile hint. + """ + handler = Mock( + side_effect=FileNotFoundError(2, "No such file or directory", "/build") + ) + processor = _run(handler, lines=("PC: 0x4010496e", "BT0: 0x4010496e")) + + assert handler.call_count == 1 + assert processor.backtrace_state is False + warnings = _warnings(caplog) + assert any("esphome compile" in m for m in warnings) + assert not any("this is a bug" in m for m in warnings) + + +def test_decoder_warning_uses_fallback_for_empty_error( + caplog: pytest.LogCaptureFixture, +) -> None: + """A bare EsphomeError must not render as empty parens.""" + _run(Mock(side_effect=EsphomeError())) + + warnings = _warnings(caplog) + assert any("build artifacts not found locally" in m for m in warnings) + assert not any("()" in m for m in warnings) + + +def test_decoder_bug_with_empty_message_names_the_type( + caplog: pytest.LogCaptureFixture, +) -> None: + """A decoder bug says so instead of sending the user down the + dead-end recompile path. + """ + _run(Mock(side_effect=IndexError())) + + warnings = _warnings(caplog) + assert any("IndexError" in m and "this is a bug" in m for m in warnings) + assert not any("esphome compile" in m for m in warnings) + + +def test_decoder_bug_warning_keeps_the_type_with_a_message( + caplog: pytest.LogCaptureFixture, +) -> None: + """The type must survive a non-empty message; a bare KeyError message + like 'prog_path' reads as a raised string in a bug report paste. + """ + _run(Mock(side_effect=KeyError("prog_path"))) + + warnings = _warnings(caplog) + assert any("KeyError: 'prog_path'" in m for m in warnings) + + +def test_marker_then_address_threads_state() -> None: + """A state marker resolves the decoder live and threads state to + the following stack words. + """ + handler = Mock(side_effect=[True, True]) + processor = _run( + handler, + platform=PLATFORM_ESP8266, + lines=(">>>stack>>>", "3ffffe10: 40201234 3ffe8410 00000000 40201000"), + ) + + assert _fed(handler) == [ + ">>>stack>>>", + "3ffffe10: 40201234 3ffe8410 00000000 40201000", + ] + assert handler.call_args_list[0].args[2] is False + assert handler.call_args_list[1].args[2] is True + assert processor.backtrace_state is True + + +def test_lines_before_the_gate_never_reach_the_decoder() -> None: + """Benign lines are dropped, not buffered.""" + handler = Mock(return_value=False) + quiet = tuple(f"quiet line {n}" for n in range(12)) + _run(handler, lines=quiet + ("PC: 0x4010496e",)) + + assert _fed(handler) == ["PC: 0x4010496e"] + + +def test_processor_resolves_lazily_on_address_token() -> None: + """No resolution attempt until a line carries an address token.""" + handler = Mock(return_value=False) + + with patch.object( + stacktrace.platform_hooks, "get_stacktrace_handler", return_value=handler + ) as mock_resolve: + processor = stacktrace.LogLineProcessor(CONFIG, PLATFORM_ESP32) + processor.process_line("[I][app:100] hello world") + mock_resolve.assert_not_called() + + processor.process_line("PC: 0x40104960") + mock_resolve.assert_called_once_with(PLATFORM_ESP32) + + # Later lines feed the resolved handler directly, no re-resolution. + processor.process_line("[I][app:101] back to normal") + mock_resolve.assert_called_once() + + assert _fed(handler) == ["PC: 0x40104960", "[I][app:101] back to normal"] + + +def test_processor_unexpected_resolution_error_disables_decoding( + caplog: pytest.LogCaptureFixture, +) -> None: + """Resolution is inside the containment guarantee like everything else.""" + with patch.object( + stacktrace.platform_hooks, + "get_stacktrace_handler", + side_effect=OSError("filesystem went away"), + ) as mock_resolve: + processor = stacktrace.LogLineProcessor(CONFIG, PLATFORM_ESP32) + processor.process_line("PC: 0x40104960") + processor.process_line("BT0: 0x40104960") + + mock_resolve.assert_called_once() + warnings = _warnings(caplog) + assert len(warnings) == 1 + assert "could not be loaded" in warnings[0] + assert processor.backtrace_state is False + + +def test_processor_import_failure_disables_decoding( + caplog: pytest.LogCaptureFixture, +) -> None: + """A broken platform package degrades once instead of raising.""" + caplog.set_level("INFO", logger="esphome.platform_hooks") + + with patch.object( + stacktrace.platform_hooks, + "import_module", + Mock(side_effect=ImportError("broken install")), + ) as mock_import: + processor = stacktrace.LogLineProcessor(CONFIG, PLATFORM_ESP32) + processor.process_line("PC: 0x40104960") + processor.process_line("BT0: 0x40104960") + + mock_import.assert_called_once() + assert "Stacktrace analysis is unavailable" in caplog.text + assert "broken install" in caplog.text + assert processor.backtrace_state is False + + +def test_processor_registry_miss_disables_at_construction( + caplog: pytest.LogCaptureFixture, +) -> None: + """Platforms the registry proves have no analyzer disable up front. + + The unavailable notice fires at session start (as it always did) and + the per-line gate never runs. + """ + caplog.set_level("INFO", logger="esphome.platform_hooks") + + with patch.object(stacktrace.platform_hooks, "import_module") as mock_import: + processor = stacktrace.LogLineProcessor(CONFIG, PLATFORM_BK72XX) + processor.process_line("PC: 0x40104960") + + mock_import.assert_not_called() + assert "Stacktrace analysis is unavailable" in caplog.text + assert processor.backtrace_state is False + + +def test_external_platform_resolves_at_construction( + caplog: pytest.LogCaptureFixture, +) -> None: + """External platforms resolve eagerly; the gates cannot speak for an + external decoder and the import belongs off the streaming callback. + """ + caplog.set_level("INFO", logger="esphome.platform_hooks") + module = type("ExternalPlatform", (), {}) # no process_stacktrace + + with patch.object( + stacktrace.platform_hooks, + "import_module", + Mock(return_value=module), + ) as mock_import: + processor = stacktrace.LogLineProcessor(CONFIG, "my_external_chip") + mock_import.assert_called_once() + assert "Stacktrace analysis is unavailable" in caplog.text + + processor.process_line("PC: 0x40104960") + + mock_import.assert_called_once() + assert processor.backtrace_state is False diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index baaa99f2a7..f4063237b1 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -1,3 +1,5 @@ +from collections import ChainMap +from fnmatch import fnmatchcase import logging from pathlib import Path from typing import Any @@ -368,7 +370,7 @@ def test_validate_config_captures_user_config_snapshot(tmp_path: Path) -> None: """ test_config = _get_test_minimal_valid_config(tmp_path) - result = config_module.validate_config(test_config, None) + result = config_module.validate_config(test_config, None, snapshot_user_config=True) # Snapshot is populated. assert result.user_config is not None @@ -391,7 +393,7 @@ def test_validate_config_user_config_snapshot_is_deep_copy(tmp_path: Path) -> No """ test_config = _get_test_minimal_valid_config(tmp_path) - result = config_module.validate_config(test_config, None) + result = config_module.validate_config(test_config, None, snapshot_user_config=True) assert result.user_config is not None # preload_core_config injected build_path onto the validated config. @@ -402,6 +404,32 @@ def test_validate_config_user_config_snapshot_is_deep_copy(tmp_path: Path) -> No assert result["esphome"] is not result.user_config["esphome"] +def test_validate_config_snapshot_without_substitutions(tmp_path: Path) -> None: + """The snapshot works for configs that have no substitutions block.""" + test_config = _get_test_minimal_valid_config(tmp_path) + del test_config[CONF_SUBSTITUTIONS] + + result = config_module.validate_config(test_config, None, snapshot_user_config=True) + + assert result.user_config is not None + assert CONF_SUBSTITUTIONS not in result.user_config + assert result.user_config["esphome"] == {"name": "test_device"} + + +def test_validate_config_skips_user_config_snapshot_by_default( + tmp_path: Path, +) -> None: + """Without ``snapshot_user_config`` the deep copy is skipped entirely; + only ``esphome config --no-defaults`` needs the snapshot and the copy is + too expensive to take on every load. + """ + test_config = _get_test_minimal_valid_config(tmp_path) + + result = config_module.validate_config(test_config, None) + + assert result.user_config is None + + def test_merge_config_preserves_ordered_dict() -> None: """Test that merge_config preserves OrderedDict type. @@ -961,3 +989,134 @@ def test_remote_package_scalar_yaml_raises_helpful_error( msg = str(exc_info.value) assert "mapping at the top level" in msg assert "file1.yaml" in msg + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + pytest.param("wifi.yaml", ["wifi.yaml"], id="literal_passthrough"), + pytest.param( + "keys/${system_name}.yaml", ["keys/*.yaml"], id="embedded_substitution" + ), + pytest.param( + "network/${eth_model}/config.yaml", + ["network/*/config.yaml"], + id="directory_substitution", + ), + pytest.param( + "device-$platform.yaml", ["device-*.yaml"], id="unbraced_substitution" + ), + pytest.param("${a}${b}.yaml", ["*.yaml"], id="adjacent_wildcards_collapse"), + pytest.param( + '${ "a.yaml" if x else "../empty.yaml" }', + ["a.yaml", "../empty.yaml"], + id="conditional_literals", + ), + pytest.param( + 'pre-${ "a" if c else "b" }.yaml', + ["pre-a.yaml", "pre-b.yaml"], + id="conditional_spliced", + ), + pytest.param( + '${ "x.yaml" if a else ("y.yaml" if b else "z.yaml") }', + ["x.yaml", "y.yaml", "z.yaml"], + id="nested_conditional", + ), + pytest.param( + '${ "same.yaml" if x else "same.yaml" }', + ["same.yaml"], + id="duplicate_literals_dedupe", + ), + pytest.param('${ "a.yaml" if x }', ["a.yaml"], id="conditional_no_else"), + pytest.param( + '${ "NO BLUETOOTH SUPPORT ON ESP8266.yaml"' + ' if enable_bluetooth_proxy else "../empty.yaml" }', + ["NO BLUETOOTH SUPPORT ON ESP8266.yaml", "../empty.yaml"], + id="issue_17650_verbatim", + ), + pytest.param( + '${ "" if x else "b.yaml" }', ["b.yaml"], id="empty_literal_dropped" + ), + pytest.param( + "keys\\${system_name}.yaml", + ["keys\\*.yaml"], + id="backslash_separator", + ), + pytest.param( + '${ "it\'s.yaml" if x else "b.yaml" }', + ["it's.yaml", "b.yaml"], + id="apostrophe_in_literal", + ), + pytest.param( + '${ "a-${x}.yaml" if c else "b.yaml" }', + ["a-*.yaml", "b.yaml"], + id="substitution_inside_literal", + ), + pytest.param("sensor [${x}].yaml", ["sensor [[]*].yaml"], id="bracket_escaped"), + pytest.param( + "config?${x}.yaml", ["config[?]*.yaml"], id="question_mark_escaped" + ), + pytest.param( + "../${x}/config.yaml", ["../*/config.yaml"], id="ascending_directory" + ), + pytest.param("${file}", [], id="bare_variable_dropped"), + pytest.param("../${file}", [], id="ascending_bare_variable_dropped"), + pytest.param( + '${ name ~ ".yaml" }', [".yaml"], id="dynamic_concat_extracts_literal" + ), + pytest.param("${ if }", [], id="no_literal_expression_dropped"), + pytest.param( + "<% if x %>a.yaml<% endif %>", ["*a.yaml*"], id="block_statement_globs" + ), + ], +) +def test_include_candidate_patterns(value: str, expected: list[str]) -> None: + """Templated include paths expand to glob patterns and branch literals.""" + assert substitutions.include_candidate_patterns(value) == expected + + +@pytest.mark.parametrize( + ("template", "variables"), + [ + pytest.param( + "keys/${system_name}.yaml", {"system_name": "esp-buero"}, id="embedded" + ), + pytest.param("device-$platform.yaml", {"platform": "esp32"}, id="unbraced"), + pytest.param( + "network/${eth_model}/config.yaml", {"eth_model": "eth01"}, id="directory" + ), + pytest.param( + '${ "NO BT.yaml" if bt else "../empty.yaml" }', + {"bt": True}, + id="conditional_true", + ), + pytest.param( + '${ "NO BT.yaml" if bt else "../empty.yaml" }', + {"bt": False}, + id="conditional_false", + ), + pytest.param('pre-${ "a" if c else "b" }.yaml', {"c": True}, id="spliced"), + pytest.param("${a}${b}.yaml", {"a": "x", "b": "y"}, id="adjacent"), + pytest.param("sensor [${x}].yaml", {"x": "a"}, id="bracket"), + ], +) +def test_include_candidate_patterns_cover_real_expansion( + template: str, variables: dict[str, Any] +) -> None: + """ + Lockstep pin against the real substitution machinery. + + include_candidate_patterns mirrors _expand_substitutions without + variable values (the evaluator returns the one selected branch, so it + cannot enumerate candidates itself); this asserts every filename the + real pass resolves is covered by a candidate pattern, so a change to + reference syntax or expansion order breaks here instead of silently + dropping files from bundles. + """ + resolved = str( + substitutions._expand_substitutions( + template, [], ChainMap(variables), True, None + ) + ) + patterns = substitutions.include_candidate_patterns(template) + assert any(fnmatchcase(resolved, p) or resolved == p for p in patterns) diff --git a/tests/unit_tests/test_util.py b/tests/unit_tests/test_util.py index 581b1aca99..02309fbff8 100644 --- a/tests/unit_tests/test_util.py +++ b/tests/unit_tests/test_util.py @@ -561,7 +561,7 @@ def test_run_external_process_line_callbacks() -> None: return "PROCESS CALLBACK\n" return None - with patch("esphome.util.subprocess.run") as mock_run: + with patch("subprocess.run") as mock_run: def run_side_effect(*args: Any, **kwargs: Any) -> MagicMock: # Simulate subprocess writing to the stdout RedirectText @@ -635,7 +635,7 @@ def test_detect_rp2040_bootsel_found() -> None: """Test BOOTSEL device detection when device is present.""" mock_result = MagicMock() mock_result.stdout = b"Device Information\n type: RP2040\n" - with patch("esphome.util.subprocess.run", return_value=mock_result): + with patch("subprocess.run", return_value=mock_result): result = util.detect_rp2040_bootsel("/usr/bin/picotool") assert result.device_count == 1 assert result.permission_error is False @@ -645,7 +645,7 @@ def test_detect_rp2040_bootsel_multiple() -> None: """Test BOOTSEL detection with multiple devices.""" mock_result = MagicMock() mock_result.stdout = b"type: RP2040\ntype: RP2350\n" - with patch("esphome.util.subprocess.run", return_value=mock_result): + with patch("subprocess.run", return_value=mock_result): result = util.detect_rp2040_bootsel("/usr/bin/picotool") assert result.device_count == 2 assert result.permission_error is False @@ -658,7 +658,7 @@ def test_detect_rp2040_bootsel_none() -> None: b"No accessible RP2040/RP2350 devices in BOOTSEL mode were found.\n" ) mock_result.stderr = b"" - with patch("esphome.util.subprocess.run", return_value=mock_result): + with patch("subprocess.run", return_value=mock_result): result = util.detect_rp2040_bootsel("/usr/bin/picotool") assert result.device_count == 0 assert result.permission_error is False @@ -675,7 +675,7 @@ def test_detect_rp2040_bootsel_permission_error() -> None: b"but picotool was unable to connect. " b"Maybe try 'sudo' or check your permissions.\n" ) - with patch("esphome.util.subprocess.run", return_value=mock_result): + with patch("subprocess.run", return_value=mock_result): result = util.detect_rp2040_bootsel("/usr/bin/picotool") assert result.device_count == 0 assert result.permission_error is True @@ -686,7 +686,7 @@ def test_detect_rp2040_bootsel_libusb_access_error() -> None: mock_result = MagicMock() mock_result.stdout = b"" mock_result.stderr = b"LIBUSB_ERROR_ACCESS\n" - with patch("esphome.util.subprocess.run", return_value=mock_result): + with patch("subprocess.run", return_value=mock_result): result = util.detect_rp2040_bootsel("/usr/bin/picotool") assert result.device_count == 0 assert result.permission_error is True @@ -694,7 +694,7 @@ def test_detect_rp2040_bootsel_libusb_access_error() -> None: def test_detect_rp2040_bootsel_oserror() -> None: """Test BOOTSEL detection handles OSError.""" - with patch("esphome.util.subprocess.run", side_effect=OSError("not found")): + with patch("subprocess.run", side_effect=OSError("not found")): result = util.detect_rp2040_bootsel("/usr/bin/picotool") assert result.device_count == 0 assert result.permission_error is False @@ -703,7 +703,7 @@ def test_detect_rp2040_bootsel_oserror() -> None: def test_detect_rp2040_bootsel_timeout() -> None: """Test BOOTSEL detection handles timeout.""" with patch( - "esphome.util.subprocess.run", + "subprocess.run", side_effect=subprocess.TimeoutExpired("picotool", 10), ): result = util.detect_rp2040_bootsel("/usr/bin/picotool") diff --git a/tests/unit_tests/test_yaml_util.py b/tests/unit_tests/test_yaml_util.py index 5c38fce105..e0a81652e3 100644 --- a/tests/unit_tests/test_yaml_util.py +++ b/tests/unit_tests/test_yaml_util.py @@ -282,8 +282,54 @@ test: !include_dir_named test_dir assert ".hidden_dir" not in actual["test"] +def test_include_dir_list(tmp_path: Path) -> None: + """!include_dir_list loads every .yaml file in the directory as a list.""" + test_dir = tmp_path / "test_dir" + test_dir.mkdir() + (test_dir / "a.yaml").write_text("key: value_a") + (test_dir / "b.yaml").write_text("key: value_b") + + test_yaml = tmp_path / "test.yaml" + test_yaml.write_text("test: !include_dir_list test_dir\n") + + actual = yaml_util.load_yaml(test_yaml) + + assert len(actual["test"]) == 2 + assert {entry["key"] for entry in actual["test"]} == {"value_a", "value_b"} + + +def test_include_dir_merge_list(tmp_path: Path) -> None: + """!include_dir_merge_list concatenates the lists from every .yaml file.""" + test_dir = tmp_path / "test_dir" + test_dir.mkdir() + (test_dir / "a.yaml").write_text("- item_a1\n- item_a2\n") + (test_dir / "b.yaml").write_text("- item_b1\n") + + test_yaml = tmp_path / "test.yaml" + test_yaml.write_text("test: !include_dir_merge_list test_dir\n") + + actual = yaml_util.load_yaml(test_yaml) + + assert sorted(actual["test"]) == ["item_a1", "item_a2", "item_b1"] + + +def test_include_dir_merge_named(tmp_path: Path) -> None: + """!include_dir_merge_named merges the mappings from every .yaml file.""" + test_dir = tmp_path / "test_dir" + test_dir.mkdir() + (test_dir / "a.yaml").write_text("key_a: value_a") + (test_dir / "b.yaml").write_text("key_b: value_b") + + test_yaml = tmp_path / "test.yaml" + test_yaml.write_text("test: !include_dir_merge_named test_dir\n") + + actual = yaml_util.load_yaml(test_yaml) + + assert actual["test"] == {"key_a": "value_a", "key_b": "value_b"} + + def test_find_files_recursive(fixture_path: Path, tmp_path: Path) -> None: - """Test that _find_files works recursively through include_dir_named.""" + """Test that find_files works recursively through include_dir_named.""" # Copy fixture directory to temporary location src_dir = fixture_path / "yaml_util" dst_dir = tmp_path / "yaml_util" @@ -1003,8 +1049,10 @@ class _StubInclude: load_result: object = None, raise_on_load: EsphomeError | None = None, ) -> None: + # Default parent lives in a nonexistent directory so unresolved + # stubs never glob real files during candidate expansion. self.file = Path(file) - self.parent_file = parent_file or Path("/tmp/parent.yaml") + self.parent_file = parent_file or Path("/nonexistent/parent.yaml") self._unresolved = unresolved self._load_result = load_result if load_result is not None else {} self._raise = raise_on_load @@ -1182,6 +1230,247 @@ def test_discover_user_yaml_files_deduplicates(tmp_path: Path) -> None: assert discovered.files.count(wifi_resolved) == 1 +def test_discover_user_yaml_files_expands_directory_substitution( + tmp_path: Path, +) -> None: + """A substitution spanning a directory segment globs across directories.""" + _write(tmp_path, "network/eth01/config.yaml", "ethernet:\n") + _write(tmp_path, "network/eth02/config.yaml", "ethernet:\n") + discovered = discover_user_yaml_files( + _write_entry_including(tmp_path, "network/${eth_model}/config.yaml") + ) + resolved = set(discovered.files) + assert (tmp_path / "network/eth01/config.yaml").resolve() in resolved + assert (tmp_path / "network/eth02/config.yaml").resolve() in resolved + + +def test_discover_user_yaml_files_loads_both_branches_of_issue_conditional( + tmp_path: Path, +) -> None: + """Both branch files of the issue-17650 conditional load when present, + including the filename with spaces.""" + _write(tmp_path, "empty.yaml", "{}\n") + _write(tmp_path, "boards/NO BLUETOOTH SUPPORT ON ESP8266.yaml", "api:\n") + _write( + tmp_path, + "boards/esp8266.yaml", + "packages:\n" + ' - !include ${ "NO BLUETOOTH SUPPORT ON ESP8266.yaml"' + ' if enable_bluetooth_proxy else "../empty.yaml" }\n', + ) + discovered = discover_user_yaml_files( + _write_entry_including(tmp_path, "boards/esp8266.yaml") + ) + resolved = set(discovered.files) + assert (tmp_path / "boards/NO BLUETOOTH SUPPORT ON ESP8266.yaml").resolve() in ( + resolved + ) + assert (tmp_path / "empty.yaml").resolve() in resolved + + +def test_discover_user_yaml_files_glob_matches_bracket_filenames( + tmp_path: Path, +) -> None: + """Glob metacharacters in the literal filename text stay literal.""" + _write(tmp_path, "sensor [a].yaml", "api:\n") + discovered = discover_user_yaml_files( + _write_entry_including(tmp_path, "sensor [${x}].yaml") + ) + assert "sensor [a].yaml" in {p.name for p in discovered.files} + + +def test_discover_user_yaml_files_ascending_glob(tmp_path: Path) -> None: + """A templated include reaching into a sibling directory via ``..`` globs.""" + _write(tmp_path, "shared/common.yaml", "api:\n") + _write(tmp_path, "nodes/dev.yaml", "p: !include ../shared/${x}.yaml\n") + discovered = discover_user_yaml_files( + _write_entry_including(tmp_path, "nodes/dev.yaml") + ) + assert (tmp_path / "shared/common.yaml").resolve() in discovered.files + + +def test_discover_user_yaml_files_mapping_include_with_vars(tmp_path: Path) -> None: + """The mapping !include form (file + vars) expands a templated filename.""" + _write(tmp_path, "keys/a.yaml", "pin: ${num}\n") + entry = _write( + tmp_path, + "entry.yaml", + "wifi: !include\n file: keys/${n}.yaml\n vars:\n num: 4\n", + ) + discovered = discover_user_yaml_files(entry) + assert (tmp_path / "keys/a.yaml").resolve() in discovered.files + + +def test_discover_user_yaml_files_absolute_templated_include_skipped( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An absolute templated include is skipped gracefully instead of crashing.""" + shared = tmp_path / "shared" + _write(tmp_path, "shared/common.yaml", "api:\n") + with caplog.at_level("DEBUG", logger="esphome.yaml_util"): + discovered = discover_user_yaml_files( + _write_entry_including(tmp_path, f"{shared}/${{x}}.yaml") + ) + assert (shared / "common.yaml").resolve() not in discovered.files + assert any("Cannot glob include pattern" in r.message for r in caplog.records) + + +def test_discover_user_yaml_files_glob_skips_dollar_named_files( + tmp_path: Path, +) -> None: + """An on-disk filename containing ``$`` can't load; the glob skips it.""" + _write(tmp_path, "keys/a.yaml", "api:\n") + _write(tmp_path, "keys/b$roken.yaml", "api:\n") + discovered = discover_user_yaml_files( + _write_entry_including(tmp_path, "keys/${n}.yaml") + ) + names = {p.name for p in discovered.files} + assert "a.yaml" in names + assert "b$roken.yaml" not in names + + +def test_discover_user_yaml_files_glob_error_skips_include( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A filesystem error during candidate globbing warns and skips the include.""" + entry = _write_entry_including(tmp_path, "keys/${n}.yaml") + with ( + patch.object(Path, "glob", side_effect=OSError("boom")), + caplog.at_level("DEBUG", logger="esphome.yaml_util"), + ): + discovered = discover_user_yaml_files(entry) + assert [p.name for p in discovered.files] == ["entry.yaml"] + matching = [ + r.levelname + for r in caplog.records + if "I/O error globbing include pattern" in r.message + ] + assert matching == ["WARNING"] + + +def test_force_load_candidate_failure_warns_by_default( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A broken candidate logs at WARNING outside the discovery re-parse.""" + _write(tmp_path, "keys/bad.yaml", "esphome: [unterminated\n") + entry = _write_entry_including(tmp_path, "keys/${n}.yaml") + with caplog.at_level("DEBUG", logger="esphome.yaml_util"): + force_load_include_files(yaml_util.load_yaml(entry)) + matching = [ + r.levelname for r in caplog.records if "Failed to load candidate" in r.message + ] + assert matching == ["WARNING"] + + +def test_discover_user_yaml_files_glob_skips_hidden_files(tmp_path: Path) -> None: + """Candidate globs exclude hidden files, matching ``!include_dir_*``.""" + _write(tmp_path, "keys/device-a.yaml", "api:\n") + _write(tmp_path, "keys/.hidden.yaml", "api:\n") + discovered = discover_user_yaml_files( + _write_entry_including(tmp_path, "keys/${name}.yaml") + ) + names = {p.name for p in discovered.files} + assert "device-a.yaml" in names + assert ".hidden.yaml" not in names + + +def test_discover_user_yaml_files_bare_expression_not_expanded( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A fully dynamic filename never globs the whole directory.""" + _write(tmp_path, "sibling.yaml", "api:\n") + with caplog.at_level("DEBUG", logger="esphome.yaml_util"): + discovered = discover_user_yaml_files( + _write_entry_including(tmp_path, "${file}") + ) + assert (tmp_path / "sibling.yaml").resolve() not in discovered.files + assert any( + "Cannot resolve !include" in r.message and r.levelname == "DEBUG" + for r in caplog.records + ) + + +def test_discover_user_yaml_files_self_glob_match_skipped( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A glob whose only match is the including file itself claims nothing.""" + entry = _write_entry_including(tmp_path, "${platform}.yaml") + with caplog.at_level("DEBUG", logger="esphome.yaml_util"): + discovered = discover_user_yaml_files(entry) + assert [p.name for p in discovered.files] == ["entry.yaml"] + assert any("Cannot resolve !include" in r.message for r in caplog.records) + + +def test_discover_user_yaml_files_candidate_cycle_terminates(tmp_path: Path) -> None: + """Mutually glob-matching includes expand finitely and capture both files.""" + _write(tmp_path, "sub/a.yaml", "p: !include ${x}.yaml\n") + _write(tmp_path, "sub/b.yaml", "p: !include ${y}.yaml\n") + entry = _write(tmp_path, "entry.yaml", "wifi: !include sub/a.yaml\n") + discovered = discover_user_yaml_files(entry) + names = {p.name for p in discovered.files} + assert names == {"entry.yaml", "a.yaml", "b.yaml"} + + +def test_discover_user_yaml_files_many_candidates_keep_nested_includes( + tmp_path: Path, +) -> None: + """Every candidate's nested includes are discovered. + + Regression test: the id()-based cycle guard is only safe while every + traversed tree stays alive. Candidate trees used to be freed between + loop iterations, so CPython recycled their addresses and later + candidates' fresh trees were skipped as already seen, silently dropping + their nested includes. Needs several candidates to manifest; two were + not enough to trigger the reuse.""" + count = 12 + for i in range(count): + _write( + tmp_path, f"keys/k{i}.yaml", f"sensor{i}: !include ../nested/n{i}.yaml\n" + ) + _write(tmp_path, f"nested/n{i}.yaml", f"api{i}: true\n") + discovered = discover_user_yaml_files( + _write_entry_including(tmp_path, "keys/${x}.yaml") + ) + names = {p.name for p in discovered.files} + expected = {f"n{i}.yaml" for i in range(count)} + expected |= {f"k{i}.yaml" for i in range(count)} + expected.add("entry.yaml") + assert names == expected + + +def test_discover_user_yaml_files_bad_candidate_still_tracked( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A matched candidate that fails to parse warns even during discovery, + stays tracked (the load listener fires before parsing), and doesn't block + other candidates.""" + _write(tmp_path, "keys/good.yaml", "api:\n") + _write(tmp_path, "keys/bad.yaml", "esphome: [unterminated\n") + with caplog.at_level("DEBUG", logger="esphome.yaml_util"): + discovered = discover_user_yaml_files( + _write_entry_including(tmp_path, "keys/${name}.yaml") + ) + resolved = set(discovered.files) + assert (tmp_path / "keys/good.yaml").resolve() in resolved + assert (tmp_path / "keys/bad.yaml").resolve() in resolved + matching = [ + r.levelname for r in caplog.records if "Failed to load candidate" in r.message + ] + assert matching == ["WARNING"] + + +def test_discover_user_yaml_files_tolerates_templated_top_level_include( + tmp_path: Path, +) -> None: + """A literal include whose entire content is a templated ``!include`` is + tracked and skipped instead of aborting discovery.""" + _write(tmp_path, "wrapper.yaml", "!include ${x}_settings.yaml\n") + discovered = discover_user_yaml_files( + _write_entry_including(tmp_path, "wrapper.yaml") + ) + assert (tmp_path / "wrapper.yaml").resolve() in discovered.files + + def test_track_yaml_loads_records_resolved_paths(tmp_path: Path) -> None: """`track_yaml_loads` is the building block — sanity-check it resolves symlinks so callers can dedupe by identity.""" @@ -1491,3 +1780,122 @@ def test_merge_include_no_overlap_records_nothing(tmp_path: Path) -> None: assert result["api"] == {"reboot_timeout": "5min"} assert result["logger"] == {"level": "DEBUG"} assert yaml_util.take_dropped_merge_keys() == [] + + +# --------------------------------------------------------------------------- +# track_document_range=False (validated-config-cache fast path) +# --------------------------------------------------------------------------- + +FAST_MODE_MAIN_YAML = """\ +defaults: &defaults + port: 6053 + reboot_timeout: 15min + +esphome: + name: !secret devname + +api: + <<: *defaults + port: 6054 + +number_value: 42 +float_value: 3.5 +lambda_value: !lambda 'return x * 2;' +extend_value: !extend some_id +remove_value: !remove some_id +literal_value: !literal keep_me_verbatim +included: !include included.yaml +""" + + +@pytest.fixture +def fast_mode_config_dir(tmp_path: Path) -> Path: + _write(tmp_path, "main.yaml", FAST_MODE_MAIN_YAML) + _write(tmp_path, "included.yaml", "inner_key: inner_value\ninner_num: 7\n") + _write(tmp_path, "secrets.yaml", "devname: livingroom\n") + return tmp_path + + +def _resolve_includes(config: dict) -> dict: + return { + key: value.load() if isinstance(value, yaml_util.IncludeFile) else value + for key, value in config.items() + } + + +def test_load_yaml_fast_mode_matches_default(fast_mode_config_dir: Path) -> None: + """Both modes produce equal values; only the metadata wrapping differs.""" + yaml_file = fast_mode_config_dir / "main.yaml" + + normal = _resolve_includes(yaml_util.load_yaml(yaml_file)) + fast = _resolve_includes(yaml_util.load_yaml(yaml_file, track_document_range=False)) + + # Lambda has no __eq__; compare it by value and the rest structurally. + fast_lambda = fast.pop("lambda_value") + normal_lambda = normal.pop("lambda_value") + assert fast == normal + assert isinstance(fast_lambda, core.Lambda) + assert fast_lambda.value == normal_lambda.value == "return x * 2;" + assert fast["esphome"]["name"] == "livingroom" + assert fast["api"]["port"] == 6054 + assert fast["api"]["reboot_timeout"] == "15min" + assert fast["extend_value"] == Extend("some_id") + assert fast["remove_value"] == Remove("some_id") + # !literal wraps via make_literal, independent of range tracking. + assert isinstance(fast["literal_value"], ESPLiteralValue) + assert fast["literal_value"] == "keep_me_verbatim" + + # Fast mode returns plain values; default mode keeps the range metadata. + assert not isinstance(fast["number_value"], ESPHomeDataBase) + assert not isinstance(fast["float_value"], ESPHomeDataBase) + assert all(type(key) is str for key in fast) + assert isinstance(normal["number_value"], ESPHomeDataBase) + assert normal["number_value"].esp_range is not None + assert all(isinstance(key, ESPHomeDataBase) for key in normal) + + # Nested includes inherit fast mode through the recursive loader. + included = fast["included"] + assert not isinstance(included["inner_num"], ESPHomeDataBase) + assert all(type(key) is str for key in included) + + +def test_load_yaml_fast_mode_survives_pure_python_fallback( + fast_mode_config_dir: Path, +) -> None: + """The ESPHomePurePythonLoader retry must honour fast mode too.""" + yaml_file = fast_mode_config_dir / "main.yaml" + + class _AlwaysFailingLoader(yaml_util.ESPHomeLoader): + def __init__(self, *args, **kwargs) -> None: + raise EsphomeError("forced fallback to the pure-Python loader") + + with patch.object(yaml_util, "ESPHomeLoader", _AlwaysFailingLoader): + fast = yaml_util.load_yaml(yaml_file, track_document_range=False) + + assert not isinstance(fast["number_value"], ESPHomeDataBase) + assert all(type(key) is str for key in fast) + + +def test_load_yaml_fast_mode_rejects_custom_loader() -> None: + """A caller-supplied yaml_loader cannot combine with fast mode.""" + with pytest.raises(ValueError, match="default yaml_loader"): + yaml_util.parse_yaml( + Path("x.yaml"), + io.StringIO("a: 1"), + lambda f: {}, + track_document_range=False, + ) + + +def test_load_yaml_fast_mode_records_dropped_merge_keys( + fast_mode_config_dir: Path, +) -> None: + """The duplicate-merge-key bookkeeping must not crash on plain str keys. + + With plain keys there is no esp_range, so the recorded location falls + back to the parent file name. + """ + yaml_file = fast_mode_config_dir / "main.yaml" + + yaml_util.load_yaml(yaml_file, track_document_range=False) + assert yaml_util.take_dropped_merge_keys() == [("port", str(yaml_file))] diff --git a/tests/unit_tests/test_zephyr_library.py b/tests/unit_tests/test_zephyr_library.py index 0ba3577fa7..b370fe0c47 100644 --- a/tests/unit_tests/test_zephyr_library.py +++ b/tests/unit_tests/test_zephyr_library.py @@ -59,7 +59,10 @@ def test_generate_cmakelists_txt_flags_and_includes(tmp_path): assert "-DFOO" in out assert "-Wall" in out assert "zephyr_link_libraries(" in out - assert "-Llibdir" in out + # -L paths are absolutised against the library dir (the CMakeLists lives in a + # zephyr/ subdir, so a relative path would resolve from the wrong place). + abs_libdir = str((tmp_path / "libdir").resolve()).replace("\\", "\\\\") + assert f"-L{abs_libdir}" in out assert "-lm" in out