From 5d56517e147d700114fa16085ac682c957f101e9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 31 Aug 2026 13:19:49 -0500 Subject: [PATCH 01/25] [api] Keep action dropped warning strings in flash on ESP8266 (#18905) --- esphome/components/api/api_server.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 751f2e4c3b..43d35363d3 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -433,8 +433,10 @@ void APIServer::send_homeassistant_action(const HomeassistantActionRequest &call // Home Assistant subscribes to actions shortly *after* authenticating, so actions // fired right at connection time (on_client_connected, on_time_sync, ...) can // arrive before the subscription and are lost - warn instead of failing silently. - ESP_LOGW(TAG, "Home Assistant %s '%s' dropped; %s", call.is_event ? "event" : "action", call.service.c_str(), - this->is_connected() ? "client has not subscribed to actions (yet)" : "no client connected"); + ESP_LOGW(TAG, "Home Assistant %s '%s' dropped; %s", + call.is_event ? LOG_STR_LITERAL("event") : LOG_STR_LITERAL("action"), call.service.c_str(), + this->is_connected() ? LOG_STR_LITERAL("client has not subscribed to actions (yet)") + : LOG_STR_LITERAL("no client connected")); } } #ifdef USE_API_HOMEASSISTANT_ACTION_RESPONSES From 1ba1aebfa1943d3a3b57232ee9a7c01c2d742cf2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 31 Aug 2026 13:20:47 -0500 Subject: [PATCH 02/25] [ci] Balance integration test buckets by recorded durations (#18895) --- .github/workflows/ci.yml | 23 ++- .../workflows/sync-integration-durations.yml | 98 ++++++++++++ script/determine-jobs.py | 63 ++++---- script/helpers.py | 65 ++++++++ script/update_integration_test_durations.py | 119 +++++++++++++++ tests/integration/conftest.py | 9 ++ .../integration_test_durations.json | 142 ++++++++++++++++++ tests/script/test_determine_jobs.py | 130 +++++++++++++--- tests/script/test_helpers.py | 27 ++++ .../test_update_integration_test_durations.py | 130 ++++++++++++++++ 10 files changed, 755 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/sync-integration-durations.yml create mode 100755 script/update_integration_test_durations.py create mode 100644 tests/integration/integration_test_durations.json create mode 100644 tests/script/test_update_integration_test_durations.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0df4da6386..a874a023b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,6 +92,7 @@ jobs: outputs: core-ci: ${{ steps.determine.outputs.core-ci }} integration-tests: ${{ steps.determine.outputs.integration-tests }} + integration-run-all: ${{ steps.determine.outputs.integration-run-all }} integration-test-buckets: ${{ steps.determine.outputs.integration-test-buckets }} clang-tidy: ${{ steps.determine.outputs.clang-tidy }} clang-tidy-mode: ${{ steps.determine.outputs.clang-tidy-mode }} @@ -152,6 +153,9 @@ jobs: # Extract individual fields echo "core-ci=$(echo "$output" | jq -r '.core_ci')" >> $GITHUB_OUTPUT echo "integration-tests=$(echo "$output" | jq -r '.integration_tests')" >> $GITHUB_OUTPUT + # A missing key must fail here, not silently disable the junit upload + run_all=$(echo "$output" | jq -r 'if has("integration_run_all") then .integration_run_all else error("integration_run_all missing") end') + echo "integration-run-all=${run_all}" >> $GITHUB_OUTPUT echo "integration-test-buckets=$(echo "$output" | jq -c '.integration_test_buckets')" >> $GITHUB_OUTPUT echo "clang-tidy=$(echo "$output" | jq -r '.clang_tidy')" >> $GITHUB_OUTPUT echo "clang-tidy-mode=$(echo "$output" | jq -r '.clang_tidy_mode')" >> $GITHUB_OUTPUT @@ -427,8 +431,25 @@ jobs: run: | . venv/bin/activate mapfile -t test_files < <(echo "$BUCKET_TESTS" | jq -r '.[]') + if [ "${#test_files[@]}" -eq 0 ]; then + echo "::error::Empty integration test bucket; pytest would collect the whole tree" + exit 1 + fi echo "Bucket ${{ matrix.bucket.name }}: running ${#test_files[@]} integration tests" - pytest -vv --no-cov --tb=native --durations=30 -n auto "${test_files[@]}" + pytest -vv --no-cov --tb=native --durations=30 -n auto --dist worksteal \ + --junitxml=junit-integration.xml "${test_files[@]}" + - name: Upload junit timings + # Consumed by sync-integration-durations.yml through + # script/update_integration_test_durations.py; only full matrix dev + # runs produce usable data. + if: github.ref == 'refs/heads/dev' && needs.determine-jobs.outputs.integration-run-all == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: junit-integration-${{ strategy.job-index }} + path: junit-integration.xml + if-no-files-found: error + # A full cron period of margin for the weekly refresh + retention-days: 14 - name: Print ccache statistics # esphome stores the PlatformIO ccache under the machine-global cache # dir (see _ccache_env() in esphome/platformio/toolchain.py). diff --git a/.github/workflows/sync-integration-durations.yml b/.github/workflows/sync-integration-durations.yml new file mode 100644 index 0000000000..d09a1cf242 --- /dev/null +++ b/.github/workflows/sync-integration-durations.yml @@ -0,0 +1,98 @@ +--- +name: Refresh integration test durations + +on: + workflow_dispatch: + schedule: + - cron: "45 5 * * 1" + +# Repo writes (branch push, PR open) happen via the App token minted below, +# so the workflow's GITHUB_TOKEN does not need any write scopes. +permissions: + contents: read + actions: read # gh api / gh run download for the CI junit artifacts + +jobs: + sync: + name: Refresh integration test durations + runs-on: ubuntu-latest + if: github.repository == 'esphome/esphome' + steps: + - name: Generate a token + id: generate-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.ESPHOME_GITHUB_APP_CLIENT_ID }} + private-key: ${{ secrets.ESPHOME_GITHUB_APP_PRIVATE_KEY }} + permission-contents: write # push the sync branch + permission-pull-requests: write # open or refresh the sync PR + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + + - name: Refresh from the newest usable dev run + env: + GH_TOKEN: ${{ github.token }} + run: | + # Only full matrix dev runs upload junit-integration-* artifacts + # (see the integration-tests job); the merge script re-checks + # coverage regardless. + # Newest-first candidates via their bucket-0 artifact. Fork PRs run + # their own ci.yml, so name and branch are spoofable; require + # same-repo. Assignment failures trip set -e and fail loudly. + candidates=$( + gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=junit-integration-0&per_page=100" \ + --jq '.artifacts[] | select(.expired | not) | .workflow_run + | select(.head_branch == "dev" and .head_repository_id != null + and .head_repository_id == .repository_id) + | .id' + ) + # Green runs first, then the rest newest first; a run missing a + # bucket fails the coverage check and the next one is tried + green="" + rest="" + for id in ${candidates}; do + conclusion=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${id}" --jq '.conclusion // ""') + if [ "${conclusion}" = "success" ]; then + green="${green} ${id}" + elif [ -n "${conclusion}" ]; then + rest="${rest} ${id}" + fi + done + # helpers.py imports colorama; the script needs nothing else + pip install colorama + for id in ${green} ${rest}; do + rm -rf /tmp/junit + if ! gh run download "${id}" --repo "${GITHUB_REPOSITORY}" -p "junit-integration-*" -D /tmp/junit; then + echo "::warning::Could not download artifacts for run ${id}; trying the next" + continue + fi + status=0 + python script/update_integration_test_durations.py /tmp/junit || status=$? + if [ "${status}" -eq 0 ]; then + echo "Refreshed from run ${id}" + exit 0 + fi + # Only EXIT_LOW_COVERAGE (3) from the script advances to the next run + [ "${status}" -eq 3 ] || exit 1 + echo "::warning::Run ${id} covers too few test files; trying the next" + done + echo "::error::No dev CI run with usable junit artifacts in range; the feed is starved" + exit 1 + + - name: Commit changes + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + commit-message: "[ci] Refresh integration test durations" + committer: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> + author: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> + branch: sync/integration-durations + delete-branch: true + title: "[ci] Refresh integration test durations" + body-path: .github/PULL_REQUEST_TEMPLATE.md + token: ${{ steps.generate-token.outputs.token }} diff --git a/script/determine-jobs.py b/script/determine-jobs.py index add1af5bba..f5412af21d 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -53,8 +53,10 @@ from collections import Counter from enum import StrEnum from functools import cache import json +import math import os from pathlib import Path +import statistics import sys from typing import Any @@ -67,7 +69,9 @@ from clang_tidy_hash import ( from helpers import ( CPP_FILE_EXTENSIONS, ESPHOME_TESTS_COMPONENTS_PATH, + INTEGRATION_TESTS_PATH, PYTHON_FILE_EXTENSIONS, + all_integration_test_files, base_python_changed, changed_files, core_changed, @@ -83,6 +87,8 @@ from helpers import ( get_target_branch, git_ls_files, is_validate_only_file, + load_integration_durations, + lpt_partition, root_path, ) from split_components_for_ci import create_intelligent_batches @@ -96,10 +102,13 @@ CLANG_TIDY_SPLIT_THRESHOLD = 65 # Isolated components count as 10x, groupable components count as 1x COMPONENT_TEST_BATCH_SIZE = 40 -# Integration test bucketing: when more than the threshold tests are scheduled, -# fan out across this many parallel jobs. Below the threshold, a single job runs. +# Above the threshold, fan out across up to this many jobs, balanced by the +# recorded per-file durations. The target is serial junit-time weight per +# bucket, not wall time (calibrated with the conftest compile cap); it +# sizes the bucket count for small subsets. INTEGRATION_TESTS_SPLIT_THRESHOLD = 10 -INTEGRATION_TESTS_SPLIT_BUCKETS = 3 +INTEGRATION_TESTS_SPLIT_BUCKETS = 5 +INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT = 360.0 # platformio and aioesphomeapi (requirements.txt), the pytest stack # (requirements_test.txt) and the fixture every session compiles; a change @@ -113,27 +122,13 @@ INTEGRATION_TESTS_TRIGGER_FILES = frozenset( ) -def _split_list(items: list[str], n: int) -> list[list[str]]: - """Split a list into n roughly-equal contiguous parts (matches script/clang-tidy).""" - k, m = divmod(len(items), n) - return [items[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(n)] - - -def _all_integration_test_files() -> list[str]: - """Return all integration test file paths, sorted, relative to repo root.""" - return sorted( - str(p.relative_to(root_path)) - for p in (Path(root_path) / "tests" / "integration").glob("test_*.py") - ) - - def _compute_integration_test_buckets( integration_run_all: bool, integration_test_files: list[str], ) -> tuple[bool, list[dict[str, Any]]]: """Compute (run_integration, buckets) from the determine_integration_tests result. - Pure function for unit testing — no I/O beyond `_all_integration_test_files` + Pure function for unit testing — no I/O beyond `all_integration_test_files` when `integration_run_all` is set. `buckets` is a list of `{name, tests}` dicts where `tests` is a JSON-friendly @@ -141,7 +136,7 @@ def _compute_integration_test_buckets( shell word-splitting / glob hazards. """ if integration_run_all: - files = _all_integration_test_files() + files = all_integration_test_files() else: files = sorted(integration_test_files) @@ -152,12 +147,23 @@ def _compute_integration_test_buckets( return False, [] if len(files) > INTEGRATION_TESTS_SPLIT_THRESHOLD: - parts = [ - part for part in _split_list(files, INTEGRATION_TESTS_SPLIT_BUCKETS) if part - ] + durations = load_integration_durations() + # Unrecorded files weigh the recording's median; with no recording a + # file weighs a whole bucket, which keeps the full fan-out + default = ( + statistics.median(durations.values()) + if durations + else INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT + ) + weights = {f: durations.get(f, default) for f in files} + count = min( + INTEGRATION_TESTS_SPLIT_BUCKETS, + math.ceil(sum(weights.values()) / INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT), + ) + # count <= SPLIT_BUCKETS < threshold < len(files): no group is empty + parts = [sorted(part) for part in lpt_partition(files, weights, count)] buckets = [ - {"name": f"{i + 1}/{len(parts)}", "tests": part} - for i, part in enumerate(parts) + {"name": f"{i + 1}/{count}", "tests": part} for i, part in enumerate(parts) ] else: buckets = [{"name": "1/1", "tests": files}] @@ -264,9 +270,9 @@ def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[s # If infrastructure Python files changed (conftest, utils, etc.), run all tests # Excludes test files (test_*.py), fixtures, and non-Python files (README.md) if any( - f.startswith("tests/integration/") + f.startswith(INTEGRATION_TESTS_PATH) and f.endswith(".py") - and not f.startswith("tests/integration/test_") + and not f.startswith(f"{INTEGRATION_TESTS_PATH}test_") and "/fixtures/" not in f for f in files ): @@ -277,9 +283,9 @@ def determine_integration_tests(branch: str | None = None) -> tuple[bool, list[s fixture_to_test_files = get_fixture_to_test_files() for f in files: - if f.startswith("tests/integration/test_") and f.endswith(".py"): + if f.startswith(f"{INTEGRATION_TESTS_PATH}test_") and f.endswith(".py"): test_files.add(f) - elif f.startswith("tests/integration/fixtures/"): + elif f.startswith(f"{INTEGRATION_TESTS_PATH}fixtures/"): if f.endswith(".yaml"): # Fixture YAML changed - add corresponding test file(s) test_files.update(fixture_to_test_files.get(Path(f).stem, ())) @@ -1415,6 +1421,7 @@ def main() -> None: output: dict[str, Any] = { "core_ci": run_core_ci, "integration_tests": run_integration, + "integration_run_all": integration_run_all, "integration_test_buckets": integration_test_buckets, "clang_tidy": run_clang_tidy, "clang_tidy_mode": clang_tidy_mode, diff --git a/script/helpers.py b/script/helpers.py index e648bb91bb..bf22e15808 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -43,6 +43,53 @@ ESPHOME_TESTS_COMPONENTS_PATH = "tests/components/" # Tuple of component and test paths for efficient startswith checks COMPONENT_AND_TESTS_PATHS = (ESPHOME_COMPONENTS_PATH, ESPHOME_TESTS_COMPONENTS_PATH) +# Integration tests path prefix +INTEGRATION_TESTS_PATH = "tests/integration/" + +# Per-file integration test durations from CI junit output; shared by the +# reader (determine-jobs) and writer (update_integration_test_durations) +INTEGRATION_TEST_DURATIONS_FILE = "tests/integration/integration_test_durations.json" + + +def all_integration_test_files() -> list[str]: + """Return all integration test file paths, sorted, relative to repo root.""" + return sorted( + p.relative_to(root_path).as_posix() + for p in (Path(root_path) / "tests" / "integration").glob("test_*.py") + ) + + +def load_integration_durations() -> dict[str, float]: + """Return recorded per-file pytest durations in seconds; empty when unavailable.""" + try: + raw = json.loads( + (Path(root_path) / INTEGRATION_TEST_DURATIONS_FILE).read_text() + ) + if not isinstance(raw, dict): + print( + f"integration durations unavailable: expected an object, " + f"got {type(raw).__name__}", + file=sys.stderr, + ) + return {} + except (OSError, ValueError) as err: + # The file ships in the repo; degrade to unweighted bucketing, loudly + print(f"integration durations unavailable: {err}", file=sys.stderr) + return {} + durations = { + key: seconds + for key, value in raw.items() + if isinstance(value, (int, float)) and (seconds := float(value)) > 0 + } + if len(durations) != len(raw): + # One bad entry must not discard the whole recording + print( + f"dropped {len(raw) - len(durations)} invalid duration entries", + file=sys.stderr, + ) + return durations + + # Base bus components - these ARE the bus implementations and should not # be flagged as needing migration since they are the platform/base components BASE_BUS_COMPONENTS = { @@ -1545,3 +1592,21 @@ def get_cpp_changed_components(files: list[str]) -> list[str]: if file.startswith(ESPHOME_COMPONENTS_PATH): affected.update(find_children_of_component(components_graph, component)) return sorted(c for c in affected if has_cpp_unit_tests(c, tests_dir)) + + +def lpt_partition( + items: list[str], weights: dict[str, float], count: int +) -> list[list[str]]: + """Partition items into `count` weight-balanced groups (LPT greedy). + + Heaviest item first into the lightest group. Ties keep input order, so + pass pre-sorted items for deterministic output. script/clang-tidy's + split_list is the unweighted contiguous sibling. + """ + groups: list[list[str]] = [[] for _ in range(count)] + group_weights = [0.0] * count + for item in sorted(items, key=lambda i: -weights[i]): + lightest = min(range(count), key=group_weights.__getitem__) + groups[lightest].append(item) + group_weights[lightest] += weights[item] + return groups diff --git a/script/update_integration_test_durations.py b/script/update_integration_test_durations.py new file mode 100755 index 0000000000..bbb959c0b2 --- /dev/null +++ b/script/update_integration_test_durations.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Merge CI junit output into tests/integration/integration_test_durations.json. + +The integration-tests CI job uploads one junit XML artifact per bucket on +full matrix dev runs. Download a run's artifacts and merge the per file +durations into the recording used by script/determine-jobs.py: + + gh run download --repo esphome/esphome -p "junit-integration-*" -D /tmp/junit + script/update_integration_test_durations.py /tmp/junit + +Missing files keep their previous recording and deleted files drop out; a +run covering under 90% of the test files aborts unless --allow-partial. +""" + +from __future__ import annotations + +import argparse +from collections import defaultdict +import json +from pathlib import Path +import sys +import xml.etree.ElementTree as ET + +from helpers import ( + INTEGRATION_TEST_DURATIONS_FILE, + INTEGRATION_TESTS_PATH, + all_integration_test_files, + load_integration_durations, + root_path, +) + +DURATIONS_FILE = Path(root_path) / INTEGRATION_TEST_DURATIONS_FILE +MIN_COVERAGE = 0.9 +# Exit code for the expected "run covers too few files" refusal, so the +# refresh workflow can move on to the next candidate run +EXIT_LOW_COVERAGE = 3 + + +def collect_durations(junit_dir: Path, known_files: set[str]) -> dict[str, float]: + """Sum junit testcase times per integration test file, in seconds.""" + durations: defaultdict[str, float] = defaultdict(float) + unmatched = 0 + xml_files = sorted(junit_dir.rglob("*.xml")) + if not xml_files: + raise SystemExit(f"no junit XML files found under {junit_dir}") + for xml_file in xml_files: + for testcase in ET.parse(xml_file).getroot().iter("testcase"): + # Skipped/errored testcases carry time="0"; recording them would + # overwrite a good previous duration + if any( + testcase.find(tag) is not None + for tag in ("skipped", "error", "failure") + ): + continue + # classname is the dotted module plus any test class, e.g. + # tests.integration.test_x or tests.integration.test_x.TestFoo + parts = testcase.get("classname", "").split(".") + if parts[:2] != ["tests", "integration"] or len(parts) < 3: + unmatched += 1 + continue + path = f"{INTEGRATION_TESTS_PATH}{parts[2]}.py" + if path not in known_files: + print(f"skipping unknown test module {path}", file=sys.stderr) + continue + durations[path] += float(testcase.get("time", "0")) + if unmatched: + # A junit naming change would otherwise shrink the recording silently + raise SystemExit( + f"{unmatched} testcases with unexpected classnames; the junit layout changed" + ) + # An all-skipped file totals 0.0; let the merge keep its previous entry + return {k: v for k, v in durations.items() if v > 0} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "junit_dir", type=Path, help="directory containing downloaded junit XML files" + ) + parser.add_argument( + "--allow-partial", + action="store_true", + help="merge a run covering under 90%% of the test files", + ) + args = parser.parse_args() + + on_disk = set(all_integration_test_files()) + if not on_disk: + raise SystemExit("no integration test files found; wrong checkout root?") + collected = collect_durations(args.junit_dir, on_disk) + coverage = len(collected.keys() & on_disk) / len(on_disk) + if coverage < MIN_COVERAGE and not args.allow_partial: + print( + f"artifacts cover only {coverage:.0%} of {len(on_disk)} test files; " + "use a full matrix run or pass --allow-partial to merge anyway", + file=sys.stderr, + ) + return EXIT_LOW_COVERAGE + + # Validated load: a bad previous entry cannot survive the round trip, and + # an unreadable file aborts rather than being overwritten + previous = load_integration_durations() + if DURATIONS_FILE.is_file() and not previous: + raise SystemExit(f"{DURATIONS_FILE} is unreadable; refusing to overwrite it") + # New recordings win, absent files keep theirs, deleted files drop out + merged = { + path: collected.get(path, previous.get(path)) + for path in sorted(on_disk) + if path in collected or path in previous + } + DURATIONS_FILE.write_text( + json.dumps({k: round(v, 2) for k, v in merged.items()}, indent=2) + "\n" + ) + print(f"wrote {len(merged)} entries to {DURATIONS_FILE} ({coverage:.0%} fresh)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 12b1407fe1..6777e6cabc 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -23,6 +23,7 @@ import pytest_asyncio import esphome.config from esphome.core import CORE +from esphome.helpers import get_usable_cpu_count from esphome.platformio.toolchain import get_idedata from .const import ( @@ -67,6 +68,14 @@ def _get_platformio_env(cache_dir: Path) -> dict[str, str]: env["PLATFORMIO_LIBDEPS_DIR"] = str(cache_dir / "libdeps" / worker) # Prevent cache cleaning during integration tests env["ESPHOME_SKIP_CLEAN_BUILD"] = "1" + # Cap each compile's -j so several xdist workers do not each spawn a + # full-width compiler fan-out on the same machine. An explicit env wins. + if "ESPHOME_DEFAULT_COMPILE_PROCESS_LIMIT" not in os.environ: + workers = int(os.environ.get("PYTEST_XDIST_WORKER_COUNT", "1")) + # Floor of 2 keeps a lone tail compile from running fully serial + env["ESPHOME_DEFAULT_COMPILE_PROCESS_LIMIT"] = str( + max(2, get_usable_cpu_count() // workers) + ) # Compile with THIS tree's esphome sources, not wherever the venv's editable # install points (which may be a different git worktree or checkout). repo_root = str(Path(__file__).resolve().parent.parent.parent) diff --git a/tests/integration/integration_test_durations.json b/tests/integration/integration_test_durations.json new file mode 100644 index 0000000000..9bada5cd36 --- /dev/null +++ b/tests/integration/integration_test_durations.json @@ -0,0 +1,142 @@ +{ + "tests/integration/test_action_concurrent_reentry.py": 45.23, + "tests/integration/test_addressable_light_transition.py": 74.47, + "tests/integration/test_alarm_control_panel_state_transitions.py": 74.1, + "tests/integration/test_api_action_metadata.py": 62.1, + "tests/integration/test_api_action_responses.py": 71.08, + "tests/integration/test_api_action_timeout.py": 21.64, + "tests/integration/test_api_conditional_memory.py": 13.72, + "tests/integration/test_api_custom_services.py": 24.16, + "tests/integration/test_api_get_time_response_timezone.py": 23.48, + "tests/integration/test_api_homeassistant.py": 37.87, + "tests/integration/test_api_homeassistant_action_no_subscriber.py": 14.38, + "tests/integration/test_api_list_entities_backpressure.py": 26.85, + "tests/integration/test_api_message_size_batching.py": 33.36, + "tests/integration/test_api_reboot_timeout.py": 13.63, + "tests/integration/test_api_string_lambda.py": 25.04, + "tests/integration/test_api_vv_logging.py": 16.6, + "tests/integration/test_api_zero_psk_provisioning.py": 43.14, + "tests/integration/test_areas_and_devices.py": 25.98, + "tests/integration/test_automation_wait_actions.py": 21.91, + "tests/integration/test_automations.py": 42.43, + "tests/integration/test_batch_delay_zero_rapid_transitions.py": 16.65, + "tests/integration/test_binary_sensor_autorepeat_filter.py": 28.67, + "tests/integration/test_binary_sensor_invalidate_state.py": 23.69, + "tests/integration/test_blocking_warning_log_time_not_charged_to_next_operation.py": 22.99, + "tests/integration/test_build_info.py": 24.96, + "tests/integration/test_camera_mock.py": 14.47, + "tests/integration/test_climate_control_action.py": 31.07, + "tests/integration/test_climate_custom_modes.py": 28.59, + "tests/integration/test_continuation_actions.py": 14.96, + "tests/integration/test_cover_control_action.py": 26.14, + "tests/integration/test_crc8_helper.py": 10.92, + "tests/integration/test_device_id_in_state.py": 64.97, + "tests/integration/test_duplicate_entities.py": 30.81, + "tests/integration/test_entity_icon.py": 32.85, + "tests/integration/test_fan_turn_on_action.py": 24.91, + "tests/integration/test_fnv1_hash_object_id.py": 12.54, + "tests/integration/test_fnv1a_hash.py": 21.8, + "tests/integration/test_gpio_expander_cache.py": 5.2, + "tests/integration/test_host_logger_thread_safety.py": 21.7, + "tests/integration/test_host_mode_basic.py": 13.62, + "tests/integration/test_host_mode_batch_delay.py": 14.56, + "tests/integration/test_host_mode_climate_basic_state.py": 30.95, + "tests/integration/test_host_mode_climate_control.py": 29.06, + "tests/integration/test_host_mode_empty_string_options.py": 27.22, + "tests/integration/test_host_mode_entity_fields.py": 30.95, + "tests/integration/test_host_mode_fan_preset.py": 14.44, + "tests/integration/test_host_mode_many_entities.py": 54.13, + "tests/integration/test_host_mode_many_entities_multiple_connections.py": 32.17, + "tests/integration/test_host_mode_noise_encryption.py": 42.77, + "tests/integration/test_host_mode_reconnect.py": 4.06, + "tests/integration/test_host_mode_sensor.py": 13.47, + "tests/integration/test_host_ota.py": 21.4, + "tests/integration/test_host_preferences.py": 25.43, + "tests/integration/test_host_preferences_suspend_resume.py": 19.2, + "tests/integration/test_improv_serial_uart.py": 31.52, + "tests/integration/test_large_message_batching.py": 15.64, + "tests/integration/test_legacy_area.py": 22.63, + "tests/integration/test_legacy_climate_compat.py": 26.13, + "tests/integration/test_legacy_fan_compat.py": 24.05, + "tests/integration/test_light_automations.py": 30.86, + "tests/integration/test_light_binary_effect_off_phase.py": 23.19, + "tests/integration/test_light_calls.py": 32.35, + "tests/integration/test_light_constant_brightness.py": 29.89, + "tests/integration/test_light_control_action.py": 29.06, + "tests/integration/test_light_dim_relative_action.py": 29.61, + "tests/integration/test_light_effect_zero_brightness.py": 18.68, + "tests/integration/test_light_initial_state.py": 24.49, + "tests/integration/test_light_toggle_action.py": 26.46, + "tests/integration/test_lock_automations.py": 23.28, + "tests/integration/test_logger_buffered_recursion_guard.py": 24.29, + "tests/integration/test_loop_disable_enable.py": 45.28, + "tests/integration/test_loop_interval_decoupling.py": 28.35, + "tests/integration/test_loop_interval_default_not_pulled_forward.py": 21.97, + "tests/integration/test_micros_to_millis.py": 20.79, + "tests/integration/test_multi_click_trigger.py": 26.2, + "tests/integration/test_multi_device_preferences.py": 16.87, + "tests/integration/test_noise_encryption_key_protection.py": 77.05, + "tests/integration/test_object_id_api_verification.py": 73.51, + "tests/integration/test_object_id_friendly_name_no_mac_suffix.py": 62.33, + "tests/integration/test_object_id_no_friendly_name.py": 43.47, + "tests/integration/test_online_image_auto_detects_image_bmp_mime.py": 32.21, + "tests/integration/test_online_image_auto_detects_redirected_image_bmp_mime.py": 56.86, + "tests/integration/test_online_image_bmp.py": 50.9, + "tests/integration/test_oversized_payloads.py": 53.2, + "tests/integration/test_preference_key_stability.py": 26.09, + "tests/integration/test_runtime_stats.py": 18.34, + "tests/integration/test_safe_mode_loop_runs.py": 10.07, + "tests/integration/test_scheduler_blocking_warning.py": 40.91, + "tests/integration/test_scheduler_bulk_cleanup.py": 23.14, + "tests/integration/test_scheduler_defer_cancel.py": 24.54, + "tests/integration/test_scheduler_defer_cancel_regular.py": 13.48, + "tests/integration/test_scheduler_defer_fifo_simple.py": 26.86, + "tests/integration/test_scheduler_defer_stress.py": 27.23, + "tests/integration/test_scheduler_heap_stress.py": 24.02, + "tests/integration/test_scheduler_internal_id_no_collision.py": 24.57, + "tests/integration/test_scheduler_interval_reschedule.py": 13.12, + "tests/integration/test_scheduler_interval_zero_coerced.py": 22.91, + "tests/integration/test_scheduler_null_name.py": 23.46, + "tests/integration/test_scheduler_numeric_id_test.py": 24.54, + "tests/integration/test_scheduler_pool.py": 25.0, + "tests/integration/test_scheduler_rapid_cancellation.py": 14.68, + "tests/integration/test_scheduler_recursive_timeout.py": 25.35, + "tests/integration/test_scheduler_removed_item_race.py": 26.19, + "tests/integration/test_scheduler_self_keyed.py": 23.43, + "tests/integration/test_scheduler_simultaneous_callbacks.py": 22.16, + "tests/integration/test_scheduler_string_test.py": 15.22, + "tests/integration/test_script_array_params.py": 14.67, + "tests/integration/test_script_delay_params.py": 15.65, + "tests/integration/test_script_queued.py": 24.93, + "tests/integration/test_script_queued_idle_loop.py": 5.04, + "tests/integration/test_script_wait_on_boot.py": 13.08, + "tests/integration/test_select_stringref_trigger.py": 29.6, + "tests/integration/test_sensor_filters_delta.py": 28.01, + "tests/integration/test_sensor_filters_ring_buffer.py": 25.04, + "tests/integration/test_sensor_filters_sliding_window.py": 71.5, + "tests/integration/test_sensor_filters_value_list.py": 16.94, + "tests/integration/test_sensor_timeout_filter.py": 29.48, + "tests/integration/test_socket_wake_gate_tcp.py": 20.36, + "tests/integration/test_status_flags.py": 37.42, + "tests/integration/test_strftime_to.py": 22.61, + "tests/integration/test_syslog.py": 16.34, + "tests/integration/test_template_alarm_control_panel_many_sensors.py": 29.81, + "tests/integration/test_template_text_save.py": 25.43, + "tests/integration/test_text_command.py": 23.34, + "tests/integration/test_text_sensor_raw_state.py": 69.57, + "tests/integration/test_uart_mock_ld2410.py": 37.95, + "tests/integration/test_uart_mock_ld2412.py": 93.22, + "tests/integration/test_uart_mock_ld2420.py": 43.24, + "tests/integration/test_uart_mock_ld2450.py": 31.75, + "tests/integration/test_uart_mock_modbus.py": 667.4, + "tests/integration/test_udp.py": 9.38, + "tests/integration/test_use_address_runtime.py": 37.05, + "tests/integration/test_valve_control_action.py": 24.47, + "tests/integration/test_varint_five_byte_device_id.py": 25.03, + "tests/integration/test_wait_until_mid_loop_timing.py": 23.73, + "tests/integration/test_wait_until_on_boot.py": 9.16, + "tests/integration/test_wait_until_ordering.py": 13.3, + "tests/integration/test_wait_until_reentrant_restart.py": 25.23, + "tests/integration/test_wake_loop_forces_phase_b.py": 23.34, + "tests/integration/test_water_heater_template.py": 17.67 +} diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 7b641e275e..4971821969 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -151,9 +151,14 @@ def test_main_all_tests_should_run( patch.object(determine_jobs, "_is_clang_tidy_full_scan", return_value=False), patch.object( determine_jobs, - "_all_integration_test_files", + "all_integration_test_files", return_value=fake_test_files, ), + patch.object( + determine_jobs, + "load_integration_durations", + return_value=dict.fromkeys(fake_test_files, 200.0), + ), patch.object( determine_jobs, "get_changed_components", @@ -189,24 +194,12 @@ def test_main_all_tests_should_run( output = json.loads(captured.out) assert output["integration_tests"] is True - # run_all=True expands to the full glob and pre-buckets into 3 parts. - # Each bucket's `tests` is a JSON list of file paths. + assert output["integration_run_all"] is True + # run_all=True expands to the full glob; balance and naming are pinned + # by the unit tests, main() only needs to round-trip the structure assert isinstance(output["integration_test_buckets"], list) - assert len(output["integration_test_buckets"]) == 3 - assert [b["name"] for b in output["integration_test_buckets"]] == [ - "1/3", - "2/3", - "3/3", - ] - for bucket in output["integration_test_buckets"]: - assert isinstance(bucket["tests"], list) - for path in bucket["tests"]: - assert isinstance(path, str) bucket_files = [f for b in output["integration_test_buckets"] for f in b["tests"]] - assert bucket_files == fake_test_files - # Bucket sizes are balanced (max-min difference at most 1). - sizes = [len(b["tests"]) for b in output["integration_test_buckets"]] - assert max(sizes) - min(sizes) <= 1 + assert sorted(bucket_files) == fake_test_files assert output["clang_tidy"] is True assert output["clang_tidy_mode"] in ["nosplit", "split"] assert output["clang_format"] is True @@ -509,14 +502,24 @@ def test_compute_integration_test_buckets_at_threshold_stays_single() -> None: def test_compute_integration_test_buckets_just_over_threshold_splits() -> None: - """One file over the threshold triggers the 3-bucket fan-out, balanced.""" + """One file over the threshold fans out fully when the weights demand it.""" n = determine_jobs.INTEGRATION_TESTS_SPLIT_THRESHOLD + 1 files = [f"tests/integration/test_{i:02d}.py" for i in range(n)] - run, buckets = determine_jobs._compute_integration_test_buckets(False, files) + with patch.object( + determine_jobs, + "load_integration_durations", + return_value=dict.fromkeys(files, 200.0), + ): + run, buckets = determine_jobs._compute_integration_test_buckets(False, files) assert run is True - assert [b["name"] for b in buckets] == ["1/3", "2/3", "3/3"] - union = [path for b in buckets for path in b["tests"]] + # threshold+1 files x 200s caps at the maximum bucket count. + n_buckets = determine_jobs.INTEGRATION_TESTS_SPLIT_BUCKETS + assert [b["name"] for b in buckets] == [ + f"{i + 1}/{n_buckets}" for i in range(n_buckets) + ] + union = sorted(path for b in buckets for path in b["tests"]) assert union == sorted(files) + # Equal weights => bucket sizes are balanced (difference at most 1). sizes = [len(b["tests"]) for b in buckets] assert max(sizes) - min(sizes) <= 1 @@ -526,7 +529,7 @@ def test_compute_integration_test_buckets_run_all_with_empty_glob_disables_run() ): """run_all=True but glob returns no files => run suppressed (otherwise pytest would collect tests outside tests/integration/).""" - with patch.object(determine_jobs, "_all_integration_test_files", return_value=[]): + with patch.object(determine_jobs, "all_integration_test_files", return_value=[]): run, buckets = determine_jobs._compute_integration_test_buckets(True, []) assert run is False assert buckets == [] @@ -3146,3 +3149,86 @@ def test_memory_impact_elf_layouts_are_found(tmp_path: Path) -> None: elf.write_text("") assert find_elf_path(build_path) == elf, f"{platform} ELF not found" + + +def test_compute_integration_test_buckets_no_durations_full_fanout() -> None: + """Without recorded durations the fan-out stays at the maximum.""" + files = [f"tests/integration/test_{i:03d}.py" for i in range(15)] + with patch.object(determine_jobs, "load_integration_durations", return_value={}): + run, buckets = determine_jobs._compute_integration_test_buckets(False, files) + assert run is True + assert len(buckets) == determine_jobs.INTEGRATION_TESTS_SPLIT_BUCKETS + assert sorted(f for b in buckets for f in b["tests"]) == files + + +def test_compute_integration_test_buckets_adaptive_count() -> None: + """A small recorded total weight collapses to one bucket above the threshold.""" + files = [f"tests/integration/test_{i:03d}.py" for i in range(15)] + with patch.object( + determine_jobs, + "load_integration_durations", + return_value=dict.fromkeys(files, 10.0), + ): + run, buckets = determine_jobs._compute_integration_test_buckets(False, files) + assert run is True + # 15 files x 10s recorded = 150s, under the per-bucket weight target. + assert [b["name"] for b in buckets] == ["1/1"] + assert buckets[0]["tests"] == files + + +def test_compute_integration_test_buckets_duration_weighted() -> None: + """Heavy files spread across buckets instead of clustering by sorted name.""" + files = [f"tests/integration/test_{i:03d}.py" for i in range(12)] + durations = dict.fromkeys(files, 10.0) + durations[files[0]] = 600.0 + durations[files[1]] = 600.0 + with patch.object( + determine_jobs, "load_integration_durations", return_value=durations + ): + run, buckets = determine_jobs._compute_integration_test_buckets(False, files) + assert run is True + assert len(buckets) >= 2 + heavy_buckets = [b for b in buckets if set(files[:2]) & set(b["tests"])] + assert len(heavy_buckets) == 2, "heavy files should land in different buckets" + assert sorted(f for b in buckets for f in b["tests"]) == files + + +def test_load_integration_durations_missing_or_corrupt(tmp_path: Path) -> None: + """Missing or unparsable durations data degrades to an empty mapping.""" + with patch.object(helpers, "root_path", str(tmp_path)): + assert determine_jobs.load_integration_durations() == {} + durations_file = tmp_path / helpers.INTEGRATION_TEST_DURATIONS_FILE + durations_file.parent.mkdir(parents=True) + durations_file.write_text("not json") + assert determine_jobs.load_integration_durations() == {} + durations_file.write_text('{"tests/integration/test_a.py": 12.5}') + assert determine_jobs.load_integration_durations() == { + "tests/integration/test_a.py": 12.5 + } + # Non-positive entries are dropped, valid ones survive + durations_file.write_text( + '{"tests/integration/test_a.py": 12.5, "tests/integration/test_b.py": -1}' + ) + assert determine_jobs.load_integration_durations() == { + "tests/integration/test_a.py": 12.5 + } + # One non-numeric entry cannot discard the whole recording + durations_file.write_text( + '{"tests/integration/test_a.py": 12.5, "tests/integration/test_b.py": null}' + ) + assert determine_jobs.load_integration_durations() == { + "tests/integration/test_a.py": 12.5 + } + # A non-dict top level degrades to empty + durations_file.write_text("[12.5]") + assert determine_jobs.load_integration_durations() == {} + + +def test_committed_integration_durations_are_sane() -> None: + """The committed recording itself holds positive bounded floats.""" + raw = json.loads( + (Path(helpers.root_path) / helpers.INTEGRATION_TEST_DURATIONS_FILE).read_text() + ) + assert raw, "committed durations file missing or empty" + assert all(isinstance(v, (int, float)) and 0 < v < 86400 for v in raw.values()) + assert all(k.startswith("tests/integration/test_") for k in raw) diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 38b8c57368..7d4059da2f 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -2120,3 +2120,30 @@ def test_get_cpp_changed_components_independent_of_cwd( assert helpers.get_cpp_changed_components( ["tests/components/time/__init__.py"] ) == ["time"] + + +def test_lpt_partition_balances_skewed_weights() -> None: + """Heavy items spread across groups instead of clustering.""" + items = [f"i{n}" for n in range(6)] + weights = {"i0": 100.0, "i1": 90.0, "i2": 10.0, "i3": 10.0, "i4": 5.0, "i5": 5.0} + groups = helpers.lpt_partition(items, weights, 2) + group_weights = sorted(sum(weights[i] for i in g) for g in groups) + # Contiguous split would give 200 vs 20; LPT lands at 110 vs 110 + assert group_weights == [110.0, 110.0] + assert sorted(i for g in groups for i in g) == items + + +def test_lpt_partition_more_groups_than_items() -> None: + """Surplus groups come back empty; every item still lands somewhere.""" + items = ["a", "b"] + groups = helpers.lpt_partition(items, {"a": 1.0, "b": 1.0}, 4) + assert len(groups) == 4 + assert sorted(i for g in groups for i in g) == items + assert sum(not g for g in groups) == 2 + + +def test_lpt_partition_tie_determinism() -> None: + """Equal weights assign in input order, so output is reproducible.""" + items = [f"i{n}" for n in range(4)] + weights = dict.fromkeys(items, 1.0) + assert helpers.lpt_partition(items, weights, 2) == [["i0", "i2"], ["i1", "i3"]] diff --git a/tests/script/test_update_integration_test_durations.py b/tests/script/test_update_integration_test_durations.py new file mode 100644 index 0000000000..f2f373d4bb --- /dev/null +++ b/tests/script/test_update_integration_test_durations.py @@ -0,0 +1,130 @@ +"""Unit tests for script/update_integration_test_durations.py.""" + +import json +from pathlib import Path +import sys +from unittest.mock import patch + +import pytest + +# Add the script directory to Python path so we can import the module +script_dir = str((Path(__file__).parent / ".." / ".." / "script").resolve()) +sys.path.insert(0, script_dir) + +import helpers # noqa: E402 +import update_integration_test_durations as uitd # noqa: E402 + +JUNIT_TEMPLATE = """ +{testcases} +""" + +KNOWN = { + "tests/integration/test_a.py", + "tests/integration/test_b.py", +} + + +def _write_junit(path: Path, testcases: str) -> None: + path.write_text(JUNIT_TEMPLATE.format(testcases=testcases), encoding="utf-8") + + +def test_collect_durations_sums_per_file(tmp_path: Path) -> None: + """Testcases from the same module sum.""" + _write_junit( + tmp_path / "a.xml", + '' + '' + '', + ) + assert uitd.collect_durations(tmp_path, KNOWN) == { + "tests/integration/test_a.py": 3.5, + "tests/integration/test_b.py": 4.0, + } + + +def test_collect_durations_class_based_testcase(tmp_path: Path) -> None: + """A class-based classname still maps to its module file.""" + _write_junit( + tmp_path / "a.xml", + '', + ) + assert uitd.collect_durations(tmp_path, KNOWN) == { + "tests/integration/test_a.py": 2.5 + } + + +def test_collect_durations_unknown_module_skipped( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A classname that maps to no known file is skipped with a warning.""" + _write_junit( + tmp_path / "a.xml", + '', + ) + assert uitd.collect_durations(tmp_path, KNOWN) == {} + assert "test_gone" in capsys.readouterr().err + + +def test_collect_durations_skips_skipped_testcases(tmp_path: Path) -> None: + """Skipped testcases do not record a bogus zero duration.""" + _write_junit( + tmp_path / "a.xml", + '' + "", + ) + assert uitd.collect_durations(tmp_path, KNOWN) == {} + + +def test_collect_durations_unexpected_classname_aborts(tmp_path: Path) -> None: + """A classname outside tests.integration means the junit layout changed.""" + _write_junit( + tmp_path / "a.xml", + '', + ) + with pytest.raises(SystemExit): + uitd.collect_durations(tmp_path, KNOWN) + + +def test_collect_durations_empty_dir_aborts(tmp_path: Path) -> None: + """No junit XML at all is a hard error, not an empty recording.""" + with pytest.raises(SystemExit): + uitd.collect_durations(tmp_path, KNOWN) + + +def test_main_merges_partial_run(tmp_path: Path) -> None: + """A partial run merges over the previous data instead of truncating it.""" + tests_dir = tmp_path / "tests" / "integration" + tests_dir.mkdir(parents=True) + for name in ("test_a", "test_b", "test_c"): + (tests_dir / f"{name}.py").write_text("", encoding="utf-8") + durations_file = tmp_path / helpers.INTEGRATION_TEST_DURATIONS_FILE + durations_file.write_text( + json.dumps( + { + "tests/integration/test_a.py": 5.0, + "tests/integration/test_b.py": 7.0, + "tests/integration/test_gone.py": 9.0, + } + ), + encoding="utf-8", + ) + junit_dir = tmp_path / "junit" + junit_dir.mkdir() + _write_junit( + junit_dir / "a.xml", + '', + ) + with ( + patch.object(helpers, "root_path", str(tmp_path)), + patch.object(uitd, "DURATIONS_FILE", durations_file), + ): + # 1 of 3 files covered: refused without --allow-partial + with patch.object(sys, "argv", ["uitd", str(junit_dir)]): + assert uitd.main() == uitd.EXIT_LOW_COVERAGE + with patch.object(sys, "argv", ["uitd", str(junit_dir), "--allow-partial"]): + assert uitd.main() == 0 + # test_a updated, test_b kept, deleted test_gone dropped + assert json.loads(durations_file.read_text()) == { + "tests/integration/test_a.py": 6.0, + "tests/integration/test_b.py": 7.0, + } From ea1e0c6f6897ebb901664ccc1155047dda4de0b9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 31 Aug 2026 13:27:53 -0500 Subject: [PATCH 03/25] [core] Keep scheduler dump cancelled marker string in flash on ESP8266 (#18906) --- esphome/core/scheduler.cpp | 15 ++++++++------- esphome/core/scheduler.h | 4 ++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/esphome/core/scheduler.cpp b/esphome/core/scheduler.cpp index e9c5bf2c04..afb323f78e 100644 --- a/esphome/core/scheduler.cpp +++ b/esphome/core/scheduler.cpp @@ -193,7 +193,7 @@ void HOT Scheduler::set_timer_common_(Component *component, SchedulerItem::Type } #ifdef ESPHOME_DEBUG_SCHEDULER - this->debug_log_timer_(item, name_type, static_name, hash_or_id, type, delay, now_64); + this->debug_log_timer_(item, name_type, static_name, hash_or_id, delay, now_64); #endif /* ESPHOME_DEBUG_SCHEDULER */ } @@ -438,9 +438,10 @@ uint32_t HOT Scheduler::call(uint32_t now) { SchedulerNameLog name_log; bool is_cancelled = is_item_removed_(item); ESP_LOGD(TAG, " %s '%s/%s' interval=%" PRIu32 " next_execution in %" PRIu64 "ms at %" PRIu64 "%s", - item->get_type_str(), LOG_STR_ARG(item->get_source()), + LOG_STR_ARG(item->get_type_str()), LOG_STR_ARG(item->get_source()), name_log.format(item->get_name_type(), item->get_name(), item->get_name_hash_or_id()), item->interval, - item->get_next_execution() - now_64, item->get_next_execution(), is_cancelled ? " [CANCELLED]" : ""); + item->get_next_execution() - now_64, item->get_next_execution(), + is_cancelled ? LOG_STR_LITERAL(" [CANCELLED]") : LOG_STR_LITERAL("")); old_items.push_back(item); } @@ -512,7 +513,7 @@ uint32_t HOT Scheduler::call(uint32_t now) { { SchedulerNameLog name_log; ESP_LOGV(TAG, "Running %s '%s/%s' with interval=%" PRIu32 " next_execution=%" PRIu64 " (now=%" PRIu64 ")", - item->get_type_str(), LOG_STR_ARG(item->get_source()), + LOG_STR_ARG(item->get_type_str()), LOG_STR_ARG(item->get_source()), name_log.format(item->get_name_type(), item->get_name(), item->get_name_hash_or_id()), item->interval, item->get_next_execution(), now_64); } @@ -794,7 +795,7 @@ void Scheduler::trim_freelist() { #ifdef ESPHOME_DEBUG_SCHEDULER void Scheduler::debug_log_timer_(const SchedulerItem *item, NameType name_type, const char *static_name, - uint32_t hash_or_id, SchedulerItem::Type type, uint32_t delay, uint64_t now) { + uint32_t hash_or_id, uint32_t delay, uint64_t now) { // Validate static strings in debug mode if (name_type == NameType::STATIC_STRING && static_name != nullptr) { validate_static_string(static_name); @@ -802,8 +803,8 @@ void Scheduler::debug_log_timer_(const SchedulerItem *item, NameType name_type, // Debug logging SchedulerNameLog name_log; - const char *type_str = (type == SchedulerItem::TIMEOUT) ? "timeout" : "interval"; - if (type == SchedulerItem::TIMEOUT) { + const char *type_str = LOG_STR_ARG(item->get_type_str()); + if (item->type == SchedulerItem::TIMEOUT) { ESP_LOGD(TAG, "set_%s(name='%s/%s', %s=%" PRIu32 ")", type_str, LOG_STR_ARG(item->get_source()), name_log.format(name_type, static_name, hash_or_id), type_str, delay); } else { diff --git a/esphome/core/scheduler.h b/esphome/core/scheduler.h index 8ef3499a11..56fc83f12f 100644 --- a/esphome/core/scheduler.h +++ b/esphome/core/scheduler.h @@ -254,7 +254,7 @@ class Scheduler { // This is correct because millis_major_ that creates these values is also 16 bits. next_execution_high_ = static_cast(value >> 32); } - constexpr const char *get_type_str() const { return (type == TIMEOUT) ? "timeout" : "interval"; } + const LogString *get_type_str() const { return (type == TIMEOUT) ? LOG_STR("timeout") : LOG_STR("interval"); } // The owning component, or nullptr for SELF_POINTER items (whose slot holds source_name instead). // All component access goes through this so SELF_POINTER items read as component-less. Component *get_component() const { return name_type_ == NameType::SELF_POINTER ? nullptr : component; } @@ -404,7 +404,7 @@ class Scheduler { #ifdef ESPHOME_DEBUG_SCHEDULER // Helper for debug logging in set_timer_common_ - extracted to reduce code size void debug_log_timer_(const SchedulerItem *item, NameType name_type, const char *static_name, uint32_t hash_or_id, - SchedulerItem::Type type, uint32_t delay, uint64_t now); + uint32_t delay, uint64_t now); #endif /* ESPHOME_DEBUG_SCHEDULER */ #ifndef ESPHOME_THREAD_SINGLE From 61e37cfc8614af14191864988556046d9730a944 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Tue, 1 Sep 2026 07:01:53 +1000 Subject: [PATCH 04/25] [docker] Use GITHUB_REPOSITORY instead of hardcoded esphome/esphome (#18916) --- docker/generate_tags.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/generate_tags.py b/docker/generate_tags.py index 31f98c4614..a54205f1bf 100755 --- a/docker/generate_tags.py +++ b/docker/generate_tags.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import argparse +import os import re CHANNEL_DEV = "dev" @@ -64,7 +65,8 @@ def main(): suffix = f"-{args.suffix}" if args.suffix else "" - image_name = f"esphome/esphome{suffix}" + repository = (os.environ.get("GITHUB_REPOSITORY") or "esphome/esphome").strip().lower() + image_name = f"{repository}{suffix}" print(f"channel={channel}") From 24af3fd8343ad7fb8c7db6f5c5f126af240a1e6a Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Tue, 1 Sep 2026 07:13:36 +1000 Subject: [PATCH 05/25] [lvgl] Fix user_ flags (#18902) --- esphome/components/lvgl/defines.py | 4 ---- tests/components/lvgl/lvgl-package.yaml | 26 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index 61d15752be..1eee8041f9 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -628,10 +628,6 @@ OBJ_FLAGS = ( "send_draw_task_events", "widget_1", "widget_2", - "user_1", - "user_2", - "user_3", - "user_4", ) LV_OBJ_FLAG = LvConstant("LV_OBJ_FLAG_", *OBJ_FLAGS) diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index b457ec2c0b..07c492db35 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -188,6 +188,8 @@ lvgl: dark_mode: true obj: border_width: 1 + user_1: + bg_color: black gradients: - id: color_bar @@ -717,6 +719,30 @@ lvgl: id: button_with_text text: Clicked + # Exercises the LV_STATE_USER_1..USER_4 states: setting them at creation + # (both literal and lambda), styling each of them individually, and + # setting/clearing them at runtime with lvgl.widget.update. + - button: + id: user_flags_button + text: User flags + state: + user_1: true + user_2: !lambda return true; + user_1: + bg_color: 0xFF00FF + user_2: + bg_color: 0x00FFFF + user_3: + bg_color: 0xFFFF00 + user_4: + bg_color: 0x808080 + on_click: + - lvgl.widget.update: + id: user_flags_button + state: + user_3: true + user_4: !lambda return !lv_obj_has_state(id(user_flags_button), LV_STATE_USER_4); + - button: layout: 2x1 id: button_button From 7a784d11358cf82a2566ff5585504080cb8cc2f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:16:22 -0400 Subject: [PATCH 06/25] Bump zeroconf from 0.150.0 to 0.150.4 (#18924) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a065492dfa..3b3f3029ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ esptool==5.3.1 click==8.3.3 aioesphomeapi==46.3.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi -zeroconf==0.150.0 +zeroconf==0.150.4 puremagic==2.2.0 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import From fc9afcb201acba3bd8961b1e6011528bc5b8ea0c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:17:58 -0400 Subject: [PATCH 07/25] Bump ruff from 0.16.4 to 0.16.5 (#18920) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index cf4b028b0e..df37a10cb4 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.0.7 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating -ruff==0.16.4 # also change in .pre-commit-config.yaml when updating +ruff==0.16.5 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating prek==0.4.14 # also change in .github/workflows/ci.yml when updating From 78beb75b3bfbde8b6006301611bf6114b6fbbaf8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:20:00 -0400 Subject: [PATCH 08/25] Bump github/codeql-action/analyze from 4.37.8 to 4.37.9 (#18926) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b46f9adab6..12c6c6c60c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -84,6 +84,6 @@ jobs: exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: category: "/language:${{matrix.language}}" From ebe6c2d0495af2d72e2405326fad7efde20d41bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:20:12 -0400 Subject: [PATCH 09/25] Bump github/codeql-action/init from 4.37.8 to 4.37.9 (#18927) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 12c6c6c60c..aab3dea592 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,7 +56,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@db488ddef3bf6cb639b32c2e9a7c0a7ea8271d28 # v4.37.8 + uses: github/codeql-action/init@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} From c20b619684a84e44800c56e160df8ecab0315250 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:20:43 -0400 Subject: [PATCH 10/25] Bump platformdirs from 4.11.4 to 4.11.5 (#18923) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3b3f3029ce..0d0fe9591b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ bleak==3.0.2 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 -platformdirs==4.11.4 # native esp-idf toolchain global cache dir +platformdirs==4.11.5 # native esp-idf toolchain global cache dir ninja==1.13.0 # native esp8266 arduino toolchain build driver filelock==3.32.4 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg From d43786937384df398eac11a084ef2f6e37f3d8e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:23:29 -0400 Subject: [PATCH 11/25] Bump cryptography from 48.0.1 to 50.0.1 (#18922) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0d0fe9591b..f19559dca8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # cryptography 49+ ships no Intel macOS wheels (arm64 only); esptool caps <49 there. # Keep 48.0.1, the last universal2 release, so esphome stays installable on Intel Macs. -cryptography==50.0.0; platform_system != "Darwin" or platform_machine != "x86_64" +cryptography==50.0.1; platform_system != "Darwin" or platform_machine != "x86_64" cryptography==48.0.1; platform_system == "Darwin" and platform_machine == "x86_64" voluptuous==0.16.0 PyYAML==6.0.3 From ca44db107cc63cb7104b554b6f56d94d27b2c83a Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:33:52 -0400 Subject: [PATCH 12/25] [docker] Fix generate_tags.py formatting (#18928) --- docker/generate_tags.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/generate_tags.py b/docker/generate_tags.py index a54205f1bf..b35aed91f0 100755 --- a/docker/generate_tags.py +++ b/docker/generate_tags.py @@ -65,7 +65,9 @@ def main(): suffix = f"-{args.suffix}" if args.suffix else "" - repository = (os.environ.get("GITHUB_REPOSITORY") or "esphome/esphome").strip().lower() + repository = ( + (os.environ.get("GITHUB_REPOSITORY") or "esphome/esphome").strip().lower() + ) image_name = f"{repository}{suffix}" print(f"channel={channel}") From 081ef3d30dc63811c547056c2d4d593c7d44b101 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 17:46:47 -0400 Subject: [PATCH 13/25] Bump prek from 0.4.14 to 0.5.0 (#18921) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index df37a10cb4..e837953878 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,7 @@ pylint==4.0.7 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating ruff==0.16.5 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating -prek==0.4.14 # also change in .github/workflows/ci.yml when updating +prek==0.5.0 # also change in .github/workflows/ci.yml when updating # Unit tests pytest==9.1.1 From afb0022dd0eb882a06e77190a4e4055c7dd05c16 Mon Sep 17 00:00:00 2001 From: Bonne Eggleston Date: Mon, 31 Aug 2026 16:53:39 -0700 Subject: [PATCH 14/25] [core] Lint: require braces around single ESP_LOG control-statement bodies (#18727) --- esphome/components/alpha3/alpha3.cpp | 3 +- esphome/components/api/api_connection.cpp | 3 +- esphome/components/bk72xx_ble/bdk_scan.cpp | 3 +- esphome/components/bk72xx_ble/bk72xx_ble.cpp | 15 +- .../bk72xx_ble_tracker/bk72xx_ble_tracker.cpp | 3 +- .../ble_client/output/ble_binary_output.cpp | 3 +- esphome/components/bme680/bme680.cpp | 6 +- esphome/components/climate/climate.cpp | 18 ++- esphome/components/dht/dht.cpp | 3 +- .../camera_web_server.cpp | 3 +- esphome/components/fan/fan.cpp | 3 +- .../hbridge/switch/hbridge_switch.cpp | 3 +- esphome/components/he60r/he60r.cpp | 6 +- .../components/hoermann_hcp/hoermann_hcp.cpp | 3 +- .../key_collector/key_collector.cpp | 21 ++- esphome/components/ln882h_ble/ln882h_ble.cpp | 3 +- esphome/components/lvgl/lvgl_esphome.cpp | 3 +- esphome/components/mipi_dsi/mipi_dsi.cpp | 3 +- esphome/components/mipi_rgb/mipi_rgb.cpp | 3 +- esphome/components/mipi_spi/mipi_spi.cpp | 9 +- esphome/components/modbus/modbus.cpp | 6 +- esphome/components/mqtt/mqtt_component.cpp | 6 +- esphome/components/one_wire/one_wire_bus.cpp | 3 +- .../packet_transport/packet_transport.cpp | 12 +- esphome/components/qwiic_pir/qwiic_pir.cpp | 3 +- .../components/rpi_dpi_rgb/rpi_dpi_rgb.cpp | 3 +- .../components/shelly_dimmer/stm32flash.cpp | 3 +- esphome/components/spi/spi.h | 3 +- esphome/components/spi/spi_esp_idf.cpp | 9 +- esphome/components/st7701s/st7701s.cpp | 3 +- .../tuya/water_heater/tuya_water_heater.cpp | 12 +- esphome/components/udp/udp_component.cpp | 9 +- .../uponor_smatrix/uponor_smatrix.cpp | 3 +- esphome/components/usb_uart/pl2303.cpp | 3 +- .../components/wake_on_lan/wake_on_lan.cpp | 3 +- esphome/components/weikai/weikai.cpp | 15 +- script/ci-custom.py | 148 ++++++++++++++++++ tests/script/test_ci_custom.py | 147 +++++++++++++++++ 38 files changed, 437 insertions(+), 71 deletions(-) create mode 100644 tests/script/test_ci_custom.py diff --git a/esphome/components/alpha3/alpha3.cpp b/esphome/components/alpha3/alpha3.cpp index 048c365616..92b00d87cb 100644 --- a/esphome/components/alpha3/alpha3.cpp +++ b/esphome/components/alpha3/alpha3.cpp @@ -162,8 +162,9 @@ void Alpha3::send_request_(uint8_t *request, size_t len) { auto status = esp_ble_gattc_write_char(this->parent_->get_gattc_if(), this->parent_->get_conn_id(), this->geni_handle_, len, request, ESP_GATT_WRITE_TYPE_NO_RSP, ESP_GATT_AUTH_REQ_NONE); - if (status) + if (status) { ESP_LOGW(TAG, "[%s] esp_ble_gattc_write_char failed, status=%d", this->parent_->address_str(), status); + } } void Alpha3::update() { diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index bc088ca473..9c609aa047 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -2391,8 +2391,9 @@ void APIConnection::process_batch_() { } else if (payload_size == 0) { // payload_size == 0 with remove set means encoding hit OOM and the // connection is being dropped; warn only for a genuinely oversized message - if (!this->flags_.remove) + if (!this->flags_.remove) { ESP_LOGW(TAG, "Message too large to send: type=%u", item.message_type); + } this->clear_batch_(); } return; diff --git a/esphome/components/bk72xx_ble/bdk_scan.cpp b/esphome/components/bk72xx_ble/bdk_scan.cpp index f17f21c06b..3192fc79d7 100644 --- a/esphome/components/bk72xx_ble/bdk_scan.cpp +++ b/esphome/components/bk72xx_ble/bdk_scan.cpp @@ -62,8 +62,9 @@ BdkActivityState bdk_scan_state(uint8_t activity_idx) { uint8_t bdk_scan_acquire_activity() { uint8_t idx = app_ble_get_idle_actv_idx_handle(SCAN_ACTV); - if (idx == INVALID_ACTIVITY_IDX) + if (idx == INVALID_ACTIVITY_IDX) { ESP_LOGE(TAG, "Scan start failed: no idle activity handle"); + } return idx; } diff --git a/esphome/components/bk72xx_ble/bk72xx_ble.cpp b/esphome/components/bk72xx_ble/bk72xx_ble.cpp index 52401114e6..7a4efff455 100644 --- a/esphome/components/bk72xx_ble/bk72xx_ble.cpp +++ b/esphome/components/bk72xx_ble/bk72xx_ble.cpp @@ -181,8 +181,9 @@ void BK72xxBLE::enable() { break; } } - if (!bdaddr_live) + if (!bdaddr_live) { ESP_LOGW(TAG, "Controller address still unset after init; BLE stack may not have started"); + } #endif this->state_ = BLEComponentState::ACTIVE; @@ -210,8 +211,9 @@ void BK72xxBLE::loop() { // Re-check a settled scan; scan_start() refills the bring-up budget. // WARN: the only report of a drop that recovers inside its budget. if (this->scan_start(this->requested_.interval, this->requested_.window, this->requested_.active) != - ScanOpResult::SETTLED) + ScanOpResult::SETTLED) { ESP_LOGW(TAG, "Controller dropped the scan; restarting"); + } } // Drain the lock-free ring filled by the BLE task; all per-report work runs @@ -230,8 +232,9 @@ void BK72xxBLE::loop() { // Log dropped reports — only reachable when reports were processed; drops can // only occur while the queue is full, and only this loop drains it. uint16_t dropped = this->report_queue_.get_and_reset_dropped_count(); - if (dropped > 0) + if (dropped > 0) { ESP_LOGW(TAG, "Dropped %u scan reports due to queue overflow", dropped); + } } void BK72xxBLE::get_mac_lsb_first(uint8_t out[MAC_ADDRESS_SIZE]) const { @@ -449,8 +452,9 @@ ScanOpResult BK72xxBLE::advance_stop_(BdkActivityState state, bool ready) { if (!ready) { // Acting mid-operation could delete an activity whose start lands // afterwards, leaking the slot with the radio on; wait. - if (this->last_result_ == ScanOpResult::SETTLED) + if (this->last_result_ == ScanOpResult::SETTLED) { ESP_LOGD(TAG, "Scan stop deferred (controller busy)"); + } return ScanOpResult::PENDING; } // Settled, so CREATED unambiguously means "never started". @@ -474,8 +478,9 @@ ScanOpResult BK72xxBLE::advance_start_(BdkActivityState state, bool ready) { return ScanOpResult::PENDING; } if (!ready) { - if (this->last_result_ == ScanOpResult::SETTLED) + if (this->last_result_ == ScanOpResult::SETTLED) { ESP_LOGD(TAG, "Scan start deferred (controller busy)"); + } return ScanOpResult::PENDING; } if (state == BdkActivityState::CREATED) { diff --git a/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.cpp b/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.cpp index 1b4e6245ae..0939e1259f 100644 --- a/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.cpp +++ b/esphome/components/bk72xx_ble_tracker/bk72xx_ble_tracker.cpp @@ -69,8 +69,9 @@ void BK72xxBLETracker::on_ota_global_state(ota::OTAState state, float progress, this->stop_scan(); // The transfer starves the loop; a deferred stop would leave the radio // scanning for the whole update, so drain it here, bounded. - if (!this->parent_->flush_pending_stop(OTA_STOP_FLUSH_MS)) + if (!this->parent_->flush_pending_stop(OTA_STOP_FLUSH_MS)) { ESP_LOGE(TAG, "Scan still stopping at OTA start; the radio may contend with the update"); + } } else if (state == ota::OTA_ERROR || state == ota::OTA_ABORT) { // On success the device reboots, so restore only on a failed/aborted update; // loop() restarts the scan on its next iteration (continuous idle branch). diff --git a/esphome/components/ble_client/output/ble_binary_output.cpp b/esphome/components/ble_client/output/ble_binary_output.cpp index 1cb83b9d8b..5d53c59708 100644 --- a/esphome/components/ble_client/output/ble_binary_output.cpp +++ b/esphome/components/ble_client/output/ble_binary_output.cpp @@ -80,8 +80,9 @@ void BLEBinaryOutput::write_state(bool state) { esp_err_t err = esp_ble_gattc_write_char(this->parent()->get_gattc_if(), this->parent()->get_conn_id(), this->char_handle_, sizeof(state_as_uint), &state_as_uint, this->write_type_, ESP_GATT_AUTH_REQ_NONE); - if (err != ESP_GATT_OK) + if (err != ESP_GATT_OK) { ESP_LOGW(TAG, "[%s] Write error, err=%d", this->char_uuid_.to_str(char_buf), err); + } } } // namespace esphome::ble_client diff --git a/esphome/components/bme680/bme680.cpp b/esphome/components/bme680/bme680.cpp index ef98174e06..164424de09 100644 --- a/esphome/components/bme680/bme680.cpp +++ b/esphome/components/bme680/bme680.cpp @@ -327,10 +327,12 @@ void BME680Component::read_data_() { ESP_LOGD(TAG, "Got temperature=%.1f°C pressure=%.1fhPa humidity=%.1f%% gas_resistance=%.1fΩ", temperature, pressure, humidity, gas_resistance); - if (!gas_valid) + if (!gas_valid) { ESP_LOGW(TAG, "Gas measurement unsuccessful, reading invalid!"); - if (!heat_stable) + } + if (!heat_stable) { ESP_LOGW(TAG, "Heater unstable, reading invalid! (Normal for a few readings after a power cycle)"); + } if (this->temperature_sensor_ != nullptr) this->temperature_sensor_->publish_state(temperature); diff --git a/esphome/components/climate/climate.cpp b/esphome/components/climate/climate.cpp index 6ca9e394f7..34684a87e1 100644 --- a/esphome/components/climate/climate.cpp +++ b/esphome/components/climate/climate.cpp @@ -749,33 +749,39 @@ void Climate::dump_traits_(const char *tag) { } if (!traits.get_supported_modes().empty()) { ESP_LOGCONFIG(tag, " Supported modes:"); - for (ClimateMode m : traits.get_supported_modes()) + for (ClimateMode m : traits.get_supported_modes()) { ESP_LOGCONFIG(tag, " - %s", LOG_STR_ARG(climate_mode_to_string(m))); + } } if (!traits.get_supported_fan_modes().empty()) { ESP_LOGCONFIG(tag, " Supported fan modes:"); - for (ClimateFanMode m : traits.get_supported_fan_modes()) + for (ClimateFanMode m : traits.get_supported_fan_modes()) { ESP_LOGCONFIG(tag, " - %s", LOG_STR_ARG(climate_fan_mode_to_string(m))); + } } if (!traits.get_supported_custom_fan_modes().empty()) { ESP_LOGCONFIG(tag, " Supported custom fan modes:"); - for (const char *s : traits.get_supported_custom_fan_modes()) + for (const char *s : traits.get_supported_custom_fan_modes()) { ESP_LOGCONFIG(tag, " - %s", s); + } } if (!traits.get_supported_presets().empty()) { ESP_LOGCONFIG(tag, " Supported presets:"); - for (ClimatePreset p : traits.get_supported_presets()) + for (ClimatePreset p : traits.get_supported_presets()) { ESP_LOGCONFIG(tag, " - %s", LOG_STR_ARG(climate_preset_to_string(p))); + } } if (!traits.get_supported_custom_presets().empty()) { ESP_LOGCONFIG(tag, " Supported custom presets:"); - for (const char *s : traits.get_supported_custom_presets()) + for (const char *s : traits.get_supported_custom_presets()) { ESP_LOGCONFIG(tag, " - %s", s); + } } if (!traits.get_supported_swing_modes().empty()) { ESP_LOGCONFIG(tag, " Supported swing modes:"); - for (ClimateSwingMode m : traits.get_supported_swing_modes()) + for (ClimateSwingMode m : traits.get_supported_swing_modes()) { ESP_LOGCONFIG(tag, " - %s", LOG_STR_ARG(climate_swing_mode_to_string(m))); + } } } diff --git a/esphome/components/dht/dht.cpp b/esphome/components/dht/dht.cpp index a9117be4e1..2196f3a982 100644 --- a/esphome/components/dht/dht.cpp +++ b/esphome/components/dht/dht.cpp @@ -154,8 +154,9 @@ bool HOT IRAM_ATTR DHT::read_sensor_(float *temperature, float *humidity, bool r } } if (error_code != 0) { - if (report_errors) + if (report_errors) { ESP_LOGW(TAG, ESP_LOG_MSG_COMM_FAIL); + } return false; } diff --git a/esphome/components/esp32_camera_web_server/camera_web_server.cpp b/esphome/components/esp32_camera_web_server/camera_web_server.cpp index 88579e9632..bee231d132 100644 --- a/esphome/components/esp32_camera_web_server/camera_web_server.cpp +++ b/esphome/components/esp32_camera_web_server/camera_web_server.cpp @@ -210,8 +210,9 @@ esp_err_t CameraWebServer::streaming_handler_(struct httpd_req *req) { if (!image) { // A shutdown is not a lost frame: wait_for_image_() returns empty as soon // as running_ clears, and the loop condition below ends the stream anyway. - if (this->running_) + if (this->running_) { ESP_LOGW(TAG, "STREAM: failed to acquire frame"); + } res = ESP_FAIL; } if (res == ESP_OK) { diff --git a/esphome/components/fan/fan.cpp b/esphome/components/fan/fan.cpp index 7dc0b5c6fe..65521e63d5 100644 --- a/esphome/components/fan/fan.cpp +++ b/esphome/components/fan/fan.cpp @@ -334,8 +334,9 @@ void Fan::dump_traits_(const char *tag, const char *prefix) { } if (traits.supports_preset_modes()) { ESP_LOGCONFIG(tag, "%s Supported presets:", prefix); - for (const char *s : traits.supported_preset_modes()) + for (const char *s : traits.supported_preset_modes()) { ESP_LOGCONFIG(tag, "%s - %s", prefix, s); + } } } diff --git a/esphome/components/hbridge/switch/hbridge_switch.cpp b/esphome/components/hbridge/switch/hbridge_switch.cpp index 1012a264f2..c8e472d7aa 100644 --- a/esphome/components/hbridge/switch/hbridge_switch.cpp +++ b/esphome/components/hbridge/switch/hbridge_switch.cpp @@ -29,8 +29,9 @@ void HBridgeSwitch::dump_config() { LOG_PIN(" On Pin: ", this->on_pin_); LOG_PIN(" Off Pin: ", this->off_pin_); ESP_LOGCONFIG(TAG, " Pulse length: %" PRId32 " ms", this->pulse_length_); - if (this->wait_time_) + if (this->wait_time_) { ESP_LOGCONFIG(TAG, " Wait time %" PRId32 " ms", this->wait_time_); + } } void HBridgeSwitch::write_state(bool state) { diff --git a/esphome/components/he60r/he60r.cpp b/esphome/components/he60r/he60r.cpp index ea662e3ba9..f49224f17c 100644 --- a/esphome/components/he60r/he60r.cpp +++ b/esphome/components/he60r/he60r.cpp @@ -44,8 +44,9 @@ void HE60rCover::dump_config() { " Close Duration: %.1fs", this->open_duration_ / 1e3f, this->close_duration_ / 1e3f); auto restore = this->restore_state_(); - if (restore.has_value()) + if (restore.has_value()) { ESP_LOGCONFIG(TAG, " Saved position %d%%", (int) (restore->position * 100.f)); + } } void HE60rCover::endstop_reached_(CoverOperation operation) { @@ -77,8 +78,9 @@ void HE60rCover::process_rx_(uint8_t data) { ESP_LOGV(TAG, "Process RX data %X", data); if (!this->query_seen_) { this->query_seen_ = data == QUERY_BYTE; - if (!this->query_seen_) + if (!this->query_seen_) { ESP_LOGD(TAG, "RX Byte %02X", data); + } return; } switch (data) { diff --git a/esphome/components/hoermann_hcp/hoermann_hcp.cpp b/esphome/components/hoermann_hcp/hoermann_hcp.cpp index 17df927eb7..4aa2c79bb1 100644 --- a/esphome/components/hoermann_hcp/hoermann_hcp.cpp +++ b/esphome/components/hoermann_hcp/hoermann_hcp.cpp @@ -257,8 +257,9 @@ void HoermannHcp::on_state_reg_(uint16_t value) { } } // The low byte can change on its own, so only report a state we cannot decode once. - if (state != (previous >> 8)) + if (state != (previous >> 8)) { ESP_LOGW(TAG, "Unknown door state 0x%02X", state); + } } // Low byte of register 6: bit 0x10 is the lamp, bit 0x04 the relay. The reference implementation records diff --git a/esphome/components/key_collector/key_collector.cpp b/esphome/components/key_collector/key_collector.cpp index 69b7a6a7c6..42f02d39d4 100644 --- a/esphome/components/key_collector/key_collector.cpp +++ b/esphome/components/key_collector/key_collector.cpp @@ -16,26 +16,33 @@ void KeyCollector::loop() { void KeyCollector::dump_config() { #if ESPHOME_LOG_LEVEL >= ESPHOME_LOG_LEVEL_CONFIG ESP_LOGCONFIG(TAG, "Key Collector:"); - if (this->min_length_ > 0) + if (this->min_length_ > 0) { ESP_LOGCONFIG(TAG, " min length: %d", this->min_length_); - if (this->max_length_ > 0) + } + if (this->max_length_ > 0) { ESP_LOGCONFIG(TAG, " max length: %d", this->max_length_); - if (!this->back_keys_.empty()) + } + if (!this->back_keys_.empty()) { ESP_LOGCONFIG(TAG, " erase keys '%s'", this->back_keys_.c_str()); - if (!this->clear_keys_.empty()) + } + if (!this->clear_keys_.empty()) { ESP_LOGCONFIG(TAG, " clear keys '%s'", this->clear_keys_.c_str()); - if (!this->start_keys_.empty()) + } + if (!this->start_keys_.empty()) { ESP_LOGCONFIG(TAG, " start keys '%s'", this->start_keys_.c_str()); + } if (!this->end_keys_.empty()) { ESP_LOGCONFIG(TAG, " end keys '%s'\n" " end key is required: %s", this->end_keys_.c_str(), ONOFF(this->end_key_required_)); } - if (!this->allowed_keys_.empty()) + if (!this->allowed_keys_.empty()) { ESP_LOGCONFIG(TAG, " allowed keys '%s'", this->allowed_keys_.c_str()); - if (this->timeout_ > 0) + } + if (this->timeout_ > 0) { ESP_LOGCONFIG(TAG, " entry timeout: %0.1f", this->timeout_ / 1000.0); + } #endif } diff --git a/esphome/components/ln882h_ble/ln882h_ble.cpp b/esphome/components/ln882h_ble/ln882h_ble.cpp index 021e138f08..0b15bf434c 100644 --- a/esphome/components/ln882h_ble/ln882h_ble.cpp +++ b/esphome/components/ln882h_ble/ln882h_ble.cpp @@ -333,8 +333,9 @@ void LN882HBLE::loop() { // the queue empty — from the very first report on. Checking here keeps that // failure visible instead of producing a scanner that is silently dead. uint16_t dropped = this->report_queue_.get_and_reset_dropped_count(); - if (dropped > 0) + if (dropped > 0) { ESP_LOGW(TAG, "Dropped %u scan reports (queue full or out of memory for a report slot)", dropped); + } // Drain the lock-free ring filled by the rw task; all per-report work runs // here on the main task, then the report returns to the pool. BLEScanReport *report = this->report_queue_.pop(); diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index a10fdb0582..2c988473a9 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -1059,8 +1059,9 @@ static void *lv_alloc_draw_buf(size_t size, bool internal) { void *buffer; size = LV_ROUND_UP(size, LV_DRAW_BUF_ALIGN); buffer = heap_caps_aligned_alloc(LV_DRAW_BUF_ALIGN, size, internal ? MALLOC_CAP_8BIT : cap_bits); // NOLINT - if (buffer == nullptr) + if (buffer == nullptr) { ESP_LOGW(esphome::lvgl::TAG, "Failed to allocate %zu bytes for %sdraw buffer", size, internal ? "internal " : ""); + } return buffer; } diff --git a/esphome/components/mipi_dsi/mipi_dsi.cpp b/esphome/components/mipi_dsi/mipi_dsi.cpp index 0ff934ae94..0850b50c85 100644 --- a/esphome/components/mipi_dsi/mipi_dsi.cpp +++ b/esphome/components/mipi_dsi/mipi_dsi.cpp @@ -237,8 +237,9 @@ void MipiDsi::write_to_display_(int x_start, int y_start, int w, int h, const ui xSemaphoreTake(this->io_lock_, portMAX_DELAY); } } - if (err != ESP_OK) + if (err != ESP_OK) { ESP_LOGE(TAG, "lcd_lcd_panel_draw_bitmap failed: %s", esp_err_to_name(err)); + } } bool MipiDsi::check_buffer_() { diff --git a/esphome/components/mipi_rgb/mipi_rgb.cpp b/esphome/components/mipi_rgb/mipi_rgb.cpp index aeb04c155c..f43bbab21c 100644 --- a/esphome/components/mipi_rgb/mipi_rgb.cpp +++ b/esphome/components/mipi_rgb/mipi_rgb.cpp @@ -243,8 +243,9 @@ void MipiRgb::write_to_display_(int x_start, int y_start, int w, int h, const ui ptr += stride; // next line } } - if (err != ESP_OK) + if (err != ESP_OK) { ESP_LOGE(TAG, "lcd_lcd_panel_draw_bitmap failed: %s", esp_err_to_name(err)); + } } bool MipiRgb::check_buffer_() { diff --git a/esphome/components/mipi_spi/mipi_spi.cpp b/esphome/components/mipi_spi/mipi_spi.cpp index 80ae96720b..b2658de6e8 100644 --- a/esphome/components/mipi_spi/mipi_spi.cpp +++ b/esphome/components/mipi_spi/mipi_spi.cpp @@ -31,12 +31,15 @@ void internal_dump_config(const char *model, int width, int height, int offset_w LOG_PIN(" CS Pin: ", cs); LOG_PIN(" Reset Pin: ", reset); LOG_PIN(" DC Pin: ", dc); - if (offset_width != 0) + if (offset_width != 0) { ESP_LOGCONFIG(TAG, " Offset width: %d", offset_width); - if (offset_height != 0) + } + if (offset_height != 0) { ESP_LOGCONFIG(TAG, " Offset height: %d", offset_height); - if (brightness.has_value()) + } + if (brightness.has_value()) { ESP_LOGCONFIG(TAG, " Brightness: %u", brightness.value()); + } } } // namespace esphome::mipi_spi diff --git a/esphome/components/modbus/modbus.cpp b/esphome/components/modbus/modbus.cpp index 25687ba106..f428236a82 100644 --- a/esphome/components/modbus/modbus.cpp +++ b/esphome/components/modbus/modbus.cpp @@ -1199,15 +1199,17 @@ void ModbusServerHub::send_raw_(const uint8_t *payload, uint16_t len) { this->set_timeout("deferred_send", (this->tx_delay_remaining() + US_PER_MS - 1) / US_PER_MS, [this]() { ModbusFrame frame(this->deferred_payload_[0], this->deferred_payload_.data() + 1, this->deferred_payload_len_ - 1); - if (!this->send_frame_(frame)) + if (!this->send_frame_(frame)) { ESP_LOGE(TAG, "Deferred server reply dropped: transmission still blocked"); + } }); return; } ModbusFrame frame(payload[0], payload + 1, len - 1); - if (!this->send_frame_(frame)) + if (!this->send_frame_(frame)) { ESP_LOGE(TAG, "Server reply dropped: a frame arrived during the send delay"); + } } void Modbus::clear_rx_buffer_(const LogString *reason, bool warn, size_t bytes_to_clear) { diff --git a/esphome/components/mqtt/mqtt_component.cpp b/esphome/components/mqtt/mqtt_component.cpp index 18a759725f..a80cea6bd6 100644 --- a/esphome/components/mqtt/mqtt_component.cpp +++ b/esphome/components/mqtt/mqtt_component.cpp @@ -39,10 +39,12 @@ inline char *append_char(char *p, char c) { // Function implementation of LOG_MQTT_COMPONENT macro to reduce code size void log_mqtt_component(const char *tag, MQTTComponent *obj, bool state_topic, bool command_topic) { char buf[MQTT_DEFAULT_TOPIC_MAX_LEN]; - if (state_topic) + if (state_topic) { ESP_LOGCONFIG(tag, " State Topic: '%s'", obj->get_state_topic_to_(buf).c_str()); - if (command_topic) + } + if (command_topic) { ESP_LOGCONFIG(tag, " Command Topic: '%s'", obj->get_command_topic_to_(buf).c_str()); + } } void MQTTComponent::set_qos(uint8_t qos) { this->qos_ = qos; } diff --git a/esphome/components/one_wire/one_wire_bus.cpp b/esphome/components/one_wire/one_wire_bus.cpp index c7ea59050c..b62e4f47d4 100644 --- a/esphome/components/one_wire/one_wire_bus.cpp +++ b/esphome/components/one_wire/one_wire_bus.cpp @@ -18,8 +18,9 @@ const std::vector &OneWireBus::get_devices() { return this->devices_; bool OneWireBus::reset_() { int res = this->reset_int(); - if (res == -1) + if (res == -1) { ESP_LOGE(TAG, "1-wire bus is held low"); + } return res == 1; } diff --git a/esphome/components/packet_transport/packet_transport.cpp b/esphome/components/packet_transport/packet_transport.cpp index a21f0e2f63..998e1be5fc 100644 --- a/esphome/components/packet_transport/packet_transport.cpp +++ b/esphome/components/packet_transport/packet_transport.cpp @@ -551,12 +551,14 @@ void PacketTransport::dump_config() { " Ping-pong: %s", this->platform_name_, YESNO(this->is_encrypted_()), YESNO(this->ping_pong_enable_)); #ifdef USE_SENSOR - for (const auto &sensor : this->sensors_) + for (const auto &sensor : this->sensors_) { ESP_LOGCONFIG(TAG, " Sensor: %s", sensor.id); + } #endif #ifdef USE_BINARY_SENSOR - for (const auto &sensor : this->binary_sensors_) + for (const auto &sensor : this->binary_sensors_) { ESP_LOGCONFIG(TAG, " Binary Sensor: %s", sensor.id); + } #endif for (const auto &host : this->providers_) { ESP_LOGCONFIG(TAG, " Remote host: %s", host.first.c_str()); @@ -564,15 +566,17 @@ void PacketTransport::dump_config() { #ifdef USE_SENSOR auto rs = this->remote_sensors_.find(host.first.c_str()); if (rs != this->remote_sensors_.end()) { - for (const auto &key : rs->second | std::views::keys) + for (const auto &key : rs->second | std::views::keys) { ESP_LOGCONFIG(TAG, " Sensor: %s", key.c_str()); + } } #endif #ifdef USE_BINARY_SENSOR auto rbs = this->remote_binary_sensors_.find(host.first.c_str()); if (rbs != this->remote_binary_sensors_.end()) { - for (const auto &key : rbs->second | std::views::keys) + for (const auto &key : rbs->second | std::views::keys) { ESP_LOGCONFIG(TAG, " Binary Sensor: %s", key.c_str()); + } } #endif } diff --git a/esphome/components/qwiic_pir/qwiic_pir.cpp b/esphome/components/qwiic_pir/qwiic_pir.cpp index baf8dc122d..eb338db772 100644 --- a/esphome/components/qwiic_pir/qwiic_pir.cpp +++ b/esphome/components/qwiic_pir/qwiic_pir.cpp @@ -124,8 +124,9 @@ void QwiicPIRComponent::dump_config() { void QwiicPIRComponent::clear_events_() { // Clear event status register - if (!this->write_byte(QWIIC_PIR_EVENT_STATUS, 0x00)) + if (!this->write_byte(QWIIC_PIR_EVENT_STATUS, 0x00)) { ESP_LOGW(TAG, "Failed to clear events"); + } } } // namespace esphome::qwiic_pir diff --git a/esphome/components/rpi_dpi_rgb/rpi_dpi_rgb.cpp b/esphome/components/rpi_dpi_rgb/rpi_dpi_rgb.cpp index aacb217965..c0afc0607e 100644 --- a/esphome/components/rpi_dpi_rgb/rpi_dpi_rgb.cpp +++ b/esphome/components/rpi_dpi_rgb/rpi_dpi_rgb.cpp @@ -75,8 +75,9 @@ void RpiDpiRgb::draw_pixels_at(int x_start, int y_start, int w, int h, const uin break; } } - if (err != ESP_OK) + if (err != ESP_OK) { ESP_LOGE(TAG, "lcd_lcd_panel_draw_bitmap failed: %s", esp_err_to_name(err)); + } } int RpiDpiRgb::get_width() { diff --git a/esphome/components/shelly_dimmer/stm32flash.cpp b/esphome/components/shelly_dimmer/stm32flash.cpp index c758b0a312..beb015851e 100644 --- a/esphome/components/shelly_dimmer/stm32flash.cpp +++ b/esphome/components/shelly_dimmer/stm32flash.cpp @@ -629,8 +629,9 @@ stm32_unique_ptr stm32_init(uart::UARTDevice *stream, const uint8_t flags, const stm->pid = (buf[1] << 8) | buf[2]; if (returned > 2) { ESP_LOGD(TAG, "This bootloader returns %d extra bytes in PID:", returned); - for (auto i = 2; i <= returned; i++) + for (auto i = 2; i <= returned; i++) { ESP_LOGD(TAG, " %02x", buf[i]); + } } if (stm32_get_ack(stm) != STM32_ERR_OK) { return make_stm32_with_deletor(nullptr); diff --git a/esphome/components/spi/spi.h b/esphome/components/spi/spi.h index f8233c48d1..2dfb3c75a8 100644 --- a/esphome/components/spi/spi.h +++ b/esphome/components/spi/spi.h @@ -406,8 +406,9 @@ class SPIClient { this->release_device_, this->write_only_); #ifdef USE_SPI_PSRAM_DMA this->delegate_->set_psram_dma(this->psram_dma_); - if (this->psram_dma_) + if (this->psram_dma_) { esph_log_config("spi_device", "PSRAM DMA: enabled"); + } #endif } diff --git a/esphome/components/spi/spi_esp_idf.cpp b/esphome/components/spi/spi_esp_idf.cpp index 45d38c1719..95b5e4f14b 100644 --- a/esphome/components/spi/spi_esp_idf.cpp +++ b/esphome/components/spi/spi_esp_idf.cpp @@ -42,8 +42,9 @@ class SPIDelegateHw : public SPIDelegate { if (this->release_device_) this->add_device_(); if (this->is_ready()) { - if (spi_device_acquire_bus(this->handle_, portMAX_DELAY) != ESP_OK) + if (spi_device_acquire_bus(this->handle_, portMAX_DELAY) != ESP_OK) { ESP_LOGE(TAG, "Failed to acquire SPI bus"); + } SPIDelegate::begin_transaction(); } else { ESP_LOGW(TAG, "SPI device not ready, cannot begin transaction"); @@ -63,8 +64,9 @@ class SPIDelegateHw : public SPIDelegate { ~SPIDelegateHw() override { esp_err_t const err = spi_bus_remove_device(this->handle_); - if (err != ESP_OK) + if (err != ESP_OK) { ESP_LOGE(TAG, "Remove device failed - err %X", err); + } } // do a transfer. either txbuf or rxbuf (but not both) may be null. @@ -284,8 +286,9 @@ class SPIBusHw : public SPIBus { } buscfg.max_transfer_sz = MAX_TRANSFER_SIZE; auto err = spi_bus_initialize(channel, &buscfg, SPI_DMA_CH_AUTO); - if (err != ESP_OK) + if (err != ESP_OK) { ESP_LOGE(TAG, "Bus init failed - err %X", err); + } } SPIDelegate *get_delegate(uint32_t data_rate, SPIBitOrder bit_order, SPIMode mode, GPIOPin *cs_pin, diff --git a/esphome/components/st7701s/st7701s.cpp b/esphome/components/st7701s/st7701s.cpp index 3ffef86f3e..83f7bc9ce5 100644 --- a/esphome/components/st7701s/st7701s.cpp +++ b/esphome/components/st7701s/st7701s.cpp @@ -78,8 +78,9 @@ void ST7701S::draw_pixels_at(int x_start, int y_start, int w, int h, const uint8 break; } } - if (err != ESP_OK) + if (err != ESP_OK) { esph_log_e(TAG, "lcd_lcd_panel_draw_bitmap failed: %s", esp_err_to_name(err)); + } } void ST7701S::draw_pixel_at(int x, int y, Color color) { diff --git a/esphome/components/tuya/water_heater/tuya_water_heater.cpp b/esphome/components/tuya/water_heater/tuya_water_heater.cpp index 2fca3bf581..e1c78530e3 100644 --- a/esphome/components/tuya/water_heater/tuya_water_heater.cpp +++ b/esphome/components/tuya/water_heater/tuya_water_heater.cpp @@ -177,14 +177,18 @@ water_heater::WaterHeaterMode TuyaWaterHeater::default_on_mode_() const { void TuyaWaterHeater::dump_config() { LOG_WATER_HEATER("", "Tuya Water Heater", this); - if (this->switch_id_.has_value()) + if (this->switch_id_.has_value()) { ESP_LOGCONFIG(TAG, " Switch has datapoint ID %u", *this->switch_id_); - if (this->mode_id_.has_value()) + } + if (this->mode_id_.has_value()) { ESP_LOGCONFIG(TAG, " Mode has datapoint ID %u", *this->mode_id_); - if (this->target_temperature_id_.has_value()) + } + if (this->target_temperature_id_.has_value()) { ESP_LOGCONFIG(TAG, " Target Temperature has datapoint ID %u", *this->target_temperature_id_); - if (this->current_temperature_id_.has_value()) + } + if (this->current_temperature_id_.has_value()) { ESP_LOGCONFIG(TAG, " Current Temperature has datapoint ID %u", *this->current_temperature_id_); + } } } // namespace esphome::tuya diff --git a/esphome/components/udp/udp_component.cpp b/esphome/components/udp/udp_component.cpp index c144212ecf..858516c746 100644 --- a/esphome/components/udp/udp_component.cpp +++ b/esphome/components/udp/udp_component.cpp @@ -129,8 +129,9 @@ void UDPComponent::dump_config() { " Listen Port: %u\n" " Broadcast Port: %u", this->listen_port_, this->broadcast_port_); - for (const char *address : this->addresses_) + for (const char *address : this->addresses_) { ESP_LOGCONFIG(TAG, " Address: %s", address); + } if (this->listen_address_.has_value()) { char addr_buf[network::IP_ADDRESS_BUFFER_SIZE]; ESP_LOGCONFIG(TAG, " Listen address: %s", this->listen_address_.value().str_to(addr_buf)); @@ -145,8 +146,9 @@ void UDPComponent::send_packet(const uint8_t *data, size_t size) { #if defined(USE_SOCKET_IMPL_BSD_SOCKETS) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) for (const auto &saddr : this->sockaddrs_) { auto result = this->broadcast_socket_->sendto(data, size, 0, &saddr, sizeof(saddr)); - if (result < 0) + if (result < 0) { ESP_LOGW(TAG, "sendto() error %d", errno); + } } #endif #ifdef USE_SOCKET_IMPL_LWIP_TCP @@ -155,8 +157,9 @@ void UDPComponent::send_packet(const uint8_t *data, size_t size) { if (this->udp_client_.beginPacketMulticast(saddr, this->broadcast_port_, iface, 128) != 0) { this->udp_client_.write(data, size); auto result = this->udp_client_.endPacket(); - if (result == 0) + if (result == 0) { ESP_LOGW(TAG, "udp.write() error"); + } } } #endif diff --git a/esphome/components/uponor_smatrix/uponor_smatrix.cpp b/esphome/components/uponor_smatrix/uponor_smatrix.cpp index 0ba19f5cd7..c77f3468c7 100644 --- a/esphome/components/uponor_smatrix/uponor_smatrix.cpp +++ b/esphome/components/uponor_smatrix/uponor_smatrix.cpp @@ -110,8 +110,9 @@ bool UponorSmatrixComponent::parse_byte_(uint8_t byte) { // Handle packet size_t data_len = (packet_len - 6) / 3; if (data_len == 0) { - if (packet[4] == UPONOR_ID_REQUEST) + if (packet[4] == UPONOR_ID_REQUEST) { ESP_LOGVV(TAG, "Ignoring request packet for device 0x%08" PRIX32 "", device_address); + } return true; } diff --git a/esphome/components/usb_uart/pl2303.cpp b/esphome/components/usb_uart/pl2303.cpp index a9f7348331..db177fd308 100644 --- a/esphome/components/usb_uart/pl2303.cpp +++ b/esphome/components/usb_uart/pl2303.cpp @@ -194,8 +194,9 @@ std::vector USBUartTypePL2303::parse_descriptors(usb_device_handle_t dev } } - if (cdc_devs.empty()) + if (cdc_devs.empty()) { ESP_LOGE(TAG, "PL2303: failed to find bulk IN+OUT endpoints"); + } return cdc_devs; } diff --git a/esphome/components/wake_on_lan/wake_on_lan.cpp b/esphome/components/wake_on_lan/wake_on_lan.cpp index a514a55d80..e46b96c86a 100644 --- a/esphome/components/wake_on_lan/wake_on_lan.cpp +++ b/esphome/components/wake_on_lan/wake_on_lan.cpp @@ -40,8 +40,9 @@ void WakeOnLanButton::press_action() { memcpy(buffer + i * sizeof(this->macaddr_) + sizeof(PREFIX), this->macaddr_, sizeof(this->macaddr_)); } if (this->broadcast_socket_->sendto(buffer, sizeof(buffer), 0, reinterpret_cast(&saddr), - addr_len) <= 0) + addr_len) <= 0) { ESP_LOGW(TAG, "sendto() error %d", errno); + } #else IPAddress broadcast = IPAddress(255, 255, 255, 255); for (auto ip : esphome::network::get_ip_addresses()) { diff --git a/esphome/components/weikai/weikai.cpp b/esphome/components/weikai/weikai.cpp index 043df86be9..b95d474fd3 100644 --- a/esphome/components/weikai/weikai.cpp +++ b/esphome/components/weikai/weikai.cpp @@ -348,14 +348,18 @@ size_t WeikaiChannel::rx_in_fifo_() { uint8_t const fsr = this->reg(WKREG_FSR); if (fsr & (FSR_RFOE | FSR_RFLB | FSR_RFFE | FSR_RFPE)) { char bin_buf[9]; - if (fsr & FSR_RFOE) + if (fsr & FSR_RFOE) { ESP_LOGE(TAG, "Receive data overflow FSR=%s", format_bin_to(bin_buf, fsr)); - if (fsr & FSR_RFLB) + } + if (fsr & FSR_RFLB) { ESP_LOGE(TAG, "Receive line break FSR=%s", format_bin_to(bin_buf, fsr)); - if (fsr & FSR_RFFE) + } + if (fsr & FSR_RFFE) { ESP_LOGE(TAG, "Receive frame error FSR=%s", format_bin_to(bin_buf, fsr)); - if (fsr & FSR_RFPE) + } + if (fsr & FSR_RFPE) { ESP_LOGE(TAG, "Receive parity error FSR=%s", format_bin_to(bin_buf, fsr)); + } } if ((available == 0) && (fsr & FSR_RFDAT)) { // here we should be very careful because we can have something like this: @@ -495,8 +499,9 @@ void print_buffer(std::vector buffer) { hex_buffer[(3 * 32) + 1] = 0; for (size_t i = 0; i < buffer.size(); i++) { snprintf(&hex_buffer[3 * (i % 32)], sizeof(hex_buffer), "%02X ", buffer[i]); - if (i % 32 == 31) + if (i % 32 == 31) { ESP_LOGI(TAG, " %s", hex_buffer); + } } if (buffer.size() % 32) { // null terminate if incomplete line diff --git a/script/ci-custom.py b/script/ci-custom.py index 724a350884..f481fda860 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -319,6 +319,154 @@ def lint_no_long_delays(fname, match): ) +# An if/else/for/while whose only body is an unbraced ESP_LOG*() call. When the build's compile-time +# log level drops that macro, the body expands to nothing and the compiler warns (-Wempty-body). +# clang-tidy's brace check does not catch these (ShortStatementLines allows short unbraced bodies), so +# this fills that gap. Matched against comment/string-masked content, so commented-out or quoted code +# is ignored. Both spellings are covered: core/log.h defines the uppercase ESP_LOG*() macros and +# the lowercase esph_log_*() ones, and both expand to nothing below their log level. +# 'for' allows ';' inside its parentheses (the classic C-style header); 'if'/'while' do not, so their +# condition cannot run past the statement it guards. The 'for' header permits one level of nested +# parens so it stays bounded to its own statement: without that, it can run past the loop body and +# latch onto a later ')', mis-reporting the line and skipping the '#' preprocessor check below. +ESP_LOG_NEEDS_BRACES_RE = re.compile( + r"(?:\bif\s*\([^{};]*\)|\bwhile\s*\([^{};]*\)|\bfor\s*\((?:[^{}()]|\([^{}()]*\))*\)|\belse\b)" + r"[ \t]*\n?[ \t]*(?:ESP_LOG[A-Z]*|esph_log_[a-z]+)\s*\(", + re.MULTILINE, +) + + +def _mask_cpp_comments_strings(s): + """Return s with // and /* */ comments and string/char/raw-string literals blanked to spaces + (length and newlines preserved) so a regex only matches real code. Parentheses in real code are + kept, so callers can still balance them on the masked text.""" + out = list(s) + i = 0 + n = len(s) + while i < n: + c = s[i] + # Raw string literal: an optional encoding prefix, then R"delim( ... )delim". The body may + # contain quotes, //, /* and unbalanced parens, so it must be consumed as one unit. + if c == "R" and i + 1 < n and s[i + 1] == '"': + j = i + 2 + delim = "" + while j < n and s[j] not in "( \t\r\n\\" and len(delim) < 16: + delim += s[j] + j += 1 + if j < n and s[j] == "(": + closing = ")" + delim + '"' + end = s.find(closing, j + 1) + end = n if end == -1 else end + len(closing) + for k in range(i, end): + if s[k] != "\n": + out[k] = " " + i = end + continue + i += 1 + elif c == "/" and i + 1 < n and s[i + 1] == "/": + while i < n and s[i] != "\n": + out[i] = " " + i += 1 + elif c == "/" and i + 1 < n and s[i + 1] == "*": + out[i] = out[i + 1] = " " + i += 2 + while i < n and not (s[i] == "*" and i + 1 < n and s[i + 1] == "/"): + if s[i] != "\n": + out[i] = " " + i += 1 + if i < n: + out[i] = " " + if i + 1 < n: + out[i + 1] = " " + i += 2 + # A "'" after an alphanumeric or '_' is a C++ digit separator (1'000), not a literal opener. + elif c == '"' or ( + c == "'" and not (i and (s[i - 1].isalnum() or s[i - 1] == "_")) + ): + quote = c + out[i] = " " + i += 1 + while i < n: + if s[i] == "\\": + out[i] = " " + if i + 1 < n: + out[i + 1] = " " + i += 2 + continue + if s[i] == quote: + out[i] = " " + i += 1 + break + if s[i] != "\n": + out[i] = " " + i += 1 + else: + i += 1 + return "".join(out) + + +def _log_statement_end(masked, open_paren): + """Index of the ';' ending the ESP_LOG call whose '(' is at open_paren, or None. Balanced on the + masked text so quotes/comments inside the arguments do not confuse the paren count.""" + depth = 0 + i = open_paren + n = len(masked) + while i < n: + ch = masked[i] + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + j = i + 1 + while j < n and masked[j] != ";": + if not masked[j].isspace(): + return None + j += 1 + return j if j < n else None + i += 1 + return None + + +@lint_content_check(include=cpp_include) +def lint_esp_log_needs_braces(fname, content): + # Cheap bailout: no log call means nothing to flag, and skips masking the file entirely. + if "ESP_LOG" not in content and "esph_log_" not in content: + return [] + masked = _mask_cpp_comments_strings(content) + errors = [] + for match in ESP_LOG_NEEDS_BRACES_RE.finditer(masked): + pos = match.start() + line_start = content.rfind("\n", 0, pos) + 1 + # Skip preprocessor conditionals (#if/#else/#elif): not C++ control statements. + if content[line_start:pos].lstrip().startswith("#"): + continue + # A '// NOLINT' may sit at the end of the log line (where the message says to put it) or on the + # control-statement line, so scan the whole statement rather than only up to the ESP_LOG token. + stmt_end = _log_statement_end(masked, match.end() - 1) + nolint_end = ( + content.find("\n", stmt_end) if stmt_end is not None else match.end() + ) + if nolint_end == -1: + nolint_end = len(content) + if "NOLINT" in content[pos:nolint_end]: + continue + snippet = content[pos : match.end()].replace("\n", " ").strip() + errors.append( + ( + content.count("\n", 0, pos) + 1, + pos - line_start + 1, + ( + f"{highlight(snippet)} - an if/else/for/while body that is a single log " + "call must be wrapped in braces. When the log level compiles the macro out, the " + "body becomes empty and the compiler warns (-Wempty-body). Add { } around the " + "log call (or a '// NOLINT' comment if this is genuinely intended)." + ), + ) + ) + return errors + + @lint_content_check( include=[ "esphome/const.py", diff --git a/tests/script/test_ci_custom.py b/tests/script/test_ci_custom.py new file mode 100644 index 0000000000..d340a816c6 --- /dev/null +++ b/tests/script/test_ci_custom.py @@ -0,0 +1,147 @@ +"""Unit tests for the ESP_LOG-needs-braces lint rule in script/ci-custom.py. + +The rule flags an if/else/for/while whose only body is an unbraced ESP_LOG*() call (which becomes an +empty statement -- and a -Wempty-body warning -- once the log level compiles the macro out). These +tests pin the comment/string/raw-string masker, the accepted control-statement shapes, and the +NOLINT escape hatch at both placements a contributor would try. +""" + +import importlib.util +from pathlib import Path +import sys + +SCRIPT_DIR = (Path(__file__).parent / ".." / ".." / "script").resolve() +sys.path.insert(0, str(SCRIPT_DIR)) +_spec = importlib.util.spec_from_file_location("ci_custom", SCRIPT_DIR / "ci-custom.py") +ci_custom = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(ci_custom) + +mask = ci_custom._mask_cpp_comments_strings + + +def _lint(content: str) -> list: + return ci_custom.lint_esp_log_needs_braces("test.cpp", content) + + +# --- masker --- + + +def test_mask_preserves_length_newlines_and_real_parens() -> None: + src = 'foo("bar") + baz();\nqux();\n' + masked = mask(src) + assert len(masked) == len(src) + assert masked.count("\n") == src.count("\n") + assert masked.count("(") == src.count("(") # real parens survive for balancing + + +def test_mask_blanks_line_and_block_comments() -> None: + assert "ESP_LOGD" not in mask("a; // if (x) ESP_LOGD(t);\n") + assert "ESP_LOGD" not in mask("a; /* if (x) ESP_LOGD(t); */ b;\n") + + +def test_mask_blanks_string_literals() -> None: + assert "if" not in mask('x = "if (y) ESP_LOGD";\n') + + +def test_mask_handles_raw_string_without_desync() -> None: + # A raw string full of quotes/parens must be consumed as one unit; code after it stays intact. + src = 's.print(R"()");\nreturn;\n' + masked = mask(src) + assert "href" not in masked + assert "return;" in masked # not swallowed by a desynced string scan + + +# --- rule: flags real violations --- + + +def test_flags_unbraced_if_next_line() -> None: + assert _lint("if (x)\n ESP_LOGD(t);\n") + + +def test_flags_unbraced_same_line() -> None: + assert _lint("if (x) ESP_LOGW(t);\n") + + +def test_flags_c_style_for() -> None: + assert _lint("for (int i = 0; i < n; i++)\n ESP_LOGD(t, i);\n") + + +def test_flags_range_for_and_else() -> None: + assert _lint("for (auto &x : v)\n ESP_LOGCONFIG(t);\n") + assert _lint("else\n ESP_LOGE(t);\n") + + +def test_flags_for_header_with_nested_call() -> None: + assert _lint("for (auto it = v.begin(); it != v.end(); ++it)\n ESP_LOGD(t);\n") + + +def test_for_header_does_not_reach_into_a_later_statement() -> None: + # The 'for' header is bounded to its own statement, so it cannot swallow the loop body and latch + # onto a later ')'. Without that, the '#if' line below is reported as an unbraced body even though + # the '#' preprocessor check should skip it. + assert not _lint( + "for (int i = 0; i < n; i++)\n arr[i] = 0;\n#if defined(USE_X)\n ESP_LOGD(t);\n#endif\n" + ) + + +def test_violation_after_a_for_loop_is_reported_at_its_own_line() -> None: + errors = _lint( + "for (int i = 0; i < n; i++)\n sum += a[i];\nif (verbose)\n ESP_LOGD(t, sum);\n" + ) + lines = [line for line, _col, _msg in errors] + assert lines == [3] # the 'if', not the 'for' on line 1 + + +def test_flags_lowercase_esph_log_family() -> None: + # core/log.h defines esph_log_*() alongside ESP_LOG*(); both expand to nothing below their level. + assert _lint('if (x)\n esph_log_config(t, "m");\n') + assert _lint('if (err != ESP_OK)\n esph_log_e(t, "m");\n') + + +def test_digit_separator_does_not_disable_the_rest_of_the_file() -> None: + # A "'" digit separator must not be read as a char-literal opener, which blanked everything after. + assert _lint("uint32_t x = 1'000;\nif (y)\n ESP_LOGD(t);\n") + + +def test_mask_still_blanks_real_char_literals() -> None: + assert "ESP_LOGD" not in mask("char c = '\"'; // if (x) ESP_LOGD(t);\n") + assert not _lint("char sep = ';';\nif (x) {\n ESP_LOGD(t);\n}\n") + + +def test_flags_multiline_log_body() -> None: + assert _lint('if (x)\n ESP_LOGD(t, "%d %d",\n a, b);\n') + + +def test_raw_string_before_violation_still_caught() -> None: + # Regression for the masker desyncing on a raw string and disabling the check for the rest. + assert _lint('s.print(R"()");\nif (y)\n ESP_LOGD(t);\n') + + +# --- rule: ignores non-violations --- + + +def test_ignores_braced_body() -> None: + assert not _lint("if (x) {\n ESP_LOGD(t);\n}\n") + + +def test_ignores_commented_out_code() -> None: + assert not _lint("// if (x) ESP_LOGD(t);\n") + + +def test_ignores_preprocessor_else() -> None: + assert not _lint("#else\n ESP_LOGCONFIG(t);\n#endif\n") + + +def test_ignores_non_log_body() -> None: + assert not _lint("if (x)\n return false;\n") + + +# --- NOLINT escape hatch, both placements --- + + +def test_nolint_at_end_of_log_line_suppresses() -> None: + assert not _lint("if (x)\n ESP_LOGD(t); // NOLINT\n") + + +def test_nolint_on_control_line_suppresses() -> None: + assert not _lint("if (x) // NOLINT\n ESP_LOGD(t);\n") From 0f982f03b2e2085fab26f27e7310b62a7c924578 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 31 Aug 2026 19:55:53 -0400 Subject: [PATCH 15/25] [core] Prefetch tool-scons by PlatformIO's core spec (#18831) --- esphome/platformio/prefetch.py | 18 +++++++++--------- tests/unit_tests/test_platformio_prefetch.py | 16 ++++++++++------ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index 1df0a4b328..5097239065 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -832,16 +832,16 @@ def _prefetch(build_dir: Path, env: str) -> None: for name, opts in p.packages.items() if not opts.get("optional") ] - # PIO's build engine installs outside the platform package list; - # skipped when the platform lists it itself - if not any(s.name == "tool-scons" for s in specs): - specs.append( - PackageSpec( - owner="platformio", - name="tool-scons", - requirements=get_core_dependencies()["tool-scons"], - ) + # PIO's build engine installs tool-scons by its own registry spec at build + # start; a platform URL copy has no owner to match it, so prefetch that spec + specs = [s for s in specs if s.name != "tool-scons"] + specs.append( + PackageSpec( + owner="platformio", + name="tool-scons", + requirements=get_core_dependencies()["tool-scons"], ) + ) lib_deps = config.get(f"env:{env}", "lib_deps", []) # pio run's storage dir for this env, with its compatibility # qualifiers: an unqualified library install could land a different diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index d0785d2724..379ef52ebd 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -13,6 +13,7 @@ from types import SimpleNamespace from unittest.mock import MagicMock, patch from filelock import Timeout +from platformio.dependencies import get_core_dependencies from platformio.package.manager._install import PackageManagerInstallMixin from platformio.package.manager.base import BasePackageManager from platformio.package.manager.library import LibraryPackageManager @@ -1728,30 +1729,31 @@ def test_preinstall_unlocks_even_when_pool_fails(tmp_path: Path) -> None: m.unlock.assert_called_once_with() -def test_prefetch_skips_duplicate_tool_scons(tmp_path: Path) -> None: - """A platform that lists tool-scons itself does not get it appended.""" +def test_prefetch_replaces_platform_tool_scons_with_core_spec(tmp_path: Path) -> None: + """A platform's own tool-scons spec gives way to the core's registry spec.""" _write_ini(tmp_path, "[env:testenv]\nplatform = fake/p@1\n") fake_platform = MagicMock() fake_platform.packages = {"tool-scons": {"optional": False}} fake_platform.get_package_spec.side_effect = lambda name: _FakeSpec( - uri=None, name=name + uri="https://x/scons.zip", name=name, owner=None ) config = _fake_config(tmp_path, {"platform": "fake/p@1"}) modules = _pio_modules(tmp_path, fake_platform, MagicMock(), config) - batches: list[list[str]] = [] + batches: list[list] = [] with ( patch.dict("sys.modules", modules), patch.object( pf, "_registry_jobs", side_effect=lambda mgr, specs, seen: ( - batches.append([s.name for s in specs]) or ([], 0, []) + batches.append(list(specs)) or ([], 0, []) ), ), patch.object(pf, "_uri_jobs", return_value=([], 0, [])), ): pf._prefetch(tmp_path, "testenv") - assert batches[0] == ["tool-scons"] + (spec,) = batches[0] + assert (spec.name, spec.owner, spec.uri) == ("tool-scons", "platformio", None) def test_platformio_private_api_contract() -> None: @@ -1784,6 +1786,8 @@ def test_platformio_private_api_contract() -> None: assert callable(getattr(BasePackageManager, name)) # The dependency wave mirrors install_dependency's builtin skip assert callable(LibraryPackageManager.is_builtin_lib) + # The prefetch keys tool-scons on this core dependency + assert "tool-scons" in get_core_dependencies() # The pre-install passes these positionally / by keyword assert "compatibility" in inspect.signature(BasePackageManager.__init__).parameters lib_params = inspect.signature(LibraryPackageManager.__init__).parameters From 5dbc8ffe4c249fa9353b4cbe70c1e1ab01ad9377 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:17:52 +1200 Subject: [PATCH 16/25] [epaper_spi] Add UC8179 mono driver and Seeed reTerminal E1001 model (#17568) --- .../epaper_spi/epaper_spi_uc8179.cpp | 139 ++++++++++++++++++ .../components/epaper_spi/epaper_spi_uc8179.h | 52 +++++++ .../components/epaper_spi/models/uc8179.py | 93 ++++++++++++ .../epaper_spi/config/uc8179_e1001_test.yaml | 15 ++ tests/component_tests/epaper_spi/test_init.py | 17 +++ .../epaper_spi/test.esp32-s3-idf.yaml | 42 ++++++ 6 files changed, 358 insertions(+) create mode 100644 esphome/components/epaper_spi/epaper_spi_uc8179.cpp create mode 100644 esphome/components/epaper_spi/epaper_spi_uc8179.h create mode 100644 esphome/components/epaper_spi/models/uc8179.py create mode 100644 tests/component_tests/epaper_spi/config/uc8179_e1001_test.yaml diff --git a/esphome/components/epaper_spi/epaper_spi_uc8179.cpp b/esphome/components/epaper_spi/epaper_spi_uc8179.cpp new file mode 100644 index 0000000000..2a4ff2969a --- /dev/null +++ b/esphome/components/epaper_spi/epaper_spi_uc8179.cpp @@ -0,0 +1,139 @@ +#include "epaper_spi_uc8179.h" + +#include + +#include "esphome/core/log.h" + +namespace esphome::epaper_spi { + +static constexpr const char *const TAG = "epaper_spi.uc8179"; + +bool EPaperUC8179::initialise(bool partial) { + EPaperBase::initialise(partial); // send the model init sequence + this->partial_ = partial; + ESP_LOGV(TAG, "Power on"); + // POWER ON must precede the waveform/mode registers and the data transfer + // (the original driver powers on and busy-waits before writing them). + // The state machine busy-waits before entering TRANSFER_DATA. + this->command(0x04); + // Give the busy line time to assert before the state machine polls it + this->next_delay_ = 100; + return true; +} + +// Set up the refresh mode. Must be called after power-on has completed. +void EPaperUC8179::set_refresh_mode_() { + if (!this->is_using_partial_update_()) { + return; // plain full refresh uses the mode set by the init sequence + } + // Fast and partial refresh use flipped data polarity and a floating border + this->cmd_data(0x50, {0xA9, 0x07}); + // Force the waveform via the temperature registers: 0x5A selects the fast + // full-refresh waveform, 0x6E the partial-refresh waveform + this->cmd_data(0xE0, {0x02}); + if (this->partial_) { + this->cmd_data(0xE5, {0x6E}); + this->command(0x91); // enter partial mode + // Set the partial window to the full screen + const uint16_t x_end = this->width_ - 1; + const uint16_t y_end = this->height_ - 1; + this->cmd_data(0x90, {0x00, 0x00, static_cast(x_end >> 8), static_cast(x_end & 0xFF), 0x00, 0x00, + static_cast(y_end >> 8), static_cast(y_end & 0xFF), 0x01}); + } else { + this->cmd_data(0xE5, {0x5A}); + this->command(0x92); // exit partial mode + } +} + +bool HOT EPaperUC8179::transfer_data() { + const uint32_t start_time = millis(); + const size_t buffer_length = this->buffer_length_; + if (this->current_data_index_ == 0) { + this->set_refresh_mode_(); + } + // Fast full refresh sends the previous-image plane as well, so that every pixel transitions + const bool two_pass = this->is_using_partial_update_() && !this->partial_; + // Plain full refresh sends inverted data (buffer is 1=white, the wire wants 0=white); + // in fast/partial mode the data polarity is flipped via the VCOM/data-interval + // register instead, so the new-image plane is sent unmodified + const bool invert_new_data = !this->is_using_partial_update_(); + + uint8_t bytes_to_send[MAX_TRANSFER_SIZE]; + + // Phase 1 (fast full refresh only): previous image via 0x10 (DTM1), inverse of the new image + if (two_pass && this->current_data_index_ < buffer_length) { + if (this->current_data_index_ == 0) { + this->command(0x10); // DATA START TRANSMISSION 1 (previous image) + } + this->start_data_(); + while (this->current_data_index_ < buffer_length) { + const size_t bytes_to_copy = std::min(MAX_TRANSFER_SIZE, buffer_length - this->current_data_index_); + for (size_t i = 0; i < bytes_to_copy; i++) { + bytes_to_send[i] = ~this->buffer_[this->current_data_index_ + i]; + } + this->write_array(bytes_to_send, bytes_to_copy); + this->current_data_index_ += bytes_to_copy; + if (millis() - start_time > MAX_TRANSFER_TIME) { + this->disable(); + return false; + } + } + this->disable(); + } + + // Phase 2: new image via 0x13 (DTM2) + const size_t offset = two_pass ? buffer_length : 0; + const size_t total = offset + buffer_length; + if (this->current_data_index_ < total) { + if (this->current_data_index_ == offset) { + this->command(0x13); // DATA START TRANSMISSION 2 (new image) + } + this->start_data_(); + while (this->current_data_index_ < total) { + const size_t bytes_to_copy = std::min(MAX_TRANSFER_SIZE, total - this->current_data_index_); + const size_t data_idx = this->current_data_index_ - offset; + for (size_t i = 0; i < bytes_to_copy; i++) { + const uint8_t byte = this->buffer_[data_idx + i]; + bytes_to_send[i] = invert_new_data ? ~byte : byte; + } + this->write_array(bytes_to_send, bytes_to_copy); + this->current_data_index_ += bytes_to_copy; + if (millis() - start_time > MAX_TRANSFER_TIME) { + this->disable(); + return false; + } + } + this->disable(); + } + + this->current_data_index_ = 0; + return true; +} + +void EPaperUC8179::power_on() { + // Power-on is sent at the end of initialise() instead, because the + // waveform/mode registers and the data transfer must follow it +} + +void EPaperUC8179::refresh_screen(bool /*partial*/) { + ESP_LOGV(TAG, "Refresh"); + this->command(0x12); // DISPLAY REFRESH + // Delay the next busy poll: the busy line takes a short time to assert after + // the refresh command, and polling too early would read it as already idle + this->next_delay_ = 100; +} + +void EPaperUC8179::power_off() { + ESP_LOGV(TAG, "Power off"); + this->command(0x02); // POWER OFF +} + +void EPaperUC8179::deep_sleep() { + // Deep sleep loses the previous-image RAM that partial refresh compares against + if (!this->is_using_partial_update_()) { + ESP_LOGV(TAG, "Deep sleep"); + this->cmd_data(0x07, {0xA5}); // DEEP SLEEP with check code + } +} + +} // namespace esphome::epaper_spi diff --git a/esphome/components/epaper_spi/epaper_spi_uc8179.h b/esphome/components/epaper_spi/epaper_spi_uc8179.h new file mode 100644 index 0000000000..85c0eb623e --- /dev/null +++ b/esphome/components/epaper_spi/epaper_spi_uc8179.h @@ -0,0 +1,52 @@ +#pragma once + +#include "epaper_spi.h" + +namespace esphome::epaper_spi { + +/** + * Monochrome e-paper displays using the UC8179 controller. + * Supports: 7.5" V2 (EPD_7in5_V2), 800x480 pixels, as used by the + * Waveshare 7.5" V2 HAT and the Seeed reTerminal E1001. + * + * Buffer layout: 1 bit per pixel, 1=white, 0=black (the base class default). + * + * The INITIALISE state sends the panel configuration followed by power-on + * (0x04); the state machine busy-waits for power-on to complete before + * TRANSFER_DATA, which first writes the waveform/mode registers (these are + * only accepted while powered) and then the image data. The state machine + * busy-waits again before triggering REFRESH_SCREEN (0x12). + * + * Three refresh modes are used, following the Waveshare EPD_7in5_V2 examples: + * - full_update_every == 1: plain full refresh. The new image is sent + * inverted to DTM2 (0x13) and the controller uses its normal waveform. + * - full_update_every > 1, full update: fast full refresh. The data polarity + * is flipped via the VCOM/data-interval register, a fast waveform is forced + * via the temperature registers, and the image is sent to both DTM1 (0x10, + * inverted) and DTM2 (0x13) so that every pixel transitions. + * - full_update_every > 1, partial update: partial refresh. A partial-update + * waveform is forced, partial mode is entered with a full-screen window and + * only DTM2 is sent; the controller compares against its previous-image RAM. + */ +class EPaperUC8179 final : public EPaperBase { + public: + EPaperUC8179(const char *name, uint16_t width, uint16_t height, const uint8_t *init_sequence, + size_t init_sequence_length) + : EPaperBase(name, width, height, init_sequence, init_sequence_length, DISPLAY_TYPE_BINARY) { + this->buffer_length_ = this->row_width_ * height; + } + + protected: + bool initialise(bool partial) override; + bool transfer_data() override; + void refresh_screen(bool partial) override; + void power_on() override; + void power_off() override; + void deep_sleep() override; + void set_refresh_mode_(); + + // Set by initialise() so transfer_data() knows which planes to send + bool partial_{}; +}; + +} // namespace esphome::epaper_spi diff --git a/esphome/components/epaper_spi/models/uc8179.py b/esphome/components/epaper_spi/models/uc8179.py new file mode 100644 index 0000000000..bea133c328 --- /dev/null +++ b/esphome/components/epaper_spi/models/uc8179.py @@ -0,0 +1,93 @@ +"""Monochrome e-paper displays using the UC8179 controller. + +Supported models: +- waveshare-7.5in-v2: 7.5" mono display, 800x480 pixels (EPD_7in5_V2) +- seeed-reterminal-e1001: Seeed reTerminal E1001, which uses the same + 7.5" 800x480 panel on an integrated ESP32-S3 board + +Panel configuration and power-on (0x04) are both sent during the INITIALISE +state; the state machine's built-in busy wait then covers the power-on delay +before the waveform/mode registers and image data are transferred. + +These displays support fast full and partial refresh: set ``full_update_every`` +greater than 1 to enable it. Every ``full_update_every``-th update is a fast +full refresh, with partial refreshes in between. +""" + +from typing import Any + +from esphome.const import CONF_DATA_RATE + +from . import EpaperModel + + +class UC8179(EpaperModel): + """EpaperModel class for monochrome displays using the UC8179 controller.""" + + def __init__( + self, + name: str, + class_name: str = "EPaperUC8179", + data_rate: str = "10MHz", + **defaults: Any, + ) -> None: + defaults.setdefault(CONF_DATA_RATE, data_rate) + super().__init__(name, class_name, **defaults) + + def get_init_sequence(self, config: dict) -> tuple: + """Generate the initialization sequence for UC8179 mono displays. + + Panel configuration only — the driver appends power-on (0x04) at the + end of the INITIALISE state, and the state machine busy-waits for it + to complete before the data transfer starts. + """ + width, height = self.get_dimensions(config) + return ( + # POWER SETTING + (0x01, 0x07, 0x07, 0x3F, 0x3F), + # BOOSTER SOFT START + (0x06, 0x17, 0x17, 0x28, 0x17), + # PANEL SETTING (black/white mode, LUT from OTP) + (0x00, 0x1F), + # RESOLUTION SETTING (width x height) + ( + 0x61, + (width >> 8) & 0xFF, + width & 0xFF, + (height >> 8) & 0xFF, + height & 0xFF, + ), + # DUAL SPI MODE (disabled) + (0x15, 0x00), + # VCOM AND DATA INTERVAL SETTING + (0x50, 0x10, 0x07), + # TCON SETTING + (0x60, 0x22), + ) + + +uc8179 = UC8179("uc8179") + +# Waveshare 7.5" V2 mono (EPD_7in5_V2) — 800x480, UC8179 controller +waveshare_7_5_v2 = uc8179.extend( + "waveshare-7.5in-v2", + width=800, + height=480, +) + +# Seeed reTerminal E1001 — 7.5" mono e-paper (800x480), same panel as the +# Waveshare 7.5" V2, driven by an integrated ESP32-S3 board +waveshare_7_5_v2.extend( + "seeed-reterminal-e1001", + cs_pin=10, + dc_pin=11, + reset_pin=12, + busy_pin={ + "number": 13, + "inverted": True, + "mode": { + "input": True, + "pullup": True, + }, + }, +) diff --git a/tests/component_tests/epaper_spi/config/uc8179_e1001_test.yaml b/tests/component_tests/epaper_spi/config/uc8179_e1001_test.yaml new file mode 100644 index 0000000000..73f956c8ee --- /dev/null +++ b/tests/component_tests/epaper_spi/config/uc8179_e1001_test.yaml @@ -0,0 +1,15 @@ +esphome: + name: test + +esp32: + board: esp32-s3-devkitc-1 + variant: esp32s3 + +spi: + clk_pin: GPIO7 + mosi_pin: GPIO9 + +display: + - platform: epaper_spi + id: epaper_display + model: seeed-reterminal-e1001 diff --git a/tests/component_tests/epaper_spi/test_init.py b/tests/component_tests/epaper_spi/test_init.py index 7a0507542e..5e2e7d6013 100644 --- a/tests/component_tests/epaper_spi/test_init.py +++ b/tests/component_tests/epaper_spi/test_init.py @@ -439,6 +439,23 @@ def test_enable_pin_multiple( assert all(pin["mode"]["output"] is True for pin in enable_pins) +def test_uc8179_e1001_code_generation( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Test that the reTerminal E1001 model generates the UC8179 driver and init sequence.""" + main_cpp = generate_main(component_config_path("uc8179_e1001_test.yaml")) + + # The model must instantiate the UC8179 driver class with the panel dimensions + assert "epaper_spi::EPaperUC8179" in main_cpp + assert re.search(r'"SEEED-RETERMINAL-E1001",\s*800,\s*480', main_cpp) + + # The generated init sequence must contain the UC8179 resolution setting + # for 800x480: command 0x61, 4 data bytes 0x03 0x20 0x01 0xE0 + # (rendered as decimal in the generated array) + assert "97, 4, 3, 32, 1, 224" in main_cpp + + def test_enable_pin_code_generation( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], diff --git a/tests/components/epaper_spi/test.esp32-s3-idf.yaml b/tests/components/epaper_spi/test.esp32-s3-idf.yaml index 9cca528744..602aeb8d0e 100644 --- a/tests/components/epaper_spi/test.esp32-s3-idf.yaml +++ b/tests/components/epaper_spi/test.esp32-s3-idf.yaml @@ -255,3 +255,45 @@ display: it.filled_rectangle(0, 0, it.get_width(), it.get_height(), Color::WHITE); it.circle(it.get_width() / 2, it.get_height() / 2, 100, Color::BLACK); it.circle(it.get_width() / 2, it.get_height() / 2, 60, Color(255, 0, 0)); + + # Waveshare 7.5" V2 mono (800x480, UC8179 controller, EPD_7in5_V2) + # full_update_every > 1 exercises the fast/partial refresh paths + - platform: epaper_spi + spi_id: spi_bus + model: waveshare-7.5in-v2 + full_update_every: 4 + cs_pin: + allow_other_uses: true + number: GPIO5 + dc_pin: + allow_other_uses: true + number: GPIO17 + reset_pin: + allow_other_uses: true + number: GPIO16 + busy_pin: + allow_other_uses: true + number: GPIO4 + inverted: true + lambda: |- + it.filled_rectangle(0, 0, it.get_width(), it.get_height(), Color::WHITE); + it.circle(it.get_width() / 2, it.get_height() / 2, 100, Color::BLACK); + + # Seeed reTerminal E1001 - 7.5" mono e-paper (800x480, UC8179) + # Pins overridden to avoid conflicts with the E1002 defaults above + - platform: epaper_spi + spi_id: spi_bus + model: seeed-reterminal-e1001 + cs_pin: + allow_other_uses: true + number: GPIO5 + dc_pin: + allow_other_uses: true + number: GPIO17 + reset_pin: + allow_other_uses: true + number: GPIO16 + busy_pin: + allow_other_uses: true + number: GPIO4 + inverted: true From d6758377d14a8ab63781a4c035162166d4889a12 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 31 Aug 2026 22:25:58 -0400 Subject: [PATCH 17/25] [core] Clone git libraries in parallel in the library prefetch (#18836) --- esphome/git.py | 9 ++ esphome/platformio/library.py | 103 +++++++++++++++----- tests/unit_tests/test_git.py | 19 ++++ tests/unit_tests/test_platformio_library.py | 81 ++++++++++++++- 4 files changed, 184 insertions(+), 28 deletions(-) diff --git a/esphome/git.py b/esphome/git.py index 9815377f51..14145a639b 100644 --- a/esphome/git.py +++ b/esphome/git.py @@ -457,6 +457,15 @@ def _clone_complete_marker_path(repo_dir: Path) -> Path: return repo_dir / ".git" / _CLONE_COMPLETE_MARKER +def has_complete_clone( + url: str, ref: str | None, domain: str, subpath: Path | None = None +) -> bool: + """Lock-free probe for a complete clone; can go stale immediately, so + best-effort decisions only, never a substitute for ``clone_or_update``.""" + repo_dir = _repo_entry_dir(_cache_key(url, ref), domain, subpath) + return _clone_complete_marker_path(repo_dir).is_file() + + def _clear_clone_complete_marker(repo_dir: Path) -> None: """Best-effort removal of the completion marker. diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index 0402311a9a..3ff60f8aaa 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -13,7 +13,7 @@ regardless of which toolchain consumes the result. """ from collections import deque -from collections.abc import Callable, Iterable +from collections.abc import Callable, Hashable, Iterable from dataclasses import dataclass, field from functools import partial import glob @@ -99,6 +99,17 @@ class Source: ) -> Path: raise NotImplementedError + def prefetch_key(self, dir_suffix: str) -> Hashable | None: + """Prefetch dedup identity; None = not prefetchable. Sources that + could write one cache dir must return equal keys (workers must never + share a dir); a coarser key only skips a prefetch.""" + return None + + def is_cached(self, dir_suffix: str, salt: str = "", namespace: str = "") -> bool: + """Whether a completed fetch exists; only consulted when + ``prefetch_key()`` is not None, True is the safe default.""" + return True + def source_root(self, build_path: Path) -> Path: """Directory holding the library's own files (manifest + sources). @@ -127,6 +138,9 @@ class URLSource(Source): h.update(salt.encode()) return base_dir / h.hexdigest()[:8] / dir_suffix + def prefetch_key(self, dir_suffix: str) -> Hashable | None: + return self.url if self.size else None + def is_cached(self, dir_suffix: str, salt: str = "", namespace: str = "") -> bool: """Whether a completed extraction already exists for this source.""" return ( @@ -177,14 +191,29 @@ class GitSource(Source): self.url = url self.ref = ref - def download( - self, dir_suffix: str, force: bool = False, salt: str = "", namespace: str = "" - ) -> Path: + @staticmethod + def _domain(salt: str, namespace: str) -> str: domain = DOMAIN if namespace: domain = f"{domain}/{namespace}" if salt: domain = f"{domain}/{salt}" + return domain + + def prefetch_key(self, dir_suffix: str) -> Hashable | None: + # The clone target dir is hash(url@ref)/ + return (self.url, self.ref, dir_suffix) + + def is_cached(self, dir_suffix: str, salt: str = "", namespace: str = "") -> bool: + """Whether a completed clone already exists for this source.""" + return git.has_complete_clone( + self.url, self.ref, self._domain(salt, namespace), Path(dir_suffix) + ) + + def download( + self, dir_suffix: str, force: bool = False, salt: str = "", namespace: str = "" + ) -> Path: + domain = self._domain(salt, namespace) path, _ = git.clone_or_update( url=self.url, ref=self.ref, @@ -988,56 +1017,78 @@ def _fetch_source( ) +def _clone_source( + component: ConvertedLibrary, + salt: str, + namespace: str, + tracker: Callable[[int], None], +) -> None: + # No byte progress from git; one tick so a cancelled batch stops here + tracker(0) + component.source.download( + component.get_sanitized_name(), salt=salt, namespace=namespace + ) + + def _prefetch_wave( wave: list[tuple[str, ConvertedLibrary]], salt: str, namespace: str ) -> None: - """Best-effort parallel download of a wave's registry archives. + """Best-effort parallel fetch of a wave's registry archives and git clones. - The walk's own ``download()`` stays authoritative; duplicate URLs + The walk's own ``download()`` stays authoritative; duplicate sources prefetch once so two threads never share a cache directory. Archives whose size the registry did not report are left to the sequential loop, whose per-file bars don't interleave. A node a sibling in the - same wave supersedes has its archive fetched in vain (knowing better + same wave supersedes has its source fetched in vain (knowing better would need the manifests being downloaded). """ try: - components: list[ConvertedLibrary] = [] - seen: set[str] = set() + archives: list[ConvertedLibrary] = [] + clones: list[ConvertedLibrary] = [] + seen: set[Hashable] = set() for _key, component in wave: source = component.source - if not isinstance(source, URLSource) or not source.size: + name = component.get_sanitized_name() + dedup_key = source.prefetch_key(name) + if dedup_key is None or dedup_key in seen: continue - if source.url in seen: - continue - seen.add(source.url) + seen.add(dedup_key) try: - cached = source.is_cached( - component.get_sanitized_name(), salt=salt, namespace=namespace - ) + cached = source.is_cached(name, salt=salt, namespace=namespace) except OSError as err: # Best-effort, but visibly: a systematic probe failure makes - # every warm build re-download every archive + # every warm build re-fetch every source _LOGGER.warning("Cache probe for %s failed: %s", component.name, err) cached = False if cached: # A warm build must stay silent continue - components.append(component) - if not components: + (archives if isinstance(source, URLSource) else clones).append(component) + if not archives and not clones: return # Single-item waves (a dependency chain discovers one archive per # wave) go through the same runner: one download method, one bar - _LOGGER.info( - "Downloading %d library archive(s): %s", - len(components), - ", ".join(c.name for c in components), - ) + if archives: + _LOGGER.info( + "Downloading %d library archive(s): %s", + len(archives), + ", ".join(c.name for c in archives), + ) + if clones: + _LOGGER.info( + "Cloning %d library repo(s): %s", + len(clones), + ", ".join(c.name for c in clones), + ) failures = run_batch_downloads( "Downloading libraries", [ (c.name, c.source.size, partial(_fetch_source, c, salt, namespace)) - for c in components - ], + for c in archives + ] + # Size 0: clones share the worker pool without skewing the + # byte bar, whose total stays the archive sum + + [(c.name, 0, partial(_clone_source, c, salt, namespace)) for c in clones], ) # The sequential call below retries and raises the real error warn_prefetch_failures( diff --git a/tests/unit_tests/test_git.py b/tests/unit_tests/test_git.py index e296d48a46..0f7e0339c9 100644 --- a/tests/unit_tests/test_git.py +++ b/tests/unit_tests/test_git.py @@ -714,6 +714,25 @@ def test_run_git_command_without_git_dir_raises_error( git.run_git_command(["git", "clone", "https://invalid.url/repo.git"]) +def test_has_complete_clone(tmp_path: Path) -> None: + """The lock-free probe tracks the completion marker, subpath included.""" + CORE.config_path = tmp_path / "test.yaml" + + url = "https://github.com/test/repo" + subpath = Path("lib") + assert not git.has_complete_clone(url, "v1", "test_domain", subpath) + + repo_dir = _compute_repo_dir(url, "v1", "test_domain") / subpath + (repo_dir / ".git").mkdir(parents=True) + # A directory without the marker is an incomplete clone + assert not git.has_complete_clone(url, "v1", "test_domain", subpath) + + _mark_clone_complete(repo_dir) + assert git.has_complete_clone(url, "v1", "test_domain", subpath) + # The ref is part of the cache key + assert not git.has_complete_clone(url, "v2", "test_domain", subpath) + + def test_clone_or_update_with_never_refresh( tmp_path: Path, mock_run_git_command: Mock ) -> None: diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index 0a16b118fc..3bae39b3c1 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -638,7 +638,7 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel( setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: """Registry archives in one wave download concurrently, deduped by URL; - git/local sources and failures are left to the sequential call.""" + local sources and failures are left to the sequential call.""" calls: list[str] = [] def fake_download( @@ -658,7 +658,7 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel( # into the same cache directory) ("b2", ConvertedLibrary("b2", "1.0", URLSource("https://x/b.tar.gz", 1))), ("c", ConvertedLibrary("c", "1.0", URLSource("https://x/boom.tar.gz", 1))), - ("g", ConvertedLibrary("g", "*", lib.GitSource("https://x/g.git", None))), + ("l", ConvertedLibrary("l", "*", LocalSource("/some/lib"))), ] lib._prefetch_wave(wave, "", "idf") assert sorted(calls) == [ @@ -670,6 +670,83 @@ def test_prefetch_wave_downloads_registry_archives_in_parallel( assert "Prefetch of c failed (retrying sequentially)" in caplog.text +def test_prefetch_wave_clones_git_sources_in_parallel( + setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Git sources join the same prefetch batch as the archives, deduped by + clone target; a clone failure warns and is left to the sequential call.""" + caplog.set_level("INFO") + calls: list[str] = [] + + def fake_clone(self, dir_suffix, force=False, salt="", namespace=""): + calls.append(f"{self}/{dir_suffix}") + if "boom" in self.url: + raise RuntimeError("boom") + + monkeypatch.setattr(GitSource, "download", fake_clone) + wave = [ + ("a", ConvertedLibrary("a", "1.0", URLSource("https://x/a.tar.gz", 1))), + ("g", ConvertedLibrary("g", "*", GitSource("https://x/g.git", "v1"))), + # Same url@ref and target dir must clone once + ("g2", ConvertedLibrary("g", "*", GitSource("https://x/g.git", "v1"))), + ("h", ConvertedLibrary("h", "*", GitSource("https://x/boom.git", None))), + ] + monkeypatch.setattr( + URLSource, "download", lambda self, dir_suffix, progress=None, **kw: None + ) + lib._prefetch_wave(wave, "", "idf") + assert sorted(calls) == ["https://x/boom.git/h", "https://x/g.git#v1/g"] + assert "Cloning 2 library repo(s): g, h" in caplog.text + assert "Prefetch of h failed (retrying sequentially)" in caplog.text + + +def test_source_base_prefetch_defaults() -> None: + """The base Source is not prefetchable and reports cached (nothing to do).""" + source = Source() + assert source.prefetch_key("x") is None + assert source.is_cached("x") is True + + +def test_prefetch_wave_single_clone_uses_the_batch( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """A wave with only git sources still clones through the batch runner.""" + caplog.set_level("INFO") + calls: list[str] = [] + monkeypatch.setattr(GitSource, "is_cached", lambda self, *a, **kw: False) + monkeypatch.setattr( + GitSource, + "download", + lambda self, dir_suffix, force=False, salt="", namespace="": calls.append( + self.url + ), + ) + lib._prefetch_wave( + [("g", ConvertedLibrary("g", "*", GitSource("https://x/g.git", None)))], + "", + "idf", + ) + assert calls == ["https://x/g.git"] + assert "Cloning 1 library repo(s): g" in caplog.text + assert "Downloading" not in caplog.text + + +def test_prefetch_wave_warm_git_cache_is_silent( + setup_core, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """An already-complete clone is neither re-fetched nor announced.""" + caplog.set_level("INFO") + monkeypatch.setattr( + GitSource, + "download", + lambda self, dir_suffix, **kw: (_ for _ in ()).throw(AssertionError("cloned")), + ) + monkeypatch.setattr(GitSource, "is_cached", lambda self, *a, **kw: True) + wave = [("g", ConvertedLibrary("g", "*", GitSource("https://x/g.git", None)))] + lib._prefetch_wave(wave, "", "idf") + assert "Cloning" not in caplog.text + + def test_prefetch_wave_unknown_size_left_to_sequential( setup_core, monkeypatch: pytest.MonkeyPatch ) -> None: From fe3788ff4783ab93192a3c979585bc2a4eac660f Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:56:03 +1000 Subject: [PATCH 18/25] [esp32][mipi_rgb] Add ESP32-S31 support for execute_from_psram (#18929) Co-authored-by: Claude Sonnet 5 Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- esphome/components/esp32/__init__.py | 17 +++++----- esphome/components/mipi_rgb/mipi_rgb.cpp | 7 +--- esphome/components/mipi_rgb/mipi_rgb.h | 9 ++++-- .../esp32/config/execute_from_psram_s31.yaml | 13 ++++++++ tests/component_tests/esp32/test_esp32.py | 32 ++++++++++++++++--- 5 files changed, 58 insertions(+), 20 deletions(-) create mode 100644 tests/component_tests/esp32/config/execute_from_psram_s31.yaml diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index b0290d7a84..3f5a34bc73 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -182,6 +182,13 @@ SIGNED_OTA_V1_ECDSA_VARIANTS = { VARIANT_ESP32, } +# Variants that support execution from PSRAM +PSRAM_XIP_VARIANTS = { + VARIANT_ESP32S3, + VARIANT_ESP32P4, + VARIANT_ESP32S31, +} + # NVS encryption (HMAC peripheral scheme) is only available on variants that # expose the HMAC peripheral (SOC_HMAC_SUPPORTED in soc_caps.h). The original # ESP32 and ESP32-C2 do not have it. New variants with an HMAC peripheral @@ -1523,7 +1530,7 @@ def final_validate(config) -> None: ) ) if advanced[CONF_EXECUTE_FROM_PSRAM]: - if config[CONF_VARIANT] not in {VARIANT_ESP32S3, VARIANT_ESP32P4}: + if config[CONF_VARIANT] not in PSRAM_XIP_VARIANTS: errs.append( cv.Invalid( f"'{CONF_EXECUTE_FROM_PSRAM}' is not available on this esp32 variant", @@ -2727,13 +2734,7 @@ async def to_code(config): _configure_lwip_max_sockets(conf) if advanced[CONF_EXECUTE_FROM_PSRAM]: - if variant == VARIANT_ESP32S3: - add_idf_sdkconfig_option("CONFIG_SPIRAM_FETCH_INSTRUCTIONS", True) - add_idf_sdkconfig_option("CONFIG_SPIRAM_RODATA", True) - elif variant == VARIANT_ESP32P4: - add_idf_sdkconfig_option("CONFIG_SPIRAM_XIP_FROM_PSRAM", True) - else: - raise ValueError("Unhandled ESP32 variant") + add_idf_sdkconfig_option("CONFIG_SPIRAM_XIP_FROM_PSRAM", True) # Apply LWIP core locking for better socket performance # This is already enabled by default in Arduino framework, where it provides diff --git a/esphome/components/mipi_rgb/mipi_rgb.cpp b/esphome/components/mipi_rgb/mipi_rgb.cpp index f43bbab21c..c11044c288 100644 --- a/esphome/components/mipi_rgb/mipi_rgb.cpp +++ b/esphome/components/mipi_rgb/mipi_rgb.cpp @@ -5,7 +5,7 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include -#include +#include #include namespace esphome::mipi_rgb { @@ -177,11 +177,6 @@ void MipiRgb::common_setup_() { ESP_LOGCONFIG(TAG, "MipiRgb setup complete"); } -void MipiRgb::loop() { - if (this->handle_ != nullptr) - esp_lcd_rgb_panel_restart(this->handle_); -} - void MipiRgb::update() { if (this->is_failed()) return; diff --git a/esphome/components/mipi_rgb/mipi_rgb.h b/esphome/components/mipi_rgb/mipi_rgb.h index 87b35781e2..f528943c1b 100644 --- a/esphome/components/mipi_rgb/mipi_rgb.h +++ b/esphome/components/mipi_rgb/mipi_rgb.h @@ -3,7 +3,7 @@ #if defined(USE_ESP32_VARIANT_ESP32S3) || defined(USE_ESP32_VARIANT_ESP32P4) || defined(USE_ESP32_VARIANT_ESP32S31) #include "esphome/core/gpio.h" #include "esphome/components/display/display.h" -#include "esp_lcd_panel_ops.h" +#include #ifdef USE_SPI #include "esphome/components/spi/spi.h" #endif @@ -25,7 +25,12 @@ class MipiRgb : public display::Display { public: MipiRgb(int width, int height) : width_(width), height_(height) {} void setup() override; - void loop() override; +#ifdef USE_ESP32_VARIANT_ESP32S3 + void loop() override { + if (this->handle_ != nullptr) + esp_lcd_rgb_panel_restart(this->handle_); + } +#endif void update() override; void fill(Color color) override; void draw_pixels_at(int x_start, int y_start, int w, int h, const uint8_t *ptr, display::ColorOrder order, diff --git a/tests/component_tests/esp32/config/execute_from_psram_s31.yaml b/tests/component_tests/esp32/config/execute_from_psram_s31.yaml new file mode 100644 index 0000000000..493c9f989e --- /dev/null +++ b/tests/component_tests/esp32/config/execute_from_psram_s31.yaml @@ -0,0 +1,13 @@ +esphome: + name: test + +esp32: + variant: esp32s31 + board: esp32-s31-devkitc + framework: + type: esp-idf + advanced: + execute_from_psram: true + +psram: + mode: octal diff --git a/tests/component_tests/esp32/test_esp32.py b/tests/component_tests/esp32/test_esp32.py index bef273badd..759020c732 100644 --- a/tests/component_tests/esp32/test_esp32.py +++ b/tests/component_tests/esp32/test_esp32.py @@ -203,6 +203,18 @@ def test_esp32_rejects_unsupported_cli_toolchain( r"'execute_from_psram' requires PSRAM to be configured @ data\['framework'\]\['advanced'\]\['execute_from_psram'\]", id="execute_from_psram_requires_psram_p4_config", ), + pytest.param( + { + "variant": "esp32s31", + "board": "esp32-s31-devkitc", + "framework": { + "type": "esp-idf", + "advanced": {"execute_from_psram": True}, + }, + }, + r"'execute_from_psram' requires PSRAM to be configured @ data\['framework'\]\['advanced'\]\['execute_from_psram'\]", + id="execute_from_psram_requires_psram_s31_config", + ), pytest.param( { "variant": "esp32s3", @@ -422,12 +434,12 @@ def test_execute_from_psram_s3_sdkconfig( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], ) -> None: - """Test that execute_from_psram on ESP32-S3 sets the correct sdkconfig options.""" + """Test that execute_from_psram on ESP32-S3 sets the correct sdkconfig option.""" generate_main(component_config_path("execute_from_psram_s3.yaml")) sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] - assert sdkconfig.get("CONFIG_SPIRAM_FETCH_INSTRUCTIONS") is True - assert sdkconfig.get("CONFIG_SPIRAM_RODATA") is True - assert "CONFIG_SPIRAM_XIP_FROM_PSRAM" not in sdkconfig + assert sdkconfig.get("CONFIG_SPIRAM_XIP_FROM_PSRAM") is True + assert "CONFIG_SPIRAM_FETCH_INSTRUCTIONS" not in sdkconfig + assert "CONFIG_SPIRAM_RODATA" not in sdkconfig def test_execute_from_psram_p4_sdkconfig( @@ -442,6 +454,18 @@ def test_execute_from_psram_p4_sdkconfig( assert "CONFIG_SPIRAM_RODATA" not in sdkconfig +def test_execute_from_psram_s31_sdkconfig( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """Test that execute_from_psram on ESP32-S31 sets the correct sdkconfig option.""" + generate_main(component_config_path("execute_from_psram_s31.yaml")) + sdkconfig = CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + assert sdkconfig.get("CONFIG_SPIRAM_XIP_FROM_PSRAM") is True + assert "CONFIG_SPIRAM_FETCH_INSTRUCTIONS" not in sdkconfig + assert "CONFIG_SPIRAM_RODATA" not in sdkconfig + + def test_nvs_encryption_sdkconfig( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], From 68ffd5a77324f43345d5bbcab6db1edc9c42e390 Mon Sep 17 00:00:00 2001 From: Oliver Kleinecke Date: Tue, 1 Sep 2026 15:12:49 +0200 Subject: [PATCH 19/25] [safe_mode] Uncover silent error in safe-mode (#18749) Co-authored-by: Oliver Kleinecke Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- esphome/components/safe_mode/safe_mode.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/esphome/components/safe_mode/safe_mode.cpp b/esphome/components/safe_mode/safe_mode.cpp index ce029b4f55..8fd5911ab5 100644 --- a/esphome/components/safe_mode/safe_mode.cpp +++ b/esphome/components/safe_mode/safe_mode.cpp @@ -255,12 +255,17 @@ bool SafeModeComponent::should_enter_safe_mode(uint8_t num_attempts, uint32_t en } void SafeModeComponent::write_rtc_(uint32_t val) { - this->rtc_.save(&val); - global_preferences->sync(); + if (!this->rtc_.save(&val)) { + ESP_LOGE(TAG, "Failed to set rtc value (%" PRIu32 ")", val); + return; + } + if (!global_preferences->sync()) { + ESP_LOGE(TAG, "Failed to persist rtc value (%" PRIu32 ")", val); + } } uint32_t SafeModeComponent::read_rtc_() { - uint32_t val; + uint32_t val = 0; if (!this->rtc_.load(&val)) return 0; return val; @@ -272,7 +277,9 @@ void SafeModeComponent::clean_rtc() { // before sync, the boot wasn't really successful anyway and the counter should // remain incremented. uint32_t val = 0; - this->rtc_.save(&val); + if (!this->rtc_.save(&val)) { + ESP_LOGE(TAG, "Failed to clear boot loop counter"); + } } void SafeModeComponent::on_safe_shutdown() { From f9824ee83f2c0e866bf6ca76e3fb5a8a10ef5b64 Mon Sep 17 00:00:00 2001 From: Zebble Date: Tue, 1 Sep 2026 14:23:21 -0400 Subject: [PATCH 20/25] [core] Move CONF_KEYS to the shared component constants (#18933) Co-authored-by: Claude --- esphome/components/const/__init__.py | 2 ++ esphome/components/lvgl/widgets/table.py | 3 +-- esphome/components/matrix_keypad/__init__.py | 4 +--- esphome/components/sx1509/__init__.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/esphome/components/const/__init__.py b/esphome/components/const/__init__.py index e445a4abde..49a625e3f1 100644 --- a/esphome/components/const/__init__.py +++ b/esphome/components/const/__init__.py @@ -14,6 +14,7 @@ CONF_CHANNEL_COLORS = "channel_colors" CONF_CLIMATE_ID = "climate_id" CONF_CO2_EQUIVALENT = "co2_equivalent" CONF_COLOR_DEPTH = "color_depth" +CONF_COLUMNS = "columns" CONF_CRC_ENABLE = "crc_enable" CONF_DATA_BITS = "data_bits" CONF_DESCRIPTION = "description" @@ -25,6 +26,7 @@ CONF_GYROSCOPE_RANGE = "gyroscope_range" CONF_IAQ = "iaq" CONF_IGNORE_NOT_FOUND = "ignore_not_found" CONF_IS_WRGB = "is_wrgb" +CONF_KEYS = "keys" CONF_LABEL = "label" CONF_LIBRETINY = "libretiny" CONF_LOOP = "loop" diff --git a/esphome/components/lvgl/widgets/table.py b/esphome/components/lvgl/widgets/table.py index efae2be2be..f000ea1846 100644 --- a/esphome/components/lvgl/widgets/table.py +++ b/esphome/components/lvgl/widgets/table.py @@ -2,7 +2,7 @@ from contextlib import ExitStack from esphome import automation import esphome.codegen as cg -from esphome.components.const import CONF_ROWS +from esphome.components.const import CONF_COLUMNS, CONF_ROWS import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_ITEMS, CONF_ROW, CONF_TEXT, CONF_WIDTH from esphome.core import ID @@ -20,7 +20,6 @@ from .label import CONF_LABEL CONF_TABLE = "table" CONF_CELLS = "cells" -CONF_COLUMNS = "columns" CONF_ROW_COUNT = "row_count" CONF_COLUMN_COUNT = "column_count" CONF_MERGE_RIGHT = "merge_right" diff --git a/esphome/components/matrix_keypad/__init__.py b/esphome/components/matrix_keypad/__init__.py index 47cf4793b1..2e43eaf7e2 100644 --- a/esphome/components/matrix_keypad/__init__.py +++ b/esphome/components/matrix_keypad/__init__.py @@ -1,7 +1,7 @@ from esphome import automation, pins import esphome.codegen as cg from esphome.components import key_provider -from esphome.components.const import CONF_ROWS +from esphome.components.const import CONF_COLUMNS, CONF_KEYS, CONF_ROWS import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_ON_KEY, CONF_PIN, CONF_TRIGGER_ID from esphome.types import ConfigType @@ -21,8 +21,6 @@ MatrixKeyTrigger = matrix_keypad_ns.class_( ) CONF_KEYPAD_ID = "keypad_id" -CONF_COLUMNS = "columns" -CONF_KEYS = "keys" CONF_DEBOUNCE_TIME = "debounce_time" CONF_HAS_DIODES = "has_diodes" CONF_HAS_PULLDOWNS = "has_pulldowns" diff --git a/esphome/components/sx1509/__init__.py b/esphome/components/sx1509/__init__.py index c1e4e11d54..7694b8f732 100644 --- a/esphome/components/sx1509/__init__.py +++ b/esphome/components/sx1509/__init__.py @@ -1,6 +1,7 @@ from esphome import automation, pins import esphome.codegen as cg from esphome.components import i2c, key_provider +from esphome.components.const import CONF_KEYS import esphome.config_validation as cv from esphome.const import ( CONF_ID, @@ -19,7 +20,6 @@ from esphome.cpp_generator import MockObj from esphome.types import ConfigType CONF_KEYPAD = "keypad" -CONF_KEYS = "keys" CONF_KEY_ROWS = "key_rows" CONF_KEY_COLUMNS = "key_columns" CONF_SLEEP_TIME = "sleep_time" From 0aff9e1c54f45dedaf111643c0e157dd87a669b9 Mon Sep 17 00:00:00 2001 From: Andrej Walilko <3455017+ch604@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:33:35 -0400 Subject: [PATCH 21/25] [d01] add D01 pm2.5 sensor support (#17788) Co-authored-by: Andrej Walilko Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- CODEOWNERS | 1 + esphome/components/d01/__init__.py | 0 esphome/components/d01/d01.cpp | 45 ++++++++++++++++++++++ esphome/components/d01/d01.h | 14 +++++++ esphome/components/d01/sensor.py | 45 ++++++++++++++++++++++ tests/components/d01/common.yaml | 3 ++ tests/components/d01/test.esp32-idf.yaml | 7 ++++ tests/components/d01/test.esp8266-ard.yaml | 7 ++++ tests/components/d01/test.rp2040-ard.yaml | 7 ++++ 9 files changed, 129 insertions(+) create mode 100644 esphome/components/d01/__init__.py create mode 100644 esphome/components/d01/d01.cpp create mode 100644 esphome/components/d01/d01.h create mode 100644 esphome/components/d01/sensor.py create mode 100644 tests/components/d01/common.yaml create mode 100644 tests/components/d01/test.esp32-idf.yaml create mode 100644 tests/components/d01/test.esp8266-ard.yaml create mode 100644 tests/components/d01/test.rp2040-ard.yaml diff --git a/CODEOWNERS b/CODEOWNERS index 13fae0664b..7bb7f310a3 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -131,6 +131,7 @@ esphome/components/cst816/* @clydebarrow esphome/components/cst9220/* @clydebarrow esphome/components/ct_clamp/* @jesserockz esphome/components/current_based/* @djwmarcx +esphome/components/d01/* @ch604 esphome/components/dac7678/* @NickB1 esphome/components/daikin_arc/* @MagicBear esphome/components/daikin_brc/* @hagak diff --git a/esphome/components/d01/__init__.py b/esphome/components/d01/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esphome/components/d01/d01.cpp b/esphome/components/d01/d01.cpp new file mode 100644 index 0000000000..f7a0c08ec4 --- /dev/null +++ b/esphome/components/d01/d01.cpp @@ -0,0 +1,45 @@ +#include "d01.h" +#include "esphome/core/log.h" + +// uart specification for d01 sensor from https://manuals.plus/ae/1005006417362019: +// +// A frame of serial output data includes 4 bytes, formatted as follows: +// __Characteristic byte: Fixed value 0xA5. +// __Data byte: DATAH is the high 7 bits of the concentration value, and DATAL is the low 7 bits of the concentration +// value. +// __Check byte: The low 7 bits of the sum of all bytes before the check byte. +// +// If the serial output is 4 bytes of data: 0*A5 0*01 0*2C 0*52, then DATAH = 0*01 = 1, DATAL = 0*2C = 44. +// Concentration value = 1*128 + 44 = 172 µg/m³. +// +// The PM2.5 dust concentration value obtained from the dust sensor needs to be calibrated with a K value coefficient +// based on the TSI instrument's photometric method. It is generally recommended to use 0.4. + +namespace esphome::d01 { + +static const char *const TAG = "d01"; + +static const uint8_t D01_FRAME_HEADER = 0xA5; + +void D01SensorComponent::dump_config() { LOG_SENSOR(" ", "D01 PM2.5", this); } + +void D01SensorComponent::loop() { + uint8_t buf[4]; + while (this->available() >= 4) { + if (this->peek() != D01_FRAME_HEADER) { + this->read(); + continue; + } + this->read_array(buf, 4); + uint8_t sum = (buf[0] + buf[1] + buf[2]) & 0x7F; + if (sum != buf[3]) { + ESP_LOGW(TAG, "checksum mismatch"); + continue; + } + uint16_t latest_concentration = (buf[1] & 0x7F) * 128 + (buf[2] & 0x7F); + ESP_LOGV(TAG, "Unadjusted PM2.5 Concentration: %d µg/m³", latest_concentration); + this->publish_state(latest_concentration); + } +} + +} // namespace esphome::d01 diff --git a/esphome/components/d01/d01.h b/esphome/components/d01/d01.h new file mode 100644 index 0000000000..73c7a8711d --- /dev/null +++ b/esphome/components/d01/d01.h @@ -0,0 +1,14 @@ +#pragma once +#include "esphome/core/component.h" +#include "esphome/components/sensor/sensor.h" +#include "esphome/components/uart/uart.h" + +namespace esphome::d01 { + +class D01SensorComponent final : public sensor::Sensor, public Component, public uart::UARTDevice { + public: + void dump_config() override; + void loop() override; +}; + +} // namespace esphome::d01 diff --git a/esphome/components/d01/sensor.py b/esphome/components/d01/sensor.py new file mode 100644 index 0000000000..5bc5a4e424 --- /dev/null +++ b/esphome/components/d01/sensor.py @@ -0,0 +1,45 @@ +import esphome.codegen as cg +from esphome.components import sensor, uart +import esphome.config_validation as cv +from esphome.const import ( + DEVICE_CLASS_PM25, + ICON_BLUR, + STATE_CLASS_MEASUREMENT, + UNIT_MICROGRAMS_PER_CUBIC_METER, +) +from esphome.types import ConfigType + +CODEOWNERS = ["@ch604"] +DEPENDENCIES = ["uart"] + +d01_ns = cg.esphome_ns.namespace("d01") +D01SensorComponent = d01_ns.class_( + "D01SensorComponent", sensor.Sensor, uart.UARTDevice, cg.Component +) + + +CONFIG_SCHEMA = ( + sensor.sensor_schema( + D01SensorComponent, + unit_of_measurement=UNIT_MICROGRAMS_PER_CUBIC_METER, + icon=ICON_BLUR, + accuracy_decimals=0, + device_class=DEVICE_CLASS_PM25, + state_class=STATE_CLASS_MEASUREMENT, + ) + .extend(cv.COMPONENT_SCHEMA) + .extend(uart.UART_DEVICE_SCHEMA) +) + +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "d01", + baud_rate=9600, + require_rx=True, + require_tx=False, +) + + +async def to_code(config: ConfigType) -> None: + var = await sensor.new_sensor(config) + await cg.register_component(var, config) + await uart.register_uart_device(var, config) diff --git a/tests/components/d01/common.yaml b/tests/components/d01/common.yaml new file mode 100644 index 0000000000..b59ec06ff0 --- /dev/null +++ b/tests/components/d01/common.yaml @@ -0,0 +1,3 @@ +sensor: + - platform: d01 + name: D01 PM2.5 Concentration diff --git a/tests/components/d01/test.esp32-idf.yaml b/tests/components/d01/test.esp32-idf.yaml new file mode 100644 index 0000000000..b658bfbede --- /dev/null +++ b/tests/components/d01/test.esp32-idf.yaml @@ -0,0 +1,7 @@ +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +packages: + uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + d01: !include common.yaml diff --git a/tests/components/d01/test.esp8266-ard.yaml b/tests/components/d01/test.esp8266-ard.yaml new file mode 100644 index 0000000000..876615ae9f --- /dev/null +++ b/tests/components/d01/test.esp8266-ard.yaml @@ -0,0 +1,7 @@ +substitutions: + tx_pin: GPIO0 + rx_pin: GPIO2 + +packages: + uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml + d01: !include common.yaml diff --git a/tests/components/d01/test.rp2040-ard.yaml b/tests/components/d01/test.rp2040-ard.yaml new file mode 100644 index 0000000000..00ed175b42 --- /dev/null +++ b/tests/components/d01/test.rp2040-ard.yaml @@ -0,0 +1,7 @@ +substitutions: + tx_pin: GPIO4 + rx_pin: GPIO5 + +packages: + uart: !include ../../test_build_components/common/uart/rp2040-ard.yaml + d01: !include common.yaml From 3f68930001385b4f66f5fa799eaa79a370bf3da5 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 1 Sep 2026 16:11:57 -0400 Subject: [PATCH 22/25] [ota] Add Noise encryption to the OTA platform (#18489) Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- THREAT_MODEL.md | 38 +- esphome/__main__.py | 29 +- esphome/components/esphome/ota/__init__.py | 135 +++++- .../components/esphome/ota/ota_esphome.cpp | 121 ++++-- esphome/components/esphome/ota/ota_esphome.h | 70 ++- .../esphome/ota/ota_esphome_noise.cpp | 279 ++++++++++++ esphome/components/noise/__init__.py | 9 + esphome/components/ota/ota_backend.h | 1 + esphome/core/defines.h | 1 + esphome/espota2.py | 198 ++++++++- .../noise/test_encryption_key.py | 11 +- tests/component_tests/ota/test_esphome_ota.py | 314 +++++++++++++- tests/components/ota/encryption.yaml | 9 + tests/components/ota/encryption_inherit.yaml | 12 + .../ota/test-encryption.esp32-idf.yaml | 2 + .../ota/test-encryption.esp8266-ard.yaml | 2 + .../ota/test-encryption.rp2040-ard.yaml | 2 + .../test-encryption_inherit.esp8266-ard.yaml | 2 + .../fixtures/host_ota_encrypted.yaml | 11 + tests/integration/test_host_ota.py | 57 +++ tests/unit_tests/test_espota2_noise.py | 407 ++++++++++++++++++ tests/unit_tests/test_main.py | 108 ++++- 22 files changed, 1772 insertions(+), 46 deletions(-) create mode 100644 esphome/components/esphome/ota/ota_esphome_noise.cpp create mode 100644 tests/components/ota/encryption.yaml create mode 100644 tests/components/ota/encryption_inherit.yaml create mode 100644 tests/components/ota/test-encryption.esp32-idf.yaml create mode 100644 tests/components/ota/test-encryption.esp8266-ard.yaml create mode 100644 tests/components/ota/test-encryption.rp2040-ard.yaml create mode 100644 tests/components/ota/test-encryption_inherit.esp8266-ard.yaml create mode 100644 tests/integration/fixtures/host_ota_encrypted.yaml create mode 100644 tests/unit_tests/test_espota2_noise.py diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index 5816f38176..b4f557e55b 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -23,7 +23,8 @@ For this repository there are two trusted inputs by design: 1. **The configuration.** Anyone who can supply or edit a YAML config is trusted (see below). 2. **Authenticated peers of a running device** — clients holding the device's - API encryption key / password, OTA password, or web server credentials. + API/OTA encryption key, API password, OTA password, or web server + credentials. The security boundary is therefore **unauthenticated network traffic vs. those trusted inputs.** A bug that lets an unauthenticated attacker cross it is a @@ -76,8 +77,8 @@ These *are* security bugs in this repo, and we want to hear about them privately captive portal, etc.) **without** valid credentials. - Authentication or encryption bypass on the device — reaching API calls, OTA updates, or the web server without the configured key/password. -- Flaws that weaken the device's API encryption (Noise), OTA, or web server auth - below their documented guarantees. +- Flaws that weaken the device's API or OTA encryption (Noise), OTA auth, or + web server auth below their documented guarantees. ## The web server is an open HTTP API by design @@ -121,6 +122,37 @@ and any memory-safety or protocol bug in the server reachable without credential This section documents the current design and scope; it is not a judgment that the design is optimal or that it will not change. +## OTA update encryption + +The `esphome` OTA platform optionally encrypts updates with the same Noise +`NNpsk0` pattern the native API uses; one key protects the device. With an +`encryption:` block configured the guarantees are: the firmware image is +confidential in transit, the uploader is authenticated by the pre-shared key, +and the plaintext negotiation preceding the handshake is bound into the +handshake prologue, so stripping or tampering with it fails the first MAC. +Both ends fail closed with no override: a device built with a key refuses +plaintext uploads, and the CLI refuses to send plaintext when a key is +configured. + +Defeating any of that without the key is in scope: a keyed device accepting a +plaintext or downgraded upload, getting past the MAC, or recovering image +contents from captured traffic. + +The following are **not** vulnerabilities, by design: + +- Plaintext OTA on a device with no `encryption:` block. That is the + documented default, authenticated (if at all) by the OTA password. +- The enablement window: turning encryption on takes one last upload of the + encryption-enabled firmware over the existing plaintext channel, with the + pre-existing plaintext exposure. +- The web OTA `/update` endpoint alongside encryption. The `web_server` + component keeps it always reachable, and `captive_portal:` auto-loads it + for the fallback AP window; validation warns about both combinations, and + the operator keeps the recovery path. +- CLI retry behavior on transport or MAC failures; every attempt renegotiates + a fresh handshake with fresh ephemerals, so retrying does not weaken + authentication. + ## Explicitly out of scope - Local attackers who already have shell access on the host that runs `esphome`. diff --git a/esphome/__main__.py b/esphome/__main__.py index 1ebf194205..b3d58ad13b 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -26,7 +26,9 @@ from esphome.const import ( CONF_DEASSERT_RTS_DTR, CONF_DISABLED, CONF_DISCOVER_IP, + CONF_ENCRYPTION, CONF_ESPHOME, + CONF_KEY, CONF_LEVEL, CONF_LOG, CONF_LOG_TOPIC, @@ -1336,6 +1338,19 @@ def _upload_via_native_api( remote_port = int(ota_conf[CONF_PORT]) password = ota_conf.get(CONF_PASSWORD) + # Fail closed: an encryption block whose key did not resolve must never + # fall back to a plaintext upload + noise_psk = None + if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: + noise_psk = encryption_conf.get(CONF_KEY) + if not noise_psk: + raise EsphomeError( + "OTA encryption is configured but no key was resolved; " + "set the key under 'ota: encryption:' or 'api: encryption:'" + ) + # Ensure the key is a string, as required by the underlying OTA implementation. + # It arrives here as a SensitiveStr which aioesphomeapi rejects. + noise_psk = str(noise_psk) def check_partition_access(option_string: str) -> None: if not ota_conf.get("allow_partition_access"): @@ -1366,7 +1381,9 @@ def _upload_via_native_api( if ota_type == espota2.OTA_TYPE_UPDATE_BOOTLOADER: _validate_bootloader_binary(binary) - return espota2.run_ota(network_devices, remote_port, password, binary, ota_type) + return espota2.run_ota( + network_devices, remote_port, password, binary, ota_type, noise_psk + ) def _upload_via_web_server( @@ -1375,6 +1392,16 @@ def _upload_via_web_server( from esphome import web_server_ota from esphome.web_server_helpers import get_web_server_connection + if any( + ota_item.get(CONF_PLATFORM) == CONF_ESPHOME + and ota_item.get(CONF_ENCRYPTION) is not None + for ota_item in config.get(CONF_OTA, []) + ): + _LOGGER.warning( + "This config has OTA encryption, but the web_server OTA path sends " + "the image over plaintext HTTP; use the esphome OTA platform to " + "keep it confidential" + ) remote_port, username, password = get_web_server_connection(config) return web_server_ota.run_ota( network_devices, remote_port, username, password, binary diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index 3ef4c7ba13..1fec9e5c9b 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -1,12 +1,20 @@ import logging import esphome.codegen as cg +from esphome.components.noise import ( + decode_encryption_key, + encryption_schema, + is_reserved_key, +) from esphome.components.ota import BASE_OTA_SCHEMA, OTAComponent, ota_to_code from esphome.config_helpers import merge_config import esphome.config_validation as cv from esphome.const import ( + CONF_API, + CONF_ENCRYPTION, CONF_ESPHOME, CONF_ID, + CONF_KEY, CONF_NUM_ATTEMPTS, CONF_OTA, CONF_PASSWORD, @@ -15,6 +23,7 @@ from esphome.const import ( CONF_REBOOT_TIMEOUT, CONF_SAFE_MODE, CONF_VERSION, + CONF_WEB_SERVER, ) from esphome.core import CORE, coroutine_with_priority from esphome.coroutine import CoroPriority @@ -22,6 +31,7 @@ import esphome.final_validate as fv from esphome.types import ConfigType CONF_ALLOW_PARTITION_ACCESS = "allow_partition_access" +CONF_CAPTIVE_PORTAL = "captive_portal" _LOGGER = logging.getLogger(__name__) @@ -30,7 +40,15 @@ CODEOWNERS = ["@esphome/core"] DEPENDENCIES = ["network"] -AUTO_LOAD = ["sha256", "socket"] +def AUTO_LOAD(config: ConfigType) -> list[str]: + """Auto-load noise only when encryption is configured.""" + base = ["sha256", "socket"] + # A falsy config is a tooling probe for the maximal set (None from + # dependency resolution, {} from the components-graph platform probe); + # a validated config always carries defaults, never empty + if not config or CONF_ENCRYPTION in config: + return base + ["noise"] + return base esphome = cg.esphome_ns.namespace("esphome") @@ -67,11 +85,24 @@ def ota_esphome_final_validate(config: ConfigType) -> None: CONF_PASSWORD in merged_ota_esphome_configs_by_port[conf_port] and CONF_PASSWORD in ota_conf and merged_ota_esphome_configs_by_port[conf_port][CONF_PASSWORD] - != ota_conf.get(CONF_PASSWORD) + != ota_conf[CONF_PASSWORD] ): raise cv.Invalid( f"Found multiple configurations but {CONF_PASSWORD} is inconsistent" ) + # Encryption blocks conflict only when both pin a key; a bare + # `encryption:` (a package/device split) is compatible with a + # keyed one, and merge_config yields the keyed result + merged_key = ( + merged_ota_esphome_configs_by_port[conf_port] + .get(CONF_ENCRYPTION, {}) + .get(CONF_KEY) + ) + other_key = ota_conf.get(CONF_ENCRYPTION, {}).get(CONF_KEY) + if merged_key and other_key and merged_key != other_key: + raise cv.Invalid( + f"Found multiple configurations but {CONF_ENCRYPTION} is inconsistent" + ) ports_with_merged_configs.append(conf_port) merged_ota_esphome_configs_by_port[conf_port] = merge_config( @@ -94,6 +125,20 @@ def ota_esphome_final_validate(config: ConfigType) -> None: new_ota_conf.extend(merged_ota_esphome_configs_by_port.values()) + api_conf = full_conf.get(CONF_API) or {} + for ota_conf in merged_ota_esphome_configs_by_port.values(): + # Merging same-port blocks can combine a password from one block with + # encryption from another; re-check the exclusion on the merged result. + _validate_no_password_with_encryption(ota_conf) + if (encryption_conf := ota_conf.get(CONF_ENCRYPTION)) is not None: + _resolve_encryption_key(encryption_conf, api_conf) + if any( + conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf + ) and any( + CONF_ENCRYPTION in conf for conf in merged_ota_esphome_configs_by_port.values() + ): + _warn_web_server_ota(full_conf) + full_conf[CONF_OTA] = new_ota_conf fv.full_config.set(full_conf) @@ -107,6 +152,73 @@ def ota_esphome_final_validate(config: ConfigType) -> None: ) +def _warn_web_server_ota(full_conf: ConfigType) -> None: + """The web_server ota platform accepts the same image over plaintext HTTP + with basic auth, bypassing the encryption; warn rather than fail so the + operator keeps the recovery path.""" + if CONF_CAPTIVE_PORTAL in full_conf and CONF_WEB_SERVER not in full_conf: + # The captive_portal auto-load: the endpoint only exists while the + # fallback AP is active + _LOGGER.warning( + "OTA encryption does not cover the %s OTA platform (auto-loaded " + "by captive_portal); the plaintext /update endpoint stays " + "reachable while the fallback AP is active", + CONF_WEB_SERVER, + ) + else: + _LOGGER.warning( + "OTA encryption does not cover the %s OTA platform; its " + "plaintext /update endpoint accepts the same image", + CONF_WEB_SERVER, + ) + + +def _resolve_encryption_key(encryption_conf: ConfigType, api_conf: ConfigType) -> None: + """Resolve the one encryption key per device into the ota block. + + An explicit ota key must match the api key, a bare block inherits it, + a runtime provisioned api key cannot be inherited, and the all-zeros + provisioning sentinel is rejected (the device treats it as no key). + """ + api_key = api_conf.get(CONF_ENCRYPTION, {}).get(CONF_KEY) + if ota_key := encryption_conf.get(CONF_KEY): + if api_key and ota_key != api_key: + raise cv.Invalid( + f"'{CONF_OTA}' {CONF_ENCRYPTION} {CONF_KEY} must match the " + f"'{CONF_API}' {CONF_ENCRYPTION} {CONF_KEY}; omit the " + f"'{CONF_OTA}' {CONF_KEY} to use the '{CONF_API}' one" + ) + elif not api_key: + if CONF_ENCRYPTION in api_conf: + raise cv.Invalid( + f"the '{CONF_API}' {CONF_ENCRYPTION} {CONF_KEY} is provisioned at " + f"runtime and cannot be inherited at build time; set an explicit " + f"'{CONF_OTA}' {CONF_ENCRYPTION} {CONF_KEY}" + ) + raise cv.Invalid( + f"'{CONF_OTA}' {CONF_ENCRYPTION} has no {CONF_KEY} and there is no " + f"'{CONF_API}' {CONF_ENCRYPTION} {CONF_KEY} to inherit; set one of them" + ) + else: + encryption_conf[CONF_KEY] = api_key + if is_reserved_key(encryption_conf[CONF_KEY]): + raise cv.Invalid( + f"The all-zeros {CONF_KEY} is reserved and provides no protection; " + f"generate a real key with: openssl rand -base64 32" + ) + + +# Also called on merged same-port configs in final validate, where schemas +# do not run +def _validate_no_password_with_encryption(config: ConfigType) -> ConfigType: + if CONF_PASSWORD in config and CONF_ENCRYPTION in config: + raise cv.Invalid( + f"'{CONF_PASSWORD}' cannot be combined with '{CONF_ENCRYPTION}'; the " + f"encryption key already authenticates the uploader, remove '{CONF_PASSWORD}'" + ) + return config + + def _consume_ota_sockets(config: ConfigType) -> ConfigType: """Register socket needs for OTA component.""" from esphome.components import socket @@ -134,6 +246,7 @@ CONFIG_SCHEMA = cv.All( ): cv.port, cv.Optional(CONF_ALLOW_PARTITION_ACCESS, default=False): cv.boolean, cv.Optional(CONF_PASSWORD): cv.sensitive(), + cv.Optional(CONF_ENCRYPTION): encryption_schema, cv.Optional(CONF_NUM_ATTEMPTS): cv.invalid( f"'{CONF_SAFE_MODE}' (and its related configuration variables) has moved from 'ota' to its own component. See https://esphome.io/components/safe_mode" ), @@ -147,12 +260,24 @@ CONFIG_SCHEMA = cv.All( ) .extend(BASE_OTA_SCHEMA) .extend(cv.COMPONENT_SCHEMA), + _validate_no_password_with_encryption, _consume_ota_sockets, ) FINAL_VALIDATE_SCHEMA = ota_esphome_final_validate +def FILTER_SOURCE_FILES() -> list[str]: + """Filter out the noise transport when no ota entry configures encryption.""" + for ota_conf in CORE.config.get(CONF_OTA, []): + if ( + ota_conf.get(CONF_PLATFORM) == CONF_ESPHOME + and ota_conf.get(CONF_ENCRYPTION) is not None + ): + return [] + return ["ota_esphome_noise.cpp"] + + @coroutine_with_priority(CoroPriority.OTA_UPDATES) async def to_code(config: ConfigType) -> None: var = cg.new_Pvariable(config[CONF_ID]) @@ -171,6 +296,12 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_ALLOW_PARTITION_ACCESS): cg.add_define("USE_OTA_PARTITIONS") + if (encryption_conf := config.get(CONF_ENCRYPTION)) is not None: + # A missing key was resolved from the api component in final validate. + key = encryption_conf[CONF_KEY] + cg.add_define("USE_OTA_ENCRYPTION") + cg.add(var.set_noise_psk(list(decode_encryption_key(key)))) + # Build flag so lwip_fast_select.c (a .c file that can't include defines.h) sees it. cg.add_build_flag("-DUSE_OTA_PLATFORM_ESPHOME") diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 9f15eaaede..396a47bc52 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -27,7 +27,6 @@ namespace esphome { static const char *const TAG = "esphome.ota"; static constexpr uint16_t OTA_BLOCK_SIZE = 8192; -static constexpr size_t OTA_BUFFER_SIZE = 1024; // buffer size for OTA data transfer static constexpr uint32_t OTA_SOCKET_TIMEOUT_HANDSHAKE = 20000; // milliseconds for initial handshake static constexpr uint32_t OTA_SOCKET_TIMEOUT_DATA = 90000; // milliseconds for data transfer @@ -105,6 +104,11 @@ void ESPHomeOTAComponent::dump_config() { ESP_LOGCONFIG(TAG, " Password configured"); } #endif +#ifdef USE_OTA_ENCRYPTION + if (this->noise_ctx_.has_psk()) { + ESP_LOGCONFIG(TAG, " Encryption configured"); + } +#endif #ifdef USE_OTA_PARTITIONS ESP_LOGCONFIG(TAG, " Partition access allowed\n" @@ -149,8 +153,10 @@ void ESPHomeOTAComponent::loop() { static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02; static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04; +static constexpr uint8_t CLIENT_FEATURE_SUPPORTS_NOISE = 0x08; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01; static constexpr uint8_t SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02; +static constexpr uint8_t SERVER_FEATURE_SUPPORTS_NOISE = 0x04; void ESPHomeOTAComponent::handle_handshake_() { /// Handle the OTA handshake and authentication. @@ -202,8 +208,7 @@ void ESPHomeOTAComponent::handle_handshake_() { } // Validate magic bytes - static const uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45}; - if (memcmp(this->handshake_buf_, MAGIC_BYTES, 5) != 0) { + if (memcmp(this->handshake_buf_, MAGIC_BYTES, sizeof(MAGIC_BYTES)) != 0) { ESP_LOGW(TAG, "Magic bytes mismatch! 0x%02X-0x%02X-0x%02X-0x%02X-0x%02X", this->handshake_buf_[0], this->handshake_buf_[1], this->handshake_buf_[2], this->handshake_buf_[3], this->handshake_buf_[4]); this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_MAGIC); @@ -235,6 +240,19 @@ void ESPHomeOTAComponent::handle_handshake_() { } this->ota_features_ = this->handshake_buf_[0]; ESP_LOGV(TAG, "Features: 0x%02X", this->ota_features_); + +#ifdef USE_OTA_ENCRYPTION + // Fail closed: with a PSK configured the client must negotiate encryption + // (which requires the extended protocol); refuse plaintext uploads. + static constexpr uint8_t NOISE_REQUIRED_FEATURES = + CLIENT_FEATURE_SUPPORTS_NOISE | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL; + if (this->noise_ctx_.has_psk() && (this->ota_features_ & NOISE_REQUIRED_FEATURES) != NOISE_REQUIRED_FEATURES) { + ESP_LOGW(TAG, "Client does not support encryption"); + this->send_error_and_cleanup_(ota::OTA_RESPONSE_ERROR_ENCRYPTION_REQUIRED); + return; + } +#endif + this->transition_ota_state_(OTAState::FEATURE_ACK); const bool supports_compression = @@ -250,6 +268,11 @@ void ESPHomeOTAComponent::handle_handshake_() { this->handshake_buf_[1] = (supports_compression ? SERVER_FEATURE_SUPPORTS_COMPRESSION : 0); #ifdef USE_OTA_PARTITIONS this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS; +#endif +#ifdef USE_OTA_ENCRYPTION + if (this->noise_ctx_.has_psk()) { + this->handshake_buf_[1] |= SERVER_FEATURE_SUPPORTS_NOISE; + } #endif } else { this->handshake_buf_[0] = @@ -265,6 +288,20 @@ void ESPHomeOTAComponent::handle_handshake_() { if (!this->try_write_(ack_size, LOG_STR("ack feature"))) { return; } +#ifdef USE_OTA_ENCRYPTION + // With a PSK configured the rest of the session runs inside the noise + // transport; the client sends the first handshake frame next, so there + // is nothing to do until data arrives. + if (this->noise_ctx_.has_psk()) { + // handshake_buf_ still holds the feature ack composed above; a + // would-block re-entry lands here without rebuilding it + if (!this->noise_start_session_(this->handshake_buf_[1])) { + return; + } + this->transition_ota_state_(OTAState::NOISE_HANDSHAKE); + return; + } +#endif #ifdef USE_OTA_PASSWORD // If password is set, move to auth phase if (!this->password_.empty()) { @@ -302,6 +339,16 @@ void ESPHomeOTAComponent::handle_handshake_() { this->handle_data_(); return; +#ifdef USE_OTA_ENCRYPTION + case OTAState::NOISE_HANDSHAKE: + if (!this->handle_noise_handshake_()) { + return; + } + this->transition_ota_state_(OTAState::DATA); + this->handle_data_(); + return; +#endif + default: break; } @@ -340,6 +387,8 @@ void ESPHomeOTAComponent::handle_data_() { /// Raw TCP (8266, RP2040): setblocking is no-op; SO_RCVTIMEO uses /// wakeable_delay() in read(); /// write() always returns immediately + // Backend calls overwrite this with OK; reset to UNKNOWN before any + // goto error that follows a successful begin()/write() ota::OTAResponseTypes error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; size_t total = 0; uint32_t last_progress = 0; @@ -361,11 +410,11 @@ void ESPHomeOTAComponent::handle_data_() { this->client_->setblocking(true); // Acknowledge auth OK - 1 byte - this->write_byte_(ota::OTA_RESPONSE_AUTH_OK); + this->data_write_byte_(ota::OTA_RESPONSE_AUTH_OK); if (this->extended_proto_) { // Read ota type, 1 byte - if (!this->readall_(buf, 1)) { + if (!this->data_readall_(buf, 1)) { this->log_read_error_(LOG_STR("OTA type")); goto error; // NOLINT(cppcoreguidelines-avoid-goto) } @@ -374,7 +423,7 @@ void ESPHomeOTAComponent::handle_data_() { ESP_LOGV(TAG, "OTA type is 0x%02x", ota_type); // Read size, 4 bytes MSB first - if (!this->readall_(buf, 4)) { + if (!this->data_readall_(buf, 4)) { this->log_read_error_(LOG_STR("size")); goto error; // NOLINT(cppcoreguidelines-avoid-goto) } @@ -405,11 +454,12 @@ void ESPHomeOTAComponent::handle_data_() { goto error; // NOLINT(cppcoreguidelines-avoid-goto) // Acknowledge prepare OK - 1 byte - this->write_byte_(ota::OTA_RESPONSE_UPDATE_PREPARE_OK); + this->data_write_byte_(ota::OTA_RESPONSE_UPDATE_PREPARE_OK); // Read binary MD5, 32 bytes - if (!this->readall_(buf, 32)) { + if (!this->data_readall_(buf, 32)) { this->log_read_error_(LOG_STR("MD5 checksum")); + error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; goto error; // NOLINT(cppcoreguidelines-avoid-goto) } sbuf[32] = '\0'; @@ -417,7 +467,7 @@ void ESPHomeOTAComponent::handle_data_() { this->backend_->set_update_md5(sbuf); // Acknowledge MD5 OK - 1 byte - this->write_byte_(ota::OTA_RESPONSE_BIN_MD5_OK); + this->data_write_byte_(ota::OTA_RESPONSE_BIN_MD5_OK); // Track when we last received data so a silently-vanished peer (no FIN/RST // delivered, e.g. uploader killed mid-transfer or NAT/router dropped state) @@ -433,19 +483,35 @@ void ESPHomeOTAComponent::handle_data_() { } size_t remaining = ota_size - total; size_t requested = remaining < OTA_BUFFER_SIZE ? remaining : OTA_BUFFER_SIZE; - ssize_t read = this->client_->read(buf, requested); - if (read == -1) { - const int err = errno; - if (this->would_block_(err)) { - // read() already waited up to SO_RCVTIMEO for data, just feed WDT - App.feed_wdt(); - continue; + ssize_t read; +#ifdef USE_OTA_ENCRYPTION + if (this->noise_ != nullptr) { + // One frame per call; noise_read_data_ waits internally (readall_), so + // there is no would-block retry here and failures are already logged. + read = this->noise_read_data_(buf, requested); + if (read <= 0) { + error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; + goto error; // NOLINT(cppcoreguidelines-avoid-goto) + } + } else +#endif + { + read = this->client_->read(buf, requested); + if (read == -1) { + const int err = errno; + if (this->would_block_(err)) { + // read() already waited up to SO_RCVTIMEO for data, just feed WDT + App.feed_wdt(); + continue; + } + ESP_LOGW(TAG, "Read err %d", err); + error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; + goto error; // NOLINT(cppcoreguidelines-avoid-goto) + } else if (read == 0) { + ESP_LOGW(TAG, "Remote closed"); + error_code = ota::OTA_RESPONSE_ERROR_UNKNOWN; + goto error; // NOLINT(cppcoreguidelines-avoid-goto) } - ESP_LOGW(TAG, "Read err %d", err); - goto error; // NOLINT(cppcoreguidelines-avoid-goto) - } else if (read == 0) { - ESP_LOGW(TAG, "Remote closed"); - goto error; // NOLINT(cppcoreguidelines-avoid-goto) } last_data_ms = millis(); @@ -457,7 +523,7 @@ void ESPHomeOTAComponent::handle_data_() { total += read; #if USE_OTA_VERSION == 2 while (size_acknowledged + OTA_BLOCK_SIZE <= total || (total == ota_size && size_acknowledged < ota_size)) { - this->write_byte_(ota::OTA_RESPONSE_CHUNK_OK); + this->data_write_byte_(ota::OTA_RESPONSE_CHUNK_OK); size_acknowledged += OTA_BLOCK_SIZE; } #endif @@ -476,7 +542,7 @@ void ESPHomeOTAComponent::handle_data_() { } // Acknowledge receive OK - 1 byte - this->write_byte_(ota::OTA_RESPONSE_RECEIVE_OK); + this->data_write_byte_(ota::OTA_RESPONSE_RECEIVE_OK); error_code = this->backend_->end(); if (error_code != ota::OTA_RESPONSE_OK) { @@ -485,10 +551,10 @@ void ESPHomeOTAComponent::handle_data_() { } // Acknowledge Update end OK - 1 byte - this->write_byte_(ota::OTA_RESPONSE_UPDATE_END_OK); + this->data_write_byte_(ota::OTA_RESPONSE_UPDATE_END_OK); // Read ACK - if (!this->readall_(buf, 1) || buf[0] != ota::OTA_RESPONSE_OK) { + if (!this->data_readall_(buf, 1) || buf[0] != ota::OTA_RESPONSE_OK) { this->log_read_error_(LOG_STR("ack")); // do not go to error, this is not fatal } @@ -511,7 +577,7 @@ void ESPHomeOTAComponent::handle_data_() { App.safe_reboot(); error: - this->write_byte_(static_cast(error_code)); + this->data_write_byte_(static_cast(error_code)); // Abort backend before cleanup - cleanup_connection_() destroys the backend. // Always call abort() unconditionally: backends register external partitions before @@ -678,6 +744,9 @@ void ESPHomeOTAComponent::cleanup_connection_() { this->backend_ = nullptr; #ifdef USE_OTA_PASSWORD this->cleanup_auth_(); +#endif +#ifdef USE_OTA_ENCRYPTION + this->noise_ = nullptr; #endif // Intentionally no disable_loop() — letting loop() run one more iteration catches // any connection that queued on the listener mid-session (otherwise the wake flag, diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index 979e3f2d7d..fd164b8138 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -4,6 +4,9 @@ #ifdef USE_OTA #include "esphome/components/ota/ota_backend_factory.h" #include "esphome/components/socket/socket.h" +#ifdef USE_OTA_ENCRYPTION +#include "esphome/components/noise/noise_handshake.h" +#endif #include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/preferences.h" @@ -24,7 +27,10 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { AUTH_SEND, // Sending authentication request AUTH_READ, // Reading authentication data #endif // USE_OTA_PASSWORD - DATA, // BLOCKING! Processing OTA data (update, etc.) +#ifdef USE_OTA_ENCRYPTION + NOISE_HANDSHAKE, // Exchanging Noise handshake frames +#endif + DATA, // BLOCKING! Processing OTA data (update, etc.) }; #ifdef USE_OTA_PASSWORD void set_auth_password(const std::string &password) { password_ = password; } @@ -38,6 +44,10 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { } #endif // USE_OTA_PASSWORD +#ifdef USE_OTA_ENCRYPTION + void set_noise_psk(noise::psk_t psk) { this->noise_ctx_.set_psk(psk); } +#endif + /// Manually set the port OTA should listen on void set_port(uint16_t port) { this->port_ = port; } @@ -63,6 +73,48 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { bool writeall_(const uint8_t *buf, size_t len); inline bool write_byte_(uint8_t byte) { return this->writeall_(&byte, 1); } +#ifdef USE_OTA_ENCRYPTION + // Heap-allocated only while an encrypted OTA session is active. + struct NoiseSession { + ~NoiseSession(); + noise::NoiseResponderHandshake handshake; + NoiseCipherState *send_cipher{nullptr}; + NoiseCipherState *recv_cipher{nullptr}; + uint16_t frame_len{0}; // total frame size once the header is parsed, 0 until then + uint16_t frame_pos{0}; // bytes read or written so far + bool writing{false}; // a produced handshake frame is still being flushed + uint8_t frame_buf[noise::FRAME_HEADER_SIZE + 1 + noise::MAX_HANDSHAKE_SIZE]; + }; + bool noise_start_session_(uint8_t server_feature_flags); + bool handle_noise_handshake_(); + bool noise_try_read_frame_(); + bool noise_try_write_frame_(); + void noise_send_reject_(const LogString *reason); + ssize_t noise_decrypt_(uint8_t *buf, size_t len); + ssize_t noise_read_frame_blocking_(uint8_t *buf, size_t min_ciphertext, size_t max_ciphertext); + bool noise_readall_(uint8_t *buf, size_t len); + ssize_t noise_read_data_(uint8_t *buf, size_t capacity); + bool noise_write_byte_(uint8_t byte); +#endif // USE_OTA_ENCRYPTION + + // Data-phase I/O dispatch: through the noise transport when a session is + // active, straight to the socket otherwise. + inline bool data_write_byte_(uint8_t byte) { +#ifdef USE_OTA_ENCRYPTION + if (this->noise_ != nullptr) + return this->noise_write_byte_(byte); +#endif + return this->write_byte_(byte); + } + // When encrypted, buf must have room for len + noise::MAC_SIZE bytes. + inline bool data_readall_(uint8_t *buf, size_t len) { +#ifdef USE_OTA_ENCRYPTION + if (this->noise_ != nullptr) + return this->noise_readall_(buf, len); +#endif + return this->readall_(buf, len); + } + bool try_read_(size_t to_read, const LogString *desc); bool try_write_(size_t to_write, const LogString *desc); @@ -91,6 +143,10 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { std::string password_; std::unique_ptr auth_buf_; #endif // USE_OTA_PASSWORD +#ifdef USE_OTA_ENCRYPTION + noise::NoiseContext noise_ctx_; + std::unique_ptr noise_; +#endif // USE_OTA_ENCRYPTION socket::ListenSocket *server_{nullptr}; std::unique_ptr client_; @@ -98,6 +154,18 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { uint32_t client_connect_time_{0}; static constexpr size_t HANDSHAKE_BUF_SIZE = 5; + // Buffer size for OTA data transfer. The upload client derives its maximum + // encrypted frame plaintext from this (espota2.NOISE_MAX_PLAINTEXT is this + // minus the 16-byte MAC); both must change together. + static constexpr size_t OTA_BUFFER_SIZE = 1040; +#ifdef USE_OTA_ENCRYPTION + // espota2.NOISE_MAX_PLAINTEXT; shrinking the buffer would reject every + // frame a current CLI sends + static constexpr size_t NOISE_CLIENT_MAX_PLAINTEXT = 1024; + static_assert(OTA_BUFFER_SIZE >= NOISE_CLIENT_MAX_PLAINTEXT + noise::MAC_SIZE, + "OTA_BUFFER_SIZE must fit a full encrypted data frame"); +#endif + static constexpr uint8_t MAGIC_BYTES[5] = {0x6C, 0x26, 0xF7, 0x5C, 0x45}; #ifdef USE_OTA_PARTITIONS uint32_t running_app_offset_{0}; size_t running_app_size_{0}; diff --git a/esphome/components/esphome/ota/ota_esphome_noise.cpp b/esphome/components/esphome/ota/ota_esphome_noise.cpp new file mode 100644 index 0000000000..7f8331cf96 --- /dev/null +++ b/esphome/components/esphome/ota/ota_esphome_noise.cpp @@ -0,0 +1,279 @@ +#include "ota_esphome.h" +#ifdef USE_OTA +#ifdef USE_OTA_ENCRYPTION +#include "esphome/components/noise/noise.h" +#include "esphome/components/ota/ota_backend.h" +#include "esphome/core/log.h" + +#include +#include + +#ifdef USE_ESP8266 +#include +#endif + +namespace esphome { + +static const char *const TAG = "esphome.ota"; + +#ifdef USE_ESP8266 +static constexpr char OTA_NOISE_PROLOGUE_INIT[] PROGMEM = "NoiseOTAInit"; +#else +static constexpr char OTA_NOISE_PROLOGUE_INIT[] = "NoiseOTAInit"; +#endif +static constexpr size_t OTA_NOISE_PROLOGUE_INIT_LEN = sizeof(OTA_NOISE_PROLOGUE_INIT) - 1; + +ESPHomeOTAComponent::NoiseSession::~NoiseSession() { + if (this->send_cipher != nullptr) { + noise_cipherstate_free(this->send_cipher); + } + if (this->recv_cipher != nullptr) { + noise_cipherstate_free(this->recv_cipher); + } +} + +/** Allocate the session and start the responder handshake. + * + * The prologue binds the whole plaintext preamble, so any tampering with the + * negotiation (a stripped feature flag, a changed version) breaks the first + * handshake MAC on either side: + * "NoiseOTAInit" | magic(5) | OK,version | client_features | FEATURE_FLAGS,server_flags + */ +bool ESPHomeOTAComponent::noise_start_session_(uint8_t server_feature_flags) { + // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) + this->noise_ = std::unique_ptr(new (std::nothrow) NoiseSession()); + if (this->noise_ == nullptr) { + ESP_LOGW(TAG, "Session allocation failed"); + this->cleanup_connection_(); + return false; + } + + static constexpr size_t PROLOGUE_ACK_LEN = 2; // OTA_RESPONSE_OK + version + static constexpr size_t PROLOGUE_CLIENT_FEATURES_LEN = 1; + static constexpr size_t PROLOGUE_FEATURE_ACK_LEN = 2; // OTA_RESPONSE_FEATURE_FLAGS + server flags + uint8_t prologue[OTA_NOISE_PROLOGUE_INIT_LEN + sizeof(MAGIC_BYTES) + PROLOGUE_ACK_LEN + PROLOGUE_CLIENT_FEATURES_LEN + + PROLOGUE_FEATURE_ACK_LEN]; +#ifdef USE_ESP8266 + memcpy_P(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); +#else + std::memcpy(prologue, OTA_NOISE_PROLOGUE_INIT, OTA_NOISE_PROLOGUE_INIT_LEN); +#endif + uint8_t *p = prologue + OTA_NOISE_PROLOGUE_INIT_LEN; + // Magic bytes, already validated in MAGIC_READ + std::memcpy(p, MAGIC_BYTES, sizeof(MAGIC_BYTES)); + p += sizeof(MAGIC_BYTES); + // Our magic ack + *p++ = ota::OTA_RESPONSE_OK; + *p++ = USE_OTA_VERSION; + // The feature byte the client sent + *p++ = this->ota_features_; + // The feature ack we sent (noise requires the extended protocol) + *p++ = ota::OTA_RESPONSE_FEATURE_FLAGS; + *p++ = server_feature_flags; + + int err = this->noise_->handshake.init(this->noise_ctx_.get_psk(), prologue, sizeof(prologue)); + if (err != 0) { + ESP_LOGW(TAG, "Handshake init: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + this->cleanup_connection_(); + return false; + } + return true; +} + +/** Drive the non-blocking handshake from loop(); returns true once the + * transport ciphers are ready. A would-block returns false and the next + * loop() resumes from the NoiseSession cursors; on failure the connection + * is cleaned up. + */ +bool ESPHomeOTAComponent::handle_noise_handshake_() { + NoiseSession &s = *this->noise_; + while (true) { + if (s.writing) { + if (!this->noise_try_write_frame_()) { + return false; // would block, or errored and cleaned up + } + s.writing = false; + s.frame_pos = 0; + s.frame_len = 0; + } + switch (s.handshake.action()) { + case noise::NoiseResponderHandshake::Action::ACTION_READ: { + if (!this->noise_try_read_frame_()) { + return false; + } + const uint16_t payload_len = s.frame_len - noise::FRAME_HEADER_SIZE; + s.frame_pos = 0; + s.frame_len = 0; + if (s.frame_buf[noise::FRAME_HEADER_SIZE] != noise::HANDSHAKE_STATUS_OK) { + ESP_LOGW(TAG, "Bad handshake error byte: %u", s.frame_buf[noise::FRAME_HEADER_SIZE]); + this->cleanup_connection_(); + return false; + } + int err = s.handshake.read_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, payload_len - 1); + if (err != 0) { + ESP_LOGW(TAG, "Handshake read: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + this->noise_send_reject_(noise::reject_reason_for(err)); + this->cleanup_connection_(); + return false; + } + break; + } + case noise::NoiseResponderHandshake::Action::ACTION_WRITE: { + size_t msg_len = 0; + int err = + s.handshake.write_message(s.frame_buf + noise::FRAME_HEADER_SIZE + 1, noise::MAX_HANDSHAKE_SIZE, msg_len); + if (err != 0) { + ESP_LOGW(TAG, "Handshake write: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + this->cleanup_connection_(); + return false; + } + const uint16_t payload_len = msg_len + 1; + noise::write_frame_header(s.frame_buf, payload_len); + s.frame_buf[noise::FRAME_HEADER_SIZE] = noise::HANDSHAKE_STATUS_OK; + s.frame_len = noise::FRAME_HEADER_SIZE + payload_len; + s.frame_pos = 0; + s.writing = true; + break; + } + case noise::NoiseResponderHandshake::Action::ACTION_SPLIT: { + int err = s.handshake.split(s.send_cipher, s.recv_cipher); + if (err != 0) { + ESP_LOGW(TAG, "Handshake split: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + this->cleanup_connection_(); + return false; + } + ESP_LOGD(TAG, "Noise handshake complete"); + return true; + } + default: { + ESP_LOGW(TAG, "Bad handshake state"); + this->cleanup_connection_(); + return false; + } + } + } +} + +/// Non-blocking read of one handshake frame into the session buffer. +bool ESPHomeOTAComponent::noise_try_read_frame_() { + NoiseSession &s = *this->noise_; + while (s.frame_pos < noise::FRAME_HEADER_SIZE) { + ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, noise::FRAME_HEADER_SIZE - s.frame_pos); + if (!this->handle_read_error_(read, LOG_STR("read noise header"))) { + return false; + } + s.frame_pos += read; + } + if (s.frame_len == 0) { + const uint16_t payload_len = encode_uint16(s.frame_buf[1], s.frame_buf[2]); + if (s.frame_buf[0] != noise::FRAME_INDICATOR || payload_len < 1 || payload_len > 1 + noise::MAX_HANDSHAKE_SIZE) { + ESP_LOGW(TAG, "Bad handshake frame: 0x%02X, %u bytes", s.frame_buf[0], payload_len); + this->cleanup_connection_(); + return false; + } + s.frame_len = noise::FRAME_HEADER_SIZE + payload_len; + } + while (s.frame_pos < s.frame_len) { + ssize_t read = this->client_->read(s.frame_buf + s.frame_pos, s.frame_len - s.frame_pos); + if (!this->handle_read_error_(read, LOG_STR("read noise frame"))) { + return false; + } + s.frame_pos += read; + } + return true; +} + +/// Non-blocking write of the pending session-buffer frame. +bool ESPHomeOTAComponent::noise_try_write_frame_() { + NoiseSession &s = *this->noise_; + while (s.frame_pos < s.frame_len) { + ssize_t written = this->client_->write(s.frame_buf + s.frame_pos, s.frame_len - s.frame_pos); + if (!this->handle_write_error_(written, LOG_STR("write noise frame"))) { + return false; + } + s.frame_pos += written; + } + return true; +} + +/// Best-effort explicit reject frame so the client can log a readable reason. +void ESPHomeOTAComponent::noise_send_reject_(const LogString *reason) { + // Every reason here comes from noise::reject_reason_for(), so the exported + // floor is the exact capacity needed + uint8_t data[noise::FRAME_HEADER_SIZE + noise::MAC_FAILURE_PAYLOAD_SIZE]; + const size_t payload_len = + noise::format_reject_payload(data + noise::FRAME_HEADER_SIZE, sizeof(data) - noise::FRAME_HEADER_SIZE, reason); + noise::write_frame_header(data, payload_len); + this->client_->write(data, noise::FRAME_HEADER_SIZE + payload_len); // Best effort, non-blocking +} + +/// Decrypt a ciphertext in place; returns the plaintext size or -1. +ssize_t ESPHomeOTAComponent::noise_decrypt_(uint8_t *buf, size_t len) { + NoiseBuffer mbuf; + noise_buffer_init(mbuf); + noise_buffer_set_inout(mbuf, buf, len, len); + int err = noise_cipherstate_decrypt(this->noise_->recv_cipher, &mbuf); + if (err != 0) { + ESP_LOGW(TAG, "Decrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + return -1; + } + return mbuf.size; +} + +/** Blocking read of one frame whose ciphertext size must be within the given + * bounds, decrypted in place; returns the plaintext size, or -1 on error. + * buf needs max_ciphertext capacity. + */ +ssize_t ESPHomeOTAComponent::noise_read_frame_blocking_(uint8_t *buf, size_t min_ciphertext, size_t max_ciphertext) { + uint8_t header[noise::FRAME_HEADER_SIZE]; + if (!this->readall_(header, sizeof(header))) { + return -1; + } + const size_t ciphertext_len = encode_uint16(header[1], header[2]); + if (header[0] != noise::FRAME_INDICATOR || ciphertext_len < min_ciphertext || ciphertext_len > max_ciphertext) { + ESP_LOGW(TAG, "Bad frame: 0x%02X, %zu bytes", header[0], ciphertext_len); + return -1; + } + if (!this->readall_(buf, ciphertext_len)) { + return -1; + } + return this->noise_decrypt_(buf, ciphertext_len); +} + +/** Blocking read of one frame whose plaintext must be exactly len bytes + * (control units are one unit per frame). buf needs len + noise::MAC_SIZE + * capacity; the plaintext lands at buf[0..len). + */ +bool ESPHomeOTAComponent::noise_readall_(uint8_t *buf, size_t len) { + return this->noise_read_frame_blocking_(buf, len + noise::MAC_SIZE, len + noise::MAC_SIZE) == (ssize_t) len; +} + +/** Blocking read of one data-phase frame, decrypted in place; returns the + * plaintext size, or -1 on error. buf is the OTA_BUFFER_SIZE data buffer. + * The ciphertext must fit that buffer and its plaintext must fit what the + * caller accepts (the remaining image bytes). + */ +ssize_t ESPHomeOTAComponent::noise_read_data_(uint8_t *buf, size_t capacity) { + const size_t max_ciphertext = std::min(capacity + noise::MAC_SIZE, OTA_BUFFER_SIZE); + return this->noise_read_frame_blocking_(buf, noise::MAC_SIZE + 1, max_ciphertext); +} + +/// Blocking write of one response byte as an encrypted frame. +bool ESPHomeOTAComponent::noise_write_byte_(uint8_t byte) { + uint8_t frame[noise::FRAME_HEADER_SIZE + 1 + noise::MAC_SIZE]; + frame[noise::FRAME_HEADER_SIZE] = byte; + NoiseBuffer mbuf; + noise_buffer_init(mbuf); + noise_buffer_set_inout(mbuf, frame + noise::FRAME_HEADER_SIZE, 1, 1 + noise::MAC_SIZE); + int err = noise_cipherstate_encrypt(this->noise_->send_cipher, &mbuf); + if (err != 0) { + ESP_LOGW(TAG, "Encrypt: %s", LOG_STR_ARG(noise::noise_err_to_logstr(err))); + return false; + } + noise::write_frame_header(frame, mbuf.size); + return this->writeall_(frame, noise::FRAME_HEADER_SIZE + mbuf.size); +} + +} // namespace esphome +#endif // USE_OTA_ENCRYPTION +#endif // USE_OTA diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 3a8e2609ef..0f9328a482 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -45,6 +45,15 @@ def decode_encryption_key(value: str) -> bytes: return decoded +def is_reserved_key(value: str) -> bool: + """Whether the key is the reserved all-zeros provisioning sentinel. + + The device treats it as no key configured, so consumers that require a + real key must reject it. + """ + return not any(decode_encryption_key(value)) + + ENCRYPTION_SCHEMA = cv.Schema( { cv.Optional(CONF_KEY): cv.sensitive(validate_encryption_key), diff --git a/esphome/components/ota/ota_backend.h b/esphome/components/ota/ota_backend.h index 1c24fc320a..7348a0ce90 100644 --- a/esphome/components/ota/ota_backend.h +++ b/esphome/components/ota/ota_backend.h @@ -49,6 +49,7 @@ enum OTAResponseTypes { OTA_RESPONSE_ERROR_BOOTLOADER_VERIFY = 0x91, OTA_RESPONSE_ERROR_BOOTLOADER_UPDATE = 0x92, OTA_RESPONSE_ERROR_VERSION_DOWNGRADE = 0x93, + OTA_RESPONSE_ERROR_ENCRYPTION_REQUIRED = 0x94, OTA_RESPONSE_ERROR_UNKNOWN = 0xFF, }; diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 1f5a10d47d..7af41409fd 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -242,6 +242,7 @@ #define USE_RUNTIME_IMAGE_QOI #define USE_RUNTIME_STATS #define USE_OTA +#define USE_OTA_ENCRYPTION #define USE_OTA_PASSWORD #define USE_OTA_VERSION 2 #define USE_TIME_TIMEZONE diff --git a/esphome/espota2.py b/esphome/espota2.py index ca833f1816..ac4cbeeb7c 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -53,6 +53,7 @@ RESPONSE_ERROR_PARTITION_TABLE_UPDATE = 0x90 RESPONSE_ERROR_BOOTLOADER_VERIFY = 0x91 RESPONSE_ERROR_BOOTLOADER_UPDATE = 0x92 RESPONSE_ERROR_VERSION_DOWNGRADE = 0x93 +RESPONSE_ERROR_ENCRYPTION_REQUIRED = 0x94 RESPONSE_ERROR_UNKNOWN = 0xFF OTA_VERSION_1_0 = 1 @@ -63,8 +64,20 @@ MAGIC_BYTES = [0x6C, 0x26, 0xF7, 0x5C, 0x45] CLIENT_FEATURE_SUPPORTS_COMPRESSION = 0x01 CLIENT_FEATURE_SUPPORTS_SHA256_AUTH = 0x02 CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL = 0x04 +CLIENT_FEATURE_SUPPORTS_NOISE = 0x08 SERVER_FEATURE_SUPPORTS_COMPRESSION = 0x01 SERVER_FEATURE_SUPPORTS_PARTITION_ACCESS = 0x02 +SERVER_FEATURE_SUPPORTS_NOISE = 0x04 + +NOISE_FRAME_INDICATOR = 0x01 +NOISE_HANDSHAKE_OK = 0x00 +# The device decrypts frames in its transfer buffer (OTA_BUFFER_SIZE, sized +# as this plus the 16-byte ChaCha20-Poly1305 MAC). 1024 divides the 8192-byte +# upload block exactly, so blocks tile into full frames with no runt. +NOISE_MAX_PLAINTEXT = 1024 +# Wire contract: the device sends exactly this reject reason for a bad MAC +NOISE_MAC_FAILURE_REASON = "Handshake MAC failure" +NOISE_PROLOGUE_INIT = b"NoiseOTAInit" # OTA types this client knows how to send. Future PRs that add bootloader/partition # updates extend this set. Anything outside the set is rejected up front so callers @@ -171,6 +184,12 @@ _ERROR_MESSAGES: dict[int, str] = { "enabled: the new firmware's version must be newer than the version the " "device is currently running." ), + RESPONSE_ERROR_ENCRYPTION_REQUIRED: ( + "The device requires an encrypted OTA connection but this upload has no " + "encryption key. Add 'encryption:' to the 'ota: platform: esphome' section " + "of the YAML this upload uses, or update your esphome installation if it " + "predates OTA encryption." + ), RESPONSE_ERROR_UNKNOWN: "Unknown error from ESP", } @@ -305,16 +324,149 @@ def send_check( raise OTANetworkError(f"sending {msg}: {err}") from err +class NoiseSocketWrapper: + """Runs the OTA session inside a Noise (ChaCha20-Poly1305) transport. + + Exposes the socket subset perform_ota uses. Frames are indicator 0x01, + 16-bit big-endian length, ciphertext; recv() drains one decrypted frame + at a time, sendall() keeps control units in one frame and splits data + at NOISE_MAX_PLAINTEXT. + """ + + def __init__(self, sock: socket.socket, psk: str, prologue: bytes) -> None: + # Deliberately lazy: the noise stack (noiseprotocol, cryptography) is + # only imported when an encrypted upload actually runs. + try: + from aioesphomeapi.noise import NoiseHandshake + except ImportError as err: + raise OTAError( + "OTA encryption requires a newer aioesphomeapi; update your " + "esphome installation (pip install -U esphome) and retry" + ) from err + # The aioesphomeapi import above already loaded cryptography; bind + # the exception once so recv() pays no per-frame import lookup + from cryptography.exceptions import InvalidTag + + self._invalid_tag = InvalidTag + self._sock = sock + try: + self._handshake = NoiseHandshake(psk, prologue) + except ValueError as err: + raise OTAError(f"Invalid OTA encryption key: {err}") from err + self._encrypt = None + self._decrypt = None + self._buffer = b"" + + # Only harmless socket controls pass through; byte-moving methods are + # deliberately absent so plaintext cannot leak past the transport. + def settimeout(self, timeout: float | None) -> None: + self._sock.settimeout(timeout) + + def setsockopt(self, level: int, optname: int, value: int) -> None: + self._sock.setsockopt(level, optname, value) + + def close(self) -> None: + self._sock.close() + + def do_handshake(self) -> None: + """Run the two-message NNpsk0 handshake and set up the transport ciphers.""" + try: + self._send_frame( + bytes([NOISE_HANDSHAKE_OK]) + self._handshake.write_message() + ) + payload = self._recv_frame() + except OSError as err: + raise OTANetworkError(f"noise handshake: {err}") from err + if not payload: + raise OTANetworkError("Device closed connection during the noise handshake") + if payload[0] != NOISE_HANDSHAKE_OK: + reason = payload[1:].decode("utf-8", "replace") + if reason == NOISE_MAC_FAILURE_REASON: + raise OTAError( + "Device rejected the handshake; is the OTA encryption key correct?" + ) + raise OTAError(f"Device rejected the noise handshake: {reason}") + try: + self._handshake.read_message(payload[1:]) + except (ValueError, self._invalid_tag) as err: + # InvalidTag is a wrong key; ValueError covers a device sending an + # invalid curve point, which cryptography rejects during the DH + raise OTAError( + "Noise handshake failed; is the OTA encryption key correct?" + ) from err + self._encrypt, self._decrypt = self._handshake.get_ciphers() + + def sendall(self, data: bytes) -> None: + frames: list[bytes] = [] + for offset in range(0, len(data), NOISE_MAX_PLAINTEXT): + ciphertext = self._encrypt.encrypt( + data[offset : offset + NOISE_MAX_PLAINTEXT] + ) + frames.append(self._frame_header(len(ciphertext))) + frames.append(ciphertext) + self._sock.sendall(b"".join(frames)) + + def recv(self, amount: int) -> bytes: + if not self._buffer: + ciphertext = self._recv_frame() + if not ciphertext: + return b"" # connection closed at a frame boundary + try: + self._buffer = self._decrypt.decrypt(ciphertext) + except self._invalid_tag as err: + # Retryable: a fresh connection renegotiates the session + raise OTANetworkError( + "Noise decryption failed (MAC mismatch); frame corrupted or tampered" + ) from err + if not self._buffer: + # Reject MAC-only frames so b"" always means the peer closed + raise OTANetworkError("Device sent an empty noise frame") + data = self._buffer[:amount] + self._buffer = self._buffer[amount:] + return data + + @staticmethod + def _frame_header(length: int) -> bytes: + return bytes([NOISE_FRAME_INDICATOR, (length >> 8) & 0xFF, length & 0xFF]) + + def _send_frame(self, payload: bytes) -> None: + self._sock.sendall(self._frame_header(len(payload)) + payload) + + def _recv_frame(self) -> bytes: + header = self._recv_exact(3, closed_ok=True) + if not header: + return b"" # connection closed at a frame boundary + # A malformed frame is a broken transport, not a device error; + # retryable so a fresh session is tried + if header[0] != NOISE_FRAME_INDICATOR: + raise OTANetworkError(f"Bad noise frame indicator 0x{header[0]:02X}") + length = (header[1] << 8) | header[2] + if length == 0: + raise OTANetworkError("Device sent an empty noise frame") + return self._recv_exact(length) + + def _recv_exact(self, amount: int, closed_ok: bool = False) -> bytes: + data = b"" + while len(data) < amount: + chunk = self._sock.recv(amount - len(data)) + if not chunk: + if closed_ok and not data: + return b"" + raise OSError("connection closed inside a noise frame") + data += chunk + return data + + def perform_ota( sock: socket.socket, password: str | None, file_handle: io.IOBase, filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, + noise_psk: str | None = None, ) -> None: - # Validate ota_type up front. It travels as a single byte on the wire, and - # passing an out-of-range value would only surface as a ValueError from - # bytes([ota_type]) deep inside send_check, bypassing OTAError handling. + # Validate up front; an out-of-range value would only surface as a + # ValueError deep inside send_check, bypassing OTAError handling if not isinstance(ota_type, int) or not 0 <= ota_type <= 0xFF: raise OTAError( f"Invalid ota_type {ota_type!r}; expected an integer in range 0-255" @@ -325,6 +477,11 @@ def perform_ota( f"Unsupported OTA type 0x{ota_type:02X}; this ESPHome supports: {supported}" ) + if noise_psk is not None and not noise_psk: + raise OTAError( + "An empty OTA encryption key was provided; refusing to upload in plaintext" + ) + file_contents = file_handle.read() file_size = len(file_contents) _LOGGER.info("Uploading %s (%s bytes)", filename, file_size) @@ -347,6 +504,8 @@ def perform_ota( | CLIENT_FEATURE_SUPPORTS_SHA256_AUTH | CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL ) + if noise_psk: + features_to_send |= CLIENT_FEATURE_SUPPORTS_NOISE send_check(sock, features_to_send, "features") features = receive_exactly( sock, @@ -369,6 +528,31 @@ def perform_ota( else: features = 0 + if noise_psk: + # Fail closed: never fall back to a plaintext upload when an + # encryption key is configured, an active attacker could otherwise + # strip the feature flag and capture the image (it contains the wifi + # credentials and the api encryption key). + if not (extended_proto and features & SERVER_FEATURE_SUPPORTS_NOISE): + raise OTAError( + "An OTA encryption key is configured but the device did not " + "offer encryption; refusing to send the image in plaintext. " + "If the running firmware predates OTA encryption, first update " + "it without the 'ota: encryption:' block (over a trusted " + "network or via USB), then restore the block and upload again." + ) + # The prologue binds every negotiation byte both sides saw, so any + # tampering with the plaintext preamble breaks the handshake. + prologue = ( + NOISE_PROLOGUE_INIT + + bytes(MAGIC_BYTES) + + bytes([RESPONSE_OK, version, features_to_send]) + + bytes([RESPONSE_FEATURE_FLAGS, features]) + ) + sock = NoiseSocketWrapper(sock, noise_psk, prologue) + sock.do_handshake() + _LOGGER.info("Encrypted connection established") + if ota_type != OTA_TYPE_UPDATE_APP: # Any non-app OTA type requires the extended protocol and the # partition-access server feature. Reject up front so the user gets @@ -572,6 +756,7 @@ def run_ota_impl_( password: str | None, filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, + noise_psk: str | None = None, ) -> tuple[int, str | None]: from esphome.core import CORE @@ -636,7 +821,7 @@ def run_ota_impl_( reached_device = True with contextlib.closing(sock), Path(filename).open("rb") as file_handle: try: - perform_ota(sock, password, file_handle, filename, ota_type) + perform_ota(sock, password, file_handle, filename, ota_type, noise_psk) except OTANetworkError as err: # Transient network failure; retry last_error = str(err) @@ -661,9 +846,12 @@ def run_ota( password: str | None, filename: Path, ota_type: int = OTA_TYPE_UPDATE_APP, + noise_psk: str | None = None, ) -> tuple[int, str | None]: try: - return run_ota_impl_(remote_host, remote_port, password, filename, ota_type) + return run_ota_impl_( + remote_host, remote_port, password, filename, ota_type, noise_psk + ) except OTAError as err: _LOGGER.error(err) return 1, None diff --git a/tests/component_tests/noise/test_encryption_key.py b/tests/component_tests/noise/test_encryption_key.py index 62abae6487..10f1eb3d4c 100644 --- a/tests/component_tests/noise/test_encryption_key.py +++ b/tests/component_tests/noise/test_encryption_key.py @@ -5,7 +5,11 @@ from __future__ import annotations import pytest from esphome import config_validation as cv -from esphome.components.noise import decode_encryption_key, validate_encryption_key +from esphome.components.noise import ( + decode_encryption_key, + is_reserved_key, + validate_encryption_key, +) KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" @@ -35,3 +39,8 @@ def test_decode_encryption_key_rejects_short_decode() -> None: a zero padded PSK on the device.""" with pytest.raises(cv.Invalid, match="32 bytes"): decode_encryption_key("AAECAw==") + + +def test_is_reserved_key() -> None: + assert is_reserved_key("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=") + assert not is_reserved_key(KEY) diff --git a/tests/component_tests/ota/test_esphome_ota.py b/tests/component_tests/ota/test_esphome_ota.py index cdac430ff7..873f162555 100644 --- a/tests/component_tests/ota/test_esphome_ota.py +++ b/tests/component_tests/ota/test_esphome_ota.py @@ -8,17 +8,25 @@ from typing import Any import pytest from esphome import config_validation as cv -from esphome.components.esphome.ota import ota_esphome_final_validate +from esphome.components.esphome.ota import ( + AUTO_LOAD, + FILTER_SOURCE_FILES, + _validate_no_password_with_encryption, + ota_esphome_final_validate, +) from esphome.const import ( + CONF_API, + CONF_ENCRYPTION, CONF_ESPHOME, CONF_ID, + CONF_KEY, CONF_OTA, CONF_PASSWORD, CONF_PLATFORM, CONF_PORT, CONF_VERSION, ) -from esphome.core import ID +from esphome.core import CORE, ID import esphome.final_validate as fv @@ -103,3 +111,305 @@ def test_non_esphome_ota_unaffected() -> None: assert len(updated[CONF_OTA]) == 3 finally: fv.full_config.reset(token) + + +API_KEY = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +OTHER_KEY = "AQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyA=" +ZEROS_KEY = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" + + +def test_encryption_key_inherited_from_api() -> None: + """A bare encryption block resolves to the api encryption key.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: API_KEY}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], + } + token = fv.full_config.set(full_conf) + try: + ota_esphome_final_validate({}) + updated = fv.full_config.get() + assert updated[CONF_OTA][0][CONF_ENCRYPTION][CONF_KEY] == API_KEY + finally: + fv.full_config.reset(token) + + +def test_encryption_explicit_key_matching_api_accepted() -> None: + """An explicit ota key equal to the api key validates.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: API_KEY}}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + ota_esphome_final_validate({}) + updated = fv.full_config.get() + assert updated[CONF_OTA][0][CONF_ENCRYPTION][CONF_KEY] == API_KEY + finally: + fv.full_config.reset(token) + + +def test_encryption_key_differing_from_api_rejected() -> None: + """There is one key per device; an ota key differing from the api key raises.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: API_KEY}}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="must match the 'api' encryption key"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_encryption_explicit_key_without_api_encryption_accepted() -> None: + """An explicit ota key with a plaintext api has nothing to match; it stands.""" + full_conf = { + CONF_API: {}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + ota_esphome_final_validate({}) + updated = fv.full_config.get() + assert updated[CONF_OTA][0][CONF_ENCRYPTION][CONF_KEY] == OTHER_KEY + finally: + fv.full_config.reset(token) + + +def test_encryption_without_any_key_rejected() -> None: + """A bare encryption block with no api key to inherit raises.""" + full_conf = { + CONF_API: {}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="no 'api' encryption key to inherit"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_encryption_explicit_all_zeros_key_rejected() -> None: + """The all-zeros key is the provisioning sentinel; the device would treat + it as no PSK and accept plaintext, so it must fail validation.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_encryption_inherited_all_zeros_key_rejected() -> None: + """An all-zeros api key must not silently disable ota encryption either.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {CONF_KEY: ZEROS_KEY}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="all-zeros key is reserved"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_encryption_key_mismatch_between_merged_configs_rejected() -> None: + """Same-port configs with different encryption keys raise.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}), + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + ] + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="encryption is inconsistent"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) + + +@pytest.mark.parametrize("keyed_first", [True, False]) +def test_encryption_bare_and_keyed_blocks_merge(keyed_first: bool) -> None: + """A bare encryption block (package/device split) is compatible with a + keyed one on the same port; the merge resolves to the keyed result.""" + keyed = _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + bare = _make_ota_config(port=3232, **{CONF_ENCRYPTION: {}}) + full_conf = { + CONF_OTA: [keyed, bare] if keyed_first else [bare, keyed], + } + token = fv.full_config.set(full_conf) + try: + ota_esphome_final_validate({}) + updated = fv.full_config.get() + assert len(updated[CONF_OTA]) == 1 + assert updated[CONF_OTA][0][CONF_ENCRYPTION][CONF_KEY] == OTHER_KEY + finally: + fv.full_config.reset(token) + + +def test_encryption_runtime_provisioned_api_key_not_inheritable() -> None: + """A keyless api encryption block provisions its key at runtime; a bare + ota encryption block cannot inherit it and the message says so.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {}}, + CONF_OTA: [_make_ota_config(port=3232, **{CONF_ENCRYPTION: {}})], + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="provisioned at runtime"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_encryption_explicit_key_with_runtime_provisioned_api_accepted() -> None: + """The documented remedy for a runtime-provisioned api key: set an + explicit ota key.""" + full_conf = { + CONF_API: {CONF_ENCRYPTION: {}}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}) + ], + } + token = fv.full_config.set(full_conf) + try: + ota_esphome_final_validate({}) + updated = fv.full_config.get() + assert updated[CONF_OTA][0][CONF_ENCRYPTION][CONF_KEY] == OTHER_KEY + finally: + fv.full_config.reset(token) + + +def test_encryption_with_web_server_ota_warns( + caplog: pytest.LogCaptureFixture, +) -> None: + """With the web_server component the plaintext /update endpoint is always + on; the combination validates with a warning.""" + full_conf = { + "web_server": {}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert any("plaintext /update" in record.message for record in caplog.records) + finally: + fv.full_config.reset(token) + + +def test_encryption_with_captive_portal_web_server_ota_warns( + caplog: pytest.LogCaptureFixture, +) -> None: + """captive_portal auto-loads the web_server ota platform without the + web_server component; encryption stays usable and only warns, so the + fallback AP recovery path is not lost.""" + full_conf = { + "captive_portal": {}, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert any("captive_portal" in record.message for record in caplog.records) + esphome_conf = next( + conf + for conf in fv.full_config.get()[CONF_OTA] + if conf.get(CONF_PLATFORM) == CONF_ESPHOME + ) + assert esphome_conf[CONF_ENCRYPTION][CONF_KEY] == OTHER_KEY + finally: + fv.full_config.reset(token) + + +def test_web_server_ota_without_encryption_unaffected() -> None: + """web_server ota stays valid alongside an unencrypted esphome entry.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + ota_esphome_final_validate({}) + assert len(fv.full_config.get()[CONF_OTA]) == 2 + finally: + fv.full_config.reset(token) + + +def test_auto_load_pulls_noise_only_for_encryption() -> None: + """A plain ota entry must never pull noise-c into the build.""" + assert AUTO_LOAD({CONF_PORT: 3232}) == ["sha256", "socket"] + assert "noise" in AUTO_LOAD({CONF_ENCRYPTION: {}}) + # Tooling probes must get the maximal set: None from dependency + # resolution, {} from the components-graph platform probe + assert "noise" in AUTO_LOAD(None) + assert "noise" in AUTO_LOAD({}) + + +def test_filter_source_files_excludes_noise_without_encryption() -> None: + """The noise transport source compiles only for encrypted builds.""" + old_config = CORE.config + try: + CORE.config = {CONF_OTA: [_make_ota_config(port=3232)]} + assert FILTER_SOURCE_FILES() == ["ota_esphome_noise.cpp"] + CORE.config = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}) + ] + } + assert FILTER_SOURCE_FILES() == [] + finally: + CORE.config = old_config + + +def test_password_with_encryption_rejected() -> None: + """The password and encryption options are mutually exclusive.""" + config = {CONF_PASSWORD: "pw", CONF_ENCRYPTION: {CONF_KEY: API_KEY}} + with pytest.raises(cv.Invalid, match="cannot be combined"): + _validate_no_password_with_encryption(config) + + +def test_password_alone_accepted() -> None: + """A password without encryption still validates.""" + config = {CONF_PASSWORD: "pw"} + assert _validate_no_password_with_encryption(config) is config + + +def test_merged_password_and_encryption_rejected() -> None: + """A password block and an encryption block merged on one port raise.""" + full_conf = { + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_PASSWORD: "pw"}), + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: API_KEY}}), + ] + } + token = fv.full_config.set(full_conf) + try: + with pytest.raises(cv.Invalid, match="cannot be combined"): + ota_esphome_final_validate({}) + finally: + fv.full_config.reset(token) diff --git a/tests/components/ota/encryption.yaml b/tests/components/ota/encryption.yaml new file mode 100644 index 0000000000..550d35caec --- /dev/null +++ b/tests/components/ota/encryption.yaml @@ -0,0 +1,9 @@ +wifi: + ssid: MySSID + password: password1 + +ota: + - platform: esphome + port: 3288 + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" diff --git a/tests/components/ota/encryption_inherit.yaml b/tests/components/ota/encryption_inherit.yaml new file mode 100644 index 0000000000..15ada6f810 --- /dev/null +++ b/tests/components/ota/encryption_inherit.yaml @@ -0,0 +1,12 @@ +wifi: + ssid: MySSID + password: password1 + +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + +ota: + - platform: esphome + port: 3289 + encryption: diff --git a/tests/components/ota/test-encryption.esp32-idf.yaml b/tests/components/ota/test-encryption.esp32-idf.yaml new file mode 100644 index 0000000000..000e38168e --- /dev/null +++ b/tests/components/ota/test-encryption.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include encryption.yaml diff --git a/tests/components/ota/test-encryption.esp8266-ard.yaml b/tests/components/ota/test-encryption.esp8266-ard.yaml new file mode 100644 index 0000000000..000e38168e --- /dev/null +++ b/tests/components/ota/test-encryption.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include encryption.yaml diff --git a/tests/components/ota/test-encryption.rp2040-ard.yaml b/tests/components/ota/test-encryption.rp2040-ard.yaml new file mode 100644 index 0000000000..000e38168e --- /dev/null +++ b/tests/components/ota/test-encryption.rp2040-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include encryption.yaml diff --git a/tests/components/ota/test-encryption_inherit.esp8266-ard.yaml b/tests/components/ota/test-encryption_inherit.esp8266-ard.yaml new file mode 100644 index 0000000000..71aa083e7e --- /dev/null +++ b/tests/components/ota/test-encryption_inherit.esp8266-ard.yaml @@ -0,0 +1,2 @@ +packages: + ota: !include encryption_inherit.yaml diff --git a/tests/integration/fixtures/host_ota_encrypted.yaml b/tests/integration/fixtures/host_ota_encrypted.yaml new file mode 100644 index 0000000000..0d11c99d3d --- /dev/null +++ b/tests/integration/fixtures/host_ota_encrypted.yaml @@ -0,0 +1,11 @@ +esphome: + name: host-ota-test +host: +api: +ota: + - platform: esphome + port: __OTA_PORT__ + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +logger: + level: DEBUG diff --git a/tests/integration/test_host_ota.py b/tests/integration/test_host_ota.py index e1036fdf1c..4e74814534 100644 --- a/tests/integration/test_host_ota.py +++ b/tests/integration/test_host_ota.py @@ -10,6 +10,7 @@ from __future__ import annotations import asyncio from collections.abc import Generator from contextlib import contextmanager +import functools import socket import pytest @@ -111,6 +112,62 @@ async def test_host_ota_self_update( assert proc.pid == pid_before +@pytest.mark.asyncio +async def test_host_ota_encrypted( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> None: + """Encrypted self-OTA succeeds; a plaintext upload to the same device fails.""" + pytest.importorskip("aioesphomeapi.noise") + noise_psk = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + api_port, api_socket = reserved_tcp_port + with _reserve_port() as (ota_port, ota_socket): + yaml_config = yaml_config.replace("__OTA_PORT__", str(ota_port)) + config_path = await write_yaml_config(yaml_config) + binary_path = await compile_esphome(config_path) + api_socket.close() + ota_socket.close() + + loop = asyncio.get_running_loop() + rebooted = loop.create_future() + + def on_log(line: str) -> None: + if not rebooted.done() and "Rebooting safely" in line: + rebooted.set_result(True) + + async with run_binary(binary_path, line_callback=on_log) as (proc, _lines): + await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) + pid_before = proc.pid + + # A plaintext upload must be refused with the device unharmed + rc, _ = await loop.run_in_executor( + None, espota2.run_ota, LOCALHOST, ota_port, None, binary_path + ) + assert rc == 1, "plaintext upload to an encrypted device must fail" + await asyncio.sleep(0.5) + assert proc.returncode is None, "process died on rejected plaintext OTA" + + # The encrypted upload goes through and the device re-execs + rc, _ = await loop.run_in_executor( + None, + functools.partial( + espota2.run_ota, + LOCALHOST, + ota_port, + None, + binary_path, + noise_psk=noise_psk, + ), + ) + assert rc == 0, "encrypted OTA reported failure" + await asyncio.wait_for(rebooted, timeout=10.0) + await _wait_for_port(LOCALHOST, api_port, PORT_WAIT_TIMEOUT) + assert proc.returncode is None, "process exited instead of execing" + assert proc.pid == pid_before + + @pytest.mark.asyncio async def test_host_ota_rejects_garbage( yaml_config: str, diff --git a/tests/unit_tests/test_espota2_noise.py b/tests/unit_tests/test_espota2_noise.py new file mode 100644 index 0000000000..5b43d05530 --- /dev/null +++ b/tests/unit_tests/test_espota2_noise.py @@ -0,0 +1,407 @@ +"""Unit tests for encrypted OTA uploads in esphome.espota2. + +A fake device implementing the responder side of the wire protocol (via +noiseprotocol, which esphome already has through aioesphomeapi) serves a real +TCP loopback connection, so these exercise the actual handshake, framing, and +cipher interop of the client code. Tests that need the client-side crypto skip +when the installed aioesphomeapi predates the noise module. +""" + +from __future__ import annotations + +import base64 +import hashlib +import io +from pathlib import Path +import socket +import sys +import threading +from unittest.mock import Mock, patch + +import pytest + +from esphome import espota2 + +PSK = base64.b64encode(bytes(range(32))).decode() +OTHER_PSK = base64.b64encode(bytes(range(1, 33))).decode() + +MAGIC = bytes(espota2.MAGIC_BYTES) + + +def _recv_exact(sock: socket.socket, amount: int) -> bytes: + data = b"" + while len(data) < amount: + chunk = sock.recv(amount - len(data)) + if not chunk: + raise ConnectionError("client closed") + data += chunk + return data + + +def _frame(payload: bytes) -> bytes: + return ( + bytes([espota2.NOISE_FRAME_INDICATOR, len(payload) >> 8, len(payload) & 0xFF]) + + payload + ) + + +def _send_frame(sock: socket.socket, payload: bytes) -> None: + sock.sendall(_frame(payload)) + + +def _recv_frame(sock: socket.socket) -> bytes: + header = _recv_exact(sock, 3) + assert header[0] == 0x01 + return _recv_exact(sock, (header[1] << 8) | header[2]) + + +class FakeEncryptedDevice(threading.Thread): + """Responder side of the encrypted OTA wire protocol.""" + + def __init__( + self, + psk: str = PSK, + version: int = 2, + offer_noise: bool = True, + require_noise: bool = True, + prologue_features_override: int | None = None, + ) -> None: + super().__init__(daemon=True) + self.psk = psk + self.version = version + self.offer_noise = offer_noise + self.require_noise = require_noise + self.prologue_features_override = prologue_features_override + self.received: bytes | None = None + self.error: Exception | None = None + self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen(1) + self.port = self.listener.getsockname()[1] + + def run(self) -> None: + try: + sock, _ = self.listener.accept() + sock.settimeout(10) + with sock: + self._serve(sock) + except Exception as err: # noqa: BLE001 - surfaced via join_and_check + self.error = err + finally: + self.listener.close() + + def join_and_check(self) -> None: + self.join(timeout=10) + assert not self.is_alive(), "fake device did not finish" + if self.error is not None: + raise self.error + + def _serve(self, sock: socket.socket) -> None: + assert _recv_exact(sock, 5) == MAGIC + sock.sendall(bytes([espota2.RESPONSE_OK, self.version])) + features = _recv_exact(sock, 1)[0] + noise_negotiated = bool( + features & espota2.CLIENT_FEATURE_SUPPORTS_NOISE + and features & espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + ) + if self.require_noise and not noise_negotiated: + sock.sendall(bytes([espota2.RESPONSE_ERROR_ENCRYPTION_REQUIRED])) + return + server_flags = espota2.SERVER_FEATURE_SUPPORTS_NOISE if self.offer_noise else 0 + sock.sendall(bytes([espota2.RESPONSE_FEATURE_FLAGS, server_flags])) + if not (self.offer_noise and noise_negotiated): + return # the client fails closed; nothing further arrives + + from cryptography.exceptions import InvalidTag + from noise.connection import NoiseConnection + + prologue_features = ( + features + if self.prologue_features_override is None + else self.prologue_features_override + ) + prologue = ( + espota2.NOISE_PROLOGUE_INIT + + MAGIC + + bytes([espota2.RESPONSE_OK, self.version, prologue_features]) + + bytes([espota2.RESPONSE_FEATURE_FLAGS, server_flags]) + ) + proto = NoiseConnection.from_name(b"Noise_NNpsk0_25519_ChaChaPoly_SHA256") + proto.set_as_responder() + proto.set_psks(base64.b64decode(self.psk)) + proto.set_prologue(prologue) + proto.start_handshake() + + msg1 = _recv_frame(sock) + assert msg1[0] == 0x00 + try: + proto.read_message(msg1[1:]) + except InvalidTag: + _send_frame(sock, b"\x01" + espota2.NOISE_MAC_FAILURE_REASON.encode()) + return + _send_frame(sock, b"\x00" + bytes(proto.write_message())) + + def send_byte(byte: int) -> None: + _send_frame(sock, proto.encrypt(bytes([byte]))) + + def recv_unit(length: int) -> bytes: + plaintext = proto.decrypt(_recv_frame(sock)) + assert len(plaintext) == length, "control units must be one per frame" + return plaintext + + send_byte(espota2.RESPONSE_AUTH_OK) + recv_unit(1) # ota type + size = int.from_bytes(recv_unit(4), "big") + send_byte(espota2.RESPONSE_UPDATE_PREPARE_OK) + md5_hex = recv_unit(32) + send_byte(espota2.RESPONSE_BIN_MD5_OK) + + received = b"" + acked = 0 + while len(received) < size: + plaintext = proto.decrypt(_recv_frame(sock)) + assert 0 < len(plaintext) <= espota2.NOISE_MAX_PLAINTEXT + received += plaintext + if self.version >= espota2.OTA_VERSION_2_0: + while acked + espota2.UPLOAD_BLOCK_SIZE <= len(received) or ( + len(received) == size and acked < size + ): + send_byte(espota2.RESPONSE_CHUNK_OK) + acked += espota2.UPLOAD_BLOCK_SIZE + assert hashlib.md5(received).hexdigest().encode() == md5_hex + send_byte(espota2.RESPONSE_RECEIVE_OK) + send_byte(espota2.RESPONSE_UPDATE_END_OK) + assert recv_unit(1) == bytes([espota2.RESPONSE_OK]) + self.received = received + + +def _upload( + device: FakeEncryptedDevice, firmware: bytes, noise_psk: str | None +) -> None: + device.start() + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(10) + sock.connect(("127.0.0.1", device.port)) + try: + espota2.perform_ota( + sock, None, io.BytesIO(firmware), Path("firmware.bin"), noise_psk=noise_psk + ) + finally: + sock.close() + + +def test_encrypted_upload_success() -> None: + """A full encrypted v2 upload spanning several 8192-byte blocks.""" + pytest.importorskip("aioesphomeapi.noise") + firmware = bytes(range(256)) * 80 # 20480 bytes, crosses chunk-ack boundaries + device = FakeEncryptedDevice() + with patch("time.sleep"): + _upload(device, firmware, PSK) + device.join_and_check() + assert device.received == firmware + + +def test_encrypted_upload_version_1() -> None: + """Version 1 protocol (no chunk acks) works through the noise transport.""" + pytest.importorskip("aioesphomeapi.noise") + firmware = b"v1 firmware image" * 100 + device = FakeEncryptedDevice(version=1) + with patch("time.sleep"): + _upload(device, firmware, PSK) + device.join_and_check() + assert device.received == firmware + + +def test_wrong_key_fails_with_clear_error() -> None: + """A key mismatch surfaces the device's handshake reject readably.""" + pytest.importorskip("aioesphomeapi.noise") + device = FakeEncryptedDevice(psk=OTHER_PSK) + with pytest.raises(espota2.OTAError, match="encryption key correct"): + _upload(device, b"firmware", PSK) + device.join_and_check() + + +def test_tampered_negotiation_breaks_handshake() -> None: + """A negotiation byte differing between the sides breaks the prologue MAC.""" + pytest.importorskip("aioesphomeapi.noise") + device = FakeEncryptedDevice( + prologue_features_override=espota2.CLIENT_FEATURE_SUPPORTS_EXTENDED_PROTOCOL + ) + with pytest.raises(espota2.OTAError, match="encryption key correct"): + _upload(device, b"firmware", PSK) + device.join_and_check() + + +def test_client_fails_closed_when_device_lacks_encryption() -> None: + """With a key configured, a device not offering noise aborts the upload.""" + device = FakeEncryptedDevice(offer_noise=False, require_noise=False) + with pytest.raises(espota2.OTAError, match="refusing to send the image"): + _upload(device, b"firmware", PSK) + device.join_and_check() + + +def test_plaintext_client_gets_encryption_required_error() -> None: + """A client without a key gets the device's 0x94 error message.""" + device = FakeEncryptedDevice() + with pytest.raises(espota2.OTAError, match="requires an encrypted OTA"): + _upload(device, b"firmware", None) + device.join_and_check() + + +def test_missing_aioesphomeapi_noise_module_message() -> None: + """An aioesphomeapi without the noise module produces a clear error.""" + with ( + patch.dict(sys.modules, {"aioesphomeapi.noise": None}), + pytest.raises(espota2.OTAError, match="requires a newer aioesphomeapi"), + ): + espota2.NoiseSocketWrapper(Mock(), PSK, b"prologue") + + +class ScriptedSocket: + """Serves scripted recv chunks; b"" means the peer closed.""" + + def __init__(self, *chunks: bytes | Exception) -> None: + self.chunks = list(chunks) + self.sent: list[bytes] = [] + + def sendall(self, data: bytes) -> None: + self.sent.append(data) + + def settimeout(self, timeout: float) -> None: + pass + + def recv(self, amount: int) -> bytes: + if not self.chunks: + return b"" + chunk = self.chunks[0] + if isinstance(chunk, Exception): + self.chunks.pop(0) + raise chunk + take, rest = chunk[:amount], chunk[amount:] + if rest: + self.chunks[0] = rest + else: + self.chunks.pop(0) + return take + + +def _wrapper(*chunks: bytes | Exception) -> espota2.NoiseSocketWrapper: + pytest.importorskip("aioesphomeapi.noise") + return espota2.NoiseSocketWrapper(ScriptedSocket(*chunks), PSK, b"prologue") + + +def test_wrapper_rejects_malformed_psk() -> None: + pytest.importorskip("aioesphomeapi.noise") + with pytest.raises(espota2.OTAError, match="Invalid OTA encryption key"): + espota2.NoiseSocketWrapper(ScriptedSocket(), "not-base64!!!", b"prologue") + + +def test_handshake_socket_error_is_network_error() -> None: + wrapper = _wrapper(OSError("boom")) + with pytest.raises(espota2.OTANetworkError, match="noise handshake"): + wrapper.do_handshake() + + +def test_handshake_closed_at_frame_boundary() -> None: + wrapper = _wrapper() + with pytest.raises(espota2.OTANetworkError, match="closed connection during"): + wrapper.do_handshake() + + +def test_handshake_reject_with_other_reason() -> None: + wrapper = _wrapper(_frame(b"\x01Handshake error")) + with pytest.raises( + espota2.OTAError, match="rejected the noise handshake: Handshake error" + ): + wrapper.do_handshake() + + +def test_handshake_garbage_second_message() -> None: + """A valid-looking point with a garbage MAC fails cleanly.""" + wrapper = _wrapper(_frame(b"\x00" + bytes(range(48)))) + with pytest.raises( + espota2.OTAError, match="handshake failed; is the OTA encryption key" + ): + wrapper.do_handshake() + + +def test_handshake_invalid_curve_point() -> None: + """An all-zero x25519 point is rejected as a clean error, not a crash.""" + wrapper = _wrapper(_frame(b"\x00" + bytes(48))) + with pytest.raises( + espota2.OTAError, match="handshake failed; is the OTA encryption key" + ): + wrapper.do_handshake() + + +def test_recv_closed_at_frame_boundary_returns_empty() -> None: + wrapper = _wrapper() + assert wrapper.recv(1) == b"" + + +def test_recv_corrupt_frame_is_retryable_network_error() -> None: + from cryptography.exceptions import InvalidTag + + wrapper = _wrapper(_frame(b"ciphertext")) + wrapper._decrypt = Mock(decrypt=Mock(side_effect=InvalidTag())) + with pytest.raises(espota2.OTANetworkError, match="decryption failed"): + wrapper.recv(1) + + +def test_wrapper_blocks_unencrypted_socket_methods() -> None: + """Byte-moving socket methods must not bypass the encrypted transport.""" + wrapper = _wrapper() + # The harmless socket controls pass through to the wrapped socket + wrapper._sock = Mock() + wrapper.settimeout(1) + wrapper._sock.settimeout.assert_called_once_with(1) + wrapper.setsockopt(6, 1, 1) + wrapper._sock.setsockopt.assert_called_once_with(6, 1, 1) + wrapper.close() + wrapper._sock.close.assert_called_once_with() + with pytest.raises(AttributeError): + _ = wrapper.send + with pytest.raises(AttributeError): + _ = wrapper.recv_into + + +def test_recv_empty_plaintext_frame_is_protocol_error() -> None: + """A MAC-only frame decrypts to nothing; b'' from recv must mean close.""" + wrapper = _wrapper(_frame(bytes(16))) + wrapper._decrypt = Mock(decrypt=Mock(return_value=b"")) + with pytest.raises(espota2.OTANetworkError, match="empty noise frame"): + wrapper.recv(1) + + +def test_recv_frame_bad_indicator_is_retryable() -> None: + wrapper = _wrapper(b"\x02\x00\x01x") + with pytest.raises(espota2.OTANetworkError, match="Bad noise frame indicator"): + wrapper._recv_frame() + + +def test_recv_frame_zero_length_is_retryable() -> None: + wrapper = _wrapper(bytes([espota2.NOISE_FRAME_INDICATOR, 0, 0])) + with pytest.raises(espota2.OTANetworkError, match="empty noise frame"): + wrapper._recv_frame() + + +def test_perform_ota_blank_key_refuses_plaintext() -> None: + with pytest.raises(espota2.OTAError, match="empty OTA encryption key"): + espota2.perform_ota( + ScriptedSocket(), None, io.BytesIO(b"x"), Path("f.bin"), noise_psk="" + ) + + +def test_recv_exact_closed_mid_frame() -> None: + wrapper = _wrapper(_frame(b"partial")[:5]) + with pytest.raises(OSError, match="closed inside a noise frame"): + wrapper._recv_frame() + + +def test_recv_serves_buffered_plaintext_without_new_frame() -> None: + """A second recv drains the decrypted buffer without reading another frame.""" + wrapper = _wrapper(_frame(b"ciphertext")) + wrapper._decrypt = Mock(decrypt=Mock(return_value=b"AB")) + assert wrapper.recv(1) == b"A" # reads and decrypts one frame + assert wrapper.recv(1) == b"B" # served from the buffer, no new frame + wrapper._decrypt.decrypt.assert_called_once() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 15b1105ed0..5372a7203d 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -86,7 +86,9 @@ from esphome.const import ( CONF_BROKER, CONF_DISABLED, CONF_DISCOVER_IP, + CONF_ENCRYPTION, CONF_ESPHOME, + CONF_KEY, CONF_LEVEL, CONF_LOG, CONF_LOG_TOPIC, @@ -2106,10 +2108,65 @@ def test_upload_program_ota_success( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, "secret", expected_firmware, OTA_TYPE_UPDATE_APP + ["192.168.1.100"], 3232, "secret", expected_firmware, OTA_TYPE_UPDATE_APP, None ) +def test_upload_program_ota_encryption_key( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """The resolved encryption key is passed through to run_ota.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_ota.return_value = (0, "192.168.1.100") + + key = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" + config = { + CONF_OTA: [ + { + CONF_PLATFORM: CONF_ESPHOME, + CONF_PORT: 3232, + CONF_ENCRYPTION: {CONF_KEY: key}, + } + ] + } + exit_code, host = upload_program(config, MockArgs(), ["192.168.1.100"]) + + assert exit_code == 0 + assert host == "192.168.1.100" + expected_firmware = ( + tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" + ) + mock_run_ota.assert_called_once_with( + ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, key + ) + + +def test_upload_program_ota_encryption_without_key_fails_closed( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """An encryption block with no resolved key must never upload plaintext.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + + config = { + CONF_OTA: [ + { + CONF_PLATFORM: CONF_ESPHOME, + CONF_PORT: 3232, + CONF_ENCRYPTION: {}, + } + ] + } + with pytest.raises(EsphomeError, match="no key was resolved"): + upload_program(config, MockArgs(), ["192.168.1.100"]) + mock_run_ota.assert_not_called() + + def test_upload_program_ota_with_file_arg( mock_run_ota: Mock, mock_get_port_type: Mock, @@ -2137,7 +2194,7 @@ def test_upload_program_ota_with_file_arg( assert exit_code == 0 assert host == "192.168.1.100" mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, Path("custom.bin"), OTA_TYPE_UPDATE_APP + ["192.168.1.100"], 3232, None, Path("custom.bin"), OTA_TYPE_UPDATE_APP, None ) @@ -2192,6 +2249,7 @@ def test_upload_program_ota_partition_table_with_file_arg( None, partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, + None, ) @@ -2253,6 +2311,7 @@ def test_upload_program_ota_partition_table_mqttip( None, partition_file, OTA_TYPE_UPDATE_PARTITION_TABLE, + None, ) @@ -2440,6 +2499,7 @@ def test_upload_program_ota_bootloader_with_file_arg( None, bootloader_file, OTA_TYPE_UPDATE_BOOTLOADER, + None, ) @@ -2602,6 +2662,42 @@ def test_has_web_server_logging_respects_log_disabled() -> None: assert has_web_server_logging() is False +def test_upload_program_web_server_warns_when_encryption_configured( + mock_run_web_server_ota: Mock, + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Explicitly picking web_server OTA on an encrypted config warns about + the plaintext upload path.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path) + mock_get_port_type.return_value = "NETWORK" + mock_run_web_server_ota.return_value = (0, "192.168.1.100") + + config = { + CONF_OTA: [ + { + CONF_PLATFORM: CONF_ESPHOME, + CONF_PORT: 3232, + CONF_ENCRYPTION: {CONF_KEY: "test_key"}, + }, + {CONF_PLATFORM: CONF_WEB_SERVER}, + ], + CONF_WEB_SERVER: { + CONF_PORT: 80, + CONF_AUTH: {CONF_USERNAME: "admin", CONF_PASSWORD: "pw"}, + }, + } + args = MockArgs(ota_platform=CONF_WEB_SERVER) + with caplog.at_level(logging.WARNING): + exit_code, _ = upload_program(config, args, ["192.168.1.100"]) + + assert exit_code == 0 + assert any("plaintext HTTP" in record.message for record in caplog.records) + mock_run_ota.assert_not_called() + + def test_upload_program_web_server_only_auto_dispatches( mock_run_web_server_ota: Mock, mock_run_ota: Mock, @@ -2892,7 +2988,7 @@ def test_upload_program_ota_with_mqtt_resolution( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP + ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None ) @@ -2942,7 +3038,7 @@ def test_upload_program_ota_with_mqtt_empty_broker( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.50"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP + ["192.168.1.50"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None ) # Verify warning was logged assert "MQTT IP discovery failed" in caplog.text @@ -5114,6 +5210,7 @@ def test_upload_program_ota_static_ip_with_mqttip( None, expected_firmware, OTA_TYPE_UPDATE_APP, + None, ) @@ -5163,6 +5260,7 @@ def test_upload_program_ota_multiple_mqttip_resolves_once( None, expected_firmware, OTA_TYPE_UPDATE_APP, + None, ) @@ -5340,7 +5438,7 @@ def test_upload_program_ota_mqtt_timeout_fallback( tmp_path / ".esphome" / "build" / "test" / ".pioenvs" / "test" / "firmware.bin" ) mock_run_ota.assert_called_once_with( - ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP + ["192.168.1.100"], 3232, None, expected_firmware, OTA_TYPE_UPDATE_APP, None ) From 379e077b5f4b83a844536746f1c6d8d66b6e094b Mon Sep 17 00:00:00 2001 From: Jake <106696989+JakeLC15@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:05:56 -0400 Subject: [PATCH 23/25] [ds1603l] New sensor DS1603L V1.0 (#13133) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Jesse Hills <3060199+jesserockz@users.noreply.github.com> --- CODEOWNERS | 1 + esphome/components/ds1603l/__init__.py | 0 esphome/components/ds1603l/ds1603l.cpp | 68 +++++++++++++++++++ esphome/components/ds1603l/ds1603l.h | 30 ++++++++ esphome/components/ds1603l/sensor.py | 43 ++++++++++++ tests/components/ds1603l/common.yaml | 3 + tests/components/ds1603l/test.esp32-idf.yaml | 7 ++ .../components/ds1603l/test.esp8266-ard.yaml | 7 ++ 8 files changed, 159 insertions(+) create mode 100644 esphome/components/ds1603l/__init__.py create mode 100644 esphome/components/ds1603l/ds1603l.cpp create mode 100644 esphome/components/ds1603l/ds1603l.h create mode 100644 esphome/components/ds1603l/sensor.py create mode 100644 tests/components/ds1603l/common.yaml create mode 100644 tests/components/ds1603l/test.esp32-idf.yaml create mode 100644 tests/components/ds1603l/test.esp8266-ard.yaml diff --git a/CODEOWNERS b/CODEOWNERS index 7bb7f310a3..3429a93aa7 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -149,6 +149,7 @@ esphome/components/display_menu_base/* @numo68 esphome/components/dlms_meter/* @latonita @PolarGoose @SimonFischer04 @Tomer27cz esphome/components/dps310/* @kbx81 esphome/components/ds1307/* @badbadc0ffee +esphome/components/ds1603l/* @JakeLC15 esphome/components/ds2484/* @mrk-its esphome/components/ds248x/* @tomwellnitz esphome/components/dsmr/* @glmnet @PolarGoose diff --git a/esphome/components/ds1603l/__init__.py b/esphome/components/ds1603l/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esphome/components/ds1603l/ds1603l.cpp b/esphome/components/ds1603l/ds1603l.cpp new file mode 100644 index 0000000000..b0b0ef8175 --- /dev/null +++ b/esphome/components/ds1603l/ds1603l.cpp @@ -0,0 +1,68 @@ +#include "ds1603l.h" + +#include + +#include "esphome/core/helpers.h" +#include "esphome/core/log.h" + +namespace esphome::ds1603l { + +static const char *const TAG = "ds1603l.sensor"; + +void DS1603L::loop() { + // Assemble frames one byte at a time so a stream that starts mid-frame can realign + uint8_t byte; + while (this->available() > 0 && this->read_byte(&byte)) { + if (this->rx_count_ == 0 && byte != HEADER_BYTE) { + ESP_LOGV(TAG, "Skipping byte 0x%02X while looking for header", byte); + continue; + } + + this->rx_buffer_[this->rx_count_++] = byte; + if (this->rx_count_ < FRAME_SIZE) { + continue; + } + + if (this->parse_data_()) { + this->rx_count_ = 0; + } else { + // The header byte was part of the payload of a misaligned frame, so realign instead of dropping everything + this->resync_(); + } + } +} + +void DS1603L::dump_config() { LOG_SENSOR("", "DS1603L", this); } + +bool DS1603L::parse_data_() { + uint8_t header = this->rx_buffer_[0]; + uint8_t data_h = this->rx_buffer_[1]; + uint8_t data_l = this->rx_buffer_[2]; + uint8_t checksum = this->rx_buffer_[3]; + + uint8_t computed_checksum = (header + data_h + data_l) & 0xFF; + + ESP_LOGV(TAG, "Data: Header=0x%02X, Data_H=0x%02X, Data_L=0x%02X, Checksum=0x%02X", header, data_h, data_l, checksum); + + if (checksum != computed_checksum) { + ESP_LOGW(TAG, "Checksum mismatch: received 0x%02X, expected 0x%02X", checksum, computed_checksum); + return false; + } + + this->publish_state(encode_uint16(data_h, data_l)); + return true; +} + +void DS1603L::resync_() { + // Drop the byte that was treated as the header, then look for the next candidate header in what is left + size_t start = 1; + while (start < this->rx_count_ && this->rx_buffer_[start] != HEADER_BYTE) { + start++; + } + this->rx_count_ -= start; + if (this->rx_count_ > 0) { + memmove(this->rx_buffer_, this->rx_buffer_ + start, this->rx_count_); + } +} + +} // namespace esphome::ds1603l diff --git a/esphome/components/ds1603l/ds1603l.h b/esphome/components/ds1603l/ds1603l.h new file mode 100644 index 0000000000..9041681f5e --- /dev/null +++ b/esphome/components/ds1603l/ds1603l.h @@ -0,0 +1,30 @@ +#pragma once + +#include +#include + +#include "esphome/components/sensor/sensor.h" +#include "esphome/components/uart/uart.h" +#include "esphome/core/component.h" + +namespace esphome::ds1603l { + +class DS1603L final : public sensor::Sensor, public Component, public uart::UARTDevice { + public: + void loop() override; + void dump_config() override; + + protected: + static constexpr uint8_t HEADER_BYTE = 0xFF; + static constexpr size_t FRAME_SIZE = 4; + + // Validates the checksum of the frame in rx_buffer_ and publishes it. Returns false if the frame is invalid. + bool parse_data_(); + // Drops the first buffered byte and realigns the buffer on the next possible header byte. + void resync_(); + + uint8_t rx_buffer_[FRAME_SIZE]; // Buffer for the frame being assembled + size_t rx_count_{0}; // Number of bytes currently in rx_buffer_ +}; + +} // namespace esphome::ds1603l diff --git a/esphome/components/ds1603l/sensor.py b/esphome/components/ds1603l/sensor.py new file mode 100644 index 0000000000..c4f117c603 --- /dev/null +++ b/esphome/components/ds1603l/sensor.py @@ -0,0 +1,43 @@ +import esphome.codegen as cg +from esphome.components import sensor, uart +import esphome.config_validation as cv +from esphome.const import ( + DEVICE_CLASS_DISTANCE, + STATE_CLASS_MEASUREMENT, + UNIT_MILLIMETER, +) +from esphome.types import ConfigType + +CODEOWNERS = ["@JakeLC15"] +DEPENDENCIES = ["uart"] + +ds1603l_ns = cg.esphome_ns.namespace("ds1603l") +DS1603L = ds1603l_ns.class_("DS1603L", sensor.Sensor, cg.Component, uart.UARTDevice) + + +CONFIG_SCHEMA = ( + sensor.sensor_schema( + DS1603L, + unit_of_measurement=UNIT_MILLIMETER, + accuracy_decimals=0, + device_class=DEVICE_CLASS_DISTANCE, + state_class=STATE_CLASS_MEASUREMENT, + ) + .extend(uart.UART_DEVICE_SCHEMA) + .extend(cv.COMPONENT_SCHEMA) +) + +FINAL_VALIDATE_SCHEMA = uart.final_validate_device_schema( + "ds1603l", + baud_rate=9600, + require_tx=False, + require_rx=True, + data_bits=8, + stop_bits=1, +) + + +async def to_code(config: ConfigType) -> None: + var = await sensor.new_sensor(config) + await cg.register_component(var, config) + await uart.register_uart_device(var, config) diff --git a/tests/components/ds1603l/common.yaml b/tests/components/ds1603l/common.yaml new file mode 100644 index 0000000000..d47ef1b610 --- /dev/null +++ b/tests/components/ds1603l/common.yaml @@ -0,0 +1,3 @@ +sensor: + - platform: ds1603l + name: ds1603l Distance diff --git a/tests/components/ds1603l/test.esp32-idf.yaml b/tests/components/ds1603l/test.esp32-idf.yaml new file mode 100644 index 0000000000..544827f577 --- /dev/null +++ b/tests/components/ds1603l/test.esp32-idf.yaml @@ -0,0 +1,7 @@ +substitutions: + tx_pin: GPIO1 + rx_pin: GPIO3 + +packages: + uart: !include ../../test_build_components/common/uart/esp32-idf.yaml + ds1603l: !include common.yaml diff --git a/tests/components/ds1603l/test.esp8266-ard.yaml b/tests/components/ds1603l/test.esp8266-ard.yaml new file mode 100644 index 0000000000..878e45899b --- /dev/null +++ b/tests/components/ds1603l/test.esp8266-ard.yaml @@ -0,0 +1,7 @@ +substitutions: + tx_pin: GPIO0 + rx_pin: GPIO2 + +packages: + uart: !include ../../test_build_components/common/uart/esp8266-ard.yaml + ds1603l: !include common.yaml From 8bef5b22e112503be305221b6167ea6a2b4e0780 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:36:04 -0400 Subject: [PATCH 24/25] Bump zeroconf from 0.150.4 to 0.151.2 (#18934) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f19559dca8..594b44432d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ esptool==5.3.1 click==8.3.3 aioesphomeapi==46.3.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi -zeroconf==0.150.4 +zeroconf==0.151.2 puremagic==2.2.0 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import From b8480b8424a3b6248c234ed0f78291813129e144 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:36:49 -0400 Subject: [PATCH 25/25] Bump pylint from 4.0.7 to 4.0.8 (#18935) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index e837953878..b1309ec63b 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,4 +1,4 @@ -pylint==4.0.7 +pylint==4.0.8 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating ruff==0.16.5 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating