From fc803fcc6d771c0b0e606d6fa13177b25dcd5c77 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 28 Sep 2026 20:06:53 +0200 Subject: [PATCH 001/133] [libretiny] Enable the PlatformIO precompiled header on Linux hosts (#19827) --- esphome/build_helpers/pch.py | 16 ++++++++++++++-- tests/unit_tests/build_helpers/test_pch.py | 15 ++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/esphome/build_helpers/pch.py b/esphome/build_helpers/pch.py index dc23942c8e8..bd13726d85f 100644 --- a/esphome/build_helpers/pch.py +++ b/esphome/build_helpers/pch.py @@ -9,6 +9,7 @@ import os from pathlib import Path import posixpath import re +import sys from esphome.build_helpers.ccache import effective_ccache_basedir, parse_enable_env from esphome.const import ( @@ -47,10 +48,18 @@ PCH_DEFAULT_HEADERS = ("esphome/core/pch_prefix.h",) # PlatformIO platforms that do not take the pch script PCH_SCRIPT_EXCLUDED_PLATFORMS = frozenset( + { + PLATFORM_NRF52, + } +) + +# The GCC 10.3 of LibreTiny only loads its own .gch back on Linux: elsewhere +# the compiler is loaded at a new address each run, which GCC accepts from +# version 12. Remove once LibreTiny ships GCC 12 or newer +PCH_SCRIPT_LINUX_ONLY_PLATFORMS = frozenset( { PLATFORM_BK72XX, PLATFORM_LN882X, - PLATFORM_NRF52, PLATFORM_RTL87XX, } ) @@ -94,7 +103,10 @@ def pch_script_enabled() -> bool: """Whether this PlatformIO build takes the pch script.""" from esphome.core import CORE - return pch_enabled() and CORE.target_platform not in PCH_SCRIPT_EXCLUDED_PLATFORMS + platform = CORE.target_platform + if platform in PCH_SCRIPT_LINUX_ONLY_PLATFORMS and sys.platform != "linux": + return False + return pch_enabled() and platform not in PCH_SCRIPT_EXCLUDED_PLATFORMS def pch_header_text(include_headers: Iterable[str]) -> str: diff --git a/tests/unit_tests/build_helpers/test_pch.py b/tests/unit_tests/build_helpers/test_pch.py index a64e9525e67..0ea82157381 100644 --- a/tests/unit_tests/build_helpers/test_pch.py +++ b/tests/unit_tests/build_helpers/test_pch.py @@ -109,14 +109,27 @@ def test_pch_checksum_tracks_closure_content(tmp_path: Path) -> None: assert base != pch.pch_checksum(tmp_path, ["root.h"], ["id"]) -@pytest.mark.parametrize("platform", ["esp32", "esp8266", "rp2"]) +@pytest.mark.parametrize( + "platform", ["bk72xx", "esp32", "esp8266", "ln882x", "rp2", "rtl87xx"] +) def test_pch_script_enabled(platform: str, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pch.sys, "platform", "linux") CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform} assert pch.pch_script_enabled() monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") assert not pch.pch_script_enabled() +@pytest.mark.parametrize("host", ["darwin", "win32"]) +@pytest.mark.parametrize("platform", sorted(pch.PCH_SCRIPT_LINUX_ONLY_PLATFORMS)) +def test_pch_script_linux_only_platform( + platform: str, host: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(pch.sys, "platform", host) + CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform} + assert not pch.pch_script_enabled() + + @pytest.mark.parametrize("platform", sorted(pch.PCH_SCRIPT_EXCLUDED_PLATFORMS)) def test_pch_script_excluded_platform(platform: str) -> None: CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform} From 9e8044868c4260df3f753f6a15ba373c897f13b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:30:32 -0400 Subject: [PATCH 002/133] Bump filelock from 4.0.1 to 4.0.3 (#19852) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 81f3cbcdd2d..15c14642b37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ requests==2.34.2 py7zr==1.1.3 platformdirs==4.11.12 # native esp-idf toolchain global cache dir ninja==1.13.2 # native host and esp8266 arduino toolchain build driver -filelock==4.0.1 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg +filelock==4.0.3 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this pyparsing >= 3.3.3 From 98754bedcd496e9b6925dbb3b33d5a13116b7432 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:30:51 -0400 Subject: [PATCH 003/133] Bump pylint from 4.0.8 to 4.0.9 (#19853) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 95e05d1be9c..5331c64a6b7 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,4 +1,4 @@ -pylint==4.0.8 +pylint==4.0.9 flake8==7.3.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py ruff==0.16.8 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py From eb97d15f2ec801e166c5fb9b8ae0c7ce5b0ba047 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:31:05 -0400 Subject: [PATCH 004/133] Bump github/codeql-action/init from 4.38.1 to 4.38.2 (#19855) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index efa6ccf5149..e26a006979d 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,7 +56,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 + uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} From 45f8dbb4da6ca47cc1a91eec79e76d7593567f51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:31:37 -0400 Subject: [PATCH 005/133] Bump platformdirs from 4.11.12 to 4.11.14 (#19854) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 15c14642b37..b62cd712a70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ bleak==3.0.2 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 -platformdirs==4.11.12 # native esp-idf toolchain global cache dir +platformdirs==4.11.14 # native esp-idf toolchain global cache dir ninja==1.13.2 # native host and esp8266 arduino toolchain build driver filelock==4.0.3 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg From 7e7035db729cc829515033dc819c3e40c060529d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:31:53 -0400 Subject: [PATCH 006/133] Bump github/codeql-action/analyze from 4.38.1 to 4.38.2 (#19856) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e26a006979d..da02a5dd09f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -84,6 +84,6 @@ jobs: exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 + uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 with: category: "/language:${{matrix.language}}" From 728449f69d380f6787c02862e0d24f5830bf3df7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:32:15 -0400 Subject: [PATCH 007/133] Bump ruff from 0.16.8 to 0.16.9 (#19851) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- requirements_test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0241857688..b8f06eb0143 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.16.8 + rev: v0.16.9 hooks: # Run the linter. - id: ruff diff --git a/requirements_test.txt b/requirements_test.txt index 5331c64a6b7..947d8c79c7b 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.0.9 flake8==7.3.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py -ruff==0.16.8 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +ruff==0.16.9 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py prek==0.5.3 # .github/workflows/ci.yml reads this pin yamlrocks==0.6.1 # used by script/sync_dependency_versions.py From 8cbc18d040d3c026e8f66dd17ac83ddd1e58f68d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 15:43:38 -0400 Subject: [PATCH 008/133] Bump flake8 from 7.3.0 to 7.4.1 (#19850) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: esphome[bot] <115708604+esphome[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- requirements_test.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b8f06eb0143..48c0d1d6fd4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: # Run the formatter. - id: ruff-format - repo: https://github.com/PyCQA/flake8 - rev: 7.3.0 + rev: 7.4.1 hooks: - id: flake8 additional_dependencies: diff --git a/requirements_test.txt b/requirements_test.txt index 947d8c79c7b..7f6d75f9470 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,5 +1,5 @@ pylint==4.0.9 -flake8==7.3.0 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py +flake8==7.4.1 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py ruff==0.16.9 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py prek==0.5.3 # .github/workflows/ci.yml reads this pin From 13c27429a4d54d1b2ff9fd611f9baafdc3e5f37a Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Tue, 29 Sep 2026 06:30:29 +1000 Subject: [PATCH 009/133] [split_buffer] Allow direct access to buffer chunks (#19819) --- .../components/split_buffer/split_buffer.cpp | 39 +++- .../components/split_buffer/split_buffer.h | 11 +- .../split_buffer/split_buffer_test.cpp | 213 ++++++++++++++++++ 3 files changed, 257 insertions(+), 6 deletions(-) create mode 100644 tests/components/split_buffer/split_buffer_test.cpp diff --git a/esphome/components/split_buffer/split_buffer.cpp b/esphome/components/split_buffer/split_buffer.cpp index 526a19c71cc..63a0f0d9195 100644 --- a/esphome/components/split_buffer/split_buffer.cpp +++ b/esphome/components/split_buffer/split_buffer.cpp @@ -1,5 +1,8 @@ #include "split_buffer.h" +#include +#include + #include "esphome/core/helpers.h" #include "esphome/core/log.h" @@ -8,15 +11,14 @@ static constexpr const char *const TAG = "split_buffer"; SplitBuffer::~SplitBuffer() { this->free(); } -bool SplitBuffer::init(size_t total_length) { +bool SplitBuffer::init(size_t total_length, size_t max_buffer_size) { this->free(); // Clean up any existing allocation - if (total_length == 0) { + if (total_length == 0 || max_buffer_size == 0) { return false; } - this->total_length_ = total_length; - size_t current_buffer_size = total_length; + size_t current_buffer_size = std::min(total_length, max_buffer_size); RAMAllocator ptr_allocator; RAMAllocator allocator; @@ -63,6 +65,7 @@ bool SplitBuffer::init(size_t total_length) { this->buffers_ = temp_buffers; this->buffer_count_ = needed_buffers; this->buffer_size_ = current_buffer_size; + this->total_length_ = total_length; ESP_LOGD(TAG, "Allocated %zu * %zu bytes - %zu bytes", this->buffer_count_, this->buffer_size_, this->total_length_); return true; @@ -122,6 +125,34 @@ uint8_t &SplitBuffer::operator[](size_t index) { return const_cast(static_cast(this)->operator[](index)); } +const uint8_t *SplitBuffer::get_span(size_t index, size_t &length) const { + if (index >= this->total_length_) { + length = 0; + return nullptr; + } + const size_t offset = index % this->buffer_size_; + length = std::min(this->buffer_size_ - offset, this->total_length_ - index); + return this->buffers_[index / this->buffer_size_] + offset; +} + +uint8_t *SplitBuffer::get_span(size_t index, size_t &length) { + return const_cast(static_cast(this)->get_span(index, length)); +} + +void SplitBuffer::write(size_t index, const uint8_t *data, size_t length) { + while (length != 0) { + size_t span_length; + uint8_t *span = this->get_span(index, span_length); + if (span == nullptr) + return; + span_length = std::min(span_length, length); + memcpy(span, data, span_length); + index += span_length; + data += span_length; + length -= span_length; + } +} + /** * Fill the entire buffer with a single byte value * @param value Fill value diff --git a/esphome/components/split_buffer/split_buffer.h b/esphome/components/split_buffer/split_buffer.h index b615ddce74d..6f4ab35e78d 100644 --- a/esphome/components/split_buffer/split_buffer.h +++ b/esphome/components/split_buffer/split_buffer.h @@ -16,8 +16,8 @@ class SplitBuffer { SplitBuffer() = default; ~SplitBuffer(); - // Initialize the buffer with the desired total length - bool init(size_t total_length); + // Initialize the buffer with the desired total length; no sub-buffer will be larger than `max_buffer_size` + bool init(size_t total_length, size_t max_buffer_size = SIZE_MAX); // Free all allocated buffers void free(); @@ -27,6 +27,13 @@ class SplitBuffer { const uint8_t &operator[](size_t index) const; void fill(uint8_t value) const; + // Pointer to the byte at `index`; `length` is set to how many bytes are contiguous from there. + // Returns nullptr with `length` 0 if `index` is out of range. + const uint8_t *get_span(size_t index, size_t &length) const; + uint8_t *get_span(size_t index, size_t &length); + // Copy `length` bytes from `data` into the buffer starting at `index`; bytes past the end are dropped. + void write(size_t index, const uint8_t *data, size_t length); + // Get the total length size_t size() const { return this->total_length_; } diff --git a/tests/components/split_buffer/split_buffer_test.cpp b/tests/components/split_buffer/split_buffer_test.cpp new file mode 100644 index 00000000000..e9cdb12a3b7 --- /dev/null +++ b/tests/components/split_buffer/split_buffer_test.cpp @@ -0,0 +1,213 @@ +#include + +#include +#include +#include + +#include "esphome/components/split_buffer/split_buffer.h" + +namespace esphome::split_buffer::testing { + +static std::vector make_pattern(size_t length, uint8_t seed = 1) { + std::vector data(length); + for (size_t i = 0; i != length; i++) + data[i] = static_cast(seed + i); + return data; +} + +static std::vector read_all(const SplitBuffer &buffer) { + std::vector out(buffer.size()); + for (size_t i = 0; i != buffer.size(); i++) + out[i] = buffer[i]; + return out; +} + +TEST(SplitBufferInit, SingleBufferByDefault) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(100)); + EXPECT_TRUE(buffer.is_valid()); + EXPECT_EQ(buffer.size(), 100u); + EXPECT_EQ(buffer.get_buffer_count(), 1u); +} + +TEST(SplitBufferInit, MaxBufferSizeSplits) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(100, 32)); + EXPECT_EQ(buffer.size(), 100u); + EXPECT_EQ(buffer.get_buffer_count(), 4u); +} + +TEST(SplitBufferInit, ZeroLengthOrMaxFails) { + SplitBuffer buffer; + EXPECT_FALSE(buffer.init(0)); + EXPECT_FALSE(buffer.init(100, 0)); + EXPECT_FALSE(buffer.is_valid()); +} + +TEST(SplitBufferInit, StartsZeroed) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + EXPECT_EQ(read_all(buffer), std::vector(50, 0)); +} + +TEST(SplitBufferFill, FillsShortLastBuffer) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + buffer.fill(0xA5); + EXPECT_EQ(read_all(buffer), std::vector(50, 0xA5)); +} + +TEST(SplitBufferGetSpan, SingleBufferCoversRest) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(100)); + size_t length = 0; + EXPECT_EQ(buffer.get_span(0, length), &buffer[0]); + EXPECT_EQ(length, 100u); + EXPECT_EQ(buffer.get_span(99, length), &buffer[99]); + EXPECT_EQ(length, 1u); +} + +TEST(SplitBufferGetSpan, StopsAtSubBufferBoundary) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + size_t length = 0; + EXPECT_EQ(buffer.get_span(0, length), &buffer[0]); + EXPECT_EQ(length, 16u); + EXPECT_EQ(buffer.get_span(10, length), &buffer[10]); + EXPECT_EQ(length, 6u); + EXPECT_EQ(buffer.get_span(15, length), &buffer[15]); + EXPECT_EQ(length, 1u); + EXPECT_EQ(buffer.get_span(16, length), &buffer[16]); + EXPECT_EQ(length, 16u); +} + +TEST(SplitBufferGetSpan, ShortLastBuffer) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + size_t length = 0; + EXPECT_EQ(buffer.get_span(48, length), &buffer[48]); + EXPECT_EQ(length, 2u); + EXPECT_EQ(buffer.get_span(49, length), &buffer[49]); + EXPECT_EQ(length, 1u); +} + +TEST(SplitBufferGetSpan, OutOfRange) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + size_t length = 99; + EXPECT_EQ(buffer.get_span(50, length), nullptr); + EXPECT_EQ(length, 0u); + length = 99; + EXPECT_EQ(buffer.get_span(1000, length), nullptr); + EXPECT_EQ(length, 0u); +} + +TEST(SplitBufferGetSpan, ConstBufferGivesConstSpan) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + const SplitBuffer &ref = buffer; + size_t length = 0; + const uint8_t *span = ref.get_span(20, length); + EXPECT_EQ(span, &ref[20]); + EXPECT_EQ(length, 12u); +} + +TEST(SplitBufferInit, FailedInitLeavesEmptyState) { + SplitBuffer buffer; + // One-byte pieces need a pointer array too large to allocate, so init fails straight away + EXPECT_FALSE(buffer.init(SIZE_MAX / 16, 1)); + EXPECT_EQ(buffer.size(), 0u); + size_t length = 99; + EXPECT_EQ(buffer.get_span(0, length), nullptr); + EXPECT_EQ(length, 0u); +} + +TEST(SplitBufferGetSpan, UninitializedReturnsNull) { + SplitBuffer buffer; + size_t length = 99; + EXPECT_EQ(buffer.get_span(0, length), nullptr); + EXPECT_EQ(length, 0u); +} + +TEST(SplitBufferGetSpan, SpansCoverWholeBuffer) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + size_t index = 0; + std::vector lengths; + while (index < buffer.size()) { + size_t length = 0; + ASSERT_NE(buffer.get_span(index, length), nullptr); + lengths.push_back(length); + index += length; + } + EXPECT_EQ(index, 50u); + EXPECT_EQ(lengths, (std::vector{16, 16, 16, 2})); +} + +TEST(SplitBufferWrite, WithinOneSubBuffer) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + const auto data = make_pattern(5); + buffer.write(3, data.data(), data.size()); + auto expected = std::vector(50, 0); + std::copy(data.begin(), data.end(), expected.begin() + 3); + EXPECT_EQ(read_all(buffer), expected); +} + +TEST(SplitBufferWrite, AcrossSubBuffers) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + const auto data = make_pattern(30); + buffer.write(10, data.data(), data.size()); + auto expected = std::vector(50, 0); + std::copy(data.begin(), data.end(), expected.begin() + 10); + EXPECT_EQ(read_all(buffer), expected); +} + +TEST(SplitBufferWrite, WholeBuffer) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + const auto data = make_pattern(50); + buffer.write(0, data.data(), data.size()); + EXPECT_EQ(read_all(buffer), data); +} + +TEST(SplitBufferWrite, TruncatesPastEnd) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + // The source is sized to the full request so ASan catches any read beyond it. + const auto data = make_pattern(20); + buffer.write(40, data.data(), data.size()); + auto expected = std::vector(50, 0); + std::copy(data.begin(), data.begin() + 10, expected.begin() + 40); + EXPECT_EQ(read_all(buffer), expected); +} + +TEST(SplitBufferWrite, StartOutOfRangeIsIgnored) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + const auto data = make_pattern(5); + buffer.write(50, data.data(), data.size()); + EXPECT_EQ(read_all(buffer), std::vector(50, 0)); +} + +TEST(SplitBufferWrite, ZeroLengthIsNoop) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(50, 16)); + buffer.write(0, nullptr, 0); + EXPECT_EQ(read_all(buffer), std::vector(50, 0)); +} + +TEST(SplitBufferWrite, MatchesContiguousBuffer) { + // Each sub-buffer size, including ones that divide the total evenly, must give the same result. + const auto data = make_pattern(64, 7); + for (size_t max_size : {1u, 3u, 8u, 16u, 63u, 64u, 1000u}) { + SplitBuffer buffer; + ASSERT_TRUE(buffer.init(64, max_size)); + buffer.write(0, data.data(), 20); + buffer.write(20, data.data() + 20, 44); + EXPECT_EQ(read_all(buffer), data) << "max_buffer_size=" << max_size; + } +} + +} // namespace esphome::split_buffer::testing From 89cc689864718a1a29c691d8487be963deeb9cf6 Mon Sep 17 00:00:00 2001 From: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Date: Mon, 28 Sep 2026 22:08:06 +0100 Subject: [PATCH 010/133] [esp32_hosted][esp32_ble] Support esp_hosted 3.x alongside 2.x, keep builds on 2.12.13 for now (#19621) --- esphome/components/esp32_ble/__init__.py | 9 +- esphome/components/esp32_ble/ble.cpp | 44 +++- .../esp32_ble_beacon/esp32_ble_beacon.cpp | 6 +- .../esp32_ble_beacon/esp32_ble_beacon.h | 6 +- .../esp32_ble_server/ble_server.cpp | 2 +- .../esp32_ble_tracker/esp32_ble_tracker.cpp | 6 +- esphome/components/esp32_hosted/__init__.py | 219 ++++++++++++++++-- .../update/esp32_hosted_update.cpp | 16 +- esphome/idf_component.yml | 2 +- 9 files changed, 259 insertions(+), 51 deletions(-) diff --git a/esphome/components/esp32_ble/__init__.py b/esphome/components/esp32_ble/__init__.py index 77ec17f3891..ef95243bc11 100644 --- a/esphome/components/esp32_ble/__init__.py +++ b/esphome/components/esp32_ble/__init__.py @@ -474,12 +474,17 @@ def final_validation(config: ConfigType) -> None: # Check if hosted bluetooth is being used if "esp32_hosted" in full_config: + from esphome.components.esp32_hosted import uses_esp_hosted_3x + add_idf_sdkconfig_option("CONFIG_BT_CLASSIC_ENABLED", False) add_idf_sdkconfig_option("CONFIG_BT_BLE_ENABLED", True) add_idf_sdkconfig_option("CONFIG_BT_BLUEDROID_ENABLED", True) add_idf_sdkconfig_option("CONFIG_BT_CONTROLLER_DISABLED", True) - add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID", True) - add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_BLUEDROID_HCI_VHCI", True) + if uses_esp_hosted_3x(): + add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_HOST_FEAT_BT", True) + else: + add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID", True) + add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_BLUEDROID_HCI_VHCI", True) # Check if BLE Client is needed (via esp32_ble_tracker or esp32_ble_client) has_ble_client = ( diff --git a/esphome/components/esp32_ble/ble.cpp b/esphome/components/esp32_ble/ble.cpp index 81fa328c160..a44886b8c81 100644 --- a/esphome/components/esp32_ble/ble.cpp +++ b/esphome/components/esp32_ble/ble.cpp @@ -6,16 +6,21 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" -#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifndef CONFIG_BT_CONTROLLER_DISABLED #include #else #include "esphome/components/watchdog/watchdog.h" #include extern "C" { #include +#ifndef CONFIG_ESP_HOSTED_HOST_FEAT_BT #include #include +#endif } +#ifdef CONFIG_ESP_HOSTED_HOST_FEAT_BT +#include +#endif #endif #include #include @@ -35,7 +40,7 @@ namespace esphome::esp32_ble { static const char *const TAG = "esp32_ble"; -#ifdef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifdef CONFIG_BT_CONTROLLER_DISABLED // Bringing up the remote BT controller issues synchronous RPCs to the // co-processor with 5 second response timeouts, and the default task watchdog // is also 5 seconds. If the co-processor firmware does not answer (for example @@ -43,8 +48,9 @@ static const char *const TAG = "esp32_ble"; // device before the RPC could return an error, causing a boot loop. Raise the // watchdog for the duration of the bring-up so failures surface as error // returns instead. 60 seconds covers the worst case: transport reconnect -// (up to ~20s), version preflight (1s), controller init/enable (5s each) and -// the bluedroid host bring-up over the hosted HCI transport. +// (up to ~20s), version preflight (1s), the controller init retry window +// (5s of 5s RPCs), controller enable (5s) and the bluedroid host bring-up +// over the hosted HCI transport. static constexpr uint32_t HOSTED_BT_WDT_TIMEOUT_MS = 60000; #endif @@ -202,10 +208,10 @@ void ESP32BLE::advertising_init_() { bool ESP32BLE::ble_setup_() { esp_err_t err; -#ifdef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifdef CONFIG_BT_CONTROLLER_DISABLED watchdog::WatchdogManager wdt(HOSTED_BT_WDT_TIMEOUT_MS); #endif -#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifndef CONFIG_BT_CONTROLLER_DISABLED if (esp_bt_controller_get_status() != ESP_BT_CONTROLLER_STATUS_ENABLED) { // start bt controller if (esp_bt_controller_get_status() == ESP_BT_CONTROLLER_STATUS_IDLE) { @@ -239,8 +245,7 @@ bool ESP32BLE::ble_setup_() { } // Fast preflight (1 second RPC timeout): verifies the co-processor answers - // RPCs at all before the 5 second timeout BT controller RPCs below, and - // before hosted_hci_bluedroid_open(), which aborts if the transport is down. + // RPCs at all before the 5 second timeout BT controller RPCs below. esp_hosted_coprocessor_fwver_t fw_ver{}; if (esp_hosted_get_coprocessor_fwversion(&fw_ver) != ESP_OK) { ESP_LOGE(TAG, "Co-processor not responding; BLE disabled. Update its firmware with the esp32_hosted " @@ -249,6 +254,19 @@ bool ESP32BLE::ble_setup_() { } ESP_LOGD(TAG, "Co-processor firmware %" PRIu32 ".%" PRIu32 ".%" PRIu32, fw_ver.major1, fw_ver.minor1, fw_ver.patch1); +#ifdef CONFIG_ESP_HOSTED_HOST_FEAT_BT + esp_hosted_bt_host_stack_cfg_t bt_cfg{}; + bt_cfg.stack = ESP_HOSTED_BT_HOST_STACK_BLUEDROID; + bt_cfg.bring_up_controller = true; + bt_cfg.controller_ready_timeout_ms = EH_BT_CTRL_DEFAULT_READY_TIMEOUT_MS; + if (esp_hosted_bt_host_stack_setup(&bt_cfg) != ESP_OK) { + ESP_LOGE(TAG, + "BT controller bring-up failed; co-processor firmware %" PRIu32 ".%" PRIu32 ".%" PRIu32 + " may lack BT support. Update it with the esp32_hosted update component; BLE disabled", + fw_ver.major1, fw_ver.minor1, fw_ver.patch1); + return false; + } +#else if (esp_hosted_bt_controller_init() != ESP_OK) { ESP_LOGE(TAG, "BT controller init failed; co-processor firmware %" PRIu32 ".%" PRIu32 ".%" PRIu32 @@ -273,6 +291,7 @@ bool ESP32BLE::ble_setup_() { .register_host_callback = hosted_hci_bluedroid_register_host_callback, }; esp_bluedroid_attach_hci_driver(&operations); +#endif #endif err = esp_bluedroid_init(); @@ -393,7 +412,7 @@ bool ESP32BLE::ble_setup_() { } bool ESP32BLE::ble_dismantle_() { -#ifdef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifdef CONFIG_BT_CONTROLLER_DISABLED // Same 5 second RPCs as the bring-up path; see HOSTED_BT_WDT_TIMEOUT_MS watchdog::WatchdogManager wdt(HOSTED_BT_WDT_TIMEOUT_MS); #endif @@ -416,7 +435,7 @@ bool ESP32BLE::ble_dismantle_() { ESP_LOGD(TAG, "Already deinitialized"); } -#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifndef CONFIG_BT_CONTROLLER_DISABLED if (esp_bt_controller_get_status() != ESP_BT_CONTROLLER_STATUS_IDLE) { // stop bt controller if (esp_bt_controller_get_status() == ESP_BT_CONTROLLER_STATUS_ENABLED) { @@ -440,6 +459,11 @@ bool ESP32BLE::ble_dismantle_() { return false; } } +#elif defined(CONFIG_ESP_HOSTED_HOST_FEAT_BT) + if (esp_hosted_bt_host_stack_teardown() != ESP_OK) { + ESP_LOGE(TAG, "esp_hosted_bt_host_stack_teardown failed"); + return false; + } #else if (esp_hosted_bt_controller_disable() != ESP_OK) { ESP_LOGE(TAG, "esp_hosted_bt_controller_disable failed"); diff --git a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp index ab728f9f6fc..47e4b7ebb20 100644 --- a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp +++ b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp @@ -3,7 +3,7 @@ #ifdef USE_ESP32 -#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifndef CONFIG_BT_CONTROLLER_DISABLED #include #endif #include @@ -35,7 +35,7 @@ void ESP32BLEBeacon::dump_config() { } } *bpos = '\0'; -#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifndef CONFIG_BT_CONTROLLER_DISABLED ESP_LOGCONFIG(TAG, " UUID: %s, Major: %u, Minor: %u, Min Interval: %ums, Max Interval: %ums, Measured Power: %d" ", TX Power: %ddBm", @@ -81,7 +81,7 @@ void ESP32BLEBeacon::on_advertise_() { ibeacon_adv_data.ibeacon_vendor.measured_power = static_cast(this->measured_power_); esp_err_t err; -#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifndef CONFIG_BT_CONTROLLER_DISABLED ESP_LOGD(TAG, "Setting BLE TX power"); err = esp_ble_tx_power_set(ESP_BLE_PWR_TYPE_ADV, this->tx_power_); if (err != ESP_OK) { diff --git a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.h b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.h index 986778de579..a7132a54652 100644 --- a/esphome/components/esp32_ble_beacon/esp32_ble_beacon.h +++ b/esphome/components/esp32_ble_beacon/esp32_ble_beacon.h @@ -5,7 +5,7 @@ #ifdef USE_ESP32 -#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifndef CONFIG_BT_CONTROLLER_DISABLED #include #endif #include @@ -47,7 +47,7 @@ class ESP32BLEBeacon final : public Component { void set_min_interval(uint16_t val) { this->min_interval_ = val; } void set_max_interval(uint16_t val) { this->max_interval_ = val; } void set_measured_power(int8_t val) { this->measured_power_ = val; } -#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifndef CONFIG_BT_CONTROLLER_DISABLED void set_tx_power(esp_power_level_t val) { this->tx_power_ = val; } #endif void gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_param_t *param); @@ -61,7 +61,7 @@ class ESP32BLEBeacon final : public Component { uint16_t min_interval_{}; uint16_t max_interval_{}; int8_t measured_power_{}; -#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifndef CONFIG_BT_CONTROLLER_DISABLED esp_power_level_t tx_power_{}; #endif esp_ble_adv_params_t ble_adv_params_; diff --git a/esphome/components/esp32_ble_server/ble_server.cpp b/esphome/components/esp32_ble_server/ble_server.cpp index 45679b9b982..e34a57cf37f 100644 --- a/esphome/components/esp32_ble_server/ble_server.cpp +++ b/esphome/components/esp32_ble_server/ble_server.cpp @@ -10,7 +10,7 @@ #include #include #include -#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifndef CONFIG_BT_CONTROLLER_DISABLED #include #endif #include diff --git a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp index e25b6f59fa0..9bb90026e3f 100644 --- a/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp +++ b/esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp @@ -7,7 +7,7 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" -#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifndef CONFIG_BT_CONTROLLER_DISABLED #include #endif #include @@ -635,7 +635,7 @@ void ESP32BLETracker::log_unexpected_state_(const char *operation, ScannerState #ifdef USE_ESP32_BLE_SOFTWARE_COEXISTENCE void ESP32BLETracker::update_coex_preference_(bool force_ble) { -#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#ifndef CONFIG_BT_CONTROLLER_DISABLED if (force_ble && !this->coex_prefer_ble_) { ESP_LOGD(TAG, "Setting coexistence to Bluetooth to make connection."); this->coex_prefer_ble_ = true; @@ -645,7 +645,7 @@ void ESP32BLETracker::update_coex_preference_(bool force_ble) { this->coex_prefer_ble_ = false; esp_coex_preference_set(ESP_COEX_PREFER_BALANCE); // Reset to default } -#endif // CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID +#endif // CONFIG_BT_CONTROLLER_DISABLED } #endif diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index ad01e130eb9..d78941ea4c8 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -1,5 +1,6 @@ import os from pathlib import Path +import re from esphome import pins from esphome.components import esp32 @@ -7,15 +8,22 @@ from esphome.components.const import CONF_SLOT, CONF_USE_PSRAM import esphome.config_validation as cv from esphome.const import ( CONF_CLK_PIN, + CONF_COMPONENTS, CONF_CS_PIN, + CONF_FRAMEWORK, CONF_FREQUENCY, CONF_MISO_PIN, CONF_MOSI_PIN, + CONF_NAME, + CONF_REF, CONF_RESET_PIN, CONF_TYPE, CONF_VARIANT, + KEY_ESP32, ) +from esphome.core import CORE from esphome.cpp_generator import add_define +import esphome.final_validate as fv from esphome.types import ConfigType CODEOWNERS = ["@swoboda1337"] @@ -23,6 +31,9 @@ DEPENDENCIES = ["esp32"] # esp32_ble raises the task watchdog around the remote BT controller bring-up AUTO_LOAD = ["watchdog"] +# 3.x is opt-in via a pin under esp32.framework.components (needs ESP-IDF 5.5) +ESP_HOSTED_COMPONENT = "espressif/esp_hosted" + CONF_ACTIVE_HIGH = "active_high" CONF_BUS_WIDTH = "bus_width" CONF_CMD_PIN = "cmd_pin" @@ -145,6 +156,34 @@ CONFIG_SCHEMA = cv.typed_schema( ) +# Version specs that stay within one major (exact, ^, ~, ~=); other ranges are 3.x +_PINNED_MAJOR = re.compile(r"^(?:==|\^|~=|~)?(\d+)(?:\.(?:\d+|\*))*$") + + +def user_esp_hosted_major() -> int | None: + """Major version of a user-pinned esp_hosted, or None without a pin.""" + try: + full_config = fv.full_config.get() + except LookupError: + full_config = CORE.config + esp32_config = full_config.get(KEY_ESP32) or {} + for component in esp32_config.get(CONF_FRAMEWORK, {}).get(CONF_COMPONENTS, []): + if component.get(CONF_NAME) != ESP_HOSTED_COMPONENT: + continue + ref = component.get(CONF_REF, "").strip() + if not ref: + return 2 + if match := _PINNED_MAJOR.match(ref): + return int(match.group(1)) + return 3 + return None + + +def uses_esp_hosted_3x() -> bool: + """Whether the user pinned esp_hosted 3.x.""" + return (major := user_esp_hosted_major()) is not None and major >= 3 + + def _final_validate(config: ConfigType) -> None: # The esp_hosted releases compatible with older ESP-IDF versions crash at # boot with a heap double free in the SDIO RX path (fixed in esp_hosted @@ -155,12 +194,34 @@ def _final_validate(config: ConfigType) -> None: "Remove the framework version from your configuration to use the " "recommended version, or pin a version at or above 5.3." ) + if (major := user_esp_hosted_major()) is None or major < 3: + return + if idf_ver < cv.Version(5, 5, 0): + raise cv.Invalid( + f"esp_hosted 3.x requires ESP-IDF 5.5 or newer, got {idf_ver}. " + f"Remove the {ESP_HOSTED_COMPONENT} pin from esp32.framework." + "components to stay on the 2.x line, or use ESP-IDF 5.5 or newer." + ) + if not config[CONF_ACTIVE_HIGH]: + raise cv.Invalid( + "esp_hosted 3.x always parks the reset line high with a low pulse, so " + "'active_high: false' cannot be expressed on it. Remove the " + f"{ESP_HOSTED_COMPONENT} pin from esp32.framework.components to stay " + "on the 2.x line." + ) + if config[CONF_TYPE] == "sdio" and config[CONF_BUS_WIDTH] == 1: + raise cv.Invalid( + "esp_hosted 3.x cannot be built with a 1-bit SDIO bus " + "(espressif/esp-hosted-mcu#245). Remove the " + f"{ESP_HOSTED_COMPONENT} pin from esp32.framework.components to stay " + "on the 2.x line, or use a 4-bit bus." + ) FINAL_VALIDATE_SCHEMA = _final_validate -def _configure_sdio(config: ConfigType) -> None: +def _configure_sdio_2x(config: ConfigType) -> None: slot = config[CONF_SLOT] esp32.add_idf_sdkconfig_option( f"CONFIG_ESP_HOSTED_SDIO_SLOT_{slot}", @@ -202,7 +263,42 @@ def _configure_sdio(config: ConfigType) -> None: ) -def _configure_spi(config: ConfigType) -> None: +def _configure_sdio_3x(config: ConfigType) -> None: + esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_HOST_TRANSPORT_BUS_SDIO", True) + esp32.add_idf_sdkconfig_option( + f"CONFIG_ESP_HOSTED_SDIO_SLOT_{config[CONF_SLOT]}", + True, + ) + esp32.add_idf_sdkconfig_option( + f"CONFIG_ESP_HOSTED_HOST_SDIO_BUS_WIDTH_{config[CONF_BUS_WIDTH]}", + True, + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SDIO_PIN_CLK", config[CONF_CLK_PIN] + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SDIO_PIN_CMD", config[CONF_CMD_PIN] + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SDIO_PIN_D0", config[CONF_D0_PIN] + ) + if config[CONF_BUS_WIDTH] == 4: + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SDIO_PIN_D1", config[CONF_D1_PIN] + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SDIO_PIN_D2", config[CONF_D2_PIN] + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SDIO_PIN_D3", config[CONF_D3_PIN] + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SDIO_CLK_KHZ", + int(config[CONF_SDIO_FREQUENCY] // 1000), + ) + + +def _configure_spi_2x(config: ConfigType) -> None: esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_SPI_HOST_INTERFACE", True) # SPI mode is set via per-variant choice options variant = config[CONF_VARIANT] @@ -250,11 +346,53 @@ def _configure_spi(config: ConfigType) -> None: esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_DR_ACTIVE_LOW", True) -async def to_code(config: ConfigType) -> None: - add_define("USE_ESP32_HOSTED") - transport = config[CONF_TYPE] - transport_prefix = "SDIO" if transport == "sdio" else "SPI" +def _configure_spi_3x(config: ConfigType) -> None: + esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_HOST_TRANSPORT_BUS_SPI", True) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SPI_MODE", config[CONF_SPI_MODE] + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SPI_CLK_MHZ", int(config[CONF_FREQUENCY] // 1e6) + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SPI_MOSI_GPIO", config[CONF_MOSI_PIN] + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SPI_MISO_GPIO", config[CONF_MISO_PIN] + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SPI_CLK_GPIO", config[CONF_CLK_PIN] + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_SPI_CS_GPIO", config[CONF_CS_PIN] + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_HANDSHAKE_GPIO", config[CONF_HANDSHAKE_PIN] + ) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_DATA_READY_GPIO", config[CONF_DATA_READY_PIN] + ) + # Handshake and data_ready polarity + if config[CONF_HANDSHAKE_ACTIVE_HIGH]: + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_SPI_HANDSHAKE_ACTIVE_HIGH", True + ) + else: + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_SPI_HANDSHAKE_ACTIVE_LOW", True + ) + if config[CONF_DATA_READY_ACTIVE_HIGH]: + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_SPI_DATAREADY_ACTIVE_HIGH", True + ) + else: + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_SPI_DATAREADY_ACTIVE_LOW", True + ) + +def _configure_2x(config: ConfigType) -> None: + transport_prefix = "SDIO" if config[CONF_TYPE] == "sdio" else "SPI" # Reset polarity if config[CONF_ACTIVE_HIGH]: esp32.add_idf_sdkconfig_option( @@ -274,24 +412,11 @@ async def to_code(config: ConfigType) -> None: f"CONFIG_SLAVE_IDF_TARGET_{config[CONF_VARIANT]}", # NOLINT True, ) - - # Transport-specific configuration - if transport == "sdio": - _configure_sdio(config) + if config[CONF_TYPE] == "sdio": + _configure_sdio_2x(config) else: - _configure_spi(config) - - # ESP-NOW-over-hosted shim: only the radio-less ESP32-P4 host needs it (see - # the note by _MAX_CUSTOM_MSG_HANDLERS). Enabled for every P4 host, not - # gated on the `espnow` component being present: the shim is tiny and the - # esp_now_* symbols/CustomRpc calls it defines require these Kconfig options - # to link whenever esp_now_hosted.cpp compiles (which is on any P4 host), so - # coupling the two keeps the build consistent. When `espnow` is absent the - # symbols are simply unused and never register a callback at runtime. + _configure_spi_2x(config) if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4: - add_define("USE_ESP_NOW_HOSTED") - # esp_now_hosted.cpp includes esp_now.h, which esp_wifi provides - esp32.include_builtin_idf_component("esp_wifi") # esp-hosted's CustomRpc ("peer data transfer") path — off by default. esp32.add_idf_sdkconfig_option( "CONFIG_ESP_HOSTED_ENABLE_PEER_DATA_TRANSFER", True @@ -299,18 +424,64 @@ async def to_code(config: ConfigType) -> None: esp32.add_idf_sdkconfig_option( "CONFIG_ESP_HOSTED_MAX_CUSTOM_MSG_HANDLERS", _MAX_CUSTOM_MSG_HANDLERS ) - # Place the transport mempool in PSRAM. Required on memory-tight host # configurations (e.g. P4 with a large LVGL UI) where the internal-RAM # mempool allocation fails at boot with `sdio_mempool_create` assert. if config[CONF_USE_PSRAM]: esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_MEMPOOL_PREFER_SPIRAM", True) + +def _configure_3x(config: ConfigType) -> None: + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_RESET_GPIO", config[CONF_RESET_PIN] + ) + # Co-processor variant + esp32.add_idf_sdkconfig_option( + f"CONFIG_ESP_HOSTED_CP_TARGET_{config[CONF_VARIANT]}", + True, + ) + if config[CONF_TYPE] == "sdio": + _configure_sdio_3x(config) + else: + _configure_spi_3x(config) + if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4: + esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_HOST_FEAT_PEER_DATA", True) + esp32.add_idf_sdkconfig_option( + "CONFIG_ESP_HOSTED_HOST_FEAT_PEER_DATA_MAX_CUSTOM_MSG_HANDLERS", + _MAX_CUSTOM_MSG_HANDLERS, + ) + if config[CONF_USE_PSRAM]: + esp32.add_idf_sdkconfig_option("CONFIG_EH_HOST_PORT_DMA_PREFER_SPIRAM", True) + esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_DFLT_TASK_FROM_SPIRAM", True) + + +async def to_code(config: ConfigType) -> None: + add_define("USE_ESP32_HOSTED") + use_3x = uses_esp_hosted_3x() + + if use_3x: + _configure_3x(config) + else: + _configure_2x(config) + + # ESP-NOW-over-hosted shim: only the radio-less ESP32-P4 host needs it (see + # the note by _MAX_CUSTOM_MSG_HANDLERS). Enabled for every P4 host, not + # gated on the `espnow` component being present: the shim is tiny and the + # esp_now_* symbols/CustomRpc calls it defines require the peer-data Kconfig + # options (set above) to link whenever esp_now_hosted.cpp compiles (which is + # on any P4 host), so coupling the two keeps the build consistent. When + # `espnow` is absent the symbols are simply unused and never register a + # callback at runtime. + if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4: + add_define("USE_ESP_NOW_HOSTED") + # esp_now_hosted.cpp includes esp_now.h, which esp_wifi provides + esp32.include_builtin_idf_component("esp_wifi") + # Library versions; this component set requires ESP-IDF 5.3 or newer, # which is enforced at validation time. idf_ver = esp32.idf_version() os.environ["ESP_IDF_VERSION"] = f"{idf_ver.major}.{idf_ver.minor}" - esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="1.6.3") + esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="1.6.5") esp32.add_idf_component(name="espressif/wifi_remote_over_eppp", ref="0.3.3") esp32.add_idf_component(name="espressif/eppp_link", ref="1.1.5") esp32.add_idf_component(name="espressif/esp_hosted", ref="2.12.13") diff --git a/esphome/components/esp32_hosted/update/esp32_hosted_update.cpp b/esphome/components/esp32_hosted/update/esp32_hosted_update.cpp index d9b375dd20c..c35301a6852 100644 --- a/esphome/components/esp32_hosted/update/esp32_hosted_update.cpp +++ b/esphome/components/esp32_hosted/update/esp32_hosted_update.cpp @@ -26,6 +26,8 @@ static const char *const TAG = "esp32_hosted.update"; // Older coprocessor firmware versions have a 1500-byte limit per RPC call constexpr size_t CHUNK_SIZE = 1500; +// OTA begin blocks while the coprocessor erases its partition +constexpr uint32_t OTA_WDT_TIMEOUT_MS = 60000; #ifdef USE_ESP32_HOSTED_HTTP_UPDATE // Interval/timeout IDs (uint32_t to avoid string comparison) @@ -336,7 +338,11 @@ bool Esp32HostedUpdate::stream_firmware_to_coprocessor_() { ESP_LOGI(TAG, "Firmware size: %zu bytes", total_size); // Begin OTA on coprocessor - esp_err_t err = esp_hosted_slave_ota_begin(); // NOLINT + esp_err_t err; + { + watchdog::WatchdogManager wdt(OTA_WDT_TIMEOUT_MS); + err = esp_hosted_slave_ota_begin(); // NOLINT + } if (err != ESP_OK) { ESP_LOGE(TAG, "Failed to begin OTA: %s", esp_err_to_name(err)); container->end(); @@ -425,7 +431,11 @@ bool Esp32HostedUpdate::write_embedded_firmware_to_coprocessor_() { ESP_LOGI(TAG, "Starting OTA update (%zu bytes)", this->firmware_size_); - esp_err_t err = esp_hosted_slave_ota_begin(); // NOLINT + esp_err_t err; + { + watchdog::WatchdogManager wdt(OTA_WDT_TIMEOUT_MS); + err = esp_hosted_slave_ota_begin(); // NOLINT + } if (err != ESP_OK) { ESP_LOGE(TAG, "Failed to begin OTA: %s", esp_err_to_name(err)); this->status_set_error(LOG_STR("Failed to begin OTA")); @@ -472,8 +482,6 @@ void Esp32HostedUpdate::perform(bool force) { this->update_info_.has_progress = false; this->publish_state(); - watchdog::WatchdogManager watchdog(60000); - #ifdef USE_ESP32_HOSTED_HTTP_UPDATE if (!this->stream_firmware_to_coprocessor_()) #else diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 65d56495af4..1a972896c3b 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -26,7 +26,7 @@ dependencies: espressif/mdns: version: 1.12.0 espressif/esp_wifi_remote: - version: 1.6.3 + version: 1.6.5 rules: - if: "target in [esp32h2, esp32p4]" espressif/wifi_remote_over_eppp: From 30dc453dc134b47ab7246fcd94ed3367d3dbe316 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 28 Sep 2026 23:30:51 +0200 Subject: [PATCH 011/133] [ethernet] Move received frames to PSRAM on SPI ethernet chips (#19377) --- .../ethernet/ethernet_component_esp32.cpp | 46 +++++++++++++++++++ .../test-w5500-wifi-psram.esp32-s3-idf.yaml | 15 ++++++ 2 files changed, 61 insertions(+) create mode 100644 tests/components/ethernet/test-w5500-wifi-psram.esp32-s3-idf.yaml diff --git a/esphome/components/ethernet/ethernet_component_esp32.cpp b/esphome/components/ethernet/ethernet_component_esp32.cpp index 1d9903271ec..ea732e84ff9 100644 --- a/esphome/components/ethernet/ethernet_component_esp32.cpp +++ b/esphome/components/ethernet/ethernet_component_esp32.cpp @@ -10,6 +10,9 @@ #include #include #include "esp_event.h" +#ifdef USE_PSRAM +#include +#endif // IDF 6.0 moved per-chip PHY/MAC drivers to the Espressif Component Registry; // they are no longer included via esp_eth.h and need explicit includes. @@ -71,6 +74,32 @@ static const char *const TAG = "ethernet"; // PHY register size for hex logging static constexpr size_t PHY_REG_SIZE = 2; +// Dual wifi + ethernet SPI builds: the one place internal RAM is short and lwip's other buffers are +// already in PSRAM. Not with L2 TAP, whose filter lives in the glue's input path this replaces. +#if defined(USE_PSRAM) && defined(USE_ETHERNET_SPI) && defined(USE_WIFI) && !defined(CONFIG_ESP_NETIF_L2_TAP) +#define USE_ETHERNET_RX_PSRAM +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - reported by dump_config() +static bool rx_psram_installed = false; + +// ESP-IDF ethernet drivers malloc() every received frame in internal RAM, where it stays until lwIP +// hands it to the application. Move it to PSRAM; if that fails the frame is passed on where it is. +static esp_err_t eth_input_to_psram(esp_eth_handle_t handle, uint8_t *buffer, uint32_t length, void *priv) { + auto *copy = static_cast(heap_caps_malloc(length, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (copy != nullptr) { + memcpy(copy, buffer, length); + free(buffer); // NOLINT(cppcoreguidelines-no-malloc) - allocated by the driver with malloc() + buffer = copy; + } else { + static bool warned = false; // once, this runs per frame in the driver's task + if (!warned) { + warned = true; + ESP_LOGW(TAG, "PSRAM allocation failed, frame kept in internal RAM (reported once)"); + } + } + return esp_netif_receive(static_cast(priv), buffer, length, nullptr); +} +#endif + void EthernetComponent::log_error_and_mark_failed_(esp_err_t err, const char *message) { ESP_LOGE(TAG, "%s: (%d) %s", message, err, esp_err_to_name(err)); this->mark_failed(); @@ -453,6 +482,16 @@ void EthernetComponent::ethernet_lazy_init_() { /* attach Ethernet driver to TCP/IP stack */ err = esp_netif_attach(this->eth_netif_, esp_eth_new_netif_glue(this->eth_handle_)); ESPHL_ERROR_CHECK(err, "ETH netif attach error"); +#ifdef USE_ETHERNET_RX_PSRAM + // The glue frees every receive buffer with free(), so the replacement buffer must come from the heap + if (esp_psram_is_initialized()) { + err = esp_eth_update_input_path(this->eth_handle_, eth_input_to_psram, this->eth_netif_); + rx_psram_installed = err == ESP_OK; + if (!rx_psram_installed) { + ESP_LOGW(TAG, "PSRAM RX path not installed: %s", esp_err_to_name(err)); + } + } +#endif // Register user defined event handers err = esp_event_handler_register(ETH_EVENT, ESP_EVENT_ANY_ID, &EthernetComponent::eth_event_handler, nullptr); @@ -630,6 +669,13 @@ void EthernetComponent::dump_config() { this->clk_pin_, this->mdc_pin_, this->mdio_pin_, this->phy_addr_); #endif ESP_LOGCONFIG(TAG, " Type: %s", eth_type); +#ifdef USE_ETHERNET_RX_PSRAM + // Only known once the driver is up; with enable_on_boot: false that is after this dump + if (this->ethernet_initialized_) { + ESP_LOGCONFIG(TAG, " RX frames: %s", + rx_psram_installed ? LOG_STR_LITERAL("PSRAM") : LOG_STR_LITERAL("internal RAM")); + } +#endif } network::IPAddresses EthernetComponent::get_ip_addresses() { diff --git a/tests/components/ethernet/test-w5500-wifi-psram.esp32-s3-idf.yaml b/tests/components/ethernet/test-w5500-wifi-psram.esp32-s3-idf.yaml new file mode 100644 index 00000000000..d454dbadc39 --- /dev/null +++ b/tests/components/ethernet/test-w5500-wifi-psram.esp32-s3-idf.yaml @@ -0,0 +1,15 @@ +# W5500 next to wifi with PSRAM: compiles the input path that moves received frames to PSRAM +packages: + ethernet: !include common-w5500.yaml + +psram: + mode: quad + +wifi: + ssid: MySSID + password: password1 + +network: + priority: + - ethernet + - wifi From 2d6a7dae547f0940f0c1deb76c6d201d435c5a20 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:40:39 +1000 Subject: [PATCH 012/133] [mipi_rgb] Fix SWRESET delay for st7701s (#17646) --- esphome/components/mipi/__init__.py | 10 ++- esphome/components/mipi_rgb/display.py | 2 +- esphome/components/mipi_rgb/mipi_rgb.cpp | 4 +- esphome/components/mipi_rgb/models/st7701s.py | 5 +- .../mipi_rgb/test_mipi_rgb_config.py | 51 ++++++++++++ .../mipi_rgb/test_reset_sequence.py | 81 +++++++++++++++++++ .../mipi_spi/test_get_sequence.py | 64 +++++++++++++++ 7 files changed, 211 insertions(+), 6 deletions(-) create mode 100644 tests/component_tests/mipi_rgb/test_reset_sequence.py create mode 100644 tests/component_tests/mipi_spi/test_get_sequence.py diff --git a/esphome/components/mipi/__init__.py b/esphome/components/mipi/__init__.py index 3f73f963277..50b84b630d1 100644 --- a/esphome/components/mipi/__init__.py +++ b/esphome/components/mipi/__init__.py @@ -606,11 +606,11 @@ class DriverChip: """ Create the init sequence for the display. Use the default sequence from the model, if any, and append any custom sequence provided in the config. - Append SLPOUT (if not already in the sequence) and DISPON to the end of the sequence + Append SLPOUT (if not suppressed by the model) and DISPON to the end of the sequence MADCTL will be set if add_madctl is True If add_reset is True, a reset is prepended: a software reset when no reset pin is configured (and the model doesn't skip it), followed by a settling delay that - both a software and a hardware reset require. + both a software and a hardware reset require. The delay length is set via reset_delay, and defaults to 10ms. Returns the init sequence """ sequence = list(self.initsequence or ()) @@ -620,12 +620,16 @@ class DriverChip: sequence = [x if isinstance(x, tuple) else (x,) for x in sequence] if add_reset: + # Matches the 1-255ms range map_sequence() already allows for a "delay N" entry. + reset_delay = self.get_default("reset_delay", 10) + if reset_delay < 1 or reset_delay > 255: + raise ValueError("reset_delay must be between 1 and 255ms") reset: list = [] # A software reset is only needed when there is no hardware reset pin. if CONF_RESET_PIN not in config and not self.skip_command("SWRESET"): reset.append((SWRESET,)) # Both a software and a hardware reset need a settling delay before further commands. - reset.append(delay(10)) + reset.append(delay(reset_delay)) sequence = reset + sequence # Set pixel format if not already in the custom sequence diff --git a/esphome/components/mipi_rgb/display.py b/esphome/components/mipi_rgb/display.py index b91528160e2..9cba56a9763 100644 --- a/esphome/components/mipi_rgb/display.py +++ b/esphome/components/mipi_rgb/display.py @@ -285,7 +285,7 @@ async def to_code(config: ConfigType) -> None: if CONF_SPI_ID in config: await spi.register_spi_device(var, config, write_only=True) - sequence = model.get_sequence(config) + sequence = model.get_sequence(config, add_reset=True) cg.add(var.set_init_sequence(sequence)) cg.add(var.set_color_mode(COLOR_ORDERS[config[CONF_COLOR_ORDER]])) diff --git a/esphome/components/mipi_rgb/mipi_rgb.cpp b/esphome/components/mipi_rgb/mipi_rgb.cpp index 3f83da7f803..034efb9c92a 100644 --- a/esphome/components/mipi_rgb/mipi_rgb.cpp +++ b/esphome/components/mipi_rgb/mipi_rgb.cpp @@ -44,8 +44,10 @@ void MipiRgb::setup_enables_() { void MipiRgbSpi::setup() { this->setup_enables_(); this->spi_setup(); - this->write_init_sequence_(); this->common_setup_(); + if (this->is_failed()) + return; + this->write_init_sequence_(); } void MipiRgbSpi::write_command_(uint8_t value) { this->enable(); diff --git a/esphome/components/mipi_rgb/models/st7701s.py b/esphome/components/mipi_rgb/models/st7701s.py index cad5dc8e20d..b51e7447ad4 100644 --- a/esphome/components/mipi_rgb/models/st7701s.py +++ b/esphome/components/mipi_rgb/models/st7701s.py @@ -7,6 +7,10 @@ SDIR_CMD = 0xC7 class ST7701S(RgbDriverChip): + def __init__(self, *args, reset_delay=50, **kwargs): + kwargs["reset_delay"] = reset_delay + super().__init__(*args, **kwargs) + # The ST7701s does not use the standard MADCTL bits for x/y mirroring def add_madctl(self, sequence: list, config: dict) -> int: transform = self.get_transform(config) @@ -49,7 +53,6 @@ st7701s = ST7701S( pclk_frequency="16MHz", pclk_inverted=True, initsequence=( - (0x01,), # Software Reset (0xFF, 0x77, 0x01, 0x00, 0x00, 0x10), # Page 0 (0xC0, 0x3B, 0x00), (0xC1, 0x0D, 0x02), (0xC2, 0x31, 0x05), (0xB0, 0x00, 0x11, 0x18, 0x0E, 0x11, 0x06, 0x07, 0x08, 0x07, 0x22, 0x04, 0x12, 0x0F, 0xAA, 0x31, 0x18,), diff --git a/tests/component_tests/mipi_rgb/test_mipi_rgb_config.py b/tests/component_tests/mipi_rgb/test_mipi_rgb_config.py index ac8e111ddba..e677577ec08 100644 --- a/tests/component_tests/mipi_rgb/test_mipi_rgb_config.py +++ b/tests/component_tests/mipi_rgb/test_mipi_rgb_config.py @@ -1,5 +1,7 @@ """Tests for mipi_rgb configuration validation.""" +from collections.abc import Generator + import pytest from esphome import config_validation as cv @@ -17,6 +19,7 @@ from esphome.components.esp32 import ( VARIANT_ESP32S3, VARIANT_ESP32S31, ) +from esphome.components.mipi import DriverChip import esphome.components.pca9554 # noqa: F401 import esphome.components.xl9535 # noqa: F401 from esphome.const import ( @@ -44,6 +47,19 @@ DATA_PINS = { } +@pytest.fixture(autouse=True) +def _remove_test_models() -> Generator[None]: + """Unregister chips created by a test. + + display.py modules drain DriverChip.models when first imported, so a + leftover TEST-* chip could become a selectable model there. + """ + existing = set(DriverChip.models) + yield + for name in set(DriverChip.models) - existing: + del DriverChip.models[name] + + def _set_s3(set_core_config: SetCoreConfigCallable) -> None: set_core_config( PlatformFramework.ESP32_IDF, @@ -176,6 +192,41 @@ def test_configuration_succeeds_on_supported_variants( CONFIG_SCHEMA(config) +def test_st7701s_default_reset_delay() -> None: + """ST7701S instances default to a 50ms reset delay. + + The datasheet's stated 5ms is too short in practice; ST7701S overrides the + DriverChip default of 10ms with its own default of 50ms. + """ + from esphome.components.mipi_rgb.models.st7701s import st7701s + + assert st7701s.get_default("reset_delay") == 50 + + +def test_st7701s_reset_delay_can_be_overridden() -> None: + """An explicit reset_delay overrides the ST7701S default of 50ms.""" + from esphome.components.mipi_rgb.models.st7701s import ST7701S + + chip = ST7701S("TEST-ST7701S-RESET-DELAY", width=480, height=480, reset_delay=99) + + assert chip.get_default("reset_delay") == 99 + + +def test_st7701s_extend_inherits_reset_delay_default() -> None: + """extend() carries the 50ms default forward to derived board models. + + Every shipped ST7701S variant is built via ``st7701s.extend(...)`` rather + than direct construction, so the override in ``ST7701S.__init__`` must + survive that path (see DriverChip.extend, which re-passes the copied + defaults as kwargs to the constructor). + """ + from esphome.components.mipi_rgb.models.st7701s import st7701s + + extended = st7701s.extend("TEST-ST7701S-EXTEND", width=480, height=480) + + assert extended.get_default("reset_delay") == 50 + + def test_only_on_variant_rejects_unsupported_variant( set_core_config: SetCoreConfigCallable, ) -> None: diff --git a/tests/component_tests/mipi_rgb/test_reset_sequence.py b/tests/component_tests/mipi_rgb/test_reset_sequence.py new file mode 100644 index 00000000000..50478361b14 --- /dev/null +++ b/tests/component_tests/mipi_rgb/test_reset_sequence.py @@ -0,0 +1,81 @@ +"""End-to-end tests for the mipi_rgb SPI reset sequence. + +These exercise the actual codegen path (mipi_rgb/display.py's +``model.get_sequence(config, add_reset=True)`` call) rather than calling +DriverChip.get_sequence directly, so a regression that drops add_reset or +reintroduces a hardcoded SWRESET into a model's initsequence would be caught +here. +""" + +from collections.abc import Callable +from pathlib import Path + +# A model with no reset_pin default: SWRESET ({1, 0}) is prepended ahead of the +# inherited ST7701S reset_delay ({50, 255}). +_NO_RESET_PIN_YAML = """ +esphome: + name: mipi-rgb-reset-test +esp32: + board: esp32-s3-devkitc-1 + framework: + type: esp-idf +psram: + mode: octal +spi: + id: spi_bus + clk_pin: 10 + mosi_pin: 11 +display: + - platform: mipi_rgb + id: no_reset_display + spi_id: spi_bus + model: MAKERFABS-4 +""" + +# A model with a reset_pin default: no SWRESET, just the settling delay. +_RESET_PIN_YAML = """ +esphome: + name: mipi-rgb-reset-test +esp32: + board: esp32-s3-devkitc-1 + framework: + type: esp-idf +psram: + mode: octal +spi: + id: spi_bus + clk_pin: 6 + mosi_pin: 7 +display: + - platform: mipi_rgb + id: has_reset_display + spi_id: spi_bus + model: WAVESHARE-3.16-320X820 +""" + + +def test_swreset_and_reset_delay_without_reset_pin( + generate_main: Callable[[str | Path], str], + tmp_path: Path, +) -> None: + """A model with no reset_pin gets SWRESET plus the ST7701S 50ms delay.""" + yaml_file = tmp_path / "no_reset.yaml" + yaml_file.write_text(_NO_RESET_PIN_YAML) + + main_cpp = generate_main(yaml_file) + + assert "no_reset_display->set_init_sequence({1, 0, 50, 255," in main_cpp + + +def test_reset_delay_only_with_reset_pin( + generate_main: Callable[[str | Path], str], + tmp_path: Path, +) -> None: + """A model with a reset_pin default skips SWRESET but keeps the settling delay.""" + yaml_file = tmp_path / "has_reset.yaml" + yaml_file.write_text(_RESET_PIN_YAML) + + main_cpp = generate_main(yaml_file) + + assert "has_reset_display->set_init_sequence({50, 255," in main_cpp + assert "has_reset_display->set_init_sequence({1, 0," not in main_cpp diff --git a/tests/component_tests/mipi_spi/test_get_sequence.py b/tests/component_tests/mipi_spi/test_get_sequence.py new file mode 100644 index 00000000000..263b567b9c9 --- /dev/null +++ b/tests/component_tests/mipi_spi/test_get_sequence.py @@ -0,0 +1,64 @@ +"""Tests for DriverChip.get_sequence's reset-delay handling.""" + +from collections.abc import Generator + +import pytest + +from esphome.components.mipi import CONF_INVERT_COLORS, CONF_PIXEL_MODE, DriverChip + +# A minimal config with no reset pin: enough for get_sequence(add_madctl=False) to run +# without needing a full display configuration. +_BASE_CONFIG = {CONF_PIXEL_MODE: "16bit", CONF_INVERT_COLORS: False} + + +@pytest.fixture(autouse=True) +def _remove_test_models() -> Generator[None]: + """Unregister chips created by a test.""" + existing = set(DriverChip.models) + yield + for name in set(DriverChip.models) - existing: + del DriverChip.models[name] + + +def test_get_sequence_defaults_to_10ms_reset_delay() -> None: + """A model with no reset_delay default falls back to a 10ms settling delay.""" + chip = DriverChip("TEST-GET-SEQUENCE-DEFAULT") + + sequence = chip.get_sequence(_BASE_CONFIG, add_madctl=False, add_reset=True) + + # SWRESET ({1, 0}) is prepended (no reset pin configured), followed by the + # 10ms settling delay, flattened to {10, 255}. + assert sequence[:4] == (1, 0, 10, 255) + + +def test_get_sequence_uses_model_reset_delay_default() -> None: + """A model's own reset_delay default overrides the base 10ms default.""" + chip = DriverChip("TEST-GET-SEQUENCE-CUSTOM-DELAY", reset_delay=99) + + sequence = chip.get_sequence(_BASE_CONFIG, add_madctl=False, add_reset=True) + + assert sequence[:4] == (1, 0, 99, 255) + + +@pytest.mark.parametrize("reset_delay", [0, 256]) +def test_get_sequence_rejects_out_of_range_reset_delay(reset_delay: int) -> None: + """reset_delay outside 1-255ms is rejected. + + This matches the 1-255ms range map_sequence() already allows for a + "delay N" entry in a custom init sequence. + """ + chip = DriverChip("TEST-GET-SEQUENCE-BAD-DELAY", reset_delay=reset_delay) + + with pytest.raises(ValueError, match="reset_delay must be between 1 and 255ms"): + chip.get_sequence(_BASE_CONFIG, add_madctl=False, add_reset=True) + + +def test_get_sequence_skips_reset_delay_validation_without_add_reset() -> None: + """An out-of-range reset_delay is only checked when add_reset is requested. + + mipi_dsi calls get_sequence with add_reset=False and never uses + reset_delay, so an invalid default there should not raise. + """ + chip = DriverChip("TEST-GET-SEQUENCE-NO-RESET", reset_delay=999) + + chip.get_sequence(_BASE_CONFIG, add_madctl=False, add_reset=False) From 3f065530238d4553fb8511a5fef5509c0cfee662 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 01:57:25 +0200 Subject: [PATCH 013/133] [ltr501] Use the sub_* helpers for optional child entities (#19867) --- esphome/components/ltr501/sensor.py | 30 +++++++---------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/esphome/components/ltr501/sensor.py b/esphome/components/ltr501/sensor.py index e27a7c323ef..51378e4cddf 100644 --- a/esphome/components/ltr501/sensor.py +++ b/esphome/components/ltr501/sensor.py @@ -229,29 +229,13 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if als_config := config.get(CONF_AMBIENT_LIGHT): - sens = await sensor.new_sensor(als_config) - cg.add(var.set_ambient_light_sensor(sens)) - - if infrared_cnt_config := config.get(CONF_INFRARED_COUNTS): - sens = await sensor.new_sensor(infrared_cnt_config) - cg.add(var.set_infrared_counts_sensor(sens)) - - if full_spect_cnt_config := config.get(CONF_FULL_SPECTRUM_COUNTS): - sens = await sensor.new_sensor(full_spect_cnt_config) - cg.add(var.set_full_spectrum_counts_sensor(sens)) - - if act_gain_config := config.get(CONF_ACTUAL_GAIN): - sens = await sensor.new_sensor(act_gain_config) - cg.add(var.set_actual_gain_sensor(sens)) - - if act_itime_config := config.get(CONF_ACTUAL_INTEGRATION_TIME): - sens = await sensor.new_sensor(act_itime_config) - cg.add(var.set_actual_integration_time_sensor(sens)) - - if prox_cnt_config := config.get(CONF_PS_COUNTS): - sens = await sensor.new_sensor(prox_cnt_config) - cg.add(var.set_proximity_counts_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_AMBIENT_LIGHT, var.set_ambient_light_sensor) + await sensors(CONF_INFRARED_COUNTS, var.set_infrared_counts_sensor) + await sensors(CONF_FULL_SPECTRUM_COUNTS, var.set_full_spectrum_counts_sensor) + await sensors(CONF_ACTUAL_GAIN, var.set_actual_gain_sensor) + await sensors(CONF_ACTUAL_INTEGRATION_TIME, var.set_actual_integration_time_sensor) + await sensors(CONF_PS_COUNTS, var.set_proximity_counts_sensor) await automation.build_callback_automations(var, config, _CALLBACK_AUTOMATIONS) From 08387ad289dfbff5d239b2ced914d64f90be9189 Mon Sep 17 00:00:00 2001 From: elwin loomis Date: Sat, 5 Sep 2026 16:08:21 -0500 Subject: [PATCH 014/133] [mipi_dsi] Let IDF pick the DPHY PLL reference clock (#18984) Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> --- esphome/components/mipi_dsi/mipi_dsi.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esphome/components/mipi_dsi/mipi_dsi.cpp b/esphome/components/mipi_dsi/mipi_dsi.cpp index 0850b50c85d..0150cc25442 100644 --- a/esphome/components/mipi_dsi/mipi_dsi.cpp +++ b/esphome/components/mipi_dsi/mipi_dsi.cpp @@ -35,8 +35,8 @@ void MipiDsi::setup() { .bus_id = 0, // index from 0, specify the DSI host to use .num_data_lanes = this->lanes_, // Number of data lanes to use, can't set a value that exceeds the chip's capability - .phy_clk_src = MIPI_DSI_PHY_CLK_SRC_DEFAULT, // Clock source for the DPHY - .lane_bit_rate_mbps = this->lane_bit_rate_, // Bit rate of the data lanes, in Mbps + // phy_clk_src left at 0 to enable runtime auto-select. + .lane_bit_rate_mbps = this->lane_bit_rate_, // Bit rate of the data lanes, in Mbps }; auto err = esp_lcd_new_dsi_bus(&bus_config, &this->bus_handle_); if (err != ESP_OK) { From 7e993016442bee0186d40af00ad2bdfd2fb42de0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 16 Sep 2026 17:19:04 -0500 Subject: [PATCH 015/133] [ota] Skip the web_server plaintext warning when web_server ota is disabled (#19348) --- esphome/components/esphome/ota/__init__.py | 12 +++++++-- tests/component_tests/ota/test_esphome_ota.py | 26 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index f5eb878260c..bcf2a2271cb 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -166,9 +166,17 @@ def ota_esphome_final_validate(config: ConfigType) -> None: CONF_PASSWORD, ) # web_server and prometheus keep the shared listener up; the captive - # portal's copy only exists on the fallback AP and is the recovery path + # portal's copy only exists on the fallback AP and is the recovery path. + # web_server `ota: false` gates /update behind the captive portal on + # every listener + web_server_conf = full_conf.get(CONF_WEB_SERVER) + plaintext_update_reachable = ( + web_server_conf.get(CONF_OTA) is not False + if web_server_conf is not None + else "prometheus" in full_conf + ) if ( - (CONF_WEB_SERVER in full_conf or "prometheus" in full_conf) + plaintext_update_reachable and any(conf.get(CONF_PLATFORM) == CONF_WEB_SERVER for conf in full_ota_conf) and any( CONF_ENCRYPTION in conf diff --git a/tests/component_tests/ota/test_esphome_ota.py b/tests/component_tests/ota/test_esphome_ota.py index d3092294dc1..235ad902dbc 100644 --- a/tests/component_tests/ota/test_esphome_ota.py +++ b/tests/component_tests/ota/test_esphome_ota.py @@ -319,6 +319,32 @@ def test_encryption_with_captive_portal_does_not_warn( fv.full_config.reset(token) +@pytest.mark.parametrize("extra", [{}, {"prometheus": {}}]) +def test_encryption_with_web_server_ota_disabled_does_not_warn( + caplog: pytest.LogCaptureFixture, extra: dict[str, Any] +) -> None: + """web_server `ota: false` only serves /update while the captive portal is + active, on every listener, so there is no plaintext endpoint to warn about.""" + full_conf = { + "web_server": {CONF_OTA: False}, + **extra, + CONF_OTA: [ + _make_ota_config(port=3232, **{CONF_ENCRYPTION: {CONF_KEY: OTHER_KEY}}), + {CONF_PLATFORM: "web_server", CONF_ID: ID("ota_ws", is_manual=False)}, + ], + } + token = fv.full_config.set(full_conf) + try: + with caplog.at_level(logging.WARNING): + ota_esphome_final_validate({}) + assert not any( + "OTA encryption does not cover" in record.message + for record in caplog.records + ) + finally: + fv.full_config.reset(token) + + def test_password_with_api_key_warns(caplog: pytest.LogCaptureFixture) -> None: """A static api key makes the device offer encryption and the CLI take it, so the password is dead weight; the config validates with a warning.""" From 3e5797d1531c161a1b945e4b5f35f2f06f3f6fb1 Mon Sep 17 00:00:00 2001 From: rexmoriarty Date: Thu, 17 Sep 2026 04:47:40 -0500 Subject: [PATCH 016/133] [mixer] Don't discard a start request while reaping a stopped task (#19368) --- esphome/components/mixer/speaker/mixer_speaker.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index ef21da65c5a..85fb445d408 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -385,7 +385,8 @@ void MixerSpeaker::loop() { // Retries on a subsequent loop if the task is still running on the other core if ((event_group_bits & MIXER_TASK_STATE_STOPPED) && this->task_.deallocate()) { ESP_LOGD(TAG, "Stopped"); - xEventGroupClearBits(this->event_group_, MIXER_TASK_ALL_BITS); + // Keep a start request that arrived while the task was stopping, otherwise it is lost for good + xEventGroupClearBits(this->event_group_, MIXER_TASK_ALL_BITS & ~MIXER_TASK_COMMAND_START); this->all_stopped_since_ms_ = 0; } From 964a54f7a76ffa4432588a67f73746f4bcb83886 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Sep 2026 08:17:11 -0500 Subject: [PATCH 017/133] [esp32][rp2] Print the previous boot crash report before the logger reads it (#19351) --- esphome/components/esp32/crash_handler.cpp | 14 +++++++++----- esphome/components/esp32/crash_handler.h | 7 +------ esphome/components/esp32/hal.cpp | 8 -------- esphome/components/rp2/crash_handler.cpp | 22 +++++++++++++++++----- esphome/components/rp2/crash_handler.h | 3 ++- esphome/core/application.h | 4 ++-- 6 files changed, 31 insertions(+), 27 deletions(-) diff --git a/esphome/components/esp32/crash_handler.cpp b/esphome/components/esp32/crash_handler.cpp index 6f65243aaa0..b72a2777c7a 100644 --- a/esphome/components/esp32/crash_handler.cpp +++ b/esphome/components/esp32/crash_handler.cpp @@ -173,7 +173,10 @@ static const char *const TAG = "esp32.crash"; // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) static uint32_t s_current_build_time = static_cast(ESPHOME_BUILD_TIME); -void crash_handler_read_and_clear() { +// Validate the NOINIT record. Runs on every has_data() call; re-running is +// harmless and the magic is left alone so the record survives an OTA +// rollback reboot, crash_handler_clear() drops it once an API client has it. +static void read_crash_data() { if (s_raw_crash_data.magic == CRASH_MAGIC && s_raw_crash_data.version == CRASH_DATA_VERSION) { s_crash_data_valid = true; // Clamp counts to prevent out-of-bounds reads from corrupt .noinit data @@ -194,11 +197,12 @@ void crash_handler_read_and_clear() { s_raw_crash_data.other_reg_frame_count = s_raw_crash_data.other_backtrace_count; #endif } - // Don't clear magic here — crash data must survive OTA rollback reboots. - // Magic is cleared by crash_handler_clear() after an API client receives the data. } -bool crash_handler_has_data() { return s_crash_data_valid; } +bool crash_handler_has_data() { + read_crash_data(); + return s_crash_data_valid; +} void crash_handler_clear() { // Only clear the magic so data doesn't survive the next reboot. @@ -426,7 +430,7 @@ static void log_foreign_addresses() { // crashes again during boot, and allowing the CLI's process_stacktrace to match // and decode each address individually. void crash_handler_log() { - if (!s_crash_data_valid) + if (!crash_handler_has_data()) return; ESP_LOGE(TAG, "*** CRASH DETECTED ON PREVIOUS BOOT ***"); diff --git a/esphome/components/esp32/crash_handler.h b/esphome/components/esp32/crash_handler.h index c5e7d145ece..314be80314c 100644 --- a/esphome/components/esp32/crash_handler.h +++ b/esphome/components/esp32/crash_handler.h @@ -4,11 +4,6 @@ namespace esphome::esp32 { -/// Read and validate crash data from NOINIT memory. -/// Does not clear the magic marker — call crash_handler_clear() after -/// the data has been delivered to an API client so it survives OTA rollback reboots. -void crash_handler_read_and_clear(); - /// Log crash data if a crash was detected on previous boot. void crash_handler_log(); @@ -16,7 +11,7 @@ void crash_handler_log(); /// Call after the data has been delivered to an API client. void crash_handler_clear(); -/// Returns true if crash data was found this boot. +/// Returns true if crash data was found this boot, reading it first if needed. bool crash_handler_has_data(); } // namespace esphome::esp32 diff --git a/esphome/components/esp32/hal.cpp b/esphome/components/esp32/hal.cpp index f6199d557f3..199cb89f516 100644 --- a/esphome/components/esp32/hal.cpp +++ b/esphome/components/esp32/hal.cpp @@ -1,9 +1,6 @@ #ifdef USE_ESP32 -// defines.h must come before crash_handler.h so USE_ESP32_CRASH_HANDLER is set -// before crash_handler.h's #ifdef-guarded namespace block is parsed. #include "esphome/core/defines.h" -#include "crash_handler.h" #include "esphome/core/hal.h" #include @@ -45,11 +42,6 @@ void arch_restart() { } void arch_init() { -#ifdef USE_ESP32_CRASH_HANDLER - // Read crash data from previous boot before anything else - esp32::crash_handler_read_and_clear(); -#endif - // Enable the task watchdog only on the loop task (from which we're currently running) esp_task_wdt_add(nullptr); diff --git a/esphome/components/rp2/crash_handler.cpp b/esphome/components/rp2/crash_handler.cpp index a0fea216371..9bcdc8bee4e 100644 --- a/esphome/components/rp2/crash_handler.cpp +++ b/esphome/components/rp2/crash_handler.cpp @@ -55,8 +55,7 @@ namespace esphome::rp2 { static const char *const TAG = "rp2.crash"; -// Placed in .noinit so BSS zero-init cannot race with crash_handler_read_and_clear(). -// The valid field is explicitly cleared in crash_handler_read_and_clear() instead. +// Filled from the watchdog scratch registers on the first read. static struct CrashData { bool valid; uint32_t pc; @@ -64,11 +63,24 @@ static struct CrashData { uint32_t sp; uint32_t backtrace[MAX_BACKTRACE]; uint8_t backtrace_count; -} s_crash_data __attribute__((section(".noinit"))); // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +} s_crash_data; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) -bool crash_handler_has_data() { return s_crash_data.valid; } +// Logger::pre_setup() logs the record before App.pre_setup() reaches +// arch_init(), so the first caller reads it and later calls are no-ops. +// The read clears the scratch registers, so it must not run twice, and +// arch_init() keeps its call so the read precedes watchdog_enable(), which +// overwrites scratch[4]. +static bool s_crash_data_read = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +bool crash_handler_has_data() { + crash_handler_read_and_clear(); + return s_crash_data.valid; +} void crash_handler_read_and_clear() { + if (s_crash_data_read) + return; + s_crash_data_read = true; s_crash_data.valid = false; uint32_t magic = watchdog_hw->scratch[0]; if ((magic & 0xFFFF0000) == CRASH_MAGIC_SENTINEL && (magic & 0xFFFF) == CRASH_DATA_VERSION) { @@ -97,7 +109,7 @@ void crash_handler_read_and_clear() { // the device crashes again during boot, and allowing the CLI's process_stacktrace // to match and decode each address individually. void crash_handler_log() { - if (!s_crash_data.valid) + if (!crash_handler_has_data()) return; ESP_LOGE(TAG, "*** CRASH DETECTED ON PREVIOUS BOOT ***"); diff --git a/esphome/components/rp2/crash_handler.h b/esphome/components/rp2/crash_handler.h index 8c43d9fd3b0..3aec80b63b2 100644 --- a/esphome/components/rp2/crash_handler.h +++ b/esphome/components/rp2/crash_handler.h @@ -9,12 +9,13 @@ namespace esphome::rp2 { /// Read crash data from watchdog scratch registers and clear them. +/// Only the first call reads; later calls are no-ops. void crash_handler_read_and_clear(); /// Log crash data if a crash was detected on previous boot. void crash_handler_log(); -/// Returns true if crash data was found this boot. +/// Returns true if crash data was found this boot, reading it first if needed. bool crash_handler_has_data(); } // namespace esphome::rp2 diff --git a/esphome/core/application.h b/esphome/core/application.h index a12cdc4ac88..2836587dfd2 100644 --- a/esphome/core/application.h +++ b/esphome/core/application.h @@ -67,7 +67,7 @@ static constexpr uint32_t TEARDOWN_TIMEOUT_REBOOT_MS = 1000; // 1 second for qu class Application { public: #ifdef ESPHOME_NAME_ADD_MAC_SUFFIX - // Called before Logger::pre_setup() — must not log (global_logger is not yet set). + // Runs after Logger::pre_setup() (emitted at EARLY_INIT priority), so the app name is not set yet there. /// Pre-setup with MAC suffix: overwrites placeholder in mutable static buffers with actual MAC. void pre_setup(char *name, size_t name_len, char *friendly_name, size_t friendly_name_len) { arch_init(); @@ -87,7 +87,7 @@ class Application { this->friendly_name_ = StringRef(friendly_name, friendly_name_len); } #else - // Called before Logger::pre_setup() — must not log (global_logger is not yet set). + // Runs after Logger::pre_setup() (emitted at EARLY_INIT priority), so the app name is not set yet there. /// Pre-setup without MAC suffix: StringRef points directly at const string literals in flash. void pre_setup(const char *name, size_t name_len, const char *friendly_name, size_t friendly_name_len) { arch_init(); From eaee1cdd8636c60e8cd69eac88bbf5866c645cbb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Sep 2026 09:40:22 -0500 Subject: [PATCH 018/133] [espidf] Switch the Windows console to UTF-8 while idf.py runs (#19352) --- esphome/espidf/runner.py | 99 +++++++++++++++++++---- tests/unit_tests/test_espidf_runner.py | 104 +++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 14 deletions(-) diff --git a/esphome/espidf/runner.py b/esphome/espidf/runner.py index 7ed11d75543..bf563a4f438 100644 --- a/esphome/espidf/runner.py +++ b/esphome/espidf/runner.py @@ -74,6 +74,64 @@ FILTER_IDF_LINES: list[str] = [ r"Stopping at filesystem boundary", ] +# Windows code page identifier for UTF-8, as used by ``chcp 65001``. +UTF8_CODEPAGE = 65001 + + +def _get_kernel32(): + """Return the Windows kernel32 module, or None on any other platform.""" + if sys.platform != "win32": + return None + import ctypes + + return ctypes.windll.kernel32 + + +class _Utf8Console: + """Keep an attached Windows console on UTF-8 for the length of the build. + + The build tree runs in UTF-8 mode, so esp_idf_size draws its table with + Unicode box characters. ``idf.py size`` reaches it through ``cmake -P``, + and CMake re-decodes the child's output with the console code page before + printing it, which turns the table into mojibake on any code page but + UTF-8. Every process in the build shares this console, so switching it + here covers CMake too. The old code pages go back on exit so the user's + terminal is left as it was. + + A console that is already on UTF-8 is left alone. The code page belongs to + the console, not to this process, so a build that overlaps another one + must not save UTF-8 as the page to go back to. + """ + + def __init__(self, kernel32) -> None: + self._kernel32 = kernel32 + self._codepages: tuple[int, int] | None = None + + def __enter__(self) -> None: + kernel32 = self._kernel32 + if kernel32 is None: + return + old_in = kernel32.GetConsoleCP() + old_out = kernel32.GetConsoleOutputCP() + # Both calls return 0 when no console is attached. + if not old_in or not old_out: + return + if old_in == UTF8_CODEPAGE and old_out == UTF8_CODEPAGE: + return + # Record the old pages first so a switch that fails part way through + # still gets put back on exit. + self._codepages = (old_in, old_out) + kernel32.SetConsoleCP(UTF8_CODEPAGE) + kernel32.SetConsoleOutputCP(UTF8_CODEPAGE) + + def __exit__(self, *exc_info: object) -> None: + if self._codepages is None: + return + old_in, old_out = self._codepages + self._codepages = None + self._kernel32.SetConsoleCP(old_in) + self._kernel32.SetConsoleOutputCP(old_out) + def main() -> int: # ---- sys.path fix-up --------------------------------------------------- @@ -269,8 +327,29 @@ def main() -> int: is_verbose = any(arg in ("-v", "--verbose") for arg in sys.argv[2:]) filter_lines = None if is_verbose else FILTER_IDF_LINES or None - stdout_shim = sys.stdout = _FilteringTTYStream(sys.stdout, filter_lines) # type: ignore[assignment] - stderr_shim = sys.stderr = _FilteringTTYStream(sys.stderr, filter_lines) # type: ignore[assignment] + class _FilteredStreams: + """Route ``sys.stdout`` and ``sys.stderr`` through the filtering shims. + + On exit each shim releases a last line that never got its + terminator. The shims made here are drained rather than whatever + ``sys.stdout`` holds by then, which the script is free to replace. + """ + + def __init__(self, filter_lines: list[str] | None) -> None: + self._stdout = _FilteringTTYStream(sys.stdout, filter_lines) + self._stderr = _FilteringTTYStream(sys.stderr, filter_lines) + + def __enter__(self) -> None: + sys.stdout = self._stdout # type: ignore[assignment] + sys.stderr = self._stderr # type: ignore[assignment] + + def __exit__(self, *exc_info: object) -> None: + # Drain stderr from a finally so a surprise from the first one + # cannot strand the second. + try: + self._stdout.drain() + finally: + self._stderr.drain() # Shift argv so the target script sees its own path as argv[0] and # its own arguments starting at argv[1]. runpy.run_path does not @@ -288,19 +367,11 @@ def main() -> int: # If idf.py calls sys.exit(), SystemExit propagates out of run_path # and carries the exit code back to our caller. For normal returns, - # fall through and exit with 0. Either way the streams get a chance to - # release a last line that never got its terminator. Drain the shims we - # made rather than sys.stdout, which the script is free to replace, and - # report instead of raising so cleanup cannot bury the real exit code. - try: + # fall through and exit with 0. Either way the context managers drain + # the streams and put the console back on the way out, and they report + # instead of raising so cleanup cannot bury the real exit code. + with _FilteredStreams(filter_lines), _Utf8Console(_get_kernel32()): runpy.run_path(script_path, run_name="__main__") - finally: - # Drain stderr from a finally so a surprise from the first one cannot - # strand the second. - try: - stdout_shim.drain() - finally: - stderr_shim.drain() return 0 diff --git a/tests/unit_tests/test_espidf_runner.py b/tests/unit_tests/test_espidf_runner.py index e4cc6e137e7..71ccfee1f2c 100644 --- a/tests/unit_tests/test_espidf_runner.py +++ b/tests/unit_tests/test_espidf_runner.py @@ -209,3 +209,107 @@ def test_runner_streams_output_before_the_build_finishes( # Join before leaving the block, so the reader is done rather than # racing ``Popen`` closing the pipe under it. reader.join(1.0) + + +class _FakeKernel32: + """Stand-in for the Windows kernel32 console code page calls.""" + + def __init__(self, input_cp: int, output_cp: int) -> None: + self.input_cp = input_cp + self.output_cp = output_cp + self.calls: list[tuple[str, int]] = [] + + def GetConsoleCP(self) -> int: # noqa: N802 + return self.input_cp + + def GetConsoleOutputCP(self) -> int: # noqa: N802 + return self.output_cp + + def SetConsoleCP(self, codepage: int) -> int: # noqa: N802 + self.calls.append(("SetConsoleCP", codepage)) + self.input_cp = codepage + return 1 + + def SetConsoleOutputCP(self, codepage: int) -> int: # noqa: N802 + self.calls.append(("SetConsoleOutputCP", codepage)) + self.output_cp = codepage + return 1 + + +def test_main_runs_the_build_with_a_utf8_console( + monkeypatch: pytest.MonkeyPatch, fixture_path: Path +) -> None: + """An attached console is switched to UTF-8 and then put back.""" + kernel32 = _FakeKernel32(850, 850) + monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) + + _run_main(monkeypatch, fixture_path / "espidf" / "filtering_probe.py") + + assert kernel32.calls == [ + ("SetConsoleCP", runner.UTF8_CODEPAGE), + ("SetConsoleOutputCP", runner.UTF8_CODEPAGE), + ("SetConsoleCP", 850), + ("SetConsoleOutputCP", 850), + ] + + +def test_main_restores_the_console_when_the_build_dies( + monkeypatch: pytest.MonkeyPatch, fixture_path: Path +) -> None: + """A failing build must not leave the user's console on UTF-8.""" + kernel32 = _FakeKernel32(437, 437) + monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) + _prepare_main(monkeypatch, fixture_path / "espidf" / "crashing_probe.py") + + with pytest.raises(SystemExit): + runner.main() + + assert (kernel32.input_cp, kernel32.output_cp) == (437, 437) + + +def test_main_restores_the_console_when_the_switch_fails_part_way( + monkeypatch: pytest.MonkeyPatch, fixture_path: Path +) -> None: + """A failed output page switch must not strand the changed input page.""" + kernel32 = _FakeKernel32(850, 850) + + def _refuse(codepage: int) -> int: + kernel32.calls.append(("SetConsoleOutputCP", codepage)) + return 0 + + kernel32.SetConsoleOutputCP = _refuse # type: ignore[method-assign] + monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) + + _run_main(monkeypatch, fixture_path / "espidf" / "filtering_probe.py") + + assert kernel32.input_cp == 850 + assert kernel32.calls[-2:] == [("SetConsoleCP", 850), ("SetConsoleOutputCP", 850)] + + +def test_main_leaves_the_console_alone_when_there_is_none( + monkeypatch: pytest.MonkeyPatch, fixture_path: Path +) -> None: + """Without a console the code page calls return 0 and nothing is set.""" + kernel32 = _FakeKernel32(0, 0) + monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) + + _run_main(monkeypatch, fixture_path / "espidf" / "filtering_probe.py") + + assert kernel32.calls == [] + + +def test_main_leaves_a_console_already_on_utf8_alone( + monkeypatch: pytest.MonkeyPatch, fixture_path: Path +) -> None: + """An overlapping build must not save UTF-8 as the page to restore.""" + kernel32 = _FakeKernel32(runner.UTF8_CODEPAGE, runner.UTF8_CODEPAGE) + monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) + + _run_main(monkeypatch, fixture_path / "espidf" / "filtering_probe.py") + + assert kernel32.calls == [] + + +@pytest.mark.skipif(sys.platform == "win32", reason="kernel32 exists on Windows") +def test_get_kernel32_is_none_off_windows() -> None: + assert runner._get_kernel32() is None From c732903390d940c2928021955f08284daaf7f042 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Sep 2026 09:57:59 -0500 Subject: [PATCH 019/133] [esp32_hosted] Fix ESP32-P4 build without wifi, espnow or BLE (#19374) --- esphome/components/esp32/__init__.py | 2 +- esphome/components/esp32_hosted/__init__.py | 2 ++ .../esp32_hosted/test-no-wifi.esp32-p4-idf.yaml | 12 ++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 tests/components/esp32_hosted/test-no-wifi.esp32-p4-idf.yaml diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 3f5a34bc735..4597904cc39 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -254,7 +254,7 @@ DEFAULT_EXCLUDED_IDF_COMPONENTS = ( "esp_lcd", # LCD controller drivers - only needed by display component "esp_local_ctrl", # Local control over HTTPS/BLE - ESPHome has native API "esp_phy", # RF PHY - re-included by internal_temperature on the original ESP32; esp_wifi/bt/ieee802154 pull it back - "esp_wifi", # WiFi stack - re-included by request_wifi(), espnow; bt pulls it back for BLE builds + "esp_wifi", # WiFi stack - re-included by request_wifi(), espnow, esp32_hosted; bt pulls it back for BLE builds "espcoredump", # Core dump support - ESPHome has its own debug component "fatfs", # FAT filesystem - ESPHome doesn't use filesystem storage "ieee802154", # 802.15.4 radio - IDF openthread and the Zigbee libs pull it back diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index 21626e432b6..6943efc3cb8 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -290,6 +290,8 @@ async def to_code(config: ConfigType) -> None: # symbols are simply unused and never register a callback at runtime. if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4: add_define("USE_ESP_NOW_HOSTED") + # esp_now_hosted.cpp includes esp_now.h, which esp_wifi provides + esp32.include_builtin_idf_component("esp_wifi") # esp-hosted's CustomRpc ("peer data transfer") path — off by default. esp32.add_idf_sdkconfig_option( "CONFIG_ESP_HOSTED_ENABLE_PEER_DATA_TRANSFER", True diff --git a/tests/components/esp32_hosted/test-no-wifi.esp32-p4-idf.yaml b/tests/components/esp32_hosted/test-no-wifi.esp32-p4-idf.yaml new file mode 100644 index 00000000000..2f57abd296e --- /dev/null +++ b/tests/components/esp32_hosted/test-no-wifi.esp32-p4-idf.yaml @@ -0,0 +1,12 @@ +# No wifi, espnow or BLE: nothing else re-includes esp_wifi for the ESP-NOW shim. +esp32_hosted: + variant: ESP32C6 + slot: 1 + active_high: true + reset_pin: GPIO15 + cmd_pin: GPIO13 + clk_pin: GPIO12 + d0_pin: GPIO11 + d1_pin: GPIO10 + d2_pin: GPIO9 + d3_pin: GPIO8 From 0575eb14f2fe6a99b0921e77c522d34619ac345a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 17 Sep 2026 10:20:41 -0500 Subject: [PATCH 020/133] [esphome] Keep the OTA encryption key without the api server so safe mode uploads work (#19349) --- esphome/components/api/api_server.cpp | 26 +++++- esphome/components/api/api_server.h | 5 ++ esphome/components/esphome/ota/__init__.py | 18 ++--- .../components/esphome/ota/ota_esphome.cpp | 22 +++-- esphome/components/esphome/ota/ota_esphome.h | 9 ++- esphome/components/noise/__init__.py | 18 +++-- esphome/core/defines.h | 1 - tests/component_tests/ota/test_esphome_ota.py | 26 +++--- .../host_ota_encrypted_safe_mode.yaml | 13 +++ ...ost_ota_provisioned_api_key_safe_mode.yaml | 11 +++ tests/integration/host_prefs.py | 35 ++++++-- tests/integration/test_host_ota.py | 80 +++++++++++++++++-- tests/integration/test_safe_mode_loop_runs.py | 12 +-- 13 files changed, 211 insertions(+), 65 deletions(-) create mode 100644 tests/integration/fixtures/host_ota_encrypted_safe_mode.yaml create mode 100644 tests/integration/fixtures/host_ota_provisioned_api_key_safe_mode.yaml diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 78ebe5c38e9..d74de98c75e 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -29,6 +29,29 @@ static const char *const TAG = "api"; // APIServer APIServer *global_api_server = nullptr; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +#ifdef USE_API_NOISE +static constexpr uint32_t NOISE_PSK_PREF_HASH = 88491486UL; +#endif + +#if defined(USE_API_NOISE) && defined(USE_OTA_ENCRYPTION_PROVISIONED) +bool load_saved_noise_psk(noise::psk_t &out) { + SavedNoisePsk saved; +#ifdef USE_PREFERENCE_KEY_LOOKUP + const bool loaded = + global_preferences->load_from_key(NOISE_PSK_PREF_HASH, reinterpret_cast(&saved), sizeof(saved)); +#else + // Slot backends need the reservation walk; it only lands on the record when the reservations before + // it match a normal boot, otherwise the type checked checksum fails the load + const bool loaded = global_preferences->make_preference(NOISE_PSK_PREF_HASH, true).load(&saved); +#endif + // The all-zeros record means no key + if (!loaded || noise::NoiseContext::is_all_zeros(saved.psk)) + return false; + out = saved.psk; + return true; +} +#endif + APIServer::APIServer() { global_api_server = this; } void APIServer::socket_failed_(const LogString *msg) { @@ -43,8 +66,7 @@ void APIServer::setup() { #ifdef USE_API_NOISE // Always reserve the slot: flash preferences are positional on esp8266, so // a yaml key build must keep the layout of a runtime key build - uint32_t hash = 88491486UL; - this->noise_pref_ = global_preferences->make_preference(hash, true); + this->noise_pref_ = global_preferences->make_preference(NOISE_PSK_PREF_HASH, true); #ifndef USE_API_NOISE_PSK_FROM_YAML // A cleared record loads fine but holds no key if (this->load_and_apply_noise_psk_() && this->noise_ctx_.has_psk()) { diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 618ea4eb11a..4aa0a422dd5 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -43,6 +43,11 @@ struct SavedNoisePsk { noise::psk_t psk; } PACKED; // NOLINT #endif +#if defined(USE_API_NOISE) && defined(USE_OTA_ENCRYPTION_PROVISIONED) +/// One-shot read of the provisioned key for a boot without an api server (safe mode); false when +/// there is no key +bool load_saved_noise_psk(noise::psk_t &out); +#endif class APIServer final : public Component, public Controller diff --git a/esphome/components/esphome/ota/__init__.py b/esphome/components/esphome/ota/__init__.py index bcf2a2271cb..bace13c17fc 100644 --- a/esphome/components/esphome/ota/__init__.py +++ b/esphome/components/esphome/ota/__init__.py @@ -316,20 +316,16 @@ async def to_code(config: ConfigType) -> None: # One key per device: an api encryption block supplies it (static or # runtime) and offers; the ota block only adds the requirement api_conf = CORE.config.get(CONF_API) or {} - encryption_conf = config.get(CONF_ENCRYPTION) - own_key = None - if encryption_conf is not None and static_encryption_key(api_conf) is None: - own_key = encryption_conf[CONF_KEY] - if own_key is not None: + if key := static_encryption_key(config) or static_encryption_key(api_conf): + # Build time key: the ota keeps its own pointer so safe mode, which + # has no api server, still has it cg.add_define("USE_OTA_ENCRYPTION") - cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], own_key))) + cg.add(var.set_noise_psk(new_psk_progmem(config[CONF_ID], key))) elif CONF_ENCRYPTION in api_conf: + # Runtime key: found in the api server, or in preferences in safe mode cg.add_define("USE_OTA_ENCRYPTION") - cg.add_define("USE_OTA_ENCRYPTION_FROM_API") - if static_encryption_key(api_conf) is None: - # The key arrives at runtime, so the offer has to look for it - cg.add_define("USE_OTA_ENCRYPTION_PROVISIONED") - if encryption_conf is not None: + cg.add_define("USE_OTA_ENCRYPTION_PROVISIONED") + if CONF_ENCRYPTION in config: cg.add_define("USE_OTA_ENCRYPTION_REQUIRED") # Build flag so lwip_fast_select.c (a .c file that can't include defines.h) sees it. diff --git a/esphome/components/esphome/ota/ota_esphome.cpp b/esphome/components/esphome/ota/ota_esphome.cpp index 3010df10561..d353d01d205 100644 --- a/esphome/components/esphome/ota/ota_esphome.cpp +++ b/esphome/components/esphome/ota/ota_esphome.cpp @@ -1,5 +1,5 @@ #include "ota_esphome.h" -#ifdef USE_OTA_ENCRYPTION_FROM_API +#ifdef USE_OTA_ENCRYPTION_PROVISIONED #include "esphome/components/api/api_server.h" #endif #ifdef USE_OTA @@ -32,11 +32,13 @@ static const char *const TAG = "esphome.ota"; #ifdef USE_OTA_ENCRYPTION const noise::NoiseContext &ESPHomeOTAComponent::noise_context_() const { -#ifdef USE_OTA_ENCRYPTION_FROM_API - return api::global_api_server->get_noise_ctx(); -#else - return this->noise_ctx_; +#ifdef USE_OTA_ENCRYPTION_PROVISIONED + // The api server holds the live key; safe mode never constructs it, and then + // noise_ctx_ holds the saved key setup() found, if any + if (api::global_api_server != nullptr) + return api::global_api_server->get_noise_ctx(); #endif + return this->noise_ctx_; } #endif static constexpr uint16_t OTA_BLOCK_SIZE = 8192; @@ -58,6 +60,16 @@ extern "C" void esphome_wake_ota_component_any_context() { } void ESPHomeOTAComponent::setup() { +#ifdef USE_OTA_ENCRYPTION_PROVISIONED + // Safe mode never constructs the api server, so read the key it saved + noise::psk_t psk; + if (api::global_api_server == nullptr && api::load_saved_noise_psk(psk)) { + this->saved_psk_ = RAMAllocator().make_unique(psk); + if (this->saved_psk_ != nullptr) { + this->noise_ctx_.set_psk(this->saved_psk_->data()); + } + } +#endif this->server_ = socket::socket_ip_loop_monitored(SOCK_STREAM, 0).release(); // monitored for incoming connections if (this->server_ == nullptr) { this->server_failed_(LOG_STR("creation")); diff --git a/esphome/components/esphome/ota/ota_esphome.h b/esphome/components/esphome/ota/ota_esphome.h index 68dd0ffb9ef..92ba094c8de 100644 --- a/esphome/components/esphome/ota/ota_esphome.h +++ b/esphome/components/esphome/ota/ota_esphome.h @@ -44,7 +44,7 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { } #endif // USE_OTA_PASSWORD -#if defined(USE_OTA_ENCRYPTION) && !defined(USE_OTA_ENCRYPTION_FROM_API) +#ifdef USE_OTA_ENCRYPTION /// psk points at 32 bytes that live in flash for the life of the program void set_noise_psk(const uint8_t *psk) { this->noise_ctx_.set_psk(psk); } #endif @@ -86,7 +86,8 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { bool writing{false}; // a produced handshake frame is still being flushed uint8_t frame_buf[noise::FRAME_HEADER_SIZE + 1 + noise::MAX_HANDSHAKE_SIZE]; }; - // The api server's live context when the api has encryption, else our own + // The api server's live context when it exists, otherwise our own (a build + // time key, or the saved key loaded in safe mode) const noise::NoiseContext &noise_context_() const; bool noise_start_session_(uint8_t server_feature_flags); bool handle_noise_handshake_(); @@ -148,8 +149,10 @@ class ESPHomeOTAComponent final : public ota::OTAComponent { RAMUniquePtr auth_buf_; #endif // USE_OTA_PASSWORD #ifdef USE_OTA_ENCRYPTION -#ifndef USE_OTA_ENCRYPTION_FROM_API noise::NoiseContext noise_ctx_; +#ifdef USE_OTA_ENCRYPTION_PROVISIONED + // Backs noise_ctx_ in safe mode, where no api server holds the saved key + RAMUniquePtr saved_psk_; #endif RAMUniquePtr noise_; #endif // USE_OTA_ENCRYPTION diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 6067fde1642..47cd4cfc677 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -5,11 +5,12 @@ from typing import Any import esphome.codegen as cg import esphome.config_validation as cv from esphome.const import CONF_ENCRYPTION, CONF_KEY -from esphome.core import ID +from esphome.core import CORE, ID from esphome.cpp_generator import MockObj from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] +DOMAIN = "noise" noise_ns = cg.esphome_ns.namespace("noise") @@ -70,11 +71,16 @@ def static_encryption_key(conf: ConfigType) -> str | None: def new_psk_progmem(parent_id: ID, key: str) -> MockObj: """Emit the decoded key as a PROGMEM array; the component keeps a pointer - so the key never occupies RAM.""" - return cg.progmem_array( - ID(f"{parent_id.id}_psk", is_declaration=True, type=cg.uint8), - list(decode_encryption_key(key)), - ) + so the key never occupies RAM. Components sharing one key (api and ota) + share the array.""" + decoded = decode_encryption_key(key) + arrays: dict[bytes, MockObj] = CORE.data.setdefault(DOMAIN, {}) + if (array := arrays.get(decoded)) is None: + array = arrays[decoded] = cg.progmem_array( + ID(f"{parent_id.id}_psk", is_declaration=True, type=cg.uint8), + list(decoded), + ) + return array def encryption_schema(config: ConfigType | None) -> ConfigType: diff --git a/esphome/core/defines.h b/esphome/core/defines.h index b78516c6ef8..94f7648f291 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -245,7 +245,6 @@ #define USE_RUNTIME_STATS #define USE_OTA #define USE_OTA_ENCRYPTION -#define USE_OTA_ENCRYPTION_FROM_API #define USE_OTA_ENCRYPTION_PROVISIONED #define USE_OTA_ENCRYPTION_REQUIRED #define USE_OTA_PASSWORD diff --git a/tests/component_tests/ota/test_esphome_ota.py b/tests/component_tests/ota/test_esphome_ota.py index 235ad902dbc..d8bcd7e2758 100644 --- a/tests/component_tests/ota/test_esphome_ota.py +++ b/tests/component_tests/ota/test_esphome_ota.py @@ -476,43 +476,36 @@ def test_static_encryption_key() -> None: ("yaml_name", "defines_present", "defines_absent"), [ # An api key alone compiles the transport in without requiring it; - # the device uses the api server's key, not a copy + # the ota keeps its own pointer to the key so safe mode, which never + # constructs the api server, can still use it ( "api_key_offer", - {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API"}, + {"USE_OTA_ENCRYPTION"}, {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, ), # A password still guards plaintext uploads on an offering device ( "api_key_offer_password", - {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_PASSWORD"}, + {"USE_OTA_ENCRYPTION", "USE_OTA_PASSWORD"}, {"USE_OTA_ENCRYPTION_REQUIRED", "USE_OTA_ENCRYPTION_PROVISIONED"}, ), # The ota encryption block is what makes the device refuse plaintext ( "encryption_required", - { - "USE_OTA_ENCRYPTION", - "USE_OTA_ENCRYPTION_REQUIRED", - "USE_OTA_ENCRYPTION_FROM_API", - }, + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_REQUIRED"}, {"USE_OTA_ENCRYPTION_PROVISIONED"}, ), # Without api encryption the ota key is the device's own ( "own_key", {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_REQUIRED"}, - {"USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_ENCRYPTION_PROVISIONED"}, + {"USE_OTA_ENCRYPTION_PROVISIONED"}, ), # A key provisioned at runtime lives in the api server; the device # offers with it once provisioned and never requires it ( "runtime_api_key", - { - "USE_OTA_ENCRYPTION", - "USE_OTA_ENCRYPTION_FROM_API", - "USE_OTA_ENCRYPTION_PROVISIONED", - }, + {"USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_PROVISIONED"}, {"USE_OTA_ENCRYPTION_REQUIRED"}, ), # No api encryption at all keeps the noise glue out of the build @@ -522,7 +515,6 @@ def test_static_encryption_key() -> None: { "USE_OTA_ENCRYPTION", "USE_OTA_ENCRYPTION_REQUIRED", - "USE_OTA_ENCRYPTION_FROM_API", "USE_OTA_ENCRYPTION_PROVISIONED", }, ), @@ -541,8 +533,10 @@ def test_encryption_offer_codegen( assert defines_present <= defines assert not (defines_absent & defines) encrypted = "USE_OTA_ENCRYPTION" in defines_present - own_key = encrypted and "USE_OTA_ENCRYPTION_FROM_API" not in defines_present + own_key = encrypted and "USE_OTA_ENCRYPTION_PROVISIONED" not in defines_present assert ("esphome_esphomeotacomponent_id->set_noise_psk(" in main_cpp) is own_key + # The api shares the ota's array instead of emitting the same key twice + assert main_cpp.count("_psk[] PROGMEM") == (1 if own_key else 0) assert ("set_auth_password(" in main_cpp) is ("USE_OTA_PASSWORD" in defines_present) # The noise transport source compiles only when the define is set assert FILTER_SOURCE_FILES() == ([] if encrypted else ["ota_esphome_noise.cpp"]) diff --git a/tests/integration/fixtures/host_ota_encrypted_safe_mode.yaml b/tests/integration/fixtures/host_ota_encrypted_safe_mode.yaml new file mode 100644 index 00000000000..57f3d57a010 --- /dev/null +++ b/tests/integration/fixtures/host_ota_encrypted_safe_mode.yaml @@ -0,0 +1,13 @@ +esphome: + name: host-ota-test +host: +api: + encryption: + key: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=" +ota: + - platform: esphome + port: __OTA_PORT__ + encryption: +safe_mode: +logger: + level: DEBUG diff --git a/tests/integration/fixtures/host_ota_provisioned_api_key_safe_mode.yaml b/tests/integration/fixtures/host_ota_provisioned_api_key_safe_mode.yaml new file mode 100644 index 00000000000..c00954515b1 --- /dev/null +++ b/tests/integration/fixtures/host_ota_provisioned_api_key_safe_mode.yaml @@ -0,0 +1,11 @@ +esphome: + name: host-ota-test +host: +api: + encryption: +ota: + - platform: esphome + port: __OTA_PORT__ +safe_mode: +logger: + level: DEBUG diff --git a/tests/integration/host_prefs.py b/tests/integration/host_prefs.py index c7f21d8a010..75c3ff81b01 100644 --- a/tests/integration/host_prefs.py +++ b/tests/integration/host_prefs.py @@ -14,6 +14,12 @@ from __future__ import annotations from pathlib import Path import struct +_ENTRY = struct.Struct(" Path: """Return the on-disk prefs file path for a host-platform device.""" @@ -34,16 +40,33 @@ def write_host_prefs(device_name: str, entries: dict[int, bytes]) -> Path: for key, data in entries.items(): if len(data) > 255: raise ValueError(f"Preference data too long: {len(data)} bytes (max 255)") - payload += struct.pack(" Path: - """Write a single preference entry, replacing the file's contents. +def read_host_prefs(device_name: str) -> dict[int, bytes]: + """Read the preference entries of a host-platform device; empty when + the file does not exist.""" + path = host_prefs_path(device_name) + if not path.exists(): + return {} + payload = path.read_bytes() + entries: dict[int, bytes] = {} + pos = 0 + while pos < len(payload): + key, length = _ENTRY.unpack_from(payload, pos) + pos += _ENTRY.size + entries[key] = payload[pos : pos + length] + pos += length + return entries - Returns the path that was written. - """ - return write_host_prefs(device_name, {key: data}) + +def force_safe_mode(device_name: str) -> None: + """Make the next boot of a host-platform device enter safe mode; other + saved preferences are kept.""" + entries = read_host_prefs(device_name) + entries[_SAFE_MODE_RTC_KEY] = struct.pack(" None: + """Provision PROVISIONING_PSK over the api and wait for it to activate.""" + async with api_client_connected(port=dev.api_port, noise_psk=ZERO_PSK) as client: + assert await client.noise_encryption_set_key(PROVISIONING_PSK) is True + await asyncio.sleep(KEY_ACTIVATION_DELAY) + + @pytest.mark.asyncio async def test_host_ota_self_update( yaml_config: str, @@ -227,6 +237,31 @@ async def test_host_ota_encrypted( await dev.ota(None, API_KEY, "encrypted OTA reported failure") +@pytest.mark.asyncio +async def test_host_ota_encrypted_safe_mode( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], +) -> None: + """Safe mode never constructs the api server, so an encrypted OTA with the + api key has to run on the ota component's own copy of that key.""" + pytest.importorskip("aioesphomeapi.noise") + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + # The api port never opens in safe mode, so wait for the log line instead + force_safe_mode(DEVICE_NAME) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_line(lines, "SAFE MODE IS ACTIVE", PORT_WAIT_TIMEOUT) + await _wait_for_port(LOCALHOST, dev.ota_port, PORT_WAIT_TIMEOUT) + # The safe mode boot clears the counter, so the re-exec boots normally + await dev.ota(None, API_KEY, "encrypted OTA in safe mode reported failure") + + @pytest.mark.asyncio async def test_host_ota_api_key_offer_with_password( yaml_config: str, @@ -305,11 +340,7 @@ async def test_host_ota_provisioned_api_key( None, None, "plaintext upload to an unprovisioned device must succeed" ) - async with api_client_connected( - port=dev.api_port, noise_psk=ZERO_PSK - ) as client: - assert await client.noise_encryption_set_key(PROVISIONING_PSK) is True - await asyncio.sleep(KEY_ACTIVATION_DELAY) + await _provision_key(dev, api_client_connected) key = PROVISIONING_PSK.decode() await dev.ota( @@ -319,6 +350,45 @@ async def test_host_ota_provisioned_api_key( await dev.ota(None, None, "plaintext must stay accepted on an offering device") +@pytest.mark.asyncio +async def test_host_ota_provisioned_api_key_safe_mode( + yaml_config: str, + write_yaml_config: ConfigWriter, + compile_esphome: CompileFunction, + reserved_tcp_port: tuple[int, socket.socket], + api_client_connected: APIClientConnectedFactory, +) -> None: + """Safe mode never constructs the api server, so the OTA has to load the + provisioned key from preferences itself to keep encrypting there.""" + pytest.importorskip("aioesphomeapi.noise") + dev = _Device( + *await _build( + yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port + ) + ) + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, _lines): + dev.proc = proc + await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT) + await _provision_key(dev, api_client_connected) + + # The saved key is already on disk; a host reboot outside an OTA just + # exits, so safe mode takes a second start + force_safe_mode(DEVICE_NAME) + key = PROVISIONING_PSK.decode() + async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines): + dev.proc = proc + await _wait_for_line(lines, "SAFE MODE IS ACTIVE", PORT_WAIT_TIMEOUT) + await _wait_for_port(LOCALHOST, dev.ota_port, PORT_WAIT_TIMEOUT) + await dev.ota( + None, + key, + "encrypted upload with the provisioned key must succeed in safe mode", + ) + # The re-exec boots normally and the api reads the same record + async with api_client_connected(port=dev.api_port, noise_psk=key): + pass + + @pytest.mark.asyncio async def test_host_ota_rejects_garbage( yaml_config: str, diff --git a/tests/integration/test_safe_mode_loop_runs.py b/tests/integration/test_safe_mode_loop_runs.py index 652877fc092..55756fca13d 100644 --- a/tests/integration/test_safe_mode_loop_runs.py +++ b/tests/integration/test_safe_mode_loop_runs.py @@ -25,19 +25,13 @@ from __future__ import annotations import asyncio import re -import struct import pytest from .conftest import run_binary -from .host_prefs import clear_host_prefs, write_host_pref +from .host_prefs import clear_host_prefs, force_safe_mode from .types import CompileFunction, ConfigWriter -# Must match esphome::safe_mode::RTC_KEY in safe_mode.h -SAFE_MODE_RTC_KEY = 233825507 -# Must match esphome::safe_mode::SafeModeComponent::ENTER_SAFE_MODE_MAGIC -ENTER_SAFE_MODE_MAGIC = 0x5AFE5AFE - DEVICE_NAME = "safe-mode-loop-runs" THREAD_LOG_MARKER = "looping component ran in safe mode" @@ -56,9 +50,7 @@ async def test_safe_mode_loop_runs( # Compile finished successfully; pre-populate prefs so the *next* run # enters safe mode immediately. - write_host_pref( - DEVICE_NAME, SAFE_MODE_RTC_KEY, struct.pack(" Date: Sat, 19 Sep 2026 18:08:56 -0400 Subject: [PATCH 021/133] [esp32] Disable newlib nano printf formatting on ESP32-C2 (#19418) Co-authored-by: Claude Fable 5.1 --- esphome/components/esp32/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 4597904cc39..e9a041b0b1b 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -2628,6 +2628,11 @@ async def to_code(config): config.get(CONF_ENGINEERING_SAMPLE, False), ) + # ESP32-C2 defaults to the ROM's newlib "nano" printf, which does not + # understand %zu or %lld and crashes on any %s that follows one. + if variant == VARIANT_ESP32C2: + add_idf_sdkconfig_option("CONFIG_LIBC_NEWLIB_NANO_FORMAT", False) + # Set minimum chip revision for ESP32 variant # Setting this to 3.0 or higher reduces flash size by excluding workaround code, # and for PSRAM users saves significant IRAM by keeping C library functions in ROM. From 92059abf6cc40bbfa1b7865b118c4a60b91092f1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 24 Sep 2026 23:56:28 +0100 Subject: [PATCH 022/133] [web_server] Mask the value of a password text entity, not only its state (#19385) --- esphome/components/web_server/web_server.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/esphome/components/web_server/web_server.cpp b/esphome/components/web_server/web_server.cpp index ec536910e54..58fecd04d8e 100644 --- a/esphome/components/web_server/web_server.cpp +++ b/esphome/components/web_server/web_server.cpp @@ -1412,8 +1412,11 @@ json::SerializationBuffer<> WebServer::text_json_(text::Text *obj, const std::st json::JsonBuilder builder; JsonObject root = builder.root(); - const char *state = obj->traits.get_mode() == text::TextMode::TEXT_MODE_PASSWORD ? "********" : value.c_str(); - set_json_icon_state_value(root, obj, "text", state, value.c_str(), start_config); + // A password entity shows the mask and prefills the input with nothing, so the secret never + // reaches the JSON and the mask cannot be written back as the value + const bool password = obj->traits.get_mode() == text::TextMode::TEXT_MODE_PASSWORD; + set_json_icon_state_value(root, obj, "text", password ? "********" : value.c_str(), password ? "" : value.c_str(), + start_config); root[ESPHOME_F("min_length")] = obj->traits.get_min_length(); root[ESPHOME_F("max_length")] = obj->traits.get_max_length(); root[ESPHOME_F("pattern")] = obj->traits.get_pattern_c_str(); From 4af2db7e8a597b4df118df44b4736a8e56ec1f51 Mon Sep 17 00:00:00 2001 From: tomaszduda23 Date: Fri, 25 Sep 2026 05:00:33 +0100 Subject: [PATCH 023/133] [nrf52] set WDT timeout to 30sec for Adafruit bootloader (#19596) Co-authored-by: J. Nick Koston --- esphome/components/nrf52/dfu.cpp | 1 + esphome/components/zephyr/hal.cpp | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/esphome/components/nrf52/dfu.cpp b/esphome/components/nrf52/dfu.cpp index 24dee997269..0122ed9b31b 100644 --- a/esphome/components/nrf52/dfu.cpp +++ b/esphome/components/nrf52/dfu.cpp @@ -28,6 +28,7 @@ void DeviceFirmwareUpdate::setup() { this->reset_pin_->digital_write(true); } else { NRF_POWER->GPREGRET = DFU_MAGIC_UF2_RESET; + arch_feed_wdt(); App.reboot(); } } diff --git a/esphome/components/zephyr/hal.cpp b/esphome/components/zephyr/hal.cpp index ad8ed5c95c3..10e8340a409 100644 --- a/esphome/components/zephyr/hal.cpp +++ b/esphome/components/zephyr/hal.cpp @@ -27,7 +27,11 @@ void arch_init() { if (device_is_ready(WDT)) { static wdt_timeout_cfg wdt_config{}; wdt_config.flags = WDT_FLAG_RESET_SOC; -#ifdef USE_ZIGBEE +#ifndef USE_BOOTLOADER_MCUBOOT + // Adafruit bootloader doesn't feed the WDT while + // erasing flash during a firmware update, so a shorter timeout can break the update. + wdt_config.window.max = 30000; +#elif defined(USE_ZIGBEE) // zboss thread uses a lot of CPU cycles during startup wdt_config.window.max = 10000; #else From b6d70d4e349561b787d1b3e664c7c16d2952637d Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Mon, 28 Sep 2026 00:04:17 +1000 Subject: [PATCH 024/133] [mipi_spi] Fix reversion in dc/cs sequence for spi_16 (#19745) --- esphome/components/mipi_spi/mipi_spi.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/esphome/components/mipi_spi/mipi_spi.h b/esphome/components/mipi_spi/mipi_spi.h index 2552451bd7c..ee1264cf234 100644 --- a/esphome/components/mipi_spi/mipi_spi.h +++ b/esphome/components/mipi_spi/mipi_spi.h @@ -246,20 +246,18 @@ class MipiSpi : public display::Display, this->write_cmd_addr_data(8, 0x02, 24, cmd << 8, bytes, len); this->disable(); } else if constexpr (BUS_TYPE == BUS_TYPE_OCTAL) { - // Toggle D/C only while holding the bus; on boards where D/C doubles as - // another bus signal, driving it while another device owns the bus - // corrupts that device's transfer. this->enable(); this->dc_pin_->digital_write(false); this->write_cmd_addr_data(0, 0, 0, 0, &cmd, 1, 8); this->dc_pin_->digital_write(true); - this->disable(); + // hold the bus between command and data to avoid a glitch on the D/C line if (len != 0) { - this->enable(); this->write_cmd_addr_data(0, 0, 0, 0, bytes, len, 8); - this->disable(); } + this->disable(); } else if constexpr (BUS_TYPE == BUS_TYPE_SINGLE) { + // Toggle D/C only while holding the bus; works around a quirk in the CoreS3 and W5500 ethernet combination. + // See https://github.com/esphome/esphome/pull/18529 this->enable(); this->dc_pin_->digital_write(false); this->write_byte(cmd); @@ -271,12 +269,14 @@ class MipiSpi : public display::Display, this->disable(); } } else if constexpr (BUS_TYPE == BUS_TYPE_SINGLE_16) { - this->enable(); + // DC must be stable before CS as the clock is gated by CS this->dc_pin_->digital_write(false); + this->enable(); this->write_byte(cmd); - this->dc_pin_->digital_write(true); this->disable(); + this->dc_pin_->digital_write(true); for (size_t i = 0; i != len; i++) { + // must enable and disable for each byte based on empirical testing this->enable(); this->write_byte(0); this->write_byte(bytes[i]); From 43b6b41d16fb0ddf818ae2c6cfd51e34de76f739 Mon Sep 17 00:00:00 2001 From: Suliman Abdulrazzaq <144490671+SulimanAbdulrazzaq@users.noreply.github.com> Date: Mon, 28 Sep 2026 02:12:34 +0300 Subject: [PATCH 025/133] [lvgl] Honor esphome build_flags when generating lv_conf.h (#19743) Co-authored-by: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> --- esphome/components/lvgl/__init__.py | 15 +++-- .../lvgl/test_lv_conf_build_flags.py | 59 +++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 tests/component_tests/lvgl/test_lv_conf_build_flags.py diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index 2d2f1d6288b..382e483d0cb 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -31,6 +31,7 @@ from esphome.components.psram import DOMAIN as PSRAM_DOMAIN import esphome.config_validation as cv from esphome.const import ( CONF_BUFFER_SIZE, + CONF_BUILD_FLAGS, CONF_ESPHOME, CONF_GROUP, CONF_ID, @@ -169,11 +170,17 @@ def generate_lv_conf_h(): all_defines = set( df.LV_DEFINES + tuple(f"LV_USE_{w.upper()}" for w in WIDGET_TYPES) ) - build_flags = ( - CORE.config[CONF_ESPHOME].get(CONF_PLATFORMIO_OPTIONS).get("build_flags", []) + esphome_config = CORE.config[CONF_ESPHOME] + # User build flags come from esphome->build_flags and from the deprecated + # esphome->platformio_options->build_flags (a string or a list). + # Remove before 2026.12.0 + + pio_build_flags = esphome_config.get(CONF_PLATFORMIO_OPTIONS, {}).get( + CONF_BUILD_FLAGS, [] ) - if not isinstance(build_flags, list): - build_flags = [build_flags] + if not isinstance(pio_build_flags, list): + pio_build_flags = [pio_build_flags] + build_flags = [*esphome_config.get(CONF_BUILD_FLAGS, []), *pio_build_flags] # Extract define names from build flags like '-DLV_USE_CHART=1', '-D LV_USE_CHART', # or multiple defines in one string. define_pattern = r'-D\s*([A-Z_][A-Z0-9_]*)(?:=[^\s\'"\]]*)?' diff --git a/tests/component_tests/lvgl/test_lv_conf_build_flags.py b/tests/component_tests/lvgl/test_lv_conf_build_flags.py new file mode 100644 index 00000000000..eb576fa8fac --- /dev/null +++ b/tests/component_tests/lvgl/test_lv_conf_build_flags.py @@ -0,0 +1,59 @@ +"""``generate_lv_conf_h()`` writes ``#define LV_... 0`` for every LVGL option +the configuration does not use, so ``lv_conf.h`` must leave alone the options +the user defines in build flags. Otherwise the header, which is included after +the compiler ``-D`` flags, turns the option off again. + +Build flags can come from ``esphome: build_flags:`` or from the deprecated +``esphome: platformio_options: build_flags:``; both must be honoured. +""" + +from __future__ import annotations + +import logging + +import pytest + +from esphome.components.lvgl import defines as df, generate_lv_conf_h +from esphome.const import CONF_BUILD_FLAGS, CONF_ESPHOME, CONF_PLATFORMIO_OPTIONS +from esphome.core import CORE + + +def _set_esphome_config( + build_flags: list[str] | None = None, + pio_build_flags: list[str] | str | None = None, +) -> None: + pio_options = {} if pio_build_flags is None else {"build_flags": pio_build_flags} + CORE.config = { + CONF_ESPHOME: { + CONF_PLATFORMIO_OPTIONS: pio_options, + CONF_BUILD_FLAGS: build_flags or [], + } + } + + +def test_unused_define_is_disabled_without_build_flag() -> None: + _set_esphome_config() + assert "#define LV_USE_OBSERVER 0" in generate_lv_conf_h().splitlines() + + +@pytest.mark.parametrize( + "flags", + [ + {"build_flags": ["-DLV_USE_OBSERVER=1"]}, + {"build_flags": ["-D LV_USE_OBSERVER"]}, + {"pio_build_flags": ["-DLV_USE_OBSERVER=1"]}, + {"pio_build_flags": "-DLV_USE_OBSERVER=1"}, + ], + ids=["esphome", "esphome-spaced", "platformio_options", "platformio_options-str"], +) +def test_build_flag_define_is_not_disabled(flags: dict) -> None: + _set_esphome_config(**flags) + assert "#define LV_USE_OBSERVER 0" not in generate_lv_conf_h().splitlines() + + +def test_esphome_build_flag_clash_warns(caplog: pytest.LogCaptureFixture) -> None: + _set_esphome_config(build_flags=["-DLV_USE_ARC=1"]) + df.add_define("LV_USE_ARC") + with caplog.at_level(logging.WARNING): + generate_lv_conf_h() + assert "LV_USE_ARC" in caplog.text From eca4c2d0a5de6f1c432f7b0e343339b2b96e4913 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Mon, 28 Sep 2026 23:30:51 +0200 Subject: [PATCH 026/133] [ethernet] Move received frames to PSRAM on SPI ethernet chips (#19377) --- .../ethernet/ethernet_component_esp32.cpp | 46 +++++++++++++++++++ .../test-w5500-wifi-psram.esp32-s3-idf.yaml | 15 ++++++ 2 files changed, 61 insertions(+) create mode 100644 tests/components/ethernet/test-w5500-wifi-psram.esp32-s3-idf.yaml diff --git a/esphome/components/ethernet/ethernet_component_esp32.cpp b/esphome/components/ethernet/ethernet_component_esp32.cpp index 1d9903271ec..ea732e84ff9 100644 --- a/esphome/components/ethernet/ethernet_component_esp32.cpp +++ b/esphome/components/ethernet/ethernet_component_esp32.cpp @@ -10,6 +10,9 @@ #include #include #include "esp_event.h" +#ifdef USE_PSRAM +#include +#endif // IDF 6.0 moved per-chip PHY/MAC drivers to the Espressif Component Registry; // they are no longer included via esp_eth.h and need explicit includes. @@ -71,6 +74,32 @@ static const char *const TAG = "ethernet"; // PHY register size for hex logging static constexpr size_t PHY_REG_SIZE = 2; +// Dual wifi + ethernet SPI builds: the one place internal RAM is short and lwip's other buffers are +// already in PSRAM. Not with L2 TAP, whose filter lives in the glue's input path this replaces. +#if defined(USE_PSRAM) && defined(USE_ETHERNET_SPI) && defined(USE_WIFI) && !defined(CONFIG_ESP_NETIF_L2_TAP) +#define USE_ETHERNET_RX_PSRAM +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - reported by dump_config() +static bool rx_psram_installed = false; + +// ESP-IDF ethernet drivers malloc() every received frame in internal RAM, where it stays until lwIP +// hands it to the application. Move it to PSRAM; if that fails the frame is passed on where it is. +static esp_err_t eth_input_to_psram(esp_eth_handle_t handle, uint8_t *buffer, uint32_t length, void *priv) { + auto *copy = static_cast(heap_caps_malloc(length, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); + if (copy != nullptr) { + memcpy(copy, buffer, length); + free(buffer); // NOLINT(cppcoreguidelines-no-malloc) - allocated by the driver with malloc() + buffer = copy; + } else { + static bool warned = false; // once, this runs per frame in the driver's task + if (!warned) { + warned = true; + ESP_LOGW(TAG, "PSRAM allocation failed, frame kept in internal RAM (reported once)"); + } + } + return esp_netif_receive(static_cast(priv), buffer, length, nullptr); +} +#endif + void EthernetComponent::log_error_and_mark_failed_(esp_err_t err, const char *message) { ESP_LOGE(TAG, "%s: (%d) %s", message, err, esp_err_to_name(err)); this->mark_failed(); @@ -453,6 +482,16 @@ void EthernetComponent::ethernet_lazy_init_() { /* attach Ethernet driver to TCP/IP stack */ err = esp_netif_attach(this->eth_netif_, esp_eth_new_netif_glue(this->eth_handle_)); ESPHL_ERROR_CHECK(err, "ETH netif attach error"); +#ifdef USE_ETHERNET_RX_PSRAM + // The glue frees every receive buffer with free(), so the replacement buffer must come from the heap + if (esp_psram_is_initialized()) { + err = esp_eth_update_input_path(this->eth_handle_, eth_input_to_psram, this->eth_netif_); + rx_psram_installed = err == ESP_OK; + if (!rx_psram_installed) { + ESP_LOGW(TAG, "PSRAM RX path not installed: %s", esp_err_to_name(err)); + } + } +#endif // Register user defined event handers err = esp_event_handler_register(ETH_EVENT, ESP_EVENT_ANY_ID, &EthernetComponent::eth_event_handler, nullptr); @@ -630,6 +669,13 @@ void EthernetComponent::dump_config() { this->clk_pin_, this->mdc_pin_, this->mdio_pin_, this->phy_addr_); #endif ESP_LOGCONFIG(TAG, " Type: %s", eth_type); +#ifdef USE_ETHERNET_RX_PSRAM + // Only known once the driver is up; with enable_on_boot: false that is after this dump + if (this->ethernet_initialized_) { + ESP_LOGCONFIG(TAG, " RX frames: %s", + rx_psram_installed ? LOG_STR_LITERAL("PSRAM") : LOG_STR_LITERAL("internal RAM")); + } +#endif } network::IPAddresses EthernetComponent::get_ip_addresses() { diff --git a/tests/components/ethernet/test-w5500-wifi-psram.esp32-s3-idf.yaml b/tests/components/ethernet/test-w5500-wifi-psram.esp32-s3-idf.yaml new file mode 100644 index 00000000000..d454dbadc39 --- /dev/null +++ b/tests/components/ethernet/test-w5500-wifi-psram.esp32-s3-idf.yaml @@ -0,0 +1,15 @@ +# W5500 next to wifi with PSRAM: compiles the input path that moves received frames to PSRAM +packages: + ethernet: !include common-w5500.yaml + +psram: + mode: quad + +wifi: + ssid: MySSID + password: password1 + +network: + priority: + - ethernet + - wifi From 7a4ffa98f67c6633e9cc119bb423ceb6e842fa88 Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:57:36 +1300 Subject: [PATCH 027/133] Bump version to 2026.9.1 --- Doxyfile | 2 +- esphome/const.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doxyfile b/Doxyfile index a5d6da63332..65d975b61ea 100644 --- a/Doxyfile +++ b/Doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = ESPHome # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2026.9.0 +PROJECT_NUMBER = 2026.9.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/esphome/const.py b/esphome/const.py index e6165bf16b0..0ec6a95869e 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -4,7 +4,7 @@ from enum import Enum from esphome.enum import StrEnum -__version__ = "2026.9.0" +__version__ = "2026.9.1" ALLOWED_NAME_CHARS = "abcdefghijklmnopqrstuvwxyz0123456789-_" VALID_SUBSTITUTIONS_CHARACTERS = ( From aba15f8ff6df8baf60506398369db83d6c1c56c9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 01:57:37 +0200 Subject: [PATCH 028/133] [thermopro_ble] Use the sub_* helpers for optional child entities (#19868) --- esphome/components/thermopro_ble/sensor.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/esphome/components/thermopro_ble/sensor.py b/esphome/components/thermopro_ble/sensor.py index c660e5fcef8..ed42312a26b 100644 --- a/esphome/components/thermopro_ble/sensor.py +++ b/esphome/components/thermopro_ble/sensor.py @@ -82,18 +82,9 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature(sens)) - if external_temperature_config := config.get(CONF_EXTERNAL_TEMPERATURE): - sens = await sensor.new_sensor(external_temperature_config) - cg.add(var.set_external_temperature(sens)) - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity(sens)) - if battery_level_config := config.get(CONF_BATTERY_LEVEL): - sens = await sensor.new_sensor(battery_level_config) - cg.add(var.set_battery_level(sens)) - if signal_strength_config := config.get(CONF_SIGNAL_STRENGTH): - sens = await sensor.new_sensor(signal_strength_config) - cg.add(var.set_signal_strength(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_EXTERNAL_TEMPERATURE, var.set_external_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) + await sensors(CONF_SIGNAL_STRENGTH, var.set_signal_strength) From 8032542076c6fda21112f53365a95ffb48f5a786 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 01:57:51 +0200 Subject: [PATCH 029/133] [as5600] Use the sub_* helpers for optional child entities (#19869) --- esphome/components/as5600/sensor/__init__.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/esphome/components/as5600/sensor/__init__.py b/esphome/components/as5600/sensor/__init__.py index 847b89f1214..4cf1e95f695 100644 --- a/esphome/components/as5600/sensor/__init__.py +++ b/esphome/components/as5600/sensor/__init__.py @@ -87,18 +87,8 @@ async def to_code(config: ConfigType) -> None: if out_of_range_mode_config := config.get(CONF_OUT_OF_RANGE_MODE): cg.add(var.set_out_of_range_mode(out_of_range_mode_config)) - if raw_position_config := config.get(CONF_RAW_POSITION): - sens = await sensor.new_sensor(raw_position_config) - cg.add(var.set_raw_position_sensor(sens)) - - if gain_config := config.get(CONF_GAIN): - sens = await sensor.new_sensor(gain_config) - cg.add(var.set_gain_sensor(sens)) - - if magnitude_config := config.get(CONF_MAGNITUDE): - sens = await sensor.new_sensor(magnitude_config) - cg.add(var.set_magnitude_sensor(sens)) - - if status_config := config.get(CONF_STATUS): - sens = await sensor.new_sensor(status_config) - cg.add(var.set_status_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_RAW_POSITION, var.set_raw_position_sensor) + await sensors(CONF_GAIN, var.set_gain_sensor) + await sensors(CONF_MAGNITUDE, var.set_magnitude_sensor) + await sensors(CONF_STATUS, var.set_status_sensor) From 571cbb72ebb07b2f982c3c2a8302029bd05deb19 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 01:58:05 +0200 Subject: [PATCH 030/133] [xiaomi_lywsd03mmc] Use the sub_* helpers for optional child entities (#19870) --- esphome/components/xiaomi_lywsd03mmc/sensor.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/xiaomi_lywsd03mmc/sensor.py b/esphome/components/xiaomi_lywsd03mmc/sensor.py index f4e24a97d0b..32977d0ed6f 100644 --- a/esphome/components/xiaomi_lywsd03mmc/sensor.py +++ b/esphome/components/xiaomi_lywsd03mmc/sensor.py @@ -68,12 +68,7 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) cg.add(var.set_bindkey(config[CONF_BINDKEY])) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From bf7aa1dc605e3aa769c51822806b31b9bc60a533 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 01:58:53 +0200 Subject: [PATCH 031/133] [sm300d2] Use the sub_* helpers for optional child entities (#19866) --- esphome/components/sm300d2/sensor.py | 29 ++++++++-------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/esphome/components/sm300d2/sensor.py b/esphome/components/sm300d2/sensor.py index e0f2b7b280e..4245283ab48 100644 --- a/esphome/components/sm300d2/sensor.py +++ b/esphome/components/sm300d2/sensor.py @@ -99,24 +99,11 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await uart.register_uart_device(var, config) - if CONF_CO2 in config: - sens = await sensor.new_sensor(config[CONF_CO2]) - cg.add(var.set_co2_sensor(sens)) - if CONF_FORMALDEHYDE in config: - sens = await sensor.new_sensor(config[CONF_FORMALDEHYDE]) - cg.add(var.set_formaldehyde_sensor(sens)) - if CONF_TVOC in config: - sens = await sensor.new_sensor(config[CONF_TVOC]) - cg.add(var.set_tvoc_sensor(sens)) - if CONF_PM_2_5 in config: - sens = await sensor.new_sensor(config[CONF_PM_2_5]) - cg.add(var.set_pm_2_5_sensor(sens)) - if CONF_PM_10_0 in config: - sens = await sensor.new_sensor(config[CONF_PM_10_0]) - cg.add(var.set_pm_10_0_sensor(sens)) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_CO2, var.set_co2_sensor) + await sensors(CONF_FORMALDEHYDE, var.set_formaldehyde_sensor) + await sensors(CONF_TVOC, var.set_tvoc_sensor) + await sensors(CONF_PM_2_5, var.set_pm_2_5_sensor) + await sensors(CONF_PM_10_0, var.set_pm_10_0_sensor) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) From 21c96049189ed325de677b9916bdc881b8efc42b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 08:56:17 +0200 Subject: [PATCH 032/133] [network] Keep lwip off the PSRAM tier when Ethernet is configured (#19367) --- esphome/components/network/__init__.py | 35 ++++++++++-- esphome/components/wifi/__init__.py | 2 +- tests/component_tests/network/__init__.py | 11 ++++ .../config/high_perf_ethernet_no_psram.yaml | 20 +++++++ .../config/high_perf_ethernet_psram.yaml | 23 ++++++++ .../config/high_perf_wifi_ethernet_psram.yaml | 30 +++++++++++ .../config/high_perf_wifi_no_psram.yaml | 14 +++++ .../network/config/high_perf_wifi_psram.yaml | 18 +++++++ .../network/test_high_performance.py | 54 +++++++++++++++++++ .../component_tests/network/test_priority.py | 9 +--- .../network/test_tcp_send_buffer.py | 18 ++----- .../network/test-priority.esp32-idf.yaml | 6 +++ 12 files changed, 215 insertions(+), 25 deletions(-) create mode 100644 tests/component_tests/network/config/high_perf_ethernet_no_psram.yaml create mode 100644 tests/component_tests/network/config/high_perf_ethernet_psram.yaml create mode 100644 tests/component_tests/network/config/high_perf_wifi_ethernet_psram.yaml create mode 100644 tests/component_tests/network/config/high_perf_wifi_no_psram.yaml create mode 100644 tests/component_tests/network/config/high_perf_wifi_psram.yaml create mode 100644 tests/component_tests/network/test_high_performance.py diff --git a/esphome/components/network/__init__.py b/esphome/components/network/__init__.py index 9a47679ba3c..decee27bcf8 100644 --- a/esphome/components/network/__init__.py +++ b/esphome/components/network/__init__.py @@ -9,9 +9,11 @@ from esphome.components.zephyr import zephyr_add_prj_conf import esphome.config_validation as cv from esphome.const import ( CONF_ENABLE_IPV6, + CONF_ETHERNET, CONF_ID, CONF_MIN_IPV6_ADDR_COUNT, CONF_PRIORITY, + CONF_WIFI, ) from esphome.core import CORE, CoroPriority, coroutine_with_priority import esphome.final_validate as fv @@ -28,6 +30,16 @@ KEY_HIGH_PERFORMANCE_NETWORKING = "high_performance_networking" CONF_ENABLE_HIGH_PERFORMANCE = "enable_high_performance" CONF_TCP_SEND_BUFFER = "tcp_send_buffer" +# TCP receive window of the optimized lwip tier (PSRAM not guaranteed) +TCP_WND_OPTIMIZED = 65534 +# Ethernet drivers keep every queued frame in internal RAM, so ethernet-only builds +# get a window sized for a LAN round trip and the stock lwip input mailbox. Dual +# wifi + ethernet builds keep wifi's sizes; the ethernet component moves their +# received frames to PSRAM instead. +TCP_WND_ETHERNET = 16384 +TCPIP_RECVMBOX_ETHERNET = 32 +TCPIP_RECVMBOX_OPTIMIZED = 64 + # lwIP queues at most this many unsent/unacked bytes per TCP socket; the # stock ESP-IDF default (5744 bytes) stalls bursty senders like a Bluetooth # proxy streaming GATT notifications. Bounds follow the lwIP guidance for the @@ -166,6 +178,8 @@ def require_high_performance_networking() -> None: Configuration is PSRAM-aware: - With PSRAM guaranteed: Aggressive settings (512 RX buffers, 512KB TCP windows) - Without PSRAM: Conservative optimized settings (64 buffers, 65KB TCP windows) + - Ethernet only: 16KB TCP windows regardless of PSRAM, because ESP-IDF + ethernet drivers keep received frames in internal RAM Example: from esphome.components import network @@ -417,8 +431,11 @@ async def to_code(config: ConfigType) -> None: if CORE.is_esp32 and should_enable: # Check if PSRAM is guaranteed (set by psram component during final validation) psram_guaranteed = psram_is_guaranteed() + # ESP-IDF ethernet drivers malloc() received frames into internal RAM, so + # lwip is never sized for PSRAM on an ethernet-only build. + ethernet_only = CONF_ETHERNET in CORE.config and CONF_WIFI not in CORE.config - if psram_guaranteed: + if psram_guaranteed and not ethernet_only: _LOGGER.info( "Applying high-performance lwip settings (PSRAM guaranteed): 512KB TCP windows, 512 mailbox sizes" ) @@ -451,15 +468,25 @@ async def to_code(config: ConfigType) -> None: add_idf_sdkconfig_option("CONFIG_LWIP_TCP_OVERSIZE_MSS", True) add_idf_sdkconfig_option("CONFIG_LWIP_TCP_QUEUE_OOSEQ", True) else: + # Every queued byte is internal RAM on ethernet and a LAN round trip needs + # little window; wifi keeps the larger sizes. + if ethernet_only: + tcp_window, tcpip_mailbox = TCP_WND_ETHERNET, TCPIP_RECVMBOX_ETHERNET + else: + tcp_window, tcpip_mailbox = TCP_WND_OPTIMIZED, TCPIP_RECVMBOX_OPTIMIZED _LOGGER.info( - "Applying optimized lwip settings: 65KB TCP windows, 64 mailbox sizes" + "Applying optimized lwip settings: %dKB TCP windows, %d entry input mailbox", + tcp_window // 1000, + tcpip_mailbox, ) # PSRAM not guaranteed - use more conservative, but still optimized settings # Based on https://github.com/espressif/esp-idf/blob/release/v5.4/examples/wifi/iperf/sdkconfig.defaults.esp32 + # The send buffer stays at 65534 on ethernet too: it only fills under outbound + # load, and bursty senders such as a Bluetooth proxy stall on the IDF default. add_idf_sdkconfig_option("CONFIG_LWIP_TCP_SND_BUF_DEFAULT", 65534) - add_idf_sdkconfig_option("CONFIG_LWIP_TCP_WND_DEFAULT", 65534) + add_idf_sdkconfig_option("CONFIG_LWIP_TCP_WND_DEFAULT", tcp_window) add_idf_sdkconfig_option("CONFIG_LWIP_TCP_RECVMBOX_SIZE", 64) - add_idf_sdkconfig_option("CONFIG_LWIP_TCPIP_RECVMBOX_SIZE", 64) + add_idf_sdkconfig_option("CONFIG_LWIP_TCPIP_RECVMBOX_SIZE", tcpip_mailbox) # After the high performance block so an explicit size wins over the # bundle's 65534 (last write wins in the sdkconfig store). diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index c9616a6c290..1ba89468413 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -771,7 +771,7 @@ async def to_code(config): "Applying high-performance WiFi settings (PSRAM guaranteed): 512 RX buffers, 32 TX buffers" ) # PSRAM is guaranteed - use aggressive settings - # Higher maximum values are allowed because CONFIG_LWIP_WND_SCALE is set to true in networking component + # Higher maximum values are allowed because CONFIG_LWIP_WND_SCALE may be set to true in networking component # Based on https://github.com/espressif/esp-adf/issues/297#issuecomment-783811702 # Large dynamic RX buffers (requires PSRAM) diff --git a/tests/component_tests/network/__init__.py b/tests/component_tests/network/__init__.py index e69de29bb2d..b4ed442ff8d 100644 --- a/tests/component_tests/network/__init__.py +++ b/tests/component_tests/network/__init__.py @@ -0,0 +1,11 @@ +"""Shared helpers for the network component tests.""" + +from esphome.components.esp32.const import KEY_ESP32, KEY_SDKCONFIG_OPTIONS +from esphome.core import CORE + + +def sdkconfig_option(name: str) -> int | bool | None: + """Return a generated sdkconfig value, or None when unset or not ESP32.""" + if KEY_ESP32 not in CORE.data: + return None + return CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS].get(name) diff --git a/tests/component_tests/network/config/high_perf_ethernet_no_psram.yaml b/tests/component_tests/network/config/high_perf_ethernet_no_psram.yaml new file mode 100644 index 00000000000..31858c3fc5e --- /dev/null +++ b/tests/component_tests/network/config/high_perf_ethernet_no_psram.yaml @@ -0,0 +1,20 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +ethernet: + type: LAN8720 + mdc_pin: 23 + mdio_pin: 32 + clk: + pin: 0 + mode: CLK_EXT_IN + phy_addr: 0 + power_pin: 33 + +network: + enable_high_performance: true diff --git a/tests/component_tests/network/config/high_perf_ethernet_psram.yaml b/tests/component_tests/network/config/high_perf_ethernet_psram.yaml new file mode 100644 index 00000000000..3407fd5ffe8 --- /dev/null +++ b/tests/component_tests/network/config/high_perf_ethernet_psram.yaml @@ -0,0 +1,23 @@ +esphome: + name: test + +esp32: + variant: esp32s3 + framework: + type: esp-idf + +psram: + mode: octal + ignore_not_found: false + +ethernet: + type: W5500 + clk_pin: GPIO12 + mosi_pin: GPIO11 + miso_pin: GPIO13 + cs_pin: GPIO10 + interrupt_pin: GPIO14 + reset_pin: GPIO9 + +network: + enable_high_performance: true diff --git a/tests/component_tests/network/config/high_perf_wifi_ethernet_psram.yaml b/tests/component_tests/network/config/high_perf_wifi_ethernet_psram.yaml new file mode 100644 index 00000000000..b9b49c825db --- /dev/null +++ b/tests/component_tests/network/config/high_perf_wifi_ethernet_psram.yaml @@ -0,0 +1,30 @@ +esphome: + name: test + +esp32: + variant: esp32s3 + framework: + type: esp-idf + +psram: + mode: octal + ignore_not_found: false + +wifi: + ssid: "test_ssid" + password: "test_password" + +ethernet: + type: W5500 + clk_pin: GPIO12 + mosi_pin: GPIO11 + miso_pin: GPIO13 + cs_pin: GPIO10 + interrupt_pin: GPIO14 + reset_pin: GPIO9 + +network: + priority: + - wifi + - ethernet + enable_high_performance: true diff --git a/tests/component_tests/network/config/high_perf_wifi_no_psram.yaml b/tests/component_tests/network/config/high_perf_wifi_no_psram.yaml new file mode 100644 index 00000000000..c24e4b62539 --- /dev/null +++ b/tests/component_tests/network/config/high_perf_wifi_no_psram.yaml @@ -0,0 +1,14 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + type: esp-idf + +wifi: + ssid: "test_ssid" + password: "test_password" + +network: + enable_high_performance: true diff --git a/tests/component_tests/network/config/high_perf_wifi_psram.yaml b/tests/component_tests/network/config/high_perf_wifi_psram.yaml new file mode 100644 index 00000000000..efdad282a44 --- /dev/null +++ b/tests/component_tests/network/config/high_perf_wifi_psram.yaml @@ -0,0 +1,18 @@ +esphome: + name: test + +esp32: + variant: esp32s3 + framework: + type: esp-idf + +psram: + mode: octal + ignore_not_found: false + +wifi: + ssid: "test_ssid" + password: "test_password" + +network: + enable_high_performance: true diff --git a/tests/component_tests/network/test_high_performance.py b/tests/component_tests/network/test_high_performance.py new file mode 100644 index 00000000000..c291806ac4b --- /dev/null +++ b/tests/component_tests/network/test_high_performance.py @@ -0,0 +1,54 @@ +"""Tests for the ``network: enable_high_performance:`` lwip tier selection. + +Ethernet drivers keep received frames in internal RAM, so an ethernet-only +build must keep lwip off the PSRAM tier even when PSRAM is guaranteed. +""" + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome.components.network import require_high_performance_networking +from tests.component_tests.network import sdkconfig_option + + +@pytest.mark.parametrize( + ( + "fixture", + "window", + "tcp_mailbox", + "tcpip_mailbox", + "window_scale", + "wifi_rx_buffers", + ), + [ + ("high_perf_wifi_psram.yaml", 512000, 512, 512, True, 512), + ("high_perf_wifi_no_psram.yaml", 65534, 64, 64, None, 64), + # dual builds keep wifi's sizes; the ethernet component moves frames to PSRAM + ("high_perf_wifi_ethernet_psram.yaml", 512000, 512, 512, True, 512), + ("high_perf_ethernet_no_psram.yaml", 16384, 64, 32, None, None), + # ethernet only with PSRAM guaranteed: the cell the guard exists for + ("high_perf_ethernet_psram.yaml", 16384, 64, 32, None, None), + ], +) +def test_lwip_tier( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + fixture: str, + window: int, + tcp_mailbox: int, + tcpip_mailbox: int, + window_scale: bool | None, + wifi_rx_buffers: int | None, +) -> None: + # The wifi component only reacts to a component request, so request it the + # way sendspin does; the config key alone decides the lwip tier. + require_high_performance_networking() + generate_main(component_config_path(fixture)) + assert sdkconfig_option("CONFIG_LWIP_TCP_WND_DEFAULT") == window + assert sdkconfig_option("CONFIG_LWIP_TCP_RECVMBOX_SIZE") == tcp_mailbox + assert sdkconfig_option("CONFIG_LWIP_TCPIP_RECVMBOX_SIZE") == tcpip_mailbox + assert sdkconfig_option("CONFIG_LWIP_WND_SCALE") is window_scale + # Wifi RX buffers really go to PSRAM, so the wifi tier is never downgraded + assert sdkconfig_option("CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM") == wifi_rx_buffers diff --git a/tests/component_tests/network/test_priority.py b/tests/component_tests/network/test_priority.py index 041b358ddaf..57e412ea198 100644 --- a/tests/component_tests/network/test_priority.py +++ b/tests/component_tests/network/test_priority.py @@ -19,6 +19,7 @@ from esphome.components.network import ( from esphome.const import CONF_PRIORITY, PlatformFramework from esphome.core import CORE import esphome.final_validate as fv +from tests.component_tests.network import sdkconfig_option from tests.component_tests.types import SetCoreConfigCallable @@ -217,13 +218,7 @@ def test_no_primary_interface_define_without_priority( def _dns_per_default_netif_option() -> bool | None: - from esphome.components.esp32.const import KEY_ESP32, KEY_SDKCONFIG_OPTIONS - - if KEY_ESP32 not in CORE.data: # non-ESP32 configs have no sdkconfig at all - return None - return CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS].get( - "CONFIG_ESP_NETIF_SET_DNS_PER_DEFAULT_NETIF" - ) + return sdkconfig_option("CONFIG_ESP_NETIF_SET_DNS_PER_DEFAULT_NETIF") @pytest.mark.parametrize( diff --git a/tests/component_tests/network/test_tcp_send_buffer.py b/tests/component_tests/network/test_tcp_send_buffer.py index 64989212439..bf68c7c11cd 100644 --- a/tests/component_tests/network/test_tcp_send_buffer.py +++ b/tests/component_tests/network/test_tcp_send_buffer.py @@ -14,32 +14,24 @@ import pytest from voluptuous import Invalid from esphome import config_validation as cv -from esphome.components.esp32.const import ( - KEY_SDKCONFIG_OPTIONS, - KEY_VARIANT, - VARIANT_ESP32, -) +from esphome.components.esp32.const import KEY_VARIANT, VARIANT_ESP32 from esphome.components.network import ( CONF_TCP_SEND_BUFFER, CONFIG_SCHEMA, TCP_SEND_BUFFER_MAX, TCP_SEND_BUFFER_MIN, ) -from esphome.const import KEY_ESP32, KEY_FRAMEWORK_VERSION, PlatformFramework -from esphome.core import CORE +from esphome.const import KEY_FRAMEWORK_VERSION, PlatformFramework +from tests.component_tests.network import sdkconfig_option from tests.component_tests.types import SetCoreConfigCallable -def _sdkconfig_option(name: str) -> int | None: - return CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS].get(name) - - def test_tcp_send_buffer_sets_sdkconfig( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], ) -> None: generate_main(component_config_path("tcp_send_buffer.yaml")) - assert _sdkconfig_option("CONFIG_LWIP_TCP_SND_BUF_DEFAULT") == 32000 + assert sdkconfig_option("CONFIG_LWIP_TCP_SND_BUF_DEFAULT") == 32000 def test_tcp_send_buffer_overrides_high_performance( @@ -48,7 +40,7 @@ def test_tcp_send_buffer_overrides_high_performance( ) -> None: """An explicit size wins over the high performance bundle's 65534.""" generate_main(component_config_path("tcp_send_buffer_high_perf.yaml")) - assert _sdkconfig_option("CONFIG_LWIP_TCP_SND_BUF_DEFAULT") == 16384 + assert sdkconfig_option("CONFIG_LWIP_TCP_SND_BUF_DEFAULT") == 16384 @pytest.mark.parametrize("value", [TCP_SEND_BUFFER_MIN, TCP_SEND_BUFFER_MAX]) diff --git a/tests/components/network/test-priority.esp32-idf.yaml b/tests/components/network/test-priority.esp32-idf.yaml index baa821a2348..ccafd74543f 100644 --- a/tests/components/network/test-priority.esp32-idf.yaml +++ b/tests/components/network/test-priority.esp32-idf.yaml @@ -3,6 +3,11 @@ # it must actually compile (not just validate) to guard the reconciler wiring. # WiFi is listed first so the build also exercises the wifi-primary branch in # network/util.cpp (the ethernet-primary branch matches the legacy order). +# PSRAM guaranteed plus high performance networking: a dual build keeps wifi's +# PSRAM tier, so this compiles the sdkconfig a wifi + ethernet audio device gets. +psram: + ignore_not_found: false + wifi: ssid: MySSID password: password1 @@ -21,3 +26,4 @@ network: priority: - wifi - ethernet + enable_high_performance: true From 54a923842505b41209010341db4d2eb9a91d9bbf Mon Sep 17 00:00:00 2001 From: FuNK3Y Date: Tue, 29 Sep 2026 11:38:01 +0200 Subject: [PATCH 033/133] [epaper_spi] reTerminal Sticky: Four-level grayscale for SSD1677 panels, and a full-refresh action (#19213) Co-authored-by: Pierre Co-authored-by: Claude Opus 5 Co-authored-by: clydebarrow <2366188+clydebarrow@users.noreply.github.com> Co-authored-by: J. Nick Koston --- esphome/components/epaper_spi/colorconv.h | 41 +- esphome/components/epaper_spi/display.py | 38 +- esphome/components/epaper_spi/epaper_spi.cpp | 13 +- esphome/components/epaper_spi/epaper_spi.h | 16 +- .../components/epaper_spi/epaper_spi_mono.h | 4 +- .../epaper_spi/epaper_spi_ssd1677.cpp | 101 +++++ .../epaper_spi/epaper_spi_ssd1677.h | 49 +++ .../epaper_spi/epaper_spi_ssd1677_gray4.cpp | 129 ++++++ .../epaper_spi/epaper_spi_ssd1677_gray4.h | 49 +++ .../components/epaper_spi/models/__init__.py | 35 ++ .../components/epaper_spi/models/jd79660.py | 4 +- .../components/epaper_spi/models/ssd1677.py | 67 +++- .../components/epaper_spi/models/ssd1683.py | 3 +- .../components/epaper_spi/models/uc8179.py | 3 +- .../components/epaper_spi/models/waveshare.py | 8 +- .../config/full_update_next_test.yaml | 29 ++ .../config/ssd1677_border_waveform_test.yaml | 22 + .../epaper_spi/config/ssd1677_gray4_test.yaml | 19 + tests/component_tests/epaper_spi/test_init.py | 70 ++++ .../epaper_spi/test_model_requirements.py | 376 ++++++++++++++++++ tests/components/epaper_spi/common.h | 50 +++ .../display/test_ssd1677_gray4_transfer.cpp | 298 ++++++++++++++ .../display/test_ssd1677_transfer.cpp | 233 +++++++++++ .../epaper_spi/test.esp32-s3-idf.yaml | 20 + 24 files changed, 1635 insertions(+), 42 deletions(-) create mode 100644 esphome/components/epaper_spi/epaper_spi_ssd1677.cpp create mode 100644 esphome/components/epaper_spi/epaper_spi_ssd1677.h create mode 100644 esphome/components/epaper_spi/epaper_spi_ssd1677_gray4.cpp create mode 100644 esphome/components/epaper_spi/epaper_spi_ssd1677_gray4.h create mode 100644 tests/component_tests/epaper_spi/config/full_update_next_test.yaml create mode 100644 tests/component_tests/epaper_spi/config/ssd1677_border_waveform_test.yaml create mode 100644 tests/component_tests/epaper_spi/config/ssd1677_gray4_test.yaml create mode 100644 tests/component_tests/epaper_spi/test_model_requirements.py create mode 100644 tests/components/epaper_spi/display/test_ssd1677_gray4_transfer.cpp create mode 100644 tests/components/epaper_spi/display/test_ssd1677_transfer.cpp diff --git a/esphome/components/epaper_spi/colorconv.h b/esphome/components/epaper_spi/colorconv.h index 7b7c48c0b0f..0240aa5cccf 100644 --- a/esphome/components/epaper_spi/colorconv.h +++ b/esphome/components/epaper_spi/colorconv.h @@ -16,6 +16,28 @@ namespace esphome::epaper_spi { /** Delta for when to regard as gray */ static constexpr uint8_t COLORCONV_GRAY_THRESHOLD = 50; +/** Rec.601 luma (0.299/0.587/0.114 weights, scaled by 256) for optimum perceptual brightness */ +constexpr uint8_t rec601_luma(Color color) { + return (uint8_t) ((77u * color.r + 150u * color.g + 29u * color.b + 128u) >> 8); +} + +/** Map RGB color to a single monochrome bit + * + * @param color RGB color to convert from + * @return 1 = white, 0 = black + */ +constexpr uint8_t color_to_mono(Color color) { return rec601_luma(color) >= 128 ? 1 : 0; } + +/** Map RGB color to one of 4 discrete gray levels (2 bits per pixel) + * + * @param color RGB color to convert from + * @return Gray level: 0 = black, 3 = white + */ +constexpr uint8_t color_to_gray4(Color color) { + const uint8_t level = (uint8_t) ((rec601_luma(color) + 32u) >> 6); // quantize 0..255 to 0..3, rounded + return level > 3 ? 3 : level; +} + /** Map RGB color to discrete BWYR hex 4 color key * * @tparam NATIVE_COLOR Type of native hardware color values @@ -25,7 +47,6 @@ static constexpr uint8_t COLORCONV_GRAY_THRESHOLD = 50; * @param hw_yellow Native value for yellow * @param hw_red Native value for red * @return Converted native hardware color value - * @internal Constexpr. Does not depend on side effects ("pure"). */ template constexpr NATIVE_COLOR color_to_bwyr(Color color, NATIVE_COLOR hw_black, NATIVE_COLOR hw_white, NATIVE_COLOR hw_yellow, @@ -38,11 +59,7 @@ constexpr NATIVE_COLOR color_to_bwyr(Color color, NATIVE_COLOR hw_black, NATIVE_ if ((max_rgb - min_rgb) < COLORCONV_GRAY_THRESHOLD) { // It's a shade of gray. Map to BLACK or WHITE. - // We split the luminance at the halfway point (382 = (255*3)/2) - if ((static_cast(color.r) + color.g + color.b) > 382) { - return hw_white; - } - return hw_black; + return color_to_mono(color) ? hw_white : hw_black; } // --- Step 2: Check for Primary/Secondary Colors --- @@ -96,7 +113,6 @@ constexpr NATIVE_COLOR color_to_bwr(Color color, NATIVE_COLOR hw_black, NATIVE_C * @param hw_green Native value for green * @param hw_blue Native value for blue * @return Converted native hardware color value - * @internal Constexpr. Does not depend on side effects ("pure"). */ template constexpr NATIVE_COLOR color_to_bwyrgb(Color color, NATIVE_COLOR hw_black, NATIVE_COLOR hw_white, @@ -105,10 +121,7 @@ constexpr NATIVE_COLOR color_to_bwyrgb(Color color, NATIVE_COLOR hw_black, NATIV const auto [min_rgb, max_rgb] = std::minmax({color.r, color.g, color.b}); if ((max_rgb - min_rgb) < COLORCONV_GRAY_THRESHOLD) { - if ((static_cast(color.r) + color.g + color.b) > 382) { - return hw_white; - } - return hw_black; + return color_to_mono(color) ? hw_white : hw_black; } const bool r_on = (color.r > 128); @@ -158,7 +171,6 @@ constexpr NATIVE_COLOR color_to_bwyrgb(Color color, NATIVE_COLOR hw_black, NATIV * @param hw_blue Native value for blue * @param hw_orange Native value for orange * @return Converted native hardware color value - * @internal Constexpr. Does not depend on side effects ("pure"). */ template constexpr NATIVE_COLOR color_to_bwyrgbo(Color color, NATIVE_COLOR hw_black, NATIVE_COLOR hw_white, @@ -167,10 +179,7 @@ constexpr NATIVE_COLOR color_to_bwyrgbo(Color color, NATIVE_COLOR hw_black, NATI const auto [min_rgb, max_rgb] = std::minmax({color.r, color.g, color.b}); if ((max_rgb - min_rgb) < COLORCONV_GRAY_THRESHOLD) { - if ((static_cast(color.r) + color.g + color.b) > 382) { - return hw_white; - } - return hw_black; + return color_to_mono(color) ? hw_white : hw_black; } const bool r_on = (color.r > 128); diff --git a/esphome/components/epaper_spi/display.py b/esphome/components/epaper_spi/display.py index e9da924de57..3fa504539be 100644 --- a/esphome/components/epaper_spi/display.py +++ b/esphome/components/epaper_spi/display.py @@ -1,7 +1,9 @@ +from collections.abc import Callable import importlib import pkgutil +from typing import Any -from esphome import core, pins +from esphome import automation, core, pins import esphome.codegen as cg from esphome.components import display, spi from esphome.components.display import CONF_SHOW_TEST_CARD, validate_rotation @@ -54,6 +56,12 @@ EPaperBase = epaper_spi_ns.class_( ) Transform = epaper_spi_ns.enum("Transform") +automation.register_apply_action( + "epaper_spi.full_update_next", + automation.maybe_simple_id({cv.Required(CONF_ID): cv.use_id(EPaperBase)}), + automation.ApplyCall("request_full_update()"), +) + # Import all models dynamically from the models package for module_info in pkgutil.iter_modules(models.__path__): importlib.import_module(f".models.{module_info.name}", package=__package__) @@ -70,6 +78,23 @@ DIMENSION_SCHEMA = cv.Schema( TRANSFORM_OPTIONS = {CONF_MIRROR_X, CONF_MIRROR_Y, CONF_SWAP_XY} +def _full_update_every_validator( + model: models.EpaperModel, +) -> Callable[[Any], int]: + if model.get_default("partial_update"): + return cv.int_range(1, 255) + + def validate(value: Any) -> int: + value = cv.int_range(1, 255)(value) + if value != 1: + raise cv.Invalid( + f"{model.name} does not support partial update; full_update_every must be 1" + ) + return value + + return validate + + def model_schema(config): model = MODELS[config[CONF_MODEL]] class_name = epaper_spi_ns.class_(model.class_name, EPaperBase) @@ -96,7 +121,9 @@ def model_schema(config): cv.Required(CONF_MIRROR_Y): cv.boolean, } ), - cv.Optional(CONF_FULL_UPDATE_EVERY, default=1): cv.int_range(1, 255), + cv.Optional( + CONF_FULL_UPDATE_EVERY, default=1 + ): _full_update_every_validator(model), model.option(CONF_BUSY_PIN): pins.gpio_input_pin_schema, model.option(CONF_CS_PIN): pins.gpio_output_pin_schema, model.option(CONF_DC_PIN, fallback=None): pins.gpio_output_pin_schema, @@ -132,8 +159,15 @@ def customise_schema(config): extra=cv.ALLOW_EXTRA, )(config) model = MODELS[config[CONF_MODEL]] + model.check_requirements() config = model_schema(config)(config) + config = model.validate_config(config) width, height = model.get_dimensions(config) + if width % (width_multiple := model.get_default("width_multiple", 1)): + raise cv.Invalid( + f"{model.name} requires a width that is a multiple of {width_multiple}", + path=[CONF_DIMENSIONS], + ) display.add_metadata( config[CONF_ID], width, diff --git a/esphome/components/epaper_spi/epaper_spi.cpp b/esphome/components/epaper_spi/epaper_spi.cpp index 3b3418d911f..1fab038c283 100644 --- a/esphome/components/epaper_spi/epaper_spi.cpp +++ b/esphome/components/epaper_spi/epaper_spi.cpp @@ -196,6 +196,15 @@ void EPaperBase::process_state_() { break; case EPaperState::UPDATE: this->do_update_(); // Calls ESPHome (current page) lambda + if (this->full_update_requested_) { + // Refresh the whole panel even if nothing was drawn + this->full_update_requested_ = false; + this->update_count_ = 0; + this->x_low_ = 0; + this->y_low_ = 0; + this->x_high_ = this->width_; + this->y_high_ = this->height_; + } if (this->x_high_ < this->x_low_ || this->y_high_ < this->y_low_) { this->set_state_(EPaperState::IDLE); return; @@ -327,9 +336,9 @@ void HOT EPaperBase::draw_pixel_at(int x, int y, Color color) { return; const size_t byte_position = y * this->row_width_ + x / 8; const uint8_t bit_position = x % 8; - const uint8_t pixel_bit = 0x80 >> bit_position; + const uint8_t pixel_bit = 0x80u >> bit_position; const auto original = this->buffer_[byte_position]; - if ((color_to_bit(color) == 0)) { + if (color_to_mono(color) == 0) { this->buffer_[byte_position] = original & ~pixel_bit; } else { this->buffer_[byte_position] = original | pixel_bit; diff --git a/esphome/components/epaper_spi/epaper_spi.h b/esphome/components/epaper_spi/epaper_spi.h index 8e2fd78e621..0040c867b20 100644 --- a/esphome/components/epaper_spi/epaper_spi.h +++ b/esphome/components/epaper_spi/epaper_spi.h @@ -1,5 +1,6 @@ #pragma once +#include "colorconv.h" #include "esphome/components/display/display.h" #include "esphome/components/spi/spi.h" #include "esphome/components/split_buffer/split_buffer.h" @@ -80,15 +81,7 @@ class EPaperBase : public Display, DisplayType get_display_type() override { return this->display_type_; }; - // Default implementations for monochrome displays - static uint8_t color_to_bit(Color color) { - // It's always a shade of gray. Map to BLACK or WHITE. - // We split the luminance at a suitable point - if ((color.r + color.g + color.b) >= 382) { - return 1; - } - return 0; - } + // Default implementation for monochrome displays void fill(Color color) override { // If clipping is active, fall back to base implementation if (this->get_clipping().is_set()) { @@ -96,7 +89,7 @@ class EPaperBase : public Display, return; } - auto pixel_color = color_to_bit(color) ? 0xFF : 0x00; + auto pixel_color = color_to_mono(color) ? 0xFF : 0x00; // We store 8 pixels per byte this->buffer_.fill(pixel_color); @@ -114,6 +107,8 @@ class EPaperBase : public Display, int get_width() override { return this->effective_transform_ & SWAP_XY ? this->height_ : this->width_; } int get_height() override { return this->effective_transform_ & SWAP_XY ? this->width_ : this->height_; } void draw_pixel_at(int x, int y, Color color) override; + // Make the next update a full one. Applied when that update starts, so one in progress is not affected. + void request_full_update() { this->full_update_requested_ = true; } protected: int get_height_internal() override { return this->height_; }; @@ -185,6 +180,7 @@ class EPaperBase : public Display, uint8_t transform_{}; uint8_t effective_transform_{}; uint8_t update_count_{}; + bool full_update_requested_{}; // these values represent the bounds of the updated buffer. Note that x_high and y_high // point to the pixel past the last one updated, i.e. may range up to width/height. uint16_t x_low_{}, y_low_{}, x_high_{}, y_high_{}; diff --git a/esphome/components/epaper_spi/epaper_spi_mono.h b/esphome/components/epaper_spi/epaper_spi_mono.h index f44b59e803e..d0740595fc5 100644 --- a/esphome/components/epaper_spi/epaper_spi_mono.h +++ b/esphome/components/epaper_spi/epaper_spi_mono.h @@ -9,8 +9,8 @@ namespace esphome::epaper_spi { class EPaperMono : public EPaperBase { public: EPaperMono(const char *name, uint16_t width, uint16_t height, const uint8_t *init_sequence, - size_t init_sequence_length) - : EPaperBase(name, width, height, init_sequence, init_sequence_length, DISPLAY_TYPE_BINARY) { + size_t init_sequence_length, DisplayType display_type = DISPLAY_TYPE_BINARY) + : EPaperBase(name, width, height, init_sequence, init_sequence_length, display_type) { this->buffer_length_ = (width + 7) / 8 * height; // 8 pixels per byte, rounded up } diff --git a/esphome/components/epaper_spi/epaper_spi_ssd1677.cpp b/esphome/components/epaper_spi/epaper_spi_ssd1677.cpp new file mode 100644 index 00000000000..e845a5d5858 --- /dev/null +++ b/esphome/components/epaper_spi/epaper_spi_ssd1677.cpp @@ -0,0 +1,101 @@ +#include "epaper_spi_ssd1677.h" + +#include + +#include "esphome/core/helpers.h" +#include "esphome/core/log.h" + +namespace esphome::epaper_spi { +static constexpr const char *const TAG = "epaper_spi.ssd1677"; + +void EPaperSSD1677::setup() { + EPaperMono::setup(); + if (!this->is_failed()) + this->init_comparison_frame_(); +} + +void EPaperSSD1677::init_comparison_frame_() { + if (!this->is_using_partial_update_()) + return; + if (!this->sent_.init(this->plane_row_length_() * this->height_)) { + ESP_LOGW(TAG, "No memory for the comparison frame; partial updates will degrade unchanged areas"); + } +} + +void EPaperSSD1677::plane_row(size_t y, uint8_t *out) { + const size_t row_length = this->plane_row_length_(); + const size_t data_idx = y * row_length; + for (size_t i = 0; i != row_length; i++) + out[i] = this->buffer_[data_idx + i]; +} + +// Nothing a partial update needs is kept in controller RAM any more, so skip the reset for those. +bool EPaperSSD1677::reset() { + if (this->update_count_ != 0 && this->sent_.is_valid()) + return true; + return EPaperMono::reset(); +} + +// The window always covers the whole panel, so each plane is the frame's bytes in order. Where +// those bytes are already stored as the plane needs them (the comparison frame, and a 1-bit +// buffer) they are written straight from the buffer, as many at a time as the time slice allows; +// otherwise they are built a row at a time by plane_row(). +bool HOT EPaperSSD1677::transfer_data() { + if (!this->sent_.is_valid()) + return EPaperMono::transfer_data(); + + const auto start_time = millis(); + if (this->current_data_index_ == 0) { + if (this->plane_ == 0) { + this->x_low_ = 0; + this->x_high_ = this->width_; + this->y_low_ = 0; + this->y_high_ = this->height_; + } + this->set_window(); + this->command(this->plane_ == 0 ? 0x26 : 0x24); + } + // A full update ignores 0x26, and the copy may not hold a real frame yet (first update after + // boot): send the new frame to both planes. + const bool send_copy = this->plane_ == 0 && this->update_count_ != 0; + const bool direct = send_copy || this->buffer_is_plane(); + const auto &source = send_copy ? this->sent_ : this->buffer_; + const size_t row_length = this->plane_row_length_(); + const size_t plane_length = row_length * this->height_; + // Roughly what the bus moves in one time slice, so a slice is not overrun by much + const size_t max_chunk = std::max(this->data_rate_ / 8000 * MAX_TRANSFER_TIME, MAX_TRANSFER_SIZE); + SmallBufferWithHeapFallback<128> row_alloc(direct ? 0 : row_length); + this->start_data_(); + while (this->current_data_index_ != plane_length) { + size_t length; + const uint8_t *data; + if (direct) { + data = source.get_span(this->current_data_index_, length); + length = std::min(length, max_chunk); + } else { + // Always at the start of a row here, since this path sends whole rows only + this->plane_row(this->current_data_index_ / row_length, row_alloc.get()); + data = row_alloc.get(); + length = row_length; + } + this->write_array(data, length); + if (this->plane_ == 1) + this->sent_.write(this->current_data_index_, data, length); + this->current_data_index_ += length; + if (this->current_data_index_ != plane_length && millis() - start_time > MAX_TRANSFER_TIME) { + // Let the main loop run and come back next loop + this->disable(); + return false; + } + } + this->disable(); + this->current_data_index_ = 0; + if (this->plane_ == 0) { + this->plane_ = 1; + return false; + } + this->plane_ = 0; + return true; +} + +} // namespace esphome::epaper_spi diff --git a/esphome/components/epaper_spi/epaper_spi_ssd1677.h b/esphome/components/epaper_spi/epaper_spi_ssd1677.h new file mode 100644 index 00000000000..ddf3d4e0892 --- /dev/null +++ b/esphome/components/epaper_spi/epaper_spi_ssd1677.h @@ -0,0 +1,49 @@ +#pragma once + +#include "epaper_spi_mono.h" + +namespace esphome::epaper_spi { + +/** + * Monochrome SSD1677 with partial refreshes that leave unchanged pixels alone. + * + * A partial refresh drives each pixel from the pair (RAM 0x26 = the image on the panel, + * RAM 0x24 = the new image) across the whole panel; the RAM window only scopes a write. + * EPaperMono writes 0x26 once and afterwards only the changed window of 0x24, which relies on + * the controller's RAM being unchanged from one update to the next. On this controller it is not: + * the hardware reset at the start of each update loses it, and even without resets, keeping 0x26 + * in step one window at a time left unchanged areas alternating between older frames. Either way + * unchanged pixels get driven on every partial and wash out. + * + * So before every refresh this class writes both planes over the whole panel: 0x26 from a copy of + * the frame last sent, 0x24 from the buffer. GxEPD2 likewise rewrites both planes after each + * partial on this controller. The copy is taken as the data goes out, not from the buffer, which + * may already hold the next frame by the time the refresh completes. + */ +class EPaperSSD1677 : public EPaperMono { + public: + EPaperSSD1677(const char *name, uint16_t width, uint16_t height, const uint8_t *init_sequence, + size_t init_sequence_length, DisplayType display_type = DISPLAY_TYPE_BINARY) + : EPaperMono(name, width, height, init_sequence, init_sequence_length, display_type) {} + + void setup() override; + + protected: + // Allocates the comparison frame when partial updates are enabled. Separate from setup() so it + // can run without a bus. + void init_comparison_frame_(); + // Bytes in one row of a RAM plane: 8 pixels per byte, whatever the buffer holds. + size_t plane_row_length_() const { return (this->width_ + 7) / 8; } + // Whether the buffer already holds the frame as a RAM plane does, so it can be sent as it is. + // A subclass with a deeper buffer returns false and overrides plane_row(). + virtual bool buffer_is_plane() const { return true; } + // Row y of the frame as a RAM plane holds it, 1 bit per pixel with 1 = white. + virtual void plane_row(size_t y, uint8_t *out); + bool reset() override; + bool transfer_data() override; + + split_buffer::SplitBuffer sent_{}; // the frame last sent to 0x24, i.e. what the panel shows + uint8_t plane_{0}; // 0 while sending 0x26, 1 while sending 0x24 +}; + +} // namespace esphome::epaper_spi diff --git a/esphome/components/epaper_spi/epaper_spi_ssd1677_gray4.cpp b/esphome/components/epaper_spi/epaper_spi_ssd1677_gray4.cpp new file mode 100644 index 00000000000..18fbb016f3b --- /dev/null +++ b/esphome/components/epaper_spi/epaper_spi_ssd1677_gray4.cpp @@ -0,0 +1,129 @@ +#include "epaper_spi_ssd1677_gray4.h" + +#include "esphome/core/log.h" + +namespace esphome::epaper_spi { + +static constexpr const char *const TAG = "epaper_spi.ssd1677_gray4"; + +// Combine two source bytes (4 pixels each, 2 bits per pixel, most significant pixel first) into +// one plane byte covering the same 8 pixels (1 bit per pixel), choosing high or low bit +static uint8_t plane_byte(uint8_t first, uint8_t second, bool high_bit) { + uint8_t out = 0; + for (const uint8_t src : {first, second}) { + for (uint8_t shift = 6;; shift -= 2) { + const uint8_t level = (src >> shift) & 0x03; + out = (uint8_t) ((out << 1) | (high_bit ? (level >> 1) : (level & 1))); + if (shift == 0) + break; + } + } + return out; +} + +void EPaperSSD1677Gray4::fill(Color color) { + if (this->get_clipping().is_set()) { + // Falls back to the generic per-pixel implementation for the clipped rectangle. + EPaperBase::fill(color); + return; + } + const uint8_t level = color_to_gray4(color); + this->buffer_.fill((uint8_t) (level | (level << 2) | (level << 4) | (level << 6))); + this->x_low_ = 0; + this->y_low_ = 0; + this->x_high_ = this->width_; + this->y_high_ = this->height_; +} + +void HOT EPaperSSD1677Gray4::draw_pixel_at(int x, int y, Color color) { + if (!this->rotate_coordinates_(x, y)) + return; + const uint8_t level = color_to_gray4(color); + const size_t byte_position = (size_t) y * this->row_width_ + x / 4; + const uint8_t shift = (uint8_t) (6 - 2 * (x % 4)); // most significant pixel first + const uint8_t original = this->buffer_[byte_position]; + this->buffer_[byte_position] = (uint8_t) ((original & ~(0x03 << shift)) | (level << shift)); +} + +// A partial update reduces each pixel to its high bit: levels 2 and 3 are light, 0 and 1 dark. +void EPaperSSD1677Gray4::plane_row(size_t y, uint8_t *out) { + const size_t src_row = y * this->row_width_; + for (size_t i = 0; i != this->plane_row_length_(); i++) + out[i] = plane_byte(this->buffer_[src_row + 2 * i], this->buffer_[src_row + 2 * i + 1], true); +} + +// the high bit of every pixel's level goes to the new (bw) plane +// (0x24), the low bit to the old (red) plane (0x26) +bool HOT EPaperSSD1677Gray4::transfer_data() { + if (this->is_partial_push_()) + return EPaperSSD1677::transfer_data(); + + auto start_time = millis(); + const bool first_pass = this->send_red_; + if (this->current_data_index_ == 0) { + if (first_pass) { + // With partial updates enabled the window follows the changed area, but the four-level + // refresh drives every pixel from both planes, and the reset before it does not keep RAM. + this->x_low_ = 0; + this->x_high_ = this->width_; + this->y_low_ = 0; + this->y_high_ = this->height_; + this->set_window(); + } + this->command(first_pass ? 0x24 : 0x26); + this->current_data_index_ = this->y_low_; + } + const size_t plane_row_length = (this->x_high_ - this->x_low_) / 8; + // Stack-backed for every panel width in practice; only a custom `dimensions:` far wider than any + // supported panel would fall back to the heap. + SmallBufferWithHeapFallback<128> bytes_to_send_alloc(plane_row_length); + uint8_t *bytes_to_send = bytes_to_send_alloc.get(); + ESP_LOGV(TAG, "Writing %u bytes at line %zu at %ums", plane_row_length, this->current_data_index_, + (unsigned) millis()); + this->start_data_(); + while (this->current_data_index_ != this->y_high_) { + const size_t src_row = this->current_data_index_ * this->row_width_ + this->x_low_ / 4; + for (size_t i = 0; i != plane_row_length; i++) { + const uint8_t plane = plane_byte(this->buffer_[src_row + 2 * i], this->buffer_[src_row + 2 * i + 1], first_pass); + // The OTP grayscale waveform treats data as inverted relative to monochrome + bytes_to_send[i] = (uint8_t) ~plane; + // What the next partial update compares against: the high bits, as a black-and-white frame. + if (first_pass && this->sent_.is_valid()) + this->sent_[this->current_data_index_ * plane_row_length + i] = plane; + } + ++this->current_data_index_; + this->write_array(bytes_to_send, plane_row_length); + if (millis() - start_time > MAX_TRANSFER_TIME) { + // Let the main loop run and come back next loop + this->disable(); + return false; + } + } + + this->disable(); + this->current_data_index_ = 0; + if (first_pass) { + this->send_red_ = false; + return false; + } + this->send_red_ = true; + return true; +} + +void EPaperSSD1677Gray4::refresh_screen(bool partial) { + if (this->is_partial_push_()) { + ESP_LOGV(TAG, "Black-and-white partial refresh"); + // The border follows the LUT selected in 0x3C. The model's setting (sent with the init + // sequence) picks the LUT that is white under the four-level waveform's inverted data; under + // the black-and-white waveform that LUT drives black and the border darkens, so use LUT1. + this->cmd_data(0x3C, {0x01}); + EPaperSSD1677::refresh_screen(true); + return; + } + ESP_LOGV(TAG, "Four-level refresh"); + this->cmd_data(0x1A, {0x67, 0x00}); // force temperature by OTP + this->cmd_data(0x22, {0xD7}); // four-level update sequence, panel's OTP waveform + this->command(0x20); // master activation +} + +} // namespace esphome::epaper_spi diff --git a/esphome/components/epaper_spi/epaper_spi_ssd1677_gray4.h b/esphome/components/epaper_spi/epaper_spi_ssd1677_gray4.h new file mode 100644 index 00000000000..47436d2db1d --- /dev/null +++ b/esphome/components/epaper_spi/epaper_spi_ssd1677_gray4.h @@ -0,0 +1,49 @@ +#pragma once + +#include "colorconv.h" +#include "epaper_spi_ssd1677.h" + +namespace esphome::epaper_spi { + +/** + * Four-level grayscale for SSD1677 panels. + * + * The SSD1677 has two independent 1-bit RAM planes, normally used for a + * black/white and a red plane. This class writes image data to both, + * splitting each pixel's 2-bit gray level across them, and triggers the + * panel's own OTP grayscale waveform instead of the normal monochrome update + * sequence. No custom LUT upload needed for any currently supported panel, + * the OTP waveform is used instead. + * + * The framebuffer therefore packs 2 bits per pixel (4 per byte, most + * significant pixel first) instead of EPaperMono's 1 bit. + * + * The grayscale waveform has no partial form: master activation redraws the whole panel + * regardless of the RAM window. So a full update is a four-level refresh, and when partial updates + * are enabled (full_update_every > 1, which the model only allows on explicit request) a partial + * update is EPaperSSD1677's black-and-white one, each pixel reduced to light or dark. The partial + * waveform also drives unchanged pixels towards black or white, so from the first partial update + * the whole panel loses its gray levels until the next full update. + */ +class EPaperSSD1677Gray4 : public EPaperSSD1677 { + public: + EPaperSSD1677Gray4(const char *name, uint16_t width, uint16_t height, const uint8_t *init_sequence, + size_t init_sequence_length) + : EPaperSSD1677(name, width, height, init_sequence, init_sequence_length, DISPLAY_TYPE_GRAYSCALE) { + this->row_width_ = (width + 3) / 4; // 4 pixels per byte + this->buffer_length_ = (size_t) this->row_width_ * height; + } + + void fill(Color color) override; + void draw_pixel_at(int x, int y, Color color) override; + + protected: + // A partial update if partial updates are enabled and this is not a full one. + bool is_partial_push_() const { return this->update_count_ != 0 && this->sent_.is_valid(); } + bool buffer_is_plane() const override { return false; } + void plane_row(size_t y, uint8_t *out) override; + void refresh_screen(bool partial) override; + bool transfer_data() override; +}; + +} // namespace esphome::epaper_spi diff --git a/esphome/components/epaper_spi/models/__init__.py b/esphome/components/epaper_spi/models/__init__.py index 34e65061f31..fbedd6371d1 100644 --- a/esphome/components/epaper_spi/models/__init__.py +++ b/esphome/components/epaper_spi/models/__init__.py @@ -2,7 +2,9 @@ from typing import Any, Self import esphome.config_validation as cv from esphome.const import CONF_DIMENSIONS, CONF_HEIGHT, CONF_WIDTH +from esphome.core import CORE from esphome.cpp_generator import MockObj +from esphome.types import ConfigType class EpaperModel: @@ -48,6 +50,16 @@ class EpaperModel: """ return {} + def validate_config(self, config: ConfigType) -> ConfigType: + """ + Validate the configuration as a whole, once the schema has been applied. + The base implementation accepts it unchanged; specific models override this for + rules that span several options. + :param config: The validated configuration + :return: The configuration, possibly updated + """ + return config + async def to_code(self, var: MockObj, config: dict) -> dict: """ Generate model-specific code for the options added by add_options(). @@ -89,3 +101,26 @@ class EpaperModel: defaults = self.defaults.copy() defaults.update(kwargs) return self.__class__(name, initsequence=tuple(initsequence), **defaults) + + def check_requirements(self) -> None: + """ + Raise a friendly error if any component this model requires is not configured. + + This runs during schema validation (before ID references are resolved) so that a + model whose default pins live on a pin expander reports the missing expander clearly + instead of a cryptic "Couldn't find ID" from the unresolved pin reference. + """ + if requirements := self.get_default("requires", set()): + # ``raw_config`` is populated before any component schema runs during a real + # validation, so presence of a required component is simply a top-level key. + # When it is absent (e.g. a unit test that invokes the schema directly) there + # is no config to check against, so skip. + global_config = CORE.raw_config + if global_config is None: + return + missing = {x for x in requirements if x not in global_config} + if missing: + reqstr = ", ".join(f"'{x}'" for x in sorted(missing)) + raise cv.Invalid( + f"{self.name} requires component{'s' if len(missing) > 1 else ''} {reqstr} to be configured" + ) diff --git a/esphome/components/epaper_spi/models/jd79660.py b/esphome/components/epaper_spi/models/jd79660.py index a0457c5812a..ca824dccd4d 100644 --- a/esphome/components/epaper_spi/models/jd79660.py +++ b/esphome/components/epaper_spi/models/jd79660.py @@ -10,7 +10,9 @@ from . import EpaperModel class JD79660(EpaperModel): def __init__(self, name, class_name="EPaperJD79660", fast_update=None, **kwargs): - super().__init__(name, class_name, **kwargs) + # Only a fast_update sequence lets the driver do anything but a full refresh + kwargs.setdefault("partial_update", fast_update is not None) + super().__init__(name, class_name=class_name, **kwargs) self.fast_update = fast_update def option(self, name, fallback=cv.UNDEFINED) -> cv.Optional | cv.Required: diff --git a/esphome/components/epaper_spi/models/ssd1677.py b/esphome/components/epaper_spi/models/ssd1677.py index 13f10350457..2249529611e 100644 --- a/esphome/components/epaper_spi/models/ssd1677.py +++ b/esphome/components/epaper_spi/models/ssd1677.py @@ -1,12 +1,56 @@ -from esphome.const import CONF_DATA_RATE +from typing import Any + +import esphome.config_validation as cv +from esphome.const import CONF_DATA_RATE, CONF_FULL_UPDATE_EVERY +from esphome.types import ConfigType from . import EpaperModel +CONF_BORDER_WAVEFORM = "border_waveform" +CONF_MONOCHROME_PARTIAL_UPDATES = "monochrome_partial_updates" + +# partial_update value for models whose partial updates are black and white only +MONOCHROME = "monochrome" + class SSD1677(EpaperModel): - def __init__(self, name, class_name="EPaperMono", data_rate="20MHz", **defaults): + def __init__( + self, + name: str, + class_name: str = "EPaperSSD1677", + data_rate: str = "20MHz", + border_waveform: int = 0x01, + **defaults: Any, + ) -> None: defaults[CONF_DATA_RATE] = data_rate - super().__init__(name, class_name, **defaults) + defaults[CONF_BORDER_WAVEFORM] = border_waveform + defaults.setdefault("partial_update", True) + super().__init__(name, class_name=class_name, **defaults) + + def get_config_options(self) -> dict: + options = { + self.option(CONF_BORDER_WAVEFORM): cv.hex_uint8_t, + } + if self.get_default("partial_update") == MONOCHROME: + options[cv.Optional(CONF_MONOCHROME_PARTIAL_UPDATES, default=False)] = ( + cv.boolean + ) + return options + + def validate_config(self, config: ConfigType) -> ConfigType: + if ( + self.get_default("partial_update") == MONOCHROME + and config[CONF_FULL_UPDATE_EVERY] > 1 + and not config[CONF_MONOCHROME_PARTIAL_UPDATES] + ): + raise cv.Invalid( + f"{self.name} can only update partially in black and white, and a partial " + "update reduces the whole panel to black and white until the next full " + f"update. Set '{CONF_MONOCHROME_PARTIAL_UPDATES}: true' to accept this, " + "or leave full_update_every at 1", + path=[CONF_FULL_UPDATE_EVERY], + ) + return config # fmt: off def get_init_sequence(self, config: dict): @@ -15,13 +59,14 @@ class SSD1677(EpaperModel): (0x18, 0x80), # Select internal Temp sensor (0x0C, 0xAE, 0xC7, 0xC3, 0xC0, 0x80), # inrush current level 2 (0x01, (height - 1) % 256, (height - 1) // 256, 0x02), # Set gate limit (number of rows-1) - (0x3C, 0x01), # Set border waveform + (0x3C, config[CONF_BORDER_WAVEFORM]), # Set border waveform (0x11, 3), # Set transform ) ssd1677 = SSD1677("ssd1677") + wave_4_26 = ssd1677.extend( "waveshare-4.26in", width=800, @@ -52,7 +97,8 @@ ssd1677.extend( mirror_x=True, ) -ssd1677.extend( +# Sticky - monochrome version +seeed_sticky = ssd1677.extend( "seeed-reterminal-sticky", width=800, height=480, @@ -63,4 +109,15 @@ ssd1677.extend( reset_pin=17, busy_pin=18, data_rate="10MHz", + requires={"psram"}, +) + +# Sticky - 4 level grayscale; partial updates only in black and white, on request +seeed_sticky.extend( + "seeed-reterminal-sticky-gray4", + class_name="EPaperSSD1677Gray4", + border_waveform=0x00, + partial_update=MONOCHROME, + # each plane byte is built from two whole buffer bytes + width_multiple=8, ) diff --git a/esphome/components/epaper_spi/models/ssd1683.py b/esphome/components/epaper_spi/models/ssd1683.py index 983f5bb382e..b43168a336e 100644 --- a/esphome/components/epaper_spi/models/ssd1683.py +++ b/esphome/components/epaper_spi/models/ssd1683.py @@ -6,7 +6,8 @@ from . import EpaperModel class SSD1683(EpaperModel): def __init__(self, name, class_name="EPaperSSD1683", data_rate="20MHz", **defaults): defaults[CONF_DATA_RATE] = data_rate - super().__init__(name, class_name, **defaults) + defaults.setdefault("partial_update", True) + super().__init__(name, class_name=class_name, **defaults) # fmt: off def get_init_sequence(self, config: dict): diff --git a/esphome/components/epaper_spi/models/uc8179.py b/esphome/components/epaper_spi/models/uc8179.py index bea133c3282..91c649808a8 100644 --- a/esphome/components/epaper_spi/models/uc8179.py +++ b/esphome/components/epaper_spi/models/uc8179.py @@ -32,7 +32,8 @@ class UC8179(EpaperModel): **defaults: Any, ) -> None: defaults.setdefault(CONF_DATA_RATE, data_rate) - super().__init__(name, class_name, **defaults) + defaults.setdefault("partial_update", True) + super().__init__(name, class_name=class_name, **defaults) def get_init_sequence(self, config: dict) -> tuple: """Generate the initialization sequence for UC8179 mono displays. diff --git a/esphome/components/epaper_spi/models/waveshare.py b/esphome/components/epaper_spi/models/waveshare.py index 74a288977d5..aecda723643 100644 --- a/esphome/components/epaper_spi/models/waveshare.py +++ b/esphome/components/epaper_spi/models/waveshare.py @@ -6,8 +6,12 @@ from . import EpaperModel class WaveshareModel(EpaperModel): - def __init__(self, name, lut, lut_partial=None, **defaults): - super().__init__(name, "EpaperWaveshare", **defaults) + def __init__( + self, name, lut, lut_partial=None, class_name="EpaperWaveshare", **defaults + ): + # A partial LUT is what lets EpaperWaveshare do partial refresh + defaults.setdefault("partial_update", lut_partial is not None) + super().__init__(name, class_name=class_name, **defaults) self.lut = lut self.lut_partial = lut_partial diff --git a/tests/component_tests/epaper_spi/config/full_update_next_test.yaml b/tests/component_tests/epaper_spi/config/full_update_next_test.yaml new file mode 100644 index 00000000000..de5e678a700 --- /dev/null +++ b/tests/component_tests/epaper_spi/config/full_update_next_test.yaml @@ -0,0 +1,29 @@ +esphome: + name: test + +esp32: + board: esp32dev + +spi: + clk_pin: GPIO18 + mosi_pin: GPIO19 + +display: + - platform: epaper_spi + id: epaper_display + model: ssd1677 + dc_pin: GPIO21 + busy_pin: GPIO22 + reset_pin: GPIO23 + cs_pin: GPIO5 + full_update_every: 5 + dimensions: + width: 200 + height: 200 + +binary_sensor: + - platform: gpio + pin: GPIO27 + name: Trigger + on_press: + - epaper_spi.full_update_next: epaper_display diff --git a/tests/component_tests/epaper_spi/config/ssd1677_border_waveform_test.yaml b/tests/component_tests/epaper_spi/config/ssd1677_border_waveform_test.yaml new file mode 100644 index 00000000000..3d15ab8f905 --- /dev/null +++ b/tests/component_tests/epaper_spi/config/ssd1677_border_waveform_test.yaml @@ -0,0 +1,22 @@ +esphome: + name: test + +esp32: + board: esp32dev + +spi: + clk_pin: GPIO18 + mosi_pin: GPIO19 + +display: + - platform: epaper_spi + id: epaper_display + model: ssd1677 + dc_pin: GPIO21 + busy_pin: GPIO22 + reset_pin: GPIO23 + cs_pin: GPIO5 + dimensions: + width: 200 + height: 200 + border_waveform: 0x1A diff --git a/tests/component_tests/epaper_spi/config/ssd1677_gray4_test.yaml b/tests/component_tests/epaper_spi/config/ssd1677_gray4_test.yaml new file mode 100644 index 00000000000..d1078072986 --- /dev/null +++ b/tests/component_tests/epaper_spi/config/ssd1677_gray4_test.yaml @@ -0,0 +1,19 @@ +esphome: + name: test + +esp32: + board: esp32-s3-devkitc-1 + variant: esp32s3 + +psram: + mode: octal + speed: 80MHz + +spi: + clk_pin: GPIO7 + mosi_pin: GPIO9 + +display: + - platform: epaper_spi + id: epaper_display + model: seeed-reterminal-sticky-gray4 diff --git a/tests/component_tests/epaper_spi/test_init.py b/tests/component_tests/epaper_spi/test_init.py index 5e2e7d6013b..9d6ebe831a0 100644 --- a/tests/component_tests/epaper_spi/test_init.py +++ b/tests/component_tests/epaper_spi/test_init.py @@ -312,6 +312,66 @@ def test_model_with_full_update_every( ) +def test_update_interval_below_model_minimum_rejected( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """update_interval faster than the model's minimum_update_interval is rejected.""" + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_BOARD: "esp32dev", KEY_VARIANT: VARIANT_ESP32}, + ) + + set_component_config("spi", {"id": "spi_bus", "clk_pin": 18, "mosi_pin": 19}) + + with pytest.raises(cv.Invalid, match="at least"): + run_schema_validation( + { + "id": "test_display", + "model": "ssd1677", + "dc_pin": 21, + "busy_pin": 22, + "reset_pin": 23, + "cs_pin": 5, + "dimensions": { + "width": 200, + "height": 200, + }, + "update_interval": "500ms", + } + ) + + +def test_reset_duration_over_max_rejected( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """reset_duration over the 500ms cap is rejected.""" + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_BOARD: "esp32dev", KEY_VARIANT: VARIANT_ESP32}, + ) + + set_component_config("spi", {"id": "spi_bus", "clk_pin": 18, "mosi_pin": 19}) + + with pytest.raises(cv.Invalid, match="at most"): + run_schema_validation( + { + "id": "test_display", + "model": "ssd1677", + "dc_pin": 21, + "busy_pin": 22, + "reset_pin": 23, + "cs_pin": 5, + "dimensions": { + "width": 200, + "height": 200, + }, + "reset_duration": "600ms", + } + ) + + def test_busy_pin_input_mode_ssd1677( set_core_config: SetCoreConfigCallable, set_component_config: Callable[[str, Any], None], @@ -481,6 +541,16 @@ def test_enable_pin_code_generation( assert f"set_enable_pins({{{pin_25}, {pin_26}}});" in main_cpp +def test_full_update_next_action_code_generation( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """The epaper_spi.full_update_next action targets the configured display.""" + main_cpp = generate_main(component_config_path("full_update_next_test.yaml")) + + assert "epaper_display->request_full_update();" in main_cpp + + def test_model_with_no_default_init_sequence_generates( generate_main: Callable[[str | Path], str], component_config_path: Callable[[str], Path], diff --git a/tests/component_tests/epaper_spi/test_model_requirements.py b/tests/component_tests/epaper_spi/test_model_requirements.py new file mode 100644 index 00000000000..26c50a133a0 --- /dev/null +++ b/tests/component_tests/epaper_spi/test_model_requirements.py @@ -0,0 +1,376 @@ +"""Tests for the SSD1677 border_waveform option and EpaperModel.check_requirements().""" + +from collections.abc import Callable, Generator +from pathlib import Path +import re +from typing import Any + +import pytest + +from esphome import config_validation as cv +from esphome.components.epaper_spi.display import CONFIG_SCHEMA, MODELS +from esphome.components.epaper_spi.models import EpaperModel +from esphome.components.epaper_spi.models.ssd1677 import CONF_BORDER_WAVEFORM +from esphome.components.esp32 import ( + KEY_BOARD, + KEY_VARIANT, + VARIANT_ESP32, + VARIANT_ESP32S3, +) +from esphome.const import PlatformFramework +from esphome.core import CORE +from esphome.types import ConfigType +from tests.component_tests.types import SetCoreConfigCallable + + +def _ssd1677_config(**overrides: Any) -> ConfigType: + config: ConfigType = { + "id": "test_display", + "model": "ssd1677", + "dc_pin": 21, + "busy_pin": 22, + "reset_pin": 23, + "cs_pin": 5, + "dimensions": {"width": 200, "height": 200}, + } + config.update(overrides) + return config + + +def _setup_esp32( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], + variant: str = VARIANT_ESP32, + board: str = "esp32dev", +) -> None: + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={KEY_BOARD: board, KEY_VARIANT: variant}, + ) + set_component_config("spi", {"id": "spi_bus", "clk_pin": 18, "mosi_pin": 19}) + + +@pytest.fixture +def temp_model() -> Generator[Callable[..., EpaperModel]]: + """Register a throwaway EpaperModel for a test and remove it from the shared registry after.""" + created: list[EpaperModel] = [] + + def _make(name: str, **defaults: Any) -> EpaperModel: + model = EpaperModel(name, class_name="EPaperMono", **defaults) + created.append(model) + return model + + yield _make + for model in created: + MODELS.pop(model.name, None) + + +# --- border_waveform --------------------------------------------------------- + + +def test_border_waveform_default_mono( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """ssd1677 defaults border_waveform to 0x01.""" + _setup_esp32(set_core_config, set_component_config) + + result = CONFIG_SCHEMA(_ssd1677_config()) + + assert result[CONF_BORDER_WAVEFORM] == 0x01 + + +def test_border_waveform_default_gray4( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """The 4-level grayscale variant defaults border_waveform to 0x00, independently of mono.""" + _setup_esp32( + set_core_config, set_component_config, VARIANT_ESP32S3, "esp32-s3-devkitc-1" + ) + CORE.raw_config = {"psram": {}} + + result = CONFIG_SCHEMA( + {"id": "test_display", "model": "seeed-reterminal-sticky-gray4"} + ) + + assert result[CONF_BORDER_WAVEFORM] == 0x00 + + +def test_border_waveform_override( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """An explicit border_waveform overrides the model default.""" + _setup_esp32(set_core_config, set_component_config) + + result = CONFIG_SCHEMA(_ssd1677_config(border_waveform=0x1A)) + + assert result[CONF_BORDER_WAVEFORM] == 0x1A + + +def test_border_waveform_accepts_hex_string( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """border_waveform accepts a hex string like the YAML author would write.""" + _setup_esp32(set_core_config, set_component_config) + + result = CONFIG_SCHEMA(_ssd1677_config(border_waveform="0x1A")) + + assert result[CONF_BORDER_WAVEFORM] == 0x1A + + +def test_border_waveform_out_of_range( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """border_waveform rejects values that don't fit in a byte.""" + _setup_esp32(set_core_config, set_component_config) + + with pytest.raises(cv.Invalid): + CONFIG_SCHEMA(_ssd1677_config(border_waveform=0x100)) + + +def test_border_waveform_in_generated_init_sequence( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """The configured border_waveform byte reaches the generated init sequence. + + Command 0x3C (60) is followed by a length of 1 and the waveform byte. + """ + main_cpp = generate_main(component_config_path("ssd1677_border_waveform_test.yaml")) + + assert re.search(r"60,\s*1,\s*0x1A,", main_cpp) + + +# --- full_update_every / supports_partial_update ------------------------------ + + +def test_full_update_every_rejected_for_gray4( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """The gray4 driver's partial updates are black and white and flatten the whole + panel, so full_update_every > 1 is refused unless that is explicitly accepted.""" + _setup_esp32( + set_core_config, set_component_config, VARIANT_ESP32S3, "esp32-s3-devkitc-1" + ) + CORE.raw_config = {"psram": {}} + + with pytest.raises(cv.Invalid, match="monochrome_partial_updates: true"): + CONFIG_SCHEMA( + { + "id": "test_display", + "model": "seeed-reterminal-sticky-gray4", + "full_update_every": 5, + } + ) + + +def test_full_update_every_accepted_for_gray4_with_monochrome_partials( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """With the trade-off accepted, the gray4 driver takes full_update_every > 1.""" + _setup_esp32( + set_core_config, set_component_config, VARIANT_ESP32S3, "esp32-s3-devkitc-1" + ) + CORE.raw_config = {"psram": {}} + + result = CONFIG_SCHEMA( + { + "id": "test_display", + "model": "seeed-reterminal-sticky-gray4", + "full_update_every": 5, + "monochrome_partial_updates": True, + } + ) + + assert result["full_update_every"] == 5 + + +def test_monochrome_partial_updates_not_offered_for_mono_sticky( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """The option only exists where partial updates lose something.""" + _setup_esp32( + set_core_config, set_component_config, VARIANT_ESP32S3, "esp32-s3-devkitc-1" + ) + CORE.raw_config = {"psram": {}} + + with pytest.raises(cv.Invalid, match="monochrome_partial_updates"): + CONFIG_SCHEMA( + { + "id": "test_display", + "model": "seeed-reterminal-sticky", + "monochrome_partial_updates": True, + } + ) + + +def test_full_update_every_default_accepted_for_gray4( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """Leaving full_update_every at its default of 1 is fine for the gray4 driver.""" + _setup_esp32( + set_core_config, set_component_config, VARIANT_ESP32S3, "esp32-s3-devkitc-1" + ) + CORE.raw_config = {"psram": {}} + + result = CONFIG_SCHEMA( + {"id": "test_display", "model": "seeed-reterminal-sticky-gray4"} + ) + + assert result["full_update_every"] == 1 + + +def test_full_update_every_accepted_for_mono_sticky( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """The mono sibling model supports partial update, unaffected by the gray4 restriction.""" + _setup_esp32( + set_core_config, set_component_config, VARIANT_ESP32S3, "esp32-s3-devkitc-1" + ) + CORE.raw_config = {"psram": {}} + + result = CONFIG_SCHEMA( + { + "id": "test_display", + "model": "seeed-reterminal-sticky", + "full_update_every": 5, + } + ) + + assert result["full_update_every"] == 5 + + +# --- check_requirements ------------------------------------------------------- + + +def test_requirement_missing_raises( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """seeed-reterminal-sticky requires psram; without it, config validation fails.""" + _setup_esp32( + set_core_config, set_component_config, VARIANT_ESP32S3, "esp32-s3-devkitc-1" + ) + CORE.raw_config = {} + + with pytest.raises(cv.Invalid, match="requires component 'psram'"): + CONFIG_SCHEMA({"id": "test_display", "model": "seeed-reterminal-sticky"}) + + +def test_requirement_satisfied_does_not_raise( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """With psram present at the top level, seeed-reterminal-sticky validates.""" + _setup_esp32( + set_core_config, set_component_config, VARIANT_ESP32S3, "esp32-s3-devkitc-1" + ) + CORE.raw_config = {"psram": {}} + + result = CONFIG_SCHEMA({"id": "test_display", "model": "seeed-reterminal-sticky"}) + + assert result["model"] == "SEEED-RETERMINAL-STICKY" + + +def test_requirement_check_skipped_without_raw_config( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """With no raw_config (e.g. a schema invoked directly, as in these tests), the + requirement check is a no-op rather than a false failure.""" + _setup_esp32( + set_core_config, set_component_config, VARIANT_ESP32S3, "esp32-s3-devkitc-1" + ) + assert CORE.raw_config is None + + # Should not raise even though "psram" is required and nothing was configured. + CONFIG_SCHEMA({"id": "test_display", "model": "seeed-reterminal-sticky"}) + + +def test_requirement_missing_multiple_pluralised( + temp_model: Callable[..., EpaperModel], +) -> None: + """The error message pluralises "component(s)" and lists every missing one.""" + model = temp_model("test-multi-requirement", requires={"aaa", "bbb"}) + CORE.raw_config = {} + + with pytest.raises(cv.Invalid, match="requires components 'aaa', 'bbb'"): + model.check_requirements() + + +# --- width_multiple ----------------------------------------------------------- + + +@pytest.mark.parametrize("width", [804, 801]) +def test_gray4_width_not_multiple_of_8_rejected( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], + width: int, +) -> None: + """The gray4 plane split reads two whole buffer bytes per plane byte, so width must be a multiple of 8.""" + _setup_esp32( + set_core_config, set_component_config, VARIANT_ESP32S3, "esp32-s3-devkitc-1" + ) + + with pytest.raises(cv.Invalid, match="multiple of 8"): + CONFIG_SCHEMA( + { + "id": "test_display", + "model": "seeed-reterminal-sticky-gray4", + "dimensions": {"width": width, "height": 480}, + } + ) + + +def test_gray4_width_multiple_of_8_accepted( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + _setup_esp32( + set_core_config, set_component_config, VARIANT_ESP32S3, "esp32-s3-devkitc-1" + ) + + result = CONFIG_SCHEMA( + { + "id": "test_display", + "model": "seeed-reterminal-sticky-gray4", + "dimensions": {"width": 808, "height": 480}, + } + ) + + assert result["dimensions"]["width"] == 808 + + +def test_mono_ssd1677_accepts_any_width( + set_core_config: SetCoreConfigCallable, + set_component_config: Callable[[str, Any], None], +) -> None: + """The width restriction applies only to the gray4 model.""" + _setup_esp32(set_core_config, set_component_config) + + result = CONFIG_SCHEMA(_ssd1677_config(dimensions={"width": 204, "height": 200})) + + assert result["dimensions"]["width"] == 204 + + +# --- extend(class_name=...) -------------------------------------------------- + + +def test_gray4_code_generation( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """seeed-reterminal-sticky-gray4 generates the EPaperSSD1677Gray4 driver, not EPaperMono.""" + main_cpp = generate_main(component_config_path("ssd1677_gray4_test.yaml")) + + assert "epaper_spi::EPaperSSD1677Gray4" in main_cpp + assert "epaper_spi::EPaperMono" not in main_cpp diff --git a/tests/components/epaper_spi/common.h b/tests/components/epaper_spi/common.h index 5ac12afa6a6..f21e674fde2 100644 --- a/tests/components/epaper_spi/common.h +++ b/tests/components/epaper_spi/common.h @@ -2,6 +2,9 @@ #include +#include +#include + #include "esphome/components/spi/spi.h" #include "esphome/core/hal.h" @@ -47,4 +50,51 @@ class RecordingPin : public GPIOPin { bool level{true}; }; +/// SPI delegate that records what reaches the bus, filing each payload under the command it +/// followed; commands are the bytes written while D/C is low. Optionally burns wall-clock time on +/// each data row, so a transfer can be driven past its yield deadline. +class RecordingDelegate : public spi::SPIDelegate { + public: + explicit RecordingDelegate(const RecordingPin *dc, uint32_t row_transfer_ms = 0) + : dc_(dc), row_transfer_ms_(row_transfer_ms) {} + + uint8_t transfer(uint8_t data) override { + this->record_(&data, 1); + return 0; + } + void write_array(const uint8_t *ptr, size_t length) override { + this->record_(ptr, length); + if (this->dc_->level && this->row_transfer_ms_ != 0) { + const uint32_t until = millis() + this->row_transfer_ms_; + while (millis() < until) { + } + } + } + + void clear() { + this->commands.clear(); + this->data.clear(); + } + + std::vector commands; + std::map> data; + + protected: + void record_(const uint8_t *ptr, size_t length) { + if (!this->dc_->level) { + for (size_t i = 0; i != length; i++) { + this->commands.push_back(ptr[i]); + this->last_command_ = ptr[i]; + } + return; + } + auto &payload = this->data[this->last_command_]; + payload.insert(payload.end(), ptr, ptr + length); + } + + const RecordingPin *dc_; + uint32_t row_transfer_ms_; + uint8_t last_command_{0}; +}; + } // namespace esphome::epaper_spi::testing diff --git a/tests/components/epaper_spi/display/test_ssd1677_gray4_transfer.cpp b/tests/components/epaper_spi/display/test_ssd1677_gray4_transfer.cpp new file mode 100644 index 00000000000..1a66efe9839 --- /dev/null +++ b/tests/components/epaper_spi/display/test_ssd1677_gray4_transfer.cpp @@ -0,0 +1,298 @@ +#include + +#include +#include + +#include "../common.h" +#include "esphome/components/epaper_spi/epaper_spi_ssd1677_gray4.h" + +namespace esphome::epaper_spi::testing { + +class TestableSSD1677Gray4 : public EPaperSSD1677Gray4 { + public: + TestableSSD1677Gray4(uint16_t width, uint16_t height) : EPaperSSD1677Gray4("test", width, height, nullptr, 0) {} + + void install(spi::SPIDelegate *delegate) { + this->delegate_ = delegate; + this->set_dc_pin(&this->dc); + ASSERT_TRUE(this->init_buffer_(this->buffer_length_)); + } + + /// As configured with monochrome_partial_updates: full_update_every > 1. + void install_with_partials(spi::SPIDelegate *delegate) { + this->install(delegate); + this->set_full_update_every(5); + this->init_comparison_frame_(); + ASSERT_TRUE(this->sent_.is_valid()); + } + + /// What the base class would decide; 0 means the next push is a full one. + void set_update_count(uint8_t count) { this->update_count_ = count; } + + /// Pretend only this rectangle changed. + void set_dirty(uint16_t x_low, uint16_t y_low, uint16_t x_high, uint16_t y_high) { + this->x_low_ = x_low; + this->y_low_ = y_low; + this->x_high_ = x_high; + this->y_high_ = y_high; + } + + /// Both planes of one push; returns how many calls it took. + int run_push() { + int calls = 1; + while (!this->transfer_data()) + calls++; + return calls; + } + + using EPaperSSD1677Gray4::refresh_screen; + using EPaperSSD1677Gray4::transfer_data; + + RecordingPin dc; +}; + +using Bytes = std::vector; + +namespace { + +/// A gray that lands squarely on each of the four levels. +Color color_for_level(uint8_t level) { + static const uint8_t GRAYS[4] = {0, 64, 128, 255}; + const uint8_t v = GRAYS[level]; + return Color(v, v, v); +} + +void draw_row(TestableSSD1677Gray4 &display, int y, const std::vector &levels) { + for (size_t x = 0; x != levels.size(); x++) + display.draw_pixel_at((int) x, y, color_for_level(levels[x])); +} + +} // namespace + +/// Each pixel's 2-bit level is split across the RAM planes: the high bit to 0x24, the low bit to +/// 0x26, both inverted because the four-level waveform reads 1 as white. +TEST(EPaperSSD1677Gray4, SplitsEachLevelAcrossBothPlanes) { + TestableSSD1677Gray4 display(8, 1); + RecordingDelegate bus(&display.dc); + display.install(&bus); + + draw_row(display, 0, {0, 1, 2, 3, 0, 1, 2, 3}); + display.run_push(); + + // levels 0 1 2 3 0 1 2 3 + // high bit 0 0 1 1 0 0 1 1 = 0x33, inverted 0xCC + // low bit 0 1 0 1 0 1 0 1 = 0x55, inverted 0xAA + EXPECT_EQ(bus.data[0x24], (Bytes{0xCC})); + EXPECT_EQ(bus.data[0x26], (Bytes{0xAA})); +} + +/// Two buffer bytes (4 pixels each) make one plane byte (8 pixels), leftmost pixel in the most +/// significant bit. An asymmetric row catches a swapped pair or reversed bit order. +TEST(EPaperSSD1677Gray4, PacksPixelsLeftmostFirstAcrossSourceBytes) { + TestableSSD1677Gray4 display(16, 1); + RecordingDelegate bus(&display.dc); + display.install(&bus); + + draw_row(display, 0, {3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}); + display.run_push(); + + // high bits: pixel 0 (3) and pixel 15 (2) -> 0x80 0x01, inverted 0x7F 0xFE + // low bits: pixel 0 (3) only -> 0x80 0x00, inverted 0x7F 0xFF + EXPECT_EQ(bus.data[0x24], (Bytes{0x7F, 0xFE})); + EXPECT_EQ(bus.data[0x26], (Bytes{0x7F, 0xFF})); +} + +/// The high-bit plane goes out first, each plane exactly once per push. +TEST(EPaperSSD1677Gray4, WritesTheHighBitPlaneBeforeTheLowBitPlane) { + TestableSSD1677Gray4 display(8, 1); + RecordingDelegate bus(&display.dc); + display.install(&bus); + + display.run_push(); + + const auto &cmds = bus.commands; + ASSERT_EQ(std::count(cmds.begin(), cmds.end(), 0x24), 1); + ASSERT_EQ(std::count(cmds.begin(), cmds.end(), 0x26), 1); + EXPECT_LT(std::find(cmds.begin(), cmds.end(), 0x24) - cmds.begin(), + std::find(cmds.begin(), cmds.end(), 0x26) - cmds.begin()); +} + +/// A push that yields partway through must resume the right plane at the right row. +TEST(EPaperSSD1677Gray4, ResumesBothPlanesAfterYielding) { + TestableSSD1677Gray4 display(8, 4); + RecordingDelegate bus(&display.dc, 6); // two rows exceed MAX_TRANSFER_TIME + display.install(&bus); + + draw_row(display, 0, {0, 0, 0, 0, 0, 0, 0, 0}); + draw_row(display, 1, {1, 1, 1, 1, 1, 1, 1, 1}); + draw_row(display, 2, {2, 2, 2, 2, 2, 2, 2, 2}); + draw_row(display, 3, {3, 3, 3, 3, 3, 3, 3, 3}); + const int calls = display.run_push(); + + EXPECT_GT(calls, 2) << "the transfer never yielded, so this test proves nothing"; + // rows at levels 0..3: high bits 0 0 1 1, low bits 0 1 0 1, each inverted across the row + EXPECT_EQ(bus.data[0x24], (Bytes{0xFF, 0xFF, 0x00, 0x00})); + EXPECT_EQ(bus.data[0x26], (Bytes{0xFF, 0x00, 0xFF, 0x00})); +} + +/// Without partial updates enabled (the default) every refresh is the four-level sequence, even if +/// the update count says otherwise. +TEST(EPaperSSD1677Gray4, WithoutPartialUpdatesEveryRefreshIsFourLevel) { + TestableSSD1677Gray4 display(8, 1); + RecordingDelegate bus(&display.dc); + display.install(&bus); + + display.set_update_count(1); + display.refresh_screen(true); + + EXPECT_EQ(bus.commands, (Bytes{0x1A, 0x22, 0x20})); + EXPECT_EQ(bus.data[0x1A], (Bytes{0x67, 0x00})); + EXPECT_EQ(bus.data[0x22], (Bytes{0xD7})); +} + +// --- With monochrome partial updates ------------------------------------------------------------ + +/// A full update is still four-level. It also records, as the frame the next partial update +/// compares against, what the panel shows in black-and-white terms: the high bit of each level. +TEST(EPaperSSD1677Gray4, FullPushRecordsTheHighBitsForTheNextPartial) { + TestableSSD1677Gray4 display(8, 1); + RecordingDelegate bus(&display.dc); + display.install_with_partials(&bus); + + draw_row(display, 0, {0, 1, 2, 3, 0, 1, 2, 3}); + display.set_update_count(0); + display.run_push(); + EXPECT_EQ(bus.data[0x24], (Bytes{0xCC})) << "full update is no longer the four-level split"; + EXPECT_EQ(bus.data[0x26], (Bytes{0xAA})); + bus.clear(); + + // Nothing changed: old and new planes must match, or the partial drives every pixel. + display.set_update_count(1); + display.run_push(); + EXPECT_EQ(bus.data[0x26], (Bytes{0x33})) << "comparison frame is not the high bits"; + EXPECT_EQ(bus.data[0x24], (Bytes{0x33})); +} + +/// A partial update sends the comparison frame to 0x26 and the new frame's high bits to 0x24, +/// not inverted (it runs the black-and-white waveform), over the whole panel. +TEST(EPaperSSD1677Gray4, PartialPushSendsTheHighBitsInBlackAndWhite) { + TestableSSD1677Gray4 display(16, 2); + RecordingDelegate bus(&display.dc); + display.install_with_partials(&bus); + + draw_row(display, 0, {0, 1, 2, 3, 0, 1, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3}); + draw_row(display, 1, {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + display.set_update_count(0); + display.run_push(); + bus.clear(); + + draw_row(display, 1, {3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + display.set_dirty(0, 1, 8, 2); // only the start of the second row changed + display.set_update_count(1); + display.run_push(); + + EXPECT_EQ(bus.data[0x26], (Bytes{0x33, 0xFF, 0x00, 0x00})) << "old plane is not the frame on the panel"; + EXPECT_EQ(bus.data[0x24], (Bytes{0x33, 0xFF, 0xF0, 0x00})) << "new plane is not the whole frame's high bits"; +} + +/// The new plane of a partial update is built a row at a time; a push that yields partway through +/// must resume at the right row. +TEST(EPaperSSD1677Gray4, PartialPushResumesAfterYielding) { + TestableSSD1677Gray4 display(8, 4); + RecordingDelegate bus(&display.dc, 6); // two rows exceed MAX_TRANSFER_TIME + display.install_with_partials(&bus); + + display.set_update_count(0); + display.run_push(); + bus.clear(); + + // The buffer starts white; darken all of row 0 and the right half of row 2 + draw_row(display, 0, {0, 0, 0, 0, 0, 0, 0, 0}); + draw_row(display, 2, {3, 3, 3, 3, 0, 0, 0, 0}); + display.set_update_count(1); + const int calls = display.run_push(); + + EXPECT_GT(calls, 2) << "the transfer never yielded, so this test proves nothing"; + EXPECT_EQ(bus.data[0x26], (Bytes{0xFF, 0xFF, 0xFF, 0xFF})); + EXPECT_EQ(bus.data[0x24], (Bytes{0x00, 0xFF, 0xF0, 0xFF})); +} + +/// Regression test: a full update requested while a partial one is being sent must not switch the +/// push to the four-level transfer halfway, which misread the partial's progress and never finished. +TEST(EPaperSSD1677Gray4, FullUpdateRequestDuringAPartialPushWaitsForTheNextUpdate) { + TestableSSD1677Gray4 display(8, 4); + RecordingDelegate bus(&display.dc, 6); // two rows exceed MAX_TRANSFER_TIME + display.install_with_partials(&bus); + + display.set_update_count(0); + display.run_push(); + bus.clear(); + + draw_row(display, 0, {0, 0, 0, 0, 0, 0, 0, 0}); + display.set_update_count(1); + ASSERT_FALSE(display.transfer_data()); + display.request_full_update(); + int calls = 1; + while (!display.transfer_data()) + ASSERT_LT(++calls, 20) << "partial push never finished"; + + EXPECT_EQ(bus.data[0x26], (Bytes{0xFF, 0xFF, 0xFF, 0xFF})); + EXPECT_EQ(bus.data[0x24], (Bytes{0x00, 0xFF, 0xFF, 0xFF})); + bus.clear(); + display.refresh_screen(true); + EXPECT_EQ(bus.data[0x22], (Bytes{0xFF})) << "refresh does not match the partial data sent"; +} + +/// The four-level refresh follows a reset, which loses controller RAM, so it must send the whole +/// panel even when partial updates are enabled but the comparison frame could not be allocated. +TEST(EPaperSSD1677Gray4, FourLevelPushCoversTheWholePanelWithoutAComparisonFrame) { + TestableSSD1677Gray4 display(16, 2); + RecordingDelegate bus(&display.dc); + display.install(&bus); + display.set_full_update_every(5); // partial updates on, but no comparison frame + + display.set_dirty(8, 1, 16, 2); + display.set_update_count(0); + display.run_push(); + + EXPECT_EQ(bus.data[0x24].size(), 4u) << "four-level update did not send the whole new plane"; + EXPECT_EQ(bus.data[0x26].size(), 4u) << "four-level update did not send the whole old plane"; +} + +/// A full update resets the controller, which does not keep RAM, so even when only part of the +/// frame changed it must send the whole panel. +TEST(EPaperSSD1677Gray4, FullPushWithPartialsEnabledCoversTheWholePanel) { + TestableSSD1677Gray4 display(16, 2); + RecordingDelegate bus(&display.dc); + display.install_with_partials(&bus); + + display.set_dirty(8, 1, 16, 2); + display.set_update_count(0); + display.run_push(); + + EXPECT_EQ(bus.data[0x24].size(), 4u) << "full update did not send the whole new plane"; + EXPECT_EQ(bus.data[0x26].size(), 4u) << "full update did not send the whole old plane"; +} + +/// The refresh matches what was sent: black-and-white for a partial update, four-level for a full. +TEST(EPaperSSD1677Gray4, PartialRefreshIsBlackAndWhiteAndFullIsFourLevel) { + TestableSSD1677Gray4 display(8, 1); + RecordingDelegate bus(&display.dc); + display.install_with_partials(&bus); + + display.set_update_count(1); + display.refresh_screen(true); + EXPECT_EQ(bus.commands, (Bytes{0x3C, 0x22, 0x20})); + EXPECT_EQ(bus.data[0x22], (Bytes{0xFF})) << "partial update did not use the black-and-white waveform"; + // The model's border setting is right for the four-level waveform only; under this one it + // would drive the border black on every partial. + EXPECT_EQ(bus.data[0x3C], (Bytes{0x01})) << "partial update did not switch the border to LUT1"; + bus.clear(); + + display.set_update_count(0); + display.refresh_screen(false); + EXPECT_EQ(bus.data[0x22], (Bytes{0xD7})) << "full update did not use the four-level waveform"; + EXPECT_EQ(bus.data.count(0x3C), 0u) << "full update overrode the model's border setting"; +} + +} // namespace esphome::epaper_spi::testing diff --git a/tests/components/epaper_spi/display/test_ssd1677_transfer.cpp b/tests/components/epaper_spi/display/test_ssd1677_transfer.cpp new file mode 100644 index 00000000000..cad853b1973 --- /dev/null +++ b/tests/components/epaper_spi/display/test_ssd1677_transfer.cpp @@ -0,0 +1,233 @@ +#include + +#include +#include + +#include "../common.h" +#include "esphome/components/epaper_spi/epaper_spi_ssd1677.h" + +namespace esphome::epaper_spi::testing { + +class TestableSSD1677 : public EPaperSSD1677 { + public: + TestableSSD1677(uint16_t width, uint16_t height) : EPaperSSD1677("test", width, height, nullptr, 0) {} + + void install(spi::SPIDelegate *delegate, uint8_t full_update_every) { + this->delegate_ = delegate; + this->set_dc_pin(&this->dc); + this->set_reset_pin(&this->reset_pin); + ASSERT_TRUE(this->init_buffer_(this->buffer_length_)); + this->set_full_update_every(full_update_every); + this->init_comparison_frame_(); + } + + bool has_comparison_frame() const { return this->sent_.is_valid(); } + + void set_frame(std::initializer_list bytes) { + size_t i = 0; + for (const uint8_t byte : bytes) + this->buffer_[i++] = byte; + } + + /// Fill the frame with a byte pattern that differs per seed; returns it. + std::vector set_pattern(uint8_t seed) { + std::vector frame; + for (size_t i = 0; i != this->buffer_length_; i++) { + frame.push_back((uint8_t) (seed + i * 7)); + this->buffer_[i] = frame.back(); + } + return frame; + } + + /// What the base class would decide; 0 means the next push is a full one. + void set_update_count(uint8_t count) { this->update_count_ = count; } + + /// Pretend only this rectangle changed. + void set_dirty(uint16_t x_low, uint16_t y_low, uint16_t x_high, uint16_t y_high) { + this->x_low_ = x_low; + this->y_low_ = y_low; + this->x_high_ = x_high; + this->y_high_ = y_high; + } + + /// One call into the transfer; false while there is more to send. + bool step() { return this->transfer_data(); } + + /// Both planes of one push; returns how many calls it took. + int run_push() { + int calls = 1; + while (!this->transfer_data()) + calls++; + return calls; + } + + /// Run the UPDATE state, with nothing drawn, and report whether a push follows. + bool run_update_state() { + this->set_auto_clear(false); + this->set_dirty(this->width_, this->height_, 0, 0); + this->state_ = EPaperState::UPDATE; + this->process_state_(); + return this->state_ == EPaperState::RESET; + } + uint8_t update_count() const { return this->update_count_; } + + bool reset_in(EPaperState state) { + this->state_ = state; + return this->reset(); + } + + RecordingPin dc; + RecordingPin reset_pin; +}; + +using Bytes = std::vector; + +/// A full push ignores the old-image plane, and on the first push after boot the comparison frame +/// holds nothing real yet, so the new frame goes to both planes. +TEST(EPaperSSD1677, FullPushSendsTheNewFrameToBothPlanes) { + TestableSSD1677 display(16, 2); + RecordingDelegate bus(&display.dc); + display.install(&bus, 5); + + display.set_frame({0x0F, 0xF0, 0x3C, 0xC3}); + display.set_update_count(0); + display.run_push(); + + EXPECT_EQ(bus.data[0x26], (Bytes{0x0F, 0xF0, 0x3C, 0xC3})); + EXPECT_EQ(bus.data[0x24], (Bytes{0x0F, 0xF0, 0x3C, 0xC3})); +} + +/// Regression test. +/// +/// A partial refresh drives every pixel from the pair (0x26 = the image on the panel, 0x24 = the +/// new image), across the whole panel whatever RAM window was written. The controller does not +/// keep its RAM intact between updates, so sending only the changed window of 0x24 - and 0x26 once +/// - leaves the pair wrong outside that window: unchanged pixels get driven on every partial and +/// wash out. Both planes must go out whole, 0x26 holding the frame actually on the panel. +TEST(EPaperSSD1677, PartialPushComparesAgainstTheFrameOnThePanel) { + TestableSSD1677 display(16, 2); + RecordingDelegate bus(&display.dc); + display.install(&bus, 5); + + display.set_frame({0x0F, 0xF0, 0x3C, 0xC3}); + display.set_update_count(0); + display.run_push(); + bus.clear(); + + display.set_frame({0x0F, 0xF0, 0x3C, 0x00}); + display.set_dirty(8, 1, 16, 2); // only the last byte changed + display.set_update_count(1); + display.run_push(); + + EXPECT_EQ(bus.data[0x26], (Bytes{0x0F, 0xF0, 0x3C, 0xC3})) << "old plane is not the frame on the panel"; + EXPECT_EQ(bus.data[0x24], (Bytes{0x0F, 0xF0, 0x3C, 0x00})) << "new plane is not the whole new frame"; + // The RAM window, set once per plane, must span the panel too, not the changed rectangle. + EXPECT_EQ(bus.data[0x44], (Bytes{0, 0, 15, 0, 0, 0, 15, 0})) << "x window is not the whole panel"; + EXPECT_EQ(bus.data[0x45], (Bytes{0, 0, 1, 0, 0, 0, 1, 0})) << "y window is not the whole panel"; +} + +/// The comparison frame must record the bytes that went to 0x24, not whatever the buffer holds +/// later: LVGL can draw into the buffer while a push is in progress. Here the buffer changes +/// between the two planes of a push; the next push must compare against what was actually sent. +TEST(EPaperSSD1677, ComparisonFrameIsWhatWasSentNotTheBuffer) { + TestableSSD1677 display(16, 2); + RecordingDelegate bus(&display.dc); + display.install(&bus, 5); + + display.set_frame({0x11, 0x11, 0x11, 0x11}); + display.set_update_count(0); + display.run_push(); + + display.set_frame({0x22, 0x22, 0x22, 0x22}); + display.set_update_count(1); + ASSERT_FALSE(display.step()) << "expected the old plane to go out on its own first"; + display.set_frame({0x33, 0x33, 0x33, 0x33}); // drawn mid-push, before the new plane + while (!display.step()) { + } + ASSERT_EQ(bus.data[0x24].size(), 8u); + EXPECT_EQ(Bytes(bus.data[0x24].begin() + 4, bus.data[0x24].end()), (Bytes{0x33, 0x33, 0x33, 0x33})); + bus.clear(); + + display.set_frame({0x44, 0x44, 0x44, 0x44}); + display.set_update_count(2); + display.run_push(); + + EXPECT_EQ(bus.data[0x26], (Bytes{0x33, 0x33, 0x33, 0x33})) << "old plane is not what was last sent"; +} + +/// Two full planes can take several loop iterations to send; each resumed call must continue the +/// right plane at the right byte. Planes go out in runs sized to the time slice, not row by row. +TEST(EPaperSSD1677, ResumesTheRightPlaneAfterYielding) { + // 400x100 is 5000 bytes per plane: two runs at the default 2 MHz bus + TestableSSD1677 display(400, 100); + RecordingDelegate bus(&display.dc, MAX_TRANSFER_TIME + 1); // every run overruns the time slice + display.install(&bus, 5); + + const auto old_frame = display.set_pattern(1); + display.set_update_count(0); + display.run_push(); + bus.clear(); + + const auto new_frame = display.set_pattern(2); + display.set_update_count(1); + const int calls = display.run_push(); + + EXPECT_EQ(calls, 4) << "expected two runs per plane, one per call"; + EXPECT_EQ(bus.data[0x26], old_frame); + EXPECT_EQ(bus.data[0x24], new_frame); +} + +/// A requested full update takes effect when the next update starts, and pushes the whole panel +/// even if nothing was drawn. +TEST(EPaperSSD1677, RequestedFullUpdateAppliesWhenTheNextUpdateStarts) { + TestableSSD1677 display(16, 2); + RecordingDelegate bus(&display.dc); + display.install(&bus, 5); + + display.set_update_count(3); + EXPECT_FALSE(display.run_update_state()) << "an update with nothing drawn should not push"; + + display.request_full_update(); + EXPECT_EQ(display.update_count(), 3) << "request changed the update in progress"; + EXPECT_TRUE(display.run_update_state()) << "requested full update did not push"; + EXPECT_EQ(display.update_count(), 0) << "requested update is not a full one"; + + display.set_update_count(3); + EXPECT_FALSE(display.run_update_state()) << "request was applied more than once"; +} + +/// Nothing a partial needs lives in controller RAM any more, so a partial push skips the reset +/// altogether; a full one still gets the hardware pulse and the software reset. +TEST(EPaperSSD1677, PartialPushSkipsTheResetAndAFullPushKeepsIt) { + TestableSSD1677 display(16, 2); + RecordingDelegate bus(&display.dc); + display.install(&bus, 5); + + display.set_update_count(1); + EXPECT_TRUE(display.reset_in(EPaperState::RESET)) << "partial push waited on a reset"; + EXPECT_TRUE(display.reset_pin.level) << "partial push pulsed the reset pin"; + EXPECT_TRUE(bus.commands.empty()) << "partial push sent a software reset"; + + display.set_update_count(0); + EXPECT_FALSE(display.reset_in(EPaperState::RESET)); + EXPECT_FALSE(display.reset_pin.level) << "full push did not pulse the reset pin"; + EXPECT_TRUE(display.reset_in(EPaperState::RESET_END)); + EXPECT_TRUE(display.reset_pin.level); + EXPECT_EQ(bus.commands, (Bytes{0x12})) << "full push did not send a software reset"; +} + +/// With every update a full one nothing is ever compared against 0x26, so no comparison frame is +/// allocated and the transfer is EPaperMono's. +TEST(EPaperSSD1677, NoComparisonFrameWhenEveryUpdateIsFull) { + TestableSSD1677 display(16, 2); + RecordingDelegate bus(&display.dc); + display.install(&bus, 1); + + EXPECT_FALSE(display.has_comparison_frame()); + display.set_frame({0x0F, 0xF0, 0x3C, 0xC3}); + display.set_update_count(0); + display.run_push(); + EXPECT_EQ(bus.data[0x24], (Bytes{0x0F, 0xF0, 0x3C, 0xC3})); +} + +} // namespace esphome::epaper_spi::testing diff --git a/tests/components/epaper_spi/test.esp32-s3-idf.yaml b/tests/components/epaper_spi/test.esp32-s3-idf.yaml index 602aeb8d0e1..1678f463311 100644 --- a/tests/components/epaper_spi/test.esp32-s3-idf.yaml +++ b/tests/components/epaper_spi/test.esp32-s3-idf.yaml @@ -1,6 +1,14 @@ packages: spi: !include ../../test_build_components/common/spi/esp32-s3-idf.yaml +psram: + mode: octal + +esphome: + on_boot: + then: + - epaper_spi.full_update_next: epaper_partial + display: - platform: epaper_spi spi_id: spi_bus @@ -85,12 +93,24 @@ display: busy_pin: 37 enable_pin: 39 - platform: epaper_spi + id: epaper_partial model: seeed-ee04-mono-4.26 full_update_every: 10 # Override pins to avoid conflict with other display configs busy_pin: 43 dc_pin: 42 + # Seeed reTerminal Sticky, four-level grayscale (800x480, SSD1677) + # dc_pin/reset_pin overridden to avoid conflict with other display configs + # full_update_every is not supported by this model, so left at its default of 1 + - platform: epaper_spi + model: seeed-reterminal-sticky-gray4 + dc_pin: 45 + reset_pin: 9 + lambda: |- + it.filled_rectangle(0, 0, it.get_width(), it.get_height(), Color(170, 170, 170)); + it.circle(it.get_width() / 2, it.get_height() / 2, 100, Color::BLACK); + # WeAct 2.13" 3-color e-paper (122x250, SSD1680) - platform: epaper_spi spi_id: spi_bus From eb9aae04d0e9775f9d6d362652de7e14b5b4f255 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 12:51:03 +0200 Subject: [PATCH 034/133] [rp2] Bump arduino-pico framework to 6.1.1 (#19881) --- esphome/components/rp2/__init__.py | 11 +++--- esphome/components/rp2/boards.py | 37 +++++++++++++++++++ .../components/rp2040_ble/btstack_memory.cpp | 2 +- esphome/core/defines.h | 2 +- platformio.ini | 2 +- 5 files changed, 46 insertions(+), 8 deletions(-) diff --git a/esphome/components/rp2/__init__.py b/esphome/components/rp2/__init__.py index a1bbf6a3d66..23346ca6f4b 100644 --- a/esphome/components/rp2/__init__.py +++ b/esphome/components/rp2/__init__.py @@ -197,20 +197,21 @@ def _parse_platform_version(value: Any) -> str: # The default/recommended arduino framework version # - https://github.com/earlephilhower/arduino-pico/releases -RECOMMENDED_ARDUINO_FRAMEWORK_VERSION = cv.Version(6, 1, 0) +RECOMMENDED_ARDUINO_FRAMEWORK_VERSION = cv.Version(6, 1, 1) # The raspberrypi platform version to use for arduino frameworks # - https://github.com/maxgerhardt/platform-raspberrypi/tags -# develop-branch commit carrying the arduino-pico 6.1.0 update and the board -# JSON files it adds; replace with a release tag when one is cut +# develop-branch commit carrying the arduino-pico 6.1.0 update; it predates +# 6.1.1, so the boards that release adds have no board JSON on it yet; replace +# with a release tag when one is cut RECOMMENDED_ARDUINO_PLATFORM_VERSION = "5d4561a05e3b212660ac6fdd3fbfb328d1988aa1" def _arduino_check_versions(value: ConfigType) -> ConfigType: value = value.copy() lookups = { - "dev": (cv.Version(6, 1, 0), "https://github.com/earlephilhower/arduino-pico"), - "latest": (cv.Version(6, 1, 0), None), + "dev": (cv.Version(6, 1, 1), "https://github.com/earlephilhower/arduino-pico"), + "latest": (cv.Version(6, 1, 1), None), "recommended": (RECOMMENDED_ARDUINO_FRAMEWORK_VERSION, None), } diff --git a/esphome/components/rp2/boards.py b/esphome/components/rp2/boards.py index a9ce11c33dc..08536fd0537 100644 --- a/esphome/components/rp2/boards.py +++ b/esphome/components/rp2/boards.py @@ -1442,6 +1442,19 @@ RP2_BOARD_PINS = { "SS": 5, "TX": 0, }, + "waveshare_rp2350_core": { + "LED": 39, + "MISO": 16, + "MOSI": 19, + "RX": 1, + "SCK": 18, + "SCL": 9, + "SCL1": 11, + "SDA": 8, + "SDA1": 10, + "SS": 17, + "TX": 0, + }, "waveshare_rp2350_lcd_0_96": { "MISO": 16, "MOSI": 19, @@ -1479,6 +1492,18 @@ RP2_BOARD_PINS = { "SS": 17, "TX": 0, }, + "waveshare_rp2350_poe_eth": { + "MISO": 16, + "MOSI": 19, + "RX": 1, + "SCK": 18, + "SCL": 5, + "SCL1": 27, + "SDA": 4, + "SDA1": 26, + "SS": 17, + "TX": 0, + }, "waveshare_rp2350_zero": { "MISO": 4, "MOSI": 3, @@ -2286,6 +2311,12 @@ BOARDS = { "mcu": "rp2040", "max_pin": 29, }, + "waveshare_rp2350_core": { + "name": "Waveshare RP2350 Core", + "mcu": "rp2350", + "max_pin": 47, + "die": "B", + }, "waveshare_rp2350_lcd_0_96": { "name": "Waveshare RP2350 LCD 0.96", "mcu": "rp2350", @@ -2304,6 +2335,12 @@ BOARDS = { "max_pin": 29, "die": "A", }, + "waveshare_rp2350_poe_eth": { + "name": "Waveshare RP2350-POE-ETH", + "mcu": "rp2350", + "max_pin": 29, + "die": "A", + }, "waveshare_rp2350_zero": { "name": "Waveshare RP2350 Zero", "mcu": "rp2350", diff --git a/esphome/components/rp2040_ble/btstack_memory.cpp b/esphome/components/rp2040_ble/btstack_memory.cpp index 699555f623f..adf45eeaf7f 100644 --- a/esphome/components/rp2040_ble/btstack_memory.cpp +++ b/esphome/components/rp2040_ble/btstack_memory.cpp @@ -20,7 +20,7 @@ namespace esphome::rp2040_ble { namespace { -// Pinned against arduino-pico 6.1.0's prebuilt archives: a framework bump (or +// Pinned against arduino-pico 6.1.1's prebuilt archives: a framework bump (or // a changed ENABLE_* macro) shifting the struct layout must fail the build // here, not overrun the pool blocks at runtime. Sizes differ per core // architecture (measured from each archive's own storage symbols). GCC only: diff --git a/esphome/core/defines.h b/esphome/core/defines.h index da03f0e3d16..115bae96c6e 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -546,7 +546,7 @@ // rp2/__init__.py codegen also defines USE_RP2040 as a back-compat alias // for external custom components that may still test for it. #ifdef USE_RP2 -#define USE_ARDUINO_VERSION_CODE VERSION_CODE(6, 1, 0) +#define USE_ARDUINO_VERSION_CODE VERSION_CODE(6, 1, 1) #define USE_RP2_CRASH_HANDLER #define USE_HTTP_REQUEST_RESPONSE #define USE_I2C diff --git a/platformio.ini b/platformio.ini index 102c7973b97..dd2a5724935 100644 --- a/platformio.ini +++ b/platformio.ini @@ -207,7 +207,7 @@ platform = https://github.com/maxgerhardt/platform-raspberrypi.git#5d4561a05e3b2 platform_packages = ; The framework-arduinopico package is no longer published to the PlatformIO ; registry, so install the framework straight from the GitHub release - earlephilhower/framework-arduinopico@https://github.com/earlephilhower/arduino-pico/releases/download/6.1.0/rp2040-6.1.0.zip + earlephilhower/framework-arduinopico@https://github.com/earlephilhower/arduino-pico/releases/download/6.1.1/rp2040-6.1.1.zip framework = arduino lib_deps = From 9b9d548ad5d5857b157db53d51173b6e6a65af2b Mon Sep 17 00:00:00 2001 From: Roman Priesol Date: Tue, 29 Sep 2026 13:56:34 +0200 Subject: [PATCH 035/133] [speaker] Skip a playlist item that fails instead of retrying it forever (#19862) --- .../media_player/speaker_media_player.cpp | 26 +++++++++++++++++-- .../media_player/speaker_media_player.h | 4 +++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/esphome/components/speaker/media_player/speaker_media_player.cpp b/esphome/components/speaker/media_player/speaker_media_player.cpp index 9ce50d7b762..06525235d47 100644 --- a/esphome/components/speaker/media_player/speaker_media_player.cpp +++ b/esphome/components/speaker/media_player/speaker_media_player.cpp @@ -154,6 +154,7 @@ void SpeakerMediaPlayer::watch_media_commands_() { // Ensure the loaded next item doesn't start playing, clear the queue, start the file, and unpause this->cancel_timeout("next_ann"); this->announcement_playlist_.clear(); + this->announcement_item_failed_ = false; if (media_command.file.has_value()) { this->announcement_pipeline_->start_file(playlist_item.file.value()); } else if (media_command.url.has_value()) { @@ -167,6 +168,7 @@ void SpeakerMediaPlayer::watch_media_commands_() { // Ensure the loaded next item doesn't start playing, clear the queue, start the file, and unpause this->cancel_timeout("next_media"); this->media_playlist_.clear(); + this->media_item_failed_ = false; if (this->is_paused_) { // If paused, stop the media pipeline and unpause it after confirming its stopped. This avoids playing a // short segment of the paused file before starting the new one. @@ -236,6 +238,7 @@ void SpeakerMediaPlayer::watch_media_commands_() { if (this->announcement_pipeline_ != nullptr) { this->cancel_timeout("next_ann"); this->announcement_playlist_.clear(); + this->announcement_item_failed_ = false; this->announcement_pipeline_->stop(); this->unpause_announcement_remaining_ = 3; this->set_interval("unpause_ann", 50, [this]() { @@ -251,6 +254,7 @@ void SpeakerMediaPlayer::watch_media_commands_() { if (this->media_pipeline_ != nullptr) { this->cancel_timeout("next_media"); this->media_playlist_.clear(); + this->media_item_failed_ = false; this->stop_and_unpause_media_(); } } @@ -351,8 +355,10 @@ void SpeakerMediaPlayer::loop() { if (this->media_pipeline_state_ == AudioPipelineState::ERROR_READING) { ESP_LOGE(TAG, "The media pipeline's file reader encountered an error."); + this->media_item_failed_ = true; } else if (this->media_pipeline_state_ == AudioPipelineState::ERROR_DECODING) { ESP_LOGE(TAG, "The media pipeline's audio decoder encountered an error."); + this->media_item_failed_ = true; } AudioPipelineState old_announcement_pipeline_state = this->announcement_pipeline_state_; @@ -362,8 +368,10 @@ void SpeakerMediaPlayer::loop() { if (this->announcement_pipeline_state_ == AudioPipelineState::ERROR_READING) { ESP_LOGE(TAG, "The announcement pipeline's file reader encountered an error."); + this->announcement_item_failed_ = true; } else if (this->announcement_pipeline_state_ == AudioPipelineState::ERROR_DECODING) { ESP_LOGE(TAG, "The announcement pipeline's audio decoder encountered an error."); + this->announcement_item_failed_ = true; } if (this->announcement_pipeline_state_ != AudioPipelineState::STOPPED) { @@ -371,7 +379,12 @@ void SpeakerMediaPlayer::loop() { } else { if (!this->announcement_playlist_.empty()) { uint32_t timeout_ms = 0; - if (old_announcement_pipeline_state == AudioPipelineState::PLAYING) { + if (this->announcement_item_failed_) { + // Drop the item that failed, even with repeat enabled; otherwise it is restarted as soon as the pipeline + // stops, which after an error is usually on the next loop + this->announcement_item_failed_ = false; + this->announcement_playlist_.pop_front(); + } else if (old_announcement_pipeline_state == AudioPipelineState::PLAYING) { // Finished the current announcement file if (!this->announcement_repeat_one_) { // Pop item off the playlist if repeat is disabled @@ -399,6 +412,8 @@ void SpeakerMediaPlayer::loop() { } } } else { + // Nothing left to retry + this->announcement_item_failed_ = false; if (this->is_paused_) { #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF if (this->state != media_player::MEDIA_PLAYER_STATE_OFF) { @@ -412,7 +427,12 @@ void SpeakerMediaPlayer::loop() { } else if (this->media_pipeline_state_ == AudioPipelineState::STOPPED) { if (!media_playlist_.empty()) { uint32_t timeout_ms = 0; - if (old_media_pipeline_state == AudioPipelineState::PLAYING) { + if (this->media_item_failed_) { + // Drop the item that failed, even with repeat enabled; otherwise it is restarted as soon as the pipeline + // stops. The flag also covers an error that happened while an announcement was playing. + this->media_item_failed_ = false; + this->media_playlist_.pop_front(); + } else if (old_media_pipeline_state == AudioPipelineState::PLAYING) { // Finished the current media file if (!this->media_repeat_one_) { // Pop item off the playlist if repeat is disabled @@ -439,6 +459,8 @@ void SpeakerMediaPlayer::loop() { } } } else { + // Nothing left to retry + this->media_item_failed_ = false; #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF if (this->state != media_player::MEDIA_PLAYER_STATE_OFF) { this->state = media_player::MEDIA_PLAYER_STATE_IDLE; diff --git a/esphome/components/speaker/media_player/speaker_media_player.h b/esphome/components/speaker/media_player/speaker_media_player.h index 6470fb925c2..889aedbc9ac 100644 --- a/esphome/components/speaker/media_player/speaker_media_player.h +++ b/esphome/components/speaker/media_player/speaker_media_player.h @@ -125,11 +125,15 @@ class SpeakerMediaPlayer final : public Component, optional media_format_; AudioPipelineState media_pipeline_state_{AudioPipelineState::STOPPED}; bool media_repeat_one_{false}; + // Set when the media pipeline reports an error, consumed when it stops, so the failed item is dropped + bool media_item_failed_{false}; uint32_t media_playlist_delay_ms_{0}; optional announcement_format_; AudioPipelineState announcement_pipeline_state_{AudioPipelineState::STOPPED}; bool announcement_repeat_one_{false}; + // Set when the announcement pipeline reports an error, consumed when it stops, so the failed item is dropped + bool announcement_item_failed_{false}; uint32_t announcement_playlist_delay_ms_{0}; QueueHandle_t media_control_command_queue_; From 757a1c42f9112a92cad9154198c7854adf08ea42 Mon Sep 17 00:00:00 2001 From: Christopher Pruijsen Date: Tue, 29 Sep 2026 14:03:19 +0100 Subject: [PATCH 036/133] [dfrobot_sen0395] Fix output_latency treating milliseconds as seconds (#19873) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- .../components/dfrobot_sen0395/__init__.py | 25 +++++++++---------- .../config/output_latency.yaml | 22 ++++++++++++++++ .../dfrobot_sen0395/test_output_latency.py | 25 +++++++++++++++++++ 3 files changed, 59 insertions(+), 13 deletions(-) create mode 100644 tests/component_tests/dfrobot_sen0395/config/output_latency.yaml create mode 100644 tests/component_tests/dfrobot_sen0395/test_output_latency.py diff --git a/esphome/components/dfrobot_sen0395/__init__.py b/esphome/components/dfrobot_sen0395/__init__.py index 42627a30f6f..6334e10ac5f 100644 --- a/esphome/components/dfrobot_sen0395/__init__.py +++ b/esphome/components/dfrobot_sen0395/__init__.py @@ -187,19 +187,18 @@ async def dfrobot_sen0395_settings_to_code( template_ = await cg.templatable(segments[7], args, cg.float_) cg.add(var.set_det_max4(template_)) if CONF_OUTPUT_LATENCY in config: - template_ = await cg.templatable( - config[CONF_OUTPUT_LATENCY][CONF_DELAY_AFTER_DETECT], args, float - ) - if isinstance(template_, cv.TimePeriod): - template_ = template_.total_milliseconds / 1000 - cg.add(var.set_delay_after_detect(template_)) - - template_ = await cg.templatable( - config[CONF_OUTPUT_LATENCY][CONF_DELAY_AFTER_DISAPPEAR], args, float - ) - if isinstance(template_, cv.TimePeriod): - template_ = template_.total_milliseconds / 1000 - cg.add(var.set_delay_after_disappear(template_)) + latency = config[CONF_OUTPUT_LATENCY] + for key, setter in ( + (CONF_DELAY_AFTER_DETECT, var.set_delay_after_detect), + (CONF_DELAY_AFTER_DISAPPEAR, var.set_delay_after_disappear), + ): + template_ = await cg.templatable( + latency[key], + args, + float, + to_exp=lambda period: period.total_milliseconds / 1000, + ) + cg.add(setter(template_)) if CONF_SENSITIVITY in config: template_ = await cg.templatable(config[CONF_SENSITIVITY], args, cg.int8) cg.add(var.set_sensitivity(template_)) diff --git a/tests/component_tests/dfrobot_sen0395/config/output_latency.yaml b/tests/component_tests/dfrobot_sen0395/config/output_latency.yaml new file mode 100644 index 00000000000..a74527e4b19 --- /dev/null +++ b/tests/component_tests/dfrobot_sen0395/config/output_latency.yaml @@ -0,0 +1,22 @@ +esphome: + name: test + on_boot: + then: + - dfrobot_sen0395.settings: + id: mmwave + output_latency: + delay_after_detect: 0.1s + delay_after_disappear: 5.0s + +esp32: + board: esp32dev + framework: + type: esp-idf + +uart: + tx_pin: GPIO17 + rx_pin: GPIO16 + baud_rate: 115200 + +dfrobot_sen0395: + id: mmwave diff --git a/tests/component_tests/dfrobot_sen0395/test_output_latency.py b/tests/component_tests/dfrobot_sen0395/test_output_latency.py new file mode 100644 index 00000000000..6ce6b6f9743 --- /dev/null +++ b/tests/component_tests/dfrobot_sen0395/test_output_latency.py @@ -0,0 +1,25 @@ +"""Codegen tests for dfrobot_sen0395 output_latency units.""" + +from collections.abc import Callable +from pathlib import Path +import re + + +def _latency_return(main_cpp: str, setter: str) -> str: + match = re.search( + rf"{re.escape(setter)}\(\[\]\(\) -> float \{{\s*return ([^;]+);", + main_cpp, + ) + assert match is not None, f"{setter} not found in generated main" + return match.group(1) + + +def test_output_latency_emits_seconds_not_milliseconds( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """YAML 0.1s / 5.0s must reach SetLatencyCommand as seconds, not milliseconds.""" + main_cpp = generate_main(component_config_path("output_latency.yaml")) + + assert _latency_return(main_cpp, "set_delay_after_detect") == "0.1f" + assert _latency_return(main_cpp, "set_delay_after_disappear") == "5.0f" From 627832f7ba08fc33508cc668a2cf8dd1fb3a14cc Mon Sep 17 00:00:00 2001 From: n-IA-hane <49248235+n-IA-hane@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:36:37 +0200 Subject: [PATCH 037/133] [speaker_source] Account for playback callbacks during speaker writes (#19499) Co-authored-by: n-IA-hane --- .../speaker_source_media_player.cpp | 15 ++++++++++++--- .../components/speaker_source/test.esp32-idf.yaml | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/esphome/components/speaker_source/speaker_source_media_player.cpp b/esphome/components/speaker_source/speaker_source_media_player.cpp index 215f3942d58..6a000b8f215 100644 --- a/esphome/components/speaker_source/speaker_source_media_player.cpp +++ b/esphome/components/speaker_source/speaker_source_media_player.cpp @@ -196,10 +196,19 @@ size_t SpeakerSourceMediaPlayer::handle_media_output_(uint8_t pipeline, media_so vTaskDelay(pdMS_TO_TICKS(timeout_ms)); return 0; } + // Reserve frames before the speaker can play them and report their progress, + // then release the portion it did not accept. + const uint32_t reserved_frames = stream_info.bytes_to_frames(length); + ps.pending_frames.fetch_add(reserved_frames, std::memory_order_relaxed); size_t bytes_written = ps.speaker->play(data, length, pdMS_TO_TICKS(timeout_ms)); - if (bytes_written > 0) { - // Track frames sent to speaker for this source - ps.pending_frames.fetch_add(stream_info.bytes_to_frames(bytes_written), std::memory_order_relaxed); + const uint32_t unused_frames = reserved_frames - stream_info.bytes_to_frames(bytes_written); + if (unused_frames > 0) { + // The main loop may reset pending_frames while play() is blocked. + // Clamp the release so that a concurrent reset cannot cause underflow. + uint32_t pending = ps.pending_frames.load(std::memory_order_relaxed); + while (!ps.pending_frames.compare_exchange_weak(pending, pending - std::min(pending, unused_frames), + std::memory_order_relaxed)) { + } } return bytes_written; } diff --git a/tests/components/speaker_source/test.esp32-idf.yaml b/tests/components/speaker_source/test.esp32-idf.yaml index 5a2fd16938d..a8d47dd2538 100644 --- a/tests/components/speaker_source/test.esp32-idf.yaml +++ b/tests/components/speaker_source/test.esp32-idf.yaml @@ -4,4 +4,4 @@ substitutions: packages: i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-idf.yaml -<<: !include common.yaml + speaker_source: !include common.yaml From c1303c048bd98b6c86f3948d62973f651013be87 Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:50:07 +0200 Subject: [PATCH 038/133] [esp32][rp2][esp8266][libretiny]mask gpio flags (1/3) (#19393) --- esphome/components/esp32/gpio.cpp | 2 +- esphome/components/esp8266/gpio.cpp | 2 ++ esphome/components/libretiny/gpio_arduino.cpp | 2 ++ esphome/components/rp2/gpio.cpp | 2 ++ 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/esphome/components/esp32/gpio.cpp b/esphome/components/esp32/gpio.cpp index 74665f3126a..a00c984f764 100644 --- a/esphome/components/esp32/gpio.cpp +++ b/esphome/components/esp32/gpio.cpp @@ -30,7 +30,7 @@ static const gpio_hal_context_t GPIO_HAL = {.dev = GPIO_HAL_GET_HW(GPIO_PORT_0)} bool ESP32InternalGPIOPin::isr_service_installed = false; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) static gpio_mode_t flags_to_mode(gpio::Flags flags) { - flags = (gpio::Flags)(flags & ~(gpio::FLAG_PULLUP | gpio::FLAG_PULLDOWN)); + flags = flags & (gpio::FLAG_INPUT | gpio::FLAG_OUTPUT | gpio::FLAG_OPEN_DRAIN); if (flags == gpio::FLAG_INPUT) return GPIO_MODE_INPUT; if (flags == gpio::FLAG_OUTPUT) diff --git a/esphome/components/esp8266/gpio.cpp b/esphome/components/esp8266/gpio.cpp index a85f054dfe9..66ac122f000 100644 --- a/esphome/components/esp8266/gpio.cpp +++ b/esphome/components/esp8266/gpio.cpp @@ -8,6 +8,8 @@ namespace esphome::esp8266 { static const char *const TAG = "esp8266"; static int flags_to_mode(gpio::Flags flags, uint8_t pin) { + flags = + flags & (gpio::FLAG_INPUT | gpio::FLAG_OUTPUT | gpio::FLAG_OPEN_DRAIN | gpio::FLAG_PULLUP | gpio::FLAG_PULLDOWN); if (flags == gpio::FLAG_OUTPUT || flags == (gpio::FLAG_OUTPUT | gpio::FLAG_INPUT)) { return OUTPUT; } diff --git a/esphome/components/libretiny/gpio_arduino.cpp b/esphome/components/libretiny/gpio_arduino.cpp index b1a37cb2258..ca88835f695 100644 --- a/esphome/components/libretiny/gpio_arduino.cpp +++ b/esphome/components/libretiny/gpio_arduino.cpp @@ -8,6 +8,8 @@ namespace esphome::libretiny { static const char *const TAG = "libretiny.gpio"; static int IRAM_ATTR flags_to_mode(gpio::Flags flags) { + flags = + flags & (gpio::FLAG_INPUT | gpio::FLAG_OUTPUT | gpio::FLAG_OPEN_DRAIN | gpio::FLAG_PULLUP | gpio::FLAG_PULLDOWN); if (flags == gpio::FLAG_INPUT) { return INPUT; } else if (flags == gpio::FLAG_OUTPUT) { diff --git a/esphome/components/rp2/gpio.cpp b/esphome/components/rp2/gpio.cpp index 0dbb124a267..b95da9bcdbc 100644 --- a/esphome/components/rp2/gpio.cpp +++ b/esphome/components/rp2/gpio.cpp @@ -9,6 +9,8 @@ namespace rp2 { static const char *const TAG = "rp2"; static int flags_to_mode(gpio::Flags flags, uint8_t pin) { + flags = + flags & (gpio::FLAG_INPUT | gpio::FLAG_OUTPUT | gpio::FLAG_OPEN_DRAIN | gpio::FLAG_PULLUP | gpio::FLAG_PULLDOWN); if (flags == gpio::FLAG_INPUT) { // NOLINT(bugprone-branch-clone) return INPUT; } else if (flags == gpio::FLAG_OUTPUT) { From 5fdb73aceb5df8af4c3b0689213dd5d59195ad1d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 18:26:40 +0200 Subject: [PATCH 039/133] [libretiny] Load the precompiled header on Apple Silicon Macs (#19857) --- esphome/build_helpers/pch.py | 28 +--- esphome/platformio/pch.py.script | 74 ++++++++++- esphome/writer.py | 5 +- tests/unit_tests/build_helpers/test_pch.py | 11 -- .../unit_tests/test_platformio_pch_script.py | 122 +++++++++++++++++- tests/unit_tests/test_writer.py | 10 +- 6 files changed, 208 insertions(+), 42 deletions(-) diff --git a/esphome/build_helpers/pch.py b/esphome/build_helpers/pch.py index bd13726d85f..ec0ff416e50 100644 --- a/esphome/build_helpers/pch.py +++ b/esphome/build_helpers/pch.py @@ -9,15 +9,9 @@ import os from pathlib import Path import posixpath import re -import sys from esphome.build_helpers.ccache import effective_ccache_basedir, parse_enable_env -from esphome.const import ( - PLATFORM_BK72XX, - PLATFORM_LN882X, - PLATFORM_NRF52, - PLATFORM_RTL87XX, -) +from esphome.const import PLATFORM_NRF52 from esphome.helpers import write_file_if_changed _LOGGER = logging.getLogger(__name__) @@ -40,8 +34,12 @@ PCH_GUARD_TEXT = f"""\ #endif """ +# The cc1plus wrapper the PlatformIO script writes on arm64 macOS +PCH_CC1_DIR = "pch_cc1" + # What the PlatformIO script leaves in the project root, for cleanup PCH_ARTIFACT_NAMES = (PCH_HEADER_NAME, PCH_GCH_NAME, PCH_SUM_NAME, PCH_SOURCE_NAME) +PCH_ARTIFACT_DIRS = (PCH_CC1_DIR,) # The core headers every backend precompiles PCH_DEFAULT_HEADERS = ("esphome/core/pch_prefix.h",) @@ -53,17 +51,6 @@ PCH_SCRIPT_EXCLUDED_PLATFORMS = frozenset( } ) -# The GCC 10.3 of LibreTiny only loads its own .gch back on Linux: elsewhere -# the compiler is loaded at a new address each run, which GCC accepts from -# version 12. Remove once LibreTiny ships GCC 12 or newer -PCH_SCRIPT_LINUX_ONLY_PLATFORMS = frozenset( - { - PLATFORM_BK72XX, - PLATFORM_LN882X, - PLATFORM_RTL87XX, - } -) - # What ccache needs to cache compiles that load a .gch _CCACHE_PCH_SLOPPINESS = ("pch_defines", "time_macros") @@ -103,10 +90,7 @@ def pch_script_enabled() -> bool: """Whether this PlatformIO build takes the pch script.""" from esphome.core import CORE - platform = CORE.target_platform - if platform in PCH_SCRIPT_LINUX_ONLY_PLATFORMS and sys.platform != "linux": - return False - return pch_enabled() and platform not in PCH_SCRIPT_EXCLUDED_PLATFORMS + return pch_enabled() and CORE.target_platform not in PCH_SCRIPT_EXCLUDED_PLATFORMS def pch_header_text(include_headers: Iterable[str]) -> str: diff --git a/esphome/platformio/pch.py.script b/esphome/platformio/pch.py.script index 962f5c3f2c2..960d5bd7d9e 100644 --- a/esphome/platformio/pch.py.script +++ b/esphome/platformio/pch.py.script @@ -1,10 +1,12 @@ import hashlib import os from pathlib import Path +import platform import posixpath import re import shlex import subprocess +import sys # pylint: disable=E0602 Import("env") # noqa: F821 @@ -29,6 +31,35 @@ _GUARD_TEXT = f"""\ #endif """ _INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+["<]([^">]+)[">]', re.MULTILINE) +# Before GCC 12 a .gch only loads at the address it was saved from. Checked +# for this PlatformIO platform, whose compiler is built for a random address +_OLD_GCC_PIO_PLATFORM = "libretiny" +_MOVABLE_PCH_GCC = 12 +# Relative: compiles run in the project root +_CC1_DIR = "pch_cc1" +_CC1PLUS_WRAPPER = """\ +#!/usr/bin/env -S "{python}" -IS +# Written by ESPHome: starts cc1plus without address randomisation, so it +# loads the precompiled header. See esphome/platformio/pch.py.script. +import ctypes +import sys + +real = {real!r} +libc = ctypes.CDLL("/usr/lib/libSystem.B.dylib") +attr = ctypes.create_string_buffer(64) +libc.posix_spawnattr_init(attr) +# POSIX_SPAWN_SETEXEC | _POSIX_SPAWN_DISABLE_ASLR +libc.posix_spawnattr_setflags(attr, ctypes.c_short(0x0040 | 0x0100)) +# What os.fsencode does, without the import +encoding = sys.getfilesystemencoding() +argv = [real.encode(), *(a.encode(encoding, "surrogateescape") for a in sys.argv[1:]), None] +libc._NSGetEnviron.restype = ctypes.POINTER(ctypes.POINTER(ctypes.c_char_p)) +# Replaces this process on success; the return value is the error number +rc = libc.posix_spawn(None, argv[0], None, attr, (ctypes.c_char_p * len(argv))(*argv), + libc._NSGetEnviron().contents) +libc.strerror.restype = ctypes.c_char_p +sys.exit("ESPHome: cannot start " + real + ": " + libc.strerror(rc).decode()) +""" _CXX_SUFFIXES = (".cpp", ".cc", ".cxx") _HEADER_SUFFIXES = (".h", ".hpp", ".hh", ".inc") _INCLUDE_DIR_OPTIONS = ("-I", "-isystem", "-iquote", "-idirafter") @@ -81,6 +112,35 @@ def _src_envs(src_dir: Path) -> list: return list(found.values()) or [projenv] # noqa: F821 +def _driver_output(cxx: str, option: str) -> str: + result = subprocess.run([cxx, option], capture_output=True, text=True) # noqa: PLW1510 + return result.stdout.strip() + + +def _old_gcc_flags(proj_dir: Path, pio_platform: str, cxx: str): + """Extra driver flags for an old GCC, or None where its .gch cannot be + used. Linux loads it as is. arm64 macOS loads every program at a random + address, so there the driver gets a -B directory whose cc1plus starts + the real one without that randomisation.""" + if pio_platform != _OLD_GCC_PIO_PLATFORM or sys.platform == "linux": + return [] + major = _driver_output(cxx, "-dumpversion").partition(".")[0] + if major.isdigit() and int(major) >= _MOVABLE_PCH_GCC: + return [] + if sys.platform != "darwin" or platform.machine() != "arm64": + return None + # The driver prints the bare name when it has no cc1plus of its own + real = _driver_output(cxx, "-print-prog-name=cc1plus") + if not os.path.isabs(real): + return None + wrapper = proj_dir / _CC1_DIR / "cc1plus" + wrapper.parent.mkdir(exist_ok=True) + real = os.path.normpath(real) + wrapper.write_text(_CC1PLUS_WRAPPER.format(python=sys.executable, real=real)) + wrapper.chmod(0o755) + return [f"-B{_CC1_DIR}/"] + + def _setup_pch() -> None: if projenv is None: if "nobuild" in COMMAND_LINE_TARGETS: # noqa: F821 @@ -97,6 +157,11 @@ def _setup_pch() -> None: src_envs = _src_envs(src_dir) cxx = src_envs[0].subst("$CXX") + pio_platform = env.PioPlatform() # noqa: F821 + extra = _old_gcc_flags(proj_dir, pio_platform.name, cxx) + if extra is None: + print("ESPHome: this compiler cannot load a precompiled header here; compiling without it") + return # The header holds the -include entries itself, so the .gch compile must # not see them; consumers keep theirs, which the .gch then satisfies flags = [] @@ -133,9 +198,8 @@ def _setup_pch() -> None: digest.update(flags_id.encode()) # GCC does not check a .gch against its headers and PlatformIO package # paths carry no version - platform = env.PioPlatform() # noqa: F821 - for package in sorted(platform.packages): - digest.update(f"{package}={platform.get_package_version(package)}\0".encode()) + for package in sorted(pio_platform.packages): + digest.update(f"{package}={pio_platform.get_package_version(package)}\0".encode()) # Libraries carry their version in lib_deps digest.update(repr(env.GetProjectOption("lib_deps", [])).encode()) # noqa: F821 closure = _include_closure(src_dir, headers) @@ -183,7 +247,7 @@ def _setup_pch() -> None: header.write_text(_GUARD_TEXT, encoding="utf-8") source.write_text(content, encoding="utf-8") result = subprocess.run( # noqa: PLW1510 - [cxx, "-x", "c++-header", *flags, "-c", str(source), "-o", str(gch)], + [cxx, "-x", "c++-header", *flags, *extra, "-c", str(source), "-o", str(gch)], cwd=proj_dir, capture_output=True, text=True, @@ -196,7 +260,7 @@ def _setup_pch() -> None: # Prepended: GCC only loads a .gch for the first -include for src_env in src_envs: - src_env.Prepend(CXXFLAGS=_CONSUMER_FLAGS) + src_env.Prepend(CXXFLAGS=[*extra, *_CONSUMER_FLAGS]) print( "ESPHome: Compiling with a precompiled header " "(set ESPHOME_PCH_ENABLE=0 to disable)" diff --git a/esphome/writer.py b/esphome/writer.py index 9f2ff85cff5..0bd485bfd31 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -611,10 +611,13 @@ def clean_build(clear_pio_cache: bool = True, *, full: bool = False): _LOGGER.info("Deleting %s", idf_path) rmtree(idf_path) # The PlatformIO pch files live in the project root - from esphome.build_helpers.pch import PCH_ARTIFACT_NAMES + from esphome.build_helpers.pch import PCH_ARTIFACT_DIRS, PCH_ARTIFACT_NAMES for name in PCH_ARTIFACT_NAMES: CORE.relative_build_path(name).unlink(missing_ok=True) + for name in PCH_ARTIFACT_DIRS: + if (pch_dir := CORE.relative_build_path(name)).is_dir(): + rmtree(pch_dir) # idedata caches live under the data dir, not the build path; globbed # so a future backend suffix cannot drift out of clean-all diff --git a/tests/unit_tests/build_helpers/test_pch.py b/tests/unit_tests/build_helpers/test_pch.py index 0ea82157381..a03b9c3a11d 100644 --- a/tests/unit_tests/build_helpers/test_pch.py +++ b/tests/unit_tests/build_helpers/test_pch.py @@ -113,23 +113,12 @@ def test_pch_checksum_tracks_closure_content(tmp_path: Path) -> None: "platform", ["bk72xx", "esp32", "esp8266", "ln882x", "rp2", "rtl87xx"] ) def test_pch_script_enabled(platform: str, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(pch.sys, "platform", "linux") CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform} assert pch.pch_script_enabled() monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") assert not pch.pch_script_enabled() -@pytest.mark.parametrize("host", ["darwin", "win32"]) -@pytest.mark.parametrize("platform", sorted(pch.PCH_SCRIPT_LINUX_ONLY_PLATFORMS)) -def test_pch_script_linux_only_platform( - platform: str, host: str, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(pch.sys, "platform", host) - CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform} - assert not pch.pch_script_enabled() - - @pytest.mark.parametrize("platform", sorted(pch.PCH_SCRIPT_EXCLUDED_PLATFORMS)) def test_pch_script_excluded_platform(platform: str) -> None: CORE.data[KEY_CORE] = {KEY_TARGET_PLATFORM: platform} diff --git a/tests/unit_tests/test_platformio_pch_script.py b/tests/unit_tests/test_platformio_pch_script.py index bccfc17477b..e7ea1a34201 100644 --- a/tests/unit_tests/test_platformio_pch_script.py +++ b/tests/unit_tests/test_platformio_pch_script.py @@ -6,6 +6,8 @@ from collections.abc import Callable import os from pathlib import Path import stat +import subprocess +import sys from types import SimpleNamespace from unittest.mock import patch @@ -22,6 +24,7 @@ _SCRIPT = Path(toolchain.__file__).parent / "pch.py.script" class _FakePlatform: + name = "fake" packages = {"framework-x": {}, "toolchain-y": {}} def get_package_version(self, name: str) -> str | None: @@ -72,7 +75,13 @@ class _FakeSConsEnv(dict): def _fake_cxx(tmp_path: Path, fail: bool = False) -> Path: """A compiler stand-in that records its argv and writes the -o target.""" cxx = tmp_path / "fake-gxx" + # As GCC: the program's path when it exists next to the driver, else its + # name; the version from FAKE_GCC_VERSION body = ( + 'case "$1" in -print-prog-name=*) n=${1#*=};' + ' p="$(dirname "$0")/../libexec/gcc/arm-none-eabi/10.3.1/$n";' + ' [ -x "$p" ] && echo "$p" || echo "$n"; exit 0;;' + ' -dumpversion) echo "${FAKE_GCC_VERSION:-10.3.1}"; exit 0;; esac\n' 'printf -- ---call---\\\\n >> "$0.argv"; printf \'%s\\n\' "$@" >> "$0.argv"\n' ) if fail: @@ -93,6 +102,7 @@ def _run_script( name: str = "dev", platform_cls: type[_FakePlatform] = _FakePlatform, build_files: Callable[[tuple], list] | None = None, + cxx: Path | None = None, ) -> _FakeSConsEnv: proj = tmp_path / name src = proj / "src" @@ -101,7 +111,8 @@ def _run_script( (src / "esphome" / "core" / "pch_prefix.h").write_text( '#include "esphome/core/defines.h"\n' ) - cxx = _fake_cxx(tmp_path, fail=fail) + if cxx is None: + cxx = _fake_cxx(tmp_path, fail=fail) args = (proj, src, str(cxx), flags or ["-DX=1"], platform_cls) # Distinct objects: the -include flags must land on projenv only global_env = _FakeSConsEnv(*args) @@ -137,6 +148,114 @@ def test_pch_script_builds_and_prepends_relative_include(tmp_path: Path) -> None assert scons_env.global_env.prepended == [] +class _LibreTinyPlatform(_FakePlatform): + name = "libretiny" + + +def _run_on_host( + tmp_path: Path, + host: str, + machine: str, + platform_cls: type[_FakePlatform], + cxx: Path | None = None, + gcc_version: str = "10.3.1", +) -> _FakeSConsEnv: + with ( + patch.object(sys, "platform", host), + patch("platform.machine", return_value=machine), + ): + return _run_script( + tmp_path, + platform_cls=platform_cls, + cxx=cxx, + env_vars={"FAKE_GCC_VERSION": gcc_version}, + ) + + +def _fake_toolchain(tmp_path: Path) -> tuple[Path, Path]: + """A driver in bin/ and a cc1plus that records its argv in libexec/.""" + toolchain = tmp_path / "toolchain" + (toolchain / "bin").mkdir(parents=True) + cxx = _fake_cxx(toolchain / "bin") + real = toolchain / "libexec" / "gcc" / "arm-none-eabi" / "10.3.1" / "cc1plus" + real.parent.mkdir(parents=True) + real.write_text('#!/bin/sh\nprintf \'%s\\n\' "$@" > "$0.argv"\n') + real.chmod(0o755) + return cxx, real + + +def test_pch_script_gcc10_wrapper_on_apple_silicon(tmp_path: Path) -> None: + """The LibreTiny .gch compile and the consumers get a -B directory + holding a cc1plus that starts the real one.""" + cxx, real = _fake_toolchain(tmp_path) + scons_env = _run_on_host(tmp_path, "darwin", "arm64", _LibreTinyPlatform, cxx) + wrapper = tmp_path / "dev" / pch.PCH_CC1_DIR / "cc1plus" + assert wrapper.stat().st_mode & stat.S_IXUSR + assert repr(str(real)) in wrapper.read_text(encoding="utf-8") + argv = Path(f"{cxx}.argv").read_text(encoding="utf-8").split("\n") + assert f"-B{pch.PCH_CC1_DIR}/" in argv + assert scons_env.prepended == [f"-B{pch.PCH_CC1_DIR}/", *pch.pch_consumer_flags()] + + +@pytest.mark.skipif(sys.platform != "darwin", reason="the wrapper is macOS only") +def test_pch_script_gcc10_wrapper_starts_the_real_cc1plus(tmp_path: Path) -> None: + cxx, real = _fake_toolchain(tmp_path) + _run_on_host(tmp_path, "darwin", "arm64", _LibreTinyPlatform, cxx) + wrapper = tmp_path / "dev" / pch.PCH_CC1_DIR / "cc1plus" + result = subprocess.run( + [str(wrapper), "-quiet", "x.cpp"], capture_output=True, text=True, check=False + ) + assert result.returncode == 0, result.stderr + assert Path(f"{real}.argv").read_text(encoding="utf-8") == "-quiet\nx.cpp\n" + + +def test_pch_script_gcc10_without_cc1plus_builds_plainly( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A driver without a cc1plus of its own gets no wrapper and no header.""" + scons_env = _run_on_host(tmp_path, "darwin", "arm64", _LibreTinyPlatform) + assert not (tmp_path / "dev" / "esphome_pch.h.gch").exists() + assert scons_env.prepended == [] + assert "compiling without it" in capsys.readouterr().out + + +@pytest.mark.parametrize( + ("host", "machine"), [("darwin", "x86_64"), ("win32", "AMD64")] +) +def test_pch_script_gcc10_skipped_elsewhere( + tmp_path: Path, host: str, machine: str, capsys: pytest.CaptureFixture[str] +) -> None: + """Where the GCC 10 .gch cannot load, the build runs without it.""" + scons_env = _run_on_host(tmp_path, host, machine, _LibreTinyPlatform) + assert not (tmp_path / "dev" / "esphome_pch.h.gch").exists() + assert scons_env.prepended == [] + assert "compiling without it" in capsys.readouterr().out + + +@pytest.mark.parametrize( + ("host", "machine", "platform_cls", "gcc_version"), + [ + ("darwin", "arm64", _FakePlatform, "10.3.1"), + ("linux", "aarch64", _LibreTinyPlatform, "10.3.1"), + # From GCC 12 the .gch loads at any address, so the rule retires itself + ("darwin", "arm64", _LibreTinyPlatform, "12.2.0"), + ("win32", "AMD64", _LibreTinyPlatform, "14.2.0"), + ], +) +def test_pch_script_no_wrapper_where_the_gch_loads( + tmp_path: Path, + host: str, + machine: str, + platform_cls: type[_FakePlatform], + gcc_version: str, +) -> None: + scons_env = _run_on_host( + tmp_path, host, machine, platform_cls, gcc_version=gcc_version + ) + assert not (tmp_path / "dev" / pch.PCH_CC1_DIR).exists() + assert scons_env.prepended == pch.pch_consumer_flags() + + def test_pch_script_names_match_the_python_side(tmp_path: Path) -> None: """The script cannot import esphome, so its copies are pinned.""" namespace: dict[str, object] = { @@ -152,6 +271,7 @@ def test_pch_script_names_match_the_python_side(tmp_path: Path) -> None: assert namespace["_CONSUMER_FLAGS"] == pch.pch_consumer_flags() assert namespace["_GUARD_TEXT"] == pch.PCH_GUARD_TEXT assert namespace["_INCLUDE_RE"].pattern == pch._INCLUDE_RE.pattern + assert namespace["_CC1_DIR"] == pch.PCH_CC1_DIR def test_pch_script_compile_failure_stops_the_build(tmp_path: Path) -> None: diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index 432a2065bff..a1ae7f9ec77 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -684,11 +684,17 @@ def test_clean_build_partial_removes_pch_artifacts( ) -> None: """The PlatformIO pch sidecars live at the project root and must go in a partial clean, like the native backend's under .pioenvs.""" - from esphome.build_helpers.pch import PCH_ARTIFACT_NAMES as names + from esphome.build_helpers.pch import ( + PCH_ARTIFACT_DIRS as dirs, + PCH_ARTIFACT_NAMES as names, + ) assert "esphome_pch.h.gch" in names for name in names: (tmp_path / name).write_text("x") + for name in dirs: + (tmp_path / name).mkdir() + (tmp_path / name / "cc1plus").write_text("x") mock_core.relative_pioenvs_path.return_value = tmp_path / ".pioenvs" mock_core.relative_piolibdeps_path.return_value = tmp_path / ".piolibdeps" mock_core.relative_build_path.side_effect = lambda name: tmp_path / name @@ -696,7 +702,7 @@ def test_clean_build_partial_removes_pch_artifacts( clean_build() - for name in names: + for name in (*names, *dirs): assert not (tmp_path / name).exists() From 5be41dec15d408855cf71e0ce757707ed1f1079e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 18:31:24 +0200 Subject: [PATCH 040/133] [esp8266] Use our own compiler builds for the native toolchain (#19849) --- esphome/__main__.py | 2 +- esphome/arduino8266/framework.py | 80 ++++++++++++++-- esphome/components/esp8266/__init__.py | 11 ++- esphome/platformio/registry.py | 44 ++++++--- .../unit_tests/test_arduino8266_framework.py | 96 ++++++++++++++++++- tests/unit_tests/test_main.py | 20 ++++ tests/unit_tests/test_platformio_registry.py | 79 ++++++++++++++- 7 files changed, 301 insertions(+), 31 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 8ed97ccae88..e67bc42764a 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -838,7 +838,7 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int: # Keep this here, NOT in codegen: config-hash and --only-generate must keep # working on machines that cannot run the toolchain. - if CORE.is_esp8266: + if CORE.is_esp8266 and CORE.using_toolchain_platformio: from esphome.components.esp8266 import check_rosetta check_rosetta() diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index d32d61375bb..0c411a9d5d9 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -4,11 +4,12 @@ Artifacts land in a machine-global cache (shared across projects, like the ESP-IDF install in ``esphome.espidf.framework``): /arduino8266/frameworks// framework-arduinoespressif8266 - /arduino8266/toolchains// toolchain-xtensa (gcc 10.3) + /arduino8266/toolchains// xtensa-lx106-elf gcc 10.3 -Packages come from the PlatformIO registry (identical bits to the PlatformIO -backend); ``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs. ninja comes -from PATH or the ninja PyPI wheel. +The framework comes from the PlatformIO registry, the toolchain from +https://github.com/esphome-libs/xtensa-lx106-elf-toolchain/releases/; +``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs. ninja comes from PATH +or the ninja PyPI wheel. """ from __future__ import annotations @@ -22,13 +23,45 @@ from esphome.build_helpers.ninja import find_ninja from esphome.build_helpers.tools_cache import ARDUINO8266_TOOLS_CACHE, tools_cache_path from esphome.core import EsphomeError, Version from esphome.framework_helpers import str_to_lst_of_str -from esphome.platformio.registry import install_package, prefetch_packages +from esphome.platformio.registry import ( + Download, + Resolver, + get_systype, + install_package, + prefetch_packages, +) FRAMEWORK_PACKAGE = "framework-arduinoespressif8266" -TOOLCHAIN_PACKAGE = "toolchain-xtensa" +TOOLCHAIN_PACKAGE = "toolchain-xtensa-lx106-elf" # gcc 10.3, the toolchain Arduino core 3.x builds with; the build # generator's compile flags are tuned to it. -TOOLCHAIN_VERSION = "2.100300.220621" +TOOLCHAIN_VERSION = "10.3.0-esphome.2" +_TOOLCHAIN_RELEASES = ( + "https://github.com/esphome-libs/xtensa-lx106-elf-toolchain/releases/" +) +# Registry system tag -> (sha256, size) of that host's archive +TOOLCHAIN_BUILDS: dict[str, tuple[str, int]] = { + "darwin_arm64": ( + "849cede44d4d5c6ea0f14099783239f559f46327bea314281814f2652b486201", + 60830321, + ), + "darwin_x86_64": ( + "ca69904daabf0c5983b372423e5e62f49182a793e992c052e94666852470c897", + 64149487, + ), + "linux_aarch64": ( + "60a49a4f082bf246544bd409a9517dbbcab19bb30ac9decbee544b896aaccbd6", + 67573397, + ), + "linux_x86_64": ( + "1fba33ca1494ec79f2776e0e37eca93282d30f8bb9992f5f4f9a655d6fff1db4", + 68431336, + ), + "windows_amd64": ( + "af9066b0e5bf036f04f2bd9d08b89b81a7f183c57dac0abcaff71dd861cf5f3b", + 67664137, + ), +} ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS = str_to_lst_of_str( os.environ.get("ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS", "") @@ -74,6 +107,22 @@ def get_toolchain_path() -> Path: return get_arduino8266_tools_path() / "toolchains" / TOOLCHAIN_VERSION +def toolchain_download() -> Download: + """The toolchain archive for the current host.""" + systype = get_systype() + if (build := TOOLCHAIN_BUILDS.get(systype)) is None: + raise EsphomeError( + f"There is no ESP8266 toolchain for this system ({systype}); " + f"supported systems are {', '.join(sorted(TOOLCHAIN_BUILDS))}. " + "Either set 'toolchain: platformio' under 'esp8266:', or point " + "ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS at a toolchain archive" + ) + sha256, size = build + archive = f"{TOOLCHAIN_PACKAGE}-{TOOLCHAIN_VERSION}-{systype}.tar.gz" + url = f"{_TOOLCHAIN_RELEASES}download/{TOOLCHAIN_VERSION}/{archive}" + return Download(url, sha256, size) + + class InstalledPaths(NamedTuple): """Locations of the installed framework, toolchain, and ninja binary.""" @@ -115,10 +164,23 @@ def check_and_install(framework_version: Version) -> InstalledPaths: ("bin", "xtensa-lx106-elf"), ), ) + # Resolved only when a download is needed, so an installed toolchain + # keeps working on a host without a build; a mirror override wins + resolvers: dict[str, Resolver] = {} + if not ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS: + resolvers[TOOLCHAIN_PACKAGE] = toolchain_download # Fetch both archives at once; the installs below verify and extract - prefetch_packages([spec[:4] for spec in specs], downloads_dir) + prefetch_packages([spec[:4] for spec in specs], downloads_dir, resolvers) for name, version, dest, mirrors, expect in specs: - install_package(name, version, dest, mirrors, downloads_dir, expect=expect) + install_package( + name, + version, + dest, + mirrors, + downloads_dir, + expect=expect, + resolve=resolvers.get(name), + ) return InstalledPaths( framework=framework_path, toolchain=toolchain_path, ninja=ninja_path ) diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 808d7ca7a9d..1016c49152b 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -314,8 +314,8 @@ CONFIG_SCHEMA = cv.All( def check_rosetta() -> None: """Fail fast when the x86_64 ESP8266 toolchain cannot run on this Mac. - There is no native arm64 build of the xtensa-lx106 toolchain; on Apple - Silicon it runs under Rosetta 2, which macOS updates can remove. + PlatformIO's xtensa-lx106 toolchain has no arm64 build; on Apple Silicon + it runs under Rosetta 2, which macOS updates can remove. """ if not IS_MACOS or platform.machine() != "arm64": return @@ -330,9 +330,10 @@ def check_rosetta() -> None: return # arch(1) unavailable; let the build proceed if result.returncode != 0: raise EsphomeError( - "ESP8266 builds on Apple Silicon Macs use an Intel (x86_64) " - "compiler that requires Rosetta 2, which is not installed on " - "this system. Install it with:\n" + "ESP8266 builds with PlatformIO on Apple Silicon Macs use an Intel " + "(x86_64) compiler that requires Rosetta 2, which is not installed " + "on this system. Either set 'toolchain: arduino' under 'esp8266:', " + "which needs no Rosetta, or install it with:\n" " softwareupdate --install-rosetta --agree-to-license" ) diff --git a/esphome/platformio/registry.py b/esphome/platformio/registry.py index 75df82da0e9..cab536c8da7 100644 --- a/esphome/platformio/registry.py +++ b/esphome/platformio/registry.py @@ -31,6 +31,18 @@ _REGISTRY_URL = ( ) +class Download(NamedTuple): + """A package archive to fetch and verify.""" + + url: str + sha256: str + size: int | None + + +# Looks a package's download up; called only when a download is needed +Resolver = Callable[[], Download] + + def get_systype() -> str: """The registry system tag for the current host. @@ -56,7 +68,7 @@ def get_systype() -> str: @cache -def registry_download(package: str, version: str) -> tuple[str, str, int | None]: +def registry_download(package: str, version: str) -> Download: """Resolve a package's download URL, sha256, and size via the registry. The metadata fetch goes through ``http_request``/``fetch_with_retry`` @@ -144,7 +156,7 @@ def registry_download(package: str, version: str) -> tuple[str, str, int | None] f"The package registry returned no download URL for " f"{package} {version}" ) - return (url, sha256, file.get("size")) + return Download(url, sha256, file.get("size")) raise EsphomeError( f"No {package} {version} build for this platform ({systype})" ) @@ -177,17 +189,20 @@ def _archive_path(downloads_dir: Path, name: str, version: str) -> Path: return downloads_dir / f"{name}-{version}" -def _already_installed(dest: Path) -> bool: +def is_installed(dest: Path) -> bool: """Whether ``dest`` holds a completed install (extraction marker).""" return (dest / ".esphome_extracted").is_file() def prefetch_packages( - packages: list[tuple[str, str, Path, list[str]]], downloads_dir: Path + packages: list[tuple[str, str, Path, list[str]]], + downloads_dir: Path, + resolvers: dict[str, Resolver] | None = None, ) -> None: """Download pending package archives in parallel under one combined bar. - ``packages`` holds ``(name, version, dest, mirrors)`` per package. Purely + ``packages`` holds ``(name, version, dest, mirrors)`` per package; + ``resolvers`` replaces the registry lookup by name. Purely an optimization: ``install_package`` verifies every archive and re-downloads anything this pass left unfinished. Mirror overrides and registry entries without a size stay on the sequential path so its @@ -200,15 +215,18 @@ def prefetch_packages( pending: list[_PendingArchive] = [] seen: set[Path] = set() for name, version, dest, mirrors in packages: - if mirrors or (dest / ".esphome_extracted").is_file(): + if mirrors or is_installed(dest): continue archive = _archive_path(downloads_dir, name, version) if archive in seen: # A duplicate entry would race itself between two workers continue seen.add(archive) + resolve = (resolvers or {}).get(name) or partial( + registry_download, name, version + ) try: - url, sha256, size = registry_download(name, version) + url, sha256, size = resolve() except EsphomeError as err: # The sequential install reports the real failure with context _LOGGER.debug("Prefetch resolve for %s failed: %s", name, err) @@ -234,7 +252,7 @@ def prefetch_packages( if done := downloaded_bytes(entry.archive, entry.size): return done # The holder deletes the archive once it has installed it - return entry.size if _already_installed(entry.dest) else 0 + return entry.size if is_installed(entry.dest) else 0 lock = FileLock(f"{entry.dest}.lock", fallback_to_soft=False) try: @@ -245,7 +263,7 @@ def prefetch_packages( _LOGGER.debug("Leaving %s to its current downloader", entry.name) return try: - if _already_installed(entry.dest): + if is_installed(entry.dest): # A concurrent build installed it while we waited; a # re-download would orphan a fresh copy in downloads_dir tracker(entry.size) @@ -282,13 +300,15 @@ def install_package( mirrors: list[str], downloads_dir: Path, expect: Collection[str], + resolve: Resolver | None = None, ) -> None: """Download, verify, and extract one package if not already installed. The registry path is integrity-checked against the sha256 the registry publishes; a mirror override (URL templates with ``{VERSION}``/``{SYSTEM}`` substitution) is trusted as configured. ``downloads_dir`` holds the - archive between runs so an interrupted download resumes. + archive between runs so an interrupted download resumes. ``resolve`` + replaces the registry lookup. """ if not expect: # Layout validation before marker.touch() is the only guard against @@ -323,7 +343,9 @@ def install_package( mirrors, {"VERSION": version, "SYSTEM": get_systype()}, archive ) else: - url, sha256, size = registry_download(name, version) + url, sha256, size = ( + resolve() if resolve else registry_download(name, version) + ) download_with_resume(url, archive, sha256=sha256, size=size) _LOGGER.info("Extracting %s ...", name) archive_extract_all(archive, dest, progress_header="Extracting") diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index ab11c41afb4..6f43f0bf40c 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -54,6 +54,91 @@ def test_tools_path_default_and_prefix(tmp_path: Path) -> None: assert path != Path.cwd() +def test_toolchain_builds_are_pinned() -> None: + """A release must not be pinned without its checksums.""" + for sha256, size in framework.TOOLCHAIN_BUILDS.values(): + assert len(sha256) == 64 + assert size > 0 + + +def test_toolchain_download() -> None: + sha256, size = framework.TOOLCHAIN_BUILDS["darwin_arm64"] + with ( + patch.object(framework, "TOOLCHAIN_VERSION", "1.2.3"), + patch.object(framework, "get_systype", return_value="darwin_arm64"), + ): + download = framework.toolchain_download() + releases = "https://github.com/esphome-libs/xtensa-lx106-elf-toolchain/releases/" + archive = "toolchain-xtensa-lx106-elf-1.2.3-darwin_arm64.tar.gz" + assert download == (f"{releases}download/1.2.3/{archive}", sha256, size) + + +def test_toolchain_download_unsupported_system() -> None: + with ( + patch.object(framework, "get_systype", return_value="linux_armv7l"), + pytest.raises( + EsphomeError, match=r"linux_armv7l.*darwin_arm64.*toolchain: platformio" + ), + ): + framework.toolchain_download() + + +def _fake_framework(tmp_path: Path) -> None: + """The layout install_package expects of an installed framework.""" + for sub in ("cores/esp8266", "tools/sdk", "libraries"): + (tmp_path / "frameworks" / "3.30102.0" / sub).mkdir(parents=True) + (tmp_path / "frameworks" / "3.30102.0" / ".esphome_extracted").touch() + + +def test_check_and_install_mirror_skips_pinned_toolchain(tmp_path: Path) -> None: + """With a mirror override an unsupported host can bring its own toolchain.""" + with ( + patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}), + patch.object(framework, "ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS", ["http://m"]), + patch.object(framework, "install_package") as mock_install, + patch.object(framework, "prefetch_packages") as mock_prefetch, + patch.object(framework, "find_ninja", return_value=tmp_path / "ninja"), + ): + framework.check_and_install(cv.Version(3, 1, 2)) + assert mock_prefetch.call_args.args[2] == {} + assert mock_install.call_args_list[1].kwargs["resolve"] is None + + +def test_check_and_install_installed_toolchain_on_unsupported_host( + tmp_path: Path, +) -> None: + """The host is only checked when the toolchain must be downloaded.""" + _fake_framework(tmp_path) + toolchain = tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION + for sub in ("bin", "xtensa-lx106-elf"): + (toolchain / sub).mkdir(parents=True) + (toolchain / ".esphome_extracted").touch() + with ( + patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}), + patch.object(framework, "get_systype", return_value="linux_armv7l"), + patch.object(framework, "prefetch_packages"), + patch.object(framework, "find_ninja", return_value=tmp_path / "ninja"), + ): + paths = framework.check_and_install(cv.Version(3, 1, 2)) + assert paths.toolchain == toolchain + + +def test_check_and_install_unsupported_host_without_toolchain_raises( + tmp_path: Path, +) -> None: + """Without an installed toolchain the download is needed, and the host + error names the ways out.""" + _fake_framework(tmp_path) + with ( + patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}), + patch.object(framework, "get_systype", return_value="linux_armv7l"), + patch.object(framework, "prefetch_packages"), + patch.object(framework, "find_ninja", return_value=tmp_path / "ninja"), + pytest.raises(EsphomeError, match=r"linux_armv7l.*toolchain: platformio"), + ): + framework.check_and_install(cv.Version(3, 1, 2)) + + def test_check_and_install_returns_paths(tmp_path: Path) -> None: with ( patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}), @@ -76,7 +161,10 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, tmp_path / "downloads", ) - assert fw_call.kwargs["expect"] == ("cores/esp8266", "tools/sdk", "libraries") + assert fw_call.kwargs == { + "expect": ("cores/esp8266", "tools/sdk", "libraries"), + "resolve": None, + } assert tc_call.args == ( framework.TOOLCHAIN_PACKAGE, framework.TOOLCHAIN_VERSION, @@ -84,7 +172,10 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: framework.ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, tmp_path / "downloads", ) - assert tc_call.kwargs["expect"] == ("bin", "xtensa-lx106-elf") + assert tc_call.kwargs == { + "expect": ("bin", "xtensa-lx106-elf"), + "resolve": framework.toolchain_download, + } # The prefetch sees the same package specs as the installs assert mock_prefetch.call_args.args == ( [ @@ -102,6 +193,7 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: ), ], tmp_path / "downloads", + {framework.TOOLCHAIN_PACKAGE: framework.toolchain_download}, ) diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 2ed61e5d26e..bf5d1566c05 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -5999,6 +5999,26 @@ def test_compile_program_esp8266_runs_rosetta_check(tmp_path: Path) -> None: mock_check.assert_called_once() +def test_compile_program_esp8266_native_skips_rosetta_check(tmp_path: Path) -> None: + """The native toolchain has an arm64 macOS build, so nothing is probed.""" + setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test_device") + CORE.toolchain = Toolchain.ARDUINO + + config: dict[str, Any] = {CONF_ESPHOME: {CONF_NAME: "test_device"}} + + with ( + patch("esphome.components.esp8266.check_rosetta") as mock_check, + patch( + "esphome.components.esp8266.run_compile", + side_effect=EsphomeError("compile reached"), + ), + pytest.raises(EsphomeError, match="compile reached"), + ): + compile_program(MockArgs(), config) + + mock_check.assert_not_called() + + def test_compile_program_skips_rosetta_check_on_other_platforms( tmp_path: Path, mock_compile_build_info_run_compile: Mock, diff --git a/tests/unit_tests/test_platformio_registry.py b/tests/unit_tests/test_platformio_registry.py index 9d5f6c4ce59..9f1c1408469 100644 --- a/tests/unit_tests/test_platformio_registry.py +++ b/tests/unit_tests/test_platformio_registry.py @@ -311,6 +311,56 @@ def test_install_package_downloads_via_registry(tmp_path: Path) -> None: assert mock_download.call_args[1] == {"sha256": "abc123", "size": 42} +def test_install_package_downloads_pinned(tmp_path: Path) -> None: + """A resolver's download is verified like a registry one, with no lookup.""" + dest = tmp_path / "pkg" + pinned = registry.Download("http://y/pinned.tar.gz", "def456", 7) + with ( + patch.object(registry, "download_with_resume") as mock_download, + patch.object(registry, "archive_extract_all") as mock_extract, + patch.object(registry, "registry_download") as mock_registry, + ): + mock_extract.side_effect = lambda *_a, **_kw: (dest / "payload").mkdir( + parents=True + ) + registry.install_package( + "pkg", + "1.0.0", + dest, + [], + tmp_path / "dl", + expect=("payload",), + resolve=lambda: pinned, + ) + mock_registry.assert_not_called() + assert mock_download.call_args[0][0] == "http://y/pinned.tar.gz" + assert mock_download.call_args[1] == {"sha256": "def456", "size": 7} + + +def test_install_package_mirror_wins_over_pinned(tmp_path: Path) -> None: + """A mirror override replaces the pinned download.""" + dest = tmp_path / "pkg" + with ( + patch.object(registry, "download_from_mirrors") as mock_mirrors, + patch.object(registry, "download_with_resume") as mock_download, + patch.object(registry, "archive_extract_all") as mock_extract, + ): + mock_extract.side_effect = lambda *_a, **_kw: (dest / "payload").mkdir( + parents=True + ) + registry.install_package( + "pkg", + "1.0.0", + dest, + ["http://mirror/{VERSION}"], + tmp_path / "dl", + expect=("payload",), + resolve=lambda: registry.Download("http://y/pinned.tar.gz", "def456", 7), + ) + mock_mirrors.assert_called_once() + mock_download.assert_not_called() + + def test_install_package_validates_expected_layout(tmp_path: Path) -> None: """The success marker is only written when the extracted tree is usable.""" dest = tmp_path / "pkg" @@ -535,6 +585,29 @@ def test_prefetch_packages_downloads_pending_in_parallel(tmp_path: Path) -> None assert callable(call[1]["progress"]) +def test_prefetch_packages_uses_pinned_download(tmp_path: Path) -> None: + """A package with a resolver skips the registry; the rest use it.""" + with ( + patch.object(registry, "download_with_resume") as mock_download, + patch.object( + registry, "registry_download", side_effect=_resolve_for({"a": 10}) + ) as mock_registry, + ): + registry.prefetch_packages( + [ + ("a", "1.0", tmp_path / "a", []), + ("b", "2.0", tmp_path / "b", []), + ], + tmp_path / "dl", + {"b": lambda: registry.Download("http://y/b.tar.gz", "def456", 20)}, + ) + mock_registry.assert_called_once_with("a", "1.0") + calls = sorted(mock_download.call_args_list, key=lambda c: c[0][0]) + assert [c[0][0] for c in calls] == ["http://x/a.tar.gz", "http://y/b.tar.gz"] + assert calls[1][1]["sha256"] == "def456" + assert calls[1][1]["size"] == 20 + + def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: """A dest whose marker appeared while the worker waited on the lock is already installed; re-downloading would orphan an archive copy.""" @@ -620,13 +693,13 @@ def test_prefetch_packages_leaves_a_long_held_lock_to_its_holder( mock_download.assert_not_called() -def test_already_installed_probe(tmp_path: Path) -> None: +def test_is_installed_probe(tmp_path: Path) -> None: """Both arms of the marker probe the prefetch worker keys on.""" dest = tmp_path / "pkg" dest.mkdir() - assert registry._already_installed(dest) is False + assert registry.is_installed(dest) is False (dest / ".esphome_extracted").touch() - assert registry._already_installed(dest) is True + assert registry.is_installed(dest) is True def test_prefetch_packages_dedupes_duplicate_entries(tmp_path: Path) -> None: From a7e0afeb46367cfe6412f76c584a7576f78dc85b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 18:38:51 +0200 Subject: [PATCH 041/133] [esp8266] Use our own Arduino core builds for the native toolchain (#19876) --- .github/actions/cache-arduino8266/action.yml | 2 +- esphome/arduino8266/framework.py | 79 +++++++++--------- esphome/components/esp8266/__init__.py | 37 ++++++--- .../esp8266/test_framework_version.py | 31 ++++++- .../esp8266/test_toolchain_validation.py | 19 +++-- .../unit_tests/test_arduino8266_framework.py | 83 ++++++++++--------- 6 files changed, 152 insertions(+), 99 deletions(-) diff --git a/.github/actions/cache-arduino8266/action.yml b/.github/actions/cache-arduino8266/action.yml index affe62b5aa2..c9e194d667e 100644 --- a/.github/actions/cache-arduino8266/action.yml +++ b/.github/actions/cache-arduino8266/action.yml @@ -22,7 +22,7 @@ runs: # cached path cannot diverge. echo "ESPHOME_ARDUINO8266_PREFIX=$HOME/.esphome-arduino8266" >> "$GITHUB_ENV" . venv/bin/activate - key=$(python -c 'from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION as f; from esphome.arduino8266.framework import TOOLCHAIN_VERSION as t; print(f"{f}-{t}")') + key=$(python -c 'from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION as f; from esphome.arduino8266.framework import FRAMEWORK_RELEASES, TOOLCHAIN_VERSION as t; print(f"{FRAMEWORK_RELEASES[f].tag}-{t}")') [ -n "$key" ] || exit 1 echo "key=$key" >> "$GITHUB_OUTPUT" - name: Cache the native toolchain (write on dev) diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index 0c411a9d5d9..a8f8c65c066 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -3,13 +3,12 @@ Artifacts land in a machine-global cache (shared across projects, like the ESP-IDF install in ``esphome.espidf.framework``): - /arduino8266/frameworks// framework-arduinoespressif8266 - /arduino8266/toolchains// xtensa-lx106-elf gcc 10.3 + /arduino8266/frameworks// the Arduino core + /arduino8266/toolchains// xtensa-lx106-elf gcc 10.3 -The framework comes from the PlatformIO registry, the toolchain from -https://github.com/esphome-libs/xtensa-lx106-elf-toolchain/releases/; -``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs. ninja comes from PATH -or the ninja PyPI wheel. +Both come from esphome-libs releases pinned below; +``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs, with ``{VERSION}`` +standing for the release tag. ninja comes from PATH or the ninja PyPI wheel. """ from __future__ import annotations @@ -31,7 +30,30 @@ from esphome.platformio.registry import ( prefetch_packages, ) -FRAMEWORK_PACKAGE = "framework-arduinoespressif8266" +FRAMEWORK_PACKAGE = "arduino-esp8266" +_FRAMEWORK_RELEASES = "https://github.com/esphome-libs/arduino-esp8266/releases/" + + +class FrameworkRelease(NamedTuple): + tag: str + sha256: str + size: int + + def download(self) -> Download: + archive = f"{FRAMEWORK_PACKAGE}-{self.tag}.tar.gz" + url = f"{_FRAMEWORK_RELEASES}download/{self.tag}/{archive}" + return Download(url, self.sha256, self.size) + + +# Arduino core version -> its build in esphome-libs/arduino-esp8266 +FRAMEWORK_RELEASES: dict[Version, FrameworkRelease] = { + Version(3, 1, 2): FrameworkRelease( + "3.1.2-esphome.1", + "e80751e3123676b967143e39c61f2d8693946db4c7806f2a83dcaaf797ecd582", + 37189311, + ), +} + TOOLCHAIN_PACKAGE = "toolchain-xtensa-lx106-elf" # gcc 10.3, the toolchain Arduino core 3.x builds with; the build # generator's compile flags are tuned to it. @@ -77,30 +99,18 @@ def get_arduino8266_tools_path() -> Path: return tools_cache_path(*ARDUINO8266_TOOLS_CACHE) -# 3.1.1 rather than 3.1.0: the registry has no packages for 3.0.0, 3.0.1 or 3.1.0 -MIN_FRAMEWORK_VERSION = Version(3, 1, 1) - - -def framework_package_version(ver: Version) -> str: - """Map an Arduino core version to its registry package version (3.1.2 -> - 3.30102.0; the leading 3 is the package major). - - Exact registry names for 3.x cores; callers floor at MIN_FRAMEWORK_VERSION. - """ - if ver.major > 3: +def framework_release(version: Version) -> FrameworkRelease: + if (release := FRAMEWORK_RELEASES.get(version)) is None: raise EsphomeError( - f"Arduino core {ver} is not supported yet; " - "the newest known core series is 3.x" + f"'toolchain: arduino' has no build of Arduino core {version}; " + f"available: {', '.join(str(v) for v in FRAMEWORK_RELEASES)}. " + "Use one of those or 'toolchain: platformio'" ) - if ver.major < 3: - raise EsphomeError( - f"Arduino core {ver} is not supported; ESPHome requires core 3.x" - ) - return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" + return release -def get_framework_path(package_version: str) -> Path: - return get_arduino8266_tools_path() / "frameworks" / package_version +def get_framework_path(tag: str) -> Path: + return get_arduino8266_tools_path() / "frameworks" / tag def get_toolchain_path() -> Path: @@ -133,24 +143,17 @@ class InstalledPaths(NamedTuple): def check_and_install(framework_version: Version) -> InstalledPaths: """Ensure framework, toolchain, and ninja are installed; return their paths.""" - if framework_version < MIN_FRAMEWORK_VERSION: - # Config validation enforces this too; keep the module honest when - # called directly. - raise EsphomeError( - f"The native toolchain requires the Arduino core " - f">= {MIN_FRAMEWORK_VERSION}, got {framework_version}" - ) + release = framework_release(framework_version) # Probe the cheap local dependency before ~110 MB of downloads ninja_path = find_ninja() - package_version = framework_package_version(framework_version) - framework_path = get_framework_path(package_version) + framework_path = get_framework_path(release.tag) downloads_dir = get_arduino8266_tools_path() / "downloads" toolchain_path = get_toolchain_path() # One spec per package: the prefetch and the installs must agree specs = ( ( FRAMEWORK_PACKAGE, - package_version, + release.tag, framework_path, ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, ("cores/esp8266", "tools/sdk", "libraries"), @@ -167,6 +170,8 @@ def check_and_install(framework_version: Version) -> InstalledPaths: # Resolved only when a download is needed, so an installed toolchain # keeps working on a host without a build; a mirror override wins resolvers: dict[str, Resolver] = {} + if not ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS: + resolvers[FRAMEWORK_PACKAGE] = release.download if not ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS: resolvers[TOOLCHAIN_PACKAGE] = toolchain_download # Fetch both archives at once; the installs below verify and extract diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 1016c49152b..0e5303e8df1 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -118,15 +118,14 @@ def _validate_native_toolchain(config: ConfigType) -> ConfigType: """Constraints of the native (non-PlatformIO) Arduino toolchain.""" if not CORE.using_toolchain_arduino: return config - from esphome.arduino8266.framework import MIN_FRAMEWORK_VERSION + from esphome.arduino8266.framework import framework_release conf = config[CONF_FRAMEWORK] version = cv.Version.parse(conf[CONF_VERSION]) - if version < MIN_FRAMEWORK_VERSION: - raise cv.Invalid( - "'toolchain: arduino' requires framework version " - f"{MIN_FRAMEWORK_VERSION} or newer" - ) + try: + framework_release(version) + except EsphomeError as err: + raise cv.Invalid(str(err), path=[CONF_FRAMEWORK, CONF_VERSION]) from err # platform_version is a PlatformIO concept; drop it, warning when a # custom pin is discarded if ( @@ -137,6 +136,8 @@ def _validate_native_toolchain(config: ConfigType) -> ConfigType: "'platform_version' is ignored by 'toolchain: arduino'; the native " "toolchain downloads the framework and compiler directly" ) + # The native path fetches its own build; source may only be the + # PlatformIO default the schema filled in if conf[CONF_SOURCE] != _format_framework_arduino_version(version): raise cv.Invalid( "'toolchain: arduino' does not support a custom framework source; " @@ -174,16 +175,27 @@ def get_download_types(storage_json: StorageJSON) -> list[dict[str, str]]: ] +def _framework_package_version(ver: cv.Version) -> str: + """Map an Arduino core version to its PlatformIO registry package version + (3.1.2 -> 3.30102.0; the leading 3 is the package major).""" + if ver.major > 3: + raise EsphomeError( + f"Arduino core {ver} is not supported yet; " + "the newest known core series is 3.x" + ) + if ver.major < 3: + raise EsphomeError( + f"Arduino core {ver} is not supported; ESPHome requires core 3.x" + ) + return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0" + + def _format_framework_arduino_version(ver: cv.Version) -> str: # format the given arduino (https://github.com/esp8266/Arduino/releases) version to # a PIO platformio/framework-arduinoespressif8266 value # List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266 - # Same encoding the native toolchain uses for its package download, so a - # custom-source check against this value cannot drift from what it fetches. - from esphome.arduino8266.framework import framework_package_version - try: - return f"~{framework_package_version(ver)}" + return f"~{_framework_package_version(ver)}" except EsphomeError as err: # Anchor the 4.x rejection to the framework version line instead of # aborting with a bare traceback-level error @@ -194,7 +206,8 @@ def _format_framework_arduino_version(ver: cv.Version) -> str: # * New framework historically have had some regressions, especially for WiFi. # The new version needs to be thoroughly validated before changing the # recommended version as otherwise a bunch of devices could be bricked -# * For all constants below, update platformio.ini (in this repo) +# * For all constants below, update platformio.ini (in this repo) and +# FRAMEWORK_RELEASES in esphome/arduino8266/framework.py # The default/recommended arduino framework version # - https://github.com/esp8266/Arduino/releases diff --git a/tests/unit_tests/components/esp8266/test_framework_version.py b/tests/unit_tests/components/esp8266/test_framework_version.py index 0107aff8dd3..11590f77cf2 100644 --- a/tests/unit_tests/components/esp8266/test_framework_version.py +++ b/tests/unit_tests/components/esp8266/test_framework_version.py @@ -2,9 +2,14 @@ import pytest -from esphome.components.esp8266 import _arduino_check_versions +from esphome.components.esp8266 import ( + _arduino_check_versions, + _format_framework_arduino_version, + _framework_package_version, +) import esphome.config_validation as cv from esphome.const import CONF_PLATFORM_VERSION, CONF_VERSION +from esphome.core import EsphomeError def test_versions_before_3_are_rejected() -> None: @@ -21,3 +26,27 @@ def test_supported_versions_pass() -> None: value = _arduino_check_versions({CONF_VERSION: "recommended"}) assert value[CONF_VERSION] == "3.1.2" assert "espressif8266@4.2.1" in value[CONF_PLATFORM_VERSION] + + +def test_framework_package_version() -> None: + assert _framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0" + assert _framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0" + # A future major bump needs its own encoding, not a doomed registry lookup + with pytest.raises(EsphomeError, match="not supported yet"): + _framework_package_version(cv.Version(4, 0, 0)) + # Cores before 3.x cannot build ESPHome (C++20) and are rejected + with pytest.raises(EsphomeError, match="requires core 3"): + _framework_package_version(cv.Version(2, 7, 4)) + + +def test_format_framework_arduino_version_pins_all_series() -> None: + """The PlatformIO source formatter across every encoding era.""" + + assert _format_framework_arduino_version(cv.Version(3, 1, 2)) == "~3.30102.0" + # Pre-3 cores are rejected with the version line anchored + with pytest.raises(cv.Invalid, match="requires core 3"): + _format_framework_arduino_version(cv.Version(2, 7, 4)) + # Anchored to the framework version line, not a bare EsphomeError + with pytest.raises(cv.Invalid, match="not supported yet") as excinfo: + _format_framework_arduino_version(cv.Version(4, 0, 0)) + assert excinfo.value.path == ["version"] diff --git a/tests/unit_tests/components/esp8266/test_toolchain_validation.py b/tests/unit_tests/components/esp8266/test_toolchain_validation.py index 951217c8a10..d54c2c35334 100644 --- a/tests/unit_tests/components/esp8266/test_toolchain_validation.py +++ b/tests/unit_tests/components/esp8266/test_toolchain_validation.py @@ -67,21 +67,24 @@ def test_valid_config_passes() -> None: def test_platformio_toolchain_skips_checks() -> None: - # 3.0.2 is pio-legal (>= the global 3.0.0 floor) but below the native - # toolchain's own 3.1.1 floor; the bogus board only the native path checks + # 3.0.2 is pio-legal (>= the global 3.0.0 floor) but has no native build; + # the bogus board only the native path checks CORE.toolchain = Toolchain.PLATFORMIO config = _config(board="not_a_board", version="3.0.2") assert _validate_native_toolchain(config) is config -def test_version_below_floor_rejected() -> None: - # 3.1.0 has no registry package, so the native floor is 3.1.1 - with pytest.raises(cv.Invalid, match="3.1.1 or newer"): - _validate_native_toolchain(_config(version="3.1.0")) +def test_version_without_build_rejected() -> None: + """Only the core versions built in esphome-libs/arduino-esp8266 work.""" + with pytest.raises( + cv.Invalid, match=r"3\.1\.1.*available: 3\.1\.2.*platformio" + ) as excinfo: + _validate_native_toolchain(_config(version="3.1.1")) + assert excinfo.value.path == [CONF_FRAMEWORK, CONF_VERSION] -def test_version_at_floor_accepted() -> None: - _validate_native_toolchain(_config(version="3.1.1")) +def test_built_version_accepted() -> None: + _validate_native_toolchain(_config(version="3.1.2")) def test_custom_platform_version_warns_and_is_dropped( diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index 6f43f0bf40c..a4193b9a748 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -9,6 +9,7 @@ from unittest.mock import patch import pytest from esphome.arduino8266 import framework +from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION import esphome.config_validation as cv from esphome.core import CORE, EsphomeError @@ -18,32 +19,6 @@ def _build_path(tmp_path: Path) -> None: CORE.build_path = tmp_path -def test_framework_package_version() -> None: - assert framework.framework_package_version(cv.Version(3, 1, 2)) == "3.30102.0" - assert framework.framework_package_version(cv.Version(3, 2, 0)) == "3.30200.0" - # A future major bump needs its own encoding, not a doomed registry lookup - with pytest.raises(EsphomeError, match="not supported yet"): - framework.framework_package_version(cv.Version(4, 0, 0)) - # Cores before 3.x cannot build ESPHome (C++20) and are rejected - with pytest.raises(EsphomeError, match="requires core 3"): - framework.framework_package_version(cv.Version(2, 7, 4)) - - -def test_format_framework_arduino_version_pins_all_series() -> None: - """The esp8266 component's PIO source formatter across every encoding - era, including the 4.x rejection it now shares with the installer.""" - from esphome.components.esp8266 import _format_framework_arduino_version as fmt - - assert fmt(cv.Version(3, 1, 2)) == "~3.30102.0" - # Pre-3 cores are rejected with the version line anchored - with pytest.raises(cv.Invalid, match="requires core 3"): - fmt(cv.Version(2, 7, 4)) - # Anchored to the framework version line, not a bare EsphomeError - with pytest.raises(cv.Invalid, match="not supported yet") as excinfo: - fmt(cv.Version(4, 0, 0)) - assert excinfo.value.path == ["version"] - - def test_tools_path_default_and_prefix(tmp_path: Path) -> None: with patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}): assert framework.get_arduino8266_tools_path() == tmp_path.resolve() @@ -54,6 +29,27 @@ def test_tools_path_default_and_prefix(tmp_path: Path) -> None: assert path != Path.cwd() +def _recommended() -> framework.FrameworkRelease: + return framework.FRAMEWORK_RELEASES[RECOMMENDED_ARDUINO_FRAMEWORK_VERSION] + + +def test_framework_releases_are_pinned() -> None: + """A release must not be pinned without its checksum, and the recommended + core must have one.""" + assert RECOMMENDED_ARDUINO_FRAMEWORK_VERSION in framework.FRAMEWORK_RELEASES + for version, release in framework.FRAMEWORK_RELEASES.items(): + assert release.tag.startswith(f"{version}-esphome.") + assert len(release.sha256) == 64 + assert release.size > 0 + + +def test_framework_download() -> None: + download = framework.FrameworkRelease("1.2.3-esphome.4", "a" * 64, 5).download() + releases = "https://github.com/esphome-libs/arduino-esp8266/releases/" + archive = "arduino-esp8266-1.2.3-esphome.4.tar.gz" + assert download == (f"{releases}download/1.2.3-esphome.4/{archive}", "a" * 64, 5) + + def test_toolchain_builds_are_pinned() -> None: """A release must not be pinned without its checksums.""" for sha256, size in framework.TOOLCHAIN_BUILDS.values(): @@ -86,14 +82,15 @@ def test_toolchain_download_unsupported_system() -> None: def _fake_framework(tmp_path: Path) -> None: """The layout install_package expects of an installed framework.""" for sub in ("cores/esp8266", "tools/sdk", "libraries"): - (tmp_path / "frameworks" / "3.30102.0" / sub).mkdir(parents=True) - (tmp_path / "frameworks" / "3.30102.0" / ".esphome_extracted").touch() + (tmp_path / "frameworks" / _recommended().tag / sub).mkdir(parents=True) + (tmp_path / "frameworks" / _recommended().tag / ".esphome_extracted").touch() def test_check_and_install_mirror_skips_pinned_toolchain(tmp_path: Path) -> None: """With a mirror override an unsupported host can bring its own toolchain.""" with ( patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}), + patch.object(framework, "ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS", ["http://f"]), patch.object(framework, "ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS", ["http://m"]), patch.object(framework, "install_package") as mock_install, patch.object(framework, "prefetch_packages") as mock_prefetch, @@ -101,7 +98,10 @@ def test_check_and_install_mirror_skips_pinned_toolchain(tmp_path: Path) -> None ): framework.check_and_install(cv.Version(3, 1, 2)) assert mock_prefetch.call_args.args[2] == {} - assert mock_install.call_args_list[1].kwargs["resolve"] is None + assert [call.kwargs["resolve"] for call in mock_install.call_args_list] == [ + None, + None, + ] def test_check_and_install_installed_toolchain_on_unsupported_host( @@ -147,7 +147,7 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: patch.object(framework, "find_ninja", return_value=tmp_path / "ninja"), ): paths = framework.check_and_install(cv.Version(3, 1, 2)) - assert paths.framework == tmp_path / "frameworks" / "3.30102.0" + assert paths.framework == tmp_path / "frameworks" / _recommended().tag assert paths.toolchain == tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION assert paths.ninja == tmp_path / "ninja" assert mock_install.call_count == 2 @@ -156,14 +156,14 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: fw_call, tc_call = mock_install.call_args_list assert fw_call.args == ( framework.FRAMEWORK_PACKAGE, - "3.30102.0", - tmp_path / "frameworks" / "3.30102.0", + _recommended().tag, + tmp_path / "frameworks" / _recommended().tag, framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, tmp_path / "downloads", ) assert fw_call.kwargs == { "expect": ("cores/esp8266", "tools/sdk", "libraries"), - "resolve": None, + "resolve": _recommended().download, } assert tc_call.args == ( framework.TOOLCHAIN_PACKAGE, @@ -181,8 +181,8 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: [ ( framework.FRAMEWORK_PACKAGE, - "3.30102.0", - tmp_path / "frameworks" / "3.30102.0", + _recommended().tag, + tmp_path / "frameworks" / _recommended().tag, framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, ), ( @@ -193,7 +193,10 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: ), ], tmp_path / "downloads", - {framework.TOOLCHAIN_PACKAGE: framework.toolchain_download}, + { + framework.FRAMEWORK_PACKAGE: _recommended().download, + framework.TOOLCHAIN_PACKAGE: framework.toolchain_download, + }, ) @@ -204,10 +207,10 @@ def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None: assert env["CCACHE_DIR"] == "x" -def test_check_and_install_rejects_old_core(tmp_path: Path) -> None: - """Calling the installer below the floor fails before any download.""" - with pytest.raises(EsphomeError, match=">= 3.1.1"): - framework.check_and_install(cv.Version(3, 0, 2)) +def test_check_and_install_rejects_unbuilt_core(tmp_path: Path) -> None: + """A core version without a build fails before any download.""" + with pytest.raises(EsphomeError, match=r"3\.1\.1.*available: 3\.1\.2"): + framework.check_and_install(cv.Version(3, 1, 1)) def test_get_build_env_without_path_has_no_empty_entry(tmp_path: Path) -> None: From b22f2e6b4c50e9cb5a78c41aa8d4c7ff56e98ce2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 18:51:12 +0200 Subject: [PATCH 042/133] [core] Leave the precompiled header off on Windows for a GCC with bug 14940 (#19875) --- esphome/build_gen/arduino8266.py | 8 +- esphome/build_gen/espidf.py | 26 ++++++- esphome/build_gen/host.py | 1 + esphome/build_helpers/ninja_gen.py | 12 +-- esphome/build_helpers/pch.py | 74 ++++++++++++++++++- esphome/components/nrf52/__init__.py | 14 +++- esphome/espidf/toolchain.py | 7 ++ esphome/platformio/pch.py.script | 32 +++++++- esphome/platformio/toolchain.py | 9 ++- .../unit_tests/build_gen/test_arduino8266.py | 13 ++++ tests/unit_tests/build_gen/test_espidf.py | 48 ++++++++++++ tests/unit_tests/build_helpers/test_pch.py | 67 +++++++++++++++++ tests/unit_tests/components/nrf52/test_pch.py | 31 +++++--- tests/unit_tests/conftest.py | 15 +++- tests/unit_tests/test_espidf_toolchain.py | 8 ++ .../unit_tests/test_platformio_pch_script.py | 55 +++++++++++++- tests/unit_tests/test_platformio_toolchain.py | 25 +++++++ 17 files changed, 410 insertions(+), 35 deletions(-) diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index c66292a8363..fd198b2e0c7 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -1053,12 +1053,9 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: + ["stdc++-exc" if config.exceptions else "stdc++", "m", "c", "gcc"] ) + cxx = (toolchain_tool(paths.toolchain, "g++"),) lines = [ - *tool_lines( - (toolchain_tool(paths.toolchain, "gcc"),), - (toolchain_tool(paths.toolchain, "g++"),), - ccache, - ), + *tool_lines((toolchain_tool(paths.toolchain, "gcc"),), cxx, ccache), *compile_rule_lines(), *ar_rule_lines(toolchain_tool(paths.toolchain, "ar")), *pch_rule_lines(), @@ -1146,6 +1143,7 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: flag_sets.cxxflags, src_other, (str(paths.framework), str(paths.toolchain)), + cxx, ) src_objs = compile_edges( lines, diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 766c945467f..9f48013a4e8 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -3,6 +3,7 @@ import json import logging from pathlib import Path +import textwrap from esphome.build_helpers import pch from esphome.components.esp32 import ( @@ -289,6 +290,8 @@ target_link_options(${{COMPONENT_LIB}} PUBLIC # Where CMake puts the .gch of the src component; ccache reads the checksum # next to it in place of the .gch _PCH_SUM_PATH = "build/esp-idf/src/CMakeFiles/__idf_src.dir/cmake_pch.hxx.gch.sum" +# Where the Windows gate records its choice +_PCH_CHOICE_VAR = "ESPHOME_PCH" def _pch_cmake_block() -> str: @@ -300,11 +303,21 @@ def _pch_cmake_block() -> str: f' "$<$:${{CMAKE_CURRENT_SOURCE_DIR}}/{header}>"' for header in pch.PCH_DEFAULT_HEADERS ) - return f""" -# ESPHome precompiled header -target_precompile_headers(${{COMPONENT_LIB}} PRIVATE + block = f"""target_precompile_headers(${{COMPONENT_LIB}} PRIVATE {headers} -) +)""" + if not pch.pch_needs_gcc_check(): + return f"\n# ESPHome precompiled header\n{block}\n" + # Before the first configure only CMake knows the compiler version + return f""" +# ESPHome precompiled header, unless GCC bug 14940 keeps it from loading +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND ({pch.PCH_WINDOWS_CMAKE_OLD_GCC})) + message(STATUS "ESPHome: GCC ${{CMAKE_CXX_COMPILER_VERSION}} cannot load a precompiled header on Windows; compiling without it") + set({_PCH_CHOICE_VAR} OFF CACHE BOOL "ESPHome precompiled header in use" FORCE) +else() + set({_PCH_CHOICE_VAR} ON CACHE BOOL "ESPHome precompiled header in use" FORCE) +{textwrap.indent(block, " ")} +endif() """ @@ -315,8 +328,13 @@ def _read_if_exists(path: Path) -> str: def write_pch_checksum() -> None: """Write the checksum ccache uses in place of the .gch: the core headers, the framework version, the sdkconfig and the managed component versions.""" + from esphome.espidf.toolchain import get_cmake_cache_value + if not pch.pch_enabled(): return + # The gate's choice, cached by configure + if pch.pch_needs_gcc_check() and get_cmake_cache_value(_PCH_CHOICE_VAR) != "ON": + return pch.log_pch_in_use() checksum = pch.pch_checksum( CORE.relative_src_path(), diff --git a/esphome/build_gen/host.py b/esphome/build_gen/host.py index 4f7c556a488..f1e53e1ccb2 100644 --- a/esphome/build_gen/host.py +++ b/esphome/build_gen/host.py @@ -274,6 +274,7 @@ def write_project(compilers: HostCompilers, ccache: str | None) -> bool: [*compilers.cxx[1:], *cxxflags], (), (compilers.cxx[0], _compiler_version(compilers.cxx)) if pch_enabled() else (), + compilers.cxx, ) src_objs = compile_edges( lines, diff --git a/esphome/build_helpers/ninja_gen.py b/esphome/build_helpers/ninja_gen.py index 40cc8e7400a..8e72b36b3fd 100644 --- a/esphome/build_helpers/ninja_gen.py +++ b/esphome/build_helpers/ninja_gen.py @@ -24,8 +24,8 @@ from esphome.build_helpers.pch import ( PCH_SUM_NAME, log_pch_in_use, pch_consumer_flags, - pch_enabled, pch_identity, + pch_usable, write_pch_headers, ) from esphome.framework_helpers import strip_win_long_path_prefix @@ -125,15 +125,17 @@ def pch_edges( cxxflags: Sequence[str], src_flags: Sequence[str], identity: Sequence[str], + cxx: Sequence[Path | str], ) -> tuple[str, str] | None: """Emit the precompiled header for the C++ src edges. ``headers`` are folded into one prefix header, ``src_flags`` are the - flags every src edge carries and ``identity`` names what else the - compile depends on, the compiler included. Returns the ``cxx_override`` - for ``compile_edges``, or None without a pch. + flags every src edge carries, ``identity`` names what else the compile + depends on, the compiler included, and ``cxx`` is what the host rule + asks. Returns the ``cxx_override`` for ``compile_edges``, or None + without a pch. """ - if not pch_enabled(): + if not pch_usable(cxx): return None if any( tok == "-include" or tok.startswith("--include") or is_joined_include(tok) diff --git a/esphome/build_helpers/pch.py b/esphome/build_helpers/pch.py index ec0ff416e50..c2e04c9c651 100644 --- a/esphome/build_helpers/pch.py +++ b/esphome/build_helpers/pch.py @@ -2,13 +2,15 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Sequence import hashlib import logging import os from pathlib import Path import posixpath import re +import subprocess +import sys from esphome.build_helpers.ccache import effective_ccache_basedir, parse_enable_env from esphome.const import PLATFORM_NRF52 @@ -63,6 +65,76 @@ def pch_enabled() -> bool: return parse_enable_env("ESPHOME_PCH_ENABLE") is not False +def pch_forced() -> bool: + """``ESPHOME_PCH_ENABLE=1``: wanted even where the host rule says no.""" + return parse_enable_env("ESPHOME_PCH_ENABLE") is True + + +# GCC bug 14940: before these releases the Windows loader maps a .gch only +# at its saved address. First fixed release per major, 16 on always fixed; +# PCH_WINDOWS_CMAKE_OLD_GCC and the pch_usable message spell the same table +PCH_WINDOWS_GCC_FIXED = {14: (14, 4), 15: (15, 3)} +PCH_WINDOWS_GCC_FIXED_DEFAULT = (16, 0) +# The same rule for CMake, which alone knows the version before configure +PCH_WINDOWS_CMAKE_OLD_GCC = ( + "CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14.4 OR " + "(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15 AND " + "CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15.3)" +) + + +def gcc_relocates_pch_on_windows(version: Sequence[int]) -> bool: + """Whether a GCC of this version loads a .gch on Windows.""" + if not version: + return False + fixed = PCH_WINDOWS_GCC_FIXED.get(version[0], PCH_WINDOWS_GCC_FIXED_DEFAULT) + return tuple(version[:2]) >= fixed + + +# GCC ends the first --version line with its version; clang names itself +_VERSION_RE = re.compile(r"\d+(?:\.\d+)+") + + +def gcc_version(cxx: Sequence[Path | str]) -> tuple[int, ...] | None: + """The GCC version from ``--version``: () when it cannot be read, None + for a compiler that is not GCC.""" + try: + result = subprocess.run( + [*cxx, "--version"], capture_output=True, text=True, check=False + ) + except OSError as err: + _LOGGER.debug("Cannot run %s: %s", cxx[0], err) + return () + banner = result.stdout.partition("\n")[0] + if "clang" in banner.lower(): + return None + found = _VERSION_RE.findall(banner) + return tuple(int(part) for part in found[-1].split(".")) if found else () + + +def pch_needs_gcc_check() -> bool: + """Windows host with the knob unset: the compiler version decides.""" + return sys.platform == "win32" and parse_enable_env("ESPHOME_PCH_ENABLE") is None + + +def pch_usable(cxx: Sequence[Path | str]) -> bool: + """The knob plus the host rule; ``ESPHOME_PCH_ENABLE=1`` skips the rule.""" + if not pch_enabled(): + return False + if not pch_needs_gcc_check(): + return True + version = gcc_version(cxx) + if version is None or gcc_relocates_pch_on_windows(version): + return True + _LOGGER.info( + "GCC %s cannot load a precompiled header on Windows (GCC bug 14940, " + "fixed in 14.4, 15.3 and 16); compiling without it " + "(set ESPHOME_PCH_ENABLE=1 to force)", + ".".join(map(str, version)) or "of unknown version", + ) + return False + + def pch_consumer_flags() -> list[str]: """Flags a C++ src compile loads the pch with. The -include stays relative: an absolute path would enter the ccache key.""" diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 488984ff70d..8e91cff0497 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -75,6 +75,7 @@ from .framework import ( get_build_env, get_build_paths, setup_platformio_python_env, + toolchain_tool, ) # force import gpio to register pin schema @@ -843,7 +844,11 @@ def _write_pch_checksum(build_dir: Path, source_dir: Path) -> None: write_file_if_changed(app_dir / _PCH_SUM_PATH, checksum + "\n") -def _generate_cmake_lists() -> bool: +def _pch_usable() -> bool: + return pch.pch_usable((toolchain_tool("g++"),)) + + +def _generate_cmake_lists(pch_on: bool) -> bool: """Write the project CMakeLists.txt, returning True if it changed.""" compile_flags = get_project_compile_flags() link_flags = get_project_link_flags() @@ -886,7 +891,7 @@ def _generate_cmake_lists() -> bool: ")", ] - if pch.pch_enabled(): + if pch_on: lines += _PCH_CMAKE_LINES if link_flags: @@ -945,7 +950,8 @@ def run_compile(args, config: ConfigType) -> bool: paths = get_build_paths() env = get_build_env() - cmake_lists_changed = _generate_cmake_lists() + pch_on = _pch_usable() + cmake_lists_changed = _generate_cmake_lists(pch_on) board = zephyr_data()[KEY_BOARD] build_dir = CORE.relative_pioenvs_path(CORE.name) @@ -962,7 +968,7 @@ def run_compile(args, config: ConfigType) -> bool: _LOGGER.info("Build inputs changed, cleaning %s", build_dir) rmtree(build_dir) - if pch.pch_enabled(): + if pch_on: pch.log_pch_in_use() # Zephyr turns ccache on by itself when it is installed env.update(pch.ccache_pch_env()) diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index f8575234f88..3bb1f5b83d5 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -605,6 +605,13 @@ def get_elf_path() -> Path: return build_dir / "firmware.elf" +def get_cmake_cache_value(var_name: str) -> str | None: + """One entry of the configured build's CMake cache, or None when unset.""" + cmake_output = _get_cmake_output(CORE.relative_build_path("build")) + match = re.search(rf"^{var_name}:\w+=(.*)$", cmake_output, re.MULTILINE) + return match.group(1).strip() if match else None + + def get_objdump_path() -> Path: return _get_cmake_tool_path("CMAKE_OBJDUMP") diff --git a/esphome/platformio/pch.py.script b/esphome/platformio/pch.py.script index 960d5bd7d9e..36751074d96 100644 --- a/esphome/platformio/pch.py.script +++ b/esphome/platformio/pch.py.script @@ -60,6 +60,10 @@ rc = libc.posix_spawn(None, argv[0], None, attr, (ctypes.c_char_p * len(argv))(* libc.strerror.restype = ctypes.c_char_p sys.exit("ESPHome: cannot start " + real + ": " + libc.strerror(rc).decode()) """ +# GCC bug 14940: before these releases the Windows loader maps a .gch only +# at its saved address. First fixed release per major, 16 on always fixed +_WINDOWS_GCC_FIXED = {14: (14, 4), 15: (15, 3)} +_WINDOWS_GCC_FIXED_DEFAULT = (16, 0) _CXX_SUFFIXES = (".cpp", ".cc", ".cxx") _HEADER_SUFFIXES = (".h", ".hpp", ".hh", ".inc") _INCLUDE_DIR_OPTIONS = ("-I", "-isystem", "-iquote", "-idirafter") @@ -113,7 +117,11 @@ def _src_envs(src_dir: Path) -> list: def _driver_output(cxx: str, option: str) -> str: - result = subprocess.run([cxx, option], capture_output=True, text=True) # noqa: PLW1510 + """What the driver prints for ``option``, or "" when it cannot run.""" + try: + result = subprocess.run([cxx, option], capture_output=True, text=True) # noqa: PLW1510 + except OSError: + return "" return result.stdout.strip() @@ -141,6 +149,26 @@ def _old_gcc_flags(proj_dir: Path, pio_platform: str, cxx: str): return [f"-B{_CC1_DIR}/"] +def _gcc_loads_pch_here(cxx: str) -> bool: + """Whether a .gch loads here; ESPHOME_PCH_ENABLE=1, as esphome + normalizes it, insists.""" + if sys.platform != "win32" or os.environ.get("ESPHOME_PCH_ENABLE") == "1": + return True + # GCC ends the first --version line with its version; clang names itself + banner = _driver_output(cxx, "--version").partition("\n")[0] + if "clang" in banner.lower(): + return True + found = re.findall(r"\d+(?:\.\d+)+", banner) + version = tuple(int(p) for p in found[-1].split(".")) if found else () + if version and version[:2] >= _WINDOWS_GCC_FIXED.get(version[0], _WINDOWS_GCC_FIXED_DEFAULT): + return True + print( + f"ESPHome: GCC {'.'.join(map(str, version)) or 'of unknown version'} cannot load a " + "precompiled header on Windows (GCC bug 14940); compiling without it" + ) + return False + + def _setup_pch() -> None: if projenv is None: if "nobuild" in COMMAND_LINE_TARGETS: # noqa: F821 @@ -157,6 +185,8 @@ def _setup_pch() -> None: src_envs = _src_envs(src_dir) cxx = src_envs[0].subst("$CXX") + if not _gcc_loads_pch_here(cxx): + return pio_platform = env.PioPlatform() # noqa: F821 extra = _old_gcc_flags(proj_dir, pio_platform.name, cxx) if extra is None: diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 6a906a42d06..0eec532c7c7 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -336,10 +336,17 @@ def run_platformio_cli(*args, **kwargs) -> str | int: env = dict(os.environ if base_env is None else base_env) env.update(_ccache_env()) if CORE.is_configured: - from esphome.build_helpers.pch import ccache_pch_env, pch_script_enabled + from esphome.build_helpers.pch import ( + ccache_pch_env, + pch_forced, + pch_script_enabled, + ) if pch_script_enabled(): env.update(ccache_pch_env()) + if pch_forced(): + # Normalized for the script, like ESPHOME_CCACHE_ENABLE + env["ESPHOME_PCH_ENABLE"] = "1" # The runner offers the out-of-flash tip but has no configured CORE, so # tell it. Ask CORE, not is_esp32_arduino_build(), which reads this same # variable; clear an inherited one so it cannot reach the wrong build. diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py index 8bd304b42be..4a0355fd010 100644 --- a/tests/unit_tests/build_gen/test_arduino8266.py +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -447,6 +447,19 @@ def test_write_project_pch_disabled( assert " flags = $srcflags" in content +def test_write_project_pch_asks_the_toolchain_compiler_on_windows( + windows_gcc_rule: None, tmp_path: Path +) -> None: + from esphome.build_helpers import pch + + paths = _make_framework(tmp_path) + _set_flags("-DPIO_FRAMEWORK_ARDUINO_LWIP2_HIGHER_BANDWIDTH_LOW_FLASH") + with patch.object(pch, "gcc_version", return_value=(10, 3, 0)) as asked: + content = _write_ninja(paths) + assert asked.call_args.args[0] == (toolchain_tool(paths.toolchain, "g++"),) + assert "esphome_pch" not in content + + def test_write_project_scanf_float_and_waveform_kept(tmp_path: Path) -> None: paths = _make_framework(tmp_path) CORE.data[KEY_ESP8266][KEY_SCANF_FLOAT] = True diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index ec8e2f3a364..4a842847962 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -538,6 +538,54 @@ def test_component_cmakelists_pch_block(monkeypatch: pytest.MonkeyPatch) -> None assert "target_precompile_headers" not in get_component_cmakelists() +def test_component_cmakelists_pch_gate_on_windows( + windows_gcc_rule: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """The block carries the rule and records its choice; the knob drops + the gate.""" + from esphome.build_gen.espidf import get_component_cmakelists + + content = get_component_cmakelists() + assert ( + 'if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND ' + "(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14.4 OR " + "(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15 AND " + "CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15.3)))\n" + " message(STATUS " in content + ) + assert ( + ' set(ESPHOME_PCH OFF CACHE BOOL "ESPHome precompiled header in use" FORCE)\nelse()\n' + in content + ) + assert ( + ' set(ESPHOME_PCH ON CACHE BOOL "ESPHome precompiled header in use" FORCE)\n target_precompile_headers(${COMPONENT_LIB} PRIVATE\n' + in content + ) + assert "endif()" in content + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "1") + content = get_component_cmakelists() + assert "if(CMAKE_CXX_COMPILER_VERSION" not in content + assert "\ntarget_precompile_headers(${COMPONENT_LIB} PRIVATE\n" in content + + +@pytest.mark.parametrize(("choice", "written"), [("OFF", False), ("ON", True)]) +def test_pch_checksum_follows_the_cmake_choice_on_windows( + windows_gcc_rule: None, tmp_path: Path, choice: str, written: bool +) -> None: + from esphome.build_gen.espidf import _PCH_SUM_PATH, write_pch_checksum + + _make_pch_project(tmp_path) + with ( + patch( + "esphome.espidf.toolchain.get_cmake_cache_value", return_value=choice + ) as asked, + patch.object(CORE, "name", "test"), + ): + write_pch_checksum() + assert asked.call_args.args == ("ESPHOME_PCH",) + assert CORE.relative_build_path(_PCH_SUM_PATH).exists() is written + + @pytest.mark.parametrize( ("file", "content"), [ diff --git a/tests/unit_tests/build_helpers/test_pch.py b/tests/unit_tests/build_helpers/test_pch.py index a03b9c3a11d..1182dc6f8cd 100644 --- a/tests/unit_tests/build_helpers/test_pch.py +++ b/tests/unit_tests/build_helpers/test_pch.py @@ -2,8 +2,10 @@ from __future__ import annotations +import logging import os from pathlib import Path +import sys from unittest.mock import patch import pytest @@ -35,6 +37,71 @@ def test_pch_enabled(value: str | None, expected: bool) -> None: assert pch.pch_enabled() is expected +@pytest.mark.parametrize( + ("version", "expected"), + [ + ((), False), + ((10, 3), False), + ((12, 2, 1), False), + ((14, 2, 0), False), + ((14, 3), False), + ((14, 4), True), + ((14,), False), + ((15, 2, 0), False), + ((15, 3), True), + ((16, 0), True), + ((17, 1), True), + ], +) +def test_gcc_relocates_pch_on_windows(version: tuple[int, ...], expected: bool) -> None: + assert pch.gcc_relocates_pch_on_windows(version) is expected + + +@pytest.mark.parametrize( + ("banner", "expected"), + [ + ("xtensa-esp32-elf-g++ (crosstool-NG esp-14.2.0_20260121) 14.2.0", (14, 2, 0)), + ("arm-zephyr-eabi-g++ (Zephyr SDK 0.16.8) 12.2.0", (12, 2, 0)), + ("g++.exe (Rev3, Built by MSYS2 project) 14.2.0", (14, 2, 0)), + ("clang version 15.0.0", None), + ("Apple clang version 17.0.0 (clang-1700.0.13.3)", None), + ("something else", ()), + ], +) +def test_gcc_version_reads_the_banner(banner: str, expected: object) -> None: + cxx = (sys.executable, "-c", f"print({banner!r}); print('more')") + assert pch.gcc_version(cxx) == expected + + +def test_gcc_version_of_a_compiler_that_cannot_run() -> None: + assert pch.gcc_version(("/nonexistent/g++",)) == () + + +def test_pch_usable_asks_the_compiler_on_windows_only( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + caplog.set_level(logging.INFO, logger=pch.__name__) + monkeypatch.delenv("ESPHOME_PCH_ENABLE") + monkeypatch.setattr(pch.sys, "platform", "darwin") + with patch.object(pch, "gcc_version", side_effect=AssertionError("off Windows")): + assert pch.pch_usable(("g++",)) + monkeypatch.setattr(pch.sys, "platform", "win32") + with patch.object(pch, "gcc_version", return_value=(14, 2, 0)): + assert not pch.pch_usable(("g++",)) + assert "GCC 14.2.0 cannot load a precompiled header on Windows" in caplog.text + with patch.object(pch, "gcc_version", return_value=(14, 4, 0)): + assert pch.pch_usable(("g++",)) + # The bug is GCC's; another compiler is not held to its table + with patch.object(pch, "gcc_version", return_value=None): + assert pch.pch_usable(("clang++",)) + # The knob overrides the rule both ways + with patch.object(pch, "gcc_version", side_effect=AssertionError("forced")): + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "1") + assert pch.pch_usable(("g++",)) + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") + assert not pch.pch_usable(("g++",)) + + def test_ccache_pch_env_enabled() -> None: with patch.dict(os.environ, {}, clear=True): env = pch.ccache_pch_env() diff --git a/tests/unit_tests/components/nrf52/test_pch.py b/tests/unit_tests/components/nrf52/test_pch.py index 0052df07720..5af55586c33 100644 --- a/tests/unit_tests/components/nrf52/test_pch.py +++ b/tests/unit_tests/components/nrf52/test_pch.py @@ -13,7 +13,7 @@ from esphome.core import CORE, EsphomeError SUM = "CMakeFiles/app.dir/cmake_pch.hxx.gch.sum" -def _generate_cmake(tmp_path: Path) -> str: +def _generate_cmake(tmp_path: Path, pch_on: bool = True) -> str: CORE.config_path = tmp_path / "test.yaml" CORE.build_path = tmp_path / "build" CORE.name = "livingroom" @@ -25,7 +25,7 @@ def _generate_cmake(tmp_path: Path) -> str: patch.object(nrf52, "get_project_compile_flags", return_value=["-Os"]), patch.object(nrf52, "get_project_link_flags", return_value=[]), ): - nrf52._generate_cmake_lists() + nrf52._generate_cmake_lists(pch_on) return (tmp_path / "build" / "zephyr" / "CMakeLists.txt").read_text() @@ -43,15 +43,27 @@ def test_cmake_lists_precompile_the_core_headers(tmp_path: Path) -> None: assert "if(NOT esphome_pch_headers)\n message(FATAL_ERROR" in text -def test_cmake_lists_pch_block_disabled( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - monkeypatch.setenv("ESPHOME_PCH_ENABLE", "0") - text = _generate_cmake(tmp_path) +def test_cmake_lists_pch_block_disabled(tmp_path: Path) -> None: + text = _generate_cmake(tmp_path, pch_on=False) assert "precompile" not in text assert "zephyr_interface" not in text +@pytest.mark.parametrize(("version", "on"), [((12, 2, 0), False), ((14, 4, 0), True)]) +def test_the_zephyr_compiler_decides_on_windows( + windows_gcc_rule: None, version: tuple[int, ...], on: bool +) -> None: + from esphome.build_helpers import pch + + # platformdirs would pick its Windows backend from the patched sys.platform + with ( + patch.object(nrf52, "toolchain_tool", lambda name: Path(f"/sdk/{name}.exe")), + patch.object(pch, "gcc_version", return_value=version) as asked, + ): + assert nrf52._pch_usable() is on + assert asked.call_args.args[0] == (Path("/sdk/g++.exe"),) + + def _write_checksum(tmp_path: Path, app: str, conf: str = "CONFIG_X=y\n") -> Path: """Write the checksum for a build dir whose app image sits in ``app``.""" CORE.build_path = tmp_path @@ -114,9 +126,10 @@ def run_cmd(tmp_path: Path) -> Mock: def test_ccache_pch_settings_reach_west(run_cmd: Mock) -> None: - # clear=True also drops ambient CCACHE_* and ESPHOME_PCH_* overrides + # clear=True also drops ambient CCACHE_* overrides; the header is on + # explicitly since Windows hosts start with it off with ( - patch.dict("os.environ", {}, clear=True), + patch.dict("os.environ", {"ESPHOME_PCH_ENABLE": "1"}, clear=True), pytest.raises(EsphomeError, match="nRF52 native build failed"), ): nrf52.run_compile(None, {}) diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py index 0353c8afc83..ff5564e8150 100644 --- a/tests/unit_tests/conftest.py +++ b/tests/unit_tests/conftest.py @@ -178,6 +178,15 @@ def held_lock() -> Callable[..., Callable[..., None]]: @pytest.fixture(autouse=True) def _default_pch_knobs(monkeypatch: pytest.MonkeyPatch) -> None: - """The precompiled header knobs change what the generators emit; a value - exported in the developer's or CI's shell must not reach the tests.""" - monkeypatch.delenv("ESPHOME_PCH_ENABLE", raising=False) + """The knob changes what the generators emit: no ambient value, and on, + so the Windows runner asks no compiler.""" + monkeypatch.setenv("ESPHOME_PCH_ENABLE", "1") + + +@pytest.fixture +def windows_gcc_rule(monkeypatch: pytest.MonkeyPatch) -> None: + """Windows host, knob unset: the compiler version decides.""" + from esphome.build_helpers import pch + + monkeypatch.delenv("ESPHOME_PCH_ENABLE") + monkeypatch.setattr(pch.sys, "platform", "win32") diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 2d655636632..3e1e88d66da 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -725,3 +725,11 @@ def test_get_core_framework_version_from_core_data(): CORE.data = {KEY_ESP32: {KEY_IDF_VERSION: cv.Version(5, 5, 4)}} assert toolchain._get_core_framework_version() == "5.5.4" + + +def test_get_cmake_cache_value_reads_the_configured_cache(setup_core: Path) -> None: + _setup_build(setup_core) + output = "CMAKE_OBJDUMP:FILEPATH=/tools/objdump\nESPHOME_PCH:BOOL=OFF\n" + with patch.object(toolchain, "_get_cmake_output", return_value=output): + assert toolchain.get_cmake_cache_value("ESPHOME_PCH") == "OFF" + assert toolchain.get_cmake_cache_value("ESPHOME_MISSING") is None diff --git a/tests/unit_tests/test_platformio_pch_script.py b/tests/unit_tests/test_platformio_pch_script.py index e7ea1a34201..af9ec4a8377 100644 --- a/tests/unit_tests/test_platformio_pch_script.py +++ b/tests/unit_tests/test_platformio_pch_script.py @@ -81,7 +81,9 @@ def _fake_cxx(tmp_path: Path, fail: bool = False) -> Path: 'case "$1" in -print-prog-name=*) n=${1#*=};' ' p="$(dirname "$0")/../libexec/gcc/arm-none-eabi/10.3.1/$n";' ' [ -x "$p" ] && echo "$p" || echo "$n"; exit 0;;' - ' -dumpversion) echo "${FAKE_GCC_VERSION:-10.3.1}"; exit 0;; esac\n' + ' -dumpversion) echo "${FAKE_GCC_VERSION:-10.3.1}"; exit 0;;' + ' --version) echo "${FAKE_GCC_BANNER:-fake-g++ (test) ${FAKE_GCC_VERSION:-10.3.1}}";' + " exit 0;; esac\n" 'printf -- ---call---\\\\n >> "$0.argv"; printf \'%s\\n\' "$@" >> "$0.argv"\n' ) if fail: @@ -239,7 +241,8 @@ def test_pch_script_gcc10_skipped_elsewhere( ("linux", "aarch64", _LibreTinyPlatform, "10.3.1"), # From GCC 12 the .gch loads at any address, so the rule retires itself ("darwin", "arm64", _LibreTinyPlatform, "12.2.0"), - ("win32", "AMD64", _LibreTinyPlatform, "14.2.0"), + # Windows needs the fix for GCC bug 14940 as well + ("win32", "AMD64", _LibreTinyPlatform, "14.4.0"), ], ) def test_pch_script_no_wrapper_where_the_gch_loads( @@ -272,6 +275,54 @@ def test_pch_script_names_match_the_python_side(tmp_path: Path) -> None: assert namespace["_GUARD_TEXT"] == pch.PCH_GUARD_TEXT assert namespace["_INCLUDE_RE"].pattern == pch._INCLUDE_RE.pattern assert namespace["_CC1_DIR"] == pch.PCH_CC1_DIR + assert namespace["_WINDOWS_GCC_FIXED"] == pch.PCH_WINDOWS_GCC_FIXED + assert namespace["_WINDOWS_GCC_FIXED_DEFAULT"] == pch.PCH_WINDOWS_GCC_FIXED_DEFAULT + + +@pytest.mark.parametrize( + ("version", "env_vars", "on"), + [ + ("14.2.0", {}, False), + ("14.4.0", {}, True), + ("14.2.0", {"ESPHOME_PCH_ENABLE": "1"}, True), + ], +) +def test_pch_script_asks_the_compiler_on_windows( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + version: str, + env_vars: dict[str, str], + on: bool, +) -> None: + """The Python rule again, plus the normalized knob.""" + monkeypatch.setattr(sys, "platform", "win32") + scons_env = _run_script( + tmp_path, env_vars={"FAKE_GCC_VERSION": version, **env_vars} + ) + assert (scons_env.prepended == pch.pch_consumer_flags()) is on + assert (tmp_path / "dev" / "esphome_pch.h.gch").is_file() is on + out = capsys.readouterr().out + assert ("cannot load a precompiled header on Windows" in out) is not on + + +def test_pch_script_spares_another_compiler_the_gcc_rule( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(sys, "platform", "win32") + scons_env = _run_script( + tmp_path, env_vars={"FAKE_GCC_BANNER": "clang version 15.0.0"} + ) + assert scons_env.prepended == pch.pch_consumer_flags() + + +def test_pch_script_leaves_the_header_off_when_the_compiler_cannot_run( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr(sys, "platform", "win32") + scons_env = _run_script(tmp_path, cxx=tmp_path / "missing-g++") + assert scons_env.prepended == [] + assert "GCC of unknown version cannot load" in capsys.readouterr().out def test_pch_script_compile_failure_stops_the_build(tmp_path: Path) -> None: diff --git a/tests/unit_tests/test_platformio_toolchain.py b/tests/unit_tests/test_platformio_toolchain.py index d218f3201f5..60a5d1d4235 100644 --- a/tests/unit_tests/test_platformio_toolchain.py +++ b/tests/unit_tests/test_platformio_toolchain.py @@ -354,6 +354,31 @@ def test_run_platformio_cli_exports_the_pch_ccache_settings( assert env.get("CCACHE_PCH_EXTSUM") == expected +@pytest.mark.parametrize( + ("value", "expected"), [("yes", "1"), ("0", "0"), (None, None)] +) +def test_run_platformio_cli_normalizes_a_forced_pch_for_the_script( + setup_core: Path, + mock_run_external_process: Mock, + value: str | None, + expected: str | None, +) -> None: + """The script cannot import the knob parser, so it only reads a ``1``.""" + CORE.build_path = str(setup_core / "build" / "test") + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: "esp32", + KEY_TARGET_FRAMEWORK: "esp-idf", + } + + env_vars = {} if value is None else {"ESPHOME_PCH_ENABLE": value} + with patch.dict(os.environ, env_vars, clear=True): + mock_run_external_process.return_value = 0 + toolchain.run_platformio_cli("test", "arg") + + env = mock_run_external_process.call_args[1]["env"] + assert env.get("ESPHOME_PCH_ENABLE") == expected + + @pytest.mark.parametrize( ("platform", "framework", "expected"), [ From 55a7340d0672b74710754f632023ab76a71dc580 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 18:52:28 +0200 Subject: [PATCH 043/133] [esp8266] Make the native arduino toolchain the default (#19886) --- docker/test_configs/esp8266-arduino.yaml | 2 ++ esphome/__main__.py | 3 +- esphome/components/esp8266/__init__.py | 14 +++++++- .../esp8266/test_toolchain_validation.py | 34 +++++++++++++++++-- tests/unit_tests/test_vscode.py | 4 +-- 5 files changed, 50 insertions(+), 7 deletions(-) diff --git a/docker/test_configs/esp8266-arduino.yaml b/docker/test_configs/esp8266-arduino.yaml index 80b52260e4d..5a80fd5b1f1 100644 --- a/docker/test_configs/esp8266-arduino.yaml +++ b/docker/test_configs/esp8266-arduino.yaml @@ -3,5 +3,7 @@ esphome: esp8266: board: d1_mini + # The PlatformIO path stays covered whatever the default is + toolchain: platformio logger: diff --git a/esphome/__main__.py b/esphome/__main__.py index e67bc42764a..01ca8ca5d96 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2367,7 +2367,8 @@ def parse_args(argv): metavar="{" + ",".join(t.value for t in Toolchain) + "}", help=( "Select toolchain for compiling. Overrides '.toolchain' in YAML. " - f"Default: {Toolchain.PLATFORMIO.value}." + "Default: the platform's native toolchain where it has one, else " + f"{Toolchain.PLATFORMIO.value}." ), ) diff --git a/esphome/components/esp8266/__init__.py b/esphome/components/esp8266/__init__.py index 0e5303e8df1..e4d8dfc4928 100644 --- a/esphome/components/esp8266/__init__.py +++ b/esphome/components/esp8266/__init__.py @@ -111,7 +111,18 @@ def set_core_data(config: ConfigType) -> ConfigType: _TOOLCHAINS = (Toolchain.PLATFORMIO, Toolchain.ARDUINO) _validate_toolchain = cv.toolchain_enum(_TOOLCHAINS) -_resolve_toolchain = cv.resolve_toolchain("ESP8266", _TOOLCHAINS, Toolchain.PLATFORMIO) +_resolve_toolchain = cv.resolve_toolchain("ESP8266", _TOOLCHAINS, Toolchain.ARDUINO) + + +def _warn_platformio_toolchain(config: ConfigType) -> ConfigType: + # Remove before 2027.4.0 + if CORE.using_toolchain_platformio: + _LOGGER.warning( + "The 'platformio' toolchain for ESP8266 is deprecated and will be " + "removed in ESPHome 2027.4.0; the native 'arduino' toolchain is the " + "default." + ) + return config def _validate_native_toolchain(config: ConfigType) -> ConfigType: @@ -319,6 +330,7 @@ CONFIG_SCHEMA = cv.All( } ), _resolve_toolchain, + _warn_platformio_toolchain, _validate_native_toolchain, set_core_data, ) diff --git a/tests/unit_tests/components/esp8266/test_toolchain_validation.py b/tests/unit_tests/components/esp8266/test_toolchain_validation.py index d54c2c35334..e7f443c2d23 100644 --- a/tests/unit_tests/components/esp8266/test_toolchain_validation.py +++ b/tests/unit_tests/components/esp8266/test_toolchain_validation.py @@ -14,6 +14,7 @@ from esphome.components.esp8266 import ( ARDUINO_FRAMEWORK_SCHEMA, _resolve_toolchain, _validate_native_toolchain, + _warn_platformio_toolchain, ) import esphome.config_validation as cv from esphome.const import ( @@ -125,10 +126,22 @@ def test_yaml_toolchain_key_resolves() -> None: assert CORE.using_toolchain_arduino -def test_yaml_toolchain_key_defaults_to_platformio() -> None: +@pytest.mark.parametrize( + ("config_toolchain", "expected"), + [ + (None, Toolchain.ARDUINO), + # An explicit `toolchain:` still wins over the default + (Toolchain.PLATFORMIO, Toolchain.PLATFORMIO), + (Toolchain.ARDUINO, Toolchain.ARDUINO), + ], +) +def test_default_toolchain_is_arduino( + config_toolchain: Toolchain | None, expected: Toolchain +) -> None: CORE.toolchain = None - _resolve_toolchain({}) - assert CORE.toolchain == Toolchain.PLATFORMIO + config = {} if config_toolchain is None else {CONF_TOOLCHAIN: config_toolchain} + _resolve_toolchain(config) + assert CORE.toolchain == expected def test_decode_pc_native_missing_tools_warns_once( @@ -198,3 +211,18 @@ def test_copy_files_native_skips_platformio_scripts(tmp_path: Path) -> None: CORE.build_path = tmp_path esp8266.copy_files() assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize( + ("toolchain", "warns"), + [(Toolchain.PLATFORMIO, True), (Toolchain.ARDUINO, False)], +) +def test_platformio_toolchain_deprecation_warning( + toolchain: Toolchain, warns: bool, caplog: pytest.LogCaptureFixture +) -> None: + CORE.toolchain = toolchain + config = _config() + assert _warn_platformio_toolchain(config) is config + assert ( + "deprecated and will be removed in ESPHome 2027.4.0" in caplog.text + ) is warns diff --git a/tests/unit_tests/test_vscode.py b/tests/unit_tests/test_vscode.py index 9b7d1e95040..7d401eaa653 100644 --- a/tests/unit_tests/test_vscode.py +++ b/tests/unit_tests/test_vscode.py @@ -58,7 +58,7 @@ esp8266: board: !secret my_secret_board """), # read_file secrets.yaml - _file_response("""my_secret_board: esp1f"""), + _file_response("""my_secret_board: d1_mini"""), ] ) @@ -83,7 +83,7 @@ esp8266: broad: !secret my_secret_board # typo here """), # read_file secrets.yaml - _file_response("""my_secret_board: esp1f"""), + _file_response("""my_secret_board: d1_mini"""), ] ) From 1a9f43909a42e19d958cda0bbb87bb47e9a560f2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 19:53:59 +0200 Subject: [PATCH 044/133] [mopeka_std_check] Use the sub_* helpers for optional child entities (#19897) --- esphome/components/mopeka_std_check/sensor.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/esphome/components/mopeka_std_check/sensor.py b/esphome/components/mopeka_std_check/sensor.py index d5c5d4135d6..b3e55766d8a 100644 --- a/esphome/components/mopeka_std_check/sensor.py +++ b/esphome/components/mopeka_std_check/sensor.py @@ -128,15 +128,8 @@ async def to_code(config: ConfigType) -> None: if CONF_PROPANE_BUTANE_MIX in config: cg.add(var.set_propane_butane_mix(config[CONF_PROPANE_BUTANE_MIX])) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_LEVEL]) - cg.add(var.set_level(sens)) - if CONF_DISTANCE in config: - sens = await sensor.new_sensor(config[CONF_DISTANCE]) - cg.add(var.set_distance(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_LEVEL, var.set_level) + await sensors(CONF_DISTANCE, var.set_distance) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From 1b86959ee0e8dac8f791061299b6d844e4d5872e Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 19:54:11 +0200 Subject: [PATCH 045/133] [inkbird_ibsth1_mini] Use the sub_* helpers for optional child entities (#19896) --- .../components/inkbird_ibsth1_mini/sensor.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/esphome/components/inkbird_ibsth1_mini/sensor.py b/esphome/components/inkbird_ibsth1_mini/sensor.py index 84a207020ea..f6524021c08 100644 --- a/esphome/components/inkbird_ibsth1_mini/sensor.py +++ b/esphome/components/inkbird_ibsth1_mini/sensor.py @@ -71,15 +71,8 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_EXTERNAL_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_EXTERNAL_TEMPERATURE]) - cg.add(var.set_external_temperature(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_EXTERNAL_TEMPERATURE, var.set_external_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From 4618c73b0539885307badd14f8daf6a89bf77618 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 19:54:33 +0200 Subject: [PATCH 046/133] [ccs811] Use the sub_* helpers for optional child entities (#19895) --- esphome/components/ccs811/sensor.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/esphome/components/ccs811/sensor.py b/esphome/components/ccs811/sensor.py index d134d2cf21f..95e51c825a8 100644 --- a/esphome/components/ccs811/sensor.py +++ b/esphome/components/ccs811/sensor.py @@ -65,17 +65,12 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if eco2_config := config.get(CONF_ECO2): - sens = await sensor.new_sensor(eco2_config) - cg.add(var.set_co2(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_ECO2, var.set_co2) + await sensors(CONF_TVOC, var.set_tvoc) - if tvoc_config := config.get(CONF_TVOC): - sens = await sensor.new_sensor(tvoc_config) - cg.add(var.set_tvoc(sens)) - - if version_config := config.get(CONF_VERSION): - sens = await text_sensor.new_text_sensor(version_config) - cg.add(var.set_version(sens)) + text_sensors = text_sensor.sub_text_sensors(config) + await text_sensors(CONF_VERSION, var.set_version) if (baseline := config.get(CONF_BASELINE)) is not None: cg.add(var.set_baseline(baseline)) From b6cd92e915a56d4e7c0aa6a82bd300b119fd2ab1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 19:54:52 +0200 Subject: [PATCH 047/133] [hmc5883l] Use the sub_* helpers for optional child entities (#19893) --- esphome/components/hmc5883l/sensor.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/esphome/components/hmc5883l/sensor.py b/esphome/components/hmc5883l/sensor.py index a2e1f8054ad..229d6c8b623 100644 --- a/esphome/components/hmc5883l/sensor.py +++ b/esphome/components/hmc5883l/sensor.py @@ -136,15 +136,8 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_oversampling(config[CONF_OVERSAMPLING])) cg.add(var.set_datarate(auto_data_rate(config))) cg.add(var.set_range(config[CONF_RANGE])) - if CONF_FIELD_STRENGTH_X in config: - sens = await sensor.new_sensor(config[CONF_FIELD_STRENGTH_X]) - cg.add(var.set_x_sensor(sens)) - if CONF_FIELD_STRENGTH_Y in config: - sens = await sensor.new_sensor(config[CONF_FIELD_STRENGTH_Y]) - cg.add(var.set_y_sensor(sens)) - if CONF_FIELD_STRENGTH_Z in config: - sens = await sensor.new_sensor(config[CONF_FIELD_STRENGTH_Z]) - cg.add(var.set_z_sensor(sens)) - if CONF_HEADING in config: - sens = await sensor.new_sensor(config[CONF_HEADING]) - cg.add(var.set_heading_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_FIELD_STRENGTH_X, var.set_x_sensor) + await sensors(CONF_FIELD_STRENGTH_Y, var.set_y_sensor) + await sensors(CONF_FIELD_STRENGTH_Z, var.set_z_sensor) + await sensors(CONF_HEADING, var.set_heading_sensor) From a0f933ea604e4f8d271488b034a3fcea6d7709a2 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 19:54:55 +0200 Subject: [PATCH 048/133] [htu21d] Use the sub_* helpers for optional child entities (#19894) --- esphome/components/htu21d/sensor.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/esphome/components/htu21d/sensor.py b/esphome/components/htu21d/sensor.py index 08888c17e5c..d78267964b0 100644 --- a/esphome/components/htu21d/sensor.py +++ b/esphome/components/htu21d/sensor.py @@ -67,17 +67,10 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) - - if CONF_HEATER in config: - sens = await sensor.new_sensor(config[CONF_HEATER]) - cg.add(var.set_heater(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_HEATER, var.set_heater) cg.add(var.set_sensor_model(config[CONF_MODEL])) From b8ba8aa1a80d12788d27616d826e4dcc39dccea6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 19:55:22 +0200 Subject: [PATCH 049/133] [espidf] Run cmake and ninja directly instead of through idf.py (#19871) --- .github/workflows/ci.yml | 12 + esphome/analyze_memory/toolchain.py | 2 +- esphome/build_helpers/tool_runner.py | 180 ++++++ esphome/build_helpers/tools_cache.py | 4 +- esphome/espidf/clang_tidy.py | 8 +- esphome/espidf/framework.py | 17 +- esphome/espidf/runner.py | 379 ------------- esphome/espidf/size_summary.py | 2 +- esphome/espidf/toolchain.py | 354 ++++++++---- esphome/util.py | 22 +- script/check_idf_py_equivalence.py | 198 +++++++ script/determine-jobs.py | 30 +- tests/script/test_check_idf_py_equivalence.py | 248 +++++++++ tests/script/test_determine_jobs.py | 89 ++- .../build_helpers/test_tool_runner.py | 338 ++++++++++++ .../fixtures/espidf/closing_probe.py | 11 - .../fixtures/espidf/crashing_probe.py | 11 - .../fixtures/espidf/filtering_probe.py | 15 - .../fixtures/espidf/formfeed_probe.py | 12 - .../fixtures/espidf/partial_noise_probe.py | 10 - .../fixtures/espidf/streaming_probe.py | 14 - tests/unit_tests/test_espidf_clang_tidy.py | 38 ++ tests/unit_tests/test_espidf_framework.py | 12 +- tests/unit_tests/test_espidf_runner.py | 315 ----------- tests/unit_tests/test_espidf_toolchain.py | 515 +++++++++++++++--- 25 files changed, 1840 insertions(+), 996 deletions(-) create mode 100644 esphome/build_helpers/tool_runner.py delete mode 100644 esphome/espidf/runner.py create mode 100755 script/check_idf_py_equivalence.py create mode 100644 tests/script/test_check_idf_py_equivalence.py create mode 100644 tests/unit_tests/build_helpers/test_tool_runner.py delete mode 100644 tests/unit_tests/fixtures/espidf/closing_probe.py delete mode 100644 tests/unit_tests/fixtures/espidf/crashing_probe.py delete mode 100644 tests/unit_tests/fixtures/espidf/filtering_probe.py delete mode 100644 tests/unit_tests/fixtures/espidf/formfeed_probe.py delete mode 100644 tests/unit_tests/fixtures/espidf/partial_noise_probe.py delete mode 100644 tests/unit_tests/fixtures/espidf/streaming_probe.py delete mode 100644 tests/unit_tests/test_espidf_runner.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dac899a41a..d9507c59381 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1226,8 +1226,20 @@ jobs: if [ -n "$compile_csv" ]; then # Run compilation with grouping and isolation python3 script/test_build_components.py -e compile -c "$compile_csv" -f --isolate "$directly_changed_csv" + if [[ "${{ matrix.batch.check_idf_py }}" == "true" ]]; then + # The real idf.py must find nothing to configure or build in a + # tree built above; catches drift on ESP-IDF bumps. + echo "Checking the native ESP-IDF build matches idf.py" + python3 script/check_idf_py_equivalence.py + fi else echo "All components in this batch are validate-only -- skipping compile stage." + if [[ "${{ matrix.batch.check_idf_py }}" == "true" ]]; then + # determine-jobs and this step disagree on what compiles; fail + # rather than let the check run nowhere. + echo "::error::This batch was picked for the idf.py check but compiled nothing" + exit 1 + fi fi - name: Print ccache statistics diff --git a/esphome/analyze_memory/toolchain.py b/esphome/analyze_memory/toolchain.py index b64af5ea5b8..7d8195a17b1 100644 --- a/esphome/analyze_memory/toolchain.py +++ b/esphome/analyze_memory/toolchain.py @@ -37,7 +37,7 @@ def find_elf_path(build_path: Path) -> Path | None: """ name = build_path.name for candidate in ( - # Native ESP-IDF: idf.py writes build/.elf, which ESPHome copies + # Native ESP-IDF: the build writes build/.elf, which ESPHome copies # to build/firmware.elf (see espidf.toolchain.create_elf_copy) build_path / "build" / "firmware.elf", # PlatformIO diff --git a/esphome/build_helpers/tool_runner.py b/esphome/build_helpers/tool_runner.py new file mode 100644 index 00000000000..52cd6c937a9 --- /dev/null +++ b/esphome/build_helpers/tool_runner.py @@ -0,0 +1,180 @@ +"""Run a native build tool (cmake, ninja) and relay its output. + +Output is read from a pipe so it can be filtered here: a child that inherits +our stdout writes straight to the file descriptor, past any Python wrapper. +""" + +from __future__ import annotations + +import codecs +from contextlib import suppress +import logging +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +from typing import Any, TextIO + +from esphome.util import ANSI_ESCAPE, RedirectText, shlex_quote + +_LOGGER = logging.getLogger(__name__) + +# Windows code page identifier for UTF-8, as used by ``chcp 65001``. +UTF8_CODEPAGE = 65001 + +# Same pattern idf.py uses to spot ninja status lines (``is_progression``). +_PROGRESS = re.compile(r"^\[\d+/\d+\]|.*\(\d+ \%\)$") +_READ_SIZE = 65536 + + +def _get_kernel32() -> Any | None: + """Return the Windows kernel32 module, or None on any other platform.""" + if sys.platform != "win32": + return None + import ctypes + + return ctypes.windll.kernel32 + + +class Utf8Console: + """Keep an attached Windows console on UTF-8 while a build tool runs. + + esp_idf_size draws its table with Unicode box characters, and CMake + re-decodes a child's output with the console code page, which garbles the + table on any page but UTF-8. A console already on UTF-8 is left alone so + an overlapping build never records UTF-8 as the page to go back to. + """ + + def __init__(self, kernel32: Any | None) -> None: + self._kernel32 = kernel32 + self._codepages: tuple[int, int] | None = None + + def __enter__(self) -> None: + kernel32 = self._kernel32 + if kernel32 is None: + return + old_in = kernel32.GetConsoleCP() + old_out = kernel32.GetConsoleOutputCP() + # Both calls return 0 when no console is attached. + if not old_in or not old_out: + return + if old_in == UTF8_CODEPAGE and old_out == UTF8_CODEPAGE: + return + # Record first so a switch that fails part way is still undone. + self._codepages = (old_in, old_out) + kernel32.SetConsoleCP(UTF8_CODEPAGE) + kernel32.SetConsoleOutputCP(UTF8_CODEPAGE) + + def __exit__(self, *exc_info: object) -> None: + if self._codepages is None: + return + old_in, old_out = self._codepages + self._codepages = None + self._kernel32.SetConsoleCP(old_in) + self._kernel32.SetConsoleOutputCP(old_out) + + +def _fit_terminal(text: str) -> str: + """Elide the middle of ``text`` to fit the terminal, as idf.py does. + + A width of 0 (a pipe, the dashboard) leaves the text whole. + """ + width = shutil.get_terminal_size((0, 0)).columns + if not width: + return text + if width <= 3: + return "." * width + if len(text) >= width: + keep = (width - 3) // 2 + return f"{text[:keep]}...{text[len(text) - keep :]}" + return text + + +class ToolOutput(RedirectText): + """RedirectText that can collapse ninja status lines into one line. + + With ``progress`` each ``[n/m]`` line overwrites the previous one, the + way idf.py shows a build. + """ + + def __init__( + self, out: TextIO, filter_lines: list[str] | None, progress: bool + ) -> None: + super().__init__(out, filter_lines=filter_lines) + self._progress = progress + self._on_progress_line = False + + def _splits_lines(self) -> bool: + return self._progress or super()._splits_lines() + + def _emit_line(self, line: str) -> None: + if self._progress and _PROGRESS.match(line): + if not self._is_filtered(line): + text = _fit_terminal(line.strip("\r\n")) + self._write_color_replace(f"\r{text}\x1b[K") + self._on_progress_line = True + return + self._end_progress_line() + super()._emit_line(line) + + def _end_progress_line(self) -> None: + if self._on_progress_line: + self._on_progress_line = False + self._write_color_replace(os.linesep) + + def drain(self) -> None: + super().drain() + # Called from cleanup, so a broken stream must not hide the exit code. + with suppress(OSError, ValueError): + self._end_progress_line() + self._out.flush() + + +def run_build_tool( + cmd: list[str], + *, + cwd: Path, + env: dict[str, str], + filter_lines: list[str] | None = None, + progress: bool = False, + log_path: Path | None = None, +) -> int: + """Run ``cmd`` and relay stdout and stderr, merged, to our stdout. + + ``log_path`` also gets the full, unfiltered output without color codes, as + idf.py wrote its logs (its hint patterns expect plain text). Returns the + exit code. + """ + _LOGGER.debug("Running: %s", " ".join(shlex_quote(arg) for arg in cmd)) + _LOGGER.debug(" in directory: %s", cwd) + output = ToolOutput(sys.stdout, filter_lines, progress) + decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + if log_path is not None: + log_path.parent.mkdir(parents=True, exist_ok=True) + with ( + Path(log_path or os.devnull).open("w", encoding="utf-8", newline="") as log, + Utf8Console(_get_kernel32()), + subprocess.Popen( + cmd, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + close_fds=False, + ) as proc, + ): + try: + # read1 returns as soon as anything is available, so output + # streams while the tool runs. + while chunk := proc.stdout.read1(_READ_SIZE): + text = decoder.decode(chunk) + log.write(ANSI_ESCAPE.sub("", text)) + output.write(text) + if tail := decoder.decode(b"", final=True): + log.write(ANSI_ESCAPE.sub("", tail)) + output.write(tail) + finally: + output.drain() + return proc.returncode diff --git a/esphome/build_helpers/tools_cache.py b/esphome/build_helpers/tools_cache.py index e3d38e7a98e..85036c166fe 100644 --- a/esphome/build_helpers/tools_cache.py +++ b/esphome/build_helpers/tools_cache.py @@ -16,8 +16,8 @@ def tools_cache_path(env_var: str, subdir: str) -> Path: from esphome.helpers import get_str_env if prefix := get_str_env(env_var, "").strip(): - # resolve(): symlinked prefixes otherwise trip idf.py's - # venv-mismatch warning on every build + # resolve(): a symlinked prefix would otherwise record a second + # spelling of the same paths in the build tree return Path(prefix).expanduser().resolve() # appauthor=False keeps the Windows path short (no vendor segment); # deep IDF trees run into MAX_PATH otherwise diff --git a/esphome/espidf/clang_tidy.py b/esphome/espidf/clang_tidy.py index 9a70f1a99aa..3e1a510dee9 100644 --- a/esphome/espidf/clang_tidy.py +++ b/esphome/espidf/clang_tidy.py @@ -11,7 +11,7 @@ running codegen on a config, it generates a minimal ESP-IDF CMake project: component, so their public include dirs land on the translation unit; * the repo ``sdkconfig.defaults`` enables sdkconfig-gated components (bt, ...). -then runs ``idf.py reconfigure`` (configure only, no compile) and reads the +then runs the CMake configure (no compile) and reads the resulting ``build/compile_commands.json``. The IDF version is the esp32 component's recommended version. @@ -367,7 +367,7 @@ def _write_tidy_project( def _generate_compile_commands( work_dir: Path, settings: _Settings, platformio_ini: Path ) -> Path: - """Generate the tidy project and run ``idf.py reconfigure`` (no build). + """Generate the tidy project and run the CMake configure (no build). Two-phase, like a real ESPHome build: a first configure with no builtin requires discovers which components actually register for the target (e.g. @@ -398,7 +398,7 @@ def _generate_compile_commands( # Phase 1: discover the components available for this target. _write_tidy_project(work_dir, [], extra_deps, settings) if toolchain.run_reconfigure() != 0: - raise RuntimeError("idf.py reconfigure (discovery) failed") + raise RuntimeError("ESP-IDF CMake configure (discovery) failed") requires = sorted( set(get_available_components() or []) - _NON_REQUIRABLE_COMPONENTS @@ -407,7 +407,7 @@ def _generate_compile_commands( # Phase 2: require every available builtin component. _write_tidy_project(work_dir, requires, extra_deps, settings) if toolchain.run_reconfigure() != 0: - raise RuntimeError("idf.py reconfigure failed") + raise RuntimeError("ESP-IDF CMake configure failed") return work_dir / "build" / "compile_commands.json" diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 8885da05cca..1a4b17efdb3 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -1211,38 +1211,37 @@ def _ccache_env() -> dict[str, str]: # ESPHOME_CCACHE_ENABLE. idf_knob = parse_enable_env("IDF_CCACHE_ENABLE") if idf_knob is False: - # The raw value (e.g. "disable") is still inherited by idf.py via - # os.environ, where a non-false-constant string reads as truthy; - # export the canonical off spelling instead + # Replace the inherited raw value (e.g. "disable") with the canonical + # off spelling, so every reader of the env sees the same answer return {"IDF_CCACHE_ENABLE": "0"} if idf_knob is True: # Forced on ignores the runnability verdict, but the outcome is # worth saying out loud. Probed directly (not via the resolver, # whose failure message says "compiling without ccache" -- exactly # what forced-on does NOT do): only the truly-missing case means - # idf.py compiles without ccache; a broken binary is still used, - # since idf.py does its own PATH lookup. + # the build compiles without ccache; a broken binary is still used, + # since IDF's CMake does its own PATH lookup. if (ccache := shutil.which("ccache")) is None: _LOGGER.warning( "IDF_CCACHE_ENABLE=1 but no ccache binary is on PATH; " - "idf.py will compile without ccache" + "the build will compile without ccache" ) else: # The probe warns with this message iff the binary fails tool_version_runs( ccache, "IDF_CCACHE_ENABLE=1 forces on the ccache at %s even though " - "it failed to run; idf.py will use it anyway", + "it failed to run; the build will use it anyway", ) elif resolve_ccache_path() is None: # ESP-IDF silently skips ccache without the binary; export the # canonical off spelling so an unparsable inherited value (or a - # probe-rejected ccache idf.py would still find) cannot enable it + # probe-rejected ccache CMake would still find) cannot enable it return {"IDF_CCACHE_ENABLE": "0"} env = ccache_defaults_env(get_idf_tools_path() / "ccache") env.update(ccache_pch_env()) - # Exactly one canonical spelling ever reaches idf.py, whatever the + # Exactly one canonical spelling ever reaches the build, whatever the # accepted input spelling was ("enable", "yes", ...) env["IDF_CCACHE_ENABLE"] = "1" return env diff --git a/esphome/espidf/runner.py b/esphome/espidf/runner.py deleted file mode 100644 index bf563a4f438..00000000000 --- a/esphome/espidf/runner.py +++ /dev/null @@ -1,379 +0,0 @@ -r"""Subprocess entry point for running ``idf.py`` with stdio wrapping. - -Invoked as ``python runner.py [script args...]``. - -Wraps ``sys.stdout`` and ``sys.stderr`` with a ``_FilteringTTYStream`` -shim so that: - -1. ``isatty()`` unconditionally returns True. CMake, Ninja, and idf.py's - own progress-bar code all check ``stream.isatty()`` to decide between - TTY-format output (``\\r`` cursor moves, ANSI colors, fancy progress - bars) and a plain fallback. With the wrapper in place they always - emit TTY format, even when our real stdout is a pipe to the parent - process (e.g. running under the Home Assistant dashboard add-on). - Downstream consumers — local terminals and the HA dashboard log - viewer — render the TTY control sequences correctly. - -2. ``FILTER_IDF_LINES`` is applied inside the shim's ``write()`` so - noisy idf.py output is dropped before it leaves this subprocess. - Filtering is skipped when ``-v`` / ``--verbose`` appears in argv so - verbose mode still shows everything. - -ESP-IDF runs under its own Python virtual environment which does not -have the ``esphome`` package installed, so the runner is intentionally -self-contained: no imports from ``esphome`` at all. The line-filtering -wrapper is inlined below rather than imported from -``esphome.util.RedirectText`` for that reason. -""" - -import sys - -# Regex patterns matched against each line of idf.py / CMake / Ninja -# output. Lines that match are dropped before reaching the parent -# process. Patterns are anchored at the start of the line (the shim -# uses ``re.match``). Disabled when the user passes ``-v`` / -# ``--verbose`` to ``esphome compile``. -FILTER_IDF_LINES: list[str] = [ - # idf.py's "how to flash" block at the end of a successful build. - # ESPHome handles flashing itself, so these instructions just clutter - # the output. - r"Project build complete\.", - r" idf\.py ", - r" python -m esptool ", - r"or$", - r"or from the ", - # CMake dumps the full list of IDF component paths on one giant line. - # It's purely informational and bloats the log. - r"-- Component paths:", - # CMake lists every linker script it adds (dozens of lines) and the - # complete flat list of IDF components on one giant line. Neither - # has diagnostic value for end users. - r"-- Adding linker script ", - r"-- Components:", - # IDF component manager notices: emitted on first build (no lock), - # once per stubbed dependency, plus the final "Processing N - # dependencies" enumeration. Patterns allow a leading run of dots - # because the component manager prints progress dots on the same - # line, so a NOTICE often arrives prefixed with ".NOTICE:" or - # "...........NOTICE:". - r"\.*NOTICE: ", - # ``idf.py size`` prefaces its table with a centered banner; the - # per-region table below already makes the structure obvious. - r"\s*Memory Type Usage Summary", - # Prefix match for esp-idf-size's trailing "Note:" paragraph (no - # upstream flag suppresses it). - r"Note: The reported total sizes may be smaller than those in the", - # Drop the blank line rich emits after the note so the build log - # doesn't end with an orphan gap before ESPHome's own status lines. - r"\s*$", - # ESP-IDF shells out to ``git rev-parse`` to embed a commit hash; - # esphome-libs strips ``.git`` from the tarball so those probes fail - # noisily without affecting the build. - r"-- git rev-parse returned ", - r"fatal: not a git repository", - r"Stopping at filesystem boundary", -] - -# Windows code page identifier for UTF-8, as used by ``chcp 65001``. -UTF8_CODEPAGE = 65001 - - -def _get_kernel32(): - """Return the Windows kernel32 module, or None on any other platform.""" - if sys.platform != "win32": - return None - import ctypes - - return ctypes.windll.kernel32 - - -class _Utf8Console: - """Keep an attached Windows console on UTF-8 for the length of the build. - - The build tree runs in UTF-8 mode, so esp_idf_size draws its table with - Unicode box characters. ``idf.py size`` reaches it through ``cmake -P``, - and CMake re-decodes the child's output with the console code page before - printing it, which turns the table into mojibake on any code page but - UTF-8. Every process in the build shares this console, so switching it - here covers CMake too. The old code pages go back on exit so the user's - terminal is left as it was. - - A console that is already on UTF-8 is left alone. The code page belongs to - the console, not to this process, so a build that overlaps another one - must not save UTF-8 as the page to go back to. - """ - - def __init__(self, kernel32) -> None: - self._kernel32 = kernel32 - self._codepages: tuple[int, int] | None = None - - def __enter__(self) -> None: - kernel32 = self._kernel32 - if kernel32 is None: - return - old_in = kernel32.GetConsoleCP() - old_out = kernel32.GetConsoleOutputCP() - # Both calls return 0 when no console is attached. - if not old_in or not old_out: - return - if old_in == UTF8_CODEPAGE and old_out == UTF8_CODEPAGE: - return - # Record the old pages first so a switch that fails part way through - # still gets put back on exit. - self._codepages = (old_in, old_out) - kernel32.SetConsoleCP(UTF8_CODEPAGE) - kernel32.SetConsoleOutputCP(UTF8_CODEPAGE) - - def __exit__(self, *exc_info: object) -> None: - if self._codepages is None: - return - old_in, old_out = self._codepages - self._codepages = None - self._kernel32.SetConsoleCP(old_in) - self._kernel32.SetConsoleOutputCP(old_out) - - -def main() -> int: - # ---- sys.path fix-up --------------------------------------------------- - # - # When Python runs this file as ``python runner.py``, it prepends the - # script's directory — ``/esphome/espidf/`` — to - # ``sys.path[0]``. That directory is part of the esphome package whose - # sibling ``types.py`` (in ``esphome/``) collides with stdlib ``types``. - # Any subsequent import that transitively touches ``types`` (``runpy``, - # ``pathlib``, ``functools``, ``typing``, ...) could resolve the wrong - # module. Drop the entry pre-emptively. ``sys`` is a built-in so - # importing it at module level earlier did not trigger the shadow. - if sys.path and sys.path[0]: - sys.path.pop(0) - # ---- end sys.path fix-up ----------------------------------------------- - - import contextlib - import os - from pathlib import Path - import re - import runpy - - # Patch ``os.get_terminal_size`` to return a fallback size instead - # of raising ``OSError`` when the underlying fd isn't a real - # terminal. - # - # idf.py's ``fit_text_in_terminal`` (in ``idf_py_actions/tools.py``) - # unconditionally calls ``os.get_terminal_size()`` to format ninja - # progress lines. When that raises ``[Errno 25] Inappropriate - # ioctl for device`` on our pipe-backed stdout, idf.py catches the - # exception as ``EnvironmentError`` and silently exits its stdout - # reader coroutine — dropping all ninja build output from that - # point on. Returning a valid value keeps the coroutine alive so - # progress and error lines continue to flow through to the parent - # process. - # - # Honour the ``COLUMNS`` / ``LINES`` env vars if the caller set - # them explicitly. Otherwise fall back to ``(0, 0)``, which - # ``fit_text_in_terminal`` treats as "unknown width, don't - # truncate" (see the ``if not terminal_width: return out`` guard). - # Downstream log viewers (local terminals, the HA dashboard) wrap - # or scroll long lines themselves, so we'd rather emit the full - # file path than have idf.py elide its middle. - _orig_get_terminal_size = os.get_terminal_size - - def _get_terminal_size_fallback(fd: int = 1) -> os.terminal_size: - try: - return _orig_get_terminal_size(fd) - except OSError: - try: - columns = int(os.environ.get("COLUMNS", "0")) - except ValueError: - columns = 0 - try: - lines = int(os.environ.get("LINES", "0")) - except ValueError: - lines = 0 - return os.terminal_size((columns, lines)) - - os.get_terminal_size = _get_terminal_size_fallback # type: ignore[assignment] - - # Strip ANSI escape sequences before comparing a line against the filter - # patterns, so colorized lines still match plain-text patterns. - ansi_escape = re.compile(r"\033[@-_][0-?]*[ -/]*[@-~]") - - class _FilteringTTYStream: - r"""Minimal stdout/stderr wrapper. - - * ``isatty()`` unconditionally returns True, tricking downstream - code into emitting TTY-format output. - * Input is split with ``str.splitlines(keepends=True)``, which - breaks on more than ``\\n`` and ``\\r``; form feed and a few - other control characters count too. Any piece whose - ANSI-stripped, right-stripped form matches one of - ``filter_lines`` is dropped. - * Only the final piece can still be waiting for more text, so - that one is held until a ``\\n`` or ``\\r`` arrives. A piece - that ended on one of the other breaks goes out as it is. - - Mirrors the matching semantics of ``esphome.util.RedirectText`` - so filter patterns behave identically in both the PlatformIO - and IDF runner paths. - """ - - def __init__(self, stream, filter_lines: list[str] | None) -> None: - self._stream = stream - if filter_lines: - combined = r"|".join(r"(?:" + p + r")" for p in filter_lines) - self._filter_pattern: re.Pattern[str] | None = re.compile(combined) - else: - self._filter_pattern = None - self._line_buffer = "" - - def __getattr__(self, name: str): - # Hide ``buffer`` so consumers that use either - # ``getattr(stream, 'buffer', None)`` or - # ``hasattr(stream, 'buffer')`` see this as a text-only stream - # and skip writing raw bytes (which would bypass the filter). - if name == "buffer": - raise AttributeError(name) - return getattr(self._stream, name) - - def isatty(self) -> bool: - return True - - def flush(self) -> None: - self._stream.flush() - - def _emit(self, line: str) -> None: - if self._filter_pattern is not None: - stripped = ansi_escape.sub("", line).rstrip() - if self._filter_pattern.match(stripped) is not None: - return - self._stream.write(line) - - def drain(self) -> None: - """Write out a held-back line that never got its terminator. - - idf.py and CMake do not always end their last line with a - newline, and a build that dies part way through can stop mid - line. Without this the user is left staring at a build that - ended with no explanation. - """ - if not self._line_buffer: - return - line, self._line_buffer = self._line_buffer, "" - try: - # Add the terminator the line never got, so whatever ESPHome - # prints next does not run onto the same line. - self._emit(line + "\n") - self._stream.flush() - except (OSError, ValueError) as err: - # We are called from cleanup, so raising would replace the - # build's real exit code. Saying so must not raise either: - # under the dashboard our stdout and stderr are the same - # pipe, so whatever broke the write has most likely broken - # the report, and ``sys.__stderr__`` is None on some - # interpreters. Carry the line along; it is usually the - # message saying why the build failed. - if (real_stderr := sys.__stderr__) is not None: - with contextlib.suppress(OSError, ValueError): - print( - f"Could not write out remaining output ({err}): {line}", - file=real_stderr, - ) - - def write(self, data) -> int: - # Text streams normally hand us ``str``; decode in case - # somebody writes bytes directly. - if not isinstance(data, str): - data = data.decode(errors="replace") - - if self._filter_pattern is None: - # Nothing to match against, so no need to wait for a full line. - self._emit(data) - else: - lines = (self._line_buffer + data).splitlines(keepends=True) - # Every piece but the last ends with something - # ``str.splitlines`` treats as a break, so only the last one - # can still be waiting for more text. Hold that one, write - # out the rest. - # - # Some of those breaks are not line endings to us, a form - # feed for one, so a piece can go out without ending in a - # newline. That beats what we did before, which was to stop - # at the first such piece and drop every complete line - # behind it. - if lines and not lines[-1].endswith(("\n", "\r")): - self._line_buffer = lines.pop() - else: - self._line_buffer = "" - for line in lines: - self._emit(line) - - # We tell idf.py it is talking to a terminal, so it sends progress - # bars and cursor moves. Our own stdout is usually a pipe, which is - # block buffered, so without this the build looks frozen until - # 8 KiB of output piles up. - self._stream.flush() - return len(data) - - if len(sys.argv) < 2: - print( - "usage: runner.py [args...]", - file=sys.stderr, - ) - return 2 - - script_path = sys.argv[1] - - # Mirror the platformio runner behaviour: verbose mode disables the - # line filter so all output reaches the user. - is_verbose = any(arg in ("-v", "--verbose") for arg in sys.argv[2:]) - filter_lines = None if is_verbose else FILTER_IDF_LINES or None - - class _FilteredStreams: - """Route ``sys.stdout`` and ``sys.stderr`` through the filtering shims. - - On exit each shim releases a last line that never got its - terminator. The shims made here are drained rather than whatever - ``sys.stdout`` holds by then, which the script is free to replace. - """ - - def __init__(self, filter_lines: list[str] | None) -> None: - self._stdout = _FilteringTTYStream(sys.stdout, filter_lines) - self._stderr = _FilteringTTYStream(sys.stderr, filter_lines) - - def __enter__(self) -> None: - sys.stdout = self._stdout # type: ignore[assignment] - sys.stderr = self._stderr # type: ignore[assignment] - - def __exit__(self, *exc_info: object) -> None: - # Drain stderr from a finally so a surprise from the first one - # cannot strand the second. - try: - self._stdout.drain() - finally: - self._stderr.drain() - - # Shift argv so the target script sees its own path as argv[0] and - # its own arguments starting at argv[1]. runpy.run_path does not - # modify sys.argv itself. - sys.argv = [script_path] + sys.argv[2:] - - # Emulate Python's default behaviour of prepending the script's - # directory to sys.path[0] when running ``python script.py``. - # runpy.run_path does not do this automatically, but idf.py relies - # on it to import its sibling modules (python_version_checker, - # idf_py_actions, ...). - script_dir = str(Path(script_path).resolve().parent) - if script_dir not in sys.path: - sys.path.insert(0, script_dir) - - # If idf.py calls sys.exit(), SystemExit propagates out of run_path - # and carries the exit code back to our caller. For normal returns, - # fall through and exit with 0. Either way the context managers drain - # the streams and put the console back on the way out, and they report - # instead of raising so cleanup cannot bury the real exit code. - with _FilteredStreams(filter_lines), _Utf8Console(_get_kernel32()): - runpy.run_path(script_path, run_name="__main__") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/esphome/espidf/size_summary.py b/esphome/espidf/size_summary.py index d98363dd67e..ffe97ba6186 100644 --- a/esphome/espidf/size_summary.py +++ b/esphome/espidf/size_summary.py @@ -1,6 +1,6 @@ """PlatformIO-format RAM/Flash one-liners after a native ESP-IDF build. -``idf.py size`` (chained onto ``idf.py build`` in +The ninja ``size`` target (run after ``all`` in ``toolchain.run_compile``) prints the per-region table inline as part of the build. This module adds two summary lines underneath, byte-identical to PlatformIO's output: diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index 3bb1f5b83d5..d1aeae96031 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -10,6 +10,7 @@ import re import shutil import subprocess +from esphome.build_helpers.tool_runner import run_build_tool from esphome.const import ( CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, @@ -189,72 +190,235 @@ def _get_idf_tool(name: str) -> str: return executable -def run_idf_py( - *args, - cwd: Path | None = None, - capture_output: bool = False, - jobs: int | None = None, -) -> int | str: - """Run idf.py with the given arguments.""" - idf_path = _get_idf_path() - if idf_path is None: - raise EsphomeError("ESP-IDF not found") +# Lines dropped from cmake and ninja output unless ``-v`` is given; matched +# with ``re.match`` against the line without ANSI codes or trailing space. +FILTER_IDF_LINES: list[str] = [ + # Full component path and linker script lists, one giant line each. + r"-- Component paths:", + r"-- Adding linker script ", + r"-- Components:", + # Component manager notices; progress dots can precede them. + r"\.*NOTICE: ", + # esp_idf_size banner and trailing note around the size table. + r"\s*Memory Type Usage Summary", + r"Note: The reported total sizes may be smaller than those in the", + r"\s*$", + # esphome-libs tarballs have no .git, so IDF's commit probes fail noisily. + r"-- git rev-parse returned ", + r"fatal: not a git repository", + r"Stopping at filesystem boundary", +] - env = _get_idf_env() - if jobs is not None: - env = {**env, "IDF_PY_BUILD_JOBS": str(jobs)} - python_executable = _get_idf_tool("python") - idf_py = idf_path / "tools" / "idf.py" - # Dispatch idf.py through esphome.espidf.runner, which wraps - # sys.stdout/sys.stderr so ``isatty()`` reports True. This keeps CMake, - # Ninja, and idf.py's own progress-bar code emitting TTY-format output - # (``\r`` cursor moves, ANSI colors, fancy progress bars) even when our - # real stdout is a pipe — e.g. when esphome is running under the Home - # Assistant dashboard add-on. The runner is a plain script (not a - # ``python -m`` module) because IDF's Python venv does not have the - # esphome package installed. - runner_py = Path(__file__).parent / "runner.py" - - cmd = [python_executable, str(runner_py), str(idf_py)] + list(args) - - if cwd is None: - cwd = CORE.build_path - - _LOGGER.debug("Running: %s", " ".join(cmd)) - _LOGGER.debug(" in directory: %s", cwd) - - if capture_output: - result = subprocess.run( - cmd, - cwd=cwd, - env=env, - capture_output=True, - text=True, - check=False, - ) - if result.returncode != 0: - _LOGGER.error("idf.py failed:\n%s", result.stderr) - return result.stdout - result = subprocess.run( - cmd, - cwd=cwd, - env=env, - check=False, - ) - return result.returncode +# click's boolean spellings, which idf.py applies to IDF_CCACHE_ENABLE. +_CLICK_TRUE = frozenset({"1", "true", "t", "yes", "y", "on"}) +_CMAKECACHE_LINE = re.compile(r"^([^#/:=]+):([^:=]+)=(.*)$") -def _get_sdkconfig_args() -> list[str]: - """Get cmake -D flags for the sdkconfig file, if it exists.""" +@dataclass(frozen=True, kw_only=True) +class _IdfPyContract: + """How the pinned idf.py drives cmake and ninja (tools/idf_py_actions).""" + + binary_dir_arg: bool # cmake gets -B + ccache_as_bool: bool # CCACHE_ENABLE=True/False instead of 1/0 + size_ng: bool # size target gets ESP_IDF_SIZE_NG=1 + + +_IDF_PY_5 = _IdfPyContract( + binary_dir_arg=False, + ccache_as_bool=False, + size_ng=True, +) +_IDF_PY_6 = _IdfPyContract( + binary_dir_arg=True, + ccache_as_bool=True, + size_ng=False, +) + + +def _idf_py() -> _IdfPyContract: + from esphome.components.esp32 import idf_version + import esphome.config_validation as cv + + return _IDF_PY_6 if idf_version() >= cv.Version(6, 0, 0) else _IDF_PY_5 + + +def _build_dir() -> Path: + """The CMake binary dir; idf.py resolves the project dir the same way.""" + return Path(os.path.realpath(CORE.build_path)) / "build" + + +def _cache_entries() -> dict[str, str]: + """The ``-D`` entries idf.py passes to cmake, in idf.py's order.""" + entries = {} sdkconfig_path = CORE.relative_build_path(f"sdkconfig.{CORE.name}") if sdkconfig_path.is_file(): - return ["-D", f"SDKCONFIG={sdkconfig_path}"] - return [] + entries["SDKCONFIG"] = str(sdkconfig_path) + ccache = _get_idf_env().get("IDF_CCACHE_ENABLE", "").strip().lower() in _CLICK_TRUE + entries["CCACHE_ENABLE"] = str(ccache if _idf_py().ccache_as_bool else int(ccache)) + return entries -def run_reconfigure() -> int: - """Run cmake reconfigure only (no build).""" - return run_idf_py(*_get_sdkconfig_args(), "reconfigure") +def _parse_cmakecache(path: Path) -> dict[str, str]: + """Map each ``NAME:TYPE=VALUE`` line of a CMakeCache.txt to NAME: VALUE.""" + result = {} + for line in path.read_text(encoding="utf-8").splitlines(): + if m := _CMAKECACHE_LINE.match(line): + result[m.group(1)] = m.group(3) + return result + + +def _cache_entries_changed() -> bool: + """True when a ``-D`` entry is missing from or differs in CMakeCache.txt. + + idf.py reconfigures on this before every build; ESPHome's own staleness + check does not cover it (for example ccache switched on or off). ESPHome + also compares ``PYTHON``, so a moved IDF prefix reconfigures; idf.py + stopped with an error instead. + """ + cache_path = _build_dir() / "CMakeCache.txt" + if not cache_path.is_file(): + return True + cache = _parse_cmakecache(cache_path) + return any(cache.get(k) != v for k, v in _configure_defines().items()) + + +def _configure_defines() -> dict[str, str]: + """Every ``-D`` idf.py passes to cmake, in its order.""" + return { + "PYTHON_DEPS_CHECKED": "1", + "PYTHON": _get_idf_tool("python"), + "ESP_PLATFORM": "1", + **_cache_entries(), + } + + +def _tool_env() -> dict[str, str]: + """The IDF env plus color, as idf.py 6.x gives every tool. + + Also used on 5.x (which forced CLICOLOR_FORCE for ninja only); color + changes only what is printed, and this way NO_COLOR is respected. + """ + env = dict(_get_idf_env()) + if not env.get("NO_COLOR"): + env.setdefault("CLICOLOR_FORCE", "1") + env.setdefault("FORCE_COLOR", "1") + return env + + +def run_reconfigure(verbose: bool = False) -> int: + """Run the CMake configure, with the arguments idf.py uses.""" + build_dir = _build_dir() + build_dir.mkdir(parents=True, exist_ok=True) + cmd = [_get_idf_tool("cmake"), "-G", "Ninja"] + if _idf_py().binary_dir_arg: + cmd += ["-B", str(build_dir)] + cmd += [f"-D{name}={value}" for name, value in _configure_defines().items()] + cmd.append(str(build_dir.parent)) + log_path = build_dir / "log" / "cmake_output.log" + rc = run_build_tool( + cmd, + cwd=build_dir, + env=_tool_env(), + filter_lines=None if verbose else FILTER_IDF_LINES, + log_path=log_path, + ) + if rc != 0: + # As idf.py does: a partial cache must not look configured. + (build_dir / "CMakeCache.txt").unlink(missing_ok=True) + _LOGGER.error("CMake configure failed with exit code %d", rc) + _print_hints(log_path) + return rc + + +def _size_env() -> dict[str, str]: + """Environment idf.py gives the ``size`` target.""" + env = {"ESP_IDF_SIZE_FORCE_TERMINAL": "1", "SIZE_OUTPUT_FORMAT": "default"} + if _idf_py().size_ng: + env["ESP_IDF_SIZE_NG"] = "1" + return env + + +def _build_jobs(config) -> int | None: + """Ninja's -j: compile_process_limit, else IDF_PY_BUILD_JOBS as idf.py read it.""" + if (limit := config[CONF_ESPHOME].get(CONF_COMPILE_PROCESS_LIMIT)) is not None: + return limit + if not (value := os.environ.get("IDF_PY_BUILD_JOBS")): + return None + try: + jobs = int(value) + except ValueError: + jobs = 0 + if jobs <= 0: + raise EsphomeError("IDF_PY_BUILD_JOBS must be a positive integer") + return jobs + + +def _run_ninja( + target: str, + *, + verbose: bool, + jobs: int | None, + progress: bool = False, + extra_env: dict[str, str] | None = None, +) -> int: + """Build one ninja target, with the flags and env idf.py uses.""" + cmd = [_get_idf_tool("ninja")] + if jobs is not None: + cmd += ["-j", str(jobs)] + if verbose: + cmd.append("-v") + cmd.append(target) + log_path = _build_dir() / "log" / f"ninja_{Path(target).name}_output.log" + rc = run_build_tool( + cmd, + cwd=_build_dir(), + env={**_tool_env(), **(extra_env or {})}, + filter_lines=None if verbose else FILTER_IDF_LINES, + progress=progress and not verbose, + log_path=log_path, + ) + if rc != 0: + _LOGGER.error("ninja %s failed with exit code %d", target, rc) + _print_hints(log_path) + return rc + + +# Runs IDF's own hint matcher (hints.yml plus its hint modules) on a failed +# tool's output, as idf.py did; it only lives in the IDF venv. +_HINTS_SCRIPT = """ +import sys +sys.path.insert(0, sys.argv[1]) +from idf_py_actions.tools import generate_hints +for hint in generate_hints(sys.argv[2]): + print(hint) +""" + + +def _print_hints(log_path: Path) -> None: + """Print ESP-IDF's advice for a failed build; never fails the build itself.""" + try: + result = subprocess.run( + [ + _get_idf_tool("python"), + "-c", + _HINTS_SCRIPT, + str(_get_idf_path() / "tools"), + str(log_path), + ], + env=_get_idf_env(), + capture_output=True, + text=True, + timeout=60, + check=False, + ) + except (OSError, subprocess.SubprocessError, EsphomeError) as err: + _LOGGER.debug("Could not get ESP-IDF hints: %s", err) + return + if result.returncode != 0: + _LOGGER.debug("Could not get ESP-IDF hints:\n%s", result.stderr) + return + if hints := result.stdout.strip(): + _LOGGER.warning("%s", hints) def _builtin_component_cache_path() -> Path | None: @@ -313,7 +477,9 @@ def save_cached_builtin_components(components: list[str]) -> None: _LOGGER.warning("Could not write component list cache %s: %s", path, err) -def _write_project_and_reconfigure(builtin_components: list[str] | None) -> int: +def _write_project_and_reconfigure( + builtin_components: list[str] | None, verbose: bool = False +) -> int: """Write the full CMakeLists.txt and run the configure for it.""" from esphome.build_gen.espidf import write_project @@ -321,12 +487,12 @@ def _write_project_and_reconfigure(builtin_components: list[str] | None) -> int: write_project(minimal=False, builtin_components=builtin_components) # Explicit reconfigure: ninja only re-runs cmake when CMakeLists.txt # is strictly newer than build.ninja, which fails on coarse-mtime - # filesystems (#18682). Also keeps idf.py from regenerating memory.ld + # filesystems (#18682). Also keeps ninja from regenerating memory.ld # in testing mode. - return run_reconfigure() + return run_reconfigure(verbose) -def _configure_project() -> int: +def _configure_project(verbose: bool = False) -> int: """Configure the project, discovering the built-in components if needed. A cached component list skips the discovery configure. If the configure @@ -337,20 +503,20 @@ def _configure_project() -> int: if (cached := load_cached_builtin_components()) is not None: _LOGGER.info("Using cached ESP-IDF component list") - if _write_project_and_reconfigure(cached) == 0: + if _write_project_and_reconfigure(cached, verbose) == 0: return 0 _LOGGER.warning("Cached component list failed; rediscovering") _builtin_component_cache_path().unlink(missing_ok=True) _LOGGER.info("Discovering available ESP-IDF components...") write_project(minimal=True) - if (rc := run_reconfigure()) != 0: + if (rc := run_reconfigure(verbose)) != 0: _LOGGER.error("Component discovery failed") return rc discovered = get_available_components() if not discovered: _LOGGER.error("Component discovery found no built-in ESP-IDF components") return 1 - if (rc := _write_project_and_reconfigure(discovered)) != 0: + if (rc := _write_project_and_reconfigure(discovered, verbose)) != 0: _LOGGER.error("Reconfigure with discovered components failed") return rc save_cached_builtin_components(discovered) @@ -489,11 +655,9 @@ def run_compile(config, verbose: bool) -> int: 2. Regenerate CMakeLists.txt with discovered components 3. Run full build """ - # Check if we need to do discovery phase - if not need_reconfigure(): - _LOGGER.info("Build configuration is up to date") - else: - if (rc := _configure_project()) != 0: + jobs = _build_jobs(config) + if need_reconfigure(): + if (rc := _configure_project(verbose)) != 0: return rc # cmake does not rewrite CMakeCache.txt when only properties change, # so restamp it or every build repeats discovery. Only after success, @@ -504,45 +668,31 @@ def run_compile(config, verbose: bool) -> int: path = CORE.relative_build_path(name) if path.is_file(): os.utime(path) + elif _cache_entries_changed(): + _LOGGER.info("CMake cache options changed, reconfiguring") + if (rc := run_reconfigure(verbose)) != 0: + return rc + else: + _LOGGER.info("Build configuration is up to date") # In testing mode, generate the linker script first, patch DRAM/IRAM sizes, # then build. memory.ld is regenerated by ninja during the build phase, # so we must patch after it's generated but before linking (same timing # as iram_fix.py.script's AddPreAction hook in the PlatformIO path). if CORE.testing_mode: - memory_ld = CORE.relative_build_path( - "build", "esp-idf", "esp_system", "ld", "memory.ld" - ) - build_dir = CORE.relative_build_path("build") - # Build just the memory.ld target - ninja needs the path relative to build dir - memory_ld_target = os.path.relpath(str(memory_ld), str(build_dir)) - env = _get_idf_env() - ninja_executable = _get_idf_tool("ninja") - result = subprocess.run( - [ninja_executable, "-C", str(build_dir), memory_ld_target], - env=env, - check=False, - ) - if result.returncode != 0: - _LOGGER.error("Failed to generate linker script") - return result.returncode + memory_ld = str(Path("esp-idf", "esp_system", "ld", "memory.ld")) + if (rc := _run_ninja(memory_ld, verbose=verbose, jobs=jobs)) != 0: + return rc _patch_memory_segments() from esphome.build_gen.espidf import write_pch_checksum write_pch_checksum() - # Build - args = [] - - if verbose: - args.append("-v") - - args.extend(_get_sdkconfig_args()) - args.append("build") - args.append("size") - - rc = run_idf_py(*args, jobs=config[CONF_ESPHOME].get(CONF_COMPILE_PROCESS_LIMIT)) + # idf.py's ``build size``, minus the second ``ninja all`` it runs first. + rc = _run_ninja("all", verbose=verbose, jobs=jobs, progress=True) + if rc == 0: + rc = _run_ninja("size", verbose=verbose, jobs=jobs, extra_env=_size_env()) if rc == 0: size_json = CORE.relative_build_path("build", "esp_idf_size.json") partitions = CORE.relative_build_path("partitions.csv") @@ -553,7 +703,7 @@ def run_compile(config, verbose: bool) -> int: def get_firmware_path() -> Path: """Get the path to the compiled firmware binary. - This is the file idf.py writes directly (named after the project), + This is the file the build writes directly (named after the project), not the copy used for OTA/factory downloads below. """ build_dir = CORE.relative_build_path("build") @@ -584,7 +734,7 @@ def get_ota_firmware_path() -> Path: def get_built_elf_path() -> Path: - """Path to the ELF idf.py writes directly, ``/.elf``. + """Path to the ELF the build writes directly, ``/.elf``. Exists as soon as the build finishes, unlike the ``firmware.elf`` copy that ``create_elf_copy`` makes later. @@ -596,7 +746,7 @@ def get_built_elf_path() -> Path: def get_elf_path() -> Path: """Get the path to the firmware ELF file. - idf.py writes ``/.elf`` directly; this returns the + The build writes ``/.elf`` directly; this returns the ``/firmware.elf`` copy created by ``create_elf_copy`` so the dashboard's "download ELF" link can find it under the PlatformIO-convention name. @@ -723,7 +873,7 @@ def create_ota_bin() -> bool: def create_elf_copy() -> bool: """Copy the ELF binary to firmware.elf for dashboard compatibility. - idf.py writes the ELF at ``/.elf``; the dashboard's + The build writes the ELF at ``/.elf``; the dashboard's "download ELF" link requests the literal filename ``firmware.elf`` (PlatformIO convention), so copy it to that name. """ diff --git a/esphome/util.py b/esphome/util.py index b8ffa048ca1..8aa321d908c 100644 --- a/esphome/util.py +++ b/esphome/util.py @@ -179,15 +179,21 @@ class RedirectText: s = s.replace("\033", "\\033") self._out.write(s) - def _emit_line(self, line: str) -> None: - line_without_ansi = ANSI_ESCAPE.sub("", line) - line_without_end = line_without_ansi.rstrip() - if ( + def _is_filtered(self, line: str) -> bool: + return ( self._filter_pattern is not None - and self._filter_pattern.match(line_without_end) is not None - ): - # Filter pattern matched, ignore the line + and self._filter_pattern.match(ANSI_ESCAPE.sub("", line).rstrip()) + is not None + ) + + def _splits_lines(self) -> bool: + """Whether output is handled line by line rather than passed through.""" + return self._filter_pattern is not None or bool(self._line_callbacks) + + def _emit_line(self, line: str) -> None: + if self._is_filtered(line): return + line_without_end = ANSI_ESCAPE.sub("", line).rstrip() self._write_color_replace(line) # Check for flash size error and provide helpful guidance @@ -233,7 +239,7 @@ class RedirectText: if not isinstance(s, str): s = s.decode() - if self._filter_pattern is not None or self._line_callbacks: + if self._splits_lines(): lines = (self._line_buffer + s).splitlines(True) # Every piece but the last ends with something # ``str.splitlines`` treats as a break, so only the last one can diff --git a/script/check_idf_py_equivalence.py b/script/check_idf_py_equivalence.py new file mode 100755 index 00000000000..fa349e223a4 --- /dev/null +++ b/script/check_idf_py_equivalence.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Check that a native ESP-IDF build tree is what idf.py itself would produce. + +ESPHome runs cmake and ninja directly with the arguments idf.py uses. This +runs the real ``idf.py reconfigure`` and ``idf.py build`` on a finished tree +(in place: CMake rejects a moved cache) and fails if either one changes the +cache, the generated build files or the firmware, or recompiles anything. +It catches drift when the pinned ESP-IDF version changes what idf.py does. +The color and ``size`` environment only change what is printed, so those +parts of the contract are pinned by the unit tests instead. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import re +import subprocess +import sys + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT)) + +WATCHED = ( + "build/CMakeCache.txt", + "build/build.ninja", + "build/compile_commands.json", + "build/project_description.json", + "build/config/sdkconfig.h", + "build/bootloader/bootloader.bin", +) +# Ninja logs whose outputs mean real work when their recorded mtime changes. +# The top level re-logs the bootloader step's byproducts on every build, so +# the bootloader is judged by its own sub-build log instead. +TOP_NINJA_LOG = "build/.ninja_log" +NINJA_LOGS = (TOP_NINJA_LOG, "build/bootloader/.ninja_log") +BOOTLOADER_BYPRODUCT = re.compile(r"(^|/build/)bootloader/") +WORK_SUFFIXES = (".obj", ".o", ".a", ".elf", ".map", ".bin", ".ld") +DEFAULT_GLOB = "tests/test_build_components/build/.esphome/build/*" + + +def _digest(path: Path) -> str | None: + return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None + + +def watched(name: str) -> list[str]: + """Files that change if idf.py configures or builds differently.""" + return [*WATCHED, f"build/{name}.elf", f"build/{name}.bin"] + + +def _snapshot(build_path: Path, name: str) -> dict[str, str | None]: + return {f: _digest(build_path / f) for f in watched(name)} + + +def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]: + """(log, output) -> recorded mtime; compaction-safe, unlike a line count.""" + mtimes = {} + for name in NINJA_LOGS: + log = build_path / name + lines = log.read_text(encoding="utf-8").splitlines() if log.is_file() else [] + for fields in (line.split("\t") for line in lines if not line.startswith("#")): + if len(fields) >= 4 and ( + name != TOP_NINJA_LOG or not BOOTLOADER_BYPRODUCT.search(fields[3]) + ): + mtimes[name, fields[3]] = fields[2] + return mtimes + + +def _log_problems(build_path: Path, mtimes: dict[tuple[str, str], str]) -> list[str]: + """A missing or unparsable ninja log would otherwise compare as unchanged.""" + problems = [] + for log in NINJA_LOGS: + if not (build_path / log).is_file(): + problems.append(f"missing {log}") + elif not any(k[0] == log and k[1].endswith(WORK_SUFFIXES) for k in mtimes): + problems.append(f"no build entries parsed from {log}") + return problems + + +def _setup_core(build_path: Path, description: dict) -> tuple[str, str]: + """Point CORE at the tree so ESPHome resolves the same IDF env as the build.""" + from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT + import esphome.config_validation as cv + from esphome.core import CORE + + name = description["project_name"] + version = Path(description["idf_path"]).name + CORE.config_path = build_path.parents[2] / f"{name}.yaml" + CORE.build_path = build_path + CORE.name = name + CORE.data[KEY_ESP32] = { + KEY_IDF_VERSION: cv.Version.parse(version), + KEY_VARIANT: description["target"].upper(), + } + return name, version + + +def check(build_path: Path) -> list[str]: + """Return the problems found in one build tree.""" + # pylint: disable=protected-access + from esphome.espidf import toolchain + + description = json.loads( + (build_path / "build" / "project_description.json").read_text(encoding="utf-8") + ) + name, version = _setup_core(build_path, description) + env = toolchain._get_idf_env(version) + python = toolchain._get_idf_tool("python") + idf_py = toolchain._get_idf_path(version) / "tools" / "idf.py" + sdkconfig = build_path / f"sdkconfig.{name}" + sdkconfig_args = ["-D", f"SDKCONFIG={sdkconfig}"] if sdkconfig.is_file() else [] + + # CMake writes a different build.ninja on a tree's first configure than on + # a reconfigure, so the baseline is ESPHome's own reconfigure and build. + if (rc := toolchain.run_reconfigure()) != 0: + return [f"ESPHome's CMake configure failed with exit code {rc}"] + if (rc := toolchain._run_ninja("all", verbose=False, jobs=None)) != 0: + return [f"ESPHome's ninja build failed with exit code {rc}"] + before = _snapshot(build_path, name) + mtimes_before = _ninja_mtimes(build_path) + # A moved or renamed output would otherwise compare as "unchanged". + problems = [f"missing {f}" for f, digest in before.items() if digest is None] + if problems := problems + _log_problems(build_path, mtimes_before): + return problems + for action in ("reconfigure", "build"): + result = subprocess.run( + [python, str(idf_py), *sdkconfig_args, action], + cwd=build_path, + env=env, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + return [f"idf.py {action} failed:\n{result.stdout}{result.stderr}"] + after = _snapshot(build_path, name) + mtimes_after = _ninja_mtimes(build_path) + problems = [f"idf.py changed {f}" for f in before if before[f] != after[f]] + problems += _log_problems(build_path, mtimes_after) + for key in sorted(mtimes_before.keys() | mtimes_after.keys()): + log, out = key + if not out.endswith(WORK_SUFFIXES): + continue + if key not in mtimes_after: + problems.append(f"idf.py dropped {out} from {log}") + elif mtimes_before.get(key) != mtimes_after[key]: + problems.append(f"idf.py rebuilt {out}") + return problems + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + parser.add_argument( + "build_paths", + nargs="*", + type=Path, + help=f"ESPHome build dirs (default: the first native ESP-IDF tree in {DEFAULT_GLOB})", + ) + args = parser.parse_args() + paths = args.build_paths or sorted(REPO_ROOT.glob(DEFAULT_GLOB)) + # Not resolved: SDKCONFIG must be spelled as the build spelled it. + trees = [ + p + for p in paths + if (p / "build" / "project_description.json").is_file() + and (p / "build" / "CMakeCache.txt").is_file() + ] + if rejected := [p for p in args.build_paths if p not in trees]: + for path in rejected: + print(f"{path}: not a configured native ESP-IDF build tree") + return 1 + if not trees: + print("No native ESP-IDF build tree found") + return 1 + if not args.build_paths: + # The contract does not depend on the target, so one tree is enough. + trees = trees[:1] + + failed = False + for tree in trees: + problems = check(tree) + print(f"{tree}: {'OK' if not problems else 'DIFFERS'}") + for problem in problems: + print(f" {problem}") + failed |= bool(problems) + if failed: + print( + "The direct cmake/ninja build no longer matches idf.py. Compare " + "_IdfPyContract and its users in esphome/espidf/toolchain.py with the " + "pinned ESP-IDF tools/idf_py_actions." + ) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index e3a0ff688fb..bd127a7a435 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -592,10 +592,17 @@ def _esp32_platformio_path_or_file_trigger(files: list[str]) -> bool: ) +# Checks the native ESP-IDF build in CI but does not shape it, so it is kept +# out of ESP_IDF_INFRA_TRIGGER_FILES (hashed into the clang-tidy cache key). +_ESP_IDF_CHECK_TRIGGER_FILES = frozenset({"script/check_idf_py_equivalence.py"}) + + def _esp_idf_infra_changed(files: list[str]) -> bool: """Whether any changed file is ESP-IDF build/runner infrastructure.""" return _path_or_file_trigger( - files, ESP_IDF_INFRA_TRIGGER_FILES, ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES + files, + ESP_IDF_INFRA_TRIGGER_FILES | _ESP_IDF_CHECK_TRIGGER_FILES, + ESP_IDF_INFRA_TRIGGER_PATH_PREFIXES, ) @@ -1533,19 +1540,34 @@ def main() -> None: # Convert batches to CI matrix entries: the component list plus which # native toolchain installs the batch's test platforms need, so the # workflow only restores the matching multi-GB toolchain caches. + # The idf.py check does not depend on the components, so it runs once + # per workflow, in the first batch that compiles an esp32 test (a + # validate-only component is never compiled). + idf_py_check_assigned = False + skip_compile = set(validate_only_components) for batch in batches: platforms: set[str] = set() + compiled_platforms: set[str] = set() for component in batch: # Variants included: the compile stage builds them, so a # component tested only by test-..yaml # still needs that platform's toolchain - platforms.update( - get_component_test_platforms(component, base_only=False) + component_platforms = get_component_test_platforms( + component, base_only=False ) + platforms.update(component_platforms) + if component not in skip_compile: + compiled_platforms.update(component_platforms) + needs_idf = any(p.startswith("esp32") for p in platforms) + check_idf_py = not idf_py_check_assigned and any( + p.startswith("esp32") for p in compiled_platforms + ) + idf_py_check_assigned |= check_idf_py component_test_batches.append( { "components": " ".join(batch), - "needs_idf": any(p.startswith("esp32") for p in platforms), + "needs_idf": needs_idf, + "check_idf_py": check_idf_py, "needs_nrf": any(p.startswith("nrf52") for p in platforms), "needs_arduino8266": any( p.startswith("esp8266") for p in platforms diff --git a/tests/script/test_check_idf_py_equivalence.py b/tests/script/test_check_idf_py_equivalence.py new file mode 100644 index 00000000000..f8135772d0c --- /dev/null +++ b/tests/script/test_check_idf_py_equivalence.py @@ -0,0 +1,248 @@ +"""Tests for script/check_idf_py_equivalence.py.""" + +from collections.abc import Callable, Iterator +import json +from pathlib import Path +import subprocess +import sys +from unittest.mock import patch + +import pytest + +sys.path.insert(0, str(Path(__file__).parent.parent.parent / "script")) + +import check_idf_py_equivalence as guard # noqa: E402 + +from esphome.core import CORE # noqa: E402 +from esphome.espidf import toolchain # noqa: E402 + + +@pytest.fixture(autouse=True) +def _reset_core() -> Iterator[None]: + """check() points the global CORE at the tree it inspects.""" + yield + CORE.reset() + + +def _make_tree(tmp_path: Path) -> Path: + tree = tmp_path / "config" / ".esphome" / "build" / "dev" + build = tree / "build" + for name in (*guard.watched("dev"), *guard.NINJA_LOGS): + (tree / name).parent.mkdir(parents=True, exist_ok=True) + (tree / name).write_bytes(b"x") + (build / "project_description.json").write_text( + json.dumps( + { + "project_name": "dev", + "idf_path": "/idf/frameworks/5.5.5", + "target": "esp32", + } + ) + ) + (build / ".ninja_log").write_text( + "# ninja log v7\n1\t2\t10\tesp-idf/a.obj\t0\n" + "1\t2\t10\tbootloader/bootloader.bin\t0\n" + ) + (build / "bootloader" / ".ninja_log").write_text( + "# ninja log v7\n1\t2\t10\tbootloader.elf\t0\n" + ) + (tree / "sdkconfig.dev").write_text("") + return tree + + +def _run_check( + tree: Path, + side_effect: Callable[[list[str]], None] = lambda cmd: None, + rc: int = 0, + esphome_rcs: tuple[int, int] = (0, 0), +) -> tuple[list[str], list[list[str]]]: + """Run check() with idf.py replaced by ``side_effect``; return problems, calls.""" + calls: list[list[str]] = [] + + def run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess: + calls.append(cmd) + side_effect(cmd) + return subprocess.CompletedProcess(cmd, rc, "out\n", "err\n") + + with ( + patch.object(toolchain, "_get_idf_env", return_value={}), + patch.object(toolchain, "_get_idf_tool", return_value="/py"), + patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), + patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]), + patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]), + patch.object(guard.subprocess, "run", side_effect=run), + ): + return guard.check(tree), calls + + +def test_check_passes_when_idf_py_changes_nothing(tmp_path: Path) -> None: + tree = _make_tree(tmp_path) + problems, calls = _run_check(tree) + assert problems == [] + sdkconfig = f"SDKCONFIG={tree / 'sdkconfig.dev'}" + assert calls == [ + ["/py", str(Path("/idf/tools/idf.py")), "-D", sdkconfig, "reconfigure"], + ["/py", str(Path("/idf/tools/idf.py")), "-D", sdkconfig, "build"], + ] + + +def test_check_reports_changed_files_and_rebuilt_outputs(tmp_path: Path) -> None: + tree = _make_tree(tmp_path) + build = tree / "build" + + def drift(cmd: list[str]) -> None: + if cmd[-1] == "reconfigure": + (build / "CMakeCache.txt").write_text("changed") + return + # Compacted log. The re-logged bootloader byproduct and a stamp are + # not work; a new object mtime is. + (build / ".ninja_log").write_text( + "# ninja log v7\n3\t4\t20\tesp-idf/a.obj\t0\n" + "5\t6\t30\tbootloader/bootloader.bin\t0\n" + "5\t6\t30\tbootloader-stamp\t0\n" + ) + # The bootloader sub-build is judged by its own log. + (build / "bootloader" / ".ninja_log").write_text( + "# ninja log v7\n1\t2\t40\tbootloader.elf\t0\n" + ) + + problems, _ = _run_check(tree, drift) + assert problems == [ + "idf.py changed build/CMakeCache.txt", + "idf.py rebuilt esp-idf/a.obj", + "idf.py rebuilt bootloader.elf", + ] + + +@pytest.mark.parametrize( + ("after_build", "problem"), + [ + (lambda log: log.unlink(), "missing build/.ninja_log"), + ( + lambda log: log.write_text("# ninja log v7\n"), + "no build entries parsed from build/.ninja_log", + ), + ( + lambda log: log.write_text("# ninja log v7\n1\t2\t10\tesp-idf/b.obj\t0\n"), + "idf.py dropped esp-idf/a.obj from build/.ninja_log", + ), + ], + ids=["log-removed", "log-emptied", "entry-dropped"], +) +def test_check_reports_a_log_idf_py_left_unusable( + tmp_path: Path, after_build: Callable[[Path], None], problem: str +) -> None: + """The comparison side gets the same log checks as the baseline.""" + tree = _make_tree(tmp_path) + log = tree / guard.TOP_NINJA_LOG + + def run(cmd: list[str]) -> None: + if cmd[-1] == "build": + after_build(log) + + problems, _ = _run_check(tree, run) + assert problem in problems + + +def test_check_stops_when_idf_py_fails(tmp_path: Path) -> None: + problems, calls = _run_check(_make_tree(tmp_path), rc=2) + assert problems == ["idf.py reconfigure failed:\nout\nerr\n"] + assert len(calls) == 1 + + +@pytest.mark.parametrize( + "remove", ["build/build.ninja", "build/dev.bin", *guard.NINJA_LOGS] +) +def test_check_fails_when_an_input_is_missing(tmp_path: Path, remove: str) -> None: + """A moved or renamed output must not compare as unchanged.""" + tree = _make_tree(tmp_path) + (tree / remove).unlink() + problems, calls = _run_check(tree) + assert problems == [f"missing {remove}"] + assert calls == [] + + +@pytest.mark.parametrize( + ("esphome_rcs", "problem"), + [ + ((3, 0), "ESPHome's CMake configure failed with exit code 3"), + ((0, 4), "ESPHome's ninja build failed with exit code 4"), + ], + ids=["configure", "build"], +) +def test_check_stops_when_the_esphome_baseline_fails( + tmp_path: Path, esphome_rcs: tuple[int, int], problem: str +) -> None: + """The baseline is ESPHome's own reconfigure and build.""" + problems, calls = _run_check(_make_tree(tmp_path), esphome_rcs=esphome_rcs) + assert problems == [problem] + assert calls == [] + + +@pytest.mark.parametrize("log", guard.NINJA_LOGS) +def test_check_fails_when_a_ninja_log_has_no_entries(tmp_path: Path, log: str) -> None: + """A log format change must not leave the rebuild check with nothing to compare.""" + tree = _make_tree(tmp_path) + (tree / log).write_text("# ninja log v99\n1 2 3\n") + problems, calls = _run_check(tree) + assert problems == [f"no build entries parsed from {log}"] + assert calls == [] + + +def test_main_rejects_a_path_that_is_not_a_tree( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + tree = _make_tree(tmp_path / "a") + stale = tmp_path / "stale" + with ( + patch.object(sys, "argv", ["check", str(tree), str(stale)]), + patch.object(guard, "check", return_value=[]) as mock_check, + ): + assert guard.main() == 1 + assert f"{stale}: not a configured native ESP-IDF build tree" in ( + capsys.readouterr().out + ) + mock_check.assert_not_called() + + +def test_main_without_build_trees( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + with ( + patch.object(sys, "argv", ["check"]), + patch.object(guard, "REPO_ROOT", tmp_path), + ): + assert guard.main() == 1 + assert "No native ESP-IDF build tree found" in capsys.readouterr().out + + +@pytest.mark.parametrize(("problems", "rc"), [([], 0), (["idf.py changed x"], 1)]) +def test_main_reports_each_tree( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + problems: list[str], + rc: int, +) -> None: + tree = _make_tree(tmp_path) + with ( + patch.object(sys, "argv", ["check", str(tree)]), + patch.object(guard, "check", return_value=problems), + ): + assert guard.main() == rc + out = capsys.readouterr().out + assert f"{tree}: {'DIFFERS' if problems else 'OK'}" in out + assert ("no longer matches idf.py" in out) is bool(problems) + + +def test_main_checks_only_the_first_found_tree(tmp_path: Path) -> None: + """The contract does not depend on the target; one tree per batch is enough.""" + first = _make_tree(tmp_path / "a") + _make_tree(tmp_path / "b") + with ( + patch.object(sys, "argv", ["check"]), + patch.object(guard, "REPO_ROOT", tmp_path), + patch.object(guard, "DEFAULT_GLOB", "*/config/.esphome/build/*"), + patch.object(guard, "check", return_value=[]) as mock_check, + ): + assert guard.main() == 0 + mock_check.assert_called_once_with(first) diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 49f101d6792..cea2a2bfe6f 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -249,11 +249,61 @@ def test_main_all_tests_should_run( # Should contain at least one component (no empty batches) assert len(batch["components"]) > 0 assert isinstance(batch["needs_idf"], bool) + assert isinstance(batch["check_idf_py"], bool) assert isinstance(batch["needs_nrf"], bool) assert isinstance(batch["needs_arduino8266"], bool) -def test_main_batch_flags_count_variant_tests( +def _batch(components: str, idf: bool, check: bool, a8266: bool) -> dict: + return { + "components": components, + "needs_idf": idf, + "check_idf_py": check, + "needs_nrf": False, + "needs_arduino8266": a8266, + } + + +@pytest.mark.parametrize( + ("batches", "component_platforms", "validate_only", "expected"), + [ + # The compile stage builds test-..yaml too, so a + # component tested on esp8266 only by a variant still needs the toolchain. + ( + [["safe_mode"]], + {"safe_mode": {"esp8266-ard"}}, + set(), + [_batch("safe_mode", idf=False, check=False, a8266=True)], + ), + # Only the first esp32 batch runs the idf.py equivalence check. + ( + [["a"], ["b"], ["c"]], + {"a": {"esp8266-ard"}, "b": {"esp32-idf"}, "c": {"esp32-c3-idf"}}, + set(), + [ + _batch("a", idf=False, check=False, a8266=True), + _batch("b", idf=True, check=True, a8266=False), + _batch("c", idf=True, check=False, a8266=False), + ], + ), + # A validate-only batch never compiles, so the next esp32 batch runs it. + ( + [["b"], ["c"]], + {"b": {"esp32-idf"}, "c": {"esp32-c3-idf"}}, + {"b"}, + [ + _batch("b", idf=True, check=False, a8266=False), + _batch("c", idf=True, check=True, a8266=False), + ], + ), + ], + ids=["variant", "idf-check-once", "idf-check-skips-validate-only"], +) +def test_main_batch_flags( + batches: list[list[str]], + component_platforms: dict[str, set[str]], + validate_only: set[str], + expected: list[dict], mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, @@ -267,8 +317,7 @@ def test_main_batch_flags_count_variant_tests( capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - """The compile stage builds test-..yaml too, so a - component tested on esp8266 only by a variant still needs the toolchain.""" + """Each batch carries the toolchain flags its test platforms need.""" monkeypatch.delenv("GITHUB_ACTIONS", raising=False) mock_determine_integration_tests.return_value = (False, []) mock_should_run_clang_tidy.return_value = False @@ -282,13 +331,20 @@ def test_main_batch_flags_count_variant_tests( mock_changed_files.return_value = ["esphome/components/safe_mode/__init__.py"] def platforms(component: str, *, base_only: bool = True) -> set[str]: - return set() if base_only else {"esp8266-ard"} + return set() if base_only else component_platforms[component] with ( patch("sys.argv", ["determine-jobs.py"]), patch.object(determine_jobs, "_is_clang_tidy_full_scan", return_value=False), patch.object( - determine_jobs, "get_changed_components", return_value=["safe_mode"] + determine_jobs, + "get_changed_components", + return_value=[c for batch in batches for c in batch], + ), + patch.object( + determine_jobs, + "_component_change_is_validate_only", + side_effect=lambda component, changed: component in validate_only, ), patch.object( determine_jobs, @@ -298,7 +354,7 @@ def test_main_batch_flags_count_variant_tests( patch.object( determine_jobs, "get_components_with_dependencies", - return_value=["safe_mode"], + return_value=[c for batch in batches for c in batch], ), patch.object(determine_jobs, "_component_has_tests", return_value=True), patch.object( @@ -307,9 +363,7 @@ def test_main_batch_flags_count_variant_tests( return_value={"should_run": "false"}, ), patch.object( - determine_jobs, - "create_intelligent_batches", - return_value=([["safe_mode"]], {}), + determine_jobs, "create_intelligent_batches", return_value=(batches, {}) ), patch.object( determine_jobs, "get_component_test_platforms", side_effect=platforms @@ -318,14 +372,7 @@ def test_main_batch_flags_count_variant_tests( determine_jobs.main() output = json.loads(capsys.readouterr().out) - assert output["component_test_batches"] == [ - { - "components": "safe_mode", - "needs_idf": False, - "needs_nrf": False, - "needs_arduino8266": True, - } - ] + assert output["component_test_batches"] == expected def test_main_no_tests_should_run( @@ -1231,8 +1278,10 @@ def test_should_run_esp32_platformio_with_branch() -> None: @pytest.mark.parametrize( ("changed_files", "expected"), [ - # ESP-IDF runner / framework / build generator -> trigger - (["esphome/espidf/runner.py"], True), + # ESP-IDF toolchain / framework / build generator -> trigger + (["esphome/espidf/toolchain.py"], True), + (["esphome/build_helpers/tool_runner.py"], True), + (["script/check_idf_py_equivalence.py"], True), (["esphome/espidf/framework.py"], True), (["esphome/build_gen/espidf.py"], True), # Shared native-build modules the IDF build imports -> trigger @@ -1250,7 +1299,7 @@ def test_should_run_esp32_platformio_with_branch() -> None: ], ) def test_esp_idf_infra_changed(changed_files: list[str], expected: bool) -> None: - """ESP-IDF build/runner infra paths are detected; other paths are not.""" + """ESP-IDF build infra paths are detected; other paths are not.""" assert determine_jobs._esp_idf_infra_changed(changed_files) is expected diff --git a/tests/unit_tests/build_helpers/test_tool_runner.py b/tests/unit_tests/build_helpers/test_tool_runner.py new file mode 100644 index 00000000000..e2c7e44341a --- /dev/null +++ b/tests/unit_tests/build_helpers/test_tool_runner.py @@ -0,0 +1,338 @@ +"""Tests for esphome.build_helpers.tool_runner.""" + +# pylint: disable=protected-access + +import io +import os +from pathlib import Path +import sys +import threading +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from esphome.build_helpers import tool_runner +from esphome.build_helpers.tool_runner import ( + UTF8_CODEPAGE, + ToolOutput, + Utf8Console, + run_build_tool, +) +from esphome.core import CORE + +FILTER = [r"-- Component paths:", r"\s*$"] + + +def _child(code: str) -> list[str]: + return [sys.executable, "-c", code] + + +def _run( + capsys: pytest.CaptureFixture[str], + tmp_path: Path, + code: str, + **kwargs: object, +) -> tuple[int, str]: + rc = run_build_tool(_child(code), cwd=tmp_path, env=dict(os.environ), **kwargs) + # The child's print() ends lines with \r\n on Windows. + return rc, capsys.readouterr().out.replace("\r\n", "\n") + + +def test_run_build_tool_filters_and_returns_exit_code( + capsys: pytest.CaptureFixture[str], tmp_path: Path +) -> None: + rc, out = _run( + capsys, + tmp_path, + "import sys\n" + "print('-- Component paths: /a /b')\n" + "print('')\n" + "print('Compiling main.cpp')\n" + "sys.exit(3)", + filter_lines=FILTER, + ) + assert rc == 3 + assert out == "Compiling main.cpp\n" + + +def test_run_build_tool_passes_everything_without_filter( + capsys: pytest.CaptureFixture[str], tmp_path: Path +) -> None: + rc, out = _run(capsys, tmp_path, "print('-- Component paths: /a')\nprint('')") + assert rc == 0 + assert out == "-- Component paths: /a\n\n" + + +def test_run_build_tool_merges_stderr_and_runs_in_cwd( + capsys: pytest.CaptureFixture[str], tmp_path: Path +) -> None: + _, out = _run( + capsys, + tmp_path, + "import os, sys\nprint(os.getcwd(), flush=True)\nprint('oops', file=sys.stderr)", + filter_lines=FILTER, + ) + assert out.splitlines() == [os.path.realpath(tmp_path), "oops"] + + +def test_run_build_tool_drains_a_partial_line( + capsys: pytest.CaptureFixture[str], tmp_path: Path +) -> None: + """A tool that dies mid line still shows that line, terminated.""" + rc, out = _run( + capsys, + tmp_path, + "import sys\nsys.stdout.write('ld returned 1 exit status')\nsys.exit(1)", + filter_lines=FILTER, + ) + assert rc == 1 + assert out == "ld returned 1 exit status\n" + + +def test_run_build_tool_replaces_invalid_utf8( + capsys: pytest.CaptureFixture[str], tmp_path: Path +) -> None: + _, out = _run( + capsys, + tmp_path, + # A multi-byte character split across writes must survive too. + "import sys\nb = sys.stdout.buffer\n" + "b.write(b'\\xc3'); b.flush(); b.write(b'\\xa9 ok\\n\\xff bad\\n')", + filter_lines=FILTER, + ) + assert out == "é ok\n� bad\n" + + +def test_run_build_tool_streams_before_the_tool_exits(tmp_path: Path) -> None: + """Output must reach the user while the tool runs, not when it ends.""" + seen = threading.Event() + lines: list[str] = [] + + class _Out(io.StringIO): + def write(self, s: str) -> int: + lines.append(s) + if "first" in s: + seen.set() + return len(s) + + release = tmp_path / "release" + code = ( + "import os, time\nprint('first', flush=True)\n" + f"while not os.path.exists({str(release)!r}): time.sleep(0.02)\n" + ) + with patch.object(tool_runner.sys, "stdout", _Out()): + thread = threading.Thread( + target=run_build_tool, + args=(_child(code),), + kwargs={"cwd": tmp_path, "env": dict(os.environ), "filter_lines": FILTER}, + ) + thread.start() + try: + assert seen.wait(30) + finally: + release.touch() + thread.join(30) + assert "first" in "".join(lines) + + +def _tool_output(progress: bool = True) -> tuple[ToolOutput, io.StringIO]: + out = io.StringIO() + return ToolOutput(out, FILTER, progress), out + + +def test_tool_output_collapses_progress_lines() -> None: + """Ninja status lines overwrite each other, as idf.py shows them.""" + output, out = _tool_output() + output.write("[1/2] Building a.o\n[2/2] Linking app\nwarning: x\n") + assert out.getvalue() == ( + "\r[1/2] Building a.o\x1b[K\r[2/2] Linking app\x1b[K" + + os.linesep + + "warning: x\n" + ) + + +def test_tool_output_drain_ends_a_progress_line() -> None: + output, out = _tool_output() + output.write("[1/1] Linking app\n") + output.drain() + assert out.getvalue() == "\r[1/1] Linking app\x1b[K" + os.linesep + # Nothing is pending any more. + output.drain() + assert out.getvalue().count(os.linesep) == 1 + + +def test_tool_output_drain_survives_a_broken_stream() -> None: + output, out = _tool_output() + output.write("[1/1] Linking app\n") + out.close() + output.drain() + + +def test_tool_output_filters_a_matching_progress_line() -> None: + out = io.StringIO() + output = ToolOutput(out, [r"\[1/2\]"], True) + output.write("[1/2] hidden\n[2/2] shown\n") + assert out.getvalue() == "\r[2/2] shown\x1b[K" + + +def test_tool_output_without_progress_keeps_status_lines() -> None: + output, out = _tool_output(progress=False) + output.write("[1/2] Building a.o\n") + assert out.getvalue() == "[1/2] Building a.o\n" + + +def test_tool_output_escapes_colors_for_the_dashboard() -> None: + CORE.dashboard = True + output, out = _tool_output() + output.write("[1/1] \x1b[1mLinking\x1b[0m\n") + assert "\x1b" not in out.getvalue() + assert "\\033[K" in out.getvalue() + + +@pytest.mark.parametrize( + ("width", "text", "expected"), + [ + (0, "x" * 50, "x" * 50), + (3, "abcdef", "..."), + (20, "short", "short"), + (11, "abcdefghijklmnop", "abcd...mnop"), + ], + ids=["unknown", "tiny", "fits", "elided"], +) +def test_fit_terminal(width: int, text: str, expected: str) -> None: + with patch.object( + tool_runner.shutil, + "get_terminal_size", + return_value=os.terminal_size((width, 24)), + ): + assert tool_runner._fit_terminal(text) == expected + + +class _FakeKernel32: + """Stand-in for the Windows kernel32 console code page calls.""" + + def __init__(self, input_cp: int, output_cp: int) -> None: + self.input_cp = input_cp + self.output_cp = output_cp + self.calls: list[tuple[str, int]] = [] + + def GetConsoleCP(self) -> int: # noqa: N802 + return self.input_cp + + def GetConsoleOutputCP(self) -> int: # noqa: N802 + return self.output_cp + + def SetConsoleCP(self, codepage: int) -> int: # noqa: N802 + self.calls.append(("SetConsoleCP", codepage)) + self.input_cp = codepage + return 1 + + def SetConsoleOutputCP(self, codepage: int) -> int: # noqa: N802 + self.calls.append(("SetConsoleOutputCP", codepage)) + self.output_cp = codepage + return 1 + + +def test_run_build_tool_switches_the_console_to_utf8( + capsys: pytest.CaptureFixture[str], tmp_path: Path +) -> None: + """An attached console runs the tool on UTF-8 and is then put back.""" + kernel32 = _FakeKernel32(850, 850) + with patch.object(tool_runner, "_get_kernel32", return_value=kernel32): + _run(capsys, tmp_path, "print('x')") + assert kernel32.calls == [ + ("SetConsoleCP", UTF8_CODEPAGE), + ("SetConsoleOutputCP", UTF8_CODEPAGE), + ("SetConsoleCP", 850), + ("SetConsoleOutputCP", 850), + ] + + +def test_utf8_console_restores_after_an_error() -> None: + kernel32 = _FakeKernel32(437, 437) + with pytest.raises(RuntimeError), Utf8Console(kernel32): + raise RuntimeError + assert (kernel32.input_cp, kernel32.output_cp) == (437, 437) + + +def test_utf8_console_restores_when_the_switch_fails_part_way() -> None: + kernel32 = _FakeKernel32(850, 850) + + def _refuse(codepage: int) -> int: + kernel32.calls.append(("SetConsoleOutputCP", codepage)) + return 0 + + kernel32.SetConsoleOutputCP = _refuse # type: ignore[method-assign] + with Utf8Console(kernel32): + pass + assert kernel32.input_cp == 850 + assert kernel32.calls[-2:] == [("SetConsoleCP", 850), ("SetConsoleOutputCP", 850)] + + +@pytest.mark.parametrize( + "codepages", [(0, 0), (UTF8_CODEPAGE, UTF8_CODEPAGE)], ids=["none", "utf8"] +) +def test_utf8_console_leaves_the_console_alone(codepages: tuple[int, int]) -> None: + """No console, or one already on UTF-8 (an overlapping build), is untouched.""" + kernel32 = _FakeKernel32(*codepages) + with Utf8Console(kernel32): + pass + assert kernel32.calls == [] + + +def test_utf8_console_without_kernel32() -> None: + with Utf8Console(None): + pass + + +@pytest.mark.skipif(sys.platform == "win32", reason="kernel32 exists on Windows") +def test_get_kernel32_is_none_off_windows() -> None: + assert tool_runner._get_kernel32() is None + + +def test_tool_output_collapses_progress_without_a_filter() -> None: + """Progress mode splits lines on its own; it does not need a filter.""" + out = io.StringIO() + output = ToolOutput(out, None, True) + output.write("[1/1] Linking app\ndone\n") + assert out.getvalue() == "\r[1/1] Linking app\x1b[K" + os.linesep + "done\n" + + +def test_run_build_tool_flushes_a_truncated_character( + capsys: pytest.CaptureFixture[str], tmp_path: Path +) -> None: + """Output that ends inside a multi-byte character still shows up.""" + _, out = _run( + capsys, + tmp_path, + "import sys\nsys.stdout.buffer.write(b'end \\xc3')", + filter_lines=FILTER, + ) + assert out == "end �\n" + + +def test_get_kernel32_loads_it_on_windows(monkeypatch: pytest.MonkeyPatch) -> None: + kernel32 = object() + fake_ctypes = SimpleNamespace(windll=SimpleNamespace(kernel32=kernel32)) + monkeypatch.setattr(tool_runner.sys, "platform", "win32") + monkeypatch.setitem(sys.modules, "ctypes", fake_ctypes) + assert tool_runner._get_kernel32() is kernel32 + + +def test_run_build_tool_logs_the_unfiltered_output( + capsys: pytest.CaptureFixture[str], tmp_path: Path +) -> None: + """The log gets every line, filtered or not, for idf.py's hint matcher.""" + log = tmp_path / "log" / "ninja_all_output.log" + _, out = _run( + capsys, + tmp_path, + "print('-- Component paths: /a')\nprint('\\x1b[1merror:\\x1b[0m boom')", + filter_lines=FILTER, + log_path=log, + ) + assert out == "\x1b[1merror:\x1b[0m boom\n" + assert log.read_text(encoding="utf-8").replace("\r\n", "\n") == ( + "-- Component paths: /a\nerror: boom\n" + ) diff --git a/tests/unit_tests/fixtures/espidf/closing_probe.py b/tests/unit_tests/fixtures/espidf/closing_probe.py deleted file mode 100644 index a77d5c8f281..00000000000 --- a/tests/unit_tests/fixtures/espidf/closing_probe.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Leave a partial line behind and then close the stream under the runner. - -Run through ``esphome/espidf/runner.py`` by test_espidf_runner.py. Draining -cannot work here; the point is that the failure is reported rather than -raised out of the runner's cleanup, where it would bury the exit code. -""" - -import sys - -sys.stdout.write("partial before close") -sys.stdout.close() diff --git a/tests/unit_tests/fixtures/espidf/crashing_probe.py b/tests/unit_tests/fixtures/espidf/crashing_probe.py deleted file mode 100644 index bf434cc24e1..00000000000 --- a/tests/unit_tests/fixtures/espidf/crashing_probe.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Die part way through a line, the way a build that blows up does. - -Run through ``esphome/espidf/runner.py`` by test_espidf_runner.py. The -message has no trailing newline, so the runner's shim is holding it when -the process exits; nothing else will ever come to release it. -""" - -import sys - -sys.stdout.write("FATAL: ld returned 1 exit status") -sys.exit(2) diff --git a/tests/unit_tests/fixtures/espidf/filtering_probe.py b/tests/unit_tests/fixtures/espidf/filtering_probe.py deleted file mode 100644 index 04c2b2ed8cc..00000000000 --- a/tests/unit_tests/fixtures/espidf/filtering_probe.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Write a mix of noisy and useful build lines, without flushing. - -Run through ``esphome/espidf/runner.py`` by test_espidf_runner.py. The -runner's shim owns both the filtering and the flushing, so this script -only writes. -""" - -import sys - -sys.stdout.write("Project build complete.\n") -sys.stdout.write("Compiling main.cpp\n") -sys.stdout.write("-- Component paths: /a /b /c\n") -sys.stdout.write("[2/9] Building C object\n") -# No terminator, so the shim has to hold this one back. -sys.stdout.write("still going") diff --git a/tests/unit_tests/fixtures/espidf/formfeed_probe.py b/tests/unit_tests/fixtures/espidf/formfeed_probe.py deleted file mode 100644 index 727cda25ce7..00000000000 --- a/tests/unit_tests/fixtures/espidf/formfeed_probe.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Write a form feed part way through the output. - -Run through ``esphome/espidf/runner.py`` by test_espidf_runner.py. A form -feed is not a line terminator here, so everything written must still come -out, including the complete lines that follow it. -""" - -import sys - -sys.stdout.write("Compiling main.cpp\n") -sys.stdout.write("page one\x0cpage two\n") -sys.stdout.write("[2/9] Building C object\n") diff --git a/tests/unit_tests/fixtures/espidf/partial_noise_probe.py b/tests/unit_tests/fixtures/espidf/partial_noise_probe.py deleted file mode 100644 index 9c81f8eb7b3..00000000000 --- a/tests/unit_tests/fixtures/espidf/partial_noise_probe.py +++ /dev/null @@ -1,10 +0,0 @@ -"""End on an unterminated line that the filter is supposed to drop. - -Run through ``esphome/espidf/runner.py`` by test_espidf_runner.py, to -check that releasing a held-back line still applies the filter. -""" - -import sys - -sys.stdout.write("Compiling main.cpp\n") -sys.stdout.write("Project build complete.") diff --git a/tests/unit_tests/fixtures/espidf/streaming_probe.py b/tests/unit_tests/fixtures/espidf/streaming_probe.py deleted file mode 100644 index c05741e3117..00000000000 --- a/tests/unit_tests/fixtures/espidf/streaming_probe.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Print one line, then stay alive so the caller can prove it streamed. - -Run through ``esphome/espidf/runner.py`` by test_espidf_runner.py. The -runner wraps stdout in its filtering shim, so this script deliberately -does not flush: the shim has to do it. The long sleep keeps the process -running, so anything the caller reads must have arrived while the build -was still going rather than at exit. -""" - -import sys -import time - -sys.stdout.write("Compiling main.cpp\n") -time.sleep(60) diff --git a/tests/unit_tests/test_espidf_clang_tidy.py b/tests/unit_tests/test_espidf_clang_tidy.py index 4cc445e29be..e0cfec88c85 100644 --- a/tests/unit_tests/test_espidf_clang_tidy.py +++ b/tests/unit_tests/test_espidf_clang_tidy.py @@ -1,5 +1,7 @@ """Tests for esphome.espidf.clang_tidy tidy-project generation.""" +# pylint: disable=protected-access + import json import os from pathlib import Path @@ -99,3 +101,39 @@ def test_idedata_from_tidy_project_missing_tu_raises(tmp_path) -> None: compile_commands.write_text(json.dumps([])) with pytest.raises(RuntimeError, match="tidy.cpp not found"): clang_tidy._idedata_from_tidy_project(compile_commands) + + +@pytest.mark.parametrize( + ("reconfigure_rcs", "error"), + [ + ((1,), "ESP-IDF CMake configure \\(discovery\\) failed"), + ((0, 1), "ESP-IDF CMake configure failed"), + ((0, 0), None), + ], + ids=["discovery", "full", "ok"], +) +def test_generate_compile_commands_configures_twice( + tmp_path: Path, reconfigure_rcs: tuple[int, ...], error: str | None +) -> None: + """Discovery configure, then a configure requiring what it found.""" + with ( + patch.object(clang_tidy, "_setup_core"), + patch.object(clang_tidy, "_convert_pio_libs", return_value={}), + patch.object(clang_tidy, "_write_tidy_project") as mock_write, + patch("esphome.espidf.toolchain.run_reconfigure", side_effect=reconfigure_rcs), + patch( + "esphome.build_gen.espidf.get_available_components", + return_value=["lwip", "esp_timer"], + ), + ): + if error: + with pytest.raises(RuntimeError, match=error): + clang_tidy._generate_compile_commands( + tmp_path, _settings(), tmp_path / "platformio.ini" + ) + return + result = clang_tidy._generate_compile_commands( + tmp_path, _settings(), tmp_path / "platformio.ini" + ) + assert result == tmp_path / "build" / "compile_commands.json" + assert mock_write.call_args_list[1].args[1] == ["esp_timer", "lwip"] diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py index 9654c7cfc57..c785d56fa41 100644 --- a/tests/unit_tests/test_espidf_framework.py +++ b/tests/unit_tests/test_espidf_framework.py @@ -1627,8 +1627,8 @@ def test_ccache_env_opt_out_via_env(tmp_path: Path) -> None: # short-circuits before build_path is needed. p1, p2, p3 = _ccache_patches(tmp_path, "/usr/bin/ccache", None) with patch.dict("os.environ", {"IDF_CCACHE_ENABLE": "0"}, clear=True), p1, p2, p3: - # The canonical off spelling is exported: the raw value is inherited - # by idf.py, where a spelling like "disable" would read as truthy + # The canonical off spelling is exported, so every reader of the + # env sees the same answer assert _ccache_env() == {"IDF_CCACHE_ENABLE": "0"} @@ -1636,7 +1636,7 @@ def test_ccache_env_opt_in_without_binary( tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: # Explicit IDF_CCACHE_ENABLE=1 forces it on; without a usable binary - # idf.py silently skips ccache, so this branch must say so out loud. + # IDF's CMake silently skips ccache, so this branch must say so out loud. p1, p2, p3 = _ccache_patches(tmp_path, None, tmp_path / "build") with patch.dict("os.environ", {"IDF_CCACHE_ENABLE": "1"}, clear=True), p1, p2, p3: env = _ccache_env() @@ -1669,7 +1669,7 @@ def test_ccache_env_opt_in_with_working_binary( def test_ccache_env_opt_in_with_rejected_binary( tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: - # Forced on with a present-but-rejected binary: idf.py does its own + # Forced on with a present-but-rejected binary: IDF's CMake does its own # PATH lookup and uses it anyway; the warning must say so, not claim # the build runs without ccache. # A present but non-executable file: the real probe fails and logs @@ -1686,7 +1686,7 @@ def test_ccache_env_opt_in_with_rejected_binary( ): env = _ccache_env() assert env["IDF_CCACHE_ENABLE"] == "1" - assert "idf.py will use it anyway" in caplog.text + assert "the build will use it anyway" in caplog.text # Exactly one story: the resolver's contradictory "compiling without # ccache" must not precede it assert "compiling without ccache" not in caplog.text @@ -1715,7 +1715,7 @@ def test_ccache_env_idf_knob_unrecognized_warns_and_defers( tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: """An unparsable IDF_CCACHE_ENABLE warns, defers to the shared resolver, - and is not forwarded to idf.py as truthy.""" + and is not forwarded to the build as truthy.""" p1, p2, p3 = _ccache_patches(tmp_path, "/usr/bin/ccache", tmp_path / "build") env_vars = {"IDF_CCACHE_ENABLE": "enabled"} with patch.dict("os.environ", env_vars, clear=True), p1, p2, p3: diff --git a/tests/unit_tests/test_espidf_runner.py b/tests/unit_tests/test_espidf_runner.py deleted file mode 100644 index 71ccfee1f2c..00000000000 --- a/tests/unit_tests/test_espidf_runner.py +++ /dev/null @@ -1,315 +0,0 @@ -"""Tests for esphome.espidf.runner.""" - -from __future__ import annotations - -import io -import os -from pathlib import Path -import subprocess -import sys -import threading - -import pytest - -from esphome.espidf import runner - -# A flushing runner delivers the first line in well under a second; this is -# only ever waited out when the shim has gone back to buffering, so keep it -# just long enough to cover interpreter startup on a loaded CI machine. -FIRST_LINE_TIMEOUT = 10.0 - - -def _prepare_main( - monkeypatch: pytest.MonkeyPatch, probe: Path, *args: str -) -> tuple[io.BytesIO, io.TextIOWrapper]: - """Point ``runner.main()`` at *probe* with a buffered fake stdout. - - ``main`` rewrites ``sys.path``, ``sys.argv``, both std streams and - ``os.get_terminal_size``; every one of those is monkeypatched so it is - put back afterwards. The fake stdout is block buffered like a pipe, so - the caller can tell whether the shim flushed. The wrapper comes back with - the buffer because dropping it would close the buffer underneath us. - """ - buf = io.BytesIO() - stream = io.TextIOWrapper(buf, encoding="utf-8", newline="\n", line_buffering=False) - - monkeypatch.setattr(sys, "path", list(sys.path)) - monkeypatch.setattr(sys, "argv", ["runner.py", str(probe), *args]) - monkeypatch.setattr(sys, "stdout", stream) - monkeypatch.setattr(sys, "stderr", stream) - monkeypatch.setattr(os, "get_terminal_size", os.get_terminal_size) - - return buf, stream - - -def _run_main( - monkeypatch: pytest.MonkeyPatch, probe: Path, *args: str -) -> tuple[io.BytesIO, io.TextIOWrapper]: - """Run ``runner.main()`` against *probe* and expect a clean exit.""" - buf, stream = _prepare_main(monkeypatch, probe, *args) - assert runner.main() == 0 - return buf, stream - - -def test_main_filters_noise_and_flushes_each_write( - monkeypatch: pytest.MonkeyPatch, fixture_path: Path -) -> None: - """Useful lines reach the stream right away; noisy ones are dropped.""" - buf, _stream = _run_main( - monkeypatch, fixture_path / "espidf" / "filtering_probe.py" - ) - - # Read before any flush of our own: the shim has to have flushed. - output = buf.getvalue().decode("utf-8") - - assert "Compiling main.cpp\n" in output - assert "[2/9] Building C object\n" in output - # Matched by FILTER_IDF_LINES, so they never leave the runner. - assert "Project build complete." not in output - assert "-- Component paths:" not in output - # Held back until the end because no terminator arrived. - assert output.endswith("still going\n") - - -def test_main_keeps_output_after_a_form_feed( - monkeypatch: pytest.MonkeyPatch, fixture_path: Path -) -> None: - """A form feed is text, not a line break, so nothing after it is lost.""" - buf, _stream = _run_main(monkeypatch, fixture_path / "espidf" / "formfeed_probe.py") - - assert buf.getvalue().decode("utf-8") == ( - "Compiling main.cpp\npage one\x0cpage two\n[2/9] Building C object\n" - ) - - -def test_main_drains_a_partial_line_when_the_build_dies( - monkeypatch: pytest.MonkeyPatch, fixture_path: Path -) -> None: - """A build that stops mid line must still show that line. - - This is the whole point of draining: the message explaining why the - build failed is exactly the one most likely to arrive without a - trailing newline. - """ - buf, _stream = _prepare_main( - monkeypatch, fixture_path / "espidf" / "crashing_probe.py" - ) - - with pytest.raises(SystemExit) as excinfo: - runner.main() - - assert excinfo.value.code == 2 - assert buf.getvalue().decode("utf-8") == "FATAL: ld returned 1 exit status\n" - - -def test_main_reports_rather_than_raises_when_draining_fails( - monkeypatch: pytest.MonkeyPatch, - fixture_path: Path, - capfd: pytest.CaptureFixture[str], -) -> None: - """A stream that closed under us must not crash the runner's cleanup. - - The drain runs from a ``finally``, so an exception there would replace - whatever exit code the build was carrying back. - """ - _prepare_main(monkeypatch, fixture_path / "espidf" / "closing_probe.py") - - assert runner.main() == 0 - reported = capfd.readouterr().err - assert "Could not write out remaining output" in reported - # The held line has to come along; the stream it was meant for is gone. - assert "partial before close" in reported - - -def test_main_survives_a_drain_failure_with_nowhere_to_report_it( - monkeypatch: pytest.MonkeyPatch, fixture_path: Path -) -> None: - """With no real stderr to report to, cleanup still must not raise. - - ``sys.__stderr__`` is None on some interpreters, and ``print(file=None)`` - falls back to ``sys.stdout``, which here is the shim wrapping the stream - that just failed. - """ - monkeypatch.setattr(sys, "__stderr__", None) - _prepare_main(monkeypatch, fixture_path / "espidf" / "closing_probe.py") - - assert runner.main() == 0 - - -def test_main_still_filters_a_drained_partial_line( - monkeypatch: pytest.MonkeyPatch, fixture_path: Path -) -> None: - """Releasing a held line does not smuggle noise past the filter.""" - buf, _stream = _run_main( - monkeypatch, fixture_path / "espidf" / "partial_noise_probe.py" - ) - - assert buf.getvalue().decode("utf-8") == "Compiling main.cpp\n" - - -def test_main_keeps_everything_in_verbose_mode( - monkeypatch: pytest.MonkeyPatch, fixture_path: Path -) -> None: - """``-v`` turns the filter off so the noisy lines survive.""" - buf, _stream = _run_main( - monkeypatch, fixture_path / "espidf" / "filtering_probe.py", "-v" - ) - - output = buf.getvalue().decode("utf-8") - - assert "Project build complete.\n" in output - assert "-- Component paths: /a /b /c\n" in output - # With no filter there is no line buffering, so the partial line goes - # straight through as well. - assert output.endswith("still going") - - -def test_runner_streams_output_before_the_build_finishes( - fixture_path: Path, probe_env: dict[str, str] -) -> None: - """The runner must flush, or a dashboard build looks frozen. - - ``toolchain.py`` spawns the runner as a plain script with no ``-u``, and - hands it a pipe when esphome itself is running under the dashboard. A - pipe is block buffered, so without a flush in the shim's ``write()`` the - output sits in the child until 8 KiB piles up or the build ends. - """ - runner_py = Path(runner.__file__) - probe = fixture_path / "espidf" / "streaming_probe.py" - - with subprocess.Popen( - [sys.executable, str(runner_py), str(probe)], - stdout=subprocess.PIPE, - # Keep stderr: if the runner dies on startup, its traceback is the - # only clue about why no line showed up. - stderr=subprocess.PIPE, - env=probe_env, - text=True, - ) as proc: - assert proc.stdout is not None - assert proc.stderr is not None - first_line: list[str] = [] - reader = threading.Thread( - target=lambda: first_line.append(proc.stdout.readline()), daemon=True - ) - try: - reader.start() - reader.join(FIRST_LINE_TIMEOUT) - still_running = proc.poll() is None - - # The probe sleeps for a minute after writing, so reaching us at - # all means the line was flushed rather than released at exit. - assert first_line == ["Compiling main.cpp\n"], ( - f"runner stderr: {'' if still_running else proc.stderr.read()}" - ) - assert still_running - finally: - proc.kill() - proc.wait() - # Join before leaving the block, so the reader is done rather than - # racing ``Popen`` closing the pipe under it. - reader.join(1.0) - - -class _FakeKernel32: - """Stand-in for the Windows kernel32 console code page calls.""" - - def __init__(self, input_cp: int, output_cp: int) -> None: - self.input_cp = input_cp - self.output_cp = output_cp - self.calls: list[tuple[str, int]] = [] - - def GetConsoleCP(self) -> int: # noqa: N802 - return self.input_cp - - def GetConsoleOutputCP(self) -> int: # noqa: N802 - return self.output_cp - - def SetConsoleCP(self, codepage: int) -> int: # noqa: N802 - self.calls.append(("SetConsoleCP", codepage)) - self.input_cp = codepage - return 1 - - def SetConsoleOutputCP(self, codepage: int) -> int: # noqa: N802 - self.calls.append(("SetConsoleOutputCP", codepage)) - self.output_cp = codepage - return 1 - - -def test_main_runs_the_build_with_a_utf8_console( - monkeypatch: pytest.MonkeyPatch, fixture_path: Path -) -> None: - """An attached console is switched to UTF-8 and then put back.""" - kernel32 = _FakeKernel32(850, 850) - monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) - - _run_main(monkeypatch, fixture_path / "espidf" / "filtering_probe.py") - - assert kernel32.calls == [ - ("SetConsoleCP", runner.UTF8_CODEPAGE), - ("SetConsoleOutputCP", runner.UTF8_CODEPAGE), - ("SetConsoleCP", 850), - ("SetConsoleOutputCP", 850), - ] - - -def test_main_restores_the_console_when_the_build_dies( - monkeypatch: pytest.MonkeyPatch, fixture_path: Path -) -> None: - """A failing build must not leave the user's console on UTF-8.""" - kernel32 = _FakeKernel32(437, 437) - monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) - _prepare_main(monkeypatch, fixture_path / "espidf" / "crashing_probe.py") - - with pytest.raises(SystemExit): - runner.main() - - assert (kernel32.input_cp, kernel32.output_cp) == (437, 437) - - -def test_main_restores_the_console_when_the_switch_fails_part_way( - monkeypatch: pytest.MonkeyPatch, fixture_path: Path -) -> None: - """A failed output page switch must not strand the changed input page.""" - kernel32 = _FakeKernel32(850, 850) - - def _refuse(codepage: int) -> int: - kernel32.calls.append(("SetConsoleOutputCP", codepage)) - return 0 - - kernel32.SetConsoleOutputCP = _refuse # type: ignore[method-assign] - monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) - - _run_main(monkeypatch, fixture_path / "espidf" / "filtering_probe.py") - - assert kernel32.input_cp == 850 - assert kernel32.calls[-2:] == [("SetConsoleCP", 850), ("SetConsoleOutputCP", 850)] - - -def test_main_leaves_the_console_alone_when_there_is_none( - monkeypatch: pytest.MonkeyPatch, fixture_path: Path -) -> None: - """Without a console the code page calls return 0 and nothing is set.""" - kernel32 = _FakeKernel32(0, 0) - monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) - - _run_main(monkeypatch, fixture_path / "espidf" / "filtering_probe.py") - - assert kernel32.calls == [] - - -def test_main_leaves_a_console_already_on_utf8_alone( - monkeypatch: pytest.MonkeyPatch, fixture_path: Path -) -> None: - """An overlapping build must not save UTF-8 as the page to restore.""" - kernel32 = _FakeKernel32(runner.UTF8_CODEPAGE, runner.UTF8_CODEPAGE) - monkeypatch.setattr(runner, "_get_kernel32", lambda: kernel32) - - _run_main(monkeypatch, fixture_path / "espidf" / "filtering_probe.py") - - assert kernel32.calls == [] - - -@pytest.mark.skipif(sys.platform == "win32", reason="kernel32 exists on Windows") -def test_get_kernel32_is_none_off_windows() -> None: - assert runner._get_kernel32() is None diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 3e1e88d66da..ee21cd77a9c 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -8,11 +8,12 @@ import json import os from pathlib import Path import subprocess -from unittest.mock import patch +from unittest.mock import call, patch import pytest -from esphome.components.esp32.const import KEY_ESP32, KEY_VARIANT +from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT +import esphome.config_validation as cv from esphome.const import ( CONF_COMPILE_PROCESS_LIMIT, CONF_ESPHOME, @@ -104,11 +105,27 @@ def _setup_build(setup_core: Path) -> tuple[Path, Path]: """Point CORE at a build dir; return (compile_commands, idedata cache) paths.""" CORE.name = "test" CORE.build_path = setup_core / "build" / "test" + CORE.data.setdefault(KEY_ESP32, {})[KEY_IDF_VERSION] = cv.Version(5, 5, 5) compile_commands = CORE.relative_build_path("build", "compile_commands.json") cache = CORE.relative_internal_path("idedata", "test.json") return compile_commands, cache +@contextmanager +def _up_to_date_compile(ninja_side_effect=None) -> Iterator[tuple]: + """Patch run_compile's staleness checks to "up to date"; yield the ninja + and print_summary mocks.""" + with ( + patch.object(toolchain, "need_reconfigure", return_value=False), + patch.object(toolchain, "_cache_entries_changed", return_value=False), + patch.object( + toolchain, "_run_ninja", return_value=0, side_effect=ninja_side_effect + ) as mock_ninja, + patch.object(toolchain, "print_summary") as mock_summary, + ): + yield mock_ninja, mock_summary + + def test_has_outdated_files_detects_exclusion_change(setup_core: Path) -> None: """A newer exclude_components.esphomeinternal stamp forces a reconfigure so components that leave the exclusion set get rediscovered.""" @@ -279,28 +296,6 @@ def test_get_cmake_output_missing_build_does_not_resolve_idf_env( mock_run.assert_not_called() -def test_run_idf_py_jobs_sets_build_jobs_env(setup_core: Path) -> None: - """The jobs argument is exported to idf.py as IDF_PY_BUILD_JOBS.""" - _setup_build(setup_core) - - with ( - patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), - patch.object(toolchain, "_get_idf_env", return_value={"PATH": "/bin"}), - patch.object(toolchain, "_get_idf_tool", return_value="python"), - patch.object(toolchain.subprocess, "run") as mock_run, - ): - mock_run.return_value.returncode = 0 - - toolchain.run_idf_py("build", jobs=2) - env = mock_run.call_args.kwargs["env"] - assert env["IDF_PY_BUILD_JOBS"] == "2" - assert env["PATH"] == "/bin" - - toolchain.run_idf_py("build") - env = mock_run.call_args.kwargs["env"] - assert "IDF_PY_BUILD_JOBS" not in env - - def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> None: """After a successful discovery reconfigure the reference CMakeCache.txt is restamped; cmake does not rewrite it when only properties or plain @@ -325,7 +320,7 @@ def test_run_compile_restamps_cmakecache_after_discovery(setup_core: Path) -> No ), patch("esphome.build_gen.espidf.write_project"), patch.object(toolchain, "run_reconfigure", return_value=0), - patch.object(toolchain, "run_idf_py", return_value=0), + patch.object(toolchain, "_run_ninja", return_value=0), patch.object(toolchain, "print_summary"), ): assert toolchain.run_compile(config, verbose=False) == 0 @@ -350,7 +345,7 @@ def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None: ), patch("esphome.build_gen.espidf.write_project"), patch.object(toolchain, "run_reconfigure", return_value=0), - patch.object(toolchain, "run_idf_py", return_value=0), + patch.object(toolchain, "_run_ninja", return_value=0), patch.object(toolchain, "print_summary"), ): assert toolchain.run_compile(config, verbose=False) == 0 @@ -376,7 +371,7 @@ def test_run_compile_reconfigures_after_full_write_outside_testing_mode( def record_write(minimal: bool = False, builtin_components=None) -> None: calls.append(("write_project", minimal)) - def record_reconfigure() -> int: + def record_reconfigure(verbose: bool = False) -> int: nonlocal reconfigures reconfigures += 1 calls.append(("run_reconfigure",)) @@ -391,7 +386,7 @@ def test_run_compile_reconfigures_after_full_write_outside_testing_mode( ), patch("esphome.build_gen.espidf.write_project", side_effect=record_write), patch.object(toolchain, "run_reconfigure", side_effect=record_reconfigure), - patch.object(toolchain, "run_idf_py", return_value=0) as mock_build, + patch.object(toolchain, "_run_ninja", return_value=0) as mock_build, patch.object(toolchain, "print_summary"), ): assert not CORE.testing_mode @@ -421,7 +416,7 @@ def _record_compile_calls( calls: list[tuple] = [] rcs = iter(reconfigure_rcs) - def record_reconfigure() -> int: + def record_reconfigure(verbose: bool = False) -> int: calls.append(("run_reconfigure",)) return next(rcs, 0) @@ -431,6 +426,11 @@ def _record_compile_calls( def record_save(components: list[str]) -> None: calls.append(("save", components)) + def record_ninja(target: str, **kwargs: object) -> int: + if target == "all": + calls.append(("build",)) + return 0 + with ( patch.object(toolchain, "need_reconfigure", return_value=True), patch.object(toolchain, "load_cached_builtin_components", return_value=cached), @@ -443,11 +443,7 @@ def _record_compile_calls( patch.object( toolchain, "_builtin_component_cache_path", return_value=cache_file ), - patch.object( - toolchain, - "run_idf_py", - side_effect=lambda *a, **kw: calls.append(("build",)) or 0, - ), + patch.object(toolchain, "_run_ninja", side_effect=record_ninja), patch.object(toolchain, "print_summary"), ): rc = toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) @@ -630,19 +626,21 @@ def test_component_cache_ignores_corrupt_file(setup_core: Path, tmp_path: Path) assert toolchain.load_cached_builtin_components() is None -def test_run_compile_passes_compile_process_limit(setup_core: Path) -> None: - """compile_process_limit is forwarded to run_idf_py as the job limit.""" +@pytest.mark.parametrize("limit", [1, None]) +def test_run_compile_passes_compile_process_limit( + setup_core: Path, limit: int | None +) -> None: + """compile_process_limit is the job limit for both ninja runs.""" _setup_build(setup_core) - config = {CONF_ESPHOME: {CONF_COMPILE_PROCESS_LIMIT: 1}} + esphome = {} if limit is None else {CONF_COMPILE_PROCESS_LIMIT: limit} - with ( - patch.object(toolchain, "need_reconfigure", return_value=False), - patch.object(toolchain, "run_idf_py", return_value=0) as mock_run, - patch.object(toolchain, "print_summary"), - ): - assert toolchain.run_compile(config, verbose=False) == 0 + with _up_to_date_compile() as (mock_run, _): + assert toolchain.run_compile({CONF_ESPHOME: esphome}, verbose=False) == 0 - mock_run.assert_called_once_with("build", "size", jobs=1) + assert mock_run.call_args_list == [ + call("all", verbose=False, jobs=limit, progress=True), + call("size", verbose=False, jobs=limit, extra_env=toolchain._size_env()), + ] def test_run_compile_passes_size_summary_paths(setup_core: Path) -> None: @@ -650,14 +648,8 @@ def test_run_compile_passes_size_summary_paths(setup_core: Path) -> None: ELF from get_built_elf_path, which must stay in lockstep with the project() name in the generated CMakeLists.""" _setup_build(setup_core) - config = {CONF_ESPHOME: {}} - - with ( - patch.object(toolchain, "need_reconfigure", return_value=False), - patch.object(toolchain, "run_idf_py", return_value=0), - patch.object(toolchain, "print_summary") as mock_summary, - ): - assert toolchain.run_compile(config, verbose=False) == 0 + with _up_to_date_compile() as (_, mock_summary): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 mock_summary.assert_called_once_with( CORE.relative_build_path("build", "esp_idf_size.json"), @@ -682,21 +674,6 @@ def test_create_elf_copy_missing_source(setup_core: Path) -> None: assert toolchain.create_elf_copy() is False -def test_run_compile_without_compile_process_limit(setup_core: Path) -> None: - """When no compile_process_limit is set, no job limit is passed to idf.py.""" - _setup_build(setup_core) - config = {CONF_ESPHOME: {}} - - with ( - patch.object(toolchain, "need_reconfigure", return_value=False), - patch.object(toolchain, "run_idf_py", return_value=0) as mock_run, - patch.object(toolchain, "print_summary"), - ): - assert toolchain.run_compile(config, verbose=False) == 0 - - mock_run.assert_called_once_with("build", "size", jobs=None) - - def test_run_compile_writes_the_pch_checksum_before_the_build( setup_core: Path, ) -> None: @@ -704,11 +681,7 @@ def test_run_compile_writes_the_pch_checksum_before_the_build( order: list[str] = [] with ( - patch.object(toolchain, "need_reconfigure", return_value=False), - patch.object( - toolchain, "run_idf_py", side_effect=lambda *a, **k: order.append("build") - ), - patch.object(toolchain, "print_summary"), + _up_to_date_compile(lambda *a, **k: order.append("build") or 0), patch( "esphome.build_gen.espidf.write_pch_checksum", side_effect=lambda: order.append("checksum"), @@ -727,6 +700,404 @@ def test_get_core_framework_version_from_core_data(): assert toolchain._get_core_framework_version() == "5.5.4" +@contextmanager +def _fake_tools(env: dict[str, str] | None = None) -> Iterator: + """Stub the IDF env and tool lookup; yield the run_build_tool mock.""" + with ( + patch.object( + toolchain, + "_get_idf_env", + return_value={"PATH": "/bin", "IDF_CCACHE_ENABLE": "0", **(env or {})}, + ), + patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"), + patch.object(toolchain, "run_build_tool", return_value=0) as mock_run, + ): + yield mock_run + + +def test_run_reconfigure_cmake_argv_matches_idf_py(setup_core: Path) -> None: + """The configure is the command idf.py runs. + + IDF 5.5.5 tools/idf_py_actions/tools.py ensure_build_directory: cmake + -G Ninja -DPYTHON_DEPS_CHECKED=1 -DPYTHON= -DESP_PLATFORM=1, + then the -D entries (SDKCONFIG, CCACHE_ENABLE), then the real project + path, in the build dir. IDF 6.1 adds -B , the same as the cwd. + """ + _setup_build(setup_core) + sdkconfig = CORE.relative_build_path("sdkconfig.test") + sdkconfig.parent.mkdir(parents=True) + sdkconfig.write_text("") + project = os.path.realpath(CORE.build_path) + + with _fake_tools() as mock_run: + assert toolchain.run_reconfigure() == 0 + + assert mock_run.call_args.args[0] == [ + "/tools/cmake", + "-G", + "Ninja", + "-DPYTHON_DEPS_CHECKED=1", + "-DPYTHON=/tools/python", + "-DESP_PLATFORM=1", + f"-DSDKCONFIG={sdkconfig}", + "-DCCACHE_ENABLE=0", + project, + ] + kwargs = mock_run.call_args.kwargs + assert kwargs["cwd"] == Path(project) / "build" + assert kwargs["cwd"].is_dir() + assert kwargs["env"]["CLICOLOR_FORCE"] == "1" + assert kwargs["filter_lines"] is toolchain.FILTER_IDF_LINES + + +@pytest.mark.parametrize(("ccache", "expected"), [("1", "True"), ("0", "False")]) +def test_run_reconfigure_cmake_argv_matches_idf6_py( + setup_core: Path, ccache: str, expected: str +) -> None: + """IDF 6 idf.py adds -B and formats CCACHE_ENABLE as True/False. + + IDF 6.1 tools/idf_py_actions/tools.py ensure_build_directory. + """ + _setup_build(setup_core) + CORE.data[KEY_ESP32][KEY_IDF_VERSION] = cv.Version(6, 1, 0) + build_dir = Path(os.path.realpath(CORE.build_path)) / "build" + + with _fake_tools({"IDF_CCACHE_ENABLE": ccache}) as mock_run: + assert toolchain.run_reconfigure() == 0 + + assert mock_run.call_args.args[0] == [ + "/tools/cmake", + "-G", + "Ninja", + "-B", + str(build_dir), + "-DPYTHON_DEPS_CHECKED=1", + "-DPYTHON=/tools/python", + "-DESP_PLATFORM=1", + f"-DCCACHE_ENABLE={expected}", + str(build_dir.parent), + ] + + +def test_run_reconfigure_without_sdkconfig_or_filter(setup_core: Path) -> None: + """No sdkconfig file means no SDKCONFIG entry; -v shows every line.""" + _setup_build(setup_core) + with _fake_tools() as mock_run: + assert toolchain.run_reconfigure(verbose=True) == 0 + cmd = mock_run.call_args.args[0] + assert not any(arg.startswith("-DSDKCONFIG=") for arg in cmd) + assert cmd[-2] == "-DCCACHE_ENABLE=0" + assert mock_run.call_args.kwargs["filter_lines"] is None + + +def test_run_reconfigure_failure_removes_cmakecache( + setup_core: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Like idf.py, a failed configure must not leave a cache that looks valid.""" + _setup_build(setup_core) + cache = Path(os.path.realpath(CORE.build_path)) / "build" / "CMakeCache.txt" + cache.parent.mkdir(parents=True) + cache.write_text("") + with ( + _fake_tools() as mock_run, + patch.object(toolchain, "_print_hints") as mock_hints, + ): + mock_run.return_value = 4 + assert toolchain.run_reconfigure() == 4 + assert not cache.exists() + assert "CMake configure failed with exit code 4" in caplog.text + log_path = mock_run.call_args.kwargs["log_path"] + assert log_path == cache.parent / "log" / "cmake_output.log" + mock_hints.assert_called_once_with(log_path) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("1", "1"), + ("on", "1"), + (" Yes ", "1"), + ("0", "0"), + ("", "0"), + ("disable", "0"), + ], +) +def test_cache_entries_ccache_follows_click_booleans( + setup_core: Path, value: str, expected: str +) -> None: + """IDF_CCACHE_ENABLE maps to CCACHE_ENABLE the way idf.py's click flag does.""" + _setup_build(setup_core) + with _fake_tools({"IDF_CCACHE_ENABLE": value}): + assert toolchain._cache_entries() == {"CCACHE_ENABLE": expected} + + +def test_parse_cmakecache(tmp_path: Path) -> None: + cache = tmp_path / "CMakeCache.txt" + cache.write_text( + "# comment\n" + "// help text\n" + "SDKCONFIG:UNINITIALIZED=/a/sdkconfig.test\n" + "CCACHE_ENABLE:UNINITIALIZED=1\n" + "FLAGS:STRING=-DX=1\n" + "\n" + ) + assert toolchain._parse_cmakecache(cache) == { + "SDKCONFIG": "/a/sdkconfig.test", + "CCACHE_ENABLE": "1", + "FLAGS": "-DX=1", + } + + +# The constant -D entries every configure writes (see _configure_defines). +_CONFIGURED = ( + "PYTHON_DEPS_CHECKED:UNINITIALIZED=1\n" + "PYTHON:UNINITIALIZED=/tools/python\n" + "ESP_PLATFORM:UNINITIALIZED=1\n" +) + + +@pytest.mark.parametrize( + ("cache_text", "expected"), + [ + (None, True), + (_CONFIGURED + "CCACHE_ENABLE:UNINITIALIZED=0\n", False), + (_CONFIGURED + "CCACHE_ENABLE:UNINITIALIZED=1\n", True), + ( + _CONFIGURED.replace("/tools/python", "/old/python") + + "CCACHE_ENABLE:UNINITIALIZED=0\n", + True, + ), + (_CONFIGURED, True), + ], + ids=["no_cache", "same", "changed", "python_moved", "missing"], +) +def test_cache_entries_changed( + setup_core: Path, cache_text: str | None, expected: bool +) -> None: + """Mirrors idf.py's _new_cmakecache_entries, plus the cached PYTHON.""" + _setup_build(setup_core) + if cache_text is not None: + cache = Path(os.path.realpath(CORE.build_path)) / "build" / "CMakeCache.txt" + cache.parent.mkdir(parents=True) + cache.write_text(cache_text) + with _fake_tools(): + assert toolchain._cache_entries_changed() is expected + + +@pytest.mark.parametrize( + ("version", "size_ng"), [(cv.Version(5, 5, 5), True), (cv.Version(6, 1, 0), False)] +) +def test_size_env(setup_core: Path, version: cv.Version, size_ng: bool) -> None: + """IDF 5.x idf.py sets ESP_IDF_SIZE_NG; 6.x dropped it (core_ext.py size_target).""" + CORE.data[KEY_ESP32] = {KEY_IDF_VERSION: version} + env = toolchain._size_env() + assert env["ESP_IDF_SIZE_FORCE_TERMINAL"] == "1" + assert env["SIZE_OUTPUT_FORMAT"] == "default" + assert ("ESP_IDF_SIZE_NG" in env) is size_ng + + +def test_run_ninja_matches_idf_py_run_target(setup_core: Path) -> None: + """The command is ninja [-j N] [-v] in the build dir.""" + _setup_build(setup_core) + with _fake_tools() as mock_run: + assert ( + toolchain._run_ninja( + "size", verbose=True, jobs=2, progress=True, extra_env={"A": "b"} + ) + == 0 + ) + assert mock_run.call_args.args[0] == ["/tools/ninja", "-j", "2", "-v", "size"] + kwargs = mock_run.call_args.kwargs + assert kwargs["cwd"] == Path(os.path.realpath(CORE.build_path)) / "build" + assert kwargs["env"]["CLICOLOR_FORCE"] == "1" + assert kwargs["env"]["A"] == "b" + # -v shows every line as it comes, as idf.py does + assert kwargs["filter_lines"] is None + assert kwargs["progress"] is False + + +def test_run_ninja_filters_and_reports_failure( + setup_core: Path, caplog: pytest.LogCaptureFixture +) -> None: + _setup_build(setup_core) + with ( + _fake_tools() as mock_run, + patch.object(toolchain, "_print_hints") as mock_hints, + ): + mock_run.return_value = 1 + assert toolchain._run_ninja("all", verbose=False, jobs=None, progress=True) == 1 + log_path = mock_run.call_args.kwargs["log_path"] + assert log_path.name == "ninja_all_output.log" + mock_hints.assert_called_once_with(log_path) + assert mock_run.call_args.args[0] == ["/tools/ninja", "all"] + assert mock_run.call_args.kwargs["filter_lines"] is toolchain.FILTER_IDF_LINES + assert mock_run.call_args.kwargs["progress"] is True + assert "ninja all failed with exit code 1" in caplog.text + + +@pytest.mark.parametrize("reconfigure_rc", [0, 5]) +def test_run_compile_reconfigures_when_cache_entries_change( + setup_core: Path, reconfigure_rc: int +) -> None: + """A changed -D entry (ccache toggled) reconfigures, as idf.py build did.""" + _setup_build(setup_core) + with ( + patch.object(toolchain, "need_reconfigure", return_value=False), + patch.object(toolchain, "_cache_entries_changed", return_value=True), + patch.object( + toolchain, "run_reconfigure", return_value=reconfigure_rc + ) as mock_reconfigure, + patch.object(toolchain, "_run_ninja", return_value=0) as mock_ninja, + patch.object(toolchain, "print_summary"), + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=True) == reconfigure_rc + mock_reconfigure.assert_called_once_with(True) + assert mock_ninja.called is (reconfigure_rc == 0) + + +@pytest.mark.parametrize("failing", ["all", "size"]) +def test_run_compile_stops_on_ninja_failure(setup_core: Path, failing: str) -> None: + """A failed build skips size; either failure skips the summary.""" + _setup_build(setup_core) + with _up_to_date_compile(lambda target, **kw: 7 if target == failing else 0) as ( + mock_ninja, + mock_summary, + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 7 + targets = [c.args[0] for c in mock_ninja.call_args_list] + assert targets == (["all"] if failing == "all" else ["all", "size"]) + mock_summary.assert_not_called() + + +@pytest.mark.parametrize("memory_ld_rc", [0, 3]) +def test_run_compile_testing_mode_builds_memory_ld_first( + setup_core: Path, memory_ld_rc: int +) -> None: + """Testing mode builds and patches memory.ld before the main build.""" + _setup_build(setup_core) + CORE.testing_mode = True + targets: list[str] = [] + + def record(target: str, **kwargs: object) -> int: + targets.append(target) + return memory_ld_rc if target.endswith("memory.ld") else 0 + + with ( + _up_to_date_compile(record), + patch.object(toolchain, "_patch_memory_segments") as mock_patch, + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == memory_ld_rc + memory_ld = str(Path("esp-idf", "esp_system", "ld", "memory.ld")) + if memory_ld_rc: + assert targets == [memory_ld] + mock_patch.assert_not_called() + else: + assert targets == [memory_ld, "all", "size"] + mock_patch.assert_called_once() + + +@pytest.mark.parametrize("version", [cv.Version(5, 5, 5), cv.Version(6, 1, 0)]) +@pytest.mark.parametrize( + ("env", "expected"), + [ + ({}, {"CLICOLOR_FORCE": "1", "FORCE_COLOR": "1"}), + ({"CLICOLOR_FORCE": "0"}, {"CLICOLOR_FORCE": "0", "FORCE_COLOR": "1"}), + ({"NO_COLOR": "1"}, {}), + ], + ids=["default", "user-value", "no-color"], +) +def test_tool_env_colors( + setup_core: Path, version: cv.Version, env: dict[str, str], expected: dict[str, str] +) -> None: + """IDF 6 RunTool.__call__ color handling, used for every IDF version.""" + _setup_build(setup_core) + CORE.data[KEY_ESP32][KEY_IDF_VERSION] = version + with _fake_tools(env): + result = toolchain._tool_env() + colors = {k: v for k, v in result.items() if k in ("CLICOLOR_FORCE", "FORCE_COLOR")} + assert colors == expected + + +@pytest.mark.parametrize( + ("limit", "env_value", "expected"), + [ + (2, "8", 2), + (None, "8", 8), + (None, "", None), + (None, None, None), + ], + ids=["limit-wins", "env", "empty-env", "unset"], +) +def test_build_jobs_honors_idf_py_build_jobs( + monkeypatch: pytest.MonkeyPatch, + limit: int | None, + env_value: str | None, + expected: int | None, +) -> None: + """compile_process_limit first, else IDF_PY_BUILD_JOBS as idf.py read it.""" + if env_value is None: + monkeypatch.delenv("IDF_PY_BUILD_JOBS", raising=False) + else: + monkeypatch.setenv("IDF_PY_BUILD_JOBS", env_value) + esphome = {} if limit is None else {CONF_COMPILE_PROCESS_LIMIT: limit} + assert toolchain._build_jobs({CONF_ESPHOME: esphome}) == expected + + +@pytest.mark.parametrize("value", ["0", "-1", "many"]) +def test_build_jobs_rejects_invalid_idf_py_build_jobs( + monkeypatch: pytest.MonkeyPatch, value: str +) -> None: + """Like idf.py, a value that is not a positive integer is an error.""" + monkeypatch.setenv("IDF_PY_BUILD_JOBS", value) + with pytest.raises(EsphomeError, match="positive integer"): + toolchain._build_jobs({CONF_ESPHOME: {}}) + + +@contextmanager +def _hint_env(tmp_path: Path, **run_kwargs: object) -> Iterator: + with ( + patch.object(toolchain, "_get_idf_tool", return_value="/py"), + patch.object(toolchain, "_get_idf_path", return_value=tmp_path / "idf"), + patch.object(toolchain, "_get_idf_env", return_value={}), + patch.object(toolchain.subprocess, "run", **run_kwargs) as mock_run, + ): + yield mock_run + + +def test_print_hints_shows_idf_advice( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The hint step runs IDF's own generate_hints on the failed tool's log.""" + log = tmp_path / "ninja_all_output.log" + done = subprocess.CompletedProcess([], 0, "HINT: the binary is too big\n", "") + with _hint_env(tmp_path, return_value=done) as mock_run: + toolchain._print_hints(log) + cmd = mock_run.call_args.args[0] + assert cmd[0] == "/py" + assert "generate_hints" in cmd[2] + assert cmd[3:] == [str(tmp_path / "idf" / "tools"), str(log)] + assert "HINT: the binary is too big" in caplog.text + + +@pytest.mark.parametrize( + "run_kwargs", + [ + {"return_value": subprocess.CompletedProcess([], 1, "", "ImportError")}, + {"return_value": subprocess.CompletedProcess([], 0, " \n", "")}, + {"side_effect": subprocess.TimeoutExpired("py", 60)}, + {"side_effect": OSError("gone")}, + ], + ids=["script-fails", "no-hints", "timeout", "oserror"], +) +def test_print_hints_never_fails_the_build( + tmp_path: Path, caplog: pytest.LogCaptureFixture, run_kwargs: dict +) -> None: + """No hints is fine; the hint step must not raise or add warnings.""" + with caplog.at_level("WARNING"), _hint_env(tmp_path, **run_kwargs): + toolchain._print_hints(tmp_path / "log") + assert caplog.records == [] + + def test_get_cmake_cache_value_reads_the_configured_cache(setup_core: Path) -> None: _setup_build(setup_core) output = "CMAKE_OBJDUMP:FILEPATH=/tools/objdump\nESPHOME_PCH:BOOL=OFF\n" From a43d3382b537d3146471c0cb531b1e1305fa00fe Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:58:12 +0200 Subject: [PATCH 050/133] [socket] Let Zephyr clang-tidy see the BSD socket implementation (#19901) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- esphome/core/defines.h | 1 + 1 file changed, 1 insertion(+) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 115bae96c6e..bec97fc82ca 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -623,6 +623,7 @@ #define USE_NRF52_REG0_VOUT 5 #define USE_NRF52_UICR_ERASE #define USE_OTA_ROLLBACK +#define USE_SOCKET_IMPL_BSD_SOCKETS #define USE_SOFTDEVICE_ID 7 #define USE_SOFTDEVICE_VERSION 1 #define USE_ZIGBEE From 46be513a39cb4bddd93ac5f034821fb916eec792 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Tue, 29 Sep 2026 21:34:38 +0200 Subject: [PATCH 051/133] [core] Create the PlatformIO HTTP cache dir before the first registry lookup (#19906) --- esphome/platformio/library.py | 5 +++++ tests/unit_tests/test_platformio_library.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index af2010711f6..e551d8f1c04 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -620,8 +620,13 @@ def _make_registry_client() -> Any: elsewhere, not by the PlatformIO registry. """ from platformio.package.manager._registry import PackageManagerRegistryMixin + from platformio.project.helpers import get_project_cache_dir from platformio.registry.client import RegistryClient + # PlatformIO creates its HTTP cache dir without exist_ok, so two builds + # making their first registry lookup at once race on it + (Path(get_project_cache_dir()) / "http").mkdir(parents=True, exist_ok=True) + class _Registry(PackageManagerRegistryMixin): def __init__(self) -> None: self.pkg_type = "library" diff --git a/tests/unit_tests/test_platformio_library.py b/tests/unit_tests/test_platformio_library.py index dba8977f1a4..658c7211ab4 100644 --- a/tests/unit_tests/test_platformio_library.py +++ b/tests/unit_tests/test_platformio_library.py @@ -247,6 +247,20 @@ def test_make_registry_client_skips_private_package_probe(monkeypatch): assert RegistryClient.__dict__["allowed_private_packages"] is pio_probe +def test_make_registry_client_creates_http_cache_dir( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The HTTP cache dir exists before PlatformIO's non-exist_ok makedirs runs.""" + from platformio.project import helpers + + monkeypatch.setattr(helpers, "get_project_cache_dir", lambda: str(tmp_path)) + + lib._make_registry_client() + assert (tmp_path / "http").is_dir() + # A second client (another build) must not trip over the existing dir + lib._make_registry_client() + + def _patch_registry_resolve(monkeypatch: pytest.MonkeyPatch) -> None: """Stub the registry lookup so tests never touch the network.""" monkeypatch.setattr( From fc270cb90f897d3ed29cb8f9fa1c7e0fc5fdf49e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 21:48:15 +0200 Subject: [PATCH 052/133] Bump platformdirs from 4.11.14 to 4.12.0 (#19904) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b62cd712a70..3d5dd8d5baa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ bleak==3.0.2 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 -platformdirs==4.11.14 # native esp-idf toolchain global cache dir +platformdirs==4.12.0 # native esp-idf toolchain global cache dir ninja==1.13.2 # native host and esp8266 arduino toolchain build driver filelock==4.0.3 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg From 6b465443bc4b62ff0d5444a735c1c26856e9cc56 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 21:48:38 +0200 Subject: [PATCH 053/133] Bump filelock from 4.0.3 to 4.0.4 (#19905) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3d5dd8d5baa..2a0b7b31f47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ requests==2.34.2 py7zr==1.1.3 platformdirs==4.12.0 # native esp-idf toolchain global cache dir ninja==1.13.2 # native host and esp8266 arduino toolchain build driver -filelock==4.0.3 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg +filelock==4.0.4 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this pyparsing >= 3.3.3 From 662c7db2223f8190a292619fd51d247769a403d1 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Tue, 29 Sep 2026 15:00:23 -0500 Subject: [PATCH 054/133] [nfc] Pass tags to triggers by reference and keep the listener list in static storage (#19902) Co-authored-by: Claude Fable 5.1 --- esphome/components/nfc/__init__.py | 3 ++- esphome/components/nfc/automation.h | 3 ++- esphome/components/nfc/binary_sensor/__init__.py | 6 ++++-- .../nfc/binary_sensor/nfc_binary_sensor.cpp | 5 +---- .../nfc/binary_sensor/nfc_binary_sensor.h | 5 +---- esphome/components/nfc/nfc.h | 4 +++- esphome/components/nfc/nfc_helpers.cpp | 8 ++++---- esphome/components/nfc/nfc_helpers.h | 4 ++-- esphome/components/nfc/nfc_tag.h | 7 ++++--- esphome/components/pn532/__init__.py | 4 ++-- esphome/components/pn71xx/__init__.py | 4 ++-- esphome/components/pn71xx/pn71xx.cpp | 4 ++++ esphome/core/defines.h | 1 + tests/components/pn532_spi/common.yaml | 9 +++++++++ tests/components/pn7160_spi/common.yaml | 15 +++++++++++++-- 15 files changed, 54 insertions(+), 28 deletions(-) diff --git a/esphome/components/nfc/__init__.py b/esphome/components/nfc/__init__.py index eea1a47b24b..6afc2eca092 100644 --- a/esphome/components/nfc/__init__.py +++ b/esphome/components/nfc/__init__.py @@ -7,7 +7,8 @@ nfc_ns = cg.esphome_ns.namespace("nfc") Nfcc = nfc_ns.class_("Nfcc") NfcTag = nfc_ns.class_("NfcTag") +NfcTagConstRef = NfcTag.operator("ref").operator("const") NfcTagListener = nfc_ns.class_("NfcTagListener") NfcOnTagTrigger = nfc_ns.class_( - "NfcOnTagTrigger", automation.Trigger.template(cg.std_string, NfcTag) + "NfcOnTagTrigger", automation.Trigger.template(cg.std_string, NfcTagConstRef) ) diff --git a/esphome/components/nfc/automation.h b/esphome/components/nfc/automation.h index ec3a979b647..6e03430d6fa 100644 --- a/esphome/components/nfc/automation.h +++ b/esphome/components/nfc/automation.h @@ -7,7 +7,8 @@ namespace esphome::nfc { -class NfcOnTagTrigger final : public Trigger { +/// Fires with the formatted UID and the tag itself; the tag is passed by reference so no copy is made per trigger +class NfcOnTagTrigger final : public Trigger { public: void process(const std::unique_ptr &tag); }; diff --git a/esphome/components/nfc/binary_sensor/__init__.py b/esphome/components/nfc/binary_sensor/__init__.py index e9747d7a142..2fde6804da9 100644 --- a/esphome/components/nfc/binary_sensor/__init__.py +++ b/esphome/components/nfc/binary_sensor/__init__.py @@ -20,7 +20,6 @@ NfcTagBinarySensor = nfc_ns.class_( binary_sensor.BinarySensor, cg.Component, NfcTagListener, - cg.Parented.template(Nfcc), ) @@ -59,12 +58,15 @@ CONFIG_SCHEMA = cv.All( ) +_request_listener_slot = cg.slot_counter("NFC_TAG_LISTENER_COUNT") + + async def to_code(config: ConfigType) -> None: var = await binary_sensor.new_binary_sensor(config) await cg.register_component(var, config) - await cg.register_parented(var, config[CONF_NFCC_ID]) hub = await cg.get_variable(config[CONF_NFCC_ID]) + _request_listener_slot(str(hub)) cg.add(hub.register_listener(var)) if CONF_NDEF_CONTAINS in config: cg.add(var.set_ndef_match_string(config[CONF_NDEF_CONTAINS])) diff --git a/esphome/components/nfc/binary_sensor/nfc_binary_sensor.cpp b/esphome/components/nfc/binary_sensor/nfc_binary_sensor.cpp index 6e8162fc912..995255d1e45 100644 --- a/esphome/components/nfc/binary_sensor/nfc_binary_sensor.cpp +++ b/esphome/components/nfc/binary_sensor/nfc_binary_sensor.cpp @@ -7,10 +7,7 @@ namespace esphome::nfc { static const char *const TAG = "nfc.binary_sensor"; -void NfcTagBinarySensor::setup() { - this->parent_->register_listener(this); - this->publish_initial_state(false); -} +void NfcTagBinarySensor::setup() { this->publish_initial_state(false); } void NfcTagBinarySensor::dump_config() { std::string match_str = "name"; diff --git a/esphome/components/nfc/binary_sensor/nfc_binary_sensor.h b/esphome/components/nfc/binary_sensor/nfc_binary_sensor.h index 6354e169670..f5def158348 100644 --- a/esphome/components/nfc/binary_sensor/nfc_binary_sensor.h +++ b/esphome/components/nfc/binary_sensor/nfc_binary_sensor.h @@ -8,10 +8,7 @@ namespace esphome::nfc { -class NfcTagBinarySensor final : public binary_sensor::BinarySensor, - public Component, - public NfcTagListener, - public Parented { +class NfcTagBinarySensor final : public binary_sensor::BinarySensor, public Component, public NfcTagListener { public: void setup() override; void dump_config() override; diff --git a/esphome/components/nfc/nfc.h b/esphome/components/nfc/nfc.h index 603cb8d3148..e0f73e934ca 100644 --- a/esphome/components/nfc/nfc.h +++ b/esphome/components/nfc/nfc.h @@ -83,10 +83,12 @@ class NfcTagListener { class Nfcc { public: +#ifdef NFC_TAG_LISTENER_COUNT void register_listener(NfcTagListener *listener) { this->tag_listeners_.push_back(listener); } protected: - std::vector tag_listeners_; + StaticVector tag_listeners_; +#endif }; } // namespace esphome::nfc diff --git a/esphome/components/nfc/nfc_helpers.cpp b/esphome/components/nfc/nfc_helpers.cpp index 6c8a5b626d5..82f4591bdf4 100644 --- a/esphome/components/nfc/nfc_helpers.cpp +++ b/esphome/components/nfc/nfc_helpers.cpp @@ -4,14 +4,14 @@ namespace esphome::nfc { static const char *const TAG = "nfc.helpers"; -bool has_ha_tag_ndef(NfcTag &tag) { return !get_ha_tag_ndef(tag).empty(); } +bool has_ha_tag_ndef(const NfcTag &tag) { return !get_ha_tag_ndef(tag).empty(); } -std::string get_ha_tag_ndef(NfcTag &tag) { +std::string get_ha_tag_ndef(const NfcTag &tag) { if (!tag.has_ndef_message()) { return std::string(); } - auto message = tag.get_ndef_message(); - auto records = message->get_records(); + const auto &message = tag.get_ndef_message(); + const auto &records = message->get_records(); for (const auto &record : records) { std::string payload = record->get_payload(); size_t pos = payload.find(HA_TAG_ID_PREFIX); diff --git a/esphome/components/nfc/nfc_helpers.h b/esphome/components/nfc/nfc_helpers.h index dedc602bf19..eff6db0ea2a 100644 --- a/esphome/components/nfc/nfc_helpers.h +++ b/esphome/components/nfc/nfc_helpers.h @@ -8,8 +8,8 @@ static const char HA_TAG_ID_EXT_RECORD_TYPE[] = "android.com:pkg"; static const char HA_TAG_ID_EXT_RECORD_PAYLOAD[] = "io.homeassistant.companion.android"; static const char HA_TAG_ID_PREFIX[] = "https://www.home-assistant.io/tag/"; -std::string get_ha_tag_ndef(NfcTag &tag); +std::string get_ha_tag_ndef(const NfcTag &tag); std::string get_random_ha_tag_ndef(); -bool has_ha_tag_ndef(NfcTag &tag); +bool has_ha_tag_ndef(const NfcTag &tag); } // namespace esphome::nfc diff --git a/esphome/components/nfc/nfc_tag.h b/esphome/components/nfc/nfc_tag.h index 6cc1a00c627..a20c435e21f 100644 --- a/esphome/components/nfc/nfc_tag.h +++ b/esphome/components/nfc/nfc_tag.h @@ -42,9 +42,10 @@ class NfcTag { } NfcTagUid &get_uid() { return this->uid_; }; - const std::string &get_tag_type() { return this->tag_type_; }; - bool has_ndef_message() { return this->ndef_message_ != nullptr; }; - const std::shared_ptr &get_ndef_message() { return this->ndef_message_; }; + const NfcTagUid &get_uid() const { return this->uid_; }; + const std::string &get_tag_type() const { return this->tag_type_; }; + bool has_ndef_message() const { return this->ndef_message_ != nullptr; }; + const std::shared_ptr &get_ndef_message() const { return this->ndef_message_; }; void set_ndef_message(std::unique_ptr ndef_message) { this->ndef_message_ = std::move(ndef_message); }; protected: diff --git a/esphome/components/pn532/__init__.py b/esphome/components/pn532/__init__.py index f4f16a8b890..c9118834c56 100644 --- a/esphome/components/pn532/__init__.py +++ b/esphome/components/pn532/__init__.py @@ -66,7 +66,7 @@ async def setup_pn532(var: MockObj, config: ConfigType) -> None: _request_ontag_trigger_slot(str(var)) cg.add(var.register_ontag_trigger(trigger)) await automation.build_automation( - trigger, [(cg.std_string, "x"), (nfc.NfcTag, "tag")], conf + trigger, [(cg.std_string, "x"), (nfc.NfcTagConstRef, "tag")], conf ) for conf in config.get(CONF_ON_TAG_REMOVED, []): @@ -74,7 +74,7 @@ async def setup_pn532(var: MockObj, config: ConfigType) -> None: _request_ontagremoved_trigger_slot(str(var)) cg.add(var.register_ontagremoved_trigger(trigger)) await automation.build_automation( - trigger, [(cg.std_string, "x"), (nfc.NfcTag, "tag")], conf + trigger, [(cg.std_string, "x"), (nfc.NfcTagConstRef, "tag")], conf ) await automation.build_callback_automations(var, config, _CALLBACK_AUTOMATIONS) diff --git a/esphome/components/pn71xx/__init__.py b/esphome/components/pn71xx/__init__.py index a8825747d17..8472d4192c4 100644 --- a/esphome/components/pn71xx/__init__.py +++ b/esphome/components/pn71xx/__init__.py @@ -144,7 +144,7 @@ async def setup_pn71xx(var: MockObj, config: ConfigType) -> None: _request_ontag_trigger_slot(str(var)) cg.add(var.register_ontag_trigger(trigger)) await automation.build_automation( - trigger, [(cg.std_string, "x"), (nfc.NfcTag, "tag")], conf + trigger, [(cg.std_string, "x"), (nfc.NfcTagConstRef, "tag")], conf ) for conf in config.get(CONF_ON_TAG_REMOVED, []): @@ -152,7 +152,7 @@ async def setup_pn71xx(var: MockObj, config: ConfigType) -> None: _request_ontagremoved_trigger_slot(str(var)) cg.add(var.register_ontagremoved_trigger(trigger)) await automation.build_automation( - trigger, [(cg.std_string, "x"), (nfc.NfcTag, "tag")], conf + trigger, [(cg.std_string, "x"), (nfc.NfcTagConstRef, "tag")], conf ) await automation.build_callback_automations(var, config, _CALLBACK_AUTOMATIONS) diff --git a/esphome/components/pn71xx/pn71xx.cpp b/esphome/components/pn71xx/pn71xx.cpp index 3d0b46efb47..6a24acd843e 100644 --- a/esphome/components/pn71xx/pn71xx.cpp +++ b/esphome/components/pn71xx/pn71xx.cpp @@ -569,9 +569,11 @@ void PN71xx::erase_tag_(const uint8_t tag_index) { trigger->process(this->discovered_endpoint_[tag_index].tag); } #endif +#ifdef NFC_TAG_LISTENER_COUNT for (auto *listener : this->tag_listeners_) { listener->tag_off(*this->discovered_endpoint_[tag_index].tag); } +#endif char uid_buf[nfc::FORMAT_UID_BUFFER_SIZE]; ESP_LOGI(TAG, "Tag %s removed", nfc::format_uid_to(uid_buf, this->discovered_endpoint_[tag_index].tag->get_uid())); // keep the remaining entries in order; selecting_endpoint_ indexes into this list @@ -908,9 +910,11 @@ void PN71xx::process_rf_intf_activated_oid_(nfc::NciMessage &rx) { // an endpoi trigger->process(working_endpoint.tag); } #endif +#ifdef NFC_TAG_LISTENER_COUNT for (auto *listener : this->tag_listeners_) { listener->tag_on(*working_endpoint.tag); } +#endif working_endpoint.trig_called = true; break; } diff --git a/esphome/core/defines.h b/esphome/core/defines.h index bec97fc82ca..3397214c9b6 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -141,6 +141,7 @@ #define MICRONOVA_LISTENER_COUNT 1 #define USE_MICRONOVA_WRITER #define MK2PVROUTER_LISTENER_COUNT 1 +#define NFC_TAG_LISTENER_COUNT 1 #define PN532_BINARY_SENSOR_COUNT 1 #define PN532_ON_TAG_REMOVED_TRIGGER_COUNT 1 #define PN532_ON_TAG_TRIGGER_COUNT 1 diff --git a/tests/components/pn532_spi/common.yaml b/tests/components/pn532_spi/common.yaml index f9149af35fd..7af4beac6ed 100644 --- a/tests/components/pn532_spi/common.yaml +++ b/tests/components/pn532_spi/common.yaml @@ -1,6 +1,15 @@ pn532_spi: id: pn532_nfcc_spi cs_pin: ${cs_pin} + on_tag: + then: + - delay: 10ms + - lambda: |- + ESP_LOGD("test", "%s %zu %d %d", tag.get_tag_type().c_str(), tag.get_uid().size(), tag.has_ndef_message(), + nfc::has_ha_tag_ndef(tag)); + on_tag_removed: + then: + - lambda: ESP_LOGD("test", "%s", tag.get_tag_type().c_str()); binary_sensor: - platform: pn532 diff --git a/tests/components/pn7160_spi/common.yaml b/tests/components/pn7160_spi/common.yaml index 53b37b38f45..18438a2e0da 100644 --- a/tests/components/pn7160_spi/common.yaml +++ b/tests/components/pn7160_spi/common.yaml @@ -24,8 +24,19 @@ pn7160_spi: emulation_message: https://www.home-assistant.io/tag/pulse_ce tag_ttl: 1000ms on_tag: - - logger.log: "Tag" + then: + - delay: 10ms + - lambda: |- + ESP_LOGD("test", "%s %zu %d %s", tag.get_tag_type().c_str(), tag.get_uid().size(), tag.has_ndef_message(), + nfc::get_ha_tag_ndef(tag).c_str()); on_tag_removed: - - logger.log: "Tag removed" + then: + - lambda: ESP_LOGD("test", "%s", tag.get_tag_type().c_str()); on_emulated_tag_scan: - logger.log: "Tag emulated" + +binary_sensor: + - platform: nfc + nfcc_id: nfcc_pn7160_spi + name: Pulse tag + ndef_contains: pulse From 35d41adf8606883080d368633f53b5a56ba102f9 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 30 Sep 2026 07:20:13 +1000 Subject: [PATCH 055/133] [lvgl] Animation updates and debug borders option (#19420) --- esphome/components/lvgl/__init__.py | 3 ++ esphome/components/lvgl/animation.h | 25 +++++++++++++ esphome/components/lvgl/animation.py | 8 +++++ esphome/components/lvgl/defines.py | 28 +++++++++++++++ esphome/components/lvgl/lv_validation.py | 6 ++-- esphome/components/lvgl/lvgl_esphome.cpp | 1 + esphome/components/lvgl/widgets/__init__.py | 19 ++++++++++ esphome/components/lvgl/widgets/lv_list.py | 2 ++ .../lvgl/config/debug_outline.yaml | 36 +++++++++++++++++++ .../lvgl/config/no_debug_outline.yaml | 35 ++++++++++++++++++ tests/component_tests/lvgl/test_animation.py | 18 ++++++++++ .../lvgl/test_debug_outline.py | 33 +++++++++++++++++ .../lvgl/test_multi_conf_validate.py | 22 +++++++++++- .../lvgl/test-debug-outline.esp32-idf.yaml | 34 ++++++++++++++++++ tests/components/lvgl/test.host.yaml | 4 +++ 15 files changed, 271 insertions(+), 3 deletions(-) create mode 100644 tests/component_tests/lvgl/config/debug_outline.yaml create mode 100644 tests/component_tests/lvgl/config/no_debug_outline.yaml create mode 100644 tests/component_tests/lvgl/test_debug_outline.py create mode 100644 tests/components/lvgl/test-debug-outline.esp32-idf.yaml diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index 382e483d0cb..3822664c948 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -238,6 +238,7 @@ def multi_conf_validate(configs: list[dict]): CONF_COLOR_DEPTH, CONF_BYTE_ORDER, df.CONF_TRANSPARENCY_KEY, + df.CONF_DEBUG_OUTLINE, ): if base_config[item] != config[item]: raise cv.Invalid( @@ -393,6 +394,7 @@ async def to_code(configs): df.add_define("LV_FONT_DEFAULT", await lvalid.lv_font.process(default_font)) cg.add(lvgl_static.esphome_lvgl_init()) default_group = get_default_group(config_0) + df.get_options()[df.CONF_DEBUG_OUTLINE] = config_0[df.CONF_DEBUG_OUTLINE] for config in configs: frac = config[CONF_BUFFER_SIZE] @@ -634,6 +636,7 @@ LVGL_TOP_LEVEL_SCHEMA = ( cv.GenerateID(df.CONF_DEFAULT_GROUP): cv.declare_id(lv_group_t), cv.Optional(df.CONF_RESUME_ON_INPUT, default=True): cv.boolean, cv.Optional(df.CONF_PAUSED, default=False): cv.boolean, + cv.Optional(df.CONF_DEBUG_OUTLINE, default=False): cv.boolean, } ) .extend(DISP_BG_SCHEMA) diff --git a/esphome/components/lvgl/animation.h b/esphome/components/lvgl/animation.h index 05b84926eb6..1ef22f9b91d 100644 --- a/esphome/components/lvgl/animation.h +++ b/esphome/components/lvgl/animation.h @@ -101,6 +101,31 @@ class LvAnimationTimingEaseInOut : public LvAnimationTiming { float slope_; }; +class LvAnimationTimingEaseIn : public LvAnimationTiming { + public: + LvAnimationTimingEaseIn(float slope) : slope_(slope) {} + float map_progress(float value) override { + const float sqr = value * value; + return this->slope_ * sqr + (1.0f - this->slope_) * value; + } + + protected: + float slope_; +}; + +class LvAnimationTimingEaseOut : public LvAnimationTiming { + public: + LvAnimationTimingEaseOut(float slope) : slope_(slope) {} + float map_progress(float value) override { + const float inv = 1.0f - value; + const float sqr = 1.0f - inv * inv; + return this->slope_ * sqr + (1.0f - this->slope_) * value; + } + + protected: + float slope_; +}; + template class LvAnimation : public Component { public: LvAnimation(void (*update_callback)(const lv_coord_t *data), std::vector> from, diff --git a/esphome/components/lvgl/animation.py b/esphome/components/lvgl/animation.py index 95d45de5ea2..bab413f04d7 100644 --- a/esphome/components/lvgl/animation.py +++ b/esphome/components/lvgl/animation.py @@ -76,6 +76,14 @@ TIMING_SCHEMA = cv.maybe_simple_value( "ease_in_out", {cv.Optional(CONF_WEIGHT, default=1.0): cv.zero_to_one_float}, ), + timing_class( + "ease_in", + {cv.Optional(CONF_WEIGHT, default=1.0): cv.zero_to_one_float}, + ), + timing_class( + "ease_out", + {cv.Optional(CONF_WEIGHT, default=1.0): cv.zero_to_one_float}, + ), timing_class( "gravity", { diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index f601b4b57a0..fbb621800b5 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -35,6 +35,24 @@ KEY_WIDGET_MAP = "widget_map" KEY_WIDGETS_COMPLETED = "widgets_completed" KEY_OPTIONS = "options" KEY_WARNINGS = "warnings" +KEY_DEBUG_OUTLINE_COUNT = "debug_outline_count" + +# Colours for the debug outline, in (red, green, blue) order. They are picked to stay +# distinct from each other and to show up on both light and dark backgrounds. +DEBUG_OUTLINE_COLORS = ( + (255, 0, 0), + (0, 160, 0), + (0, 0, 255), + (255, 140, 0), + (200, 0, 200), + (0, 190, 190), + (160, 100, 0), + (255, 0, 120), + (110, 110, 110), + (140, 200, 0), + (0, 110, 255), + (130, 0, 255), +) # Initial set of LVGL features that are always enabled. _INITIAL_LV_USES = frozenset( @@ -102,6 +120,15 @@ def get_options() -> dict[str, Any]: return _get_data(KEY_OPTIONS, {}) +def next_debug_outline_color() -> tuple[int, int, int]: + """Return the next debug outline colour, cycling through the palette.""" + # A one-element list so that the count can be updated in place. + count = _get_data(KEY_DEBUG_OUTLINE_COUNT, [0]) + color = DEBUG_OUTLINE_COLORS[count[0] % len(DEBUG_OUTLINE_COLORS)] + count[0] += 1 + return color + + def get_defines() -> dict[str, str]: return _get_data(KEY_LV_DEFINES, {}) @@ -684,6 +711,7 @@ CONF_COLOR_END = "color_end" CONF_COLOR_START = "color_start" CONF_CONTAINER = "container" CONF_CONTROL = "control" +CONF_DEBUG_OUTLINE = "debug_outline" CONF_DEFAULT_FONT = "default_font" CONF_DEFAULT_GROUP = "default_group" CONF_DIR = "dir" diff --git a/esphome/components/lvgl/lv_validation.py b/esphome/components/lvgl/lv_validation.py index b3285afa1af..961fa4079d7 100644 --- a/esphome/components/lvgl/lv_validation.py +++ b/esphome/components/lvgl/lv_validation.py @@ -157,10 +157,12 @@ padding = LValidator(padding_validator, int32, retmapper=literal) def scale_validator(value): - return cv.float_range(0.1, 10.0)(value) + return cv.float_range(0.0, 10.0)(value) -scale = LValidator(scale_validator, uint32, retmapper=lambda x: int(x * 256)) +scale = LValidator( + scale_validator, uint32, retmapper=lambda x: int(x * 256), animatable=True +) def angle(value): diff --git a/esphome/components/lvgl/lvgl_esphome.cpp b/esphome/components/lvgl/lvgl_esphome.cpp index 2c988473a93..90a770ac3b0 100644 --- a/esphome/components/lvgl/lvgl_esphome.cpp +++ b/esphome/components/lvgl/lvgl_esphome.cpp @@ -994,6 +994,7 @@ lv_point_t LvglComponent::get_touch_relative_to_obj(lv_obj_t *obj) { static void lv_container_constructor(const lv_obj_class_t *class_p, lv_obj_t *obj) { LV_TRACE_OBJ_CREATE("begin"); + lv_obj_remove_flag(obj, LV_OBJ_FLAG_CLICKABLE); LV_UNUSED(class_p); } diff --git a/esphome/components/lvgl/widgets/__init__.py b/esphome/components/lvgl/widgets/__init__.py index a524fe761f9..da513e05a85 100644 --- a/esphome/components/lvgl/widgets/__init__.py +++ b/esphome/components/lvgl/widgets/__init__.py @@ -21,6 +21,7 @@ from esphome.schema_extractors import EnableSchemaExtraction from esphome.types import Expression from ..defines import ( + CONF_DEBUG_OUTLINE, CONF_FLEX_ALIGN_CROSS, CONF_FLEX_ALIGN_MAIN, CONF_FLEX_ALIGN_TRACK, @@ -42,12 +43,14 @@ from ..defines import ( STATES, LValidator, add_lv_use, + get_options, get_styles_used, get_theme_widget_map, get_widget_map, get_widgets_completed, join_enums, literal, + next_debug_outline_color, ) from ..lv_validation import lv_int from ..lvcode import ( @@ -191,6 +194,7 @@ class WidgetType: w = Widget.create(wid, var, self, config) apply_theme_styles(w) await set_obj_properties(w, config) + apply_debug_outline(w) await add_widgets(w, config) await self.to_code(w, config) return w @@ -270,6 +274,21 @@ def apply_theme_styles(w: "Widget") -> None: w.add_style(style, lv_state) +def apply_debug_outline(w: "Widget") -> None: + """ + When `debug_outline` is set, outline this widget in the next palette colour. + An outline is drawn outside the widget's own box, so it doesn't take up layout + space and doesn't touch the widget's own `border_*` style, unlike a border. + """ + if not get_options().get(CONF_DEBUG_OUTLINE): + return + r, g, b = next_debug_outline_color() + w.set_style("outline_width", 1) + w.set_style("outline_pad", 0) + w.set_style("outline_color", f"lv_color_make({r}, {g}, {b})") + w.set_style("outline_opa", "LV_OPA_COVER") + + class Widget: """ Represents a Widget. diff --git a/esphome/components/lvgl/widgets/lv_list.py b/esphome/components/lvgl/widgets/lv_list.py index 7711e8bfe4f..5d94c2565a2 100644 --- a/esphome/components/lvgl/widgets/lv_list.py +++ b/esphome/components/lvgl/widgets/lv_list.py @@ -57,6 +57,7 @@ from ..types import LV_EVENT, LvType, ObjUpdateAction, lv_obj_t from . import ( Widget, WidgetType, + apply_debug_outline, apply_theme_styles, collect_parts, get_widgets, @@ -445,6 +446,7 @@ async def _finish_dynamic_widget( await w.type.on_create(w.obj, w_conf) apply_theme_styles(w) await set_obj_properties(w, w_conf) + apply_debug_outline(w) await w.type.to_code(w, w_conf) await _wire_dynamic_triggers(w, w_conf) for child in w_conf.get(CONF_WIDGETS, ()): diff --git a/tests/component_tests/lvgl/config/debug_outline.yaml b/tests/component_tests/lvgl/config/debug_outline.yaml new file mode 100644 index 00000000000..eb8a488c139 --- /dev/null +++ b/tests/component_tests/lvgl/config/debug_outline.yaml @@ -0,0 +1,36 @@ +esphome: + name: test-debug-outline + +esp32: + board: lolin_c3_mini + +spi: + mosi_pin: + number: GPIO2 + ignore_strapping_warning: true + clk_pin: GPIO1 + +display: + - platform: mipi_spi + data_rate: 20MHz + model: st7735 + cs_pin: + number: GPIO8 + ignore_strapping_warning: true + dc_pin: + number: GPIO3 + +lvgl: + debug_outline: true + widgets: + - obj: + id: root_obj + widgets: + - label: + id: title_label + text: Title + - button: + id: ok_button + widgets: + - label: + text: OK diff --git a/tests/component_tests/lvgl/config/no_debug_outline.yaml b/tests/component_tests/lvgl/config/no_debug_outline.yaml new file mode 100644 index 00000000000..2ea646bcb23 --- /dev/null +++ b/tests/component_tests/lvgl/config/no_debug_outline.yaml @@ -0,0 +1,35 @@ +esphome: + name: test-debug-outline + +esp32: + board: lolin_c3_mini + +spi: + mosi_pin: + number: GPIO2 + ignore_strapping_warning: true + clk_pin: GPIO1 + +display: + - platform: mipi_spi + data_rate: 20MHz + model: st7735 + cs_pin: + number: GPIO8 + ignore_strapping_warning: true + dc_pin: + number: GPIO3 + +lvgl: + widgets: + - obj: + id: root_obj + widgets: + - label: + id: title_label + text: Title + - button: + id: ok_button + widgets: + - label: + text: OK diff --git a/tests/component_tests/lvgl/test_animation.py b/tests/component_tests/lvgl/test_animation.py index ce9a162d993..a150f66a40e 100644 --- a/tests/component_tests/lvgl/test_animation.py +++ b/tests/component_tests/lvgl/test_animation.py @@ -191,6 +191,24 @@ class TestTimingSchema: result = TIMING_SCHEMA({"type": "ease_in_out", "weight": 0.5}) assert result["weight"] == pytest.approx(0.5) + def test_ease_in_default_weight(self) -> None: + result = TIMING_SCHEMA("ease_in") + assert result["type"] == "ease_in" + assert result["weight"] == pytest.approx(1.0) + + def test_ease_in_custom_weight(self) -> None: + result = TIMING_SCHEMA({"type": "ease_in", "weight": 0.5}) + assert result["weight"] == pytest.approx(0.5) + + def test_ease_out_default_weight(self) -> None: + result = TIMING_SCHEMA("ease_out") + assert result["type"] == "ease_out" + assert result["weight"] == pytest.approx(1.0) + + def test_ease_out_custom_weight(self) -> None: + result = TIMING_SCHEMA({"type": "ease_out", "weight": 0.5}) + assert result["weight"] == pytest.approx(0.5) + def test_gravity_defaults(self) -> None: result = TIMING_SCHEMA("gravity") assert result["type"] == "gravity" diff --git a/tests/component_tests/lvgl/test_debug_outline.py b/tests/component_tests/lvgl/test_debug_outline.py new file mode 100644 index 00000000000..78eb3c077f7 --- /dev/null +++ b/tests/component_tests/lvgl/test_debug_outline.py @@ -0,0 +1,33 @@ +"""Tests for the LVGL ``debug_outline`` option code generation.""" + +from __future__ import annotations + +import re + +_OUTLINE_COLOR_RE = re.compile( + r"lv_obj_set_style_outline_color\(.+?, (lv_color_make\(.+?\))," +) +_OUTLINE_WIDTH_RE = re.compile(r"lv_obj_set_style_outline_width\(") +_OUTLINE_PAD_RE = re.compile(r"lv_obj_set_style_outline_pad\(.+?, 0,") + + +class TestDebugOutlineCodeGeneration: + """Verify that ``debug_outline`` outlines every widget in a distinct colour.""" + + def test_debug_outline_styles_every_widget( + self, generate_main, component_config_path + ): + """Four widgets are declared, so four outlines with different colours appear.""" + main_cpp = generate_main(component_config_path("debug_outline.yaml")) + assert len(_OUTLINE_WIDTH_RE.findall(main_cpp)) == 4 + assert len(_OUTLINE_PAD_RE.findall(main_cpp)) == 4 + colors = _OUTLINE_COLOR_RE.findall(main_cpp) + assert len(colors) == 4 + assert len(set(colors)) == 4 + + def test_debug_outline_default_emits_nothing( + self, generate_main, component_config_path + ): + """Without ``debug_outline`` no outline styles are generated.""" + main_cpp = generate_main(component_config_path("no_debug_outline.yaml")) + assert "lv_obj_set_style_outline_" not in main_cpp diff --git a/tests/component_tests/lvgl/test_multi_conf_validate.py b/tests/component_tests/lvgl/test_multi_conf_validate.py index b63b7618e7d..c44b40317e8 100644 --- a/tests/component_tests/lvgl/test_multi_conf_validate.py +++ b/tests/component_tests/lvgl/test_multi_conf_validate.py @@ -9,13 +9,16 @@ from esphome.components.lvgl.schemas import theme_schema from esphome.config_validation import Invalid -def _config(displays: list[str], theme: dict | None = None) -> dict: +def _config( + displays: list[str], theme: dict | None = None, debug_outline: bool = False +) -> dict: config = { df.CONF_DISPLAYS: displays, "log_level": "WARN", "color_depth": 16, "byte_order": "big_endian", df.CONF_TRANSPARENCY_KEY: 0x000400, + df.CONF_DEBUG_OUTLINE: debug_outline, } if theme is not None: config[df.CONF_THEME] = theme @@ -53,3 +56,20 @@ class TestThemeOnMultipleInstances: def test_passes_when_no_instance_has_theme(self) -> None: configs = [_config(["disp_a"]), _config(["disp_b"])] multi_conf_validate(configs) + + +class TestDebugOutlineOnMultipleInstances: + def test_raises_when_instances_differ(self) -> None: + configs = [ + _config(["disp_a"], debug_outline=True), + _config(["disp_b"], debug_outline=False), + ] + with pytest.raises(Invalid, match="'debug_outline' must be the same"): + multi_conf_validate(configs) + + def test_passes_when_instances_match(self) -> None: + configs = [ + _config(["disp_a"], debug_outline=True), + _config(["disp_b"], debug_outline=True), + ] + multi_conf_validate(configs) diff --git a/tests/components/lvgl/test-debug-outline.esp32-idf.yaml b/tests/components/lvgl/test-debug-outline.esp32-idf.yaml new file mode 100644 index 00000000000..73418962ffd --- /dev/null +++ b/tests/components/lvgl/test-debug-outline.esp32-idf.yaml @@ -0,0 +1,34 @@ +# Kept separate from the main lvgl-package.yaml so that enabling debug_outline +# here doesn't add its extra style calls to the memory-impact baseline that +# every other lvgl test change is measured against. +packages: + spi: !include ../../test_build_components/common/spi/esp32-idf.yaml + +display: + - platform: mipi_spi + spi_id: spi_bus + model: st7789v + id: debug_outline_display + dimensions: + width: 240 + height: 320 + data_rate: 80MHz + cs_pin: GPIO22 + dc_pin: GPIO21 + auto_clear_enabled: false + update_interval: never + +lvgl: + debug_outline: true + displays: + - debug_outline_display + widgets: + - obj: + id: debug_outline_root + widgets: + - label: + text: Title + - button: + widgets: + - label: + text: OK diff --git a/tests/components/lvgl/test.host.yaml b/tests/components/lvgl/test.host.yaml index 3fa54fa3d62..22d051da8eb 100644 --- a/tests/components/lvgl/test.host.yaml +++ b/tests/components/lvgl/test.host.yaml @@ -40,6 +40,10 @@ lvgl: - round_trip - type: ease_in_out weight: 0.5 + - type: ease_in + weight: 0.3 + - type: ease_out + weight: 0.7 on_start: - logger.log: anim started on_stop: From 99c7f9e625e8ab2cff17e6ef8c5754506fdc8aab Mon Sep 17 00:00:00 2001 From: guillempages Date: Tue, 29 Sep 2026 23:28:28 +0200 Subject: [PATCH 056/133] [runtime_image] Improve format logs (#19903) Co-authored-by: J. Nick Koston --- .../components/online_image/online_image.cpp | 4 ++-- .../components/runtime_image/image_format.cpp | 7 +++++++ .../components/runtime_image/image_format.h | 3 +++ .../runtime_image/runtime_image.cpp | 9 +++++---- .../runtime_image/test_format_names.cpp | 19 +++++++++++++++++++ .../runtime_image/test_mime_types.cpp | 2 +- 6 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 tests/components/runtime_image/test_format_names.cpp diff --git a/esphome/components/online_image/online_image.cpp b/esphome/components/online_image/online_image.cpp index 3f2382accd1..a21849d618c 100644 --- a/esphome/components/online_image/online_image.cpp +++ b/esphome/components/online_image/online_image.cpp @@ -122,11 +122,11 @@ void OnlineImage::update() { return; } } - ESP_LOGD(TAG, "Using image format: %d", format); + ESP_LOGD(TAG, "Using image format: %s", LOG_STR_ARG(runtime_image::get_format_name(format))); // Initialize decoder with the known format if (!this->begin_decode(total_size, format)) { - ESP_LOGE(TAG, "Failed to initialize decoder for format %d", format); + ESP_LOGE(TAG, "Failed to initialize decoder for format %s", LOG_STR_ARG(runtime_image::get_format_name(format))); this->end_connection_(); this->download_error_callback_.call(); return; diff --git a/esphome/components/runtime_image/image_format.cpp b/esphome/components/runtime_image/image_format.cpp index 9db84904158..368c103d15f 100644 --- a/esphome/components/runtime_image/image_format.cpp +++ b/esphome/components/runtime_image/image_format.cpp @@ -1,5 +1,6 @@ #include "esphome/core/defines.h" #include "esphome/core/helpers.h" +#include "esphome/core/progmem.h" #include "image_format.h" namespace esphome::runtime_image { @@ -26,6 +27,8 @@ static constexpr MimeLookup MIME_LOOKUP_TABLE[] = { #endif }; +PROGMEM_STRING_TABLE(ImageFormatStrings, "AUTO", "BMP", "JPEG", "PNG", "QOI", "UNKNOWN"); + const char *get_mime_type_for_format(ImageFormat format) { for (const auto &entry : MIME_LOOKUP_TABLE) { if (entry.format == format) { @@ -44,4 +47,8 @@ std::optional get_format_for_mime_type(const char *mime_type) { return std::nullopt; } +const LogString *get_format_name(ImageFormat format) { + return ImageFormatStrings::get_log_str(static_cast(format), ImageFormatStrings::LAST_INDEX); +} + } // namespace esphome::runtime_image diff --git a/esphome/components/runtime_image/image_format.h b/esphome/components/runtime_image/image_format.h index 72bd81a06ea..60dbee39bfa 100644 --- a/esphome/components/runtime_image/image_format.h +++ b/esphome/components/runtime_image/image_format.h @@ -1,6 +1,7 @@ #pragma once #include +#include "esphome/core/log.h" namespace esphome::runtime_image { @@ -25,5 +26,7 @@ enum ImageFormat { const char *get_mime_type_for_format(ImageFormat format); /// Case-insensitive substring match of known media types; nullopt if none found std::optional get_format_for_mime_type(const char *mime_type); +/// Human-readable name for a format; "UNKNOWN" if not recognized +const LogString *get_format_name(ImageFormat format); } // namespace esphome::runtime_image diff --git a/esphome/components/runtime_image/runtime_image.cpp b/esphome/components/runtime_image/runtime_image.cpp index ef92d0d7073..5c14d398071 100644 --- a/esphome/components/runtime_image/runtime_image.cpp +++ b/esphome/components/runtime_image/runtime_image.cpp @@ -186,14 +186,15 @@ bool RuntimeImage::begin_decode(size_t expected_size, ImageFormat format) { // An idle decoder for a different format cannot be reused if (this->decoder_ != nullptr && this->decoder_->get_format() != format) { - ESP_LOGD(TAG, "Decoder format mismatch: current: %d, new: %d", this->decoder_->get_format(), format); + ESP_LOGD(TAG, "Decoder format mismatch: current: %s, new: %s", + LOG_STR_ARG(get_format_name(this->decoder_->get_format())), LOG_STR_ARG(get_format_name(format))); this->decoder_ = nullptr; } if (!this->decoder_) { this->decoder_ = this->create_decoder_(format); if (!this->decoder_) { - ESP_LOGE(TAG, "Failed to create decoder for format %d", format); + ESP_LOGE(TAG, "Failed to create decoder for format %s", LOG_STR_ARG(get_format_name(format))); return false; } } @@ -357,7 +358,7 @@ size_t RuntimeImage::get_buffer_size(int width, int height) const { int RuntimeImage::get_position_(int x, int y) const { return (x + y * this->buffer_width_) * this->get_bpp() / 8; } std::unique_ptr RuntimeImage::create_decoder_(ImageFormat format) { - ESP_LOGV(TAG, "Creating decoder for format %d", format); + ESP_LOGV(TAG, "Creating decoder for format %s", LOG_STR_ARG(get_format_name(format))); switch (format) { #ifdef USE_RUNTIME_IMAGE_BMP case BMP: @@ -379,7 +380,7 @@ std::unique_ptr RuntimeImage::create_decoder_(ImageFormat format) ESP_LOGE(TAG, "Image format could not be determined; set `format:` explicitly in the configuration"); return nullptr; default: - ESP_LOGE(TAG, "Unsupported image format: %d", format); + ESP_LOGE(TAG, "Unsupported image format: %s", LOG_STR_ARG(get_format_name(format))); return nullptr; } } diff --git a/tests/components/runtime_image/test_format_names.cpp b/tests/components/runtime_image/test_format_names.cpp new file mode 100644 index 00000000000..16299a60896 --- /dev/null +++ b/tests/components/runtime_image/test_format_names.cpp @@ -0,0 +1,19 @@ +#include + +#include "esphome/components/runtime_image/image_format.h" + +namespace esphome::runtime_image::testing { + +TEST(RuntimeImageFormatNames, FormatNames) { + EXPECT_STREQ(LOG_STR_ARG(get_format_name(AUTO)), "AUTO"); + EXPECT_STREQ(LOG_STR_ARG(get_format_name(BMP)), "BMP"); + EXPECT_STREQ(LOG_STR_ARG(get_format_name(JPEG)), "JPEG"); + EXPECT_STREQ(LOG_STR_ARG(get_format_name(PNG)), "PNG"); + EXPECT_STREQ(LOG_STR_ARG(get_format_name(QOI)), "QOI"); +} + +TEST(RuntimeImageFormatNames, OutOfRangeName) { + EXPECT_STREQ(LOG_STR_ARG(get_format_name(static_cast(254))), "UNKNOWN"); +} + +} // namespace esphome::runtime_image::testing diff --git a/tests/components/runtime_image/test_mime_types.cpp b/tests/components/runtime_image/test_mime_types.cpp index 22b825cff08..5f702d36aff 100644 --- a/tests/components/runtime_image/test_mime_types.cpp +++ b/tests/components/runtime_image/test_mime_types.cpp @@ -2,7 +2,7 @@ #include -#include "esphome/components/runtime_image/runtime_image.h" +#include "esphome/components/runtime_image/image_format.h" namespace esphome::runtime_image::testing { From 5502f620d4c757591baa0c2bb48b4c51b4098cc8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 00:10:40 +0200 Subject: [PATCH 057/133] [web_server_idf] Send events straight from the caller's buffers instead of a std::string (#19376) --- esphome/components/json/json_util.cpp | 32 +- esphome/components/json/json_util.h | 4 + esphome/components/web_server/web_server.cpp | 454 +++++++++--------- esphome/components/web_server/web_server.h | 146 +++--- .../components/web_server_idf/sse_chunk.cpp | 66 +++ esphome/components/web_server_idf/sse_chunk.h | 40 ++ .../web_server_idf/web_server_idf.cpp | 423 +++++++++------- .../web_server_idf/web_server_idf.h | 57 ++- .../web_server_idf/test_sse_chunk.cpp | 131 +++++ 9 files changed, 850 insertions(+), 503 deletions(-) create mode 100644 esphome/components/web_server_idf/sse_chunk.cpp create mode 100644 esphome/components/web_server_idf/sse_chunk.h create mode 100644 tests/components/web_server_idf/test_sse_chunk.cpp diff --git a/esphome/components/json/json_util.cpp b/esphome/components/json/json_util.cpp index 1b1eefe59b3..193390e4782 100644 --- a/esphome/components/json/json_util.cpp +++ b/esphome/components/json/json_util.cpp @@ -1,4 +1,6 @@ #include "json_util.h" + +#include #include "esphome/core/log.h" // ArduinoJson::Allocator is included via ArduinoJson.h in json_util.h @@ -68,6 +70,22 @@ JsonDocument parse_json(const uint8_t *data, size_t len) { JsonBuilder::JsonBuilder() = default; +size_t JsonBuilder::serialize_to(char *buf, size_t cap) { + if (doc_.overflowed()) { + ESP_LOGE(TAG, "JSON document overflow"); + // Same contract as serializeJson; written by hand so no "{}" literal lives in RAM on ESP8266 + size_t n = 0; + if (n < cap) + buf[n++] = '{'; + if (n < cap) + buf[n++] = '}'; + if (n < cap) + buf[n] = '\0'; + return n; + } + return serializeJson(doc_, buf, cap); +} + SerializationBuffer<> JsonBuilder::serialize() { // =========================================================================================== // CRITICAL: NRVO (Named Return Value Optimization) - DO NOT REFACTOR WITHOUT UNDERSTANDING @@ -109,17 +127,7 @@ SerializationBuffer<> JsonBuilder::serialize() { constexpr size_t buf_size = SerializationBuffer<>::BUFFER_SIZE; SerializationBuffer<> result(buf_size - 1); // Max content size (reserve 1 for null) - if (doc_.overflowed()) { - ESP_LOGE(TAG, "JSON document overflow"); - auto *buf = result.data_writable_(); - buf[0] = '{'; - buf[1] = '}'; - buf[2] = '\0'; - result.set_size_(2); - return result; - } - - size_t size = serializeJson(doc_, result.data_writable_(), buf_size); + size_t size = this->serialize_to(result.data_writable_(), buf_size); if (size < buf_size) { // Fits in stack buffer - update size to actual length result.set_size_(size); @@ -134,7 +142,7 @@ SerializationBuffer<> JsonBuilder::serialize() { size_t heap_size = buf_size * 2; while (heap_size <= max_heap_size) { result.reallocate_heap_(heap_size - 1); - size = serializeJson(doc_, result.data_writable_(), heap_size); + size = this->serialize_to(result.data_writable_(), heap_size); if (size < heap_size) { result.set_size_(size); return result; diff --git a/esphome/components/json/json_util.h b/esphome/components/json/json_util.h index 130e1503321..5823e993085 100644 --- a/esphome/components/json/json_util.h +++ b/esphome/components/json/json_util.h @@ -179,6 +179,10 @@ class JsonBuilder { return root_; } + /// Serialize into a caller owned buffer. Returns the length; cap or more means it did not fit + /// and buf holds a truncated, unterminated copy. An overflowed document serializes as "{}". + size_t serialize_to(char *buf, size_t cap); + /// Serialize the JSON document to a SerializationBuffer (stack-first allocation) /// Uses 512-byte stack buffer by default, falls back to heap for larger JSON SerializationBuffer<> serialize(); diff --git a/esphome/components/web_server/web_server.cpp b/esphome/components/web_server/web_server.cpp index fef484cd669..74641738ec3 100644 --- a/esphome/components/web_server/web_server.cpp +++ b/esphome/components/web_server/web_server.cpp @@ -1,4 +1,6 @@ #include "web_server.h" + +#include #ifdef USE_WEBSERVER #include "esphome/components/json/json_util.h" #include "esphome/core/progmem.h" @@ -58,6 +60,12 @@ static const char *const TAG = "web_server"; // View a state LogString as a ProgmemStr so ArduinoJson serializes it PROGMEM-aware on ESP8266. [[maybe_unused]] static ProgmemStr json_state_str(const LogString *s) { return reinterpret_cast(s); } +// Out of line: every GET handler ends with this +[[maybe_unused]] static void send_json(AsyncWebServerRequest *request, json::JsonBuilder &builder) { + auto data = builder.serialize(); + request->send(200, ESPHOME_F("application/json"), data.c_str()); +} + // Parse URL and return match info // URL formats (disambiguated by HTTP method for 3-segment case): // GET /{domain}/{entity_name} - main device state @@ -193,13 +201,15 @@ DeferredUpdateEventSource::deq_push_back_with_dedup_(void *source, message_gener void DeferredUpdateEventSource::process_deferred_queue_() { while (!deferred_queue_.empty()) { DeferredEvent &de = deferred_queue_.front(); - auto message = de.message_generator_(web_server_, de.source_); + json::JsonBuilder builder; + de.message_generator_(web_server_, de.source_, builder); + auto message = builder.serialize(); if (this->send(message.c_str(), "state") != DISCARDED) { // O(n) but memory efficiency is more important than speed here which is why std::vector was chosen deferred_queue_.erase(deferred_queue_.begin()); this->consecutive_send_failures_ = 0; // Reset failure count on successful send } else { - // NOTE: Similar logic exists in web_server_idf/web_server_idf.cpp in AsyncEventSourceResponse::process_buffer_(). + // NOTE: Similar logic exists in web_server_idf/web_server_idf.cpp in AsyncEventSourceResponse::drain_tail_(). // The close mechanisms are platform-specific (this path calls close() directly; the IDF path is time-based and // closes through HTTPD to preserve session ownership), but both drop a client after roughly 20 seconds without // send progress. Keep that stall policy in sync when changing either side. @@ -250,7 +260,9 @@ void DeferredUpdateEventSource::deferrable_send_state(void *source, const char * // deferred queue still not empty which means downstream event queue full, no point trying to send first deq_push_back_with_dedup_(source, message_generator); } else { - auto message = message_generator(web_server_, source); + json::JsonBuilder builder; + message_generator(web_server_, source, builder); + auto message = builder.serialize(); if (this->send(message.c_str(), "state") == DISCARDED) { deq_push_back_with_dedup_(source, message_generator); } else { @@ -405,10 +417,11 @@ void WebServer::loop() { } #ifdef USE_LOGGER + void WebServer::on_log(uint8_t level, const char *tag, const char *message, size_t message_len) { (void) level; (void) tag; - this->events_.try_send_nodefer(message, message_len, "log", millis()); + this->events_.try_send_nodefer(message, std::min(message_len, LOG_EVENT_MAX_LEN), "log", millis()); } #endif @@ -634,21 +647,21 @@ void WebServer::handle_sensor_request(AsyncWebServerRequest *request, const UrlM // Note: request->method() is always HTTP_GET here (canHandle ensures this) if (entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->sensor_json_(obj, obj->state, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->sensor_json_(obj, obj->state, detail, builder); + send_json(request, builder); return; } } request->send(404); } -json::SerializationBuffer<> WebServer::sensor_state_json_generator(WebServer *web_server, void *source) { - return web_server->sensor_json_((sensor::Sensor *) (source), ((sensor::Sensor *) (source))->state, DETAIL_STATE); +void WebServer::sensor_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->sensor_json_((sensor::Sensor *) (source), ((sensor::Sensor *) (source))->state, DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::sensor_all_json_generator(WebServer *web_server, void *source) { - return web_server->sensor_json_((sensor::Sensor *) (source), ((sensor::Sensor *) (source))->state, DETAIL_ALL); +void WebServer::sensor_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->sensor_json_((sensor::Sensor *) (source), ((sensor::Sensor *) (source))->state, DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::sensor_json_(sensor::Sensor *obj, float value, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::sensor_json_(sensor::Sensor *obj, float value, JsonDetail start_config, json::JsonBuilder &builder) { JsonObject root = builder.root(); const auto uom_ref = obj->get_unit_of_measurement_ref(); @@ -662,8 +675,6 @@ json::SerializationBuffer<> WebServer::sensor_json_(sensor::Sensor *obj, float v if (!uom_ref.empty()) root[ESPHOME_F("uom")] = uom_ref.c_str(); } - - return builder.serialize(); } #endif @@ -681,32 +692,30 @@ void WebServer::handle_text_sensor_request(AsyncWebServerRequest *request, const // Note: request->method() is always HTTP_GET here (canHandle ensures this) if (entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->text_sensor_json_(obj, obj->state, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->text_sensor_json_(obj, obj->state, detail, builder); + send_json(request, builder); return; } } request->send(404); } -json::SerializationBuffer<> WebServer::text_sensor_state_json_generator(WebServer *web_server, void *source) { - return web_server->text_sensor_json_((text_sensor::TextSensor *) (source), - ((text_sensor::TextSensor *) (source))->state, DETAIL_STATE); +void WebServer::text_sensor_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->text_sensor_json_((text_sensor::TextSensor *) (source), ((text_sensor::TextSensor *) (source))->state, + DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::text_sensor_all_json_generator(WebServer *web_server, void *source) { - return web_server->text_sensor_json_((text_sensor::TextSensor *) (source), - ((text_sensor::TextSensor *) (source))->state, DETAIL_ALL); +void WebServer::text_sensor_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->text_sensor_json_((text_sensor::TextSensor *) (source), ((text_sensor::TextSensor *) (source))->state, + DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::text_sensor_json_(text_sensor::TextSensor *obj, const std::string &value, - JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::text_sensor_json_(text_sensor::TextSensor *obj, const std::string &value, JsonDetail start_config, + json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_icon_state_value(root, obj, "text_sensor", value.c_str(), value.c_str(), start_config); if (start_config == DETAIL_ALL) { this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -742,8 +751,9 @@ void WebServer::handle_switch_request(AsyncWebServerRequest *request, const UrlM if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->switch_json_(obj, obj->state, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->switch_json_(obj, obj->state, detail, builder); + send_json(request, builder); return; } @@ -767,14 +777,13 @@ void WebServer::handle_switch_request(AsyncWebServerRequest *request, const UrlM } request->send(404); } -json::SerializationBuffer<> WebServer::switch_state_json_generator(WebServer *web_server, void *source) { - return web_server->switch_json_((switch_::Switch *) (source), ((switch_::Switch *) (source))->state, DETAIL_STATE); +void WebServer::switch_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->switch_json_((switch_::Switch *) (source), ((switch_::Switch *) (source))->state, DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::switch_all_json_generator(WebServer *web_server, void *source) { - return web_server->switch_json_((switch_::Switch *) (source), ((switch_::Switch *) (source))->state, DETAIL_ALL); +void WebServer::switch_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->switch_json_((switch_::Switch *) (source), ((switch_::Switch *) (source))->state, DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::switch_json_(switch_::Switch *obj, bool value, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::switch_json_(switch_::Switch *obj, bool value, JsonDetail start_config, json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_icon_state_value(root, obj, "switch", value ? "ON" : "OFF", value, start_config); @@ -782,8 +791,6 @@ json::SerializationBuffer<> WebServer::switch_json_(switch_::Switch *obj, bool v root[ESPHOME_F("assumed_state")] = obj->assumed_state(); this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -795,8 +802,9 @@ void WebServer::handle_button_request(AsyncWebServerRequest *request, const UrlM continue; if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->button_json_(obj, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->button_json_(obj, detail, builder); + send_json(request, builder); } else if (match.method_equals(ESPHOME_F("press"))) { DEFER_ACTION(obj, obj->press()); request->send(200); @@ -808,19 +816,16 @@ void WebServer::handle_button_request(AsyncWebServerRequest *request, const UrlM } request->send(404); } -json::SerializationBuffer<> WebServer::button_all_json_generator(WebServer *web_server, void *source) { - return web_server->button_json_((button::Button *) (source), DETAIL_ALL); +void WebServer::button_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->button_json_((button::Button *) (source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::button_json_(button::Button *obj, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::button_json_(button::Button *obj, JsonDetail start_config, json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_id(root, obj, "button", start_config); if (start_config == DETAIL_ALL) { this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -838,32 +843,30 @@ void WebServer::handle_binary_sensor_request(AsyncWebServerRequest *request, con // Note: request->method() is always HTTP_GET here (canHandle ensures this) if (entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->binary_sensor_json_(obj, obj->state, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->binary_sensor_json_(obj, obj->state, detail, builder); + send_json(request, builder); return; } } request->send(404); } -json::SerializationBuffer<> WebServer::binary_sensor_state_json_generator(WebServer *web_server, void *source) { - return web_server->binary_sensor_json_((binary_sensor::BinarySensor *) (source), - ((binary_sensor::BinarySensor *) (source))->state, DETAIL_STATE); +void WebServer::binary_sensor_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->binary_sensor_json_((binary_sensor::BinarySensor *) (source), + ((binary_sensor::BinarySensor *) (source))->state, DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::binary_sensor_all_json_generator(WebServer *web_server, void *source) { - return web_server->binary_sensor_json_((binary_sensor::BinarySensor *) (source), - ((binary_sensor::BinarySensor *) (source))->state, DETAIL_ALL); +void WebServer::binary_sensor_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->binary_sensor_json_((binary_sensor::BinarySensor *) (source), + ((binary_sensor::BinarySensor *) (source))->state, DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::binary_sensor_json_(binary_sensor::BinarySensor *obj, bool value, - JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::binary_sensor_json_(binary_sensor::BinarySensor *obj, bool value, JsonDetail start_config, + json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_icon_state_value(root, obj, "binary_sensor", value ? "ON" : "OFF", value, start_config); if (start_config == DETAIL_ALL) { this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -881,8 +884,9 @@ void WebServer::handle_fan_request(AsyncWebServerRequest *request, const UrlMatc if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->fan_json_(obj, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->fan_json_(obj, detail, builder); + send_json(request, builder); } else if (match.method_equals(ESPHOME_F("toggle"))) { DEFER_ACTION(obj, obj->toggle().perform()); request->send(200); @@ -922,14 +926,13 @@ void WebServer::handle_fan_request(AsyncWebServerRequest *request, const UrlMatc } request->send(404); } -json::SerializationBuffer<> WebServer::fan_state_json_generator(WebServer *web_server, void *source) { - return web_server->fan_json_((fan::Fan *) (source), DETAIL_STATE); +void WebServer::fan_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->fan_json_((fan::Fan *) (source), DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::fan_all_json_generator(WebServer *web_server, void *source) { - return web_server->fan_json_((fan::Fan *) (source), DETAIL_ALL); +void WebServer::fan_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->fan_json_((fan::Fan *) (source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::fan_json_(fan::Fan *obj, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::fan_json_(fan::Fan *obj, JsonDetail start_config, json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_icon_state_value(root, obj, "fan", obj->state ? "ON" : "OFF", obj->state, start_config); @@ -943,8 +946,6 @@ json::SerializationBuffer<> WebServer::fan_json_(fan::Fan *obj, JsonDetail start if (start_config == DETAIL_ALL) { this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -962,8 +963,9 @@ void WebServer::handle_light_request(AsyncWebServerRequest *request, const UrlMa if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->light_json_(obj, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->light_json_(obj, detail, builder); + send_json(request, builder); } else if (match.method_equals(ESPHOME_F("toggle"))) { DEFER_ACTION(obj, obj->toggle().perform()); request->send(200); @@ -1003,14 +1005,13 @@ void WebServer::handle_light_request(AsyncWebServerRequest *request, const UrlMa } request->send(404); } -json::SerializationBuffer<> WebServer::light_state_json_generator(WebServer *web_server, void *source) { - return web_server->light_json_((light::LightState *) (source), DETAIL_STATE); +void WebServer::light_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->light_json_((light::LightState *) (source), DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::light_all_json_generator(WebServer *web_server, void *source) { - return web_server->light_json_((light::LightState *) (source), DETAIL_ALL); +void WebServer::light_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->light_json_((light::LightState *) (source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::light_json_(light::LightState *obj, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::light_json_(light::LightState *obj, JsonDetail start_config, json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_value(root, obj, "light", obj->get_reported_values().is_on() ? "ON" : "OFF", start_config); @@ -1024,8 +1025,6 @@ json::SerializationBuffer<> WebServer::light_json_(light::LightState *obj, JsonD } this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -1043,8 +1042,9 @@ void WebServer::handle_cover_request(AsyncWebServerRequest *request, const UrlMa if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->cover_json_(obj, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->cover_json_(obj, detail, builder); + send_json(request, builder); return; } @@ -1091,14 +1091,13 @@ void WebServer::handle_cover_request(AsyncWebServerRequest *request, const UrlMa } request->send(404); } -json::SerializationBuffer<> WebServer::cover_state_json_generator(WebServer *web_server, void *source) { - return web_server->cover_json_((cover::Cover *) (source), DETAIL_STATE); +void WebServer::cover_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->cover_json_((cover::Cover *) (source), DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::cover_all_json_generator(WebServer *web_server, void *source) { - return web_server->cover_json_((cover::Cover *) (source), DETAIL_ALL); +void WebServer::cover_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->cover_json_((cover::Cover *) (source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::cover_json_(cover::Cover *obj, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::cover_json_(cover::Cover *obj, JsonDetail start_config, json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_icon_state_value(root, obj, "cover", obj->is_fully_closed() ? "CLOSED" : "OPEN", obj->position, @@ -1113,8 +1112,6 @@ json::SerializationBuffer<> WebServer::cover_json_(cover::Cover *obj, JsonDetail root[ESPHOME_F("assumed_state")] = obj->get_traits().get_is_assumed_state(); this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -1132,8 +1129,9 @@ void WebServer::handle_number_request(AsyncWebServerRequest *request, const UrlM if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->number_json_(obj, obj->state, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->number_json_(obj, obj->state, detail, builder); + send_json(request, builder); return; } if (!match.method_equals(ESPHOME_F("set"))) { @@ -1151,14 +1149,13 @@ void WebServer::handle_number_request(AsyncWebServerRequest *request, const UrlM request->send(404); } -json::SerializationBuffer<> WebServer::number_state_json_generator(WebServer *web_server, void *source) { - return web_server->number_json_((number::Number *) (source), ((number::Number *) (source))->state, DETAIL_STATE); +void WebServer::number_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->number_json_((number::Number *) (source), ((number::Number *) (source))->state, DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::number_all_json_generator(WebServer *web_server, void *source) { - return web_server->number_json_((number::Number *) (source), ((number::Number *) (source))->state, DETAIL_ALL); +void WebServer::number_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->number_json_((number::Number *) (source), ((number::Number *) (source))->state, DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::number_json_(number::Number *obj, float value, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::number_json_(number::Number *obj, float value, JsonDetail start_config, json::JsonBuilder &builder) { JsonObject root = builder.root(); const auto uom_ref = obj->get_unit_of_measurement_ref(); @@ -1181,8 +1178,6 @@ json::SerializationBuffer<> WebServer::number_json_(number::Number *obj, float v root[ESPHOME_F("uom")] = uom_ref.c_str(); this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -1199,8 +1194,9 @@ void WebServer::handle_date_request(AsyncWebServerRequest *request, const UrlMat continue; if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->date_json_(obj, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->date_json_(obj, detail, builder); + send_json(request, builder); return; } if (!match.method_equals(ESPHOME_F("set"))) { @@ -1225,14 +1221,13 @@ void WebServer::handle_date_request(AsyncWebServerRequest *request, const UrlMat request->send(404); } -json::SerializationBuffer<> WebServer::date_state_json_generator(WebServer *web_server, void *source) { - return web_server->date_json_((datetime::DateEntity *) (source), DETAIL_STATE); +void WebServer::date_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->date_json_((datetime::DateEntity *) (source), DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::date_all_json_generator(WebServer *web_server, void *source) { - return web_server->date_json_((datetime::DateEntity *) (source), DETAIL_ALL); +void WebServer::date_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->date_json_((datetime::DateEntity *) (source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::date_json_(datetime::DateEntity *obj, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::date_json_(datetime::DateEntity *obj, JsonDetail start_config, json::JsonBuilder &builder) { JsonObject root = builder.root(); // Format: YYYY-MM-DD (max 10 chars + null) @@ -1242,8 +1237,6 @@ json::SerializationBuffer<> WebServer::date_json_(datetime::DateEntity *obj, Jso if (start_config == DETAIL_ALL) { this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif // USE_DATETIME_DATE @@ -1260,8 +1253,9 @@ void WebServer::handle_time_request(AsyncWebServerRequest *request, const UrlMat continue; if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->time_json_(obj, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->time_json_(obj, detail, builder); + send_json(request, builder); return; } if (!match.method_equals(ESPHOME_F("set"))) { @@ -1285,14 +1279,13 @@ void WebServer::handle_time_request(AsyncWebServerRequest *request, const UrlMat } request->send(404); } -json::SerializationBuffer<> WebServer::time_state_json_generator(WebServer *web_server, void *source) { - return web_server->time_json_((datetime::TimeEntity *) (source), DETAIL_STATE); +void WebServer::time_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->time_json_((datetime::TimeEntity *) (source), DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::time_all_json_generator(WebServer *web_server, void *source) { - return web_server->time_json_((datetime::TimeEntity *) (source), DETAIL_ALL); +void WebServer::time_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->time_json_((datetime::TimeEntity *) (source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::time_json_(datetime::TimeEntity *obj, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::time_json_(datetime::TimeEntity *obj, JsonDetail start_config, json::JsonBuilder &builder) { JsonObject root = builder.root(); // Format: HH:MM:SS (8 chars + null) @@ -1302,8 +1295,6 @@ json::SerializationBuffer<> WebServer::time_json_(datetime::TimeEntity *obj, Jso if (start_config == DETAIL_ALL) { this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif // USE_DATETIME_TIME @@ -1320,8 +1311,9 @@ void WebServer::handle_datetime_request(AsyncWebServerRequest *request, const Ur continue; if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->datetime_json_(obj, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->datetime_json_(obj, detail, builder); + send_json(request, builder); return; } if (!match.method_equals(ESPHOME_F("set"))) { @@ -1345,14 +1337,13 @@ void WebServer::handle_datetime_request(AsyncWebServerRequest *request, const Ur } request->send(404); } -json::SerializationBuffer<> WebServer::datetime_state_json_generator(WebServer *web_server, void *source) { - return web_server->datetime_json_((datetime::DateTimeEntity *) (source), DETAIL_STATE); +void WebServer::datetime_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->datetime_json_((datetime::DateTimeEntity *) (source), DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::datetime_all_json_generator(WebServer *web_server, void *source) { - return web_server->datetime_json_((datetime::DateTimeEntity *) (source), DETAIL_ALL); +void WebServer::datetime_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->datetime_json_((datetime::DateTimeEntity *) (source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::datetime_json_(datetime::DateTimeEntity *obj, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::datetime_json_(datetime::DateTimeEntity *obj, JsonDetail start_config, json::JsonBuilder &builder) { JsonObject root = builder.root(); // Format: YYYY-MM-DD HH:MM:SS (max 19 chars + null) @@ -1363,8 +1354,6 @@ json::SerializationBuffer<> WebServer::datetime_json_(datetime::DateTimeEntity * if (start_config == DETAIL_ALL) { this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif // USE_DATETIME_DATETIME @@ -1382,8 +1371,9 @@ void WebServer::handle_text_request(AsyncWebServerRequest *request, const UrlMat if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->text_json_(obj, obj->state, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->text_json_(obj, obj->state, detail, builder); + send_json(request, builder); return; } if (!match.method_equals(ESPHOME_F("set"))) { @@ -1403,14 +1393,14 @@ void WebServer::handle_text_request(AsyncWebServerRequest *request, const UrlMat request->send(404); } -json::SerializationBuffer<> WebServer::text_state_json_generator(WebServer *web_server, void *source) { - return web_server->text_json_((text::Text *) (source), ((text::Text *) (source))->state, DETAIL_STATE); +void WebServer::text_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->text_json_((text::Text *) (source), ((text::Text *) (source))->state, DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::text_all_json_generator(WebServer *web_server, void *source) { - return web_server->text_json_((text::Text *) (source), ((text::Text *) (source))->state, DETAIL_ALL); +void WebServer::text_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->text_json_((text::Text *) (source), ((text::Text *) (source))->state, DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::text_json_(text::Text *obj, const std::string &value, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::text_json_(text::Text *obj, const std::string &value, JsonDetail start_config, + json::JsonBuilder &builder) { JsonObject root = builder.root(); // A password entity shows the mask and prefills the input with nothing, so the secret never @@ -1425,8 +1415,6 @@ json::SerializationBuffer<> WebServer::text_json_(text::Text *obj, const std::st root[ESPHOME_F("mode")] = (int) obj->traits.get_mode(); this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -1444,8 +1432,9 @@ void WebServer::handle_select_request(AsyncWebServerRequest *request, const UrlM if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->select_json_(obj, obj->has_state() ? obj->current_option() : StringRef(), detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->select_json_(obj, obj->has_state() ? obj->current_option() : StringRef(), detail, builder); + send_json(request, builder); return; } @@ -1465,16 +1454,16 @@ void WebServer::handle_select_request(AsyncWebServerRequest *request, const UrlM } request->send(404); } -json::SerializationBuffer<> WebServer::select_state_json_generator(WebServer *web_server, void *source) { +void WebServer::select_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { auto *obj = (select::Select *) (source); - return web_server->select_json_(obj, obj->has_state() ? obj->current_option() : StringRef(), DETAIL_STATE); + web_server->select_json_(obj, obj->has_state() ? obj->current_option() : StringRef(), DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::select_all_json_generator(WebServer *web_server, void *source) { +void WebServer::select_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { auto *obj = (select::Select *) (source); - return web_server->select_json_(obj, obj->has_state() ? obj->current_option() : StringRef(), DETAIL_ALL); + web_server->select_json_(obj, obj->has_state() ? obj->current_option() : StringRef(), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::select_json_(select::Select *obj, StringRef value, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::select_json_(select::Select *obj, StringRef value, JsonDetail start_config, + json::JsonBuilder &builder) { JsonObject root = builder.root(); // value points to null-terminated string literals from codegen (via current_option()) @@ -1486,8 +1475,6 @@ json::SerializationBuffer<> WebServer::select_json_(select::Select *obj, StringR } this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -1505,8 +1492,9 @@ void WebServer::handle_climate_request(AsyncWebServerRequest *request, const Url if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->climate_json_(obj, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->climate_json_(obj, detail, builder); + send_json(request, builder); return; } @@ -1547,17 +1535,16 @@ void WebServer::handle_climate_request(AsyncWebServerRequest *request, const Url } request->send(404); } -json::SerializationBuffer<> WebServer::climate_state_json_generator(WebServer *web_server, void *source) { +void WebServer::climate_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) false positive with ArduinoJson - return web_server->climate_json_((climate::Climate *) (source), DETAIL_STATE); + web_server->climate_json_((climate::Climate *) (source), DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::climate_all_json_generator(WebServer *web_server, void *source) { +void WebServer::climate_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) false positive with ArduinoJson - return web_server->climate_json_((climate::Climate *) (source), DETAIL_ALL); + web_server->climate_json_((climate::Climate *) (source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::climate_json_(climate::Climate *obj, JsonDetail start_config) { +void WebServer::climate_json_(climate::Climate *obj, JsonDetail start_config, json::JsonBuilder &builder) { // NOLINTBEGIN(clang-analyzer-cplusplus.NewDeleteLeaks) false positive with ArduinoJson - json::JsonBuilder builder; JsonObject root = builder.root(); set_json_id(root, obj, "climate", start_config); const auto traits = obj->get_traits(); @@ -1656,7 +1643,6 @@ json::SerializationBuffer<> WebServer::climate_json_(climate::Climate *obj, Json root[ESPHOME_F("state")] = root[ESPHOME_F("target_temperature")]; } - return builder.serialize(); // NOLINTEND(clang-analyzer-cplusplus.NewDeleteLeaks) } #endif @@ -1693,8 +1679,9 @@ void WebServer::handle_lock_request(AsyncWebServerRequest *request, const UrlMat if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->lock_json_(obj, obj->state, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->lock_json_(obj, obj->state, detail, builder); + send_json(request, builder); return; } @@ -1718,22 +1705,20 @@ void WebServer::handle_lock_request(AsyncWebServerRequest *request, const UrlMat } request->send(404); } -json::SerializationBuffer<> WebServer::lock_state_json_generator(WebServer *web_server, void *source) { - return web_server->lock_json_((lock::Lock *) (source), ((lock::Lock *) (source))->state, DETAIL_STATE); +void WebServer::lock_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->lock_json_((lock::Lock *) (source), ((lock::Lock *) (source))->state, DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::lock_all_json_generator(WebServer *web_server, void *source) { - return web_server->lock_json_((lock::Lock *) (source), ((lock::Lock *) (source))->state, DETAIL_ALL); +void WebServer::lock_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->lock_json_((lock::Lock *) (source), ((lock::Lock *) (source))->state, DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::lock_json_(lock::Lock *obj, lock::LockState value, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::lock_json_(lock::Lock *obj, lock::LockState value, JsonDetail start_config, + json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_icon_state_value(root, obj, "lock", json_state_str(lock::lock_state_to_string(value)), value, start_config); if (start_config == DETAIL_ALL) { this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -1751,8 +1736,9 @@ void WebServer::handle_valve_request(AsyncWebServerRequest *request, const UrlMa if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->valve_json_(obj, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->valve_json_(obj, detail, builder); + send_json(request, builder); return; } @@ -1797,14 +1783,13 @@ void WebServer::handle_valve_request(AsyncWebServerRequest *request, const UrlMa } request->send(404); } -json::SerializationBuffer<> WebServer::valve_state_json_generator(WebServer *web_server, void *source) { - return web_server->valve_json_((valve::Valve *) (source), DETAIL_STATE); +void WebServer::valve_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->valve_json_((valve::Valve *) (source), DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::valve_all_json_generator(WebServer *web_server, void *source) { - return web_server->valve_json_((valve::Valve *) (source), DETAIL_ALL); +void WebServer::valve_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->valve_json_((valve::Valve *) (source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::valve_json_(valve::Valve *obj, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::valve_json_(valve::Valve *obj, JsonDetail start_config, json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_icon_state_value(root, obj, "valve", obj->is_fully_closed() ? "CLOSED" : "OPEN", obj->position, @@ -1816,8 +1801,6 @@ json::SerializationBuffer<> WebServer::valve_json_(valve::Valve *obj, JsonDetail if (start_config == DETAIL_ALL) { this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -1835,8 +1818,9 @@ void WebServer::handle_alarm_control_panel_request(AsyncWebServerRequest *reques if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->alarm_control_panel_json_(obj, obj->get_state(), detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->alarm_control_panel_json_(obj, obj->get_state(), detail, builder); + send_json(request, builder); return; } @@ -1878,20 +1862,21 @@ void WebServer::handle_alarm_control_panel_request(AsyncWebServerRequest *reques } request->send(404); } -json::SerializationBuffer<> WebServer::alarm_control_panel_state_json_generator(WebServer *web_server, void *source) { - return web_server->alarm_control_panel_json_((alarm_control_panel::AlarmControlPanel *) (source), - ((alarm_control_panel::AlarmControlPanel *) (source))->get_state(), - DETAIL_STATE); +void WebServer::alarm_control_panel_state_json_generator(WebServer *web_server, void *source, + json::JsonBuilder &builder) { + web_server->alarm_control_panel_json_((alarm_control_panel::AlarmControlPanel *) (source), + ((alarm_control_panel::AlarmControlPanel *) (source))->get_state(), + DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::alarm_control_panel_all_json_generator(WebServer *web_server, void *source) { - return web_server->alarm_control_panel_json_((alarm_control_panel::AlarmControlPanel *) (source), - ((alarm_control_panel::AlarmControlPanel *) (source))->get_state(), - DETAIL_ALL); +void WebServer::alarm_control_panel_all_json_generator(WebServer *web_server, void *source, + json::JsonBuilder &builder) { + web_server->alarm_control_panel_json_((alarm_control_panel::AlarmControlPanel *) (source), + ((alarm_control_panel::AlarmControlPanel *) (source))->get_state(), DETAIL_ALL, + builder); } -json::SerializationBuffer<> WebServer::alarm_control_panel_json_(alarm_control_panel::AlarmControlPanel *obj, - alarm_control_panel::AlarmControlPanelState value, - JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::alarm_control_panel_json_(alarm_control_panel::AlarmControlPanel *obj, + alarm_control_panel::AlarmControlPanelState value, JsonDetail start_config, + json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_icon_state_value(root, obj, "alarm_control_panel", @@ -1899,8 +1884,6 @@ json::SerializationBuffer<> WebServer::alarm_control_panel_json_(alarm_control_p if (start_config == DETAIL_ALL) { this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -1918,8 +1901,9 @@ void WebServer::handle_water_heater_request(AsyncWebServerRequest *request, cons if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->water_heater_json_(obj, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->water_heater_json_(obj, detail, builder); + send_json(request, builder); return; } if (!match.method_equals(ESPHOME_F("set"))) { @@ -1957,15 +1941,15 @@ void WebServer::handle_water_heater_request(AsyncWebServerRequest *request, cons request->send(404); } -json::SerializationBuffer<> WebServer::water_heater_state_json_generator(WebServer *web_server, void *source) { - return web_server->water_heater_json_(static_cast(source), DETAIL_STATE); +void WebServer::water_heater_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { + web_server->water_heater_json_(static_cast(source), DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::water_heater_all_json_generator(WebServer *web_server, void *source) { +void WebServer::water_heater_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) false positive with ArduinoJson - return web_server->water_heater_json_(static_cast(source), DETAIL_ALL); + web_server->water_heater_json_(static_cast(source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::water_heater_json_(water_heater::WaterHeater *obj, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::water_heater_json_(water_heater::WaterHeater *obj, JsonDetail start_config, + json::JsonBuilder &builder) { JsonObject root = builder.root(); const auto mode = obj->get_mode(); @@ -2012,8 +1996,6 @@ json::SerializationBuffer<> WebServer::water_heater_json_(water_heater::WaterHea if (traits.has_feature_flags(water_heater::WATER_HEATER_SUPPORTS_ON_OFF)) { root[ESPHOME_F("is_on")] = obj->is_on(); } - - return builder.serialize(); } #endif @@ -2026,8 +2008,9 @@ void WebServer::handle_infrared_request(AsyncWebServerRequest *request, const Ur if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->infrared_json_(obj, detail); - request->send(200, ESPHOME_F("application/json"), data.c_str()); + json::JsonBuilder builder; + this->infrared_json_(obj, detail, builder); + send_json(request, builder); return; } if (!match.method_equals(ESPHOME_F("transmit"))) { @@ -2086,13 +2069,12 @@ void WebServer::handle_infrared_request(AsyncWebServerRequest *request, const Ur request->send(404); } -json::SerializationBuffer<> WebServer::infrared_all_json_generator(WebServer *web_server, void *source) { +void WebServer::infrared_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) false positive with ArduinoJson - return web_server->infrared_json_(static_cast(source), DETAIL_ALL); + web_server->infrared_json_(static_cast(source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::infrared_json_(infrared::Infrared *obj, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::infrared_json_(infrared::Infrared *obj, JsonDetail start_config, json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_icon_state_value(root, obj, "infrared", "", 0, start_config); @@ -2105,8 +2087,6 @@ json::SerializationBuffer<> WebServer::infrared_json_(infrared::Infrared *obj, J if (start_config == DETAIL_ALL) { this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -2119,8 +2099,9 @@ void WebServer::handle_radio_frequency_request(AsyncWebServerRequest *request, c if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->radio_frequency_json_(obj, detail); - request->send(200, ESPHOME_F("application/json"), data.c_str()); + json::JsonBuilder builder; + this->radio_frequency_json_(obj, detail, builder); + send_json(request, builder); return; } if (!match.method_equals(ESPHOME_F("transmit"))) { @@ -2178,14 +2159,13 @@ void WebServer::handle_radio_frequency_request(AsyncWebServerRequest *request, c request->send(404); } -json::SerializationBuffer<> WebServer::radio_frequency_all_json_generator(WebServer *web_server, void *source) { +void WebServer::radio_frequency_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) false positive with ArduinoJson - return web_server->radio_frequency_json_(static_cast(source), DETAIL_ALL); + web_server->radio_frequency_json_(static_cast(source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::radio_frequency_json_(radio_frequency::RadioFrequency *obj, - JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::radio_frequency_json_(radio_frequency::RadioFrequency *obj, JsonDetail start_config, + json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_icon_state_value(root, obj, "radio_frequency", "", 0, start_config); @@ -2203,8 +2183,6 @@ json::SerializationBuffer<> WebServer::radio_frequency_json_(radio_frequency::Ra if (start_config == DETAIL_ALL) { this->add_sorting_info_(root, obj); } - - return builder.serialize(); } #endif @@ -2224,8 +2202,9 @@ void WebServer::handle_event_request(AsyncWebServerRequest *request, const UrlMa // Note: request->method() is always HTTP_GET here (canHandle ensures this) if (entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->event_json_(obj, StringRef(), detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->event_json_(obj, StringRef(), detail, builder); + send_json(request, builder); return; } } @@ -2234,17 +2213,17 @@ void WebServer::handle_event_request(AsyncWebServerRequest *request, const UrlMa static StringRef get_event_type(event::Event *event) { return event ? event->get_last_event_type() : StringRef(); } -json::SerializationBuffer<> WebServer::event_state_json_generator(WebServer *web_server, void *source) { +void WebServer::event_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { auto *event = static_cast(source); - return web_server->event_json_(event, get_event_type(event), DETAIL_STATE); + web_server->event_json_(event, get_event_type(event), DETAIL_STATE, builder); } // NOLINTBEGIN(clang-analyzer-cplusplus.NewDeleteLeaks) false positive with ArduinoJson -json::SerializationBuffer<> WebServer::event_all_json_generator(WebServer *web_server, void *source) { +void WebServer::event_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { auto *event = static_cast(source); - return web_server->event_json_(event, get_event_type(event), DETAIL_ALL); + web_server->event_json_(event, get_event_type(event), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::event_json_(event::Event *obj, StringRef event_type, JsonDetail start_config) { - json::JsonBuilder builder; +void WebServer::event_json_(event::Event *obj, StringRef event_type, JsonDetail start_config, + json::JsonBuilder &builder) { JsonObject root = builder.root(); set_json_id(root, obj, "event", start_config); @@ -2260,8 +2239,6 @@ json::SerializationBuffer<> WebServer::event_json_(event::Event *obj, StringRef root[ESPHOME_F("device_class")] = obj->get_device_class_to(dc_buf); this->add_sorting_info_(root, obj); } - - return builder.serialize(); } // NOLINTEND(clang-analyzer-cplusplus.NewDeleteLeaks) #endif @@ -2278,8 +2255,9 @@ void WebServer::handle_update_request(AsyncWebServerRequest *request, const UrlM if (request->method() == HTTP_GET && entity_match.action_is_empty) { auto detail = get_request_detail(request); - auto data = this->update_json_(obj, detail); - request->send(200, "application/json", data.c_str()); + json::JsonBuilder builder; + this->update_json_(obj, detail, builder); + send_json(request, builder); return; } @@ -2294,17 +2272,16 @@ void WebServer::handle_update_request(AsyncWebServerRequest *request, const UrlM } request->send(404); } -json::SerializationBuffer<> WebServer::update_state_json_generator(WebServer *web_server, void *source) { +void WebServer::update_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) false positive with ArduinoJson - return web_server->update_json_((update::UpdateEntity *) (source), DETAIL_STATE); + web_server->update_json_((update::UpdateEntity *) (source), DETAIL_STATE, builder); } -json::SerializationBuffer<> WebServer::update_all_json_generator(WebServer *web_server, void *source) { +void WebServer::update_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder) { // NOLINTNEXTLINE(clang-analyzer-cplusplus.NewDeleteLeaks) false positive with ArduinoJson - return web_server->update_json_((update::UpdateEntity *) (source), DETAIL_ALL); + web_server->update_json_((update::UpdateEntity *) (source), DETAIL_ALL, builder); } -json::SerializationBuffer<> WebServer::update_json_(update::UpdateEntity *obj, JsonDetail start_config) { +void WebServer::update_json_(update::UpdateEntity *obj, JsonDetail start_config, json::JsonBuilder &builder) { // NOLINTBEGIN(clang-analyzer-cplusplus.NewDeleteLeaks) false positive with ArduinoJson - json::JsonBuilder builder; JsonObject root = builder.root(); set_json_icon_state_value(root, obj, "update", json_state_str(update::update_state_to_string(obj->state)), @@ -2321,7 +2298,6 @@ json::SerializationBuffer<> WebServer::update_json_(update::UpdateEntity *obj, J this->add_sorting_info_(root, obj); } - return builder.serialize(); // NOLINTEND(clang-analyzer-cplusplus.NewDeleteLeaks) } #endif diff --git a/esphome/components/web_server/web_server.h b/esphome/components/web_server/web_server.h index 3236ccfd3c8..143e9a15eae 100644 --- a/esphome/components/web_server/web_server.h +++ b/esphome/components/web_server/web_server.h @@ -37,6 +37,10 @@ extern const size_t ESPHOME_WEBSERVER_JS_INCLUDE_SIZE; namespace esphome::web_server { +// ESP-IDF cuts a log event here: nothing a browser log view needs is longer, and it bounds the +// tail a stalled client keeps. The Arduino backend takes a C string and sends the whole line. +constexpr size_t LOG_EVENT_MAX_LEN = 512; + // Parameter names live in flash on ESP8266 using ParamNameType = ProgmemStr; @@ -101,7 +105,7 @@ enum JsonDetail { DETAIL_ALL, DETAIL_STATE }; can be forgotten. */ #if !defined(USE_ESP32) && defined(USE_ARDUINO) -using message_generator_t = json::SerializationBuffer<>(WebServer *, void *); +using message_generator_t = void(WebServer *, void *, json::JsonBuilder &); class DeferredUpdateEventSourceList; class DeferredUpdateEventSource final : public AsyncEventSource { @@ -296,8 +300,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a sensor request under '/sensor/'. void handle_sensor_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> sensor_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> sensor_all_json_generator(WebServer *web_server, void *source); + static void sensor_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void sensor_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_SWITCH @@ -306,8 +310,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a switch request under '/switch//'. void handle_switch_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> switch_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> switch_all_json_generator(WebServer *web_server, void *source); + static void switch_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void switch_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_BUTTON @@ -315,7 +319,7 @@ class WebServer final : public Component, public AsyncWebHandler { void handle_button_request(AsyncWebServerRequest *request, const UrlMatch &match); // Buttons are stateless, so there is no button_state_json_generator - static json::SerializationBuffer<> button_all_json_generator(WebServer *web_server, void *source); + static void button_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_BINARY_SENSOR @@ -324,8 +328,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a binary sensor request under '/binary_sensor/'. void handle_binary_sensor_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> binary_sensor_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> binary_sensor_all_json_generator(WebServer *web_server, void *source); + static void binary_sensor_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void binary_sensor_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_FAN @@ -334,8 +338,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a fan request under '/fan//'. void handle_fan_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> fan_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> fan_all_json_generator(WebServer *web_server, void *source); + static void fan_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void fan_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_LIGHT @@ -344,8 +348,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a light request under '/light//'. void handle_light_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> light_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> light_all_json_generator(WebServer *web_server, void *source); + static void light_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void light_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_TEXT_SENSOR @@ -354,8 +358,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a text sensor request under '/text_sensor/'. void handle_text_sensor_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> text_sensor_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> text_sensor_all_json_generator(WebServer *web_server, void *source); + static void text_sensor_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void text_sensor_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_COVER @@ -364,8 +368,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a cover request under '/cover//'. void handle_cover_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> cover_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> cover_all_json_generator(WebServer *web_server, void *source); + static void cover_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void cover_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_NUMBER @@ -373,8 +377,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a number request under '/number/'. void handle_number_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> number_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> number_all_json_generator(WebServer *web_server, void *source); + static void number_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void number_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_DATETIME_DATE @@ -382,8 +386,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a date request under '/date/'. void handle_date_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> date_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> date_all_json_generator(WebServer *web_server, void *source); + static void date_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void date_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_DATETIME_TIME @@ -391,8 +395,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a time request under '/time/'. void handle_time_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> time_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> time_all_json_generator(WebServer *web_server, void *source); + static void time_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void time_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_DATETIME_DATETIME @@ -400,8 +404,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a datetime request under '/datetime/'. void handle_datetime_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> datetime_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> datetime_all_json_generator(WebServer *web_server, void *source); + static void datetime_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void datetime_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_TEXT @@ -409,8 +413,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a text input request under '/text/'. void handle_text_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> text_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> text_all_json_generator(WebServer *web_server, void *source); + static void text_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void text_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_SELECT @@ -418,8 +422,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a select request under '/select/'. void handle_select_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> select_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> select_all_json_generator(WebServer *web_server, void *source); + static void select_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void select_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_CLIMATE @@ -427,8 +431,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a climate request under '/climate/'. void handle_climate_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> climate_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> climate_all_json_generator(WebServer *web_server, void *source); + static void climate_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void climate_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_LOCK @@ -437,8 +441,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a lock request under '/lock//'. void handle_lock_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> lock_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> lock_all_json_generator(WebServer *web_server, void *source); + static void lock_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void lock_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_VALVE @@ -447,8 +451,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a valve request under '/valve//'. void handle_valve_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> valve_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> valve_all_json_generator(WebServer *web_server, void *source); + static void valve_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void valve_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_ALARM_CONTROL_PANEL @@ -457,8 +461,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a alarm_control_panel request under '/alarm_control_panel/'. void handle_alarm_control_panel_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> alarm_control_panel_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> alarm_control_panel_all_json_generator(WebServer *web_server, void *source); + static void alarm_control_panel_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void alarm_control_panel_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_WATER_HEATER @@ -467,21 +471,21 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a water_heater request under '/water_heater//'. void handle_water_heater_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> water_heater_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> water_heater_all_json_generator(WebServer *web_server, void *source); + static void water_heater_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void water_heater_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_INFRARED /// Handle an infrared request under '/infrared//transmit'. void handle_infrared_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> infrared_all_json_generator(WebServer *web_server, void *source); + static void infrared_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_RADIO_FREQUENCY /// Handle a radio frequency request under '/radio_frequency//transmit'. void handle_radio_frequency_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> radio_frequency_all_json_generator(WebServer *web_server, void *source); + static void radio_frequency_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif #ifdef USE_MEDIA_PLAYER @@ -492,8 +496,8 @@ class WebServer final : public Component, public AsyncWebHandler { #ifdef USE_EVENT void on_event(event::Event *obj); - static json::SerializationBuffer<> event_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> event_all_json_generator(WebServer *web_server, void *source); + static void event_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void event_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); /// Handle a event request under '/event'. void handle_event_request(AsyncWebServerRequest *request, const UrlMatch &match); @@ -505,8 +509,8 @@ class WebServer final : public Component, public AsyncWebHandler { /// Handle a update request under '/update/'. void handle_update_request(AsyncWebServerRequest *request, const UrlMatch &match); - static json::SerializationBuffer<> update_state_json_generator(WebServer *web_server, void *source); - static json::SerializationBuffer<> update_all_json_generator(WebServer *web_server, void *source); + static void update_state_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); + static void update_all_json_generator(WebServer *web_server, void *source, json::JsonBuilder &builder); #endif /// Override the web handler's canHandle method. @@ -624,77 +628,77 @@ class WebServer final : public Component, public AsyncWebHandler { private: #ifdef USE_SENSOR - json::SerializationBuffer<> sensor_json_(sensor::Sensor *obj, float value, JsonDetail start_config); + void sensor_json_(sensor::Sensor *obj, float value, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_SWITCH - json::SerializationBuffer<> switch_json_(switch_::Switch *obj, bool value, JsonDetail start_config); + void switch_json_(switch_::Switch *obj, bool value, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_BUTTON - json::SerializationBuffer<> button_json_(button::Button *obj, JsonDetail start_config); + void button_json_(button::Button *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_BINARY_SENSOR - json::SerializationBuffer<> binary_sensor_json_(binary_sensor::BinarySensor *obj, bool value, - JsonDetail start_config); + void binary_sensor_json_(binary_sensor::BinarySensor *obj, bool value, JsonDetail start_config, + json::JsonBuilder &builder); #endif #ifdef USE_FAN - json::SerializationBuffer<> fan_json_(fan::Fan *obj, JsonDetail start_config); + void fan_json_(fan::Fan *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_LIGHT - json::SerializationBuffer<> light_json_(light::LightState *obj, JsonDetail start_config); + void light_json_(light::LightState *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_TEXT_SENSOR - json::SerializationBuffer<> text_sensor_json_(text_sensor::TextSensor *obj, const std::string &value, - JsonDetail start_config); + void text_sensor_json_(text_sensor::TextSensor *obj, const std::string &value, JsonDetail start_config, + json::JsonBuilder &builder); #endif #ifdef USE_COVER - json::SerializationBuffer<> cover_json_(cover::Cover *obj, JsonDetail start_config); + void cover_json_(cover::Cover *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_NUMBER - json::SerializationBuffer<> number_json_(number::Number *obj, float value, JsonDetail start_config); + void number_json_(number::Number *obj, float value, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_DATETIME_DATE - json::SerializationBuffer<> date_json_(datetime::DateEntity *obj, JsonDetail start_config); + void date_json_(datetime::DateEntity *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_DATETIME_TIME - json::SerializationBuffer<> time_json_(datetime::TimeEntity *obj, JsonDetail start_config); + void time_json_(datetime::TimeEntity *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_DATETIME_DATETIME - json::SerializationBuffer<> datetime_json_(datetime::DateTimeEntity *obj, JsonDetail start_config); + void datetime_json_(datetime::DateTimeEntity *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_TEXT - json::SerializationBuffer<> text_json_(text::Text *obj, const std::string &value, JsonDetail start_config); + void text_json_(text::Text *obj, const std::string &value, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_SELECT - json::SerializationBuffer<> select_json_(select::Select *obj, StringRef value, JsonDetail start_config); + void select_json_(select::Select *obj, StringRef value, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_CLIMATE - json::SerializationBuffer<> climate_json_(climate::Climate *obj, JsonDetail start_config); + void climate_json_(climate::Climate *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_LOCK - json::SerializationBuffer<> lock_json_(lock::Lock *obj, lock::LockState value, JsonDetail start_config); + void lock_json_(lock::Lock *obj, lock::LockState value, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_VALVE - json::SerializationBuffer<> valve_json_(valve::Valve *obj, JsonDetail start_config); + void valve_json_(valve::Valve *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_ALARM_CONTROL_PANEL - json::SerializationBuffer<> alarm_control_panel_json_(alarm_control_panel::AlarmControlPanel *obj, - alarm_control_panel::AlarmControlPanelState value, - JsonDetail start_config); + void alarm_control_panel_json_(alarm_control_panel::AlarmControlPanel *obj, + alarm_control_panel::AlarmControlPanelState value, JsonDetail start_config, + json::JsonBuilder &builder); #endif #ifdef USE_EVENT - json::SerializationBuffer<> event_json_(event::Event *obj, StringRef event_type, JsonDetail start_config); + void event_json_(event::Event *obj, StringRef event_type, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_WATER_HEATER - json::SerializationBuffer<> water_heater_json_(water_heater::WaterHeater *obj, JsonDetail start_config); + void water_heater_json_(water_heater::WaterHeater *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_INFRARED - json::SerializationBuffer<> infrared_json_(infrared::Infrared *obj, JsonDetail start_config); + void infrared_json_(infrared::Infrared *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_RADIO_FREQUENCY - json::SerializationBuffer<> radio_frequency_json_(radio_frequency::RadioFrequency *obj, JsonDetail start_config); + void radio_frequency_json_(radio_frequency::RadioFrequency *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif #ifdef USE_UPDATE - json::SerializationBuffer<> update_json_(update::UpdateEntity *obj, JsonDetail start_config); + void update_json_(update::UpdateEntity *obj, JsonDetail start_config, json::JsonBuilder &builder); #endif }; diff --git a/esphome/components/web_server_idf/sse_chunk.cpp b/esphome/components/web_server_idf/sse_chunk.cpp new file mode 100644 index 00000000000..a312b64c03e --- /dev/null +++ b/esphome/components/web_server_idf/sse_chunk.cpp @@ -0,0 +1,66 @@ +#include "sse_chunk.h" + +#include +#include + +#include "esphome/core/helpers.h" + +namespace esphome::web_server_idf { + +void for_each_chunk_piece(const char *message, size_t message_len, ChunkPieceSink sink, void *ctx) { + if (message == nullptr) { + sink(ctx, CHUNK_END, CHUNK_END_LEN); + return; + } + const char *pos = message; + const char *end = message + message_len; + for (;;) { + const size_t remaining = end - pos; + const auto *n = static_cast(memchr(pos, '\n', remaining)); + // Only a \r before the next \n can end this line, so the search stops there instead of + // rescanning the rest of the message for every line + const auto *r = static_cast(memchr(pos, '\r', n != nullptr ? n - pos : remaining)); + if (n == nullptr && r == nullptr) { + sink(ctx, pos, remaining); + break; + } + const char *brk = (r != nullptr && (n == nullptr || r < n)) ? r : n; + sink(ctx, pos, brk - pos); + pos = brk + ((brk == r && brk + 1 == n) ? 2 : 1); + if (pos >= end) { + break; + } + sink(ctx, SSE_SEP, SSE_SEP_LEN); + } + sink(ctx, SSE_SUFFIX, SSE_SUFFIX_LEN); +} + +size_t build_chunk_prefix(char *buf, size_t size, const char *event, uint32_t id, uint32_t reconnect, bool with_data) { + size_t len = CHUNK_HDR_LEN; + if (reconnect) { + len = buf_append_printf(buf, size, len, "retry: %" PRIu32 "\r\n", reconnect); + } + if (id) { + len = buf_append_printf(buf, size, len, "id: %" PRIu32 "\r\n", id); + } + if (event && *event) { + len = buf_append_str(buf, size, len, "event: "); + len = buf_append_str(buf, size, len, event); + len = buf_append_str(buf, size, len, "\r\n"); + } + if (with_data) { + len = buf_append_str(buf, size, len, "data: "); + } + return len; +} + +void write_chunk_header(char *buf, size_t chunk_len) { + // Eight lowercase hex digits; the temp keeps the terminator format_hex_to writes out of buf + char digits[9]; + format_hex_to(digits, static_cast(chunk_len)); + std::memcpy(buf, digits, 8); + buf[8] = '\r'; + buf[9] = '\n'; +} + +} // namespace esphome::web_server_idf diff --git a/esphome/components/web_server_idf/sse_chunk.h b/esphome/components/web_server_idf/sse_chunk.h new file mode 100644 index 00000000000..c8a4170008a --- /dev/null +++ b/esphome/components/web_server_idf/sse_chunk.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include + +namespace esphome::web_server_idf { + +// Wire framing of one Server-Sent-Events event inside the chunked /events response: +// "%08x\r\n" + fields + "data: " line ["\r\ndata: " line]... "\r\n" + "\r\n" + "\r\n" +// The chunk header and the final CRLF (the chunk terminator) do not count toward the +// chunk length; everything between them does. + +// HTTP chunk header "%08x\r\n" +constexpr size_t CHUNK_HDR_LEN = 10; +// Chunk terminator, also the only piece of a null message (no data line, no blank line) +constexpr char CHUNK_END[] = "\r\n"; +constexpr size_t CHUNK_END_LEN = sizeof(CHUNK_END) - 1; +// Between two data lines: the end of one and the prefix of the next +constexpr char SSE_SEP[] = "\r\ndata: "; +constexpr size_t SSE_SEP_LEN = sizeof(SSE_SEP) - 1; +// End of the last data line, the blank line ending the event, and the chunk terminator +constexpr char SSE_SUFFIX[] = "\r\n\r\n\r\n"; +constexpr size_t SSE_SUFFIX_LEN = sizeof(SSE_SUFFIX) - 1; + +// Receives one piece of the chunk; ctx is whatever the caller passed to for_each_chunk_piece() +using ChunkPieceSink = void (*)(void *ctx, const char *piece, size_t len); + +// Calls sink for each piece after the prefix: data lines split on \n, \r or \r\n with SSE_SEP +// between them and SSE_SUFFIX after the last (as ESPAsyncWebServer: a trailing break adds no +// empty line, an inner one is kept). A null message yields only the chunk terminator. +void for_each_chunk_piece(const char *message, size_t message_len, ChunkPieceSink sink, void *ctx); + +// Writes the chunk header placeholder, the retry/id/event lines and, with_data, the first +// "data: " into buf. Returns the prefix length; size - 1 or more means the event name did not fit. +size_t build_chunk_prefix(char *buf, size_t size, const char *event, uint32_t id, uint32_t reconnect, bool with_data); + +// Fills the header placeholder at the start of buf once the chunk length is known +void write_chunk_header(char *buf, size_t chunk_len); + +} // namespace esphome::web_server_idf diff --git a/esphome/components/web_server_idf/web_server_idf.cpp b/esphome/components/web_server_idf/web_server_idf.cpp index 0062556063f..837a705cd36 100644 --- a/esphome/components/web_server_idf/web_server_idf.cpp +++ b/esphome/components/web_server_idf/web_server_idf.cpp @@ -1,7 +1,9 @@ #ifdef USE_ESP32 +#include #include #include +#include #include #include #include @@ -15,6 +17,7 @@ #include #include "utils.h" +#include "sse_chunk.h" #include "web_server_idf.h" #ifdef USE_WEBSERVER_AUTH_DIGEST @@ -740,7 +743,8 @@ bool AsyncEventSource::loop() { ++i; } } - return !this->sessions_.empty(); + // A session still waiting for httpd to commit its context keeps the loop alive too + return !this->sessions_.empty() || this->has_pending_sessions_.load(std::memory_order_acquire); } void AsyncEventSource::adopt_pending_sessions_main_loop_() { @@ -756,6 +760,14 @@ void AsyncEventSource::adopt_pending_sessions_main_loop_() { delete rsp; // NOLINT(cppcoreguidelines-owning-memory) continue; } + // httpd commits the session context only after the creating handler returns, so stay + // pending until then; httpd_req_cleanup() always commits it or calls destroy() + if (httpd_sess_get_ctx(rsp->hd_, rsp->fd_.load()) != rsp) { + LockGuard guard{this->pending_mutex_}; + this->pending_sessions_.push_back(rsp); + this->has_pending_sessions_.store(true, std::memory_order_release); + continue; + } this->sessions_.push_back(rsp); // Prime first so on_connect_ observes a session that has already sent its // initial ping/config/sorting_groups, matching the pre-refactor ordering. @@ -792,7 +804,7 @@ AsyncEventSourceResponse::AsyncEventSourceResponse(const AsyncWebServerRequest * esphome::web_server_idf::AsyncEventSource *server, esphome::web_server::WebServer *ws) : server_(server), web_server_(ws), entities_iterator_(ws, server) { - // Httpd task only. start_session_main_loop_() handles event_buffer_ / iterator setup. + // Httpd task only. start_session_main_loop_() sends the greeting and starts the iterator. httpd_req_t *req = *request; httpd_resp_set_status(req, HTTPD_200); @@ -820,9 +832,14 @@ AsyncEventSourceResponse::AsyncEventSourceResponse(const AsyncWebServerRequest * void AsyncEventSourceResponse::start_session_main_loop_() { auto *ws = this->web_server_; - // tcp send buffer is empty on connect, so these should always go through + // The tcp send buffer is empty on connect. A refusal is a closing session or a failed tail + // allocation; nothing retries the greeting, so close and let the client reconnect. auto message = ws->get_config_json(); - this->try_send_nodefer(message.c_str(), message.size(), "ping", millis(), 30000); + if (!this->try_send_nodefer(message.c_str(), message.size(), "ping", millis(), 30000)) { + ESP_LOGW(TAG, "Config not sent to fd %d", this->fd_.load()); + this->request_close_(); + return; + } #ifdef USE_WEBSERVER_SORTING for (auto &group : ws->sorting_groups_) { @@ -833,8 +850,13 @@ void AsyncEventSourceResponse::start_session_main_loop_() { message = builder.serialize(); // a (very) large number of these should be able to be queued initially without defer - // since the only thing in the send buffer at this point is the initial ping/config - this->try_send_nodefer(message.c_str(), message.size(), "sorting_group"); + // since the only thing in the send buffer at this point is the initial ping/config. + // A refusal means the socket is full or closing; the remaining groups are not sent. The + // session stays up on purpose: partial grouping beats a reconnect loop on a slow link. + if (!this->try_send_nodefer(message.c_str(), message.size(), "sorting_group")) { + ESP_LOGW(TAG, "Sorting groups not sent to fd %d", this->fd_.load()); + break; + } } #endif @@ -871,8 +893,9 @@ void AsyncEventSourceResponse::process_deferred_queue_() { } while (!deferred_queue_.empty()) { DeferredEvent &de = deferred_queue_.front(); - auto message = de.message_generator_(web_server_, de.source_); - if (this->try_send_nodefer(message.c_str(), message.size(), "state")) { + json::JsonBuilder builder; + de.message_generator_(web_server_, de.source_, builder); + if (this->send_json_(builder)) { if (this->close_requested_ || deferred_queue_.empty()) { return; } @@ -888,8 +911,10 @@ void AsyncEventSourceResponse::request_close_() { if (!this->close_requested_) { this->close_requested_ = true; this->deferred_queue_.clear(); - this->event_buffer_.clear(); - this->event_bytes_sent_ = 0; + this->tail_.reset(); + this->tail_cap_ = 0; + this->tail_len_ = 0; + this->tail_sent_ = 0; this->next_close_attempt_ms_ = App.get_loop_component_start_time(); } @@ -940,21 +965,43 @@ void AsyncEventSourceResponse::close_session_work(void *arg) { response->close_work_queued_.store(false, std::memory_order_release); } -void AsyncEventSourceResponse::process_buffer_() { - if (this->close_requested_ || event_buffer_.empty()) { - return; +ssize_t AsyncEventSourceResponse::send_(struct iovec *iov, int iovcnt) { + // httpd frees a session before closing its socket, so the fd may already be a new client's. + // Treated as would-block; the stall timer ends a session that never becomes ours again. + const int fd = this->fd_.load(); + if (httpd_sess_get_ctx(this->hd_, fd) != this) { + return 0; } - if (event_bytes_sent_ == event_buffer_.size()) { - event_buffer_.resize(0); - event_bytes_sent_ = 0; - return; + struct msghdr msg {}; + msg.msg_iov = iov; + msg.msg_iovlen = iovcnt; + const ssize_t sent = sendmsg(fd, &msg, MSG_DONTWAIT); + if (sent >= 0) { + return sent; } + const int err = errno; + if (err == EAGAIN || err == EWOULDBLOCK) { + return 0; + } + ESP_LOGD(TAG, "send error: errno %d", err); + this->request_close_(); + return -1; +} - size_t remaining = event_buffer_.size() - event_bytes_sent_; - int bytes_sent = - httpd_socket_send(this->hd_, this->fd_.load(), event_buffer_.c_str() + event_bytes_sent_, remaining, 0); - if (bytes_sent == HTTPD_SOCK_ERR_TIMEOUT) { - // EAGAIN/EWOULDBLOCK - socket buffer full, try again later +void AsyncEventSourceResponse::drain_tail_() { + if (this->sending_ || this->close_requested_ || this->tail_len_ == 0) { + return; + } + SendGuard guard{*this}; + + const size_t remaining = this->tail_len_ - this->tail_sent_; + struct iovec iov = {this->tail_.get() + this->tail_sent_, remaining}; + const ssize_t sent = this->send_(&iov, 1); + if (sent < 0) { + return; + } + if (sent == 0) { + // Socket buffer full, try again later // NOTE: Similar logic exists in web_server/web_server.cpp in DeferredUpdateEventSource::process_deferred_queue_(). // The IDF path is intentionally time-based and closes through HTTPD to preserve session ownership. const uint32_t now = App.get_loop_component_start_time(); @@ -968,32 +1015,62 @@ void AsyncEventSourceResponse::process_buffer_() { } return; } - if (bytes_sent == HTTPD_SOCK_ERR_FAIL) { - // Low-level asynchronous sends do not make HTTPD close the session automatically. - this->request_close_(); - return; - } - if (bytes_sent <= 0) { - // Unexpected error or zero bytes sent - ESP_LOGW(TAG, "Unexpected send result: %d", bytes_sent); - this->request_close_(); - return; - } - // Successful send - reset stall tracking this->send_failure_started_ms_ = 0; - event_bytes_sent_ += bytes_sent; - - // Log partial sends for debugging - if (event_bytes_sent_ < event_buffer_.size()) { - ESP_LOGV(TAG, "Partial send: %d/%zu bytes (total: %zu/%zu)", bytes_sent, remaining, event_bytes_sent_, - event_buffer_.size()); + this->tail_sent_ += sent; + if (this->tail_sent_ < this->tail_len_) { + ESP_LOGV(TAG, "Partial send: %zd/%zu bytes (total: %u/%u)", sent, remaining, this->tail_sent_, this->tail_len_); + return; } + // Fully sent; the storage stays for the next stall + this->tail_len_ = 0; +} - if (event_bytes_sent_ == event_buffer_.size()) { - event_buffer_.resize(0); - event_bytes_sent_ = 0; +bool AsyncEventSourceResponse::reserve_tail_(size_t len) { + if (this->tail_cap_ >= len) { + return true; } + if (len > TAIL_MAX_SIZE) { + return false; + } + // Nothing is pending while the tail grows, so free the old block first. PREFER_INTERNAL keeps + // the tail where plain new put it. + this->tail_.reset(); + this->tail_ = RAMAllocator(RAMAllocator::PREFER_INTERNAL).make_unique_array_for_overwrite(len); + this->tail_cap_ = this->tail_ ? len : 0; + return this->tail_cap_ != 0; +} + +bool AsyncEventSourceResponse::stash_chunk_(const char *prefix, size_t prefix_len, const char *message, + size_t message_len, size_t total, size_t sent) { + // A log event of nothing but line breaks is the largest chunk; reserve_tail_() must never + // refuse it for size, only for memory + static_assert(SSE_SEP_LEN * web_server::LOG_EVENT_MAX_LEN + PREFIX_BUF_SIZE + SSE_SUFFIX_LEN <= TAIL_MAX_SIZE, + "the log cut in web_server.h must keep a worst case log event inside the tail ceiling"); + if (!this->reserve_tail_(total)) { + if (sent != 0) { + // Part of the chunk is on the wire, so the stream is broken and the client has to go + ESP_LOGW(TAG, "Cannot buffer a %zu byte chunk, closing", total); + this->request_close_(); + } else { + this->tail_alloc_failed_(total); // nothing on the wire, the caller retries on the stall clock + } + return false; + } + uint8_t *dst = this->tail_.get(); + std::memcpy(dst, prefix, prefix_len); + dst += prefix_len; + for_each_chunk_piece( + message, message_len, + [](void *ctx, const char *piece, size_t len) { + auto &out = *static_cast(ctx); + std::memcpy(out, piece, len); + out += len; + }, + &dst); + this->tail_len_ = total; + this->tail_sent_ = sent; + return true; } void AsyncEventSourceResponse::loop() { @@ -1001,7 +1078,7 @@ void AsyncEventSourceResponse::loop() { this->process_close_(); return; } - process_buffer_(); + drain_tail_(); process_deferred_queue_(); if (this->close_requested_) return; @@ -1009,142 +1086,138 @@ void AsyncEventSourceResponse::loop() { this->entities_iterator_.try_advance(1); } +bool AsyncEventSourceResponse::send_json_(json::JsonBuilder &builder) { + char buf[JSON_BUF_SIZE]; + const size_t len = builder.serialize_to(buf, sizeof(buf)); + if (len < sizeof(buf)) { + return this->try_send_nodefer(buf, len, "state"); + } + + // Too large for the stack: the tail holds the whole chunk and loop() drains it. Serialized + // JSON has no raw line break, so the body is one data line. + if (!this->ready_to_send_()) { + return false; + } + { + SendGuard guard{*this}; + char prefix[PREFIX_BUF_SIZE]; + const size_t prefix_len = build_chunk_prefix(prefix, sizeof(prefix), "state", 0, 0, true); + + // Grow the tail until the document fits. Nothing has reached the wire, so a document that + // cannot be held costs only this event, and the tail grown for it is released. + size_t json_len = 0; + size_t cap = std::max(JSON_BUF_SIZE * 2, this->tail_cap_); + for (;;) { + if (!this->reserve_tail_(cap)) { + this->tail_alloc_failed_(cap); + return false; // stays deferred, retried on a later pass + } + const size_t room = cap - prefix_len - SSE_SUFFIX_LEN; + json_len = builder.serialize_to(reinterpret_cast(this->tail_.get()) + prefix_len, room); + if (json_len < room) { + break; + } + if (cap >= TAIL_MAX_SIZE) { + ESP_LOGW(TAG, "State event does not fit %zu bytes, dropped", room); + this->tail_.reset(); + this->tail_cap_ = 0; + this->send_failure_started_ms_ = 0; + return true; // would never fit, reported as sent + } + cap = std::min(cap * 2, TAIL_MAX_SIZE); + } + + const size_t total = prefix_len + json_len + SSE_SUFFIX_LEN; + write_chunk_header(prefix, total - CHUNK_HDR_LEN - CHUNK_END_LEN); + uint8_t *dst = this->tail_.get(); + std::memcpy(dst, prefix, prefix_len); + std::memcpy(dst + prefix_len + json_len, SSE_SUFFIX, SSE_SUFFIX_LEN); + this->tail_len_ = total; + this->tail_sent_ = 0; + } + drain_tail_(); + return true; +} + +void AsyncEventSourceResponse::tail_alloc_failed_(size_t cap) { + // Same stall clock as a socket that stops draining, so a session cannot retry forever + const uint32_t now = App.get_loop_component_start_time(); + if (this->send_failure_started_ms_ == 0) { + this->send_failure_started_ms_ = now != 0 ? now : 1; // Reserve zero for no stall. + ESP_LOGW(TAG, "No memory for a %zu byte chunk", cap); + return; + } + if (static_cast(now - (this->send_failure_started_ms_ + SEND_STALL_TIMEOUT_MS)) >= 0) { + ESP_LOGW(TAG, "Closing EventSource after %" PRIu32 " ms without memory", now - this->send_failure_started_ms_); + this->request_close_(); + } +} + +bool AsyncEventSourceResponse::ready_to_send_() { + if (this->sending_ || this->fd_.load() == 0 || this->close_requested_) { + return false; + } + drain_tail_(); + return !this->close_requested_ && this->tail_len_ == 0; +} + bool AsyncEventSourceResponse::try_send_nodefer(const char *message, size_t message_len, const char *event, uint32_t id, uint32_t reconnect) { - if (this->fd_.load() == 0 || this->close_requested_) { + if (!this->ready_to_send_()) { return false; } + SendGuard guard{*this}; - process_buffer_(); - if (this->close_requested_ || !event_buffer_.empty()) { - // there is still pending event data to send first - return false; + // Everything after the prefix goes out straight from the caller's buffer + char prefix[PREFIX_BUF_SIZE]; + const size_t prefix_len = build_chunk_prefix(prefix, sizeof(prefix), event, id, reconnect, message != nullptr); + if (message == nullptr && prefix_len == CHUNK_HDR_LEN) { + return true; // Match ESPAsyncWebServer: nothing to send } - - // 8 spaces are standing in for the hexidecimal chunk length to print later - const char chunk_len_header[] = " " CRLF_STR; - const int chunk_len_header_len = sizeof(chunk_len_header) - 1; - - event_buffer_.append(chunk_len_header); - - // Use stack buffer for formatting numeric fields to avoid temporary string allocations - // Size: "retry: " (7) + max uint32 (10 digits) + CRLF (2) + null (1) = 20 bytes, use 32 for safety - constexpr size_t num_buf_size = 32; - char num_buf[num_buf_size]; - - if (reconnect) { - int len = snprintf(num_buf, num_buf_size, "retry: %" PRIu32 CRLF_STR, reconnect); - event_buffer_.append(num_buf, len); - } - - if (id) { - int len = snprintf(num_buf, num_buf_size, "id: %" PRIu32 CRLF_STR, id); - event_buffer_.append(num_buf, len); - } - - if (event && *event) { - event_buffer_.append("event: ", sizeof("event: ") - 1); - event_buffer_.append(event); - event_buffer_.append(CRLF_STR, CRLF_LEN); - } - - // Match ESPAsyncWebServer: null message means no data lines and no terminating blank line - if (message) { - // SSE spec requires each line of a multi-line message to have its own "data:" prefix - // Handle \n, \r, and \r\n line endings (matching ESPAsyncWebServer behavior) - - // Fast path: check if message contains any newlines at all - // Most SSE messages (JSON state updates) have no newlines - const char *first_n = static_cast(memchr(message, '\n', message_len)); - const char *first_r = static_cast(memchr(message, '\r', message_len)); - - if (first_n == nullptr && first_r == nullptr) { - // No newlines - fast path (most common case) - event_buffer_.append("data: ", sizeof("data: ") - 1); - event_buffer_.append(message, message_len); - event_buffer_.append(CRLF_STR CRLF_STR, CRLF_LEN * 2); // data line + blank line terminator - } else { - // Has newlines - handle multi-line message - const char *line_start = message; - const char *msg_end = message + message_len; - - // Reuse the first search results - const char *next_n = first_n; - const char *next_r = first_r; - - while (line_start <= msg_end) { - const char *line_end; - const char *next_line; - - if (next_n == nullptr && next_r == nullptr) { - // No more line breaks - output remaining text as final line - event_buffer_.append("data: ", sizeof("data: ") - 1); - event_buffer_.append(line_start, msg_end - line_start); - event_buffer_.append(CRLF_STR, CRLF_LEN); - break; - } - - // Determine line ending type and next line start - if (next_n != nullptr && next_r != nullptr) { - if (next_r + 1 == next_n) { - // \r\n sequence - line_end = next_r; - next_line = next_n + 1; - } else { - // Mixed \n and \r - use whichever comes first - line_end = (next_r < next_n) ? next_r : next_n; - next_line = line_end + 1; - } - } else if (next_n != nullptr) { - // Unix LF - line_end = next_n; - next_line = next_n + 1; - } else { - // Old Mac CR - line_end = next_r; - next_line = next_r + 1; - } - - // Output this line - event_buffer_.append("data: ", sizeof("data: ") - 1); - event_buffer_.append(line_start, line_end - line_start); - event_buffer_.append(CRLF_STR, CRLF_LEN); - - line_start = next_line; - - // Check if we've consumed all content - if (line_start >= msg_end) { - break; - } - - // Search for next newlines only in remaining string - next_n = static_cast(memchr(line_start, '\n', msg_end - line_start)); - next_r = static_cast(memchr(line_start, '\r', msg_end - line_start)); - } - - // Terminate message with blank line - event_buffer_.append(CRLF_STR, CRLF_LEN); - } - } - - if (event_buffer_.size() == static_cast(chunk_len_header_len)) { - // Nothing was added, reset buffer - event_buffer_.resize(0); + if (prefix_len >= PREFIX_BUF_SIZE - 1) { + // The appenders truncate silently, which would put a malformed event on the wire + ESP_LOGW(TAG, "Event name too long, dropped"); return true; } - event_buffer_.append(CRLF_STR, CRLF_LEN); + // Gather list: the prefix, then the data lines and their separators from the message + struct Gather { + struct iovec iov[MAX_SEND_IOV]; // left uninitialized on purpose + int iovcnt{1}; + size_t total{0}; + bool fits{true}; + } g; + g.total = prefix_len; + for_each_chunk_piece( + message, message_len, + [](void *ctx, const char *piece, size_t len) { + auto &g = *static_cast(ctx); + g.total += len; + if (len == 0) { + return; + } + if (g.iovcnt == MAX_SEND_IOV) { + g.fits = false; + return; + } + g.iov[g.iovcnt++] = {const_cast(piece), len}; + }, + &g); + // The header and the terminator are not part of the chunk length + write_chunk_header(prefix, g.total - CHUNK_HDR_LEN - CHUNK_END_LEN); + g.iov[0] = {prefix, prefix_len}; - // chunk length header itself and the final chunk terminating CRLF are not counted as part of the chunk - int chunk_len = event_buffer_.size() - CRLF_LEN - chunk_len_header_len; - char chunk_len_str[9]; - snprintf(chunk_len_str, 9, "%08x", chunk_len); - std::memcpy(&event_buffer_[0], chunk_len_str, 8); - - event_bytes_sent_ = 0; - process_buffer_(); - - return true; + // A message with more lines than the list holds skips straight to the tail + const ssize_t sent = g.fits ? this->send_(g.iov, g.iovcnt) : 0; + if (sent < 0) { + return false; + } + if (static_cast(sent) == g.total) { + this->send_failure_started_ms_ = 0; // progress, whichever stall clock was running + return true; + } + // The caller's buffers do not outlive this call, so keep the chunk and continue from loop() + return this->stash_chunk_(prefix, prefix_len, message, message_len, g.total, sent); } void AsyncEventSourceResponse::deferrable_send_state(void *source, const char *event_type, @@ -1165,20 +1238,22 @@ void AsyncEventSourceResponse::deferrable_send_state(void *source, const char *e ESP_LOGE(TAG, "Can't defer non-state event"); } - process_buffer_(); + drain_tail_(); process_deferred_queue_(); if (this->close_requested_) { return; } - if (!event_buffer_.empty() || !deferred_queue_.empty()) { + if (this->tail_len_ != 0 || !deferred_queue_.empty()) { // outgoing event buffer or deferred queue still not empty which means downstream tcp send buffer full, no point // trying to send first deq_push_back_with_dedup_(source, message_generator); } else { - auto message = message_generator(web_server_, source); - if (!this->try_send_nodefer(message.c_str(), message.size(), "state")) { + json::JsonBuilder builder; + message_generator(web_server_, source, builder); + // A send error closes the session and clears the queue; nothing is queued after that + if (!this->send_json_(builder) && !this->close_requested_) { deq_push_back_with_dedup_(source, message_generator); } } diff --git a/esphome/components/web_server_idf/web_server_idf.h b/esphome/components/web_server_idf/web_server_idf.h index 7894206102f..0f0977229c6 100644 --- a/esphome/components/web_server_idf/web_server_idf.h +++ b/esphome/components/web_server_idf/web_server_idf.h @@ -18,8 +18,11 @@ #ifdef USE_WEBSERVER #include "esphome/components/json/json_util.h" #include "esphome/components/web_server/list_entities.h" +#include "sse_chunk.h" #endif +struct iovec; // NOLINT(readability-identifier-naming) - forward decl of lwip's gather list entry + namespace esphome { #ifdef USE_WEBSERVER namespace web_server { @@ -255,7 +258,7 @@ class AsyncWebHandler { class AsyncEventSource; class AsyncEventSourceResponse; -using message_generator_t = json::SerializationBuffer<>(esphome::web_server::WebServer *, void *); +using message_generator_t = void(esphome::web_server::WebServer *, void *, json::JsonBuilder &); /* This class holds a pointer to the source component that wants to publish a state event, and a pointer to a function @@ -300,8 +303,30 @@ class AsyncEventSourceResponse { void deq_push_back_with_dedup_(void *source, message_generator_t *message_generator); void process_deferred_queue_(); - void process_buffer_(); + // A new chunk may go out: not re-entered from a log line, session alive, tail empty + bool ready_to_send_(); + // Non-blocking gather write. Returns bytes written, 0 on would-block or a socket that is not + // ours, -1 after requesting the close on any other error. + ssize_t send_(struct iovec *iov, int iovcnt); + // Push what is left of the chunk in tail_ to the socket; owns the stall timer. + void drain_tail_(); + // Grow tail_ to hold len bytes, kept at its high-water mark. False on OOM. + bool reserve_tail_(size_t len); + // Keep the whole chunk in tail_ and continue from sent; false when the tail cannot be allocated. + bool stash_chunk_(const char *prefix, size_t prefix_len, const char *message, size_t message_len, size_t total, + size_t sent); + // Send a state event; JSON too large for the stack buffer is serialized into tail_ instead + bool send_json_(json::JsonBuilder &builder); + // Warn once, and close the session once the stall timeout passes with no memory for the tail + void tail_alloc_failed_(size_t cap); void request_close_(); + + // A log line emitted inside a send re-enters try_send_nodefer on this session; refuse it + struct SendGuard { + AsyncEventSourceResponse &owner; + explicit SendGuard(AsyncEventSourceResponse &owner) : owner(owner) { owner.sending_ = true; } + ~SendGuard() { this->owner.sending_ = false; } + }; void process_close_(); static void close_session_work(void *arg); @@ -317,15 +342,33 @@ class AsyncEventSourceResponse { std::vector deferred_queue_; esphome::web_server::WebServer *web_server_; esphome::web_server::ListEntitiesIterator entities_iterator_; - std::string event_buffer_; - size_t event_bytes_sent_; + // One chunk the socket did not take whole, allocated on the first stall; the only heap use + // on the send path + RAMUniquePtr tail_; uint32_t send_failure_started_ms_{0}; // Zero means no send stall in progress. uint32_t next_close_attempt_ms_{0}; - // Main-loop only; the HTTPD task never reads or writes this flag. - bool close_requested_{false}; - bool close_retry_warning_logged_{false}; + uint16_t tail_cap_{0}; + uint16_t tail_len_{0}; // Zero means nothing pending + uint16_t tail_sent_{0}; // Set on the main loop before queueing close work, cleared by the HTTPD-task callback when done. std::atomic close_work_queued_{false}; + // Main-loop only; the HTTPD task never reads or writes these flags. + bool close_requested_{false}; + bool close_retry_warning_logged_{false}; + bool sending_{false}; + // The longest multi line log message in the tree (a climate dump_config) has 22 lines; a + // longer one goes through the tail + static constexpr size_t MAX_SEND_LINES = 22; + static constexpr size_t MAX_SEND_IOV = 1 + 2 * MAX_SEND_LINES; + // Chunk header, retry/id/event lines and the first "data: " + static constexpr size_t PREFIX_BUF_SIZE = 128; + // Stack buffer for a state event's JSON; a larger document is serialized into the tail + static constexpr size_t JSON_BUF_SIZE = 1024; + // Same ceiling JsonBuilder::serialize() applies (max_heap_size in json_util.cpp); a larger + // document is dropped before anything is on the wire + static constexpr size_t JSON_MAX_SIZE = 5120; + // Most RAM a stalled session keeps: the largest state document plus any accepted framing + static constexpr size_t TAIL_MAX_SIZE = JSON_MAX_SIZE + PREFIX_BUF_SIZE + SSE_SUFFIX_LEN; static constexpr uint32_t SEND_STALL_TIMEOUT_MS = 20000; static constexpr uint32_t CLOSE_RETRY_INTERVAL_MS = 250; static constexpr uint32_t CLOSE_CONFIRM_INTERVAL_MS = 1000; diff --git a/tests/components/web_server_idf/test_sse_chunk.cpp b/tests/components/web_server_idf/test_sse_chunk.cpp new file mode 100644 index 00000000000..4e34f90d774 --- /dev/null +++ b/tests/components/web_server_idf/test_sse_chunk.cpp @@ -0,0 +1,131 @@ +#include + +#include +#include +#include +#include +#include + +#include "esphome/components/web_server_idf/sse_chunk.h" + +namespace esphome::web_server_idf::testing { + +// The chunk as try_send_nodefer lays it out: header, fields, first "data: ", then the pieces +static std::string build_chunk(const char *message, size_t message_len, const char *event, uint32_t id, + uint32_t reconnect) { + char prefix[128]; + const size_t prefix_len = build_chunk_prefix(prefix, sizeof(prefix), event, id, reconnect, message != nullptr); + if (message == nullptr && prefix_len == CHUNK_HDR_LEN) { + return ""; + } + std::string out(prefix, prefix_len); + for_each_chunk_piece( + message, message_len, + [](void *ctx, const char *piece, size_t len) { static_cast(ctx)->append(piece, len); }, &out); + write_chunk_header(prefix, out.size() - CHUNK_HDR_LEN - CHUNK_END_LEN); + out.replace(0, CHUNK_HDR_LEN, prefix, CHUNK_HDR_LEN); + return out; +} + +// The std::string builder this framing replaced, kept as the reference for the wire format +static std::string reference_chunk(const char *message, size_t message_len, const char *event, uint32_t id, + uint32_t reconnect) { + std::string buf = " \r\n"; + char num[32]; + if (reconnect) + buf.append(num, snprintf(num, sizeof(num), "retry: %u\r\n", reconnect)); + if (id) + buf.append(num, snprintf(num, sizeof(num), "id: %u\r\n", id)); + if (event && *event) + buf.append("event: ").append(event).append("\r\n"); + if (message) { + const char *line_start = message; + const char *msg_end = message + message_len; + const char *next_n = static_cast(memchr(message, '\n', message_len)); + const char *next_r = static_cast(memchr(message, '\r', message_len)); + if (next_n == nullptr && next_r == nullptr) { + buf.append("data: ").append(message, message_len).append("\r\n\r\n"); + } else { + while (line_start <= msg_end) { + const char *line_end; + const char *next_line; + if (next_n == nullptr && next_r == nullptr) { + buf.append("data: ").append(line_start, msg_end - line_start).append("\r\n"); + break; + } + if (next_n != nullptr && next_r != nullptr) { + if (next_r + 1 == next_n) { + line_end = next_r; + next_line = next_n + 1; + } else { + line_end = (next_r < next_n) ? next_r : next_n; + next_line = line_end + 1; + } + } else if (next_n != nullptr) { + line_end = next_n; + next_line = next_n + 1; + } else { + line_end = next_r; + next_line = next_r + 1; + } + buf.append("data: ").append(line_start, line_end - line_start).append("\r\n"); + line_start = next_line; + if (line_start >= msg_end) + break; + next_n = static_cast(memchr(line_start, '\n', msg_end - line_start)); + next_r = static_cast(memchr(line_start, '\r', msg_end - line_start)); + } + buf.append("\r\n"); + } + } + if (buf.size() == 10) + return ""; + buf.append("\r\n"); + char len[9]; + snprintf(len, sizeof(len), "%08x", static_cast(buf.size() - 2 - 10)); + buf.replace(0, 8, len, 8); + return buf; +} + +static void expect_same(const std::string &message, const char *event, uint32_t id, uint32_t reconnect) { + const std::string got = build_chunk(message.data(), message.size(), event, id, reconnect); + const std::string want = reference_chunk(message.data(), message.size(), event, id, reconnect); + EXPECT_EQ(got, want) << "message=[" << message << "] event=" << (event ? event : "null") << " id=" << id + << " retry=" << reconnect; +} + +TEST(SseChunk, NullMessageHasNoDataLineAndNoBlankLine) { + EXPECT_EQ(build_chunk(nullptr, 0, "ping", 7, 30000), reference_chunk(nullptr, 0, "ping", 7, 30000)); + EXPECT_EQ(build_chunk(nullptr, 0, "ping", 7, 30000), "00000022\r\nretry: 30000\r\nid: 7\r\nevent: ping\r\n\r\n"); + EXPECT_EQ(build_chunk(nullptr, 0, nullptr, 0, 0), ""); +} + +TEST(SseChunk, SingleLine) { + EXPECT_EQ(build_chunk("{}", 2, "state", 0, 0), "0000001a\r\nevent: state\r\ndata: {}\r\n\r\n\r\n"); + expect_same("", "state", 0, 0); + expect_same(R"({"id":"light-x"})", "state_detail_all", 0, 0); +} + +TEST(SseChunk, LineBreaks) { + for (const char *m : {"a\n", "a\r", "a\r\n", "\n", "\r\n", "\r", "a\n\nb", "a\r\rb", "a\n\rb", "a\r\n\r\nb", "\n\n", + "x\r\n", "\r\nx", "one\ntwo\nthree", "tail\r\n\r\n"}) { + expect_same(m, "log", 1234, 0); + } + EXPECT_EQ(build_chunk("a\r\nb", 4, nullptr, 0, 0), "00000014\r\ndata: a\r\ndata: b\r\n\r\n\r\n"); + EXPECT_EQ(build_chunk("a\n\rb", 4, nullptr, 0, 0), "0000001c\r\ndata: a\r\ndata: \r\ndata: b\r\n\r\n\r\n"); +} + +TEST(SseChunk, MatchesReferenceOnRandomMessages) { + std::mt19937 rng(1234); // NOLINT(cert-msc32-c,cert-msc51-cpp,bugprone-random-generator-seed) reproducible + const char *events[] = {nullptr, "", "ping", "state", "log", "state_detail_all", "sorting_group"}; + const char alphabet[] = "ab{}\":,\n\r "; + for (int i = 0; i < 20000; i++) { + std::string m; + const size_t len = rng() % 120; + for (size_t k = 0; k < len; k++) + m += alphabet[rng() % (sizeof(alphabet) - 1)]; + expect_same(m, events[rng() % 7], (rng() % 3 == 0) ? 0 : rng(), (rng() % 4 == 0) ? 30000 : 0); + } +} + +} // namespace esphome::web_server_idf::testing From 25ddc94d5d0cec4ff74f632fac214e7358874aab Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 00:10:40 +0200 Subject: [PATCH 058/133] [web_server_idf] Build state documents in a stack arena so a state event does no heap allocation (#19383) --- esphome/components/json/__init__.py | 6 + esphome/components/json/json_util.cpp | 19 +- esphome/components/json/json_util.h | 84 ++++++- esphome/components/web_server/__init__.py | 4 + .../web_server_idf/web_server_idf.cpp | 27 +- .../web_server_idf/web_server_idf.h | 3 +- esphome/core/defines.h | 1 + tests/components/json/__init__.py | 12 + tests/components/json/test_json_arena.cpp | 238 ++++++++++++++++++ 9 files changed, 375 insertions(+), 19 deletions(-) create mode 100644 tests/components/json/test_json_arena.cpp diff --git a/esphome/components/json/__init__.py b/esphome/components/json/__init__.py index af7eb7e733d..618c56faa14 100644 --- a/esphome/components/json/__init__.py +++ b/esphome/components/json/__init__.py @@ -21,3 +21,9 @@ async def to_code(config: ConfigType) -> None: cg.add_library("bblanchon/ArduinoJson", "7.4.3") cg.add_define("USE_JSON") cg.add_global(json_ns.using) + + +def enable_arena() -> None: + """Compile JsonArena and the allocator constructor of JsonBuilder; only the consumers that build + documents in a stack arena pay for them.""" + cg.add_define("USE_JSON_ARENA") diff --git a/esphome/components/json/json_util.cpp b/esphome/components/json/json_util.cpp index 193390e4782..3ecfce8b699 100644 --- a/esphome/components/json/json_util.cpp +++ b/esphome/components/json/json_util.cpp @@ -46,11 +46,7 @@ JsonDocument parse_json(const uint8_t *data, size_t len) { ESP_LOGE(TAG, "No data to parse"); return JsonObject(); // return unbound object } -#ifdef USE_PSRAM - JsonDocument json_document(&global_json_allocator); -#else - JsonDocument json_document; -#endif + JsonDocument json_document(heap_json_allocator()); if (json_document.overflowed()) { ESP_LOGE(TAG, "Could not allocate memory for JSON document!"); return JsonObject(); // return unbound object @@ -68,7 +64,18 @@ JsonDocument parse_json(const uint8_t *data, size_t len) { // NOLINTEND(clang-analyzer-cplusplus.NewDeleteLeaks,clang-analyzer-core.StackAddressEscape) } -JsonBuilder::JsonBuilder() = default; +JsonBuilder::JsonBuilder() : doc_(heap_json_allocator()) {} +#ifdef USE_JSON_ARENA +JsonBuilder::JsonBuilder(ArduinoJson::Allocator *allocator) : doc_(allocator) {} +#endif + +ArduinoJson::Allocator *heap_json_allocator() { +#ifdef USE_PSRAM + return &global_json_allocator; +#else + return ArduinoJson::detail::DefaultAllocator::instance(); +#endif +} size_t JsonBuilder::serialize_to(char *buf, size_t cap) { if (doc_.overflowed()) { diff --git a/esphome/components/json/json_util.h b/esphome/components/json/json_util.h index 5823e993085..be12d0aa38a 100644 --- a/esphome/components/json/json_util.h +++ b/esphome/components/json/json_util.h @@ -1,5 +1,7 @@ #pragma once +#include +#include #include #include #include @@ -165,11 +167,88 @@ inline JsonDocument parse_json(const std::string &data) { return parse_json(reinterpret_cast(data.c_str()), data.size()); } +/// The allocator a JsonBuilder uses by default (PSRAM first when available) +ArduinoJson::Allocator *heap_json_allocator(); + +#ifdef USE_JSON_ARENA +/// Size of one ArduinoJson slot pool, the first allocation every document makes (1 KB on 32 bit targets) +constexpr size_t JSON_POOL_BYTES = ARDUINOJSON_POOL_CAPACITY * sizeof(ArduinoJson::detail::VariantData); +/// One pool plus room for copied string nodes; a 40 option select with linked options fits +constexpr size_t JSON_ARENA_SIZE = JSON_POOL_BYTES + 1152; +static_assert(sizeof(void *) != 4 || JSON_ARENA_SIZE == 2176, "the arena was sized for a 1 KB pool"); + +/// Bump allocator over a fixed buffer for a document built and serialized in one scope. What the +/// buffer cannot hold goes to the heap allocator; nothing is freed until the arena goes away. +template class JsonArena final : public ArduinoJson::Allocator { + public: + // Takes what the buffer cannot hold + explicit JsonArena(ArduinoJson::Allocator *fallback = heap_json_allocator()) : fallback_(fallback) {} + // The document points into buf_ + JsonArena(const JsonArena &) = delete; + JsonArena &operator=(const JsonArena &) = delete; + + void *allocate(size_t size) override { + if (size > N) { + return this->fallback_->allocate(size); // also keeps the rounding below from wrapping + } + size = (size + ALIGN - 1) & ~(ALIGN - 1); + if (size > N - this->used_) { + return this->fallback_->allocate(size); + } + this->last_ = this->used_; + this->used_ += size; + return this->buf_ + this->last_; + } + void deallocate(void *ptr) override { + if (!this->owns_(ptr)) { + this->fallback_->deallocate(ptr); + } + } + void *reallocate(void *ptr, size_t new_size) override { + if (!this->owns_(ptr)) { + return this->fallback_->reallocate(ptr, new_size); + } + const size_t off = static_cast(ptr) - this->buf_; + const size_t size = new_size > N ? N + ALIGN : (new_size + ALIGN - 1) & ~(ALIGN - 1); + const bool newest = off == this->last_; + if (newest && size <= N - off) { + this->used_ = off + size; // the newest block grows or shrinks in place + return ptr; + } + // An older block's size is unknown; copying to the end of the buffer stays in bounds + const size_t old_size = newest ? this->used_ - off : N - off; + void *moved = this->fallback_->allocate(new_size); + if (moved == nullptr) { + return nullptr; // the caller keeps ptr, so its arena space stays reserved + } + std::memcpy(moved, ptr, std::min(new_size, old_size)); + if (newest) { + this->used_ = off; // it moved to the heap, so its arena space is free again + } + return moved; + } + /// Bytes of the buffer handed out so far + size_t used() const { return this->used_; } + + private: + static constexpr size_t ALIGN = alignof(std::max_align_t); + bool owns_(const void *ptr) const { return ptr >= this->buf_ && ptr < this->buf_ + N; } + ArduinoJson::Allocator *fallback_; + alignas(ALIGN) uint8_t buf_[N]; + size_t used_{0}; + size_t last_{0}; +}; +#endif // USE_JSON_ARENA + /// Builder class for creating JSON documents without lambdas class JsonBuilder { public: // Out of line: inlining the JsonDocument constructor duplicates it at every call site JsonBuilder(); +#ifdef USE_JSON_ARENA + // The builder must not outlive the allocator + explicit JsonBuilder(ArduinoJson::Allocator *allocator); +#endif JsonObject root() { if (!root_created_) { @@ -188,12 +267,7 @@ class JsonBuilder { SerializationBuffer<> serialize(); private: -#ifdef USE_PSRAM - SpiRamAllocator allocator_; - JsonDocument doc_{&allocator_}; -#else JsonDocument doc_; -#endif JsonObject root_; bool root_created_{false}; }; diff --git a/esphome/components/web_server/__init__.py b/esphome/components/web_server/__init__.py index 3ec365cf381..2b00a6f8de1 100644 --- a/esphome/components/web_server/__init__.py +++ b/esphome/components/web_server/__init__.py @@ -8,6 +8,7 @@ from typing import Any import esphome.codegen as cg from esphome.components import web_server_base +from esphome.components.json import enable_arena from esphome.components.logger import request_log_listener from esphome.components.web_server_base import CONF_WEB_SERVER_BASE_ID import esphome.config_validation as cv @@ -387,6 +388,9 @@ async def to_code(config: ConfigType) -> None: cg.add(paren.set_port(port)) cg.add_define("USE_WEBSERVER") cg.add_define("USE_WEBSERVER_PORT", port) + if CORE.is_esp32: + # The ESP-IDF event source builds state documents in a stack arena + enable_arena() cg.add_define("USE_WEBSERVER_VERSION", version) if version >= 2: # Don't compress the index HTML as the data sizes are almost the same. diff --git a/esphome/components/web_server_idf/web_server_idf.cpp b/esphome/components/web_server_idf/web_server_idf.cpp index 837a705cd36..510b4e50f1a 100644 --- a/esphome/components/web_server_idf/web_server_idf.cpp +++ b/esphome/components/web_server_idf/web_server_idf.cpp @@ -57,6 +57,12 @@ namespace esphome::web_server_idf { static const char *const TAG = "web_server_idf"; +// Only send_json_() may hold the JSON arena: every other frame in this file is capped below one +// arena. Measured at -Os on GCC 14; newer toolchains stay checked on purpose, older ones skip it. +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 14 && defined(__OPTIMIZE_SIZE__) +#pragma GCC diagnostic error "-Wstack-usage=2048" +#endif + // Chunk size for streaming request bodies; matches the Arduino AsyncWebServer buffer size. // Buffers of this size must live on the heap - the httpd task stack is too small. static constexpr size_t RECV_CHUNK_SIZE = 1460; @@ -893,9 +899,7 @@ void AsyncEventSourceResponse::process_deferred_queue_() { } while (!deferred_queue_.empty()) { DeferredEvent &de = deferred_queue_.front(); - json::JsonBuilder builder; - de.message_generator_(web_server_, de.source_, builder); - if (this->send_json_(builder)) { + if (this->send_json_(de.source_, de.message_generator_)) { if (this->close_requested_ || deferred_queue_.empty()) { return; } @@ -1086,7 +1090,15 @@ void AsyncEventSourceResponse::loop() { this->entities_iterator_.try_advance(1); } -bool AsyncEventSourceResponse::send_json_(json::JsonBuilder &builder) { +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 14 && defined(__OPTIMIZE_SIZE__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstack-usage=" // the one frame that holds the arena and the JSON buffer +#endif +bool AsyncEventSourceResponse::send_json_(void *source, message_generator_t *generator) { + // The arena lives only in this frame, so no call chain ever holds two of them + json::JsonArena arena; + json::JsonBuilder builder(&arena); + generator(this->web_server_, source, builder); char buf[JSON_BUF_SIZE]; const size_t len = builder.serialize_to(buf, sizeof(buf)); if (len < sizeof(buf)) { @@ -1138,6 +1150,9 @@ bool AsyncEventSourceResponse::send_json_(json::JsonBuilder &builder) { drain_tail_(); return true; } +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 14 && defined(__OPTIMIZE_SIZE__) +#pragma GCC diagnostic pop +#endif void AsyncEventSourceResponse::tail_alloc_failed_(size_t cap) { // Same stall clock as a socket that stops draining, so a session cannot retry forever @@ -1250,10 +1265,8 @@ void AsyncEventSourceResponse::deferrable_send_state(void *source, const char *e // trying to send first deq_push_back_with_dedup_(source, message_generator); } else { - json::JsonBuilder builder; - message_generator(web_server_, source, builder); // A send error closes the session and clears the queue; nothing is queued after that - if (!this->send_json_(builder) && !this->close_requested_) { + if (!this->send_json_(source, message_generator) && !this->close_requested_) { deq_push_back_with_dedup_(source, message_generator); } } diff --git a/esphome/components/web_server_idf/web_server_idf.h b/esphome/components/web_server_idf/web_server_idf.h index 0f0977229c6..2b62d68790e 100644 --- a/esphome/components/web_server_idf/web_server_idf.h +++ b/esphome/components/web_server_idf/web_server_idf.h @@ -316,7 +316,7 @@ class AsyncEventSourceResponse { bool stash_chunk_(const char *prefix, size_t prefix_len, const char *message, size_t message_len, size_t total, size_t sent); // Send a state event; JSON too large for the stack buffer is serialized into tail_ instead - bool send_json_(json::JsonBuilder &builder); + bool send_json_(void *source, message_generator_t *generator); // Warn once, and close the session once the stall timeout passes with no memory for the tail void tail_alloc_failed_(size_t cap); void request_close_(); @@ -362,6 +362,7 @@ class AsyncEventSourceResponse { static constexpr size_t MAX_SEND_IOV = 1 + 2 * MAX_SEND_LINES; // Chunk header, retry/id/event lines and the first "data: " static constexpr size_t PREFIX_BUF_SIZE = 128; + // Stack buffer for a state event's JSON; a larger document is serialized into the tail static constexpr size_t JSON_BUF_SIZE = 1024; // Same ceiling JsonBuilder::serialize() applies (max_heap_size in json_util.cpp); a larger diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 3397214c9b6..6ee6f97947d 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -87,6 +87,7 @@ #define USE_INFRARED #define USE_IR_RF #define USE_JSON +#define USE_JSON_ARENA #define USE_RADIO_FREQUENCY #define USE_LIGHT #define USE_LIGHT_FLASH_TRANSITION_LENGTH diff --git a/tests/components/json/__init__.py b/tests/components/json/__init__.py index 40ec1f996ea..37e4dba021b 100644 --- a/tests/components/json/__init__.py +++ b/tests/components/json/__init__.py @@ -1,3 +1,6 @@ +import functools + +from esphome.components.json import enable_arena from tests.testing_helpers import ComponentManifestOverride @@ -7,3 +10,12 @@ def override_manifest(manifest: ComponentManifestOverride) -> None: # library registration to happen, otherwise json_util.cpp fails to find # ArduinoJson.h. manifest.enable_codegen() + # The JsonArena host test needs the arena compiled in, as a consumer would request it + real_to_code = manifest.to_code + + @functools.wraps(real_to_code) + async def to_code_with_arena(config): + await real_to_code(config) + enable_arena() + + manifest.to_code = to_code_with_arena diff --git a/tests/components/json/test_json_arena.cpp b/tests/components/json/test_json_arena.cpp new file mode 100644 index 00000000000..8d1d44d1bb0 --- /dev/null +++ b/tests/components/json/test_json_arena.cpp @@ -0,0 +1,238 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include "esphome/components/json/json_util.h" + +using esphome::json::JsonArena; +using esphome::json::JsonBuilder; + +namespace { + +constexpr size_t ALIGN = alignof(std::max_align_t); +constexpr size_t round_up(size_t n) { return (n + ALIGN - 1) & ~(ALIGN - 1); } + +// Counts what the arena could not hold +struct Counting final : ArduinoJson::Allocator { + int allocs{0}; + void *allocate(size_t n) override { + this->allocs++; + return malloc(n); // NOLINT + } + void deallocate(void *p) override { free(p); } // NOLINT + void *reallocate(void *p, size_t n) override { + this->allocs++; + return realloc(p, n); // NOLINT + } +}; + +// Refuses everything, so a spill or a move sees the heap as exhausted +struct NoMemory final : ArduinoJson::Allocator { + void *allocate(size_t) override { return nullptr; } + void deallocate(void *) override {} + void *reallocate(void *, size_t) override { return nullptr; } +}; + +template bool inside(const JsonArena &arena, const void *p) { + auto base = reinterpret_cast(&arena); + auto addr = reinterpret_cast(p); + return addr >= base && addr < base + sizeof(arena); +} + +} // namespace + +TEST(JsonArena, BumpsAlignedInsideTheBuffer) { + JsonArena<256> arena; + auto *a = static_cast(arena.allocate(10)); + auto *b = static_cast(arena.allocate(10)); + ASSERT_NE(a, nullptr); + ASSERT_NE(b, nullptr); + EXPECT_TRUE(inside(arena, a)); + EXPECT_TRUE(inside(arena, b)); + EXPECT_EQ(reinterpret_cast(a) % ALIGN, 0u); + EXPECT_EQ(static_cast(b - a), round_up(10)); + arena.deallocate(a); + arena.deallocate(b); +} + +TEST(JsonArena, SpillsToTheHeapWhenFull) { + JsonArena<64> arena; + void *a = arena.allocate(48); + void *b = arena.allocate(48); + ASSERT_NE(a, nullptr); + ASSERT_NE(b, nullptr); + EXPECT_TRUE(inside(arena, a)); + EXPECT_FALSE(inside(arena, b)); + std::memset(b, 'b', 48); + arena.deallocate(b); // routed to the heap; a mismatch would trip the sanitizer + arena.deallocate(a); +} + +TEST(JsonArena, NewestBlockGrowsAndShrinksInPlace) { + JsonArena<256> arena; + void *a = arena.allocate(16); + std::memset(a, 'x', 16); + EXPECT_EQ(arena.reallocate(a, 96), a); + EXPECT_EQ(std::memcmp(a, "xxxxxxxxxxxxxxxx", 16), 0); + EXPECT_EQ(arena.reallocate(a, 8), a); + auto *next = static_cast(arena.allocate(8)); + EXPECT_EQ(static_cast(next - static_cast(a)), round_up(8)); +} + +TEST(JsonArena, NewestBlockMovesToTheHeapAndFreesItsSpace) { + JsonArena<64> arena; + void *a = arena.allocate(32); + std::memset(a, 'q', 32); + void *moved = arena.reallocate(a, 200); + ASSERT_NE(moved, nullptr); + EXPECT_FALSE(inside(arena, moved)); + EXPECT_EQ(std::memcmp(moved, "qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq", 32), 0); + EXPECT_EQ(arena.allocate(16), a); // the space it left is handed out again + arena.deallocate(moved); +} + +TEST(JsonArena, OlderBlockMovesToTheHeapKeepingItsBytes) { + JsonArena<256> arena; + void *a = arena.allocate(16); + std::memset(a, 'a', 16); + auto *b = static_cast(arena.allocate(16)); + void *moved = arena.reallocate(a, 64); + ASSERT_NE(moved, nullptr); + EXPECT_FALSE(inside(arena, moved)); + EXPECT_EQ(std::memcmp(moved, "aaaaaaaaaaaaaaaa", 16), 0); + auto *next = static_cast(arena.allocate(8)); + EXPECT_EQ(static_cast(next - b), round_up(16)); // b's space is untouched + arena.deallocate(moved); +} + +TEST(JsonArena, HeapBlocksReallocateOnTheHeap) { + JsonArena<32> arena; + void *a = arena.allocate(64); // never fit + EXPECT_FALSE(inside(arena, a)); + std::memset(a, 'h', 64); + void *grown = arena.reallocate(a, 128); + ASSERT_NE(grown, nullptr); + EXPECT_EQ(std::memcmp(grown, "hhhhhhhhhhhhhhhh", 16), 0); + arena.deallocate(grown); +} + +TEST(JsonArena, FailedMoveKeepsTheBlockReserved) { + NoMemory no_memory; + JsonArena<64> arena(&no_memory); + void *a = arena.allocate(32); + std::memset(a, 'k', 32); + EXPECT_EQ(arena.reallocate(a, 200), nullptr); + EXPECT_EQ(std::memcmp(a, "kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk", 32), 0); + auto *b = static_cast(arena.allocate(16)); // must not hand out a's bytes again + ASSERT_NE(b, nullptr); + EXPECT_EQ(static_cast(b - static_cast(a)), round_up(32)); + EXPECT_EQ(arena.allocate(64), nullptr); // nothing left and the fallback refuses +} + +// NOLINTBEGIN(clang-analyzer-cplusplus.NewDeleteLeaks) false positive with ArduinoJson +constexpr size_t ARENA = esphome::json::JSON_ARENA_SIZE; + +// The documents the event stream sends must fit without touching the fallback, and their copied +// strings must land in the headroom above the pool +TEST(JsonArena, StateDocumentsFitWithoutTouchingTheFallback) { + Counting counting; + JsonArena arena(&counting); + { + // A switch state event: copied id, domain and name, bool value + JsonBuilder builder(&arena); + JsonObject root = builder.root(); + char id_buf[] = "switch/SSE Toggle"; + char domain_buf[] = "switch"; + char name_buf[] = "SSE Toggle"; + root["id"] = static_cast(id_buf); + root["domain"] = static_cast(domain_buf); + root["name"] = static_cast(name_buf); + root["icon"] = ""; + root["entity_category"] = 0; + root["value"] = true; + root["state"] = "ON"; + root["assumed_state"] = false; + char out[256]; + EXPECT_LT(builder.serialize_to(out, sizeof(out)), sizeof(out)); + } + EXPECT_EQ(counting.allocs, 0); + EXPECT_GT(arena.used(), esphome::json::JSON_POOL_BYTES); + + Counting counting_select; + JsonArena select_arena(&counting_select); + { + // A 40 option select detail document: copied id, domain, name and value, linked options + JsonBuilder builder(&select_arena); + JsonObject root = builder.root(); + char id_buf[] = "select/SSE Big Select"; + char domain_buf[] = "select"; + char name_buf[] = "SSE Big Select"; + char value_buf[] = "option number 17 padded to twenty"; + root["id"] = static_cast(id_buf); + root["domain"] = static_cast(domain_buf); + root["name"] = static_cast(name_buf); + root["icon"] = ""; + root["entity_category"] = 0; + root["value"] = static_cast(value_buf); + root["state"] = static_cast(value_buf); + JsonArray options = root["option"].to(); + char option_bufs[40][34]; + for (int i = 0; i < 40; i++) { + snprintf(option_bufs[i], sizeof(option_bufs[i]), "option number %02d padded to twenty", i); + options.add(JsonString(option_bufs[i], true)); + } + char out[2048]; + EXPECT_LT(builder.serialize_to(out, sizeof(out)), sizeof(out)); + } + EXPECT_EQ(counting_select.allocs, 0); + EXPECT_GT(select_arena.used(), esphome::json::JSON_POOL_BYTES); + + // The same select with its 40 options copied, as the generator does before the strings are + // linked, does not fit: the headroom is sized for linked options and the rest spills + Counting counting_copied; + JsonArena copied_arena(&counting_copied); + { + JsonBuilder builder(&copied_arena); + JsonArray options = builder.root()["option"].to(); + char option_bufs[40][34]; + for (int i = 0; i < 40; i++) { + snprintf(option_bufs[i], sizeof(option_bufs[i]), "option number %02d padded to twenty", i); + options.add(static_cast(option_bufs[i])); + } + char out[2048]; + EXPECT_LT(builder.serialize_to(out, sizeof(out)), sizeof(out)); + } + EXPECT_GT(counting_copied.allocs, 0); + EXPECT_GT(copied_arena.used(), ARENA - 64); // the arena filled up before the spill began +} + +TEST(JsonArena, DocumentMatchesTheHeapAllocator) { + // 700 integers need six pools, which also grows ArduinoJson's pool list past its preallocated four + auto build = [](JsonBuilder &builder) { + JsonArray arr = builder.root()["a"].to(); + for (int i = 0; i < 700; i++) { + arr.add(i); + } + JsonArray strings = builder.root()["s"].to(); + char buf[32]; + for (int i = 0; i < 60; i++) { + snprintf(buf, sizeof(buf), "string number %04d padded", i); + strings.add(buf); + } + }; + JsonArena arena; + JsonBuilder with_arena(&arena); + build(with_arena); + JsonBuilder with_heap; + build(with_heap); + std::string a = with_arena.serialize(); + std::string b = with_heap.serialize(); + EXPECT_GT(a.size(), 4000u); + EXPECT_EQ(a, b); +} +// NOLINTEND(clang-analyzer-cplusplus.NewDeleteLeaks) From 7187acec04495a63f960e249c2eecc0d09497d6d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 00:15:14 +0200 Subject: [PATCH 059/133] [core] Add a line based yaml editor and use it for rename (#19410) --- esphome/__main__.py | 154 +---- esphome/cli/__init__.py | 2 + esphome/cli/rename.py | 164 ++++++ esphome/util.py | 4 + esphome/yaml_edit.py | 114 ++++ tests/unit_tests/cli/__init__.py | 0 tests/unit_tests/cli/test_rename.py | 839 ++++++++++++++++++++++++++++ tests/unit_tests/test_main.py | 631 +-------------------- tests/unit_tests/test_yaml_edit.py | 182 ++++++ 9 files changed, 1316 insertions(+), 774 deletions(-) create mode 100644 esphome/cli/__init__.py create mode 100644 esphome/cli/rename.py create mode 100644 esphome/yaml_edit.py create mode 100644 tests/unit_tests/cli/__init__.py create mode 100644 tests/unit_tests/cli/test_rename.py create mode 100644 tests/unit_tests/test_yaml_edit.py diff --git a/esphome/__main__.py b/esphome/__main__.py index 01ca8ca5d96..49fc7a3020f 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -18,7 +18,6 @@ from typing import TYPE_CHECKING, Protocol from esphome import const, platform_hooks from esphome.build_helpers.native import analysis_backend, native_backend from esphome.const import ( - ALLOWED_NAME_CHARS, ARGUMENT_HELP_DEVICE, BUNDLE_EXTENSION, CONF_API, @@ -36,7 +35,6 @@ from esphome.const import ( CONF_LOGGER, CONF_MDNS, CONF_MQTT, - CONF_NAME, CONF_NAME_ADD_MAC_SUFFIX, CONF_OTA, CONF_PASSWORD, @@ -62,6 +60,7 @@ from esphome.stacktrace import LogLineProcessor from esphome.types import ConfigType from esphome.upload_targets import PortType, get_port_type from esphome.util import ( + ESPHOME_COMMAND, PICOTOOL_PACKAGE, FlashImage, detect_rp2040_bootsel, @@ -85,7 +84,6 @@ if TYPE_CHECKING: _LOGGER = logging.getLogger(__name__) -ESPHOME_COMMAND = [sys.executable, "-m", "esphome"] # Maximum buffer size for serial log reading to prevent unbounded memory growth SERIAL_BUFFER_MAX_SIZE = 65536 @@ -2104,155 +2102,9 @@ def command_analyze_memory(args: ArgsProtocol, config: ConfigType) -> int: def command_rename(args: ArgsProtocol, config: ConfigType) -> int | None: - from esphome import yaml_util + from esphome.cli.rename import command_rename as run - new_name = args.name - for c in new_name: - if c not in ALLOWED_NAME_CHARS: - safe_print( - color( - AnsiFore.BOLD_RED, - f"'{c}' is an invalid character for names. Valid characters are: " - f"{ALLOWED_NAME_CHARS} (lowercase, no spaces)", - ) - ) - return 1 - # Load existing yaml file - raw_contents = CORE.config_path.read_text(encoding="utf-8") - - yaml = yaml_util.load_yaml(CORE.config_path) - if CONF_ESPHOME not in yaml or CONF_NAME not in yaml[CONF_ESPHOME]: - safe_print( - color( - AnsiFore.BOLD_RED, "Complex YAML files cannot be automatically renamed." - ) - ) - return 1 - old_name = yaml[CONF_ESPHOME][CONF_NAME] - match = re.match(r"^\$\{?([a-zA-Z0-9_]+)\}?$", old_name) - if match is None: - # Only swap the ``name:`` line that sits directly under the - # top-level ``esphome:`` block. A naked ``re.sub`` would - # also clobber any other ``name:`` line whose value happens - # to match (e.g. a sensor / output / wifi entry sharing the - # device's hostname), silently rewriting unrelated user - # configuration. The pattern anchors: - # - at the start of the line so ``friendly_name:``, - # ``device_name:`` etc. don't match the trailing ``name:`` - # substring; and - # - at the end of the value (lookahead for whitespace + - # comment + EOL) so ``old_name`` doesn't match as a - # prefix of a longer value (``kitchen`` vs ``kitchen2``). - name_pattern = re.compile( - rf"^(\s*)name:\s+[\"']?{re.escape(old_name)}[\"']?(?=\s*(?:#|$))" - ) - out_lines: list[str] = [] - in_esphome_block = False - for line in raw_contents.splitlines(keepends=True): - if line and not line[0].isspace() and line.strip(): - in_esphome_block = line.lstrip().startswith("esphome:") - out_lines.append(line) - continue - if in_esphome_block: - line = name_pattern.sub(rf'\1name: "{new_name}"', line, count=1) - out_lines.append(line) - new_raw = "".join(out_lines) - else: - old_name = yaml[CONF_SUBSTITUTIONS][match.group(1)] - if ( - len( - re.findall( - rf"^\s+{match.group(1)}:\s+[\"']?{old_name}[\"']?", - raw_contents, - flags=re.MULTILINE, - ) - ) - > 1 - ): - safe_print( - color(AnsiFore.BOLD_RED, "Too many matches in YAML to safely rename") - ) - return 1 - - new_raw = re.sub( - rf"^(\s+{match.group(1)}):\s+[\"']?{old_name}[\"']?", - f'\\1: "{new_name}"', - raw_contents, - flags=re.MULTILINE, - ) - - # ``new_name == old_name`` (after substitution resolution) is - # a no-op rewrite that would still queue a pointless re-flash. - # Catch it before the path-equality check below — covers the - # case where the config filename doesn't match the device name - # (e.g. ``weird-file.yaml`` whose ``esphome.name`` is - # ``kitchen``; running ``esphome rename weird-file.yaml kitchen`` - # would otherwise just re-flash the same hostname). - if new_name == old_name: - safe_print( - color( - AnsiFore.BOLD_RED, - f"'{new_name}' is already the device's name.", - ) - ) - return 1 - - new_path: Path = CORE.config_dir / (new_name + ".yaml") - if new_path.resolve() == CORE.config_path.resolve(): - safe_print( - color( - AnsiFore.BOLD_RED, - f"'{new_name}' is already the device's name.", - ) - ) - return 1 - if new_path.exists(): - safe_print( - color( - AnsiFore.BOLD_RED, - f"Cannot rename: {new_path} already exists. " - "Refusing to overwrite an existing configuration.", - ) - ) - return 1 - safe_print( - f"Updating {color(AnsiFore.CYAN, str(CORE.config_path))} to {color(AnsiFore.CYAN, str(new_path))}" - ) - print() - - new_path.write_text(new_raw, encoding="utf-8") - - rc = run_external_process(*ESPHOME_COMMAND, "config", str(new_path)) - if rc != 0: - safe_print(color(AnsiFore.BOLD_RED, "Rename failed. Reverting changes.")) - new_path.unlink() - return 1 - - cli_args = [ - "run", - str(new_path), - "--no-logs", - "--device", - CORE.address, - ] - - if args.dashboard: - cli_args.insert(0, "--dashboard") - - try: - rc = run_external_process(*ESPHOME_COMMAND, *cli_args) - except KeyboardInterrupt: - rc = 1 - if rc != 0: - new_path.unlink() - return 1 - - if CORE.config_path != new_path: - CORE.config_path.unlink() - - safe_print(color(AnsiFore.BOLD_GREEN, "SUCCESS")) - print() - return 0 + return run(args, config) PRE_CONFIG_ACTIONS = { diff --git a/esphome/cli/__init__.py b/esphome/cli/__init__.py new file mode 100644 index 00000000000..fa6e1fa266c --- /dev/null +++ b/esphome/cli/__init__.py @@ -0,0 +1,2 @@ +"""Commands of the esphome command line, one module each, imported by +__main__ only when they run so that startup stays light.""" diff --git a/esphome/cli/rename.py b/esphome/cli/rename.py new file mode 100644 index 00000000000..cbef68fb70f --- /dev/null +++ b/esphome/cli/rename.py @@ -0,0 +1,164 @@ +"""``esphome rename``.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import re + +from esphome import yaml_edit, yaml_util +from esphome.const import ( + ALLOWED_NAME_CHARS, + CONF_ESPHOME, + CONF_NAME, + CONF_SUBSTITUTIONS, +) +from esphome.core import CORE, EsphomeError +from esphome.log import AnsiFore, color +from esphome.types import ConfigType +from esphome.util import ESPHOME_COMMAND, run_external_process, safe_print + + +def _revert(new_path: Path, why: str) -> int: + """Say why the rename stopped and take the new file back; an orphan the + next attempt would trip over is reported.""" + safe_print(color(AnsiFore.BOLD_RED, f"Rename failed: {why}")) + try: + new_path.unlink(missing_ok=True) + except OSError as err: + safe_print(color(AnsiFore.BOLD_RED, f"Could not remove {new_path}: {err}")) + return 1 + + +def command_rename(args: argparse.Namespace, config: ConfigType) -> int | None: + """Rename the device: a new file with the name line rewritten, validated + and installed, then the old file removed.""" + new_name = args.name + for c in new_name: + if c not in ALLOWED_NAME_CHARS: + safe_print( + color( + AnsiFore.BOLD_RED, + f"'{c}' is an invalid character for names. Valid characters are: " + f"{ALLOWED_NAME_CHARS} (lowercase, no spaces)", + ) + ) + return 1 + + yaml = yaml_util.load_yaml(CORE.config_path) + + def name_edit() -> tuple[str, yaml_edit.LineEdit]: + """The name and the line to rewrite: the name's own line, or the + substitution's line it comes from, as a plain value in this file.""" + esphome_conf = yaml.get(CONF_ESPHOME) + if not isinstance(esphome_conf, dict) or CONF_NAME not in esphome_conf: + raise EsphomeError(f"no '{CONF_ESPHOME}: {CONF_NAME}:' in the file") + old_name = str(esphome_conf[CONF_NAME]) + mapping, field = esphome_conf, CONF_NAME + if match := re.match(r"^\$\{?([a-zA-Z0-9_]+)\}?$", old_name): + mapping, field = yaml.get(CONF_SUBSTITUTIONS), match.group(1) + if not isinstance(mapping, dict) or field not in mapping: + raise EsphomeError(f"the substitution '{field}' is not in the file") + old_name = str(mapping[field]) + # Only read here; the rewritten text goes to a new file, so the + # source may live anywhere the config path points to + source = yaml_edit.source_of(mapping, field) + if source is None or source[0].resolve() != CORE.config_path.resolve(): + raise EsphomeError(f"'{field}' was not read from {CORE.config_path}") + doc, line_no = source + text = yaml_edit.line_at(doc, line_no) + if (line_match := yaml_edit.field_line_re(field, old_name).match(text)) is None: + raise EsphomeError(f"'{field}' is not a plain value on {doc}:{line_no + 1}") + # The new value is always quoted, whatever the old line had + return old_name, yaml_edit.LineEdit( + doc, line_no, text, yaml_edit.rewrite(line_match, new_name, '"') + ) + + try: + old_name, edit = name_edit() + except EsphomeError as err: + safe_print( + color( + AnsiFore.BOLD_RED, + f"Complex YAML files cannot be automatically renamed: {err}", + ) + ) + return 1 + + # ``new_name == old_name`` (after substitution resolution) is + # a no-op rewrite that would still queue a pointless re-flash. + # Catch it before the path-equality check below — covers the + # case where the config filename doesn't match the device name + # (e.g. ``weird-file.yaml`` whose ``esphome.name`` is + # ``kitchen``; running ``esphome rename weird-file.yaml kitchen`` + # would otherwise just re-flash the same hostname). + if new_name == old_name: + safe_print( + color( + AnsiFore.BOLD_RED, + f"'{new_name}' is already the device's name.", + ) + ) + return 1 + + new_path: Path = CORE.config_dir / (new_name + ".yaml") + if new_path.resolve() == CORE.config_path.resolve(): + safe_print( + color( + AnsiFore.BOLD_RED, + f"'{new_name}' is already the device's name.", + ) + ) + return 1 + if new_path.exists(): + safe_print( + color( + AnsiFore.BOLD_RED, + f"Cannot rename: {new_path} already exists. " + "Refusing to overwrite an existing configuration.", + ) + ) + return 1 + safe_print( + f"Updating {color(AnsiFore.CYAN, str(CORE.config_path))} to {color(AnsiFore.CYAN, str(new_path))}" + ) + print() + + try: + yaml_edit.write_keeping_mode( + new_path, + yaml_edit.rewritten_text(yaml_edit.read_text(CORE.config_path), [edit]), + like=CORE.config_path, + ) + except EsphomeError as err: + return _revert(new_path, str(err)) + + if run_external_process(*ESPHOME_COMMAND, "config", str(new_path)) != 0: + return _revert(new_path, "the new configuration does not validate") + + cli_args = [ + "run", + str(new_path), + "--no-logs", + "--device", + CORE.address, + ] + + if args.dashboard: + cli_args.insert(0, "--dashboard") + + try: + rc = run_external_process(*ESPHOME_COMMAND, *cli_args) + except KeyboardInterrupt: + rc = 1 + if rc != 0: + return _revert( + new_path, + "the install did not finish; the device may already run the new name", + ) + + CORE.config_path.unlink() + + safe_print(color(AnsiFore.BOLD_GREEN, "SUCCESS")) + print() + return 0 diff --git a/esphome/util.py b/esphome/util.py index 8aa321d908c..dd1998a7c29 100644 --- a/esphome/util.py +++ b/esphome/util.py @@ -352,6 +352,10 @@ def run_external_command( return retval +# How a command starts another esphome, as a child of this one +ESPHOME_COMMAND = [sys.executable, "-m", "esphome"] + + def run_external_process(*cmd: str, **kwargs: Any) -> int | str: # Deferred: an OTA upload/logs run never spawns an external process. import subprocess diff --git a/esphome/yaml_edit.py b/esphome/yaml_edit.py new file mode 100644 index 00000000000..da66fa088fc --- /dev/null +++ b/esphome/yaml_edit.py @@ -0,0 +1,114 @@ +"""Rewrite single lines of a yaml file in place, located by the source +ranges the loader keeps, so quotes, comments, indentation and line endings +around them survive and nothing else in the file is touched.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +import re +import stat + +from esphome.core import EsphomeError +from esphome.helpers import write_file +from esphome.types import ConfigType + +# A plain scalar with no yaml indicator, so an empty value, a block scalar +# (`>-`, `|`) or a flow collection never counts; then optional matching +# quotes and the trailer, where a comment needs whitespace before its `#` +PLAIN_SCALAR = r"[^\s#\"'>|&*!%@`\[\]{},]+" +TRAILER = r"(?P[\"']?){value}(?P=quote)(?P(?:\s+#.*)?\s*)$" + + +def read_text(path: Path) -> str: + """The file as written, line endings included; read_file would fold them.""" + try: + return path.read_bytes().decode("utf-8") + except (OSError, UnicodeDecodeError) as err: + raise EsphomeError(f"Error reading file {path}: {err}") from err + + +@dataclass +class LineEdit: + """One line to rewrite; ``old_line`` is what it held when located.""" + + path: Path + line: int + old_line: str + new_line: str + + +def field_line_re( + name: str, value: str | None = None, indent: str = r"\s*" +) -> re.Pattern[str]: + """Match ``name: value``, or ``name:`` with any plain scalar, at + ``indent``; the name may be quoted. Keeps the prefix, the quotes and the + trailer for rewrite.""" + scalar = PLAIN_SCALAR if value is None else re.escape(value) + prefix = rf"{indent}[\"']?{re.escape(name)}[\"']?\s*:\s*" + return re.compile(rf"^(?P{prefix}){TRAILER.format(value=scalar)}") + + +def rewrite(match: re.Match[str], value: str, quote: str | None = None) -> str: + """The matched line with ``value`` in place of the scalar; the source + quotes stay unless ``quote`` is given.""" + quote = match["quote"] if quote is None else quote + return f"{match['prefix']}{quote}{value}{quote}{match['trail']}" + + +def line_at(doc: Path, line_no: int) -> str: + lines = read_text(doc).splitlines() + if line_no >= len(lines): + raise EsphomeError(f"{doc}:{line_no + 1} changed since it was read") + return lines[line_no] + + +def source_of(mapping: ConfigType, name: str) -> tuple[Path, int] | None: + """The file and line ``name:`` was read from, None when validation added + it or a merge key brought it in from an anchor elsewhere. Mapping keys + keep their range through validation, values may not.""" + rng = getattr(next((k for k in mapping if k == name), None), "esp_range", None) + if rng is None: + return None + # An included mapping carries the `!include` line of its parent, so only + # a key in the same document can be placed against the mapping + if (own := getattr(mapping, "esp_range", None)) is None: + return None # a mapping built in code, its keys are not on its lines + if ( + rng.start_mark.document == own.start_mark.document + and not own.start_mark.line <= rng.start_mark.line <= own.end_mark.line + ): + return None + return Path(rng.start_mark.document), rng.start_mark.line + + +def write_keeping_mode(path: Path, text: str, like: Path | None = None) -> None: + """Write with the mode of ``like`` (default: the file itself) rather + than write_file's 0644; a 0600 secrets file stays 0600.""" + try: + mode = stat.S_IMODE((like or path).stat().st_mode) + except OSError as err: + raise EsphomeError(f"Could not read the mode of {like or path}: {err}") from err + try: + write_file(path, text, private=True) + except EsphomeError as err: + # write_file keeps the reason in the cause only + raise EsphomeError(f"{err}: {err.__cause__}") from err + try: + path.chmod(mode) + except OSError as err: + raise EsphomeError( + f"{path} was written but could not get its mode back: {err}" + ) from err + + +def rewritten_text(original: str, edits: list[LineEdit]) -> str: + """``original`` with the edits applied; every edit must still find the + line it was located on.""" + lines = original.splitlines(keepends=True) + for edit in edits: + text = lines[edit.line].rstrip("\r\n") if edit.line < len(lines) else None + if text != edit.old_line: + raise EsphomeError(f"{edit.path}:{edit.line + 1} changed since it was read") + lines[edit.line] = edit.new_line + lines[edit.line][len(text) :] + return "".join(lines) diff --git a/tests/unit_tests/cli/__init__.py b/tests/unit_tests/cli/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit_tests/cli/test_rename.py b/tests/unit_tests/cli/test_rename.py new file mode 100644 index 00000000000..944d1e37ca9 --- /dev/null +++ b/tests/unit_tests/cli/test_rename.py @@ -0,0 +1,839 @@ +"""Tests for ``esphome rename``.""" + +from __future__ import annotations + +from collections.abc import Generator +from dataclasses import dataclass +from pathlib import Path +import sys +from typing import Any +from unittest.mock import Mock, patch + +import pytest +from pytest import CaptureFixture + +from esphome.cli.rename import command_rename +from esphome.const import CONF_ESPHOME, CONF_NAME, CONF_SUBSTITUTIONS +from esphome.core import CORE + + +@dataclass +class MockArgs: + name: str | None = None + dashboard: bool = False + + +def setup_core(tmp_path: Path, config: dict[str, Any] | None = None) -> None: + """Point CORE at a config in ``tmp_path``; the tests override the path.""" + CORE.config = config or {} + CORE.config_path = tmp_path / "test.yaml" + CORE.name = "test" + + +@pytest.fixture +def mock_run_external_process() -> Generator[Mock]: + """The child esphome the command starts to validate and install.""" + with patch("esphome.cli.rename.run_external_process") as mock: + mock.return_value = 0 + yield mock + + +def test_command_rename_invalid_characters( + tmp_path: Path, capfd: CaptureFixture[str] +) -> None: + """Test command_rename with invalid characters in name.""" + setup_core(tmp_path=tmp_path) + + # Test with invalid character (space) + args = MockArgs(name="invalid name") + result = command_rename(args, {}) + + assert result == 1 + captured = capfd.readouterr() + assert "invalid character" in captured.out.lower() + + +def test_command_rename_complex_yaml( + tmp_path: Path, capfd: CaptureFixture[str] +) -> None: + """Test command_rename with complex YAML that cannot be renamed.""" + config_file = tmp_path / "test.yaml" + config_file.write_text("# Complex YAML without esphome section\nsome_key: value\n") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + + args = MockArgs(name="newname") + result = command_rename(args, {}) + + assert result == 1 + captured = capfd.readouterr() + assert "complex yaml" in captured.out.lower() + + +def test_command_rename_success( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, +) -> None: + """Test successful rename of a simple configuration.""" + config_file = tmp_path / "oldname.yaml" + config_file.write_text(""" +esphome: + name: oldname + +esp32: + board: nodemcu-32s + +wifi: + ssid: "test" + password: "test1234" +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + + # Set up CORE.config to avoid ValueError when accessing CORE.address + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + + args = MockArgs(name="newname", dashboard=False) + + # Simulate successful validation and upload + mock_run_external_process.return_value = 0 + + result = command_rename(args, {}) + + assert result == 0 + + # Verify new file was created + new_file = tmp_path / "newname.yaml" + assert new_file.exists() + + # Verify old file was removed + assert not config_file.exists() + + # Verify content was updated + content = new_file.read_text() + assert ( + 'name: "newname"' in content + or "name: 'newname'" in content + or "name: newname" in content + ) + + captured = capfd.readouterr() + assert "SUCCESS" in captured.out + + +def test_command_rename_with_substitutions( + tmp_path: Path, + mock_run_external_process: Mock, +) -> None: + """Test rename with substitutions in YAML.""" + config_file = tmp_path / "oldname.yaml" + config_file.write_text(""" +substitutions: + device_name: oldname + +esphome: + name: ${device_name} + +esp32: + board: nodemcu-32s +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + + # Set up CORE.config to avoid ValueError when accessing CORE.address + CORE.config = { + CONF_ESPHOME: {CONF_NAME: "oldname"}, + CONF_SUBSTITUTIONS: {"device_name": "oldname"}, + } + + args = MockArgs(name="newname", dashboard=False) + + mock_run_external_process.return_value = 0 + + result = command_rename(args, {}) + + assert result == 0 + + # Verify substitution was updated + new_file = tmp_path / "newname.yaml" + content = new_file.read_text() + assert 'device_name: "newname"' in content + + +def test_command_rename_validation_failure( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, +) -> None: + """Test rename when validation fails.""" + config_file = tmp_path / "oldname.yaml" + config_file.write_text(""" +esphome: + name: oldname + +esp32: + board: nodemcu-32s +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + + args = MockArgs(name="newname", dashboard=False) + + # First call for validation fails + mock_run_external_process.return_value = 1 + + result = command_rename(args, {}) + + assert result == 1 + + # Verify new file was created but then removed due to failure + new_file = tmp_path / "newname.yaml" + assert not new_file.exists() + + # Verify old file still exists (not removed on failure) + assert config_file.exists() + + captured = capfd.readouterr() + assert "Rename failed" in captured.out + + +def test_command_rename_install_failure_reverts( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, +) -> None: + """Test rename when the install (esphome run) step fails.""" + config_file = tmp_path / "oldname.yaml" + config_file.write_text(""" +esphome: + name: oldname + +esp32: + board: nodemcu-32s +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + + args = MockArgs(name="newname", dashboard=False) + + # First call (config validation) succeeds; second (esphome run) fails. + mock_run_external_process.side_effect = [0, 1] + + result = command_rename(args, {}) + + assert result == 1 + + # New file was unlinked when install failed. + new_file = tmp_path / "newname.yaml" + assert not new_file.exists() + + # Old file is preserved so the device stays reachable under the + # original hostname. + assert config_file.exists() + + +def test_command_rename_target_exists_refuses( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, +) -> None: + """Test rename refuses when the target filename already exists. + + Without this guard, the rename would overwrite the unrelated + device's YAML and OTA-install our firmware to the wrong device. + """ + config_file = tmp_path / "oldname.yaml" + config_file.write_text(""" +esphome: + name: oldname + +esp32: + board: nodemcu-32s +""") + target_file = tmp_path / "newname.yaml" + target_file.write_text(""" +esphome: + name: someoneelse + +esp32: + board: nodemcu-32s +""") + target_original = target_file.read_text() + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + + args = MockArgs(name="newname", dashboard=False) + + result = command_rename(args, {}) + + assert result == 1 + # No subprocess work happened — refusal is up-front. + mock_run_external_process.assert_not_called() + # Target file untouched: same content, still on disk. + assert target_file.exists() + assert target_file.read_text() == target_original + # Source file untouched. + assert config_file.exists() + + captured = capfd.readouterr() + assert "already exists" in captured.out + + +def test_command_rename_same_name_refuses( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, +) -> None: + """Test rename refuses when the new name matches the current name. + + A same-name rename would otherwise re-write the YAML and queue + a redundant compile + install — wasted work the user almost + certainly didn't intend. + """ + config_file = tmp_path / "samename.yaml" + config_file.write_text(""" +esphome: + name: samename + +esp32: + board: nodemcu-32s +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "samename"}} + + args = MockArgs(name="samename", dashboard=False) + + result = command_rename(args, {}) + + assert result == 1 + mock_run_external_process.assert_not_called() + # File preserved verbatim — no rewrite happened. + assert config_file.exists() + + captured = capfd.readouterr() + assert "already" in captured.out.lower() + + +def test_command_rename_does_not_touch_friendly_name_substring( + tmp_path: Path, + mock_run_external_process: Mock, +) -> None: + r"""Test rename does not match the ``name:`` substring of ``friendly_name:``. + + Without anchoring the regex at line start, the pattern + ``\s*name:\s+`` could match the trailing ``name:`` + substring inside ``friendly_name: ``. The rewrite would + flip both lines to the new name, leaving the user with a + silently corrupted ``friendly_name``. + """ + config_file = tmp_path / "oldname.yaml" + config_file.write_text(""" +esphome: + name: oldname + friendly_name: oldname + +esp32: + board: nodemcu-32s +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + + args = MockArgs(name="newname", dashboard=False) + mock_run_external_process.return_value = 0 + + result = command_rename(args, {}) + + assert result == 0 + new_file = tmp_path / "newname.yaml" + content = new_file.read_text() + # esphome.name swapped. + assert 'name: "newname"' in content + # friendly_name kept verbatim. + assert "friendly_name: oldname" in content + + +def test_command_rename_does_not_match_old_name_as_value_prefix( + tmp_path: Path, + mock_run_external_process: Mock, +) -> None: + r"""Test rename does not match ``old_name`` as a prefix of a longer value. + + With ``old_name = kitchen`` the value ``kitchen2`` (a sensor + or wifi entry) would otherwise match the unanchored + ``["']?kitchen["']?`` pattern at the prefix and get + rewritten to the new name. The end-of-value lookahead keeps + the match restricted to whole tokens. + """ + config_file = tmp_path / "kitchen.yaml" + config_file.write_text(""" +esphome: + name: kitchen + +esp32: + board: nodemcu-32s + +wifi: + ap: + ssid: kitchen2 +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "kitchen"}} + + args = MockArgs(name="garage", dashboard=False) + mock_run_external_process.return_value = 0 + + result = command_rename(args, {}) + + assert result == 0 + new_file = tmp_path / "garage.yaml" + content = new_file.read_text() + assert 'name: "garage"' in content + # The wifi ssid value is unrelated and stays intact. + assert "ssid: kitchen2" in content + + +def test_command_rename_same_resolved_name_refuses( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, +) -> None: + """Test rename refuses when ``new_name`` matches the resolved device name. + + The path-equality check only catches the case where the + config filename matches the device name. For a config whose + filename and ``esphome.name`` differ (here ``weird-file.yaml`` + holds ``esphome.name: kitchen``), running + ``esphome rename weird-file.yaml kitchen`` would otherwise + fall through to the rewrite + install: the YAML's name stays + ``kitchen``, the file is renamed to ``kitchen.yaml``, and the + device gets a redundant flash. Refuse up-front so the + "already the device's name" message matches reality. + """ + config_file = tmp_path / "weird-file.yaml" + config_file.write_text(""" +esphome: + name: kitchen + +esp32: + board: nodemcu-32s +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "kitchen"}} + + args = MockArgs(name="kitchen", dashboard=False) + + result = command_rename(args, {}) + + assert result == 1 + mock_run_external_process.assert_not_called() + # Source file untouched, no derived target written. + assert config_file.exists() + assert not (tmp_path / "kitchen.yaml").exists() + + captured = capfd.readouterr() + assert "already" in captured.out.lower() + + +def test_command_rename_target_path_equals_source_refuses( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, +) -> None: + """Test rename refuses when the new path resolves to the source file. + + Reachable only when the YAML's filename and ``esphome.name`` + disagree — here ``kitchen.yaml`` holds ``esphome.name: garage`` + and the user runs ``esphome rename kitchen.yaml kitchen``. The + name-equality check above passes (``garage != kitchen``), but + ``/kitchen.yaml`` resolves to the source file + itself, so the rewrite would clobber the source mid-rename. + Refuse rather than silently overwriting. + """ + config_file = tmp_path / "kitchen.yaml" + config_file.write_text(""" +esphome: + name: garage + +esp32: + board: nodemcu-32s +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "garage"}} + + args = MockArgs(name="kitchen", dashboard=False) + + result = command_rename(args, {}) + + assert result == 1 + mock_run_external_process.assert_not_called() + # Source file still present and unmodified. + assert config_file.exists() + assert "name: garage" in config_file.read_text() + + captured = capfd.readouterr() + assert "already" in captured.out.lower() + + +def test_command_rename_does_not_touch_lookalike_name_in_other_blocks( + tmp_path: Path, + mock_run_external_process: Mock, +) -> None: + """Test rename only swaps the esphome.name line. + + A device whose name happens to match a sensor's / output's + ``name:`` value must not have those other names rewritten — + they're independent. Without an anchor for the esphome block + a naive regex would clobber every line whose value matches. + """ + config_file = tmp_path / "kitchen.yaml" + config_file.write_text(""" +esphome: + name: kitchen + +esp32: + board: nodemcu-32s + +sensor: + - platform: template + name: kitchen + lambda: 'return 0;' +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "kitchen"}} + + args = MockArgs(name="garage", dashboard=False) + mock_run_external_process.return_value = 0 + + result = command_rename(args, {}) + + assert result == 0 + + new_file = tmp_path / "garage.yaml" + content = new_file.read_text() + # esphome.name renamed. + assert 'name: "garage"' in content + # Sensor's name is the user's entity name — must not be touched. + assert " name: kitchen\n" in content + + +def test_command_rename_preserves_trailing_comment( + tmp_path: Path, + mock_run_external_process: Mock, +) -> None: + """Test rename preserves a trailing ``# comment`` on the name line.""" + config_file = tmp_path / "kitchen.yaml" + config_file.write_text(""" +esphome: + name: kitchen # primary device + +esp32: + board: nodemcu-32s +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "kitchen"}} + + args = MockArgs(name="garage", dashboard=False) + mock_run_external_process.return_value = 0 + + result = command_rename(args, {}) + + assert result == 0 + + new_file = tmp_path / "garage.yaml" + content = new_file.read_text() + assert "# primary device" in content + + +def test_command_rename_handles_double_quoted_value( + tmp_path: Path, + mock_run_external_process: Mock, +) -> None: + """Test rename matches when the existing value is double-quoted.""" + config_file = tmp_path / "kitchen.yaml" + config_file.write_text(""" +esphome: + name: "kitchen" + +esp32: + board: nodemcu-32s +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "kitchen"}} + + args = MockArgs(name="garage", dashboard=False) + mock_run_external_process.return_value = 0 + + result = command_rename(args, {}) + + assert result == 0 + new_file = tmp_path / "garage.yaml" + assert 'name: "garage"' in new_file.read_text() + + +def test_command_rename_handles_single_quoted_value( + tmp_path: Path, + mock_run_external_process: Mock, +) -> None: + """Test rename matches when the existing value is single-quoted.""" + config_file = tmp_path / "kitchen.yaml" + config_file.write_text(""" +esphome: + name: 'kitchen' + +esp32: + board: nodemcu-32s +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "kitchen"}} + + args = MockArgs(name="garage", dashboard=False) + mock_run_external_process.return_value = 0 + + result = command_rename(args, {}) + + assert result == 0 + new_file = tmp_path / "garage.yaml" + assert 'name: "garage"' in new_file.read_text() + + +def test_command_rename_leaves_a_lookalike_substitution_line_alone( + tmp_path: Path, + mock_run_external_process: Mock, +) -> None: + """Only the substitution's own line changes; another block's field of + the same name and value is not it.""" + config_file = tmp_path / "oldname.yaml" + config_file.write_text(""" +substitutions: + device_name: oldname + +esphome: + name: ${device_name} + +example: + device_name: oldname + +esp32: + board: nodemcu-32s +""") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = { + CONF_ESPHOME: {CONF_NAME: "oldname"}, + CONF_SUBSTITUTIONS: {"device_name": "oldname"}, + } + assert command_rename(MockArgs(name="newname", dashboard=False), {}) == 0 + content = (tmp_path / "newname.yaml").read_text() + assert 'device_name: "newname"' in content + assert "example:\n device_name: oldname\n" in content + + +def test_command_rename_keeps_line_endings_and_mode( + tmp_path: Path, + mock_run_external_process: Mock, +) -> None: + """A CRLF file stays CRLF and the new file gets the old one's mode.""" + config_file = tmp_path / "oldname.yaml" + config_file.write_bytes( + b"esphome:\r\n name: oldname # device\r\n\r\nesp32:\r\n board: nodemcu-32s\r\n" + ) + if sys.platform != "win32": + config_file.chmod(0o600) + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + assert command_rename(MockArgs(name="newname", dashboard=False), {}) == 0 + new_file = tmp_path / "newname.yaml" + assert new_file.read_bytes() == ( + b'esphome:\r\n name: "newname" # device\r\n\r\nesp32:\r\n board: nodemcu-32s\r\n' + ) + if sys.platform != "win32": + assert new_file.stat().st_mode & 0o777 == 0o600 + + +@pytest.mark.parametrize( + ("yaml_text", "extra"), + [ + ("esphome:\n name: ${missing}\n", {}), + ("esphome: {name: oldname}\n", {}), + ("esphome: !include base.yaml\n", {"base.yaml": "name: oldname\n"}), + ( + ( + "named: &named\n name: oldname\n\nesphome:\n <<: *named\n\n" + "sensor:\n - platform: template\n <<: *named\n" + ), + {}, + ), + ], + ids=["missing_substitution", "flow_mapping", "included_name", "merged_name"], +) +def test_command_rename_refuses_shapes_without_a_plain_name_line( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, + yaml_text: str, + extra: dict[str, str], +) -> None: + """The name line must be a plain value in the file being renamed.""" + for name, text in extra.items(): + (tmp_path / name).write_text(text) + config_file = tmp_path / "oldname.yaml" + config_file.write_text(yaml_text) + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + assert command_rename(MockArgs(name="newname", dashboard=False), {}) == 1 + mock_run_external_process.assert_not_called() + assert "complex yaml" in capfd.readouterr().out.lower() + + +def test_command_rename_removes_the_new_file_when_its_mode_cannot_be_set( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, +) -> None: + """No orphan is left for the next attempt to trip over.""" + config_file = tmp_path / "oldname.yaml" + config_file.write_text("esphome:\n name: oldname\n") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + with patch("pathlib.Path.chmod", side_effect=OSError("read-only share")): + assert command_rename(MockArgs(name="newname", dashboard=False), {}) == 1 + assert not (tmp_path / "newname.yaml").exists() + mock_run_external_process.assert_not_called() + assert "Rename failed" in capfd.readouterr().out + + +def test_command_rename_refuses_a_name_without_a_source_line( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, +) -> None: + """A key the loader did not read from a file cannot be located.""" + config_file = tmp_path / "oldname.yaml" + config_file.write_text("esphome:\n name: oldname\n") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + with patch("esphome.yaml_edit.source_of", return_value=None): + assert command_rename(MockArgs(name="newname", dashboard=False), {}) == 1 + mock_run_external_process.assert_not_called() + assert "was not read from" in capfd.readouterr().out + + +def test_command_rename_passes_dashboard_to_the_install( + tmp_path: Path, + mock_run_external_process: Mock, +) -> None: + config_file = tmp_path / "oldname.yaml" + config_file.write_text("esphome:\n name: oldname\n") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + assert command_rename(MockArgs(name="newname", dashboard=True), {}) == 0 + install = mock_run_external_process.call_args_list[-1].args + assert install[-6:-4] == ("--dashboard", "run") + + +def test_command_rename_interrupted_install_reverts( + tmp_path: Path, + mock_run_external_process: Mock, +) -> None: + config_file = tmp_path / "oldname.yaml" + config_file.write_text("esphome:\n name: oldname\n") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + mock_run_external_process.side_effect = [0, KeyboardInterrupt] + assert command_rename(MockArgs(name="newname", dashboard=False), {}) == 1 + assert not (tmp_path / "newname.yaml").exists() + assert config_file.exists() + + +def test_command_rename_reads_a_config_linked_from_outside( + tmp_path: Path, + mock_run_external_process: Mock, +) -> None: + """The source is only read; the new file lands in the config directory.""" + outside = tmp_path / "elsewhere.yaml" + outside.write_text("esphome:\n name: oldname\n") + config_dir = tmp_path / "config" + config_dir.mkdir() + config_file = config_dir / "oldname.yaml" + config_file.symlink_to(outside) + setup_core(tmp_path=config_dir) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + assert command_rename(MockArgs(name="newname", dashboard=False), {}) == 0 + assert (config_dir / "newname.yaml").read_text() == 'esphome:\n name: "newname"\n' + assert not config_file.exists() + assert outside.exists() + + +def test_command_rename_reports_an_orphan_it_could_not_remove( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, +) -> None: + """The write failure is the message; a cleanup failure is added to it.""" + config_file = tmp_path / "oldname.yaml" + config_file.write_text("esphome:\n name: oldname\n") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + with ( + patch("pathlib.Path.chmod", side_effect=OSError("read-only share")), + patch("pathlib.Path.unlink", side_effect=OSError("busy")), + ): + assert command_rename(MockArgs(name="newname", dashboard=False), {}) == 1 + out = capfd.readouterr().out + assert "Rename failed" in out + assert "Could not remove" in out and "newname.yaml" in out + + +def test_command_rename_install_failure_says_so( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, +) -> None: + """The device may already carry the new name; the user is told.""" + config_file = tmp_path / "oldname.yaml" + config_file.write_text("esphome:\n name: oldname\n") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + mock_run_external_process.side_effect = [0, 1] + assert command_rename(MockArgs(name="newname", dashboard=False), {}) == 1 + out = capfd.readouterr().out + assert "Rename failed: the install did not finish" in out + assert "may already run the new name" in out + + +def test_command_rename_validation_revert_reports_an_orphan( + tmp_path: Path, + capfd: CaptureFixture[str], + mock_run_external_process: Mock, +) -> None: + config_file = tmp_path / "oldname.yaml" + config_file.write_text("esphome:\n name: oldname\n") + setup_core(tmp_path=tmp_path) + CORE.config_path = config_file + CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} + mock_run_external_process.return_value = 1 + with patch("pathlib.Path.unlink", side_effect=OSError("busy")): + assert command_rename(MockArgs(name="newname", dashboard=False), {}) == 1 + out = capfd.readouterr().out + assert "does not validate" in out and "Could not remove" in out diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index bf5d1566c05..837643417d5 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -42,7 +42,6 @@ from esphome.__main__ import ( command_config_hash, command_dashboard, command_idedata, - command_rename, command_run, command_update_all, command_wizard, @@ -101,7 +100,6 @@ from esphome.const import ( CONF_PASSWORD, CONF_PLATFORM, CONF_PORT, - CONF_SUBSTITUTIONS, CONF_TOPIC, CONF_USE_ADDRESS, CONF_USERNAME, @@ -4448,627 +4446,6 @@ def test_command_config_hash( assert output == f"0x{CORE.config_hash:08x}" -def test_command_rename_invalid_characters( - tmp_path: Path, capfd: CaptureFixture[str] -) -> None: - """Test command_rename with invalid characters in name.""" - setup_core(tmp_path=tmp_path) - - # Test with invalid character (space) - args = MockArgs(name="invalid name") - result = command_rename(args, {}) - - assert result == 1 - captured = capfd.readouterr() - assert "invalid character" in captured.out.lower() - - -def test_command_rename_complex_yaml( - tmp_path: Path, capfd: CaptureFixture[str] -) -> None: - """Test command_rename with complex YAML that cannot be renamed.""" - config_file = tmp_path / "test.yaml" - config_file.write_text("# Complex YAML without esphome section\nsome_key: value\n") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - - args = MockArgs(name="newname") - result = command_rename(args, {}) - - assert result == 1 - captured = capfd.readouterr() - assert "complex yaml" in captured.out.lower() - - -def test_command_rename_success( - tmp_path: Path, - capfd: CaptureFixture[str], - mock_run_external_process: Mock, -) -> None: - """Test successful rename of a simple configuration.""" - config_file = tmp_path / "oldname.yaml" - config_file.write_text(""" -esphome: - name: oldname - -esp32: - board: nodemcu-32s - -wifi: - ssid: "test" - password: "test1234" -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - - # Set up CORE.config to avoid ValueError when accessing CORE.address - CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} - - args = MockArgs(name="newname", dashboard=False) - - # Simulate successful validation and upload - mock_run_external_process.return_value = 0 - - result = command_rename(args, {}) - - assert result == 0 - - # Verify new file was created - new_file = tmp_path / "newname.yaml" - assert new_file.exists() - - # Verify old file was removed - assert not config_file.exists() - - # Verify content was updated - content = new_file.read_text() - assert ( - 'name: "newname"' in content - or "name: 'newname'" in content - or "name: newname" in content - ) - - captured = capfd.readouterr() - assert "SUCCESS" in captured.out - - -def test_command_rename_with_substitutions( - tmp_path: Path, - mock_run_external_process: Mock, -) -> None: - """Test rename with substitutions in YAML.""" - config_file = tmp_path / "oldname.yaml" - config_file.write_text(""" -substitutions: - device_name: oldname - -esphome: - name: ${device_name} - -esp32: - board: nodemcu-32s -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - - # Set up CORE.config to avoid ValueError when accessing CORE.address - CORE.config = { - CONF_ESPHOME: {CONF_NAME: "oldname"}, - CONF_SUBSTITUTIONS: {"device_name": "oldname"}, - } - - args = MockArgs(name="newname", dashboard=False) - - mock_run_external_process.return_value = 0 - - result = command_rename(args, {}) - - assert result == 0 - - # Verify substitution was updated - new_file = tmp_path / "newname.yaml" - content = new_file.read_text() - assert 'device_name: "newname"' in content - - -def test_command_rename_validation_failure( - tmp_path: Path, - capfd: CaptureFixture[str], - mock_run_external_process: Mock, -) -> None: - """Test rename when validation fails.""" - config_file = tmp_path / "oldname.yaml" - config_file.write_text(""" -esphome: - name: oldname - -esp32: - board: nodemcu-32s -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - - args = MockArgs(name="newname", dashboard=False) - - # First call for validation fails - mock_run_external_process.return_value = 1 - - result = command_rename(args, {}) - - assert result == 1 - - # Verify new file was created but then removed due to failure - new_file = tmp_path / "newname.yaml" - assert not new_file.exists() - - # Verify old file still exists (not removed on failure) - assert config_file.exists() - - captured = capfd.readouterr() - assert "Rename failed" in captured.out - - -def test_command_rename_install_failure_reverts( - tmp_path: Path, - capfd: CaptureFixture[str], - mock_run_external_process: Mock, -) -> None: - """Test rename when the install (esphome run) step fails.""" - config_file = tmp_path / "oldname.yaml" - config_file.write_text(""" -esphome: - name: oldname - -esp32: - board: nodemcu-32s -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} - - args = MockArgs(name="newname", dashboard=False) - - # First call (config validation) succeeds; second (esphome run) fails. - mock_run_external_process.side_effect = [0, 1] - - result = command_rename(args, {}) - - assert result == 1 - - # New file was unlinked when install failed. - new_file = tmp_path / "newname.yaml" - assert not new_file.exists() - - # Old file is preserved so the device stays reachable under the - # original hostname. - assert config_file.exists() - - -def test_command_rename_target_exists_refuses( - tmp_path: Path, - capfd: CaptureFixture[str], - mock_run_external_process: Mock, -) -> None: - """Test rename refuses when the target filename already exists. - - Without this guard, the rename would overwrite the unrelated - device's YAML and OTA-install our firmware to the wrong device. - """ - config_file = tmp_path / "oldname.yaml" - config_file.write_text(""" -esphome: - name: oldname - -esp32: - board: nodemcu-32s -""") - target_file = tmp_path / "newname.yaml" - target_file.write_text(""" -esphome: - name: someoneelse - -esp32: - board: nodemcu-32s -""") - target_original = target_file.read_text() - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} - - args = MockArgs(name="newname", dashboard=False) - - result = command_rename(args, {}) - - assert result == 1 - # No subprocess work happened — refusal is up-front. - mock_run_external_process.assert_not_called() - # Target file untouched: same content, still on disk. - assert target_file.exists() - assert target_file.read_text() == target_original - # Source file untouched. - assert config_file.exists() - - captured = capfd.readouterr() - assert "already exists" in captured.out - - -def test_command_rename_same_name_refuses( - tmp_path: Path, - capfd: CaptureFixture[str], - mock_run_external_process: Mock, -) -> None: - """Test rename refuses when the new name matches the current name. - - A same-name rename would otherwise re-write the YAML and queue - a redundant compile + install — wasted work the user almost - certainly didn't intend. - """ - config_file = tmp_path / "samename.yaml" - config_file.write_text(""" -esphome: - name: samename - -esp32: - board: nodemcu-32s -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - CORE.config = {CONF_ESPHOME: {CONF_NAME: "samename"}} - - args = MockArgs(name="samename", dashboard=False) - - result = command_rename(args, {}) - - assert result == 1 - mock_run_external_process.assert_not_called() - # File preserved verbatim — no rewrite happened. - assert config_file.exists() - - captured = capfd.readouterr() - assert "already" in captured.out.lower() - - -def test_command_rename_does_not_touch_friendly_name_substring( - tmp_path: Path, - mock_run_external_process: Mock, -) -> None: - r"""Test rename does not match the ``name:`` substring of ``friendly_name:``. - - Without anchoring the regex at line start, the pattern - ``\s*name:\s+`` could match the trailing ``name:`` - substring inside ``friendly_name: ``. The rewrite would - flip both lines to the new name, leaving the user with a - silently corrupted ``friendly_name``. - """ - config_file = tmp_path / "oldname.yaml" - config_file.write_text(""" -esphome: - name: oldname - friendly_name: oldname - -esp32: - board: nodemcu-32s -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - CORE.config = {CONF_ESPHOME: {CONF_NAME: "oldname"}} - - args = MockArgs(name="newname", dashboard=False) - mock_run_external_process.return_value = 0 - - result = command_rename(args, {}) - - assert result == 0 - new_file = tmp_path / "newname.yaml" - content = new_file.read_text() - # esphome.name swapped. - assert 'name: "newname"' in content - # friendly_name kept verbatim. - assert "friendly_name: oldname" in content - - -def test_command_rename_does_not_match_old_name_as_value_prefix( - tmp_path: Path, - mock_run_external_process: Mock, -) -> None: - r"""Test rename does not match ``old_name`` as a prefix of a longer value. - - With ``old_name = kitchen`` the value ``kitchen2`` (a sensor - or wifi entry) would otherwise match the unanchored - ``["']?kitchen["']?`` pattern at the prefix and get - rewritten to the new name. The end-of-value lookahead keeps - the match restricted to whole tokens. - """ - config_file = tmp_path / "kitchen.yaml" - config_file.write_text(""" -esphome: - name: kitchen - -esp32: - board: nodemcu-32s - -wifi: - ap: - ssid: kitchen2 -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - CORE.config = {CONF_ESPHOME: {CONF_NAME: "kitchen"}} - - args = MockArgs(name="garage", dashboard=False) - mock_run_external_process.return_value = 0 - - result = command_rename(args, {}) - - assert result == 0 - new_file = tmp_path / "garage.yaml" - content = new_file.read_text() - assert 'name: "garage"' in content - # The wifi ssid value is unrelated and stays intact. - assert "ssid: kitchen2" in content - - -def test_command_rename_same_resolved_name_refuses( - tmp_path: Path, - capfd: CaptureFixture[str], - mock_run_external_process: Mock, -) -> None: - """Test rename refuses when ``new_name`` matches the resolved device name. - - The path-equality check only catches the case where the - config filename matches the device name. For a config whose - filename and ``esphome.name`` differ (here ``weird-file.yaml`` - holds ``esphome.name: kitchen``), running - ``esphome rename weird-file.yaml kitchen`` would otherwise - fall through to the rewrite + install: the YAML's name stays - ``kitchen``, the file is renamed to ``kitchen.yaml``, and the - device gets a redundant flash. Refuse up-front so the - "already the device's name" message matches reality. - """ - config_file = tmp_path / "weird-file.yaml" - config_file.write_text(""" -esphome: - name: kitchen - -esp32: - board: nodemcu-32s -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - CORE.config = {CONF_ESPHOME: {CONF_NAME: "kitchen"}} - - args = MockArgs(name="kitchen", dashboard=False) - - result = command_rename(args, {}) - - assert result == 1 - mock_run_external_process.assert_not_called() - # Source file untouched, no derived target written. - assert config_file.exists() - assert not (tmp_path / "kitchen.yaml").exists() - - captured = capfd.readouterr() - assert "already" in captured.out.lower() - - -def test_command_rename_target_path_equals_source_refuses( - tmp_path: Path, - capfd: CaptureFixture[str], - mock_run_external_process: Mock, -) -> None: - """Test rename refuses when the new path resolves to the source file. - - Reachable only when the YAML's filename and ``esphome.name`` - disagree — here ``kitchen.yaml`` holds ``esphome.name: garage`` - and the user runs ``esphome rename kitchen.yaml kitchen``. The - name-equality check above passes (``garage != kitchen``), but - ``/kitchen.yaml`` resolves to the source file - itself, so the rewrite would clobber the source mid-rename. - Refuse rather than silently overwriting. - """ - config_file = tmp_path / "kitchen.yaml" - config_file.write_text(""" -esphome: - name: garage - -esp32: - board: nodemcu-32s -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - CORE.config = {CONF_ESPHOME: {CONF_NAME: "garage"}} - - args = MockArgs(name="kitchen", dashboard=False) - - result = command_rename(args, {}) - - assert result == 1 - mock_run_external_process.assert_not_called() - # Source file still present and unmodified. - assert config_file.exists() - assert "name: garage" in config_file.read_text() - - captured = capfd.readouterr() - assert "already" in captured.out.lower() - - -def test_command_rename_does_not_touch_lookalike_name_in_other_blocks( - tmp_path: Path, - mock_run_external_process: Mock, -) -> None: - """Test rename only swaps the esphome.name line. - - A device whose name happens to match a sensor's / output's - ``name:`` value must not have those other names rewritten — - they're independent. Without an anchor for the esphome block - a naive regex would clobber every line whose value matches. - """ - config_file = tmp_path / "kitchen.yaml" - config_file.write_text(""" -esphome: - name: kitchen - -esp32: - board: nodemcu-32s - -sensor: - - platform: template - name: kitchen - lambda: 'return 0;' -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - CORE.config = {CONF_ESPHOME: {CONF_NAME: "kitchen"}} - - args = MockArgs(name="garage", dashboard=False) - mock_run_external_process.return_value = 0 - - result = command_rename(args, {}) - - assert result == 0 - - new_file = tmp_path / "garage.yaml" - content = new_file.read_text() - # esphome.name renamed. - assert 'name: "garage"' in content - # Sensor's name is the user's entity name — must not be touched. - assert " name: kitchen\n" in content - - -def test_command_rename_preserves_trailing_comment( - tmp_path: Path, - mock_run_external_process: Mock, -) -> None: - """Test rename preserves a trailing ``# comment`` on the name line.""" - config_file = tmp_path / "kitchen.yaml" - config_file.write_text(""" -esphome: - name: kitchen # primary device - -esp32: - board: nodemcu-32s -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - CORE.config = {CONF_ESPHOME: {CONF_NAME: "kitchen"}} - - args = MockArgs(name="garage", dashboard=False) - mock_run_external_process.return_value = 0 - - result = command_rename(args, {}) - - assert result == 0 - - new_file = tmp_path / "garage.yaml" - content = new_file.read_text() - assert "# primary device" in content - - -def test_command_rename_handles_double_quoted_value( - tmp_path: Path, - mock_run_external_process: Mock, -) -> None: - """Test rename matches when the existing value is double-quoted.""" - config_file = tmp_path / "kitchen.yaml" - config_file.write_text(""" -esphome: - name: "kitchen" - -esp32: - board: nodemcu-32s -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - CORE.config = {CONF_ESPHOME: {CONF_NAME: "kitchen"}} - - args = MockArgs(name="garage", dashboard=False) - mock_run_external_process.return_value = 0 - - result = command_rename(args, {}) - - assert result == 0 - new_file = tmp_path / "garage.yaml" - assert 'name: "garage"' in new_file.read_text() - - -def test_command_rename_handles_single_quoted_value( - tmp_path: Path, - mock_run_external_process: Mock, -) -> None: - """Test rename matches when the existing value is single-quoted.""" - config_file = tmp_path / "kitchen.yaml" - config_file.write_text(""" -esphome: - name: 'kitchen' - -esp32: - board: nodemcu-32s -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - CORE.config = {CONF_ESPHOME: {CONF_NAME: "kitchen"}} - - args = MockArgs(name="garage", dashboard=False) - mock_run_external_process.return_value = 0 - - result = command_rename(args, {}) - - assert result == 0 - new_file = tmp_path / "garage.yaml" - assert 'name: "garage"' in new_file.read_text() - - -def test_command_rename_too_many_substitution_matches_refuses( - tmp_path: Path, - capfd: CaptureFixture[str], - mock_run_external_process: Mock, -) -> None: - """Test rename refuses when ``${var}`` resolves to multiple matches. - - When ``esphome.name: ${device_name}`` and the substitution - definition ``device_name: foo`` appears more than once in the - YAML (e.g. inside multiple included blocks), the regex rewrite - can't tell which one to flip. Rather than silently picking one - or rewriting both, the command refuses. - """ - config_file = tmp_path / "oldname.yaml" - config_file.write_text(""" -substitutions: - device_name: oldname - -esphome: - name: ${device_name} - -# A copy-pasted block that re-declares the substitution at the -# same indent level - happens when users splice in a packaged -# fragment without renaming the variable. -example: - device_name: oldname - -esp32: - board: nodemcu-32s -""") - setup_core(tmp_path=tmp_path) - CORE.config_path = config_file - CORE.config = { - CONF_ESPHOME: {CONF_NAME: "oldname"}, - CONF_SUBSTITUTIONS: {"device_name": "oldname"}, - } - - args = MockArgs(name="newname", dashboard=False) - - result = command_rename(args, {}) - - assert result == 1 - mock_run_external_process.assert_not_called() - # File untouched. - assert config_file.exists() - assert "device_name: oldname" in config_file.read_text() - - captured = capfd.readouterr() - assert "Too many matches" in captured.out - - def test_command_update_all_path_string_conversion( tmp_path: Path, mock_run_external_process: Mock, @@ -8028,3 +7405,11 @@ def test_command_analyze_memory_host_refuses_before_compiling( command_analyze_memory(MockArgs(), {}) mock_write_cpp.assert_not_called() mock_compile_program.assert_not_called() + + +def test_command_rename_is_dispatched_to_the_cli_module() -> None: + """__main__ keeps a thin wrapper and imports the command when it runs.""" + args = MockArgs(name="newname") + with patch("esphome.cli.rename.command_rename", return_value=7) as run: + assert main.command_rename(args, {}) == 7 + run.assert_called_once_with(args, {}) diff --git a/tests/unit_tests/test_yaml_edit.py b/tests/unit_tests/test_yaml_edit.py new file mode 100644 index 00000000000..31df23dc46c --- /dev/null +++ b/tests/unit_tests/test_yaml_edit.py @@ -0,0 +1,182 @@ +"""Tests for rewriting single lines of a yaml file.""" + +from __future__ import annotations + +from pathlib import Path +import sys + +import pytest + +from esphome import yaml_util +from esphome.config import do_substitution_pass +from esphome.const import CONF_ESPHOME, CONF_NAME +from esphome.core import CORE, EsphomeError +from esphome.yaml_edit import ( + LineEdit, + field_line_re, + line_at, + read_text, + rewrite, + rewritten_text, + source_of, + write_keeping_mode, +) + +YAML = """esphome: + name: kitchen # the device + +wifi: + ssid: kitchen +""" + + +def _setup(tmp_path: Path, yaml_text: str) -> Path: + """Write the yaml, point CORE at it and load it the way read_config does, + so every node carries its source range.""" + CORE.reset() + CORE.config_path = tmp_path / "test.yaml" + # Bytes, so Windows does not turn the newlines into CRLF on the way in + CORE.config_path.write_bytes(yaml_text.encode()) + CORE.raw_config = do_substitution_pass(yaml_util.load_yaml(CORE.config_path), None) + return CORE.config_path + + +def _name_edit(new_name: str) -> LineEdit: + doc, line_no = source_of(CORE.raw_config[CONF_ESPHOME], CONF_NAME) + text = line_at(doc, line_no) + match = field_line_re(CONF_NAME, "kitchen").match(text) + return LineEdit(doc, line_no, text, rewrite(match, new_name)) + + +def test_rewrite_keeps_the_rest_of_the_line(tmp_path: Path) -> None: + """The value changes; indentation, quotes and the comment stay.""" + path = _setup(tmp_path, YAML.replace("name: kitchen", "name: 'kitchen'")) + edit = _name_edit("garage") + assert (edit.line, edit.new_line) == (1, " name: 'garage' # the device") + assert rewritten_text(read_text(path), [edit]) == YAML.replace( + "name: kitchen", "name: 'garage'" + ) + + +def test_rewrite_can_force_quotes() -> None: + match = field_line_re(CONF_NAME, "kitchen").match(" name: kitchen # x") + assert rewrite(match, "garage", quote='"') == ' name: "garage" # x' + + +def test_only_the_located_line_changes(tmp_path: Path) -> None: + """A lookalike `name:` under another block has its own range.""" + yaml_text = YAML + "sensor:\n - platform: template\n name: kitchen\n" + path = _setup(tmp_path, yaml_text) + text = rewritten_text(read_text(path), [_name_edit("garage")]) + assert text.endswith(" name: kitchen\n") + assert " name: garage # the device" in text + + +def test_line_endings_are_kept(tmp_path: Path) -> None: + path = _setup(tmp_path, YAML.replace("\n", "\r\n")) + assert rewritten_text(read_text(path), [_name_edit("garage")]) == YAML.replace( + "\n", "\r\n" + ).replace("name: kitchen", "name: garage") + + +def test_stale_line_is_refused(tmp_path: Path) -> None: + path = _setup(tmp_path, YAML) + edit = _name_edit("garage") + with pytest.raises(EsphomeError, match="changed since it was read"): + rewritten_text(YAML.replace("kitchen #", "pantry #"), [edit]) + with pytest.raises(EsphomeError, match="changed since it was read"): + rewritten_text("esphome:\n", [edit]) + with pytest.raises(EsphomeError, match="changed since it was read"): + line_at(path, 5) + + +def test_a_comment_needs_whitespace_and_a_scalar_is_not_empty() -> None: + """`abc#def` is one value to the loader, and a bare `key:` heads a block.""" + assert field_line_re("key", "abc").match("key: abc#def") is None + assert field_line_re("key").match("key:") is None + assert field_line_re("key").match("key: abc # c")["trail"] == " # c" + assert field_line_re("key", "abc#def").match("key: abc#def") is not None + + +def test_source_of_is_none_for_a_value_validation_added(tmp_path: Path) -> None: + """Only a key read from a file carries a range, and only a mapping read + from a file can place its keys; a mapping built in code cannot.""" + assert source_of({"name": "kitchen"}, "name") is None + _setup(tmp_path, YAML) + loaded_key = next(iter(CORE.raw_config[CONF_ESPHOME])) + assert source_of({loaded_key: "kitchen"}, CONF_NAME) is None + + +def test_source_of_is_none_for_a_merged_key(tmp_path: Path) -> None: + """A key a merge brought in points at the anchor, which other mappings + may merge as well; it is not this mapping's own line.""" + _setup( + tmp_path, + "named: &named\n name: kitchen\n\nesphome:\n <<: *named\n friendly_name: x\n", + ) + assert source_of(CORE.raw_config[CONF_ESPHOME], CONF_NAME) is None + assert source_of(CORE.raw_config[CONF_ESPHOME], "friendly_name") == ( + tmp_path / "test.yaml", + 5, + ) + + +def test_source_of_names_the_file_the_loader_read(tmp_path: Path) -> None: + """An include has its own document; a symlink is reported as given.""" + (tmp_path / "base.yaml").write_bytes(b"name: kitchen\n") + _setup(tmp_path, "esphome: !include base.yaml\n") + assert source_of(CORE.raw_config[CONF_ESPHOME], CONF_NAME) == ( + tmp_path / "base.yaml", + 0, + ) + target = tmp_path / "shared" / "test.yaml" + target.parent.mkdir() + target.write_bytes(YAML.encode()) + CORE.config_path.unlink() + CORE.config_path.symlink_to(target) + CORE.raw_config = yaml_util.load_yaml(CORE.config_path) + assert _name_edit("garage").path == CORE.config_path + + +@pytest.mark.skipif(sys.platform == "win32", reason="posix file modes") +def test_write_keeps_the_mode_of_the_file_or_another(tmp_path: Path) -> None: + path = _setup(tmp_path, YAML) + path.chmod(0o600) + write_keeping_mode(path, YAML) + assert path.stat().st_mode & 0o777 == 0o600 + other = tmp_path / "other.yaml" + write_keeping_mode(other, YAML, like=path) + assert other.stat().st_mode & 0o777 == 0o600 + + +def test_write_failures_say_which_step_and_why(tmp_path: Path) -> None: + """A missing mode source, a write that fails, and a mode that cannot be + put back after the write are three different situations.""" + from unittest.mock import patch + + path = _setup(tmp_path, YAML) + with pytest.raises(EsphomeError, match="Could not read the mode of .*gone.yaml"): + write_keeping_mode(path, YAML, like=tmp_path / "gone.yaml") + with ( + patch("pathlib.Path.chmod", side_effect=OSError("denied")), + pytest.raises( + EsphomeError, match="was written but could not get its mode back: denied" + ), + ): + write_keeping_mode(path, YAML) + + def refuse(*_args: object, **_kwargs: object) -> None: + raise EsphomeError(f"Could not write file at {path}") from OSError("disk full") + + with ( + patch("esphome.yaml_edit.write_file", side_effect=refuse), + pytest.raises(EsphomeError, match="Could not write file at .*: disk full"), + ): + write_keeping_mode(path, YAML) + + +def test_read_text_reports_a_file_it_cannot_decode(tmp_path: Path) -> None: + path = tmp_path / "latin1.yaml" + path.write_bytes(b"caf\xe9: 1\n") + with pytest.raises(EsphomeError, match="Error reading file"): + read_text(path) From ef0819f983a4af8e3b20c1838b60c7cd3580f306 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Tue, 29 Sep 2026 18:12:55 -0500 Subject: [PATCH 060/133] [nfc] Hold NDEF records in a fixed-size list (#19912) Co-authored-by: Claude Fable 5.1 --- esphome/components/nfc/ndef_message.cpp | 2 +- esphome/components/nfc/ndef_message.h | 9 +++++---- esphome/components/pn532/pn532.cpp | 6 ++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/esphome/components/nfc/ndef_message.cpp b/esphome/components/nfc/ndef_message.cpp index d9135cbdba6..e597b9192d4 100644 --- a/esphome/components/nfc/ndef_message.cpp +++ b/esphome/components/nfc/ndef_message.cpp @@ -97,7 +97,7 @@ bool NdefMessage::add_record(std::unique_ptr record) { ESP_LOGE(TAG, "Too many records. Max: %d", MAX_NDEF_RECORDS); return false; } - this->records_.emplace_back(std::move(record)); + this->records_.emplace_next() = std::move(record); return true; } diff --git a/esphome/components/nfc/ndef_message.h b/esphome/components/nfc/ndef_message.h index 0d7699e9d72..25ec7c00c7d 100644 --- a/esphome/components/nfc/ndef_message.h +++ b/esphome/components/nfc/ndef_message.h @@ -13,6 +13,8 @@ namespace esphome::nfc { static constexpr uint8_t MAX_NDEF_RECORDS = 4; +/// The records of one message, owned in place so parsing a message allocates only the records themselves +using NdefRecords = StaticVector, MAX_NDEF_RECORDS>; class NdefMessage { public: @@ -20,13 +22,12 @@ class NdefMessage { NdefMessage(std::span data); NdefMessage(std::vector &data) : NdefMessage(std::span(data)) {} NdefMessage(const NdefMessage &msg) { - records_.reserve(msg.records_.size()); for (const auto &r : msg.records_) { - records_.emplace_back(r->clone()); + records_.emplace_next() = r->clone(); } } - const std::vector> &get_records() { return this->records_; }; + const NdefRecords &get_records() const { return this->records_; }; bool add_record(std::unique_ptr record); bool add_text_record(const std::string &text); @@ -36,7 +37,7 @@ class NdefMessage { std::vector encode(); protected: - std::vector> records_; + NdefRecords records_; }; } // namespace esphome::nfc diff --git a/esphome/components/pn532/pn532.cpp b/esphome/components/pn532/pn532.cpp index 91593484c87..89392105a24 100644 --- a/esphome/components/pn532/pn532.cpp +++ b/esphome/components/pn532/pn532.cpp @@ -339,8 +339,9 @@ enum PN532ReadReady PN532::read_ready_(bool block) { return READY; } + // the non-blocking call comes from loop(), where the tick's timestamp is already cached if (!this->rd_started_) { - this->rd_start_time_ = millis(); + this->rd_start_time_ = block ? millis() : App.get_loop_component_start_time(); this->rd_started_ = true; } const uint32_t rd_start_time = this->rd_start_time_; @@ -351,7 +352,8 @@ enum PN532ReadReady PN532::read_ready_(bool block) { break; } - if (millis() - rd_start_time > 100) { + const uint32_t now = block ? millis() : App.get_loop_component_start_time(); + if (now - rd_start_time > 100) { ESP_LOGV(TAG, "Timed out waiting for readiness from PN532!"); this->rd_ready_ = TIMEOUT; break; From ed88002c3783465254117c7b5df2c3a9f59b4784 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 02:23:05 +0200 Subject: [PATCH 061/133] [core] Extract native toolchain package archives in parallel (#18840) --- esphome/arduino8266/framework.py | 23 +- esphome/espidf/framework.py | 4 +- esphome/framework_helpers.py | 100 ++++-- esphome/platformio/library.py | 4 +- esphome/platformio/prefetch.py | 9 +- esphome/platformio/registry.py | 155 +++++++++- .../unit_tests/test_arduino8266_framework.py | 51 +--- tests/unit_tests/test_framework_helpers.py | 91 +++++- tests/unit_tests/test_platformio_prefetch.py | 2 +- tests/unit_tests/test_platformio_registry.py | 289 ++++++++++++++++-- 10 files changed, 591 insertions(+), 137 deletions(-) diff --git a/esphome/arduino8266/framework.py b/esphome/arduino8266/framework.py index a8f8c65c066..b731b746751 100644 --- a/esphome/arduino8266/framework.py +++ b/esphome/arduino8266/framework.py @@ -24,9 +24,10 @@ from esphome.core import EsphomeError, Version from esphome.framework_helpers import str_to_lst_of_str from esphome.platformio.registry import ( Download, + PackageSpec, Resolver, get_systype, - install_package, + install_packages, prefetch_packages, ) @@ -151,14 +152,14 @@ def check_and_install(framework_version: Version) -> InstalledPaths: toolchain_path = get_toolchain_path() # One spec per package: the prefetch and the installs must agree specs = ( - ( + PackageSpec( FRAMEWORK_PACKAGE, release.tag, framework_path, ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, ("cores/esp8266", "tools/sdk", "libraries"), ), - ( + PackageSpec( TOOLCHAIN_PACKAGE, TOOLCHAIN_VERSION, toolchain_path, @@ -174,18 +175,10 @@ def check_and_install(framework_version: Version) -> InstalledPaths: resolvers[FRAMEWORK_PACKAGE] = release.download if not ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS: resolvers[TOOLCHAIN_PACKAGE] = toolchain_download - # Fetch both archives at once; the installs below verify and extract - prefetch_packages([spec[:4] for spec in specs], downloads_dir, resolvers) - for name, version, dest, mirrors, expect in specs: - install_package( - name, - version, - dest, - mirrors, - downloads_dir, - expect=expect, - resolve=resolvers.get(name), - ) + # Fetch both archives at once; the installs verify and extract them. + # One spec list for both, so the two phases cannot drift. + prefetch_packages(specs, downloads_dir, resolvers) + install_packages(specs, downloads_dir, resolvers) return InstalledPaths( framework=framework_path, toolchain=toolchain_path, ninja=ninja_path ) diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 1a4b17efdb3..8c377561ca6 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -34,7 +34,7 @@ from esphome.framework_helpers import ( run_command_ok, str_to_lst_of_str, tool_version_runs, - warn_prefetch_failures, + warn_batch_failures, ) from esphome.helpers import write_file_if_changed @@ -774,7 +774,7 @@ def _prefetch_idf_tool_archives( for entry in entries ], ) - warn_prefetch_failures(failures) + warn_batch_failures(failures, "Could not prefetch %s: %s") if len(failures) == len(entries): # A systematic fault, not one flaky mirror: the resume # workaround (#17703) is off for this whole install diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index fc2a18a6ecd..86010e30650 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -15,7 +15,7 @@ import threading import time from typing import IO, TYPE_CHECKING -from esphome.helpers import ProgressBar, rmtree +from esphome.helpers import ProgressBar, get_usable_cpu_count, rmtree from esphome.net_retry import ( NETWORK_MAX_ATTEMPTS, http_request, @@ -288,10 +288,25 @@ def _detect_archive_root(names: Iterable[str]) -> str | None: return root if has_descendant else None +def _resolve_progress( + progress: Callable[[float], None] | None, + progress_header: str | None, + has_work: bool, +) -> Callable[[float], None] | None: + """Fraction reporter for an extractor: the caller's callback wins over a + private ``progress_header`` bar.""" + if progress is not None: + return progress + if progress_header and has_work: + return ProgressBar(progress_header).update + return None + + def _tar_extract_all( data: io.BufferedIOBase, extract_dir: PathType = ".", progress_header: str | None = None, + progress: Callable[[float], None] | None = None, ): """ Extract a TAR archive to the specified directory. @@ -306,6 +321,7 @@ def _tar_extract_all( data: File-like object containing the TAR archive extract_dir: Directory to extract contents to progress_header: If set, show a progress bar with this header + progress: fraction callback (0..1, ends at 1.0); overrides progress_header """ import tarfile @@ -364,21 +380,23 @@ def _tar_extract_all( safe_members.append(member) total = len(safe_members) - progress = ( - ProgressBar(progress_header) if progress_header and total > 0 else None - ) + report = _resolve_progress(progress, progress_header, total > 0) for i, member in enumerate(safe_members, 1): - tar_ref.extract(member, abs_dest) - if progress is not None: - progress.update(i / total) - if progress is not None: - progress.update(1) + # Named: the default is fully_trusted on 3.12/3.13, data on + # 3.14. The pre-pass drops unsafe members; an escape past it + # gains nothing, since the build runs what these archives hold. + tar_ref.extract(member, abs_dest, filter="fully_trusted") + if report is not None: + report(i / total) + if report is not None: + report(1) def _zip_extract_all( data: io.BufferedIOBase, extract_dir: PathType = ".", progress_header: str | None = None, + progress: Callable[[float], None] | None = None, ): """ Extract a ZIP archive to the specified directory. @@ -387,6 +405,7 @@ def _zip_extract_all( data: File-like object containing the ZIP archive extract_dir: Directory to extract contents to progress_header: If set, show a progress bar with this header + progress: fraction callback (0..1, ends at 1.0); overrides progress_header """ import zipfile @@ -403,9 +422,7 @@ def _zip_extract_all( strip_prefix = f"{strip_root}/" if strip_root is not None else None total = len(all_members) - progress = ( - ProgressBar(progress_header) if progress_header and total > 0 else None - ) + report = _resolve_progress(progress, progress_header, total > 0) for i, member in enumerate(all_members, 1): # 1. Normalize name @@ -438,10 +455,10 @@ def _zip_extract_all( # 6. Extract zip_ref.extract(member, extract_dir) - if progress is not None: - progress.update(i / total) - if progress is not None: - progress.update(1) + if report is not None: + report(i / total) + if report is not None: + report(1) def _rename_with_retry( @@ -472,6 +489,7 @@ def _7z_extract_all( data: io.BufferedIOBase, extract_dir: PathType = ".", progress_header: str | None = None, + progress: Callable[[float], None] | None = None, ): """ Extract a 7z archive to the specified directory. @@ -486,6 +504,7 @@ def _7z_extract_all( data: File-like object containing the 7z archive (must be seekable) extract_dir: Directory to extract contents to progress_header: If set, show a progress bar with this header + progress: called with 1.0 on completion; overrides progress_header """ import py7zr @@ -524,19 +543,15 @@ def _7z_extract_all( continue safe_targets.append(raw) - progress = ( - ProgressBar(progress_header) - if progress_header and safe_targets - else None - ) + report = _resolve_progress(progress, progress_header, bool(safe_targets)) if len(safe_targets) == len(all_names): z.extractall(path=staging) else: z.extract(path=staging, targets=safe_targets) - if progress is not None: - progress.update(1) + if report is not None: + report(1) src_root = staging / strip_root if strip_root else staging for item in src_root.iterdir(): @@ -567,6 +582,7 @@ def archive_extract_all( archive: PathType | io.RawIOBase | IO[bytes], extract_dir: PathType = ".", progress_header: str | None = None, + progress: Callable[[float], None] | None = None, ): """ Extract an archive file to the specified directory. @@ -575,6 +591,7 @@ def archive_extract_all( archive: Path to archive file or file-like object extract_dir: Directory to extract contents to progress_header: If set, show a progress bar with this header + progress: fraction callback (0..1, ends at 1.0); overrides progress_header Raises: TypeError: If archive is not a valid type @@ -605,7 +622,9 @@ def archive_extract_all( break if matched_fct is None: raise ValueError("Unsupported archive format") - matched_fct(archive_ref, extract_dir, progress_header=progress_header) + matched_fct( + archive_ref, extract_dir, progress_header=progress_header, progress=progress + ) def _open_ranged( @@ -769,13 +788,23 @@ def _stream_response_to_file( # hammering the host or the mirrors. BATCH_DOWNLOAD_WORKERS = 4 +# Measured: gz peaks near 2 workers (8 is slower than serial), xz +# plateaus by 4 and holds ~50 MB of dictionary per worker. +BATCH_EXTRACT_WORKERS = 4 + + +def extract_workers(jobs: int | None = None) -> int: + """Worker count for an extraction batch of ``jobs`` archives.""" + workers = min(get_usable_cpu_count(), BATCH_EXTRACT_WORKERS) + return workers if jobs is None else min(workers, jobs) + def run_batch_downloads( header: str, jobs: list[tuple[str, int, Callable[[Callable[[int], None]], None]]], max_workers: int = BATCH_DOWNLOAD_WORKERS, ) -> list[tuple[str, BaseException]]: - """Run ``(name, size, fetch)`` download jobs concurrently under one bar. + """Run ``(name, size, fetch)`` jobs concurrently under one bar. Each ``fetch(tracker)`` reports absolute byte counts; the bar total is the sum of the sizes. Failures are returned after the bar is done so @@ -1005,15 +1034,26 @@ def resume_fetch_job( return fetch -def warn_prefetch_failures( +def is_expected_fetch_error(err: BaseException) -> bool: + """Download failures the callers degrade on, vs programming errors.""" + from esphome.core import EsphomeError # local import avoids circular dependency + + return isinstance(err, (EsphomeError, OSError)) + + +def warn_batch_failures( failures: list[tuple[str, BaseException]], - message: str = "Could not prefetch %s: %s", + message: str, ) -> None: - """Warn per failed batch-prefetch job; the caller's installer retries them.""" + """Warn per failed batch job, keeping the traceback of unexpected errors.""" for name, err in failures: # failure_reason: a message-less exception must not log blank - _LOGGER.warning(message, name, failure_reason(err)) - _LOGGER.debug("Prefetch failure detail", exc_info=err) + if is_expected_fetch_error(err): + _LOGGER.warning(message, name, failure_reason(err)) + _LOGGER.debug("Failure detail", exc_info=err) + else: + # A programming error must not be reduced to a bare message + _LOGGER.warning(message, name, failure_reason(err), exc_info=err) def download_with_resume( diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index e551d8f1c04..e5e4aa7245f 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -35,7 +35,7 @@ from esphome.framework_helpers import ( failure_reason, rmdir, run_batch_downloads, - warn_prefetch_failures, + warn_batch_failures, ) _LOGGER = logging.getLogger(__name__) @@ -1109,7 +1109,7 @@ def _prefetch_wave( + [(c.name, 0, partial(_clone_source, c, salt, namespace)) for c in clones], ) # The sequential call below retries and raises the real error - warn_prefetch_failures( + warn_batch_failures( failures, "Prefetch of %s failed (retrying sequentially): %s" ) except Exception as err: # noqa: BLE001 # pylint: disable=broad-exception-caught diff --git a/esphome/platformio/prefetch.py b/esphome/platformio/prefetch.py index e648192b737..fbb31ae4529 100644 --- a/esphome/platformio/prefetch.py +++ b/esphome/platformio/prefetch.py @@ -37,13 +37,14 @@ from esphome.framework_helpers import ( content_length, discard_partial_download, downloaded_bytes, + extract_workers, failure_reason, resume_fetch_job, run_batch_downloads, wait_for_download_lock, - warn_prefetch_failures, + warn_batch_failures, ) -from esphome.helpers import get_bool_env, get_usable_cpu_count, rmtree +from esphome.helpers import get_bool_env, rmtree _LOGGER = logging.getLogger(__name__) @@ -702,7 +703,7 @@ def _preinstall( would hang, not fail). Waves skip dependencies; the installed manifests feed the next wave. Any failure falls back to pio run. """ - workers = min(get_usable_cpu_count(), len(entries)) + workers = extract_workers(len(entries)) # One manager per worker (_install mutates instance state); built # serially because construction rewires the shared manager logger managers: SimpleQueue = SimpleQueue() @@ -891,7 +892,7 @@ def _prefetch(build_dir: Path, env: str) -> None: ) # PlatformIO retries failed packages itself, without resume failures = run_batch_downloads("Downloading PlatformIO packages", jobs) - warn_prefetch_failures(failures) + warn_batch_failures(failures, "Could not prefetch %s: %s") failed_names = {name for name, _ in failures} elif not groups and not unresolved: # Record the no-work run so the parent skips the next spawn. diff --git a/esphome/platformio/registry.py b/esphome/platformio/registry.py index cab536c8da7..326a587fc88 100644 --- a/esphome/platformio/registry.py +++ b/esphome/platformio/registry.py @@ -18,9 +18,12 @@ from esphome.framework_helpers import ( download_from_mirrors, download_with_resume, downloaded_bytes, + extract_workers, + is_expected_fetch_error, rmdir, run_batch_downloads, wait_for_download_lock, + warn_batch_failures, ) from esphome.net_retry import fetch_with_retry, http_request @@ -174,6 +177,16 @@ def _check_layout(name: str, dest: Path, expect: Collection[str]) -> None: ) +class PackageSpec(NamedTuple): + """One registry package to install.""" + + name: str + version: str + dest: Path + mirrors: list[str] + expect: Collection[str] = () + + class _PendingArchive(NamedTuple): name: str version: str @@ -194,27 +207,47 @@ def is_installed(dest: Path) -> bool: return (dest / ".esphome_extracted").is_file() +def _batched_download_progress( + name: str, version: str, extract_progress: Callable[[float], None] +) -> Callable[[int], None]: + """Zero-tick tracker for a batched install; announces a real download + once, since the shared bar cannot move for it.""" + ticks = 0 + + def progress(done: int) -> None: + nonlocal ticks + ticks += 1 + # A verified archive credits itself in one tick; more than one + # means bytes are streaming, including a resumed .part + if ticks == 2: + _LOGGER.info("Re-downloading %s %s ...", name, version) + extract_progress(0.0) + + return progress + + def prefetch_packages( - packages: list[tuple[str, str, Path, list[str]]], + packages: Collection[PackageSpec], downloads_dir: Path, resolvers: dict[str, Resolver] | None = None, ) -> None: """Download pending package archives in parallel under one combined bar. - ``packages`` holds ``(name, version, dest, mirrors)`` per package; - ``resolvers`` replaces the registry lookup by name. Purely - an optimization: ``install_package`` verifies every archive and - re-downloads anything this pass left unfinished. Mirror overrides and - registry entries without a size stay on the sequential path so its - per-file bars remain trustworthy. Each fetch holds the same per-dest - lock as ``install_package``: the archive's ``.part`` file is shared, and - two concurrent writers would truncate each other's bytes. + ``packages`` holds one ``PackageSpec`` per package, the same list the + install pass takes; ``expect`` is unused here and ``resolvers`` replaces + the registry lookup by name. Purely an optimization: ``install_package`` + verifies every archive and re-downloads anything this pass left + unfinished. Mirror overrides and registry entries without a size stay on + the sequential path so its per-file bars remain trustworthy. Each fetch + holds the same per-dest lock as ``install_package``: the archive's + ``.part`` file is shared, and two concurrent writers would truncate each + other's bytes. """ from filelock import FileLock, Timeout pending: list[_PendingArchive] = [] seen: set[Path] = set() - for name, version, dest, mirrors in packages: + for name, version, dest, mirrors, _expect in packages: if mirrors or is_installed(dest): continue archive = _archive_path(downloads_dir, name, version) @@ -283,7 +316,7 @@ def prefetch_packages( [(entry.name, entry.size, partial(_fetch, entry)) for entry in pending], ) for name, err in failures: - if isinstance(err, (EsphomeError, OSError)): + if is_expected_fetch_error(err): # Expected download failures: install_package retries this one # itself, with a visible bar _LOGGER.debug("Prefetch of %s failed: %s", name, err) @@ -301,6 +334,7 @@ def install_package( downloads_dir: Path, expect: Collection[str], resolve: Resolver | None = None, + extract_progress: Callable[[float], None] | None = None, ) -> None: """Download, verify, and extract one package if not already installed. @@ -309,6 +343,9 @@ def install_package( substitution) is trusted as configured. ``downloads_dir`` holds the archive between runs so an interrupted download resumes. ``resolve`` replaces the registry lookup. + + ``extract_progress`` receives extraction fractions in [0, 1] instead of + the private per-file bars (see ``install_packages``). """ if not expect: # Layout validation before marker.touch() is the only guard against @@ -332,7 +369,10 @@ def install_package( # Persistent location so an interrupted download resumes across runs. downloads_dir.mkdir(parents=True, exist_ok=True) archive = _archive_path(downloads_dir, name, version) - _LOGGER.info("Downloading %s %s ...", name, version) + # Batched runs are announced by the batch header + batched = extract_progress is not None and archive.is_file() + if not batched: + _LOGGER.info("Downloading %s %s ...", name, version) if mirrors: _LOGGER.warning( "Downloading %s from a mirror override; checksum verification " @@ -346,11 +386,96 @@ def install_package( url, sha256, size = ( resolve() if resolve else registry_download(name, version) ) - download_with_resume(url, archive, sha256=sha256, size=size) - _LOGGER.info("Extracting %s ...", name) - archive_extract_all(archive, dest, progress_header="Extracting") + download_with_resume( + url, + archive, + sha256=sha256, + size=size, + # Zero ticks: the shared bar must never run backwards + progress=None + if extract_progress is None + else _batched_download_progress(name, version, extract_progress), + ) + if not batched: + _LOGGER.info("Extracting %s ...", name) + archive_extract_all( + archive, dest, progress_header="Extracting", progress=extract_progress + ) # Validate the layout before recording success, so an unexpected # package is never cached as a working install. _check_layout(name, dest, expect) marker.touch() archive.unlink(missing_ok=True) + + +def install_packages( + specs: Collection[PackageSpec], + downloads_dir: Path, + resolvers: dict[str, Resolver] | None = None, +) -> None: + """Install several packages; prefetched archives extract in parallel under + one shared bar, the rest take the sequential ``install_package`` path. + ``resolvers`` replaces the registry lookup by name; the first failure is + re-raised.""" + resolvers = resolvers or {} + pending: list[tuple[PackageSpec, int]] = [] + rest: list[PackageSpec] = [] + for spec in specs: + name, version, dest, mirrors, _expect = spec + archive = _archive_path(downloads_dir, name, version) + if is_installed(dest) or mirrors: + rest.append(spec) + continue + try: + # Sized, not hashed: install_package still verifies the archive + size = archive.stat().st_size + except FileNotFoundError: + rest.append(spec) + continue + pending.append((spec, size)) + if len(pending) < 2: + # One archive alone gains nothing from a pool + rest = list(specs) + pending = [] + + def _install(spec: PackageSpec, size: int, tracker: Callable[[int], None]) -> None: + name, version, dest, mirrors, expect = spec + install_package( + name, + version, + dest, + mirrors, + downloads_dir, + expect=expect, + resolve=resolvers.get(name), + extract_progress=lambda frac: tracker(int(frac * size)), + ) + + if pending: + workers = extract_workers(len(pending)) + _LOGGER.info( + "Extracting %d package archive(s) with %d worker(s): %s", + len(pending), + workers, + ", ".join(spec.name for spec, _ in pending), + ) + failures = run_batch_downloads( + "Extracting packages", + [(spec[0], size, partial(_install, spec, size)) for spec, size in pending], + max_workers=workers, + ) + if failures: + # The raised exception may not name the package; nothing runs + # behind this pass to redo the work + warn_batch_failures(failures, "Could not install %s: %s") + raise failures[0][1] + for name, version, dest, mirrors, expect in rest: + install_package( + name, + version, + dest, + mirrors, + downloads_dir, + expect=expect, + resolve=resolvers.get(name), + ) diff --git a/tests/unit_tests/test_arduino8266_framework.py b/tests/unit_tests/test_arduino8266_framework.py index a4193b9a748..5ffcea0114a 100644 --- a/tests/unit_tests/test_arduino8266_framework.py +++ b/tests/unit_tests/test_arduino8266_framework.py @@ -92,16 +92,13 @@ def test_check_and_install_mirror_skips_pinned_toolchain(tmp_path: Path) -> None patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}), patch.object(framework, "ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS", ["http://f"]), patch.object(framework, "ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS", ["http://m"]), - patch.object(framework, "install_package") as mock_install, + patch.object(framework, "install_packages") as mock_install, patch.object(framework, "prefetch_packages") as mock_prefetch, patch.object(framework, "find_ninja", return_value=tmp_path / "ninja"), ): framework.check_and_install(cv.Version(3, 1, 2)) assert mock_prefetch.call_args.args[2] == {} - assert [call.kwargs["resolve"] for call in mock_install.call_args_list] == [ - None, - None, - ] + assert mock_install.call_args.args[2] == {} def test_check_and_install_installed_toolchain_on_unsupported_host( @@ -142,7 +139,7 @@ def test_check_and_install_unsupported_host_without_toolchain_raises( def test_check_and_install_returns_paths(tmp_path: Path) -> None: with ( patch.dict(os.environ, {"ESPHOME_ARDUINO8266_PREFIX": str(tmp_path)}), - patch.object(framework, "install_package") as mock_install, + patch.object(framework, "install_packages") as mock_install, patch.object(framework, "prefetch_packages") as mock_prefetch, patch.object(framework, "find_ninja", return_value=tmp_path / "ninja"), ): @@ -150,54 +147,38 @@ def test_check_and_install_returns_paths(tmp_path: Path) -> None: assert paths.framework == tmp_path / "frameworks" / _recommended().tag assert paths.toolchain == tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION assert paths.ninja == tmp_path / "ninja" - assert mock_install.call_count == 2 # Full argument pinning: a copy-paste swap between the two near-identical - # calls (mirrors, destination) must not stay green - fw_call, tc_call = mock_install.call_args_list - assert fw_call.args == ( - framework.FRAMEWORK_PACKAGE, - _recommended().tag, - tmp_path / "frameworks" / _recommended().tag, - framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, - tmp_path / "downloads", - ) - assert fw_call.kwargs == { - "expect": ("cores/esp8266", "tools/sdk", "libraries"), - "resolve": _recommended().download, - } - assert tc_call.args == ( - framework.TOOLCHAIN_PACKAGE, - framework.TOOLCHAIN_VERSION, - tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION, - framework.ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, - tmp_path / "downloads", - ) - assert tc_call.kwargs == { - "expect": ("bin", "xtensa-lx106-elf"), - "resolve": framework.toolchain_download, - } - # The prefetch sees the same package specs as the installs - assert mock_prefetch.call_args.args == ( - [ + # specs (mirrors, destination) must not stay green + assert mock_install.call_args.args == ( + ( ( framework.FRAMEWORK_PACKAGE, _recommended().tag, tmp_path / "frameworks" / _recommended().tag, framework.ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS, + ("cores/esp8266", "tools/sdk", "libraries"), ), ( framework.TOOLCHAIN_PACKAGE, framework.TOOLCHAIN_VERSION, tmp_path / "toolchains" / framework.TOOLCHAIN_VERSION, framework.ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS, + ("bin", "xtensa-lx106-elf"), ), - ], + ), tmp_path / "downloads", { framework.FRAMEWORK_PACKAGE: _recommended().download, framework.TOOLCHAIN_PACKAGE: framework.toolchain_download, }, ) + # One spec list feeds both phases, so they cannot drift + assert mock_prefetch.call_args.args == mock_install.call_args.args + # PackageSpec instances, not bare tuples: the batch header reads .name + assert all( + isinstance(spec, framework.PackageSpec) + for spec in mock_install.call_args.args[0] + ) def test_get_build_env_prepends_toolchain_bin(tmp_path: Path) -> None: diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index 22b34c9df51..f3b182073f2 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -523,6 +523,17 @@ class TestArchiveExtractAll: archive_extract_all(archive, dest) assert (dest / "file.txt").read_text() == "hi" + def test_progress_callback_passed_through(self, tmp_path: Path) -> None: + """The progress kwarg reaches the dispatched extractor.""" + archive = tmp_path / "test.tar.gz" + archive.write_bytes(_gzip_tar_bytes({"file.txt": b"hello"})) + dest = tmp_path / "out" + dest.mkdir() + fractions: list[float] = [] + archive_extract_all(archive, dest, progress=fractions.append) + assert fractions[-1] == 1 + assert (dest / "file.txt").read_bytes() == b"hello" + def test_invalid_type_raises_type_error(self) -> None: with pytest.raises(TypeError, match="archive must be"): archive_extract_all(42, ".") # type: ignore[arg-type] @@ -1951,6 +1962,19 @@ class TestTarExtractAllBranches: mock_pb.assert_called_once_with("Extracting") mock_pb.return_value.update.assert_called() + def test_progress_callback_replaces_bar(self, tmp_path: Path) -> None: + """A progress callback wins over progress_header and ends at 1.0.""" + buf = _make_tar([_reg("a.txt"), _reg("b.txt")], {"a.txt": b"x", "b.txt": b"y"}) + fractions: list[float] = [] + with patch("esphome.framework_helpers.ProgressBar") as mock_pb: + _tar_extract_all( + buf, tmp_path, progress_header="Extracting", progress=fractions.append + ) + mock_pb.assert_not_called() + assert fractions == sorted(fractions) + assert fractions[-1] == 1 + assert (tmp_path / "a.txt").is_file() + # --------------------------------------------------------------------------- # _zip_extract_all — additional branch coverage @@ -1980,6 +2004,19 @@ class TestZipExtractAllBranches: mock_pb.assert_called_once_with("Unzipping") mock_pb.return_value.update.assert_called() + def test_progress_callback_replaces_bar(self, tmp_path: Path) -> None: + """A progress callback wins over progress_header and ends at 1.0.""" + buf = _make_zip([("a.txt", "aaa"), ("b.txt", "bbb")]) + fractions: list[float] = [] + with patch("esphome.framework_helpers.ProgressBar") as mock_pb: + _zip_extract_all( + buf, tmp_path, progress_header="Unzipping", progress=fractions.append + ) + mock_pb.assert_not_called() + assert fractions == sorted(fractions) + assert fractions[-1] == 1 + assert (tmp_path / "a.txt").is_file() + # --------------------------------------------------------------------------- # _rename_with_retry @@ -2137,6 +2174,20 @@ class TestSevenZipExtractAll: mock_pb.assert_called_once_with("Unpacking 7z") mock_pb.return_value.update.assert_called() + def test_progress_callback_replaces_bar(self, tmp_path: Path) -> None: + """A progress callback wins over progress_header; 7z reports 1.0 once.""" + buf = self._make_7z({"file.txt": b"x"}) + out = tmp_path / "out" + out.mkdir() + fractions: list[float] = [] + with patch("esphome.framework_helpers.ProgressBar") as mock_pb: + _7z_extract_all( + buf, out, progress_header="Unpacking 7z", progress=fractions.append + ) + mock_pb.assert_not_called() + assert fractions == [1] + assert (out / "file.txt").is_file() + def test_absolute_path_in_names_skipped(self, tmp_path: Path) -> None: """Names that resolve as absolute are silently skipped.""" import py7zr @@ -2294,18 +2345,50 @@ def test_resume_fetch_job_threads_tracker(tmp_path: Path) -> None: ) -def test_warn_prefetch_failures_names_each_failure( +def test_warn_batch_failures_names_each_failure( caplog: pytest.LogCaptureFixture, ) -> None: """The shared failure loop warns per job with the failure reason.""" - from esphome.framework_helpers import warn_prefetch_failures + from esphome.framework_helpers import warn_batch_failures - warn_prefetch_failures([("toolchain-x@1", OSError("down"))]) + warn_batch_failures( + [("toolchain-x@1", OSError("down"))], "Could not prefetch %s: %s" + ) assert "Could not prefetch toolchain-x@1: down" in caplog.text - warn_prefetch_failures([("lib", OSError("gone"))], "Prefetch of %s failed: %s") + warn_batch_failures([("lib", OSError("gone"))], "Prefetch of %s failed: %s") assert "Prefetch of lib failed: gone" in caplog.text +def test_extract_workers_caps_and_clamps() -> None: + """Extraction stops scaling well before high core counts, and a batch + never asks for more workers than it has archives.""" + from esphome.framework_helpers import BATCH_EXTRACT_WORKERS, extract_workers + + with patch("esphome.framework_helpers.get_usable_cpu_count", return_value=64): + assert extract_workers() == BATCH_EXTRACT_WORKERS + assert extract_workers(2) == 2 + with patch("esphome.framework_helpers.get_usable_cpu_count", return_value=1): + assert extract_workers(8) == 1 + + +def test_warn_batch_failures_unexpected_error_keeps_traceback( + caplog: pytest.LogCaptureFixture, +) -> None: + """An unexpected error type is not reduced to a bare message; expected + download failures stay message-only at WARNING.""" + from esphome.framework_helpers import warn_batch_failures + + with caplog.at_level(logging.DEBUG): + warn_batch_failures( + [("pkg", TypeError("bad call")), ("lib", OSError("down"))], + "Could not install %s: %s", + ) + warnings = {r.getMessage(): r for r in caplog.records if r.levelname == "WARNING"} + assert warnings["Could not install pkg: bad call"].exc_info is not None + assert warnings["Could not install lib: down"].exc_info is None + assert "Failure detail" in caplog.text + + @pytest.mark.parametrize( ("platform", "input_path", "expected"), [ diff --git a/tests/unit_tests/test_platformio_prefetch.py b/tests/unit_tests/test_platformio_prefetch.py index 774493ecf41..ef573767f74 100644 --- a/tests/unit_tests/test_platformio_prefetch.py +++ b/tests/unit_tests/test_platformio_prefetch.py @@ -1727,7 +1727,7 @@ def test_preinstall_uses_distinct_managers_in_parallel(tmp_path: Path) -> None: barrier.wait() seed = _WaveManager(str(tmp_path)) - with patch.object(pf, "get_usable_cpu_count", return_value=2): + with patch("esphome.framework_helpers.get_usable_cpu_count", return_value=2): pf._preinstall( seed, [ diff --git a/tests/unit_tests/test_platformio_registry.py b/tests/unit_tests/test_platformio_registry.py index 9f1c1408469..c30cdc7d6c8 100644 --- a/tests/unit_tests/test_platformio_registry.py +++ b/tests/unit_tests/test_platformio_registry.py @@ -2,8 +2,10 @@ from __future__ import annotations -from contextlib import contextmanager +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, contextmanager import json +import logging import os from pathlib import Path from unittest.mock import MagicMock, patch @@ -45,7 +47,7 @@ def test_registry_download_resolves_once_per_process() -> None: @pytest.fixture(autouse=True) -def _fresh_registry_cache(): +def _fresh_registry_cache() -> Iterator[None]: # registry_download memoizes per process; tests reuse package names registry.registry_download.cache_clear() yield @@ -112,7 +114,7 @@ def _http_response(text: str) -> MagicMock: return resp -def _registry_response(files: list[dict]): +def _registry_response(files: list[dict]) -> AbstractContextManager[MagicMock]: """Patch the consolidated HTTP path to serve a canned registry response.""" payload = {"versions": [{"name": "1.0.0", "files": files}]} return patch.object( @@ -308,7 +310,11 @@ def test_install_package_downloads_via_registry(tmp_path: Path) -> None: "pkg", "1.0.0", dest, [], tmp_path / "dl", expect=("payload",) ) assert mock_download.call_args[0][0] == "http://x/pkg.tar.gz" - assert mock_download.call_args[1] == {"sha256": "abc123", "size": 42} + assert mock_download.call_args[1] == { + "sha256": "abc123", + "size": 42, + "progress": None, + } def test_install_package_downloads_pinned(tmp_path: Path) -> None: @@ -334,7 +340,11 @@ def test_install_package_downloads_pinned(tmp_path: Path) -> None: ) mock_registry.assert_not_called() assert mock_download.call_args[0][0] == "http://y/pinned.tar.gz" - assert mock_download.call_args[1] == {"sha256": "def456", "size": 7} + assert mock_download.call_args[1] == { + "sha256": "def456", + "size": 7, + "progress": None, + } def test_install_package_mirror_wins_over_pinned(tmp_path: Path) -> None: @@ -546,8 +556,10 @@ def test_registry_download_non_list_system_is_named() -> None: registry.registry_download("pkg", "1.0.0") -def _resolve_for(sizes: dict[str, int | None]): - def resolve(name: str, version: str): +def _resolve_for( + sizes: dict[str, int | None], +) -> Callable[[str, str], tuple[str, str, int | None]]: + def resolve(name: str, version: str) -> tuple[str, str, int | None]: size = sizes[name] if size == -1: raise EsphomeError("registry down") @@ -567,8 +579,8 @@ def test_prefetch_packages_downloads_pending_in_parallel(tmp_path: Path) -> None ): registry.prefetch_packages( [ - ("a", "1.0", tmp_path / "a", []), - ("b", "2.0", tmp_path / "b", []), + ("a", "1.0", tmp_path / "a", [], ()), + ("b", "2.0", tmp_path / "b", [], ()), ], tmp_path / "dl", ) @@ -595,8 +607,8 @@ def test_prefetch_packages_uses_pinned_download(tmp_path: Path) -> None: ): registry.prefetch_packages( [ - ("a", "1.0", tmp_path / "a", []), - ("b", "2.0", tmp_path / "b", []), + ("a", "1.0", tmp_path / "a", [], ("payload",)), + ("b", "2.0", tmp_path / "b", [], ("payload",)), ], tmp_path / "dl", {"b": lambda: registry.Download("http://y/b.tar.gz", "def456", 20)}, @@ -626,7 +638,7 @@ def test_prefetch_packages_skips_freshly_installed_dest(tmp_path: Path) -> None: registry, "registry_download", side_effect=_resolve_for({"a": 10}) ), ): - registry.prefetch_packages([("a", "1.0", dest, [])], tmp_path / "dl") + registry.prefetch_packages([("a", "1.0", dest, [], ())], tmp_path / "dl") mock_download.assert_not_called() @@ -666,7 +678,7 @@ def test_prefetch_packages_waits_with_the_holders_progress( ), ): registry.prefetch_packages( - [("a", "1.0", dest, []), ("b", "2.0", tmp_path / "b", [])], + [("a", "1.0", dest, [], ()), ("b", "2.0", tmp_path / "b", [], ())], tmp_path / "dl", ) assert ticks == [0, 3, 10, 10] @@ -687,7 +699,10 @@ def test_prefetch_packages_leaves_a_long_held_lock_to_its_holder( ), ): registry.prefetch_packages( - [("a", "1.0", tmp_path / "a", []), ("b", "2.0", tmp_path / "b", [])], + [ + ("a", "1.0", tmp_path / "a", [], ()), + ("b", "2.0", tmp_path / "b", [], ()), + ], tmp_path / "dl", ) mock_download.assert_not_called() @@ -713,8 +728,8 @@ def test_prefetch_packages_dedupes_duplicate_entries(tmp_path: Path) -> None: ): registry.prefetch_packages( [ - ("a", "1.0", tmp_path / "a", []), - ("a", "1.0", tmp_path / "a", []), + ("a", "1.0", tmp_path / "a", [], ()), + ("a", "1.0", tmp_path / "a", [], ()), ], tmp_path / "dl", ) @@ -735,8 +750,8 @@ def test_prefetch_packages_single_pending_skips(tmp_path: Path) -> None: ): registry.prefetch_packages( [ - ("a", "1.0", marker_dest, []), - ("b", "2.0", tmp_path / "b", []), + ("a", "1.0", marker_dest, [], ()), + ("b", "2.0", tmp_path / "b", [], ()), ], tmp_path / "dl", ) @@ -758,9 +773,9 @@ def test_prefetch_packages_mirror_and_sizeless_stay_sequential( ): registry.prefetch_packages( [ - ("a", "1.0", tmp_path / "a", ["http://mirror/{VERSION}"]), - ("b", "2.0", tmp_path / "b", []), - ("c", "3.0", tmp_path / "c", []), + ("a", "1.0", tmp_path / "a", ["http://mirror/{VERSION}"], ()), + ("b", "2.0", tmp_path / "b", [], ()), + ("c", "3.0", tmp_path / "c", [], ()), ], tmp_path / "dl", ) @@ -781,8 +796,8 @@ def test_prefetch_packages_resolve_failure_defers_to_install( ): registry.prefetch_packages( [ - ("a", "1.0", tmp_path / "a", []), - ("b", "2.0", tmp_path / "b", []), + ("a", "1.0", tmp_path / "a", [], ()), + ("b", "2.0", tmp_path / "b", [], ()), ], tmp_path / "dl", ) @@ -803,8 +818,8 @@ def test_prefetch_packages_complete_archive_skipped(tmp_path: Path) -> None: ): registry.prefetch_packages( [ - ("a", "1.0", tmp_path / "a", []), - ("b", "2.0", tmp_path / "b", []), + ("a", "1.0", tmp_path / "a", [], ()), + ("b", "2.0", tmp_path / "b", [], ()), ], dl, ) @@ -826,8 +841,8 @@ def test_prefetch_packages_download_failure_is_debug( ): registry.prefetch_packages( [ - ("a", "1.0", tmp_path / "a", []), - ("b", "2.0", tmp_path / "b", []), + ("a", "1.0", tmp_path / "a", [], ()), + ("b", "2.0", tmp_path / "b", [], ()), ], tmp_path / "dl", ) @@ -851,9 +866,225 @@ def test_prefetch_packages_unexpected_failure_warns( ): registry.prefetch_packages( [ - ("a", "1.0", tmp_path / "a", []), - ("b", "2.0", tmp_path / "b", []), + ("a", "1.0", tmp_path / "a", [], ()), + ("b", "2.0", tmp_path / "b", [], ()), ], tmp_path / "dl", ) assert "TypeError" in caplog.text + + +def _spec( + name: str, + version: str, + dest: Path, + mirrors: list[str] | None = None, + expect: tuple[str, ...] = ("payload",), +) -> registry.PackageSpec: + return registry.PackageSpec(name, version, dest, mirrors or [], expect) + + +def test_install_packages_extracts_verified_archives_in_parallel( + tmp_path: Path, +) -> None: + """Two prefetched archives install concurrently under one shared bar.""" + dl = tmp_path / "dl" + dl.mkdir() + (dl / "a-1.0").write_bytes(b"x" * 10) + (dl / "b-2.0").write_bytes(b"y" * 20) + with patch.object(registry, "install_package") as mock_install: + registry.install_packages( + [_spec("a", "1.0", tmp_path / "a"), _spec("b", "2.0", tmp_path / "b")], dl + ) + assert mock_install.call_count == 2 + calls = sorted(mock_install.call_args_list, key=lambda c: c[0][0]) + for c, (name, version) in zip(calls, [("a", "1.0"), ("b", "2.0")], strict=True): + assert c[0][:3] == (name, version, tmp_path / name) + assert c[1]["expect"] == ("payload",) + assert callable(c[1]["extract_progress"]) + # Driving the tracker exercises the fraction-to-bytes scaling + c[1]["extract_progress"](0.5) + c[1]["extract_progress"](1.0) + + +def test_install_packages_single_archive_stays_sequential(tmp_path: Path) -> None: + """One verified archive has nothing to parallelize; original order kept.""" + dl = tmp_path / "dl" + dl.mkdir() + (dl / "a-1.0").write_bytes(b"x") + specs = [_spec("a", "1.0", tmp_path / "a"), _spec("b", "2.0", tmp_path / "b")] + with patch.object(registry, "install_package") as mock_install: + registry.install_packages(specs, dl) + assert [c[0][0] for c in mock_install.call_args_list] == ["a", "b"] + for c in mock_install.call_args_list: + assert "extract_progress" not in c[1] + + +def test_batched_download_progress_announces_a_real_download_once( + caplog: pytest.LogCaptureFixture, +) -> None: + """A batched archive that fails verification streams again behind a bar + that cannot move, so it says so once; a verified archive credits itself + in one tick and stays quiet.""" + ticks: list[float] = [] + with caplog.at_level(logging.INFO): + progress = registry._batched_download_progress("pkg", "1.0.0", ticks.append) + progress(0) + progress(4096) + assert caplog.text.count("Re-downloading pkg 1.0.0") == 1 + # The shared bar never moves for a download; it tracks extraction + assert ticks == [0.0, 0.0] + + # A resumed .part starts mid-file, so the first tick is not zero + caplog.clear() + ticks.clear() + with caplog.at_level(logging.INFO): + resumed = registry._batched_download_progress("pkg", "1.0.0", ticks.append) + resumed(8192) + resumed(16384) + assert caplog.text.count("Re-downloading pkg 1.0.0") == 1 + + caplog.clear() + ticks.clear() + with caplog.at_level(logging.INFO): + verified = registry._batched_download_progress("pkg", "1.0.0", ticks.append) + verified(42) + assert "Re-downloading" not in caplog.text + assert ticks == [0.0] + + +def test_install_packages_no_batch_logs_no_header( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The batch header must not describe a batch that never ran.""" + dl = tmp_path / "dl" + dl.mkdir() + specs = [_spec("a", "1.0", tmp_path / "a")] + with ( + caplog.at_level(logging.INFO), + patch.object(registry, "install_package"), + ): + registry.install_packages(specs, dl) + assert "Extracting 0" not in caplog.text + assert "package archive(s) with" not in caplog.text + + +def test_install_packages_mirror_and_marker_stay_sequential(tmp_path: Path) -> None: + """Mirror overrides and marker hits never enter the parallel batch.""" + dl = tmp_path / "dl" + dl.mkdir() + for name, ver in (("a", "1.0"), ("b", "2.0"), ("c", "3.0"), ("d", "4.0")): + (dl / f"{name}-{ver}").write_bytes(b"x") + marked = tmp_path / "c" + marked.mkdir() + (marked / ".esphome_extracted").touch() + specs = [ + _spec("a", "1.0", tmp_path / "a"), + _spec("b", "2.0", tmp_path / "b", mirrors=["http://m"]), + _spec("c", "3.0", marked), + _spec("d", "4.0", tmp_path / "d"), + ] + with patch.object(registry, "install_package") as mock_install: + registry.install_packages(specs, dl) + sequential = [ + c for c in mock_install.call_args_list if "extract_progress" not in c[1] + ] + batched = [c for c in mock_install.call_args_list if "extract_progress" in c[1]] + assert sorted(c[0][0] for c in sequential) == ["b", "c"] + assert sorted(c[0][0] for c in batched) == ["a", "d"] + + +def test_install_packages_first_failure_reraised( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Installs are mandatory: the first failure propagates, extras are logged.""" + dl = tmp_path / "dl" + dl.mkdir() + (dl / "a-1.0").write_bytes(b"x") + (dl / "b-2.0").write_bytes(b"y") + boom = EsphomeError("bad layout") + + def _fail(name: str, *_a, **_kw) -> None: + raise boom if name == "a" else EsphomeError("also bad") + + with ( + patch.object(registry, "install_package", side_effect=_fail), + pytest.raises(EsphomeError), + ): + registry.install_packages( + [_spec("a", "1.0", tmp_path / "a"), _spec("b", "2.0", tmp_path / "b")], dl + ) + # Every failure is named, including the re-raised one: its exception + # message may not identify the package + assert "Could not install a" in caplog.text + assert "Could not install b" in caplog.text + + +@contextmanager +def _batched_install( + tmp_path: Path, + extract_progress: Callable[[float], None] | None, + prefill_archive: bool = True, +) -> Iterator[tuple[MagicMock, MagicMock]]: + """Run a batched install_package of pkg@1.0.0; yields the download and + extract mocks.""" + dest = tmp_path / "pkg" + if prefill_archive: + (tmp_path / "dl").mkdir() + (tmp_path / "dl" / "pkg-1.0.0").write_bytes(b"x") + with ( + patch.object(registry, "download_with_resume") as mock_download, + patch.object(registry, "archive_extract_all") as mock_extract, + patch.object( + registry, + "registry_download", + return_value=("http://x/pkg.tar.gz", "abc123", 42), + ), + ): + mock_extract.side_effect = lambda *_a, **_kw: (dest / "payload").mkdir( + parents=True + ) + registry.install_package( + "pkg", + "1.0.0", + dest, + [], + tmp_path / "dl", + expect=("payload",), + extract_progress=extract_progress, + ) + yield mock_download, mock_extract + + +def test_install_package_extract_progress_suppresses_bars( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A batched install routes extraction fractions to the caller and keeps + both private bars and per-package INFO lines off the shared bar.""" + fractions: list[float] = [] + with ( + caplog.at_level(logging.INFO), + _batched_install(tmp_path, fractions.append) as (mock_download, mock_extract), + ): + pass + assert mock_extract.call_args[1]["progress"] == fractions.append + # The download tracker reports zero bytes, keeping the shared bar honest + download_progress = mock_download.call_args[1]["progress"] + assert callable(download_progress) + download_progress(42) + assert fractions == [0.0] + assert "Downloading pkg" not in caplog.text + assert "Extracting pkg" not in caplog.text + + +def test_install_package_batched_missing_archive_keeps_info_log( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A batched archive that unexpectedly needs a real download keeps the + INFO line; the shared bar shows no progress for it.""" + with ( + caplog.at_level(logging.INFO), + _batched_install(tmp_path, lambda _frac: None, prefill_archive=False), + ): + pass + assert "Downloading pkg 1.0.0" in caplog.text From ac1190094a22c91d4ea6109c329355bb69b6974a Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Wed, 30 Sep 2026 01:48:10 +0100 Subject: [PATCH 062/133] [ethernet] Add KSZ8851SNL SPI ethernet support (#19883) --- esphome/components/ethernet/__init__.py | 12 +- .../components/ethernet/ethernet_component.h | 1 + .../ethernet/ethernet_component_esp32.cpp | 21 ++++ esphome/core/defines.h | 2 + .../ethernet/config/ksz8851snl_idf5.yaml | 16 +++ .../ethernet/config/ksz8851snl_idf6.yaml | 18 +++ .../ethernet/test_ksz8851snl.py | 105 ++++++++++++++++++ .../ethernet/common-ksz8851snl.yaml | 19 ++++ .../ethernet/test-ksz8851snl.esp32-idf.yaml | 2 + 9 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 tests/component_tests/ethernet/config/ksz8851snl_idf5.yaml create mode 100644 tests/component_tests/ethernet/config/ksz8851snl_idf6.yaml create mode 100644 tests/component_tests/ethernet/test_ksz8851snl.py create mode 100644 tests/components/ethernet/common-ksz8851snl.yaml create mode 100644 tests/components/ethernet/test-ksz8851snl.esp32-idf.yaml diff --git a/esphome/components/ethernet/__init__.py b/esphome/components/ethernet/__init__.py index 920f2e5e4bf..ce7206bf57a 100644 --- a/esphome/components/ethernet/__init__.py +++ b/esphome/components/ethernet/__init__.py @@ -138,6 +138,7 @@ ETHERNET_TYPES = { "GENERIC": EthernetType.ETHERNET_TYPE_GENERIC, "YT8531": EthernetType.ETHERNET_TYPE_YT8531, "CH390": EthernetType.ETHERNET_TYPE_CH390, + "KSZ8851SNL": EthernetType.ETHERNET_TYPE_KSZ8851SNL, } # PHY types that need compile-time defines for conditional compilation @@ -160,6 +161,7 @@ _PHY_TYPE_TO_DEFINE = { "GENERIC": "USE_ETHERNET_GENERIC", "YT8531": "USE_ETHERNET_YT8531", "CH390": "USE_ETHERNET_CH390", + "KSZ8851SNL": "USE_ETHERNET_KSZ8851SNL", } @@ -184,13 +186,14 @@ _IDF6_ETHERNET_COMPONENTS: dict[str, IDFRegistryComponent] = { "ENC28J60": IDFRegistryComponent("espressif/enc28j60", "1.0.1"), "LAN8670": IDFRegistryComponent("espressif/lan867x", "2.0.0"), "CH390": IDFRegistryComponent("espressif/ch390", "0.3.0"), + "KSZ8851SNL": IDFRegistryComponent("espressif/ksz8851snl", "1.2.0"), } # These types are always external IDF components (never built-in to ESP-IDF) _ALWAYS_EXTERNAL_IDF_COMPONENTS = {"LAN8670", "ENC28J60", "CH390"} # ESP32-only SPI ethernet types (W5100 is RP2040-only, no ESP-IDF driver) -SPI_ETHERNET_TYPES = {"W5500", "DM9051", "ENC28J60", "CH390"} +SPI_ETHERNET_TYPES = {"W5500", "DM9051", "ENC28J60", "CH390", "KSZ8851SNL"} # RP2-supported ethernet types (SPI and PIO QSPI). Applies to the whole # RP2 family (RP2040 and RP2350); the chip-specific W5100 caveat in the # comment above is about ESP-IDF driver coverage, not the RP2 platform. @@ -534,6 +537,12 @@ SPI_SCHEMA_ENC28J60 = _spi_schema(default_clock="20MHz", max_clock=int(20e6)) # https://www.wch-ic.com/downloads/CH390DS1_PDF.html SPI_SCHEMA_CH390 = _spi_schema(max_clock=int(72e6)) +# The KSZ8851SNL rates fSCLK at 40 MHz maximum, so the shared 80 MHz ceiling is +# out of spec while the 26.67 MHz default is not. KSZ8851SNL/SNLI datasheet +# DS00002381C, table 7-1: +# https://ww1.microchip.com/downloads/aemDocuments/documents/UNG/ProductDocuments/DataSheets/KSZ8851SNL-Single-Port-Ethernet-Controller-with-SPI-DS00002381C.pdf +SPI_SCHEMA_KSZ8851SNL = _spi_schema(max_clock=int(40e6)) + CONFIG_SCHEMA = cv.All( cv.typed_schema( { @@ -549,6 +558,7 @@ CONFIG_SCHEMA = cv.All( "OPENETH": cv.All(BASE_SCHEMA, cv.only_on([Platform.ESP32])), "DM9051": SPI_SCHEMA, "CH390": SPI_SCHEMA_CH390, + "KSZ8851SNL": SPI_SCHEMA_KSZ8851SNL, "ENC28J60": SPI_SCHEMA_ENC28J60, "W6100": cv.All(SPI_SCHEMA, cv.only_on([Platform.RP2])), "W6300": cv.All(SPI_SCHEMA, cv.only_on([Platform.RP2])), diff --git a/esphome/components/ethernet/ethernet_component.h b/esphome/components/ethernet/ethernet_component.h index 2b67b9093b7..75dcfb45f66 100644 --- a/esphome/components/ethernet/ethernet_component.h +++ b/esphome/components/ethernet/ethernet_component.h @@ -92,6 +92,7 @@ enum EthernetType : uint8_t { ETHERNET_TYPE_GENERIC, ETHERNET_TYPE_YT8531, ETHERNET_TYPE_CH390, + ETHERNET_TYPE_KSZ8851SNL, }; struct ManualIP { diff --git a/esphome/components/ethernet/ethernet_component_esp32.cpp b/esphome/components/ethernet/ethernet_component_esp32.cpp index ea732e84ff9..d15ef5a34c6 100644 --- a/esphome/components/ethernet/ethernet_component_esp32.cpp +++ b/esphome/components/ethernet/ethernet_component_esp32.cpp @@ -41,6 +41,10 @@ #include "esp_eth_mac_dm9051.h" #include "esp_eth_phy_dm9051.h" #endif +#ifdef USE_ETHERNET_KSZ8851SNL +#include "esp_eth_mac_ksz8851snl.h" +#include "esp_eth_phy_ksz8851snl.h" +#endif #endif // ESP_IDF_VERSION >= 6.0.0 // LAN867x header exists on all IDF versions (external component since IDF 5.3) @@ -264,6 +268,8 @@ void EthernetComponent::ethernet_lazy_init_() { eth_enc28j60_config_t enc28j60_config = ETH_ENC28J60_DEFAULT_CONFIG(host, &devcfg); #elif defined(USE_ETHERNET_CH390) eth_ch390_config_t ch390_config = ETH_CH390_DEFAULT_CONFIG(host, &devcfg); +#elif defined(USE_ETHERNET_KSZ8851SNL) + eth_ksz8851snl_config_t ksz8851snl_config = ETH_KSZ8851SNL_DEFAULT_CONFIG(host, &devcfg); #endif #if defined(USE_ETHERNET_W5500) @@ -290,6 +296,11 @@ void EthernetComponent::ethernet_lazy_init_() { #ifdef USE_ETHERNET_SPI_POLLING_SUPPORT ch390_config.poll_period_ms = this->polling_interval_; #endif +#elif defined(USE_ETHERNET_KSZ8851SNL) + ksz8851snl_config.int_gpio_num = this->interrupt_pin_; +#ifdef USE_ETHERNET_SPI_POLLING_SUPPORT + ksz8851snl_config.poll_period_ms = this->polling_interval_; +#endif #endif phy_config.phy_addr = this->phy_addr_spi_; @@ -420,6 +431,12 @@ void EthernetComponent::ethernet_lazy_init_() { this->phy_ = esp_eth_phy_new_ch390(&phy_config); break; } +#elif defined(USE_ETHERNET_KSZ8851SNL) + case ETHERNET_TYPE_KSZ8851SNL: { + mac = esp_eth_mac_new_ksz8851snl(&ksz8851snl_config, &mac_config); + this->phy_ = esp_eth_phy_new_ksz8851snl(&phy_config); + break; + } #endif #endif default: { @@ -593,6 +610,10 @@ void EthernetComponent::dump_config() { case ETHERNET_TYPE_CH390: eth_type = "CH390"; break; +#elif defined(USE_ETHERNET_KSZ8851SNL) + case ETHERNET_TYPE_KSZ8851SNL: + eth_type = "KSZ8851SNL"; + break; #endif #ifdef USE_ETHERNET_OPENETH case ETHERNET_TYPE_OPENETH: diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 6ee6f97947d..f845154389b 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -473,8 +473,10 @@ #define USE_ETHERNET_W6300 #define USE_ETHERNET_DM9051 #define USE_ETHERNET_CH390 +#define USE_ETHERNET_KSZ8851SNL #define CONFIG_ETH_SPI_ETHERNET_W5500 1 #define CONFIG_ETH_SPI_ETHERNET_DM9051 1 +#define CONFIG_ETH_SPI_ETHERNET_KSZ8851SNL 1 #define CONFIG_ETH_USE_ESP32_EMAC 1 #define USE_ETHERNET_MANUAL_IP #define USE_ETHERNET_IP_STATE_LISTENERS diff --git a/tests/component_tests/ethernet/config/ksz8851snl_idf5.yaml b/tests/component_tests/ethernet/config/ksz8851snl_idf5.yaml new file mode 100644 index 00000000000..7c8f3336844 --- /dev/null +++ b/tests/component_tests/ethernet/config/ksz8851snl_idf5.yaml @@ -0,0 +1,16 @@ +esphome: + name: test + +esp32: + board: esp32dev + +ethernet: + id: eth_component + type: KSZ8851SNL + clk_pin: GPIO18 + mosi_pin: GPIO23 + miso_pin: GPIO19 + cs_pin: GPIO5 + interrupt_pin: GPIO36 + reset_pin: GPIO22 + clock_speed: 20MHz diff --git a/tests/component_tests/ethernet/config/ksz8851snl_idf6.yaml b/tests/component_tests/ethernet/config/ksz8851snl_idf6.yaml new file mode 100644 index 00000000000..bc11c6db5b8 --- /dev/null +++ b/tests/component_tests/ethernet/config/ksz8851snl_idf6.yaml @@ -0,0 +1,18 @@ +esphome: + name: test + +esp32: + board: esp32dev + framework: + version: 6.0.0 + +ethernet: + id: eth_component + type: KSZ8851SNL + clk_pin: GPIO18 + mosi_pin: GPIO23 + miso_pin: GPIO19 + cs_pin: GPIO5 + interrupt_pin: GPIO36 + reset_pin: GPIO22 + clock_speed: 20MHz diff --git a/tests/component_tests/ethernet/test_ksz8851snl.py b/tests/component_tests/ethernet/test_ksz8851snl.py new file mode 100644 index 00000000000..ca150100888 --- /dev/null +++ b/tests/component_tests/ethernet/test_ksz8851snl.py @@ -0,0 +1,105 @@ +"""Tests for the KSZ8851SNL SPI ethernet type.""" + +from collections.abc import Callable +from pathlib import Path + +import pytest +from voluptuous import Invalid + +from esphome import config_validation as cv +from esphome.components.esp32 import ( + KEY_BOARD, + KEY_IDF_VERSION, + KEY_VARIANT, + VARIANT_ESP32S3, +) +from esphome.components.esp32.const import ( + KEY_COMPONENTS, + KEY_ESP32, + KEY_REF, + KEY_SDKCONFIG_OPTIONS, +) +from esphome.components.ethernet import CONF_CLOCK_SPEED, CONFIG_SCHEMA +from esphome.const import PlatformFramework +from esphome.core import CORE + +from ..types import SetCoreConfigCallable + +_BASE_CONFIG = { + "type": "KSZ8851SNL", + "clk_pin": 47, + "mosi_pin": 48, + "miso_pin": 14, + "cs_pin": 21, +} + +_SDKCONFIG_OPTION = "CONFIG_ETH_SPI_ETHERNET_KSZ8851SNL" +_IDF_COMPONENT = "espressif/ksz8851snl" + + +def _sdkconfig() -> dict[str, object]: + return CORE.data[KEY_ESP32][KEY_SDKCONFIG_OPTIONS] + + +def _idf_components() -> dict[str, dict[str, str | None]]: + return CORE.data[KEY_ESP32][KEY_COMPONENTS] + + +def test_codegen_on_idf5_wires_the_spi_driver( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """On IDF 5.x the driver is built into esp_eth and enabled by its Kconfig option.""" + main_cpp = generate_main(component_config_path("ksz8851snl_idf5.yaml")) + + assert "eth_component->set_type(ethernet::ETHERNET_TYPE_KSZ8851SNL);" in main_cpp + assert "eth_component->set_cs_pin(5);" in main_cpp + assert "eth_component->set_interrupt_pin(36);" in main_cpp + assert "USE_ETHERNET_KSZ8851SNL" in {d.name for d in CORE.defines} + assert _sdkconfig()[_SDKCONFIG_OPTION] is True + # Built into IDF 5.x, so the registry component must not be pulled in. + assert _IDF_COMPONENT not in _idf_components() + + +def test_codegen_on_idf6_pulls_the_registry_component( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> None: + """IDF 6.0 dropped the built-in driver, so the managed component is required.""" + generate_main(component_config_path("ksz8851snl_idf6.yaml")) + + assert _idf_components()[_IDF_COMPONENT][KEY_REF] == "1.2.0" + # The Kconfig option no longer exists on IDF 6.0. + assert _SDKCONFIG_OPTION not in _sdkconfig() + + +def _set_esp32_s3(set_core_config: SetCoreConfigCallable) -> None: + set_core_config( + PlatformFramework.ESP32_IDF, + platform_data={ + KEY_BOARD: "esp32-s3-devkitc-1", + KEY_VARIANT: VARIANT_ESP32S3, + KEY_IDF_VERSION: cv.Version(5, 3, 2), + }, + ) + # _validate derives use_address from the node name, which has no default here. + CORE.name = "ksz8851snl-test" + + +@pytest.mark.parametrize("clock_speed", ["26.67MHz", "40MHz"]) +def test_accepts_clock_speed_up_to_the_datasheet_maximum( + set_core_config: SetCoreConfigCallable, clock_speed: str +) -> None: + """The datasheet rates fSCLK to 40MHz, so the whole range must be accepted.""" + _set_esp32_s3(set_core_config) + config = CONFIG_SCHEMA({**_BASE_CONFIG, CONF_CLOCK_SPEED: clock_speed}) + assert config[CONF_CLOCK_SPEED] == cv.frequency(clock_speed) + + +def test_rejects_clock_speed_above_the_datasheet_maximum( + set_core_config: SetCoreConfigCallable, +) -> None: + """The shared 80MHz ceiling is out of spec for this part.""" + _set_esp32_s3(set_core_config) + with pytest.raises(Invalid, match="value must be at most 40000000"): + CONFIG_SCHEMA({**_BASE_CONFIG, CONF_CLOCK_SPEED: "80MHz"}) diff --git a/tests/components/ethernet/common-ksz8851snl.yaml b/tests/components/ethernet/common-ksz8851snl.yaml new file mode 100644 index 00000000000..aed9b744dcb --- /dev/null +++ b/tests/components/ethernet/common-ksz8851snl.yaml @@ -0,0 +1,19 @@ +ethernet: + type: KSZ8851SNL + clk_pin: 19 + mosi_pin: 21 + miso_pin: 23 + cs_pin: 18 + interrupt_pin: 36 + reset_pin: 22 + clock_speed: 10Mhz + manual_ip: + static_ip: 192.168.178.56 + gateway: 192.168.178.1 + subnet: 255.255.255.0 + domain: .local + mac_address: "02:AA:BB:CC:DD:01" + on_connect: + - logger.log: "Ethernet connected!" + on_disconnect: + - logger.log: "Ethernet disconnected!" diff --git a/tests/components/ethernet/test-ksz8851snl.esp32-idf.yaml b/tests/components/ethernet/test-ksz8851snl.esp32-idf.yaml new file mode 100644 index 00000000000..4b46dc8f538 --- /dev/null +++ b/tests/components/ethernet/test-ksz8851snl.esp32-idf.yaml @@ -0,0 +1,2 @@ +packages: + ethernet: !include common-ksz8851snl.yaml From 7d904b6330df5f7e47e0aef4a079c5e8e80523f6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 03:31:39 +0200 Subject: [PATCH 063/133] [ens160_base] Use the sub_* helpers for optional child entities (#19921) --- esphome/components/ens160_base/__init__.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/ens160_base/__init__.py b/esphome/components/ens160_base/__init__.py index 1bdfb0c0a6b..6da0d10af90 100644 --- a/esphome/components/ens160_base/__init__.py +++ b/esphome/components/ens160_base/__init__.py @@ -63,15 +63,10 @@ async def to_code_base(config: ConfigType) -> MockObj: var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) - if eco2_config := config.get(CONF_ECO2): - sens = await sensor.new_sensor(eco2_config) - cg.add(var.set_co2(sens)) - if tvoc_config := config.get(CONF_TVOC): - sens = await sensor.new_sensor(tvoc_config) - cg.add(var.set_tvoc(sens)) - if aqi_config := config.get(CONF_AQI): - sens = await sensor.new_sensor(aqi_config) - cg.add(var.set_aqi(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_ECO2, var.set_co2) + await sensors(CONF_TVOC, var.set_tvoc) + await sensors(CONF_AQI, var.set_aqi) if compensation_config := config.get(CONF_COMPENSATION): sens = await cg.get_variable(compensation_config[CONF_TEMPERATURE]) From 203a7f70a375391cec2ebe2c63b0b2f1d170f139 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 03:31:50 +0200 Subject: [PATCH 064/133] [emc2101] Use the sub_* helpers for optional child entities (#19920) --- esphome/components/emc2101/sensor/__init__.py | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/esphome/components/emc2101/sensor/__init__.py b/esphome/components/emc2101/sensor/__init__.py index cc8901cf389..8c6d469fb82 100644 --- a/esphome/components/emc2101/sensor/__init__.py +++ b/esphome/components/emc2101/sensor/__init__.py @@ -55,22 +55,12 @@ CONFIG_SCHEMA = EMC2101_COMPONENT_SCHEMA.extend( async def to_code(config: ConfigType) -> None: - paren = await cg.get_variable(config[CONF_EMC2101_ID]) - var = cg.new_Pvariable(config[CONF_ID], paren) + hub = await cg.get_variable(config[CONF_EMC2101_ID]) + var = cg.new_Pvariable(config[CONF_ID], hub) await cg.register_component(var, config) - if CONF_INTERNAL_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_INTERNAL_TEMPERATURE]) - cg.add(var.set_internal_temperature_sensor(sens)) - - if CONF_EXTERNAL_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_EXTERNAL_TEMPERATURE]) - cg.add(var.set_external_temperature_sensor(sens)) - - if CONF_SPEED in config: - sens = await sensor.new_sensor(config[CONF_SPEED]) - cg.add(var.set_speed_sensor(sens)) - - if CONF_DUTY_CYCLE in config: - sens = await sensor.new_sensor(config[CONF_DUTY_CYCLE]) - cg.add(var.set_duty_cycle_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_INTERNAL_TEMPERATURE, var.set_internal_temperature_sensor) + await sensors(CONF_EXTERNAL_TEMPERATURE, var.set_external_temperature_sensor) + await sensors(CONF_SPEED, var.set_speed_sensor) + await sensors(CONF_DUTY_CYCLE, var.set_duty_cycle_sensor) From 83d689d944b7c4b7c5eb4ee78b2e55649f027bc4 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 03:32:02 +0200 Subject: [PATCH 065/133] [xiaomi_hhccjcy10] Use the sub_* helpers for optional child entities (#19919) --- esphome/components/xiaomi_hhccjcy10/sensor.py | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/esphome/components/xiaomi_hhccjcy10/sensor.py b/esphome/components/xiaomi_hhccjcy10/sensor.py index d1ce8dfcf77..c5d256db1d3 100644 --- a/esphome/components/xiaomi_hhccjcy10/sensor.py +++ b/esphome/components/xiaomi_hhccjcy10/sensor.py @@ -81,18 +81,9 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature(sens)) - if moisture_config := config.get(CONF_MOISTURE): - sens = await sensor.new_sensor(moisture_config) - cg.add(var.set_moisture(sens)) - if illuminance_config := config.get(CONF_ILLUMINANCE): - sens = await sensor.new_sensor(illuminance_config) - cg.add(var.set_illuminance(sens)) - if conductivity_config := config.get(CONF_CONDUCTIVITY): - sens = await sensor.new_sensor(conductivity_config) - cg.add(var.set_conductivity(sens)) - if battery_level_config := config.get(CONF_BATTERY_LEVEL): - sens = await sensor.new_sensor(battery_level_config) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_MOISTURE, var.set_moisture) + await sensors(CONF_ILLUMINANCE, var.set_illuminance) + await sensors(CONF_CONDUCTIVITY, var.set_conductivity) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From ef4d06e0ea6837411d8d8e0b5c0686cc428b4213 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 03:32:13 +0200 Subject: [PATCH 066/133] [ld6002b] Use the sub_* helpers for optional child entities (#19918) --- esphome/components/ld6002b/binary_sensor.py | 5 ++--- esphome/components/ld6002b/sensor.py | 10 +++------- esphome/components/ld6002b/text_sensor.py | 9 +++------ 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/esphome/components/ld6002b/binary_sensor.py b/esphome/components/ld6002b/binary_sensor.py index 74095d5dede..36405f06602 100644 --- a/esphome/components/ld6002b/binary_sensor.py +++ b/esphome/components/ld6002b/binary_sensor.py @@ -40,9 +40,8 @@ CONFIG_SCHEMA = ( async def to_code(config: ConfigType) -> None: hub = await cg.get_variable(config[CONF_LD6002B_ID]) - if target_config := config.get(CONF_TARGET): - sens = await binary_sensor.new_binary_sensor(target_config) - cg.add(hub.set_presence_binary_sensor(sens)) + binary_sensors = binary_sensor.sub_binary_sensors(config) + await binary_sensors(CONF_TARGET, hub.set_presence_binary_sensor) for i in range(MAX_TARGETS): if target_config := config.get(f"target_{i + 1}"): diff --git a/esphome/components/ld6002b/sensor.py b/esphome/components/ld6002b/sensor.py index cceefb3837a..5653f8b9449 100644 --- a/esphome/components/ld6002b/sensor.py +++ b/esphome/components/ld6002b/sensor.py @@ -154,13 +154,9 @@ CONFIG_SCHEMA = ( async def to_code(config: ConfigType) -> None: hub = await cg.get_variable(config[CONF_LD6002B_ID]) - if target_count_config := config.get(CONF_TARGET_COUNT): - sens = await sensor.new_sensor(target_count_config) - cg.add(hub.set_target_count_sensor(sens)) - - if point_count_config := config.get(CONF_POINT_COUNT): - sens = await sensor.new_sensor(point_count_config) - cg.add(hub.set_point_count_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TARGET_COUNT, hub.set_target_count_sensor) + await sensors(CONF_POINT_COUNT, hub.set_point_count_sensor) for i in range(MAX_TARGETS): if target_config := config.get(f"target_{i + 1}"): diff --git a/esphome/components/ld6002b/text_sensor.py b/esphome/components/ld6002b/text_sensor.py index 0e8e2e80e7e..de3dde15cca 100644 --- a/esphome/components/ld6002b/text_sensor.py +++ b/esphome/components/ld6002b/text_sensor.py @@ -24,9 +24,6 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config: ConfigType) -> None: hub = await cg.get_variable(config[CONF_LD6002B_ID]) - if work_mode_config := config.get(CONF_WORK_MODE): - sens = await text_sensor.new_text_sensor(work_mode_config) - cg.add(hub.set_work_mode_text_sensor(sens)) - if ota_config := config.get(CONF_OTA_VERSION): - sens = await text_sensor.new_text_sensor(ota_config) - cg.add(hub.set_ota_version_text_sensor(sens)) + text_sensors = text_sensor.sub_text_sensors(config) + await text_sensors(CONF_WORK_MODE, hub.set_work_mode_text_sensor) + await text_sensors(CONF_OTA_VERSION, hub.set_ota_version_text_sensor) From 23afdc1d9b6573207bec3496fbc6c05da2af15d1 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 03:32:45 +0200 Subject: [PATCH 067/133] [bme680] Use the sub_* helpers for optional child entities (#19917) --- esphome/components/bme680/sensor.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/esphome/components/bme680/sensor.py b/esphome/components/bme680/sensor.py index dce5c88cfaa..ffe5c74bbd4 100644 --- a/esphome/components/bme680/sensor.py +++ b/esphome/components/bme680/sensor.py @@ -131,24 +131,15 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) - cg.add(var.set_temperature_oversampling(temperature_config[CONF_OVERSAMPLING])) - - if pressure_config := config.get(CONF_PRESSURE): - sens = await sensor.new_sensor(pressure_config) - cg.add(var.set_pressure_sensor(sens)) - cg.add(var.set_pressure_oversampling(pressure_config[CONF_OVERSAMPLING])) - - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity_sensor(sens)) - cg.add(var.set_humidity_oversampling(humidity_config[CONF_OVERSAMPLING])) - - if gas_resistance_config := config.get(CONF_GAS_RESISTANCE): - sens = await sensor.new_sensor(gas_resistance_config) - cg.add(var.set_gas_resistance_sensor(sens)) + sensors = sensor.sub_sensors(config) + if await sensors(CONF_TEMPERATURE, var.set_temperature_sensor): + oversampling = config[CONF_TEMPERATURE][CONF_OVERSAMPLING] + cg.add(var.set_temperature_oversampling(oversampling)) + if await sensors(CONF_PRESSURE, var.set_pressure_sensor): + cg.add(var.set_pressure_oversampling(config[CONF_PRESSURE][CONF_OVERSAMPLING])) + if await sensors(CONF_HUMIDITY, var.set_humidity_sensor): + cg.add(var.set_humidity_oversampling(config[CONF_HUMIDITY][CONF_OVERSAMPLING])) + await sensors(CONF_GAS_RESISTANCE, var.set_gas_resistance_sensor) cg.add(var.set_iir_filter(IIR_FILTER_OPTIONS[config[CONF_IIR_FILTER]])) From cd0a0f3fcafb9155e29324cd83642b749a52e008 Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Wed, 30 Sep 2026 03:48:08 +0200 Subject: [PATCH 068/133] [zigbee] Add cluster: device_class to expose ESP32 sensors via measurement clusters instead of Analog/Binary Input (#17388) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- esphome/components/zigbee/__init__.py | 28 +- esphome/components/zigbee/const.py | 1 + esphome/components/zigbee/const_esp32.py | 25 +- .../zigbee/zigbee_attribute_esp32.cpp | 3 +- .../zigbee/zigbee_attribute_esp32.h | 42 ++- esphome/components/zigbee/zigbee_ep_esp32.py | 273 ++++++++++++++++-- esphome/components/zigbee/zigbee_esp32.cpp | 11 +- esphome/components/zigbee/zigbee_esp32.py | 142 ++++++--- .../components/zigbee/zigbee_helpers_esp32.c | 62 ++++ .../components/zigbee/zigbee_helpers_esp32.h | 1 + esphome/components/zigbee/zigbee_zephyr.py | 12 +- tests/components/zigbee/common_esp32.yaml | 29 +- .../zigbee/validate.esp32-c6-idf.yaml | 36 +++ 13 files changed, 565 insertions(+), 100 deletions(-) create mode 100644 tests/components/zigbee/validate.esp32-c6-idf.yaml diff --git a/esphome/components/zigbee/__init__.py b/esphome/components/zigbee/__init__.py index 1fa6110f04b..e9fef59f353 100644 --- a/esphome/components/zigbee/__init__.py +++ b/esphome/components/zigbee/__init__.py @@ -18,6 +18,7 @@ from esphome.core import CORE, CoroPriority, coroutine_with_priority from esphome.types import ConfigType from .const import ( + CONF_CLUSTER, CONF_ENDPOINT, CONF_MAX_EP_NUMBER, CONF_ON_JOIN, @@ -74,6 +75,11 @@ BASE_SCHEMA = cv.Schema( _check_report_deprecation, cv.enum(REPORT, lower=True), ), + cv.OnlyWith(CONF_CLUSTER, ["esp32", "zigbee"], default="generic"): cv.All( + cv.requires_component("zigbee"), + cv.requires_component("esp32"), + cv.one_of(*["generic", "device_class"], lower=True), + ), cv.Optional(CONF_ENDPOINT): cv.All( cv.requires_component("zigbee"), cv.requires_component("esp32"), @@ -196,21 +202,27 @@ async def to_code(config: ConfigType) -> None: async def setup_binary_sensor(entity: cg.MockObj, config: ConfigType) -> None: - if not config.get(CONF_ZIGBEE_ID) or config.get(CONF_INTERNAL): + if "zigbee" not in CORE.loaded_integrations or config.get(CONF_INTERNAL): return if CORE.using_zephyr: - from .zigbee_zephyr import zephyr_setup_binary_sensor - - await zephyr_setup_binary_sensor(entity, config) + if not config.get(CONF_ZIGBEE_ID): + return + from .zigbee_zephyr import add_binary_sensor + else: + from .zigbee_esp32 import add_component as add_binary_sensor + CORE.add_job(add_binary_sensor, entity, config) async def setup_sensor(entity: cg.MockObj, config: ConfigType) -> None: - if not config.get(CONF_ZIGBEE_ID) or config.get(CONF_INTERNAL): + if "zigbee" not in CORE.loaded_integrations or config.get(CONF_INTERNAL): return if CORE.using_zephyr: - from .zigbee_zephyr import zephyr_setup_sensor - - await zephyr_setup_sensor(entity, config) + if not config.get(CONF_ZIGBEE_ID): + return + from .zigbee_zephyr import add_sensor + else: + from .zigbee_esp32 import add_component as add_sensor + CORE.add_job(add_sensor, entity, config) async def setup_switch(entity: cg.MockObj, config: ConfigType) -> None: diff --git a/esphome/components/zigbee/const.py b/esphome/components/zigbee/const.py index d922ae372f2..d6abd9d6c07 100644 --- a/esphome/components/zigbee/const.py +++ b/esphome/components/zigbee/const.py @@ -64,6 +64,7 @@ REPORT = { "default": report.ZIGBEE_REPORT_DEFAULT, } +CONF_CLUSTER = "cluster" CONF_ENDPOINT = "endpoint" CONF_MAX_EP_NUMBER = 239 CONF_ON_JOIN = "on_join" diff --git a/esphome/components/zigbee/const_esp32.py b/esphome/components/zigbee/const_esp32.py index 2e1b09fb221..32ae3e37348 100644 --- a/esphome/components/zigbee/const_esp32.py +++ b/esphome/components/zigbee/const_esp32.py @@ -1,12 +1,14 @@ import esphome.codegen as cg +ALLOWED_UNITS = "allowed_units" DEVICE_TYPE = "device_type" ROLE = "role" CONF_CLUSTERS = "clusters" CONF_ATTRIBUTES = "attributes" -CONF_CLUSTER = "cluster" -SCALE = "scale" CONF_ATTRIBUTE_ID = "attribute_id" +KEY_ZIGBEE_ATTRIBUTES = "zigbee_attributes" +CONNECT = "connect" +SCALE = "scale" KEY_ZIGBEE_EP = "zigbee_ep" KEY_ZIGBEE_EP_NO_NUM = "zigbee_ep_no_num" KEY_ZIGBEE_FIRST_EP_CL = "zigbee_first_ep_cl" @@ -14,6 +16,11 @@ KEY_ZIGBEE_FIRST_EP_CL = "zigbee_first_ep_cl" DEVICE_ID = { "RANGE_EXTENDER": cg.RawExpression("EZB_ZHA_RANGE_EXTENDER_DEVICE_ID"), "SIMPLE_SENSOR": cg.RawExpression("EZB_ZHA_SIMPLE_SENSOR_DEVICE_ID"), + "LIGHT_SENSOR": cg.RawExpression("EZB_ZHA_LIGHT_SENSOR_DEVICE_ID"), + "OCCUPANCY_SENSOR": cg.RawExpression("EZB_ZHA_OCCUPANCY_SENSOR_DEVICE_ID"), + "TEMPERATURE_SENSOR": cg.RawExpression("EZB_ZHA_TEMPERATURE_SENSOR_DEVICE_ID"), + "PRESSURE_SENSOR": cg.RawExpression("EZB_ZHA_PRESSURE_SENSOR_DEVICE_ID"), + "FLOW_SENSOR": cg.RawExpression("EZB_ZHA_FLOW_SENSOR_DEVICE_ID"), "CUSTOM_ATTR": 0xFFF2, } cluster_id = cg.esphome_ns.enum("ezb_zcl_cluster_id_e") @@ -22,6 +29,14 @@ CLUSTER_ID = { "TIME": cluster_id.EZB_ZCL_CLUSTER_ID_TIME, "BINARY_INPUT": cluster_id.EZB_ZCL_CLUSTER_ID_BINARY_INPUT, "ANALOG_INPUT": cluster_id.EZB_ZCL_CLUSTER_ID_ANALOG_INPUT, + "ILLUMINANCE_MEASUREMENT": cluster_id.EZB_ZCL_CLUSTER_ID_ILLUMINANCE_MEASUREMENT, + "TEMPERATURE_MEASUREMENT": cluster_id.EZB_ZCL_CLUSTER_ID_TEMPERATURE_MEASUREMENT, + "PRESSURE_MEASUREMENT": cluster_id.EZB_ZCL_CLUSTER_ID_PRESSURE_MEASUREMENT, + "FLOW_MEASUREMENT": cluster_id.EZB_ZCL_CLUSTER_ID_FLOW_MEASUREMENT, + "REL_HUMIDITY_MEASUREMENT": cluster_id.EZB_ZCL_CLUSTER_ID_REL_HUMIDITY_MEASUREMENT, + "OCCUPANCY_SENSING": cluster_id.EZB_ZCL_CLUSTER_ID_OCCUPANCY_SENSING, + "CARBON_DIOXIDE_MEASUREMENT": cluster_id.EZB_ZCL_CLUSTER_ID_CARBON_DIOXIDE_MEASUREMENT, + "PM2_5_MEASUREMENT": cluster_id.EZB_ZCL_CLUSTER_ID_PM2_5_MEASUREMENT, } CLUSTER_ROLE = { "SERVER": cg.RawExpression("EZB_ZCL_CLUSTER_SERVER"), @@ -31,6 +46,12 @@ attr_type = cg.esphome_ns.enum("ezb_zcl_attr_type_e") ATTR_TYPE = { "BOOL": attr_type.EZB_ZCL_ATTR_TYPE_BOOL, "MAP8": attr_type.EZB_ZCL_ATTR_TYPE_MAP8, + "UINT8": attr_type.EZB_ZCL_ATTR_TYPE_UINT8, + "UINT16": attr_type.EZB_ZCL_ATTR_TYPE_UINT16, + "INT8": attr_type.EZB_ZCL_ATTR_TYPE_INT8, + "INT16": attr_type.EZB_ZCL_ATTR_TYPE_INT16, + "ENUM8": attr_type.EZB_ZCL_ATTR_TYPE_ENUM8, + "ENUM16": attr_type.EZB_ZCL_ATTR_TYPE_ENUM16, "STRING": attr_type.EZB_ZCL_ATTR_TYPE_STRING, "SINGLE": attr_type.EZB_ZCL_ATTR_TYPE_SINGLE, "DOUBLE": attr_type.EZB_ZCL_ATTR_TYPE_DOUBLE, diff --git a/esphome/components/zigbee/zigbee_attribute_esp32.cpp b/esphome/components/zigbee/zigbee_attribute_esp32.cpp index 1fb8d1abe44..acf0321b12b 100644 --- a/esphome/components/zigbee/zigbee_attribute_esp32.cpp +++ b/esphome/components/zigbee/zigbee_attribute_esp32.cpp @@ -23,7 +23,8 @@ void ZigbeeAttribute::set_attr_() { } // Check for error if (state != EZB_ZCL_STATUS_SUCCESS) { - ESP_LOGE(TAG, "Setting attribute failed, ZCL status: %u", static_cast(state)); + ESP_LOGE(TAG, "Setting attribute 0x%04X on cluster 0x%04X failed, ZCL status: %u", this->attr_id_, + this->cluster_id_, static_cast(state)); } esp_zigbee_lock_release(); } diff --git a/esphome/components/zigbee/zigbee_attribute_esp32.h b/esphome/components/zigbee/zigbee_attribute_esp32.h index fc229b4e950..47f460406c9 100644 --- a/esphome/components/zigbee/zigbee_attribute_esp32.h +++ b/esphome/components/zigbee/zigbee_attribute_esp32.h @@ -1,5 +1,7 @@ #pragma once +#include +#include #include #include "esphome/core/automation.h" @@ -47,6 +49,7 @@ class ZigbeeAttribute final : public Component { void set_report(ZigbeeReportT report); #ifdef USE_SENSOR template void connect(sensor::Sensor *sensor); + template void connect(sensor::Sensor *sensor, F &&f); #endif #ifdef USE_BINARY_SENSOR template void connect(binary_sensor::BinarySensor *sensor); @@ -68,6 +71,8 @@ class ZigbeeAttribute final : public Component { bool set_attr_requested_{false}; bool report_requested_{false}; bool force_report_{false}; + template T scale_value_(float value); + template T invalid_value_(); }; template void ZigbeeAttribute::add_attr(T value) { @@ -85,9 +90,44 @@ template void ZigbeeAttribute::set_attr(const T &value) { this->enable_loop(); } +template T ZigbeeAttribute::scale_value_(float value) { + static_assert(sizeof(T) <= 2 || std::is_floating_point_v); + if constexpr (std::is_integral::value) { + const float scaled = this->scale_ * value; + if (std::isnan(value) || scaled < static_cast(std::numeric_limits::lowest()) || + scaled > static_cast(std::numeric_limits::max())) { + return this->invalid_value_(); // 0x8000 / 0xFFFF / 0 for bitmaps + } + return static_cast(lroundf(scaled)); + } + return static_cast(this->scale_ * value); +} + +template T ZigbeeAttribute::invalid_value_() { + if constexpr (std::is_integral_v) { + if constexpr (std::is_signed_v) { + // For signed integer types, NaN is represented by the minimum value + return static_cast(std::numeric_limits::min()); + } + + if (this->attr_type_ >= EZB_ZCL_ATTR_TYPE_UINT8 && this->attr_type_ <= EZB_ZCL_ATTR_TYPE_ENUM16) { + // For unsigned integer types and enum, NaN is represented by the maximum value + return static_cast(std::numeric_limits::max()); + } + + // For other integer types, return 0 as a fallback + return static_cast(0); + } + + return std::numeric_limits::quiet_NaN(); // For floating-point types, return NaN +} + #ifdef USE_SENSOR template void ZigbeeAttribute::connect(sensor::Sensor *sensor) { - sensor->add_on_state_callback([this](float value) { this->set_attr((T) (this->scale_ * value)); }); + sensor->add_on_state_callback([this](float value) { this->set_attr(this->scale_value_(value)); }); +} +template void ZigbeeAttribute::connect(sensor::Sensor *sensor, F &&f) { + sensor->add_on_state_callback([f = std::forward(f), this](float value) { this->set_attr((T) f(value)); }); } #endif #ifdef USE_BINARY_SENSOR diff --git a/esphome/components/zigbee/zigbee_ep_esp32.py b/esphome/components/zigbee/zigbee_ep_esp32.py index 700267ef50e..c45f5800459 100644 --- a/esphome/components/zigbee/zigbee_ep_esp32.py +++ b/esphome/components/zigbee/zigbee_ep_esp32.py @@ -1,8 +1,31 @@ from typing import Any import esphome.config_validation as cv -from esphome.const import CONF_DEVICE, CONF_ID, CONF_TYPE -from esphome.core import CORE +from esphome.const import ( + CONF_ID, + CONF_LAMBDA, + CONF_TYPE, + CONF_VALUE, + DEVICE_CLASS_ATMOSPHERIC_PRESSURE, + DEVICE_CLASS_CARBON_DIOXIDE, + DEVICE_CLASS_HUMIDITY, + DEVICE_CLASS_ILLUMINANCE, + DEVICE_CLASS_OCCUPANCY, + DEVICE_CLASS_PM25, + DEVICE_CLASS_PRESSURE, + DEVICE_CLASS_TEMPERATURE, + DEVICE_CLASS_VOLUME_FLOW_RATE, + UNIT_CELSIUS, + UNIT_CUBIC_METER_PER_HOUR, + UNIT_HECTOPASCAL, + UNIT_LITRE_PER_HOUR, + UNIT_LUX, + UNIT_MICROGRAMS_PER_CUBIC_METER, + UNIT_PARTS_PER_MILLION, + UNIT_PASCAL, + UNIT_PERCENT, +) +from esphome.core import CORE, Lambda from .const import ( CONF_MAX_EP_NUMBER, @@ -12,70 +35,264 @@ from .const import ( REPORT, ) from .const_esp32 import ( + ALLOWED_UNITS, CONF_ATTRIBUTE_ID, CONF_ATTRIBUTES, CONF_CLUSTERS, + CONNECT, DEVICE_TYPE, KEY_ZIGBEE_EP, KEY_ZIGBEE_EP_NO_NUM, KEY_ZIGBEE_FIRST_EP_CL, ROLE, + SCALE, ) # endpoint configs: -ep_configs: dict[str, dict[str, Any]] = { - "binary_input": { - DEVICE_TYPE: "SIMPLE_SENSOR", +ANALOG_INPUT_EP = { + CONF_CLUSTERS: [ + { + CONF_ID: "ANALOG_INPUT", + ROLE: "SERVER", + CONF_ATTRIBUTES: [ + { + CONF_ATTRIBUTE_ID: 0x55, + CONF_TYPE: "SINGLE", + CONF_REPORT: cv.enum(REPORT, lower=True)("default"), + CONNECT: True, + }, + { + CONF_ATTRIBUTE_ID: 0x51, + CONF_TYPE: "BOOL", + }, + { + CONF_ATTRIBUTE_ID: 0x6F, + CONF_TYPE: "MAP8", + }, + { + CONF_ATTRIBUTE_ID: 0x1C, + CONF_TYPE: "STRING", + }, + ], + }, + ], +} + +BINARY_INPUT_EP = { + DEVICE_TYPE: "SIMPLE_SENSOR", + CONF_CLUSTERS: [ + { + CONF_ID: "BINARY_INPUT", + ROLE: "SERVER", + CONF_ATTRIBUTES: [ + { + CONF_ATTRIBUTE_ID: 0x55, + CONF_TYPE: "BOOL", + CONF_REPORT: cv.enum(REPORT, lower=True)("default"), + CONNECT: True, + }, + { + CONF_ATTRIBUTE_ID: 0x51, + CONF_TYPE: "BOOL", + }, + { + CONF_ATTRIBUTE_ID: 0x6F, + CONF_TYPE: "MAP8", + }, + { + CONF_ATTRIBUTE_ID: 0x1C, + CONF_TYPE: "STRING", + }, + ], + }, + ], +} + + +def _pressure_ep(device_type: bool = False) -> dict[str, Any]: + ep = { + ALLOWED_UNITS: [UNIT_HECTOPASCAL, UNIT_PASCAL], CONF_CLUSTERS: [ { - CONF_ID: "BINARY_INPUT", + CONF_ID: "PRESSURE_MEASUREMENT", ROLE: "SERVER", CONF_ATTRIBUTES: [ { - CONF_ATTRIBUTE_ID: 0x55, - CONF_TYPE: "BOOL", + CONF_ATTRIBUTE_ID: 0x0, + CONF_TYPE: "INT16", CONF_REPORT: cv.enum(REPORT, lower=True)("default"), - CONF_DEVICE: None, + CONNECT: True, + SCALE: { + UNIT_HECTOPASCAL: 1, + UNIT_PASCAL: 0.01, + }, }, + ], + }, + ], + } + if device_type: + ep[DEVICE_TYPE] = ( + "PRESSURE_SENSOR" # Sensor that measures pressure of liquids like water + ) + return ep + + +SENSOR_EP_CONFIGS: dict[str, dict[str, Any]] = { + DEVICE_CLASS_TEMPERATURE: { + ALLOWED_UNITS: [UNIT_CELSIUS], + DEVICE_TYPE: "TEMPERATURE_SENSOR", + CONF_CLUSTERS: [ + { + CONF_ID: "TEMPERATURE_MEASUREMENT", + ROLE: "SERVER", + CONF_ATTRIBUTES: [ { - CONF_ATTRIBUTE_ID: 0x51, - CONF_TYPE: "BOOL", - }, - { - CONF_ATTRIBUTE_ID: 0x6F, - CONF_TYPE: "MAP8", - }, - { - CONF_ATTRIBUTE_ID: 0x1C, - CONF_TYPE: "STRING", + CONF_ATTRIBUTE_ID: 0x0, + CONF_TYPE: "INT16", + CONF_REPORT: cv.enum(REPORT, lower=True)("default"), + SCALE: 100, + CONNECT: True, }, ], }, ], }, - "analog_input": { + DEVICE_CLASS_HUMIDITY: { + ALLOWED_UNITS: [UNIT_PERCENT], CONF_CLUSTERS: [ { - CONF_ID: "ANALOG_INPUT", + CONF_ID: "REL_HUMIDITY_MEASUREMENT", ROLE: "SERVER", CONF_ATTRIBUTES: [ { - CONF_ATTRIBUTE_ID: 0x55, + CONF_ATTRIBUTE_ID: 0x0, + CONF_TYPE: "UINT16", + CONF_REPORT: cv.enum(REPORT, lower=True)("default"), + SCALE: 100, + CONNECT: True, + }, + ], + }, + ], + }, + DEVICE_CLASS_ATMOSPHERIC_PRESSURE: _pressure_ep(), + DEVICE_CLASS_PRESSURE: _pressure_ep(device_type=True), + DEVICE_CLASS_VOLUME_FLOW_RATE: { + ALLOWED_UNITS: [UNIT_LITRE_PER_HOUR, UNIT_CUBIC_METER_PER_HOUR], + DEVICE_TYPE: "FLOW_SENSOR", + CONF_CLUSTERS: [ + { + CONF_ID: "FLOW_MEASUREMENT", + ROLE: "SERVER", + CONF_ATTRIBUTES: [ + { + CONF_ATTRIBUTE_ID: 0x0, + CONF_TYPE: "UINT16", + CONF_REPORT: cv.enum(REPORT, lower=True)("default"), + CONNECT: True, + SCALE: { + UNIT_LITRE_PER_HOUR: 0.01, + UNIT_CUBIC_METER_PER_HOUR: 10, + }, + }, + ], + }, + ], + }, + DEVICE_CLASS_ILLUMINANCE: { + ALLOWED_UNITS: [UNIT_LUX], + DEVICE_TYPE: "LIGHT_SENSOR", + CONF_CLUSTERS: [ + { + CONF_ID: "ILLUMINANCE_MEASUREMENT", + ROLE: "SERVER", + CONF_ATTRIBUTES: [ + { + CONF_ATTRIBUTE_ID: 0x0, + CONF_TYPE: "UINT16", + CONF_REPORT: cv.enum(REPORT, lower=True)("default"), + CONF_LAMBDA: cv.lambda_( + Lambda( + "if (x < 0.0f || std::isnan(x)) return 0xFFFF;" # NaN + " if (x < 1.0f) return 0;" # too small to measure + " const float v = log10(x)*10000 + 1;" + " return v > 65534.0f ? 0xFFFE : (uint16_t) lroundf(v);" # clamp to 0xFFFE if too large + ) + ), + CONNECT: True, + }, + ], + }, + ], + }, + DEVICE_CLASS_PM25: { + ALLOWED_UNITS: [UNIT_MICROGRAMS_PER_CUBIC_METER], + CONF_CLUSTERS: [ + { + CONF_ID: "PM2_5_MEASUREMENT", + ROLE: "SERVER", + CONF_ATTRIBUTES: [ + { + CONF_ATTRIBUTE_ID: 0x0, CONF_TYPE: "SINGLE", CONF_REPORT: cv.enum(REPORT, lower=True)("default"), - CONF_DEVICE: None, + CONNECT: True, }, { - CONF_ATTRIBUTE_ID: 0x51, - CONF_TYPE: "BOOL", + CONF_ATTRIBUTE_ID: 0x2, + CONF_TYPE: "SINGLE", + CONF_VALUE: 9999, # overwrite default 1.0 }, + ], + }, + ], + }, + DEVICE_CLASS_CARBON_DIOXIDE: { + ALLOWED_UNITS: [UNIT_PARTS_PER_MILLION], + CONF_CLUSTERS: [ + { + CONF_ID: "CARBON_DIOXIDE_MEASUREMENT", + ROLE: "SERVER", + CONF_ATTRIBUTES: [ { - CONF_ATTRIBUTE_ID: 0x6F, + CONF_ATTRIBUTE_ID: 0x0000, + CONF_TYPE: "SINGLE", + CONF_REPORT: cv.enum(REPORT, lower=True)("default"), + CONNECT: True, + SCALE: 0.000001, + }, + {CONF_ATTRIBUTE_ID: 0x0001, CONF_TYPE: "SINGLE", CONF_VALUE: 0.0}, + {CONF_ATTRIBUTE_ID: 0x0002, CONF_TYPE: "SINGLE", CONF_VALUE: 0.1}, + ], + }, + ], + }, +} + +BINARY_SENSOR_EP_CONFIGS: dict[str, dict[str, Any]] = { + DEVICE_CLASS_OCCUPANCY: { + DEVICE_TYPE: "OCCUPANCY_SENSOR", + CONF_CLUSTERS: [ + { + CONF_ID: "OCCUPANCY_SENSING", + ROLE: "SERVER", + CONF_ATTRIBUTES: [ + { + CONF_ATTRIBUTE_ID: 0x0, CONF_TYPE: "MAP8", + CONF_REPORT: cv.enum(REPORT, lower=True)("default"), + CONNECT: True, }, { - CONF_ATTRIBUTE_ID: 0x1C, - CONF_TYPE: "STRING", + CONF_ATTRIBUTE_ID: 0x1, + CONF_TYPE: "ENUM8", + CONF_VALUE: 0, # hardcode PIR for now as ultrasonic or physical contact is unlikely + }, + { + CONF_ATTRIBUTE_ID: 0x2, + CONF_TYPE: "MAP8", + CONF_VALUE: 0b00000001, # hardcode PIR for now as ultrasonic or physical contact is unlikely }, ], }, diff --git a/esphome/components/zigbee/zigbee_esp32.cpp b/esphome/components/zigbee/zigbee_esp32.cpp index 65ceae8a4bf..c00b13cffd8 100644 --- a/esphome/components/zigbee/zigbee_esp32.cpp +++ b/esphome/components/zigbee/zigbee_esp32.cpp @@ -181,13 +181,8 @@ static void zb_action_handler(ezb_zcl_core_action_callback_id_t callback_id, voi } void ZigbeeComponent::create_default_cluster(uint8_t endpoint_id, uint16_t device_id) { - ezb_af_ep_config_t config = { - .ep_id = endpoint_id, - .app_profile_id = EZB_AF_HA_PROFILE_ID, - .app_device_id = device_id, - .app_device_version = 0, - }; - ezb_af_ep_desc_t ep_desc = ezb_af_create_endpoint_desc(&config); + ezb_af_ep_desc_t ep_desc = + esphome_zb_zha_default_ep_desc_create(endpoint_id, device_id, this->basic_cluster_data_.power_source); if (ezb_af_device_add_endpoint_desc(this->dev_desc_, ep_desc) != EZB_ERR_NONE) { ESP_LOGE(TAG, "Could not create endpoint %u", endpoint_id); } @@ -232,13 +227,13 @@ void ZigbeeComponent::update_basic_cluster_(ezb_af_ep_desc_t ep_desc) { .power_source = this->basic_cluster_data_.power_source, }; cluster_desc = ezb_zcl_basic_create_cluster_desc(&basic_cluster_cfg, EZB_ZCL_CLUSTER_SERVER); + ezb_af_endpoint_add_cluster_desc(ep_desc, cluster_desc); } ezb_zcl_basic_cluster_desc_add_attr(cluster_desc, EZB_ZCL_ATTR_BASIC_MANUFACTURER_NAME_ID, this->basic_cluster_data_.manufacturer); ezb_zcl_basic_cluster_desc_add_attr(cluster_desc, EZB_ZCL_ATTR_BASIC_MODEL_IDENTIFIER_ID, this->basic_cluster_data_.model); ezb_zcl_basic_cluster_desc_add_attr(cluster_desc, EZB_ZCL_ATTR_BASIC_DATE_CODE_ID, this->basic_cluster_data_.date); - ezb_af_endpoint_add_cluster_desc(ep_desc, cluster_desc); } bool ZigbeeComponent::register_device() { diff --git a/esphome/components/zigbee/zigbee_esp32.py b/esphome/components/zigbee/zigbee_esp32.py index 9aae8d4557d..d5e8ebbf5ca 100644 --- a/esphome/components/zigbee/zigbee_esp32.py +++ b/esphome/components/zigbee/zigbee_esp32.py @@ -17,9 +17,9 @@ import esphome.config_validation as cv from esphome.const import ( CONF_ACCURACY_DECIMALS, CONF_AP, - CONF_DEVICE, CONF_DEVICE_CLASS, CONF_ID, + CONF_LAMBDA, CONF_MAX_LENGTH, CONF_MODEL, CONF_NAME, @@ -38,6 +38,7 @@ from .const import ( ANALOG_INPUT_APPTYPE, BACNET_UNIT_NO_UNITS, BACNET_UNITS, + CONF_CLUSTER, CONF_ENDPOINT, CONF_POWER_SOURCE, CONF_REPORT, @@ -48,19 +49,29 @@ from .const import ( ZigbeeAttribute, ) from .const_esp32 import ( + ALLOWED_UNITS, ATTR_TYPE, CLUSTER_ID, CLUSTER_ROLE, CONF_ATTRIBUTE_ID, CONF_ATTRIBUTES, CONF_CLUSTERS, + CONNECT, DEVICE_ID, DEVICE_TYPE, + KEY_ZIGBEE_ATTRIBUTES, KEY_ZIGBEE_EP, ROLE, SCALE, ) -from .zigbee_ep_esp32 import add_ep, create_ep, ep_configs +from .zigbee_ep_esp32 import ( + ANALOG_INPUT_EP, + BINARY_INPUT_EP, + BINARY_SENSOR_EP_CONFIGS, + SENSOR_EP_CONFIGS, + add_ep, + create_ep, +) _LOGGER = logging.getLogger(__name__) @@ -79,8 +90,11 @@ def get_c_type(attr_type: str) -> Any | None: if "STRING" in attr_type: return cg.std_string test = re.match(r"^(DATA|UINT|MAP|ENUM)(\d{1,2})$", attr_type) - if test and test.group(2): + if test: return getattr(cg, "uint" + get_c_size(test.group(2), [8, 16, 32, 64])) + test = re.match(r"^INT(\d{1,2})$", attr_type) + if test: + return getattr(cg, "int" + get_c_size(test.group(1), [8, 16, 32, 64])) return None @@ -92,8 +106,11 @@ def get_cv_by_type(attr_type: str) -> Any | None: if "STRING" in attr_type: return cv.string test = re.match(r"^(DATA|UINT|MAP|ENUM)(\d{1,2})$", attr_type) - if test and test.group(2): + if test: return cv.positive_int + test = re.match(r"^INT(\d{1,2})$", attr_type) + if test: + return cv.int_ raise cv.Invalid(f"Zigbee: type {attr_type} not supported or implemented") @@ -104,6 +121,14 @@ def get_default_by_type(attr_type: str) -> str | bool | int | float: return False if attr_type in ["SINGLE", "DOUBLE"]: return float("nan") + test = re.match(r"^(UINT|ENUM)(\d{1,2})$", attr_type) + if test: + # ZCL "invalid value" sentinel for unsigned ints is the maximum (0xFFFF for UINT16) + return 2 ** (int(test.group(2))) - 1 + test = re.match(r"^INT(\d{1,2})$", attr_type) + if test: + # ZCL "invalid value" sentinel for signed ints is the minimum (0x8000 for INT16) + return -(1 << (int(test.group(1)) - 1)) return 0 @@ -167,59 +192,84 @@ def setup_attributes(config: ConfigType, clusters: list[dict[str, Any]]) -> None ) # or use unidecode attr[CONF_VALUE] = str(name) attr[CONF_MAX_LENGTH] = len(str(name)) - if CONF_DEVICE in attr: # connect device - attr[CONF_DEVICE] = config[CONF_ID] + if attr.get(CONNECT): # connect device if CONF_REPORT in config: attr[CONF_REPORT] = config[CONF_REPORT] attr[CONF_ID] = cv.declare_id(ZigbeeAttribute)(None) - if "zb_attr_ids" not in config: - config["zb_attr_ids"] = [] - config["zb_attr_ids"].append(attr[CONF_ID]) + attr_list = config.setdefault(KEY_ZIGBEE_ATTRIBUTES, []) + attr_list.append(attr) else: attr[CONF_ID] = None validate_attributes(attr) def validate_sensor_esp32(config: ConfigType) -> ConfigType: - ep = copy.deepcopy(ep_configs["analog_input"]) # get application type from device class and meas unit # if none get BACNET unit from meas unit dev_class = config.get(CONF_DEVICE_CLASS) unit = config.get(CONF_UNIT_OF_MEASUREMENT) - apptype = ANALOG_INPUT_APPTYPE.get((dev_class, unit)) - bacunit = BACNET_UNITS.get(unit, BACNET_UNIT_NO_UNITS) - accuracy = config.get(CONF_ACCURACY_DECIMALS) - if apptype is not None: + if config[CONF_CLUSTER] == "device_class": + if dev_class not in SENSOR_EP_CONFIGS: + raise cv.Invalid( + "'cluster: device_class' requires a supported 'device_class'. " + f"Supported: {', '.join(SENSOR_EP_CONFIGS)}. Use 'cluster: generic' otherwise." + ) + ep = copy.deepcopy(SENSOR_EP_CONFIGS[dev_class]) + if unit not in ep[ALLOWED_UNITS]: + raise cv.Invalid( + f"Device class '{dev_class}' requires one of units {', '.join(ep[ALLOWED_UNITS])}." + ) + for attr in ep[CONF_CLUSTERS][0][CONF_ATTRIBUTES]: + if isinstance(attr.get(SCALE), dict): + attr[SCALE] = attr[SCALE][unit] + if isinstance(attr.get(CONF_LAMBDA), dict): + attr[CONF_LAMBDA] = attr[CONF_LAMBDA][unit] + else: + ep = copy.deepcopy(ANALOG_INPUT_EP) + apptype = ANALOG_INPUT_APPTYPE.get((dev_class, unit)) + bacunit = BACNET_UNITS.get(unit, BACNET_UNIT_NO_UNITS) + accuracy = config.get(CONF_ACCURACY_DECIMALS) + if apptype is not None: + ep[CONF_CLUSTERS][0][CONF_ATTRIBUTES].append( + { + CONF_ATTRIBUTE_ID: 0x100, + CONF_VALUE: (apptype << 16) | 0xFFFF, + CONF_TYPE: "UINT32", + }, + ) ep[CONF_CLUSTERS][0][CONF_ATTRIBUTES].append( { - CONF_ATTRIBUTE_ID: 0x100, - CONF_VALUE: (apptype << 16) | 0xFFFF, - CONF_TYPE: "UINT32", - }, - ) - ep[CONF_CLUSTERS][0][CONF_ATTRIBUTES].append( - { - CONF_ATTRIBUTE_ID: 0x75, - CONF_VALUE: bacunit, - CONF_TYPE: "ENUM16", - }, - ) - if accuracy is not None: - # Analog Input Resolution (0x006A): smallest reportable change - ep[CONF_CLUSTERS][0][CONF_ATTRIBUTES].append( - { - CONF_ATTRIBUTE_ID: 0x6A, - CONF_VALUE: 10**-accuracy, - CONF_TYPE: "SINGLE", + CONF_ATTRIBUTE_ID: 0x75, + CONF_VALUE: bacunit, + CONF_TYPE: "ENUM16", }, ) + if accuracy is not None: + # Analog Input Resolution (0x006A): smallest reportable change + ep[CONF_CLUSTERS][0][CONF_ATTRIBUTES].append( + { + CONF_ATTRIBUTE_ID: 0x6A, + CONF_VALUE: 10**-accuracy, + CONF_TYPE: "SINGLE", + }, + ) setup_attributes(config, ep[CONF_CLUSTERS]) add_ep(ep, config.get(CONF_ENDPOINT), config.get(CONF_USE_DEVICE_TYPE)) return config def validate_binary_sensor_esp32(config: ConfigType) -> ConfigType: - ep = copy.deepcopy(ep_configs["binary_input"]) + dev_class = config.get(CONF_DEVICE_CLASS) + if config[CONF_CLUSTER] == "device_class": + if dev_class in BINARY_SENSOR_EP_CONFIGS: + ep = copy.deepcopy(BINARY_SENSOR_EP_CONFIGS[dev_class]) + else: + raise cv.Invalid( + "'cluster: device_class' requires a supported 'device_class'. " + f"Supported: {', '.join(BINARY_SENSOR_EP_CONFIGS)}. Use 'cluster: generic' otherwise." + ) + else: + ep = copy.deepcopy(BINARY_INPUT_EP) setup_attributes(config, ep[CONF_CLUSTERS]) add_ep(ep, config.get(CONF_ENDPOINT), config.get(CONF_USE_DEVICE_TYPE)) return config @@ -274,15 +324,11 @@ async def attributes_to_code( ) await cg.register_component(attr_var, attr) - cg.add(attr_var.add_attr(attr[CONF_VALUE])) + template_arg = cg.TemplateArguments(get_c_type(attr[CONF_TYPE])) + cg.add(attr_var.add_attr(template_arg, attr[CONF_VALUE])) if CONF_REPORT in attr: cg.add(attr_var.set_report(attr[CONF_REPORT])) - if CONF_DEVICE in attr: - device = await cg.get_variable(attr[CONF_DEVICE]) - template_arg = cg.TemplateArguments(get_c_type(attr[CONF_TYPE])) - cg.add(attr_var.connect(template_arg, device)) - async def esp32_to_code(config: ConfigType) -> "MockObj": add_idf_component( @@ -332,3 +378,19 @@ async def esp32_to_code(config: ConfigType) -> "MockObj": ) await attributes_to_code(var, ep_num, cl) return var + + +async def add_component(entity: cg.MockObj, config: ConfigType) -> None: + attrs = config.get(KEY_ZIGBEE_ATTRIBUTES, []) + for attr in attrs: + zb_attr = await cg.get_variable(attr[CONF_ID]) + template_arg = cg.TemplateArguments(get_c_type(attr[CONF_TYPE])) + if attr.get(CONF_LAMBDA) is not None: + lambda_ = await cg.process_lambda( + attr[CONF_LAMBDA], + [(cg.float_, "x")], + return_type=get_c_type(attr[CONF_TYPE]), + ) + cg.add(zb_attr.connect(template_arg, entity, lambda_)) + else: + cg.add(zb_attr.connect(template_arg, entity)) diff --git a/esphome/components/zigbee/zigbee_helpers_esp32.c b/esphome/components/zigbee/zigbee_helpers_esp32.c index 0793669955a..53f2c31e8d4 100644 --- a/esphome/components/zigbee/zigbee_helpers_esp32.c +++ b/esphome/components/zigbee/zigbee_helpers_esp32.c @@ -24,6 +24,36 @@ ezb_err_t esphome_zb_add_or_update_cluster(uint16_t cluster_id, ezb_af_ep_desc_t return ezb_af_endpoint_add_cluster_desc(ep_desc, cluster_desc); } +ezb_af_ep_desc_t esphome_zb_zha_default_ep_desc_create(uint8_t ep_id, uint16_t device_id, uint8_t power_source) { + ezb_af_ep_desc_t ep_desc; + switch (device_id) { + case EZB_ZHA_LIGHT_SENSOR_DEVICE_ID: { + ezb_zha_light_sensor_config_t config = EZB_ZHA_LIGHT_SENSOR_CONFIG(); + config.basic_cfg.power_source = power_source; + ep_desc = ezb_zha_create_light_sensor(ep_id, &config); + break; + } + case EZB_ZHA_TEMPERATURE_SENSOR_DEVICE_ID: { + ezb_zha_temperature_sensor_config_t config = EZB_ZHA_TEMPERATURE_SENSOR_CONFIG(); + config.basic_cfg.power_source = power_source; + ep_desc = ezb_zha_create_temperature_sensor(ep_id, &config); + break; + } + default: { + // For other device IDs no macro exists, so we create an empty endpoint descriptor + ezb_af_ep_config_t config = { + .ep_id = ep_id, + .app_profile_id = EZB_AF_HA_PROFILE_ID, + .app_device_id = device_id, + .app_device_version = 0, + }; + ep_desc = ezb_af_create_endpoint_desc(&config); + break; + } + } + return ep_desc; +} + ezb_zcl_cluster_desc_t esphome_zb_default_cluster_dscr_create(uint16_t cluster_id, uint8_t role_mask) { switch (cluster_id) { case EZB_ZCL_CLUSTER_ID_BASIC: @@ -36,6 +66,22 @@ ezb_zcl_cluster_desc_t esphome_zb_default_cluster_dscr_create(uint16_t cluster_i return ezb_zcl_analog_input_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_BINARY_INPUT: return ezb_zcl_binary_input_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_ILLUMINANCE_MEASUREMENT: + return ezb_zcl_illuminance_measurement_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_TEMPERATURE_MEASUREMENT: + return ezb_zcl_temperature_measurement_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_PRESSURE_MEASUREMENT: + return ezb_zcl_pressure_measurement_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_FLOW_MEASUREMENT: + return ezb_zcl_flow_measurement_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_REL_HUMIDITY_MEASUREMENT: + return ezb_zcl_rel_humidity_measurement_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_OCCUPANCY_SENSING: + return ezb_zcl_occupancy_sensing_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_CARBON_DIOXIDE_MEASUREMENT: + return ezb_zcl_carbon_dioxide_measurement_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_PM2_5_MEASUREMENT: + return ezb_zcl_pm2_5_measurement_create_cluster_desc(NULL, role_mask); default: { ezb_zcl_custom_cluster_config_t config = {0}; config.cluster_id = cluster_id; @@ -57,6 +103,22 @@ ezb_err_t esphome_zb_cluster_add_attr(uint16_t cluster_id, ezb_zcl_cluster_desc_ return ezb_zcl_analog_input_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_BINARY_INPUT: return ezb_zcl_binary_input_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_ILLUMINANCE_MEASUREMENT: + return ezb_zcl_illuminance_measurement_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_TEMPERATURE_MEASUREMENT: + return ezb_zcl_temperature_measurement_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_PRESSURE_MEASUREMENT: + return ezb_zcl_pressure_measurement_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_FLOW_MEASUREMENT: + return ezb_zcl_flow_measurement_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_REL_HUMIDITY_MEASUREMENT: + return ezb_zcl_rel_humidity_measurement_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_OCCUPANCY_SENSING: + return ezb_zcl_occupancy_sensing_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_CARBON_DIOXIDE_MEASUREMENT: + return ezb_zcl_carbon_dioxide_measurement_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_PM2_5_MEASUREMENT: + return ezb_zcl_pm2_5_measurement_cluster_desc_add_attr(cluster_desc, attr_id, value_p); default: return EZB_ERR_NOT_FOUND; } diff --git a/esphome/components/zigbee/zigbee_helpers_esp32.h b/esphome/components/zigbee/zigbee_helpers_esp32.h index 6898068b442..305ab2224c7 100644 --- a/esphome/components/zigbee/zigbee_helpers_esp32.h +++ b/esphome/components/zigbee/zigbee_helpers_esp32.h @@ -10,6 +10,7 @@ extern "C" { #include "esp_zigbee.h" +ezb_af_ep_desc_t esphome_zb_zha_default_ep_desc_create(uint8_t ep_id, uint16_t device_id, uint8_t power_source); ezb_err_t esphome_zb_cluster_add_or_update_attr(uint16_t cluster_id, ezb_zcl_cluster_desc_t cluster_desc, uint16_t attr_id, void *value_p); ezb_err_t esphome_zb_add_or_update_cluster(uint16_t cluster_id, ezb_af_ep_desc_t ep_desc, uint8_t role_mask); diff --git a/esphome/components/zigbee/zigbee_zephyr.py b/esphome/components/zigbee/zigbee_zephyr.py index f47cf6bd40e..588305da300 100644 --- a/esphome/components/zigbee/zigbee_zephyr.py +++ b/esphome/components/zigbee/zigbee_zephyr.py @@ -314,14 +314,6 @@ async def _ctx_to_code(config: ConfigType) -> None: cg.add(cg.RawExpression("ZB_AF_REGISTER_DEVICE_CTX(&zb_device_ctx)")) -async def zephyr_setup_binary_sensor(entity: cg.MockObj, config: ConfigType) -> None: - CORE.add_job(_add_binary_sensor, entity, config) - - -async def zephyr_setup_sensor(entity: cg.MockObj, config: ConfigType) -> None: - CORE.add_job(_add_sensor, entity, config) - - async def zephyr_setup_switch(entity: cg.MockObj, config: ConfigType) -> None: CORE.add_job(_add_switch, entity, config) @@ -404,7 +396,7 @@ async def _add_zigbee_ep( cg.add(var.set_parent(hub)) -async def _add_binary_sensor(entity: cg.MockObj, config: ConfigType) -> None: +async def add_binary_sensor(entity: cg.MockObj, config: ConfigType) -> None: await _add_zigbee_ep( entity, config, @@ -416,7 +408,7 @@ async def _add_binary_sensor(entity: cg.MockObj, config: ConfigType) -> None: ) -async def _add_sensor(entity: cg.MockObj, config: ConfigType) -> None: +async def add_sensor(entity: cg.MockObj, config: ConfigType) -> None: # Get BACnet engineering unit from unit_of_measurement unit = config.get(CONF_UNIT_OF_MEASUREMENT, "") bacnet_unit = BACNET_UNITS.get(unit, BACNET_UNIT_NO_UNITS) diff --git a/tests/components/zigbee/common_esp32.yaml b/tests/components/zigbee/common_esp32.yaml index ac25fb8faf7..706da1f10c1 100644 --- a/tests/components/zigbee/common_esp32.yaml +++ b/tests/components/zigbee/common_esp32.yaml @@ -3,13 +3,19 @@ packages: binary_sensor: - platform: template - name: "Garage Door Open 10" + name: "Occupancy Sensor" report: "default" - use_device_type: false + device_class: occupancy + cluster: device_class + endpoint: 3 - platform: template name: "Garage Door Open 12" report: "force" endpoint: 1 + - platform: template + name: "Garage Door Open 13" + endpoint: 2 + use_device_type: false sensor: - platform: template @@ -19,6 +25,25 @@ sensor: unit_of_measurement: "°C" endpoint: 1 use_device_type: true + cluster: device_class + - platform: template + name: "Light Sensor" + lambda: return 100.0; + device_class: illuminance + unit_of_measurement: "lx" + endpoint: 2 + use_device_type: true + cluster: device_class + - platform: template + name: "pressure_sensor" + unit_of_measurement: Pa + device_class: pressure + cluster: device_class + - platform: template + name: "flow_sensor" + unit_of_measurement: L/h + device_class: volume_flow_rate + cluster: device_class zigbee: model: zigbee_test diff --git a/tests/components/zigbee/validate.esp32-c6-idf.yaml b/tests/components/zigbee/validate.esp32-c6-idf.yaml new file mode 100644 index 00000000000..873b2286826 --- /dev/null +++ b/tests/components/zigbee/validate.esp32-c6-idf.yaml @@ -0,0 +1,36 @@ +packages: + zigbee: !include common_esp32.yaml + +sensor: + - platform: template + name: "humidity_sensor" + unit_of_measurement: "%" + device_class: humidity + cluster: device_class + endpoint: 2 + use_device_type: false + - platform: template + name: "pm25_sensor" + unit_of_measurement: 'µg/m³' + device_class: pm25 + cluster: device_class + - platform: template + name: "co2_sensor" + unit_of_measurement: ppm + device_class: carbon_dioxide + cluster: device_class + - platform: template + name: "pressure_sensor2" + unit_of_measurement: hPa + device_class: pressure + cluster: device_class + - platform: template + name: "flow_sensor2" + unit_of_measurement: m³/h + device_class: volume_flow_rate + cluster: device_class + - platform: template + name: "atmospheric_pressure_sensor" + unit_of_measurement: hPa + device_class: atmospheric_pressure + cluster: device_class From 4a00cebd61867a575de211ff499e082a17d51f25 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:13:53 +0200 Subject: [PATCH 069/133] [kuntze] Use the sub_* helpers for optional child entities (#19926) --- esphome/components/kuntze/sensor.py | 36 +++++++---------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/esphome/components/kuntze/sensor.py b/esphome/components/kuntze/sensor.py index 51d23991e28..40e7447d669 100644 --- a/esphome/components/kuntze/sensor.py +++ b/esphome/components/kuntze/sensor.py @@ -101,31 +101,11 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await modbus.register_modbus_client_device(var, config) - if CONF_PH in config: - conf = config[CONF_PH] - sens = await sensor.new_sensor(conf) - cg.add(var.set_ph_sensor(sens)) - if CONF_TEMPERATURE in config: - conf = config[CONF_TEMPERATURE] - sens = await sensor.new_sensor(conf) - cg.add(var.set_temperature_sensor(sens)) - if CONF_DIS1 in config: - conf = config[CONF_DIS1] - sens = await sensor.new_sensor(conf) - cg.add(var.set_dis1_sensor(sens)) - if CONF_DIS2 in config: - conf = config[CONF_DIS2] - sens = await sensor.new_sensor(conf) - cg.add(var.set_dis2_sensor(sens)) - if CONF_REDOX in config: - conf = config[CONF_REDOX] - sens = await sensor.new_sensor(conf) - cg.add(var.set_redox_sensor(sens)) - if CONF_EC in config: - conf = config[CONF_EC] - sens = await sensor.new_sensor(conf) - cg.add(var.set_ec_sensor(sens)) - if CONF_OCI in config: - conf = config[CONF_OCI] - sens = await sensor.new_sensor(conf) - cg.add(var.set_oci_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_PH, var.set_ph_sensor) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_DIS1, var.set_dis1_sensor) + await sensors(CONF_DIS2, var.set_dis2_sensor) + await sensors(CONF_REDOX, var.set_redox_sensor) + await sensors(CONF_EC, var.set_ec_sensor) + await sensors(CONF_OCI, var.set_oci_sensor) From 4a34315fc7c8a958112e9f1cbd34f16b34b85638 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:14:11 +0200 Subject: [PATCH 070/133] [ufm01] Use the sub_* helpers for optional child entities (#19927) --- esphome/components/ufm01/binary_sensor.py | 26 +++++++++-------------- esphome/components/ufm01/sensor.py | 17 +++++---------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/esphome/components/ufm01/binary_sensor.py b/esphome/components/ufm01/binary_sensor.py index 59583357e4f..81d8db9a52a 100644 --- a/esphome/components/ufm01/binary_sensor.py +++ b/esphome/components/ufm01/binary_sensor.py @@ -34,20 +34,14 @@ CONFIG_SCHEMA = { async def to_code(config: ConfigType) -> None: - ufm01_component = await cg.get_variable(config[CONF_UFM01_ID]) + hub = await cg.get_variable(config[CONF_UFM01_ID]) - if ufc_chip_error_config := config.get(CONF_UFC_CHIP_ERROR): - sens = await binary_sensor.new_binary_sensor(ufc_chip_error_config) - cg.add(ufm01_component.set_ufc_chip_error_binary_sensor(sens)) - - if flow_direction_wrong_config := config.get(CONF_FLOW_DIRECTION_WRONG): - sens = await binary_sensor.new_binary_sensor(flow_direction_wrong_config) - cg.add(ufm01_component.set_flow_direction_wrong_binary_sensor(sens)) - - if empty_tube_config := config.get(CONF_EMPTY_TUBE): - sens = await binary_sensor.new_binary_sensor(empty_tube_config) - cg.add(ufm01_component.set_empty_tube_binary_sensor(sens)) - - if flow_rate_out_of_range_config := config.get(CONF_FLOW_RATE_OUT_OF_RANGE): - sens = await binary_sensor.new_binary_sensor(flow_rate_out_of_range_config) - cg.add(ufm01_component.set_flow_rate_out_of_range_binary_sensor(sens)) + binary_sensors = binary_sensor.sub_binary_sensors(config) + await binary_sensors(CONF_UFC_CHIP_ERROR, hub.set_ufc_chip_error_binary_sensor) + await binary_sensors( + CONF_FLOW_DIRECTION_WRONG, hub.set_flow_direction_wrong_binary_sensor + ) + await binary_sensors(CONF_EMPTY_TUBE, hub.set_empty_tube_binary_sensor) + await binary_sensors( + CONF_FLOW_RATE_OUT_OF_RANGE, hub.set_flow_rate_out_of_range_binary_sensor + ) diff --git a/esphome/components/ufm01/sensor.py b/esphome/components/ufm01/sensor.py index e3281f0b2da..581cdd4a9ba 100644 --- a/esphome/components/ufm01/sensor.py +++ b/esphome/components/ufm01/sensor.py @@ -49,16 +49,9 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config: ConfigType) -> None: - ufm01_component = await cg.get_variable(config[CONF_UFM01_ID]) + hub = await cg.get_variable(config[CONF_UFM01_ID]) - if CONF_ACCUMULATED_FLOW in config: - sens = await sensor.new_sensor(config[CONF_ACCUMULATED_FLOW]) - cg.add(ufm01_component.set_accumulated_flow_sensor(sens)) - - if CONF_FLOW in config: - sens = await sensor.new_sensor(config[CONF_FLOW]) - cg.add(ufm01_component.set_flow_sensor(sens)) - - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(ufm01_component.set_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_ACCUMULATED_FLOW, hub.set_accumulated_flow_sensor) + await sensors(CONF_FLOW, hub.set_flow_sensor) + await sensors(CONF_TEMPERATURE, hub.set_temperature_sensor) From 0fd0f72550c3ee16055919784acd481207d522ba Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:14:27 +0200 Subject: [PATCH 071/133] [alpha3] Use the sub_* helpers for optional child entities (#19928) --- esphome/components/alpha3/sensor.py | 30 +++++++---------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/esphome/components/alpha3/sensor.py b/esphome/components/alpha3/sensor.py index 2c1a04ef274..d8d59242fac 100644 --- a/esphome/components/alpha3/sensor.py +++ b/esphome/components/alpha3/sensor.py @@ -74,26 +74,10 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await ble_client.register_ble_node(var, config) - if flow_config := config.get(CONF_FLOW): - sens = await sensor.new_sensor(flow_config) - cg.add(var.set_flow_sensor(sens)) - - if head_config := config.get(CONF_HEAD): - sens = await sensor.new_sensor(head_config) - cg.add(var.set_head_sensor(sens)) - - if power_config := config.get(CONF_POWER): - sens = await sensor.new_sensor(power_config) - cg.add(var.set_power_sensor(sens)) - - if current_config := config.get(CONF_CURRENT): - sens = await sensor.new_sensor(current_config) - cg.add(var.set_current_sensor(sens)) - - if speed_config := config.get(CONF_SPEED): - sens = await sensor.new_sensor(speed_config) - cg.add(var.set_speed_sensor(sens)) - - if voltage_config := config.get(CONF_VOLTAGE): - sens = await sensor.new_sensor(voltage_config) - cg.add(var.set_voltage_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_FLOW, var.set_flow_sensor) + await sensors(CONF_HEAD, var.set_head_sensor) + await sensors(CONF_POWER, var.set_power_sensor) + await sensors(CONF_CURRENT, var.set_current_sensor) + await sensors(CONF_SPEED, var.set_speed_sensor) + await sensors(CONF_VOLTAGE, var.set_voltage_sensor) From 4d534dc420309de2d6d1ed8ebb9d02bcb441c3c0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:14:40 +0200 Subject: [PATCH 072/133] [uponor_smatrix] Use the sub_* helpers for optional child entities (#19929) --- .../uponor_smatrix/sensor/__init__.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/esphome/components/uponor_smatrix/sensor/__init__.py b/esphome/components/uponor_smatrix/sensor/__init__.py index 52e755f0054..8167506d783 100644 --- a/esphome/components/uponor_smatrix/sensor/__init__.py +++ b/esphome/components/uponor_smatrix/sensor/__init__.py @@ -67,15 +67,8 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await register_uponor_smatrix_device(var, config) - if temperature_config := config.get(CONF_TEMPERATURE): - sens = await sensor.new_sensor(temperature_config) - cg.add(var.set_temperature_sensor(sens)) - if external_temperature_config := config.get(CONF_EXTERNAL_TEMPERATURE): - sens = await sensor.new_sensor(external_temperature_config) - cg.add(var.set_external_temperature_sensor(sens)) - if humidity_config := config.get(CONF_HUMIDITY): - sens = await sensor.new_sensor(humidity_config) - cg.add(var.set_humidity_sensor(sens)) - if target_temperature_config := config.get(CONF_TARGET_TEMPERATURE): - sens = await sensor.new_sensor(target_temperature_config) - cg.add(var.set_target_temperature_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_EXTERNAL_TEMPERATURE, var.set_external_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) + await sensors(CONF_TARGET_TEMPERATURE, var.set_target_temperature_sensor) From e111f1130cc67a7e22d6e8a65ece2b07126fa309 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:14:51 +0200 Subject: [PATCH 073/133] [max9611] Use the sub_* helpers for optional child entities (#19930) --- esphome/components/max9611/sensor.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/esphome/components/max9611/sensor.py b/esphome/components/max9611/sensor.py index 9332274a955..e6309b21ccb 100644 --- a/esphome/components/max9611/sensor.py +++ b/esphome/components/max9611/sensor.py @@ -77,19 +77,8 @@ async def to_code(config: ConfigType) -> None: await i2c.register_i2c_device(var, config) cg.add(var.set_current_resistor(config[CONF_SHUNT_RESISTANCE])) cg.add(var.set_gain(config[CONF_GAIN])) - if CONF_VOLTAGE in config: - conf = config[CONF_VOLTAGE] - sens = await sensor.new_sensor(conf) - cg.add(var.set_voltage_sensor(sens)) - if CONF_CURRENT in config: - conf = config[CONF_CURRENT] - sens = await sensor.new_sensor(conf) - cg.add(var.set_current_sensor(sens)) - if CONF_POWER in config: - conf = config[CONF_POWER] - sens = await sensor.new_sensor(conf) - cg.add(var.set_watt_sensor(sens)) - if CONF_TEMPERATURE in config: - conf = config[CONF_TEMPERATURE] - sens = await sensor.new_sensor(conf) - cg.add(var.set_temp_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_VOLTAGE, var.set_voltage_sensor) + await sensors(CONF_CURRENT, var.set_current_sensor) + await sensors(CONF_POWER, var.set_watt_sensor) + await sensors(CONF_TEMPERATURE, var.set_temp_sensor) From 5baad05926a3fe170d10d2d288e77c0c8e68e396 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:54:38 +1000 Subject: [PATCH 074/133] [lvgl] Use contextvars to improve lvgl scheduling (#19242) Co-authored-by: Claude Opus 5.5 --- esphome/components/lvgl/__init__.py | 18 ++- esphome/components/lvgl/automation.py | 19 +-- .../components/lvgl/binary_sensor/__init__.py | 3 +- esphome/components/lvgl/defines.py | 52 ++++--- esphome/components/lvgl/light/__init__.py | 3 +- esphome/components/lvgl/lvcode.py | 33 +++-- esphome/components/lvgl/number/__init__.py | 3 +- esphome/components/lvgl/schemas.py | 6 +- esphome/components/lvgl/sensor/__init__.py | 3 +- esphome/components/lvgl/styles.py | 94 +++++++------ esphome/components/lvgl/switch/__init__.py | 3 +- esphome/components/lvgl/text/__init__.py | 3 +- .../components/lvgl/text_sensor/__init__.py | 3 +- esphome/components/lvgl/widgets/__init__.py | 60 +++----- esphome/components/lvgl/widgets/button.py | 7 +- .../components/lvgl/widgets/buttonmatrix.py | 3 +- esphome/components/lvgl/widgets/canvas.py | 10 +- esphome/components/lvgl/widgets/keyboard.py | 50 +++---- esphome/components/lvgl/widgets/lv_list.py | 126 ++++++----------- esphome/components/lvgl/widgets/meter.py | 4 +- esphome/components/lvgl/widgets/page.py | 10 +- esphome/components/lvgl/widgets/table.py | 4 +- .../config/keyboard_before_textarea_test.yaml | 28 ++++ .../config/keyboard_update_textarea_test.yaml | 34 +++++ .../config/multi_instance_theme_test.yaml | 35 +++++ .../lvgl/test_keyboard_before_textarea.py | 47 +++++++ .../lvgl/test_keyboard_update_textarea.py | 33 +++++ tests/component_tests/lvgl/test_list.py | 8 +- .../lvgl/test_list_on_add_lvgl_action.py | 14 +- .../lvgl/test_multi_instance_theme.py | 44 ++++++ tests/components/lvgl/lvgl-package.yaml | 18 +++ .../test_action_to_code_context_isolation.py | 56 ++++++++ .../lvgl/test_get_part_state_selector.py | 33 +++++ .../lvgl/test_list_trigger_ordering.py | 129 ++++++++++++++++++ .../components/lvgl/test_table_codegen.py | 3 - .../lvgl/test_theme_update_ordering.py | 44 ++++++ 36 files changed, 726 insertions(+), 317 deletions(-) create mode 100644 tests/component_tests/lvgl/config/keyboard_before_textarea_test.yaml create mode 100644 tests/component_tests/lvgl/config/keyboard_update_textarea_test.yaml create mode 100644 tests/component_tests/lvgl/config/multi_instance_theme_test.yaml create mode 100644 tests/component_tests/lvgl/test_keyboard_before_textarea.py create mode 100644 tests/component_tests/lvgl/test_keyboard_update_textarea.py create mode 100644 tests/component_tests/lvgl/test_multi_instance_theme.py create mode 100644 tests/unit_tests/components/lvgl/test_action_to_code_context_isolation.py create mode 100644 tests/unit_tests/components/lvgl/test_get_part_state_selector.py create mode 100644 tests/unit_tests/components/lvgl/test_list_trigger_ordering.py create mode 100644 tests/unit_tests/components/lvgl/test_theme_update_ordering.py diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index 3822664c948..ca9a317fec0 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -61,7 +61,6 @@ from .defines import ( get_focused_widgets, get_lv_images_used, get_refreshed_widgets, - set_widgets_completed, ) from .encoders import ( ENCODERS_CONFIG, @@ -108,6 +107,7 @@ from .widgets import ( get_screen_active, set_obj_properties, ) +from .widgets.keyboard import attach_textareas # Import only what we actually use directly in this file from .widgets.msgbox import MSGBOX_SCHEMA, msgboxes_to_code @@ -396,6 +396,11 @@ async def to_code(configs): default_group = get_default_group(config_0) df.get_options()[df.CONF_DEBUG_OUTLINE] = config_0[df.CONF_DEBUG_OUTLINE] + # Create theme lambdas before any widgets. + async with LvContext(): + for config in configs: + await theme_to_code(config) + for config in configs: frac = config[CONF_BUFFER_SIZE] if frac >= 0.75: @@ -448,7 +453,6 @@ async def to_code(configs): await touchscreens_to_code(lv_component, config) await encoders_to_code(lv_component, config, default_group) await keypads_to_code(lv_component, config, default_group) - await theme_to_code(config) await gradients_to_code(config) await styles_to_code(config) await set_obj_properties(lv_scr_act, config) @@ -459,20 +463,14 @@ async def to_code(configs): await msgboxes_to_code(lv_component, config) await animations_to_code(config.get(CONF_ANIMATIONS, [])) - # Mark all widgets as completed so awaiters of ``wait_for_widgets`` proceed. - set_widgets_completed(True) async with LvContext(): - # Local import: lv_list imports meter, which imports obj_spec/set_obj_properties - # from this module's own namespace - a top-level import here would be circular. + # Local import to avoid circularity from .widgets.lv_list import finish_list_triggers - # Must run before generate_triggers(): that's what actually processes other - # widgets' on_click etc. automations, which can include lvgl.list.add/remove/ - # clear actions that fire a list's on_add/on_remove triggers - those need to - # already exist by then, not still be pending. await finish_list_triggers() await generate_triggers() await generate_align_tos(configs[0]) + await attach_textareas() for config in configs: lv_component = await cg.get_variable(config[CONF_ID]) await add_animation_triggers(config.get(CONF_ANIMATIONS, [])) diff --git a/esphome/components/lvgl/automation.py b/esphome/components/lvgl/automation.py index c23a36c3892..d797688a7e8 100644 --- a/esphome/components/lvgl/automation.py +++ b/esphome/components/lvgl/automation.py @@ -78,7 +78,6 @@ from .widgets import ( get_screen_active, get_widgets, set_obj_properties, - wait_for_widgets, ) # Widgets that are used in a focused/refreshed action are tracked in @@ -127,15 +126,7 @@ async def action_to_code( action_id, template_arg, args, - config=None, ): - # Ensure all required ids have been processed, so our LambdaContext doesn't get context-switched. - if config: - for lamb in config.values(): - if isinstance(lamb, Lambda): - for id_ in lamb.requires_ids: - await get_variable(id_) - await wait_for_widgets() async with LambdaContext(parameters=args, where=action_id) as context: for widget in widgets: await action(widget) @@ -145,7 +136,7 @@ async def action_to_code( async def update_to_code(config, action_id, template_arg, args): async def do_update(widget: Widget): await set_obj_properties(widget, config) - await widget.type.to_code(widget, config) + await widget.type.update_to_code(widget, config) if ( widget.type.w_type.value_property is not None and widget.type.w_type.value_property in config @@ -153,9 +144,7 @@ async def update_to_code(config, action_id, template_arg, args): lv_obj.send_event(widget.obj, UPDATE_EVENT, nullptr) widgets = await get_widgets(config[CONF_ID]) - return await action_to_code( - widgets, do_update, action_id, template_arg, args, config - ) + return await action_to_code(widgets, do_update, action_id, template_arg, args) @automation.register_condition( @@ -507,9 +496,7 @@ async def obj_update_to_code(config, action_id, template_arg, args): await set_obj_properties(widget, config) widgets = await get_widgets(config[CONF_ID]) - return await action_to_code( - widgets, do_update, action_id, template_arg, args, config - ) + return await action_to_code(widgets, do_update, action_id, template_arg, args) def validate_refresh_config(config): diff --git a/esphome/components/lvgl/binary_sensor/__init__.py b/esphome/components/lvgl/binary_sensor/__init__.py index aa68e764211..fc7317f8b97 100644 --- a/esphome/components/lvgl/binary_sensor/__init__.py +++ b/esphome/components/lvgl/binary_sensor/__init__.py @@ -9,7 +9,7 @@ from esphome.const import CONF_STATE from ..defines import CONF_WIDGET, LV_OBJ_FLAG, LvConstant from ..lvcode import EVENT_ARG, UPDATE_EVENT, LambdaContext, LvContext, lvgl_static from ..types import LV_EVENT, LV_STATE, lv_pseudo_button_t -from ..widgets import Widget, get_widgets, wait_for_widgets +from ..widgets import Widget, get_widgets STATE_PRESSED = "PRESSED" STATE_CHECKED = "CHECKED" @@ -33,7 +33,6 @@ async def to_code(config): widget = widget[0] assert isinstance(widget, Widget) state = await BS_STATE.process(config[CONF_STATE]) - await wait_for_widgets() is_pressed = str(state) == str(LV_STATE.PRESSED) test_expr = widget.is_pressed() if is_pressed else widget.is_checked() async with LambdaContext(EVENT_ARG) as test_ctx: diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index fbb621800b5..c019dff96e0 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -20,21 +20,22 @@ DOMAIN = "lvgl" KEY_COLOR_FORMATS = "color_formats" KEY_ESPHOME_FONTS_USED = "esphome_fonts_used" KEY_FOCUSED_WIDGETS = "focused_widgets" +KEY_LIST_TRIGGERS = "list_triggers" KEY_LV_DEFINES = "lv_defines" KEY_LV_FONTS_USED = "lv_fonts_used" KEY_LV_IMAGES_USED = "lv_images_used" KEY_LV_USES = "lv_uses" KEY_NAMED_STYLES = "named_styles" +KEY_OPTIONS = "options" KEY_REFRESHED_WIDGETS = "refreshed_widgets" KEY_REMAPPED_USES = "remapped_uses" KEY_STYLES_USED = "styles_used" KEY_THEME_UPDATE_REQUESTS = "theme_update_requests" -KEY_THEME_WIDGET_MAP = "theme_widget_map" +KEY_THEME_STYLES = "theme_styles" KEY_UPDATED_WIDGETS = "updated_widgets" -KEY_WIDGET_MAP = "widget_map" -KEY_WIDGETS_COMPLETED = "widgets_completed" -KEY_OPTIONS = "options" KEY_WARNINGS = "warnings" +KEY_WIDGET_MAP = "widget_map" +KEY_WIDGET_THEME_STYLES = "widget_theme_styles" KEY_DEBUG_OUTLINE_COUNT = "debug_outline_count" # Colours for the debug outline, in (red, green, blue) order. They are picked to stay @@ -137,8 +138,14 @@ def get_updated_widgets() -> dict: return _get_data(KEY_UPDATED_WIDGETS, {}) -def get_theme_widget_map() -> dict[str, Any]: - return _get_data(KEY_THEME_WIDGET_MAP, {}) +def get_theme_styles() -> dict[str, MockObj]: + """Get a map of already created theme style names to their corresponding style IDs.""" + return _get_data(KEY_THEME_STYLES, {}) + + +def get_widget_theme_style_data() -> dict[str, list[tuple[MockObj, MockObj]]]: + """Get the map of widget type names to the list of (style variable, part/state name)""" + return _get_data(KEY_WIDGET_THEME_STYLES, {}) def get_theme_update_requests() -> dict[str, dict[tuple[str, str], None]]: @@ -157,20 +164,6 @@ def get_widget_map() -> dict[str, Any]: return _get_data(KEY_WIDGET_MAP, {}) -def get_widgets_completed() -> bool: - # ``[value]`` rather than the bare value so that we can mutate the - # entry in place; ``CORE.data`` is reset for us between runs. - return _get_data(KEY_WIDGETS_COMPLETED, [False])[0] - - -def set_widgets_completed(value: bool) -> None: - _get_data(KEY_WIDGETS_COMPLETED, [False])[0] = value - - -def is_widget_completed(name: ID) -> bool: - return name in get_widget_map() - - def get_focused_widgets() -> set: return _get_data(KEY_FOCUSED_WIDGETS, set()) @@ -179,6 +172,10 @@ def get_refreshed_widgets() -> set: return _get_data(KEY_REFRESHED_WIDGETS, set()) +def get_list_triggers() -> dict: + return _get_data(KEY_LIST_TRIGGERS, {}) + + def add_define(macro: str, value="1"): lv_defines = get_defines() value = str(value) @@ -869,7 +866,7 @@ LV_SCALE_MODE = LvConstant( DEFAULT_ESPHOME_FONT = "esphome_lv_default_font" -def join_enums(enums, prefix=""): +def join_enums(enums: tuple[str], prefix: str = "") -> MockObj: enums = list(enums) enums.sort() # If a prefix is provided, prepend each constant with the prefix, and assume that all the constants are within the @@ -879,6 +876,19 @@ def join_enums(enums, prefix=""): return literal("|".join(f"(int){e.upper()}" for e in enums)) +def get_part_state_selector(part: str, state: str) -> MockObj: + """Combine a part and state into a single selector value, e.g. LV_PART_KNOB | LV_STATE_PRESSED.""" + state = "LV_STATE_" + state.removeprefix("LV_STATE_").upper() + part = "LV_PART_" + part.removeprefix("LV_PART_").upper() + if state == "LV_STATE_DEFAULT": + return literal(part) + if part == "LV_PART_MAIN": + return literal(state) + return MockObj( + StaticCastExpression("lv_style_selector_t", literal(state)) + ) | MockObj(StaticCastExpression("lv_style_selector_t", literal(part))) + + # fmt: off LV_COLOR_FORMATS = ( "RGB565", "SWAPPED", "RGB565A8", "RGB888", "XRGB8888", "ARGB8888", "PREMULTIPLIED", "L8", "AL88", "A8", "I1", diff --git a/esphome/components/lvgl/light/__init__.py b/esphome/components/lvgl/light/__init__.py index dcdf67a5207..0a8720f5eee 100644 --- a/esphome/components/lvgl/light/__init__.py +++ b/esphome/components/lvgl/light/__init__.py @@ -7,7 +7,7 @@ from esphome.const import CONF_GAMMA_CORRECT, CONF_OUTPUT_ID from ..defines import CONF_WIDGET from ..lvcode import LvContext from ..types import LvType, lvgl_ns -from ..widgets import get_widgets, wait_for_widgets +from ..widgets import get_widgets lv_led_t = LvType("lv_led_t") LVLight = lvgl_ns.class_("LVLight", LightOutput) @@ -26,6 +26,5 @@ async def to_code(config): widget = await get_widgets(config, CONF_WIDGET) widget = widget[0] - await wait_for_widgets() async with LvContext() as ctx: ctx.add(var.set_obj(widget.obj)) diff --git a/esphome/components/lvgl/lvcode.py b/esphome/components/lvgl/lvcode.py index 850b63a26f4..d6d1aaa4144 100644 --- a/esphome/components/lvgl/lvcode.py +++ b/esphome/components/lvgl/lvcode.py @@ -1,5 +1,5 @@ import abc -from typing import TYPE_CHECKING +import contextvars from esphome import codegen as cg from esphome.config import Config @@ -65,15 +65,33 @@ class IndentedStatement(Statement): return result -class CodeContext(abc.ABC): +class _CodeContextMeta(abc.ABCMeta): + """ + Backs `CodeContext.code_context` with a contextvar instead of a plain class + attribute, so the FakeEventLoop's per-task context isolation (see coroutine.py) + keeps interleaved to_code() jobs from clobbering each other's current context. + """ + + _context_var: contextvars.ContextVar["CodeContext | None"] = contextvars.ContextVar( + "code_context", default=None + ) + + @property + def code_context(cls) -> "CodeContext | None": + return _CodeContextMeta._context_var.get() + + @code_context.setter + def code_context(cls, value: "CodeContext | None") -> None: + _CodeContextMeta._context_var.set(value) + + +class CodeContext(abc.ABC, metaclass=_CodeContextMeta): """ A class providing a context for code generation. Generated code will be added to the current context. A new context will stack on the current context, and restore it when done. Used with the `with` statement. """ - code_context = None - @abc.abstractmethod def add(self, expression: Expression | Statement): pass @@ -115,6 +133,9 @@ class CodeContext(abc.ABC): def indented_statement(self, stmt): return IndentedStatement(stmt, self.indent_level) + def get_automation_parameters(self) -> list[tuple[SafeExpType, str]]: + return [] + class MainContext(CodeContext): """ @@ -212,10 +233,6 @@ def get_lambda_context_args() -> list[tuple[SafeExpType, str]]: """ if CodeContext.code_context is None: return [] - if TYPE_CHECKING: - # CodeContext base class doesn't define get_automation_parameters(), - # but LambdaContext and LvContext (the concrete implementations) do. - assert isinstance(CodeContext.code_context, LambdaContext) return CodeContext.code_context.get_automation_parameters() diff --git a/esphome/components/lvgl/number/__init__.py b/esphome/components/lvgl/number/__init__.py index be51963ba12..0d8d99ab218 100644 --- a/esphome/components/lvgl/number/__init__.py +++ b/esphome/components/lvgl/number/__init__.py @@ -22,7 +22,7 @@ from ..lvcode import ( ) from ..schemas import TRIGGER_EVENT_MAP, VALUE_TRIGGER_SCHEMA from ..types import LvNumber, lvgl_ns -from ..widgets import get_widgets, wait_for_widgets +from ..widgets import get_widgets LVGLNumber = lvgl_ns.class_("LVGLNumber", number.Number, cg.Component) @@ -47,7 +47,6 @@ async def to_code(config): trigger = CONF_ON_RELEASE widget = await get_widgets(config, CONF_WIDGET) widget = widget[0] - await wait_for_widgets() async with LambdaContext([], return_type=cg.float_) as value: value.add(ReturnStatement(widget.get_value())) async with LambdaContext([(cg.float_, "v")]) as control: diff --git a/esphome/components/lvgl/schemas.py b/esphome/components/lvgl/schemas.py index a546d9f1834..cf766b96c86 100644 --- a/esphome/components/lvgl/schemas.py +++ b/esphome/components/lvgl/schemas.py @@ -670,11 +670,7 @@ def theme_update_schema(value: dict) -> dict: for w_name, style in validated.items(): for part, states in collect_parts(style).items(): for state, props in states.items(): - # collect_parts() unconditionally seeds a main/default entry - # even when nothing was set for it (e.g. `{pressed: {...}}` - # alone) -- skip combos with no properties so a request for - # one state doesn't also create an unused, empty main/default - # style that gets attached to every widget of this type. + # Skip states with no properties if not props: continue df.get_theme_update_requests().setdefault(w_name, {})[(part, state)] = ( diff --git a/esphome/components/lvgl/sensor/__init__.py b/esphome/components/lvgl/sensor/__init__.py index e69ea9771a0..c5ef98b05e5 100644 --- a/esphome/components/lvgl/sensor/__init__.py +++ b/esphome/components/lvgl/sensor/__init__.py @@ -5,7 +5,7 @@ from ..defines import CONF_TRIGGER, CONF_WIDGET from ..lvcode import EVENT_ARG, LambdaContext, LvContext, lv_add, lvgl_static from ..schemas import TRIGGER_EVENT_MAP, VALUE_TRIGGER_SCHEMA from ..types import LvNumber -from ..widgets import Widget, get_widgets, wait_for_widgets +from ..widgets import Widget, get_widgets CONFIG_SCHEMA = sensor_schema(Sensor).extend( { @@ -20,7 +20,6 @@ async def to_code(config): widget = await get_widgets(config, CONF_WIDGET) widget = widget[0] assert isinstance(widget, Widget) - await wait_for_widgets() async with LambdaContext(EVENT_ARG) as lamb: lv_add(sensor.publish_state(widget.get_value())) async with LvContext(): diff --git a/esphome/components/lvgl/styles.py b/esphome/components/lvgl/styles.py index ad420283279..1fef37e2ee8 100644 --- a/esphome/components/lvgl/styles.py +++ b/esphome/components/lvgl/styles.py @@ -1,17 +1,22 @@ from esphome import automation import esphome.codegen as cg import esphome.config_validation as cv -from esphome.const import CONF_ID +from esphome.const import CONF_DEFAULT, CONF_ID from esphome.core import ID +from esphome.cpp_generator import MockObj from .defines import ( CONF_STYLE_DEFINITIONS, CONF_THEME, + PARTS, + STATES, LValidator, add_lv_use, + get_part_state_selector, get_styles_used, + get_theme_styles, get_theme_update_requests, - get_theme_widget_map, + get_widget_theme_style_data, literal, ) from .lvcode import LambdaContext, lv @@ -23,7 +28,7 @@ from .schemas import ( theme_update_schema, ) from .types import ObjUpdateAction, lv_style_t -from .widgets import collect_parts, wait_for_widgets +from .widgets import collect_parts def has_style_props(config) -> bool: @@ -89,7 +94,6 @@ async def styles_to_code(config): synchronous=True, ) async def style_update_to_code(config, action_id, template_arg, args): - await wait_for_widgets() style = await cg.get_variable(config[CONF_ID]) async with LambdaContext(parameters=args, where=action_id) as context: await style_set(style, config) @@ -101,41 +105,64 @@ async def style_update_to_code(config, action_id, template_arg, args): return cg.new_Pvariable(action_id, template_arg, await context.get_lambda()) +def _get_theme_style_name(w_name: str, part: str, state: str) -> str: + return f"_lv_theme_style_{w_name}_{part}_{state}" + + +def get_widget_theme_styles(w_name: str) -> list[tuple[MockObj, MockObj]]: + """Return a list of (style variable, part/state name) for all theme styles used by the given widget type.""" + widget_styles = get_widget_theme_style_data() + if w_name in widget_styles: + return widget_styles[w_name] + theme_styles = get_theme_styles() + style_list = [] + for part in PARTS: + for state in STATES + (CONF_DEFAULT,): + style_name = _get_theme_style_name(w_name, part, state) + if style_name in theme_styles: + style_list.append( + (theme_styles[style_name], get_part_state_selector(part, state)) + ) + widget_styles[w_name] = style_list + return style_list + + async def theme_to_code(config): + """ + Convert theme to C++ code. May be called multiple times for different LVGL instances. + A style is created for each (widget type, part, state) combo declared in the `theme:` section of the config, + or requested by a `theme.update` action. + If a style is requested but not declared, it is created as an empty placeholder. + :param config: + :return: + """ theme = config.get(CONF_THEME) or {} requests = get_theme_update_requests() - # Iterate in WIDGET_TYPES' (deterministic, registration-order) sequence rather - # than a set -- a set of strings/tuples iterates in an order that depends on - # per-process hash randomization, which would otherwise churn the order hidden - # style variables are declared in main.cpp between builds of the same config. widget_names = [ w_name for w_name in WIDGET_TYPES if w_name in theme or w_name in requests ] if not widget_names: return add_lv_use(CONF_THEME) - theme_map = get_theme_widget_map() + style_map = get_theme_styles() for w_name in widget_names: declared_parts = collect_parts(theme[w_name]) if w_name in theme else {} parts = {part: dict(states) for part, states in declared_parts.items()} for part, state in requests.get(w_name, {}): parts.setdefault(part, {}).setdefault(state, {}) - widget_styles = theme_map.setdefault(w_name, {}) for part, states in parts.items(): - part_styles = widget_styles.setdefault(part, {}) declared_states = declared_parts.get(part, {}) for state, props in states.items(): - if state not in part_styles: - part_styles[state] = await create_style( - "_lv_theme_style_" + w_name + "_" + part + "_" + state, props - ) + style_name = _get_theme_style_name(w_name, part, state) + if style_name not in style_map: + style_map[style_name] = await create_style(style_name, props) elif state in declared_states: # A `theme.update` request for this combo (possibly from # another LVGL instance) already created the style as an # empty placeholder before this instance's real `theme:` # declaration was reached -- apply the real values now # instead of silently leaving it empty. - await style_set(part_styles[state], props) + await style_set(style_map[style_name], props) @automation.register_action( @@ -144,40 +171,25 @@ async def theme_to_code(config): theme_update_schema, synchronous=True, ) -async def theme_update_to_code(config, action_id, template_arg, args): - await wait_for_widgets() - theme_map = get_theme_widget_map() - # Invariant this relies on: theme_update_schema() records every (widget - # type, part, state) combo this action targets as a request during config - # validation (which completes for the whole config tree before any - # to_code runs), and theme_to_code() -- which runs for every LVGL - # instance before any action's own to_code -- materialises a style for - # each recorded request. If that handshake is ever broken by a future - # change, fail with a diagnosable message rather than a bare KeyError. - to_update = [] +async def theme_update_to_code(config, action_id, template_arg, args) -> MockObj: + # The theme_update_schema records the requested (widget type, part, state) combos in a global dict so that + # theme_to_code() can create the corresponding styles variables. Here we await get_variable(), which will + # context switch if required so theme_to_code() can run and create the style variable. + to_update: list[tuple] = [] for w_name, style in config.items(): for part, states in collect_parts(style).items(): for state, props in states.items(): - # collect_parts() unconditionally seeds an (empty) main/default - # entry even when this action didn't target it -- skip it, both - # because there's nothing to update and because - # theme_update_schema no longer pre-creates a placeholder style - # for combos with no properties. + # Skip states with no properties to set. if not props: continue - style_var = theme_map.get(w_name, {}).get(part, {}).get(state) - if style_var is None: - raise cv.Invalid( - f"No theme style exists for '{w_name}' {part}/{state}. " - "This is an internal error -- please report it." - ) + style_var = await cg.get_variable( + ID(_get_theme_style_name(w_name, part, state)) + ) to_update.append((style_var, props)) async with LambdaContext(parameters=args, where=action_id) as context: for style_var, props in to_update: await style_set(style_var, props) - # Refresh and redraw every widget using this style -- otherwise the - # updated properties would sit unused until something else happens - # to invalidate the affected widgets. + # Trigger a redraw for affected widgets. lv.obj_report_style_change(style_var) return cg.new_Pvariable(action_id, template_arg, await context.get_lambda()) diff --git a/esphome/components/lvgl/switch/__init__.py b/esphome/components/lvgl/switch/__init__.py index 509e4f42ad7..907665eca61 100644 --- a/esphome/components/lvgl/switch/__init__.py +++ b/esphome/components/lvgl/switch/__init__.py @@ -15,7 +15,7 @@ from ..lvcode import ( lvgl_static, ) from ..types import LV_EVENT, LV_STATE, lv_pseudo_button_t, lvgl_ns -from ..widgets import get_widgets, wait_for_widgets +from ..widgets import get_widgets LVGLSwitch = lvgl_ns.class_("LVGLSwitch", Switch, Component) CONFIG_SCHEMA = switch_schema(LVGLSwitch).extend( @@ -28,7 +28,6 @@ CONFIG_SCHEMA = switch_schema(LVGLSwitch).extend( async def to_code(config): widget = await get_widgets(config, CONF_WIDGET) widget = widget[0] - await wait_for_widgets() switch_id = MockObj(config[CONF_ID], "->") v = literal("v") async with LambdaContext([(cg.bool_, "v")]) as control: diff --git a/esphome/components/lvgl/text/__init__.py b/esphome/components/lvgl/text/__init__.py index 61db5444e86..3d5e0305b98 100644 --- a/esphome/components/lvgl/text/__init__.py +++ b/esphome/components/lvgl/text/__init__.py @@ -14,7 +14,7 @@ from ..lvcode import ( lvgl_static, ) from ..types import LV_EVENT, LvText, lvgl_ns -from ..widgets import get_widgets, wait_for_widgets +from ..widgets import get_widgets LVGLText = lvgl_ns.class_("LVGLText", text.Text) @@ -29,7 +29,6 @@ async def to_code(config): textvar = await new_text(config) widget = await get_widgets(config, CONF_WIDGET) widget = widget[0] - await wait_for_widgets() async with LambdaContext([(cg.std_string, "text_value")]) as control: await widget.set_property("text", "text_value.c_str()") lv_obj.send_event(widget.obj, UPDATE_EVENT, cg.nullptr) diff --git a/esphome/components/lvgl/text_sensor/__init__.py b/esphome/components/lvgl/text_sensor/__init__.py index c3306ad57a0..f0575fb1013 100644 --- a/esphome/components/lvgl/text_sensor/__init__.py +++ b/esphome/components/lvgl/text_sensor/__init__.py @@ -8,7 +8,7 @@ import esphome.config_validation as cv from ..defines import CONF_WIDGET from ..lvcode import EVENT_ARG, UPDATE_EVENT, LambdaContext, LvContext, lvgl_static from ..types import LV_EVENT, LvText -from ..widgets import get_widgets, wait_for_widgets +from ..widgets import get_widgets CONFIG_SCHEMA = text_sensor_schema(TextSensor).extend( { @@ -21,7 +21,6 @@ async def to_code(config): sensor = await new_text_sensor(config) widget = await get_widgets(config, CONF_WIDGET) widget = widget[0] - await wait_for_widgets() async with LambdaContext(EVENT_ARG) as pressed_ctx: pressed_ctx.add(sensor.publish_state(widget.get_value())) async with LvContext() as ctx: diff --git a/esphome/components/lvgl/widgets/__init__.py b/esphome/components/lvgl/widgets/__init__.py index da513e05a85..d69e90056fd 100644 --- a/esphome/components/lvgl/widgets/__init__.py +++ b/esphome/components/lvgl/widgets/__init__.py @@ -4,7 +4,7 @@ from typing import Any from esphome import codegen as cg, config_validation as cv from esphome.automation import register_action -from esphome.config_validation import Invalid, Schema +from esphome.config_validation import Schema from esphome.const import ( CONF_DEFAULT, CONF_GROUP, @@ -44,10 +44,9 @@ from ..defines import ( LValidator, add_lv_use, get_options, + get_part_state_selector, get_styles_used, - get_theme_widget_map, get_widget_map, - get_widgets_completed, join_enums, literal, next_debug_outline_color, @@ -199,13 +198,21 @@ class WidgetType: await self.to_code(w, config) return w - async def to_code(self, w: "Widget", config: dict): + async def to_code(self, w: "Widget", config: dict) -> None: """ - Update a widget, also called when creating + Generate code for widget properties and actions. :param config: :return: """ + async def update_to_code(self, w: "Widget", config: dict) -> None: + """ + Update a widget. Defaults to calling to_code, but can be overridden + :param w: The widget to update + :param config: The configuration for the update + """ + await self.to_code(w, config) + async def obj_creator(self, parent: MockObj, config: dict): """ Create an instance of the widget type @@ -261,17 +268,10 @@ class WidgetType: def apply_theme_styles(w: "Widget") -> None: """Apply the current theme's styles for this widget's type""" - for part, states in get_theme_widget_map().get(w.type.name, {}).items(): - part = "LV_PART_" + part.upper() - for state, style in states.items(): - state = "LV_STATE_" + state.upper() - if state == "LV_STATE_DEFAULT": - lv_state = literal(part) - elif part == "LV_PART_MAIN": - lv_state = literal(state) - else: - lv_state = join_enums((state, part)) - w.add_style(style, lv_state) + from ..styles import get_widget_theme_styles + + for style, lv_state in get_widget_theme_styles(w.type.name): + w.add_style(style, lv_state) def apply_debug_outline(w: "Widget") -> None: @@ -342,8 +342,6 @@ class Widget: return lv_obj.remove_flag(self.obj, literal(flag)) def add_style(self, style_id, state=LV_STATE.DEFAULT): - if "|" in state: - state = f"(lv_state_t)({state})" lv_obj.add_style(self.obj, MockObj(style_id), literal(state)) async def set_property( @@ -476,10 +474,6 @@ def get_widget_generator(wid): while True: if obj := widget_map.get(wid): return obj - if get_widgets_completed(): - raise Invalid( - f"Widget {wid} not found, yet all widgets should be defined by now" - ) yield @@ -489,19 +483,6 @@ async def get_widget_(wid): return await FakeAwaitable(get_widget_generator(wid)) -def widgets_wait_generator(): - while True: - if get_widgets_completed(): - return - yield - - -async def wait_for_widgets(): - if get_widgets_completed(): - return - await FakeAwaitable(widgets_wait_generator()) - - async def get_widgets(config: dict | list, id: str = CONF_ID) -> list[Widget]: if not config: return [] @@ -634,15 +615,8 @@ async def set_obj_properties(w: Widget, config): _set_layout_options(w, layout, base_name) parts = collect_parts(config) for part, states in parts.items(): - part = "LV_PART_" + part.upper() for state, props in states.items(): - state = "LV_STATE_" + state.upper() - if state == "LV_STATE_DEFAULT": - lv_state = literal(part) - elif part == "LV_PART_MAIN": - lv_state = literal(state) - else: - lv_state = join_enums((state, part)) + lv_state = get_part_state_selector(part, state) for style_id in props.get(CONF_STYLES, ()): w.add_style(style_id, lv_state) for prop, value in { diff --git a/esphome/components/lvgl/widgets/button.py b/esphome/components/lvgl/widgets/button.py index 0ad512cd8bb..306f6925018 100644 --- a/esphome/components/lvgl/widgets/button.py +++ b/esphome/components/lvgl/widgets/button.py @@ -15,9 +15,7 @@ lv_button_t = LvBoolean("lv_button_t") class ButtonType(WidgetType): def __init__(self): - super().__init__( - CONF_BUTTON, lv_button_t, (CONF_MAIN,), schema=TEXT_SCHEMA, lv_name="btn" - ) + super().__init__(CONF_BUTTON, lv_button_t, (CONF_MAIN,), schema=TEXT_SCHEMA) def validate(self, value): if CONF_TEXT in value: @@ -26,9 +24,6 @@ class ButtonType(WidgetType): add_lv_use("label") return value - def get_uses(self): - return ("btn",) - async def on_create(self, var: MockObj, config: dict): if CONF_TEXT in config: lv.label_create(var) diff --git a/esphome/components/lvgl/widgets/buttonmatrix.py b/esphome/components/lvgl/widgets/buttonmatrix.py index 02dc9ed4ba8..854c5e3127d 100644 --- a/esphome/components/lvgl/widgets/buttonmatrix.py +++ b/esphome/components/lvgl/widgets/buttonmatrix.py @@ -188,7 +188,6 @@ class ButtonMatrixType(WidgetType): (CONF_MAIN, CONF_ITEMS), BUTTONMATRIX_SCHEMA, {}, - lv_name="buttonmatrix", ) async def to_code(self, w: Widget, config): @@ -272,5 +271,5 @@ async def button_update_to_code(config, action_id, template_arg, args): ) return await action_to_code( - widgets, do_button_update, action_id, template_arg, args, config + widgets, do_button_update, action_id, template_arg, args ) diff --git a/esphome/components/lvgl/widgets/canvas.py b/esphome/components/lvgl/widgets/canvas.py index 4427a3b00eb..2137acb3f6a 100644 --- a/esphome/components/lvgl/widgets/canvas.py +++ b/esphome/components/lvgl/widgets/canvas.py @@ -141,7 +141,7 @@ async def canvas_fill(config, action_id, template_arg, args): async def do_fill(w: Widget): lv.canvas_fill_bg(w.obj, color, opa) - return await action_to_code(widget, do_fill, action_id, template_arg, args, config) + return await action_to_code(widget, do_fill, action_id, template_arg, args) @automation.register_action( @@ -176,9 +176,7 @@ async def canvas_set_pixel(config, action_id, template_arg, args): x, y = point lv.canvas_set_px(w.obj, x, y, color, opa) - return await action_to_code( - widget, do_set_pixels, action_id, template_arg, args, config - ) + return await action_to_code(widget, do_set_pixels, action_id, template_arg, args) DRAW_SCHEMA = { @@ -214,9 +212,7 @@ async def draw_to_code(config, dsc_type, props, do_draw, action_id, template_arg await do_draw(addr(layer), x, y, dsc) lv.canvas_finish_layer(w.obj, addr(layer)) - return await action_to_code( - widget, action_func, action_id, template_arg, args, config - ) + return await action_to_code(widget, action_func, action_id, template_arg, args) RECT_PROPS = { diff --git a/esphome/components/lvgl/widgets/keyboard.py b/esphome/components/lvgl/widgets/keyboard.py index 65516513a6c..0356e66dac7 100644 --- a/esphome/components/lvgl/widgets/keyboard.py +++ b/esphome/components/lvgl/widgets/keyboard.py @@ -1,17 +1,9 @@ from esphome.components.key_provider import KeyProvider import esphome.config_validation as cv from esphome.const import CONF_ITEMS, CONF_MODE -from esphome.core import CORE from esphome.cpp_types import std_string -from .. import LvContext -from ..defines import ( - CONF_MAIN, - KEYBOARD_MODES, - add_lv_use, - is_widget_completed, - literal, -) +from ..defines import CONF_MAIN, KEYBOARD_MODES, get_widget_map, literal from ..types import LvCompound, LvType from . import Widget, WidgetType, get_widgets from .buttonmatrix import CONF_BUTTONMATRIX @@ -50,30 +42,32 @@ class KeyboardType(WidgetType): ) def get_uses(self): - return CONF_KEYBOARD, CONF_TEXTAREA, CONF_BUTTONMATRIX, CONF_LABEL + return ( + CONF_KEYBOARD, + CONF_TEXTAREA, + CONF_BUTTONMATRIX, + CONF_LABEL, + "KEY_LISTENER", + ) async def to_code(self, w: Widget, config: dict): - add_lv_use("KEY_LISTENER") if mode := config.get(CONF_MODE): await w.set_property(CONF_MODE, await KEYBOARD_MODES.process(mode)) - if textarea := config.get(CONF_TEXTAREA): - if not is_widget_completed(textarea): - # Can only happen for an initial config, where the keyboard is configured before the - # textarea, so it's ok to always emit into the global context - async def add_textarea(): - async with LvContext(): - await w.set_property( - CONF_TEXTAREA, - (await get_widgets(config, CONF_TEXTAREA))[0].obj, - ) - CORE.add_job(add_textarea) - else: - # Handles updates in automations, and properly ordered initial config. Code is generated - # into the enclosing context (main or lambda) - await w.set_property( - CONF_TEXTAREA, (await get_widgets(config, CONF_TEXTAREA))[0].obj - ) + async def update_to_code(self, w: "Widget", config: dict) -> None: + await self.to_code(w, config) + if config.get(CONF_TEXTAREA): + await w.set_property( + CONF_TEXTAREA, (await get_widgets(config, CONF_TEXTAREA))[0].obj + ) + + +async def attach_textareas(): + for w in get_widget_map().values(): + if w.type == keyboard_spec and w.config.get(CONF_TEXTAREA): + await w.set_property( + CONF_TEXTAREA, (await get_widgets(w.config, CONF_TEXTAREA))[0].obj + ) keyboard_spec = KeyboardType() diff --git a/esphome/components/lvgl/widgets/lv_list.py b/esphome/components/lvgl/widgets/lv_list.py index 5d94c2565a2..f41cb8e4e96 100644 --- a/esphome/components/lvgl/widgets/lv_list.py +++ b/esphome/components/lvgl/widgets/lv_list.py @@ -1,4 +1,3 @@ -from collections.abc import Generator from dataclasses import dataclass, field from typing import Any @@ -15,8 +14,6 @@ from esphome.const import ( CONF_TEXT, CONF_TRIGGER_ID, ) -from esphome.core import CORE -from esphome.coroutine import FakeAwaitable from esphome.cpp_generator import MockObj from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor @@ -31,6 +28,7 @@ from ..defines import ( SWIPE_TRIGGERS, TYPE_FLEX, add_lv_use, + get_list_triggers, literal, ) from ..lv_validation import lv_int, lv_text, padding @@ -65,6 +63,7 @@ from . import ( ) from .buttonmatrix import CONF_BUTTONMATRIX from .canvas import CONF_CANVAS +from .keyboard import CONF_KEYBOARD from .label import CONF_LABEL from .meter import CONF_METER from .tabview import CONF_TABVIEW @@ -75,8 +74,6 @@ CONF_WIDGET = "widget" CONF_ON_ADD = "on_add" CONF_ON_REMOVE = "on_remove" -DOMAIN = "lvgl_list" - lv_list_t = LvType("lv_list_t") @@ -87,76 +84,49 @@ class ListTriggers: def _get_list_triggers(list_id) -> ListTriggers: - """ - Trigger Pvariables built for a given list's `on_add`/`on_remove` config, indexed by the - list's own ID. - """ - triggers_by_list = CORE.data.setdefault(DOMAIN, {}) - return triggers_by_list.setdefault(list_id, ListTriggers()) + """Every on_add/on_remove automation config declared for a list, indexed by the list's own ID.""" + return get_list_triggers().get(list_id, ListTriggers()) -def _get_pending_list_triggers(list_id) -> ListTriggers: - """ - Same shape as _get_list_triggers(), but holding raw on_add/on_remove automation - configs, not yet built. - """ - pending_by_list = CORE.data.setdefault(DOMAIN + "_pending", {}) - return pending_by_list.setdefault(list_id, ListTriggers()) - - -def _list_triggers_completed_flag() -> list[bool]: - return CORE.data.setdefault(DOMAIN + "_completed", [False]) - - -def _list_triggers_completed_generator() -> Generator[None, None, None]: - while True: - if _list_triggers_completed_flag()[0]: - return - yield - - -async def _wait_list_triggers_completed() -> None: - """Waits until finish_list_triggers() has built every list's on_add/on_remove automations.""" - if _list_triggers_completed_flag()[0]: - return - await FakeAwaitable(_list_triggers_completed_generator()) +def _declare_list_triggers(list_id) -> ListTriggers: + """Get-or-create a list's trigger record, for ListType.on_create() to populate.""" + return get_list_triggers().setdefault(list_id, ListTriggers()) async def finish_list_triggers() -> None: """ - Builds every list's on_add/on_remove automations, collected by ListType.to_code() - instead of being built there directly. Must run after set_widgets_completed(True). + Builds every list's on_add/on_remove automations, collected by ListType.on_create() + instead of being built there directly. """ - for list_id, pending in CORE.data.get(DOMAIN + "_pending", {}).items(): - triggers = _get_list_triggers(list_id) - for conf in pending.on_add: - trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID]) - await automation.build_automation(trigger, [(cg.int_, "list_index")], conf) - triggers.on_add.append(trigger) - for conf in pending.on_remove: - trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID]) - await automation.build_automation(trigger, [(cg.int_, "list_index")], conf) - triggers.on_remove.append(trigger) - _list_triggers_completed_flag()[0] = True + # Avoid circular references by creating the trigger Pvariables first + built_triggers = [ + (cg.new_Pvariable(conf[CONF_TRIGGER_ID]), conf) + for triggers in get_list_triggers().values() + for conf in triggers.on_add + triggers.on_remove + ] + for trigger, conf in built_triggers: + await automation.build_automation(trigger, [(cg.int_, "list_index")], conf) -def _fire_index_triggers(triggers: list, index) -> None: - for trigger in triggers: +async def _fire_index_triggers(confs: list, index) -> None: + for conf in confs: + # finish_list_triggers() may not have built this trigger yet if it runs as + # part of a different component's own to_code() job than this one - + # get_variable() blocks until it does, regardless of scheduling order. + trigger = await cg.get_variable(conf[CONF_TRIGGER_ID]) lv_add(trigger.trigger(index)) async def _fire_on_add(list_id, list_obj, entry_obj) -> None: - await _wait_list_triggers_completed() - triggers = _get_list_triggers(list_id).on_add - if not triggers: + confs = _get_list_triggers(list_id).on_add + if not confs: return index = cg.RawExpression(f"lvgl::lv_list_get_row_index({list_obj}, {entry_obj})") - _fire_index_triggers(triggers, index) + await _fire_index_triggers(confs, index) async def _fire_on_remove(list_id, index) -> None: - await _wait_list_triggers_completed() - _fire_index_triggers(_get_list_triggers(list_id).on_remove, index) + await _fire_index_triggers(_get_list_triggers(list_id).on_remove, index) LIST_SCHEMA = cv.Schema( @@ -200,14 +170,15 @@ class ListType(WidgetType): def get_uses(self): return TYPE_FLEX, CONF_LABEL, CONF_BUTTON - async def to_code(self, w: Widget, config: dict): + async def on_create(self, var: MockObj, config: dict) -> None: + # Runs before Widget.create() registers this list on_add = config.get(CONF_ON_ADD, ()) on_remove = config.get(CONF_ON_REMOVE, ()) if not on_add and not on_remove: return - pending = _get_pending_list_triggers(w.config[CONF_ID]) - pending.on_add.extend(on_add) - pending.on_remove.extend(on_remove) + triggers = _declare_list_triggers(config[CONF_ID]) + triggers.on_add.extend(on_add) + triggers.on_remove.extend(on_remove) list_spec = ListType() @@ -228,7 +199,6 @@ LIST_ID_SCHEMA = cv.Schema({cv.Required(CONF_ID): cv.use_id(lv_list_t)}) ) async def list_add_text_to_code(config, action_id, template_arg, args): widgets = await get_widgets(config) - await _wait_list_triggers_completed() async def do_add_text(w: Widget): text = await lv_text.process(config[CONF_TEXT]) @@ -239,17 +209,19 @@ async def list_add_text_to_code(config, action_id, template_arg, args): lv.obj_move_to_index(entry, await lv_int.process(idx)) await _fire_on_add(config[CONF_ID], w.obj, entry) - return await action_to_code( - widgets, do_add_text, action_id, template_arg, args, config - ) + return await action_to_code(widgets, do_add_text, action_id, template_arg, args) +# These types of widgets are not compatible with lvgl.list.add, some since they manage their own children in a way +# that can't be rebuilt on every call, others are just Too Hard. _DYNAMIC_WIDGET_UNSUPPORTED = ( CONF_BUTTONMATRIX, + CONF_CANVAS, + CONF_KEYBOARD, + CONF_LIST, + CONF_METER, CONF_TABVIEW, CONF_TILEVIEW, - CONF_METER, - CONF_CANVAS, ) @@ -372,7 +344,6 @@ async def list_add_to_code(config, action_id, template_arg, args): _register_lv_uses(w_type_name, w_conf) _register_dynamic_widget_style_uses(w_conf) widgets = await get_widgets(config) - await _wait_list_triggers_completed() async def do_add(w: Widget): index = None @@ -388,7 +359,7 @@ async def list_add_to_code(config, action_id, template_arg, args): index=index, ) - return await action_to_code(widgets, do_add, action_id, template_arg, args, config) + return await action_to_code(widgets, do_add, action_id, template_arg, args) async def _build_dynamic_widget( @@ -507,7 +478,6 @@ LIST_REMOVE_SCHEMA = LIST_ID_SCHEMA.extend( ) async def list_remove_to_code(config, action_id, template_arg, args): widgets = await get_widgets(config) - await _wait_list_triggers_completed() async def do_remove(w: Widget): index = await lv_int.process(config[CONF_INDEX]) @@ -528,9 +498,7 @@ async def list_remove_to_code(config, action_id, template_arg, args): # Recursively destroys the whole subtree lv.obj_del(child) - return await action_to_code( - widgets, do_remove, action_id, template_arg, args, config - ) + return await action_to_code(widgets, do_remove, action_id, template_arg, args) @automation.register_action( @@ -541,19 +509,15 @@ async def list_remove_to_code(config, action_id, template_arg, args): ) async def list_clear_to_code(config, action_id, template_arg, args): widgets = await get_widgets(config) - await _wait_list_triggers_completed() async def do_clear(w: Widget): - await _wait_list_triggers_completed() - triggers = _get_list_triggers(config[CONF_ID]).on_remove - if triggers: + confs = _get_list_triggers(config[CONF_ID]).on_remove + if confs: # Fire on_remove for every entry, newest to oldest, before wiping them all out, # so on_remove's semantics ("an entry left the list") hold with LvCountdown("list_index", lv_expr.obj_get_child_count(w.obj)) as index: - _fire_index_triggers(triggers, index) + await _fire_index_triggers(confs, index) # lv_obj_clean recursively destroys every child's whole subtree lv.obj_clean(w.obj) - return await action_to_code( - widgets, do_clear, action_id, template_arg, args, config - ) + return await action_to_code(widgets, do_clear, action_id, template_arg, args) diff --git a/esphome/components/lvgl/widgets/meter.py b/esphome/components/lvgl/widgets/meter.py index 166e88f382a..ba322881289 100644 --- a/esphome/components/lvgl/widgets/meter.py +++ b/esphome/components/lvgl/widgets/meter.py @@ -612,9 +612,7 @@ async def indicator_update_to_code(config, action_id, template_arg, args): async def set_value(w: Widget): await set_indicator_values(w.parent, w, config) - return await action_to_code( - widget, set_value, action_id, template_arg, args, config - ) + return await action_to_code(widget, set_value, action_id, template_arg, args) async def set_indicator_values(scale: MockObj, indicator: Widget, config): diff --git a/esphome/components/lvgl/widgets/page.py b/esphome/components/lvgl/widgets/page.py index 7e75ab6a2da..9dc9123ac9e 100644 --- a/esphome/components/lvgl/widgets/page.py +++ b/esphome/components/lvgl/widgets/page.py @@ -26,14 +26,7 @@ from ..lvcode import ( ) from ..schemas import LVGL_SCHEMA from ..types import LvglAction, LvglCondition, lv_page_t -from . import ( - Widget, - WidgetType, - add_widgets, - get_widgets, - set_obj_properties, - wait_for_widgets, -) +from . import Widget, WidgetType, add_widgets, get_widgets, set_obj_properties CONF_ON_LOAD = "on_load" CONF_ON_UNLOAD = "on_unload" @@ -107,7 +100,6 @@ async def page_next_to_code(config, action_id, template_arg, args): ), ) async def page_is_showing_to_code(config, condition_id, template_arg, args): - await wait_for_widgets() page = await cg.get_variable(config[CONF_ID]) async with LambdaContext( [(lv_page_t.operator("ptr"), PAGE_ARG)], return_type=cg.bool_ diff --git a/esphome/components/lvgl/widgets/table.py b/esphome/components/lvgl/widgets/table.py index f000ea18466..019c6d4ecaf 100644 --- a/esphome/components/lvgl/widgets/table.py +++ b/esphome/components/lvgl/widgets/table.py @@ -274,6 +274,4 @@ async def table_cell_update_to_code( ) await set_cell_ctrl(w, row, column, config) - return await action_to_code( - widgets, do_update, action_id, template_arg, args, config - ) + return await action_to_code(widgets, do_update, action_id, template_arg, args) diff --git a/tests/component_tests/lvgl/config/keyboard_before_textarea_test.yaml b/tests/component_tests/lvgl/config/keyboard_before_textarea_test.yaml new file mode 100644 index 00000000000..8f80a942f5d --- /dev/null +++ b/tests/component_tests/lvgl/config/keyboard_before_textarea_test.yaml @@ -0,0 +1,28 @@ +esphome: + name: test-keyboard-before-textarea + +esp32: + board: lolin_c3_mini + +spi: + mosi_pin: + number: GPIO2 + ignore_strapping_warning: true + clk_pin: GPIO1 + +display: + - platform: mipi_spi + data_rate: 20MHz + model: st7735 + cs_pin: + number: GPIO8 + ignore_strapping_warning: true + dc_pin: GPIO3 + +lvgl: + widgets: + - keyboard: + id: kb + textarea: ta + - textarea: + id: ta diff --git a/tests/component_tests/lvgl/config/keyboard_update_textarea_test.yaml b/tests/component_tests/lvgl/config/keyboard_update_textarea_test.yaml new file mode 100644 index 00000000000..1d02a2df587 --- /dev/null +++ b/tests/component_tests/lvgl/config/keyboard_update_textarea_test.yaml @@ -0,0 +1,34 @@ +esphome: + name: test-keyboard-update-textarea + on_boot: + - lvgl.keyboard.update: + id: kb + textarea: ta2 + +esp32: + board: lolin_c3_mini + +spi: + mosi_pin: + number: GPIO2 + ignore_strapping_warning: true + clk_pin: GPIO1 + +display: + - platform: mipi_spi + data_rate: 20MHz + model: st7735 + cs_pin: + number: GPIO8 + ignore_strapping_warning: true + dc_pin: GPIO3 + +lvgl: + widgets: + - keyboard: + id: kb + textarea: ta1 + - textarea: + id: ta1 + - textarea: + id: ta2 diff --git a/tests/component_tests/lvgl/config/multi_instance_theme_test.yaml b/tests/component_tests/lvgl/config/multi_instance_theme_test.yaml new file mode 100644 index 00000000000..d30f1ff29a1 --- /dev/null +++ b/tests/component_tests/lvgl/config/multi_instance_theme_test.yaml @@ -0,0 +1,35 @@ +esphome: + name: test-multi-instance-theme + +host: + mac_address: "62:23:45:AF:B3:DD" + +display: + - platform: snapshot + id: sdl0 + dimensions: + width: 100 + height: 100 + - platform: snapshot + id: sdl1 + dimensions: + width: 100 + height: 100 + +lvgl: + - id: lvgl_0 + displays: sdl0 + widgets: + - label: + id: label_0 + text: "First" + + - id: lvgl_1 + displays: sdl1 + theme: + label: + bg_color: 0xFF0000 + widgets: + - label: + id: label_1 + text: "Second" diff --git a/tests/component_tests/lvgl/test_keyboard_before_textarea.py b/tests/component_tests/lvgl/test_keyboard_before_textarea.py new file mode 100644 index 00000000000..3da4b1cebe8 --- /dev/null +++ b/tests/component_tests/lvgl/test_keyboard_before_textarea.py @@ -0,0 +1,47 @@ +"""Regression test: a keyboard: declared before its textarea: sibling must +still get attached to it, and only after both widgets exist. + +attach_textareas() runs as a deferred pass, after every widget (across every +LVGL instance) is created, so it must emit the attach call after the +keyboard's own creation statement, not inline during widget creation. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from esphome.__main__ import generate_cpp_contents +from esphome.config import read_config +from esphome.core import CORE + + +@pytest.fixture(scope="module") +def main_cpp(request: pytest.FixtureRequest) -> str: + config_path = ( + Path(request.fspath).parent / "config" / "keyboard_before_textarea_test.yaml" + ) + original_path = CORE.config_path + try: + CORE.config_path = config_path + CORE.config = read_config({}) + generate_cpp_contents(CORE.config) + return CORE.cpp_main_section + finally: + CORE.config_path = original_path + CORE.reset() + + +def test_keyboard_and_textarea_are_created(main_cpp: str) -> None: + assert "lv_keyboard_create(" in main_cpp + assert "lv_textarea_create(" in main_cpp + + +def test_attach_call_runs_after_keyboard_and_textarea_are_created( + main_cpp: str, +) -> None: + attach_index = main_cpp.find("lv_keyboard_set_textarea(kb->obj, ta);") + assert attach_index != -1, "keyboard was never attached to its textarea" + assert attach_index > main_cpp.find("lv_keyboard_create(") + assert attach_index > main_cpp.find("lv_textarea_create(") diff --git a/tests/component_tests/lvgl/test_keyboard_update_textarea.py b/tests/component_tests/lvgl/test_keyboard_update_textarea.py new file mode 100644 index 00000000000..154d2ee2004 --- /dev/null +++ b/tests/component_tests/lvgl/test_keyboard_update_textarea.py @@ -0,0 +1,33 @@ +"""Regression test: lvgl.keyboard.update must be able to change which +textarea a keyboard is attached to. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from esphome.__main__ import generate_cpp_contents +from esphome.config import read_config +from esphome.core import CORE + + +@pytest.fixture(scope="module") +def main_cpp(request: pytest.FixtureRequest) -> str: + config_path = ( + Path(request.fspath).parent / "config" / "keyboard_update_textarea_test.yaml" + ) + original_path = CORE.config_path + try: + CORE.config_path = config_path + CORE.config = read_config({}) + generate_cpp_contents(CORE.config) + return CORE.cpp_main_section + finally: + CORE.config_path = original_path + CORE.reset() + + +def test_keyboard_update_changes_textarea(main_cpp: str) -> None: + assert "lv_keyboard_set_textarea(kb->obj, ta2);" in main_cpp diff --git a/tests/component_tests/lvgl/test_list.py b/tests/component_tests/lvgl/test_list.py index 93ee162f1af..368a3a49905 100644 --- a/tests/component_tests/lvgl/test_list.py +++ b/tests/component_tests/lvgl/test_list.py @@ -286,7 +286,7 @@ def test_add_text_with_index_moves_before_firing_on_add(main_cpp: str) -> None: def test_add_button_with_checkable_flag(main_cpp: str) -> None: - assert "lv_obj_t *dyn_button_VAR_ = lv_btn_create(test_list);" in main_cpp + assert "lv_obj_t *dyn_button_VAR_ = lv_button_create(test_list);" in main_cpp assert ( "lv_obj_add_flag(dyn_button_VAR_, (lv_obj_flag_t)(LV_OBJ_FLAG_CHECKABLE));" in main_cpp @@ -310,7 +310,7 @@ def test_add_nested_hierarchy_with_compound_child(main_cpp: str) -> None: assert ( "lv_obj_t *dyn_label_1_VAR_ = lv_label_create(dyn_obj_VAR_);\n" " lv_obj_add_style(dyn_label_1_VAR_, _lv_theme_style_label_main_default, " - "(lv_state_t)(LV_PART_MAIN));\n" + "LV_PART_MAIN);\n" ' lv_label_set_text(dyn_label_1_VAR_, "Nested");' ) in main_cpp @@ -321,7 +321,7 @@ def test_add_applies_theme_styles_to_dynamic_widget(main_cpp: str) -> None: """ assert ( "lv_obj_add_style(dyn_label_1_VAR_, _lv_theme_style_label_main_default, " - "(lv_state_t)(LV_PART_MAIN));" + "LV_PART_MAIN);" ) in main_cpp assert "LvDropdownType *dyn_dropdown_1_VAR_ = new LvDropdownType();" in main_cpp assert "lv_dropdown_create(dyn_obj_VAR_)" in main_cpp @@ -343,7 +343,7 @@ def test_add_nested_same_type_child_does_not_shadow_parent(main_cpp: str) -> Non assert ( "lv_obj_t *dyn_label_2_VAR_ = lv_label_create(dyn_obj_1_VAR_);\n" " lv_obj_add_style(dyn_label_2_VAR_, _lv_theme_style_label_main_default, " - "(lv_state_t)(LV_PART_MAIN));\n" + "LV_PART_MAIN);\n" ' lv_label_set_text(dyn_label_2_VAR_, "Grandchild");' ) in main_cpp diff --git a/tests/component_tests/lvgl/test_list_on_add_lvgl_action.py b/tests/component_tests/lvgl/test_list_on_add_lvgl_action.py index 2383adc2518..9ff7b8736e0 100644 --- a/tests/component_tests/lvgl/test_list_on_add_lvgl_action.py +++ b/tests/component_tests/lvgl/test_list_on_add_lvgl_action.py @@ -1,16 +1,4 @@ -"""Regression test: on_add:/on_remove: containing an lvgl action must not deadlock. - -ListType.to_code() used to build the on_add/on_remove automations directly, during -widget creation. Every lvgl action's to_code awaits wait_for_widgets(), which only -resolves once *all* widgets - including the list itself - have finished being -created. Building an automation containing an lvgl action from inside that same -widget-creation walk therefore could never complete: codegen deadlocked with -"Circular dependency detected!". Fixed by deferring the actual build_automation() -call to finish_list_triggers(), run after set_widgets_completed(True) - and, -critically, before generate_triggers(), which is what processes other widgets' -on_click etc. automations that might reference this list (e.g. via lvgl.list.add), -and which therefore need the list's own on_add/on_remove triggers to already exist. -""" +"""Regression test: on_add:/on_remove: containing an lvgl action must not deadlock.""" from __future__ import annotations diff --git a/tests/component_tests/lvgl/test_multi_instance_theme.py b/tests/component_tests/lvgl/test_multi_instance_theme.py new file mode 100644 index 00000000000..82f271b6059 --- /dev/null +++ b/tests/component_tests/lvgl/test_multi_instance_theme.py @@ -0,0 +1,44 @@ +"""Regression test: a widget's theme style must be attached regardless of +which LVGL instance declares theme: and which instance's widgets are built +first. + +theme_to_code() runs once per LVGL instance, interleaved with that instance's +own add_widgets(). A widget built by an earlier instance, before any instance +has declared theming for its type, must still pick up a later instance's +theme: declaration for that type - including on the later instance's own +widgets, which get_widget_theme_styles()'s memoisation can poison too. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from esphome.__main__ import generate_cpp_contents +from esphome.config import read_config +from esphome.core import CORE + + +@pytest.fixture(scope="module") +def main_cpp(request: pytest.FixtureRequest) -> str: + config_path = ( + Path(request.fspath).parent / "config" / "multi_instance_theme_test.yaml" + ) + original_path = CORE.config_path + try: + CORE.config_path = config_path + CORE.config = read_config({}) + generate_cpp_contents(CORE.config) + return CORE.cpp_main_section + finally: + CORE.config_path = original_path + CORE.reset() + + +def test_earlier_instance_widget_is_themed(main_cpp: str) -> None: + assert "lv_obj_add_style(label_0, _lv_theme_style_label_main_default," in main_cpp + + +def test_later_instance_widget_is_themed(main_cpp: str) -> None: + assert "lv_obj_add_style(label_1, _lv_theme_style_label_main_default," in main_cpp diff --git a/tests/components/lvgl/lvgl-package.yaml b/tests/components/lvgl/lvgl-package.yaml index e10ca68ac64..18a7fd0913a 100644 --- a/tests/components/lvgl/lvgl-package.yaml +++ b/tests/components/lvgl/lvgl-package.yaml @@ -42,6 +42,13 @@ script: label: text: row +esphome: + on_boot: + then: + - lvgl.theme.update: + obj: + border_width: 2 + lvgl: id: lvgl_id rotation: 90 @@ -1351,6 +1358,17 @@ lvgl: id: checkbox_id text: Checkbox align: bottom_right + - list: + id: list_a + on_add: + - lvgl.list.add: + id: list_b # fires list_b's on_add + label: {text: x} + - list: + id: list_b + on_add: + - logger.log: added + - list: id: test_list_id align: top_right diff --git a/tests/unit_tests/components/lvgl/test_action_to_code_context_isolation.py b/tests/unit_tests/components/lvgl/test_action_to_code_context_isolation.py new file mode 100644 index 00000000000..0842bc87076 --- /dev/null +++ b/tests/unit_tests/components/lvgl/test_action_to_code_context_isolation.py @@ -0,0 +1,56 @@ +"""Regression test: action_to_code(): Confirms two +actions whose own processing each suspends mid-lambda, waiting for an ID, don't +corrupt each other's LambdaContext when interleaved. +""" + +from __future__ import annotations + +import pytest + +from esphome.automation import ACTION_REGISTRY +import esphome.codegen as cg +from esphome.components.lvgl.automation import action_to_code +from esphome.components.lvgl.lvcode import lv_add +from esphome.core import CORE, ID +from esphome.cpp_generator import RawExpression, TemplateArguments + + +@pytest.mark.asyncio +async def test_action_to_code_survives_interleaved_suspended_contexts( + setup_core, +) -> None: + later_id_a = ID("later_var_a", False, cg.int_) + later_id_b = ID("later_var_b", False, cg.int_) + action_type = ACTION_REGISTRY["lvgl.list.add"].type_id + + async def action_a(_widget) -> None: + value = await cg.get_variable(later_id_a) + lv_add(RawExpression(f"action_a_marker({value})")) + + async def action_b(_widget) -> None: + value = await cg.get_variable(later_id_b) + lv_add(RawExpression(f"action_b_marker({value})")) + + async def run_action_a() -> None: + action_id = ID("test_action_a", is_declaration=True, type=action_type) + await action_to_code([None], action_a, action_id, TemplateArguments(), []) + + async def run_action_b() -> None: + action_id = ID("test_action_b", is_declaration=True, type=action_type) + await action_to_code([None], action_b, action_id, TemplateArguments(), []) + + async def define_later_ids() -> None: + # Both actions are still suspended, mid-LambdaContext, when this runs - + # resolving both at once lets each resume while the other's context is + # still open, rather than one finishing before the other starts. + cg.new_variable(later_id_a, RawExpression("1")) + cg.new_variable(later_id_b, RawExpression("2")) + + CORE.add_job(run_action_a) + CORE.add_job(run_action_b) + CORE.add_job(define_later_ids) + CORE.flush_tasks() + + text = "\n".join(str(s) for s in CORE.main_statements) + assert "action_a_marker(later_var_a)" in text + assert "action_b_marker(later_var_b)" in text diff --git a/tests/unit_tests/components/lvgl/test_get_part_state_selector.py b/tests/unit_tests/components/lvgl/test_get_part_state_selector.py new file mode 100644 index 00000000000..e7d623b7c6b --- /dev/null +++ b/tests/unit_tests/components/lvgl/test_get_part_state_selector.py @@ -0,0 +1,33 @@ +"""Tests for get_part_state_selector()'s three branches.""" + +from __future__ import annotations + +from esphome.components.lvgl.defines import get_part_state_selector + + +def test_default_state_returns_bare_part() -> None: + assert str(get_part_state_selector("main", "default")) == "LV_PART_MAIN" + assert str(get_part_state_selector("knob", "default")) == "LV_PART_KNOB" + + +def test_main_part_with_non_default_state_returns_bare_state() -> None: + assert str(get_part_state_selector("main", "pressed")) == "LV_STATE_PRESSED" + + +def test_non_main_part_with_non_default_state_combines_both() -> None: + assert str(get_part_state_selector("knob", "pressed")) == ( + "(static_cast(LV_STATE_PRESSED) | " + "static_cast(LV_PART_KNOB))" + ) + + +def test_accepts_already_prefixed_part_and_state() -> None: + assert ( + str(get_part_state_selector("LV_PART_MAIN", "LV_STATE_DEFAULT")) + == "LV_PART_MAIN" + ) + assert ( + str(get_part_state_selector("LV_PART_KNOB", "LV_STATE_PRESSED")) + == "(static_cast(LV_STATE_PRESSED) | " + "static_cast(LV_PART_KNOB))" + ) diff --git a/tests/unit_tests/components/lvgl/test_list_trigger_ordering.py b/tests/unit_tests/components/lvgl/test_list_trigger_ordering.py new file mode 100644 index 00000000000..6caf83fe330 --- /dev/null +++ b/tests/unit_tests/components/lvgl/test_list_trigger_ordering.py @@ -0,0 +1,129 @@ +"""Regression test: an lvgl.list.add action must fire a list's on_add trigger +even if it reaches _fire_on_add() before finish_list_triggers() has built that +list's Trigger Pvariable. + +ListType.on_create() only records on_add/on_remove configs (via +_declare_list_triggers()) - finish_list_triggers() is what actually builds the +Trigger Pvariables from them. An lvgl.list.add action for a list can be +scheduled as part of a different component's own to_code() coroutine, entirely +independent of lvgl's own, so it can reach _fire_on_add() before +finish_list_triggers() has run for that list. _fire_on_add()/_fire_on_remove() +resolve each trigger via cg.get_variable(), which blocks until +finish_list_triggers() builds it - regardless of which of the two jobs the +scheduler happens to run first. + +This is reproduced deterministically here (no reliance on incidental component +priority/scheduling) by scheduling the action's job before finish_list_triggers() +on ESPHome's own coroutine scheduler: without cg.get_variable()'s wait, the +action job would run to completion first and observe the trigger as not yet +built. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from esphome.automation import ACTION_REGISTRY +from esphome.components.lvgl.lvcode import LvContext +from esphome.components.lvgl.schemas import container_schema +from esphome.components.lvgl.widgets import Widget, widget_to_code +from esphome.components.lvgl.widgets.lv_list import ( + CONF_ON_ADD, + _get_list_triggers, + finish_list_triggers, + list_spec, +) +from esphome.const import CONF_AUTOMATION_ID, CONF_THEN, CONF_TRIGGER_ID, CONF_TYPE_ID +from esphome.core import CORE, ID +from esphome.cpp_generator import MockObj, TemplateArguments +from esphome.yaml_util import make_data_base + + +def _statements() -> list[str]: + return [str(s) for s in CORE.main_statements] + + +@pytest.mark.asyncio +async def test_list_add_action_running_before_finish_list_triggers_still_fires_on_add( + setup_core, +) -> None: + config = container_schema(list_spec)( + { + "id": "test_list", + CONF_ON_ADD: [{"lambda": make_data_base("return;")}], + } + ) + # Auto-generated IDs (trigger/automation/action) are normally resolved to + # unique names by esphome's full config pass before code generation; do + # that by hand here since this test only exercises the widget/trigger + # codegen slice in isolation. + automation_conf = config[CONF_ON_ADD][0] + automation_conf[CONF_TRIGGER_ID].resolve([]) + automation_conf[CONF_AUTOMATION_ID].resolve([]) + automation_conf[CONF_THEN][0][CONF_TYPE_ID].resolve([]) + + parent = MockObj("parent_obj") + async with LvContext(): + await widget_to_code(config, list_spec, parent) + + # Schedule the lvgl.list.add action's job before finish_list_triggers()'s - + # mirroring an action that lives in a different component's automation than + # lvgl's own to_code(), which can reach this action before lvgl gets to build + # this list's on_add/on_remove triggers. + entry = ACTION_REGISTRY["lvgl.list.add"] + add_config = entry.schema({"id": "test_list", "label": {"text": "row"}}) + action_id = ID("test_list_add_action", is_declaration=True, type=entry.type_id) + + async def run_add_action() -> None: + async with LvContext(): + await entry.coroutine_fun(add_config, action_id, TemplateArguments(), []) + + CORE.add_job(run_add_action) + CORE.add_job(finish_list_triggers) + CORE.flush_tasks() + + statements = _statements() + assert any("->trigger(" in s for s in statements), ( + "on_add did not fire: the lvgl.list.add action ran before " + "finish_list_triggers() built the list's on_add trigger, and " + "_fire_on_add() didn't wait for it" + ) + + +@pytest.mark.asyncio +async def test_on_add_recorded_before_widget_registered(setup_core) -> None: + """Widget.create() is what makes a list visible to get_widgets(), so an + action interleaved with its creation could resolve get_widgets() and reach + _fire_on_add() right after Widget.create() runs. Its on_add config must + already be recorded by then - ListType.on_create() (called before + Widget.create()) is what guarantees that, not ListType.to_code() (called + after). + """ + config = container_schema(list_spec)( + { + "id": "test_list", + CONF_ON_ADD: [{"lambda": make_data_base("return;")}], + } + ) + automation_conf = config[CONF_ON_ADD][0] + automation_conf[CONF_TRIGGER_ID].resolve([]) + automation_conf[CONF_AUTOMATION_ID].resolve([]) + automation_conf[CONF_THEN][0][CONF_TYPE_ID].resolve([]) + + seen_on_add_counts = [] + real_create = Widget.create + + def spy_create(name, var, wtype, config=None): + seen_on_add_counts.append(len(_get_list_triggers(name).on_add)) + return real_create(name, var, wtype, config) + + parent = MockObj("parent_obj") + with patch.object(Widget, "create", side_effect=spy_create): + async with LvContext(): + await widget_to_code(config, list_spec, parent) + + assert seen_on_add_counts == [1], ( + "on_add wasn't recorded yet when Widget.create() registered the list" + ) diff --git a/tests/unit_tests/components/lvgl/test_table_codegen.py b/tests/unit_tests/components/lvgl/test_table_codegen.py index 390f67dffcf..12ebd3a12f1 100644 --- a/tests/unit_tests/components/lvgl/test_table_codegen.py +++ b/tests/unit_tests/components/lvgl/test_table_codegen.py @@ -5,7 +5,6 @@ from __future__ import annotations import pytest from esphome.automation import ACTION_REGISTRY -from esphome.components.lvgl.defines import set_widgets_completed from esphome.components.lvgl.lvcode import LvContext from esphome.components.lvgl.schemas import container_schema from esphome.components.lvgl.trigger import generate_triggers @@ -151,7 +150,6 @@ async def test_selected_cell_omitted_entirely_when_not_configured( @pytest.mark.asyncio async def test_cell_update_action_writes_only_the_given_fields(setup_core) -> None: await _create_table({"id": "table_update", "rows": [["a", "b"], ["c", "d"]]}) - set_widgets_completed(True) # Only inspect statements emitted by the action below, not by creation. before = len(_statements()) @@ -194,7 +192,6 @@ async def test_on_value_registers_a_value_changed_event_callback(setup_core) -> parent = MockObj("parent_obj") async with LvContext(): await widget_to_code(config, table_spec, parent) - set_widgets_completed(True) await generate_triggers() statements = _statements() diff --git a/tests/unit_tests/components/lvgl/test_theme_update_ordering.py b/tests/unit_tests/components/lvgl/test_theme_update_ordering.py new file mode 100644 index 00000000000..62b8a4eaedf --- /dev/null +++ b/tests/unit_tests/components/lvgl/test_theme_update_ordering.py @@ -0,0 +1,44 @@ +"""Regression test: an lvgl.theme.update action must still apply its style +change even if it runs before theme_to_code() has built that style. + +theme_update_to_code() reads get_theme_widget_map() synchronously and raises +cv.Invalid if the requested style isn't there yet - relying on theme_to_code() +(which materialises a style for every requested widget/part/state combo) to +have always already run. That's guaranteed when the action lives inside the +lvgl: block's own automations (same to_code() job, sequential), but not when +it's scheduled as part of a different component's own to_code() job - e.g. +tests/components/lvgl/lvgl-package.yaml's `esphome: on_boot:` case, which this +test reproduces at the scheduler level. +""" + +from __future__ import annotations + +import pytest + +from esphome.automation import ACTION_REGISTRY +from esphome.components.lvgl.schemas import theme_update_schema +from esphome.components.lvgl.styles import theme_to_code +from esphome.core import CORE, ID +from esphome.cpp_generator import TemplateArguments + + +@pytest.mark.asyncio +async def test_theme_update_before_theme_to_code_still_applies(setup_core) -> None: + add_config = theme_update_schema({"obj": {"border_width": 2}}) + + entry = ACTION_REGISTRY["lvgl.theme.update"] + action_id = ID("test_theme_update_action", is_declaration=True, type=entry.type_id) + + async def run_update_action() -> None: + await entry.coroutine_fun(add_config, action_id, TemplateArguments(), []) + + # Scheduled before theme_to_code()'s job - mirrors the action being reached + # from a different component's own to_code() job than lvgl's. + CORE.add_job(run_update_action) + CORE.add_job(theme_to_code, {}) + CORE.flush_tasks() + + statements = [str(s) for s in CORE.main_statements] + assert any("style_set_border_width" in s for s in statements), ( + "theme.update's border_width change was never applied" + ) From 3cccdca7718fbcf4906ece00ecede1d1ba3c9539 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:55:50 +0200 Subject: [PATCH 075/133] [modbus] Fix a format warning when logging the frame buffer size (#19934) --- esphome/components/modbus/modbus.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/modbus/modbus.cpp b/esphome/components/modbus/modbus.cpp index 0cf57ef7aa5..5f237be8dd7 100644 --- a/esphome/components/modbus/modbus.cpp +++ b/esphome/components/modbus/modbus.cpp @@ -210,7 +210,7 @@ void ModbusServerHub::parse_modbus_frames() { this->deferred_payload_len_ = 0; } size_t size = this->rx_buffer_.size(); - ESP_LOGVV(TAG, "Parsing frames buffer size = %" PRIu32, size); + ESP_LOGVV(TAG, "Parsing frames buffer size = %zu", size); bool retry_as_client = false; // A broadcast is a client request, never a peer response; clear any stale expectation (RTU is half-duplex). const bool is_broadcast = this->rx_buffer_[0] == BROADCAST_ADDRESS; From f327cbbcfe429fdbb69adf1734626198095df69b Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:56:15 +0200 Subject: [PATCH 076/133] [dlms_meter] Fix a format warning when logging the receive timeout (#19935) --- esphome/components/dlms_meter/dlms_meter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/dlms_meter/dlms_meter.cpp b/esphome/components/dlms_meter/dlms_meter.cpp index bdbf798df52..e672fb200fc 100644 --- a/esphome/components/dlms_meter/dlms_meter.cpp +++ b/esphome/components/dlms_meter/dlms_meter.cpp @@ -86,7 +86,7 @@ void DlmsMeterComponent::setup() { this->flush_rx_buffer_(); } void DlmsMeterComponent::dump_config() { ESP_LOGCONFIG(TAG, "DLMS Meter:"); - ESP_LOGCONFIG(TAG, " Receive Timeout: %u ms", this->receive_timeout_ms_); + ESP_LOGCONFIG(TAG, " Receive Timeout: %" PRIu32 " ms", this->receive_timeout_ms_); ESP_LOGCONFIG(TAG, " Skip CRC Check: %s", YESNO(this->skip_crc_check_)); for (const auto &pattern : this->custom_patterns_) { From e7f2159f0e80da1d73b9b776b813399fbc3ef875 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:56:37 +0200 Subject: [PATCH 077/133] [api] Fix format warnings in the api test config (#19936) --- tests/components/api/common-base.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/components/api/common-base.yaml b/tests/components/api/common-base.yaml index 5e3139da48c..f55e7b176dd 100644 --- a/tests/components/api/common-base.yaml +++ b/tests/components/api/common-base.yaml @@ -92,7 +92,7 @@ api: then: - logger.log: # yamllint disable rule:line-length - format: "Bool: %s (%u), Int: %ld (%u), Float: %f (%u), String: %s (%u)" + format: "Bool: %s (%zu), Int: %ld (%zu), Float: %f (%zu), String: %s (%zu)" # yamllint enable rule:line-length args: - YESNO(bool_arr[0]) @@ -116,7 +116,7 @@ api: then: - delay: 20ms - logger.log: - format: "Delayed: %s (%u ints, %u strings)" + format: "Delayed: %s (%zu ints, %zu strings)" args: - name.c_str() - int_arr.size() @@ -139,7 +139,7 @@ api: message: !lambda 'return name;' on_success: - logger.log: - format: "Notified %s (%u ints)" + format: "Notified %s (%zu ints)" args: - name.c_str() - int_arr.size() From 987618b49e753d88796d5db082339d7985d86f02 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:57:01 +0200 Subject: [PATCH 078/133] [micro_wake_word] Fix a format warning when logging a model version mismatch (#19937) --- esphome/components/micro_wake_word/model_data.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esphome/components/micro_wake_word/model_data.cpp b/esphome/components/micro_wake_word/model_data.cpp index a7326ab77a0..c235e290ec2 100644 --- a/esphome/components/micro_wake_word/model_data.cpp +++ b/esphome/components/micro_wake_word/model_data.cpp @@ -87,7 +87,8 @@ bool ModelData::validate_and_mark_ready() { const tflite::Model *model = tflite::GetModel(this->data_); if (model->version() != TFLITE_SCHEMA_VERSION) { - ESP_LOGE(TAG, "TFLite model version mismatch (expected %d, got %d)", TFLITE_SCHEMA_VERSION, model->version()); + ESP_LOGE(TAG, "TFLite model version mismatch (expected %d, got %" PRIu32 ")", TFLITE_SCHEMA_VERSION, + model->version()); return false; } From 39f41656b9578dcc269d35a1a7938f4b02c20239 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:57:46 +0200 Subject: [PATCH 079/133] [seeed_mr60bha2] Use the sub_* helpers for optional child entities (#19939) --- .../seeed_mr60bha2/binary_sensor.py | 7 +++---- esphome/components/seeed_mr60bha2/sensor.py | 19 ++++++------------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/esphome/components/seeed_mr60bha2/binary_sensor.py b/esphome/components/seeed_mr60bha2/binary_sensor.py index 4130bac224a..b009de8381a 100644 --- a/esphome/components/seeed_mr60bha2/binary_sensor.py +++ b/esphome/components/seeed_mr60bha2/binary_sensor.py @@ -17,8 +17,7 @@ CONFIG_SCHEMA = { async def to_code(config: ConfigType) -> None: - mr60bha2_component = await cg.get_variable(config[CONF_MR60BHA2_ID]) + hub = await cg.get_variable(config[CONF_MR60BHA2_ID]) - if has_target_config := config.get(CONF_HAS_TARGET): - sens = await binary_sensor.new_binary_sensor(has_target_config) - cg.add(mr60bha2_component.set_has_target_binary_sensor(sens)) + binary_sensors = binary_sensor.sub_binary_sensors(config) + await binary_sensors(CONF_HAS_TARGET, hub.set_has_target_binary_sensor) diff --git a/esphome/components/seeed_mr60bha2/sensor.py b/esphome/components/seeed_mr60bha2/sensor.py index a2f41a90a81..d815aa1a8c3 100644 --- a/esphome/components/seeed_mr60bha2/sensor.py +++ b/esphome/components/seeed_mr60bha2/sensor.py @@ -51,16 +51,9 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config: ConfigType) -> None: - mr60bha2_component = await cg.get_variable(config[CONF_MR60BHA2_ID]) - if breath_rate_config := config.get(CONF_BREATH_RATE): - sens = await sensor.new_sensor(breath_rate_config) - cg.add(mr60bha2_component.set_breath_rate_sensor(sens)) - if heart_rate_config := config.get(CONF_HEART_RATE): - sens = await sensor.new_sensor(heart_rate_config) - cg.add(mr60bha2_component.set_heart_rate_sensor(sens)) - if distance_config := config.get(CONF_DISTANCE): - sens = await sensor.new_sensor(distance_config) - cg.add(mr60bha2_component.set_distance_sensor(sens)) - if num_targets_config := config.get(CONF_NUM_TARGETS): - sens = await sensor.new_sensor(num_targets_config) - cg.add(mr60bha2_component.set_num_targets_sensor(sens)) + hub = await cg.get_variable(config[CONF_MR60BHA2_ID]) + sensors = sensor.sub_sensors(config) + await sensors(CONF_BREATH_RATE, hub.set_breath_rate_sensor) + await sensors(CONF_HEART_RATE, hub.set_heart_rate_sensor) + await sensors(CONF_DISTANCE, hub.set_distance_sensor) + await sensors(CONF_NUM_TARGETS, hub.set_num_targets_sensor) From 3062e1f844c61c376376ad2fa5b6c43085e969bf Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:58:02 +0200 Subject: [PATCH 080/133] [smt100] Use the sub_* helpers for optional child entities (#19940) --- esphome/components/smt100/sensor.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/esphome/components/smt100/sensor.py b/esphome/components/smt100/sensor.py index 7ba7da801cd..dae98bc1897 100644 --- a/esphome/components/smt100/sensor.py +++ b/esphome/components/smt100/sensor.py @@ -83,22 +83,9 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await uart.register_uart_device(var, config) - if CONF_COUNTS in config: - sens = await sensor.new_sensor(config[CONF_COUNTS]) - cg.add(var.set_counts_sensor(sens)) - - if CONF_PERMITTIVITY in config: - sens = await sensor.new_sensor(config[CONF_PERMITTIVITY]) - cg.add(var.set_permittivity_sensor(sens)) - - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) - - if CONF_MOISTURE in config: - sens = await sensor.new_sensor(config[CONF_MOISTURE]) - cg.add(var.set_moisture_sensor(sens)) - - if CONF_VOLTAGE in config: - sens = await sensor.new_sensor(config[CONF_VOLTAGE]) - cg.add(var.set_voltage_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_COUNTS, var.set_counts_sensor) + await sensors(CONF_PERMITTIVITY, var.set_permittivity_sensor) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_MOISTURE, var.set_moisture_sensor) + await sensors(CONF_VOLTAGE, var.set_voltage_sensor) From 08ed25833d34f05ea1d097331817ca6864d74be8 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:58:14 +0200 Subject: [PATCH 081/133] [mmc5603] Use the sub_* helpers for optional child entities (#19941) --- esphome/components/mmc5603/sensor.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/esphome/components/mmc5603/sensor.py b/esphome/components/mmc5603/sensor.py index a9f240508c1..70c70877b75 100644 --- a/esphome/components/mmc5603/sensor.py +++ b/esphome/components/mmc5603/sensor.py @@ -82,17 +82,10 @@ async def to_code(config: ConfigType) -> None: await i2c.register_i2c_device(var, config) cg.add(var.set_datarate(auto_data_rate(config))) - if CONF_FIELD_STRENGTH_X in config: - sens = await sensor.new_sensor(config[CONF_FIELD_STRENGTH_X]) - cg.add(var.set_x_sensor(sens)) - if CONF_FIELD_STRENGTH_Y in config: - sens = await sensor.new_sensor(config[CONF_FIELD_STRENGTH_Y]) - cg.add(var.set_y_sensor(sens)) - if CONF_FIELD_STRENGTH_Z in config: - sens = await sensor.new_sensor(config[CONF_FIELD_STRENGTH_Z]) - cg.add(var.set_z_sensor(sens)) - if CONF_HEADING in config: - sens = await sensor.new_sensor(config[CONF_HEADING]) - cg.add(var.set_heading_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_FIELD_STRENGTH_X, var.set_x_sensor) + await sensors(CONF_FIELD_STRENGTH_Y, var.set_y_sensor) + await sensors(CONF_FIELD_STRENGTH_Z, var.set_z_sensor) + await sensors(CONF_HEADING, var.set_heading_sensor) if CONF_AUTO_SET_RESET in config: cg.add(var.set_auto_set_reset(config[CONF_AUTO_SET_RESET])) From a706fa9e6b72a2f2bf1a07203b806a686c11fd28 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:58:30 +0200 Subject: [PATCH 082/133] [gdk101] Use the sub_* helpers for optional child entities (#19942) --- esphome/components/gdk101/sensor.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/esphome/components/gdk101/sensor.py b/esphome/components/gdk101/sensor.py index 4ed081a7bef..4fb65d9c256 100644 --- a/esphome/components/gdk101/sensor.py +++ b/esphome/components/gdk101/sensor.py @@ -63,18 +63,8 @@ CONFIG_SCHEMA = cv.Schema( async def to_code(config: ConfigType) -> None: hub = await cg.get_variable(config[CONF_GDK101_ID]) - if radiation_dose_per_1m := config.get(CONF_RADIATION_DOSE_PER_1M): - sens = await sensor.new_sensor(radiation_dose_per_1m) - cg.add(hub.set_rad_1m_sensor(sens)) - - if radiation_dose_per_10m := config.get(CONF_RADIATION_DOSE_PER_10M): - sens = await sensor.new_sensor(radiation_dose_per_10m) - cg.add(hub.set_rad_10m_sensor(sens)) - - if status_config := config.get(CONF_STATUS): - sens = await sensor.new_sensor(status_config) - cg.add(hub.set_status_sensor(sens)) - - if measurement_duration_config := config.get(CONF_MEASUREMENT_DURATION): - sens = await sensor.new_sensor(measurement_duration_config) - cg.add(hub.set_measurement_duration_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_RADIATION_DOSE_PER_1M, hub.set_rad_1m_sensor) + await sensors(CONF_RADIATION_DOSE_PER_10M, hub.set_rad_10m_sensor) + await sensors(CONF_STATUS, hub.set_status_sensor) + await sensors(CONF_MEASUREMENT_DURATION, hub.set_measurement_duration_sensor) From c2d830ebd3e3afc23a2d4978608e45433fbdbb78 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:58:45 +0200 Subject: [PATCH 083/133] [pmwcs3] Use the sub_* helpers for optional child entities (#19943) --- esphome/components/pmwcs3/sensor.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/esphome/components/pmwcs3/sensor.py b/esphome/components/pmwcs3/sensor.py index 19d0083fd04..61f89afa41b 100644 --- a/esphome/components/pmwcs3/sensor.py +++ b/esphome/components/pmwcs3/sensor.py @@ -67,21 +67,11 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if CONF_E25 in config: - sens = await sensor.new_sensor(config[CONF_E25]) - cg.add(var.set_e25_sensor(sens)) - - if CONF_EC in config: - sens = await sensor.new_sensor(config[CONF_EC]) - cg.add(var.set_ec_sensor(sens)) - - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature_sensor(sens)) - - if CONF_VWC in config: - sens = await sensor.new_sensor(config[CONF_VWC]) - cg.add(var.set_vwc_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_E25, var.set_e25_sensor) + await sensors(CONF_EC, var.set_ec_sensor) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_VWC, var.set_vwc_sensor) # Actions From 6121bb3bcfe676dfe830aeca55ed35bc64bfbbac Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 10:59:23 +0200 Subject: [PATCH 084/133] [ufm01] Fix a format warning when logging the frame sync position (#19944) --- esphome/components/ufm01/ufm01.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/ufm01/ufm01.cpp b/esphome/components/ufm01/ufm01.cpp index 880132bad39..fbacd84db0f 100644 --- a/esphome/components/ufm01/ufm01.cpp +++ b/esphome/components/ufm01/ufm01.cpp @@ -261,7 +261,7 @@ bool UFM01Component::process_active_stream_() { } if ((this->read_index_ == 0 && this->data_[0] != FRAME_START_BYTE_1) || (this->read_index_ == 1 && this->data_[1] != FRAME_START_BYTE_2)) { - ESP_LOGD(TAG, "not start of data at %d (is 0x%02X)", this->read_index_, this->data_[this->read_index_]); + ESP_LOGD(TAG, "not start of data at %" PRId32 " (is 0x%02X)", this->read_index_, this->data_[this->read_index_]); this->read_index_ = 0; continue; } From 199cc08270a1e636d1557d875829ae76f3215138 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 11:00:04 +0200 Subject: [PATCH 085/133] [midea] Fix a format warning in the esp32 test config (#19946) --- tests/components/midea/test.esp32-idf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/components/midea/test.esp32-idf.yaml b/tests/components/midea/test.esp32-idf.yaml index 5ad22b5b93c..edaa35691be 100644 --- a/tests/components/midea/test.esp32-idf.yaml +++ b/tests/components/midea/test.esp32-idf.yaml @@ -12,4 +12,4 @@ wifi: interval: - interval: 10s then: - - lambda: ESP_LOGD("test", "%u", millis()); + - lambda: ESP_LOGD("test", "%" PRIu32, millis()); From 920880470730b455b090d62024fcaf828675e218 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 11:10:20 +0200 Subject: [PATCH 086/133] [json] Fix format truncation warnings in the arena tests (#19938) --- tests/components/json/test_json_arena.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/components/json/test_json_arena.cpp b/tests/components/json/test_json_arena.cpp index 8d1d44d1bb0..1d42997225e 100644 --- a/tests/components/json/test_json_arena.cpp +++ b/tests/components/json/test_json_arena.cpp @@ -181,7 +181,7 @@ TEST(JsonArena, StateDocumentsFitWithoutTouchingTheFallback) { root["value"] = static_cast(value_buf); root["state"] = static_cast(value_buf); JsonArray options = root["option"].to(); - char option_bufs[40][34]; + char option_bufs[40][44]; // room for any int, so -Wformat-truncation stays quiet for (int i = 0; i < 40; i++) { snprintf(option_bufs[i], sizeof(option_bufs[i]), "option number %02d padded to twenty", i); options.add(JsonString(option_bufs[i], true)); @@ -199,7 +199,7 @@ TEST(JsonArena, StateDocumentsFitWithoutTouchingTheFallback) { { JsonBuilder builder(&copied_arena); JsonArray options = builder.root()["option"].to(); - char option_bufs[40][34]; + char option_bufs[40][44]; // room for any int, so -Wformat-truncation stays quiet for (int i = 0; i < 40; i++) { snprintf(option_bufs[i], sizeof(option_bufs[i]), "option number %02d padded to twenty", i); options.add(static_cast(option_bufs[i])); @@ -219,7 +219,7 @@ TEST(JsonArena, DocumentMatchesTheHeapAllocator) { arr.add(i); } JsonArray strings = builder.root()["s"].to(); - char buf[32]; + char buf[40]; // room for any int, so -Wformat-truncation stays quiet for (int i = 0; i < 60; i++) { snprintf(buf, sizeof(buf), "string number %04d padded", i); strings.add(buf); From e8c5cd56a8265037c7f6438796727b1addb09119 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 11:28:46 +0200 Subject: [PATCH 087/133] [xdb401] Fix format warnings in the timing and raw pressure logs (#19945) --- esphome/components/xdb401/xdb401.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/esphome/components/xdb401/xdb401.cpp b/esphome/components/xdb401/xdb401.cpp index 3a24d637609..6deab23828d 100644 --- a/esphome/components/xdb401/xdb401.cpp +++ b/esphome/components/xdb401/xdb401.cpp @@ -74,13 +74,13 @@ void XDB401Component::check_measurement_ready_(uint8_t attempt) { // Bit 3 shall be 0 when measurement is ready if ((meas_resp[0] & MASK_MEASURE_READY) == 0) { - ESP_LOGV(TAG, "Meas mode entered after %u ms", attempt * CHECK_DELAY); + ESP_LOGV(TAG, "Meas mode entered after %" PRIu32 " ms", attempt * CHECK_DELAY); this->read_measurement_(); return; } if (attempt >= CHECK_ATTEMPTS) { - ESP_LOGE(TAG, "Device not in measurement mode after timeout of %u ms", CHECK_DELAY * CHECK_ATTEMPTS); + ESP_LOGE(TAG, "Device not in measurement mode after timeout of %" PRIu32 " ms", CHECK_DELAY * CHECK_ATTEMPTS); this->handle_comm_failure_("Measurement timeout"); return; } @@ -129,7 +129,7 @@ i2c::ErrorCode XDB401Component::read_pressure_(float &pressure) { // Sign-extend 24-bit big-endian pressure value to int32_t. int32_t raw_pressure = static_cast(encode_uint24(p_data[0], p_data[1], p_data[2]) << 8) >> 8; - ESP_LOGD(TAG, "Pressure data raw %i", raw_pressure); + ESP_LOGD(TAG, "Pressure data raw %" PRId32, raw_pressure); pressure = (static_cast(raw_pressure) / CONVERT_PRESSURE) * XDB401Component::full_scale_pressure_pa(this->pressure_range_bar_); From 6c432473958fd2e7612e3c8f83e30465b3564f65 Mon Sep 17 00:00:00 2001 From: Panda <735591750@QQ.COM> Date: Wed, 30 Sep 2026 18:41:52 +0800 Subject: [PATCH 088/133] [core] Ignore empty CLI address cache entries (#19924) --- esphome/address_cache.py | 11 ++++++-- tests/unit_tests/test_address_cache.py | 36 ++++++++++++++++++++++++++ tests/unit_tests/test_helpers.py | 20 ++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/esphome/address_cache.py b/esphome/address_cache.py index 4fb3689818b..8d7107b782d 100644 --- a/esphome/address_cache.py +++ b/esphome/address_cache.py @@ -148,6 +148,13 @@ class AddressCache: continue hostname, ips = arg.split("=", 1) # Normalize hostname for consistent lookups - normalized = normalize_hostname(hostname) - cache[normalized] = [ip.strip() for ip in ips.split(",")] + normalized = normalize_hostname(hostname.strip()) + addresses = [ip for value in ips.split(",") if (ip := value.strip())] + if not normalized or not addresses: + _LOGGER.warning( + "Invalid cache entry: %s (hostname and at least one address are required)", + arg, + ) + continue + cache[normalized] = addresses return cache diff --git a/tests/unit_tests/test_address_cache.py b/tests/unit_tests/test_address_cache.py index 1ca28c4f029..71fb632d6df 100644 --- a/tests/unit_tests/test_address_cache.py +++ b/tests/unit_tests/test_address_cache.py @@ -257,6 +257,42 @@ def test_from_cli_args_invalid_format(caplog: LogCaptureFixture) -> None: assert "Invalid cache format: also_invalid" in caplog.text +@pytest.mark.parametrize( + "args,expected", + [ + (["device.local="], {}), + (["device.local= , , "], {}), + (["=192.168.1.10"], {}), + (["...=192.168.1.10"], {}), + ( + [" device.local. = , 192.168.1.10, , fe80::1, "], + {"device.local": ["192.168.1.10", "fe80::1"]}, + ), + ], +) +def test_from_cli_args_discards_empty_addresses_and_hostnames( + args: list[str], expected: dict[str, list[str]], caplog: LogCaptureFixture +) -> None: + """Only usable cache entries may prevent normal hostname resolution.""" + cache = AddressCache.from_cli_args(args, args) + + assert cache.mdns_cache == expected + assert cache.dns_cache == expected + assert cache.has_cache() == bool(expected) + if not expected: + assert cache.get_addresses("device.local") is None + assert "Invalid cache entry" in caplog.text + + +def test_invalid_cache_entry_does_not_replace_valid_entry() -> None: + """A later empty entry must not remove an earlier usable address.""" + args = ["device.local=192.168.1.10", "device.local=", "server.com=10.0.0.1"] + cache = AddressCache.from_cli_args(args, args) + + assert cache.get_mdns_addresses("DEVICE.LOCAL.") == ["192.168.1.10"] + assert cache.get_dns_addresses("server.com") == ["10.0.0.1"] + + def test_from_cli_args_ipv6() -> None: """Test handling of IPv6 addresses.""" mdns_args: list[str] = ["device.local=fe80::1,2001:db8::1"] diff --git a/tests/unit_tests/test_helpers.py b/tests/unit_tests/test_helpers.py index ff82fa3c80f..5bdecf2fd3a 100644 --- a/tests/unit_tests/test_helpers.py +++ b/tests/unit_tests/test_helpers.py @@ -1129,6 +1129,26 @@ def test_resolve_ip_address_cache_miss() -> None: assert result[0][4][0] == "192.168.1.100" +@pytest.mark.parametrize("hostname", ["test.local", "example.com"]) +def test_resolve_ip_address_empty_cache_entry_falls_back(hostname: str) -> None: + """An empty CLI cache entry must use normal DNS or mDNS resolution.""" + cache = AddressCache.from_cli_args([f"{hostname}="], [f"{hostname}="]) + mock_addr_info = AddrInfo( + family=socket.AF_INET, + type=socket.SOCK_STREAM, + proto=socket.IPPROTO_TCP, + sockaddr=IPv4Sockaddr(address="192.168.1.100", port=6053), + ) + + with patch("esphome.resolver.AsyncResolver") as MockResolver: + MockResolver.return_value.resolve.return_value = [mock_addr_info] + + result = helpers.resolve_ip_address(hostname, 6053, address_cache=cache) + + MockResolver.assert_called_once_with([hostname], 6053) + assert result[0][4][0] == "192.168.1.100" + + def test_resolve_ip_address_mixed_cached_uncached() -> None: """Test resolution with mix of cached and uncached hosts.""" cache = AddressCache(mdns_cache={"cached.local": ["192.168.1.50"]}) From 3fac97867ec2aeca4b24f2fb0663614c7a36d31d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 30 Sep 2026 13:17:07 +0200 Subject: [PATCH 089/133] [espidf] Resolve cmake through the IDF environment when reading the CMake cache (#19947) --- esphome/espidf/toolchain.py | 4 +++- tests/unit_tests/test_espidf_toolchain.py | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index d1aeae96031..4243eff0ba2 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -138,7 +138,9 @@ def _get_cmake_output(build_dir) -> str: if not (Path(build_dir) / "CMakeCache.txt").is_file(): raise EsphomeError(f"No ESP-IDF build found in {build_dir}") - cmd = ["cmake", "-LA", "-N", "."] + # Resolve to an absolute path: Windows locates a child process + # through the parent's PATH, not the env passed to it. + cmd = [_get_idf_tool("cmake"), "-LA", "-N", "."] env = _get_idf_env() result = subprocess.run( diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index ee21cd77a9c..c012785e892 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -264,6 +264,7 @@ def test_get_cmake_output_with_configured_build(setup_core: Path) -> None: ) with ( patch.object(toolchain, "_get_idf_env", return_value={}), + patch.object(toolchain, "_get_idf_tool", return_value="/idf/tools/cmake"), patch.object(toolchain.subprocess, "run", return_value=completed) as mock_run, ): assert toolchain._get_cmake_output(build_dir) == completed.stdout @@ -271,6 +272,9 @@ def test_get_cmake_output_with_configured_build(setup_core: Path) -> None: assert toolchain._get_cmake_output(build_dir) == completed.stdout mock_run.assert_called_once() + # The resolved path, never a bare "cmake": Windows locates the child + # through the parent's PATH, where the IDF-managed cmake is missing. + assert mock_run.call_args.args[0][0] == "/idf/tools/cmake" assert toolchain._get_cmake_tool_path("CMAKE_ADDR2LINE") == Path("/tool/addr2line") From 52079dd3f3c3ecfb56b0565bc0e579e35273960c Mon Sep 17 00:00:00 2001 From: Remco van Essen Date: Wed, 30 Sep 2026 15:45:15 +0200 Subject: [PATCH 090/133] [audio_dac] Add a hook that runs once the I2S clocks are running (#19900) --- esphome/components/audio_dac/audio_dac.h | 4 ++++ esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/esphome/components/audio_dac/audio_dac.h b/esphome/components/audio_dac/audio_dac.h index 16a422f4ac7..aebdd85540c 100644 --- a/esphome/components/audio_dac/audio_dac.h +++ b/esphome/components/audio_dac/audio_dac.h @@ -14,6 +14,10 @@ class AudioDac { virtual bool is_muted() = 0; virtual float volume() = 0; + /// Called from the main loop by the speaker driving this DAC once its audio clocks are running. + /// DACs that can only be configured while clocked (e.g. DSP state lost across clock stops) override this. + virtual void on_audio_started() {} + protected: bool is_muted_{false}; }; diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index cb82b09f33a..7ae58b3e440 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -77,6 +77,12 @@ void I2SAudioSpeakerBase::loop() { ESP_LOGV(TAG, "Started"); xEventGroupClearBits(this->event_group_, SpeakerEventGroupBits::TASK_RUNNING); this->state_ = speaker::STATE_RUNNING; +#ifdef USE_AUDIO_DAC + // The task raises TASK_RUNNING only after enabling the channel, so the clocks are already running + if (this->audio_dac_ != nullptr) { + this->audio_dac_->on_audio_started(); + } +#endif // USE_AUDIO_DAC } if (event_group_bits & SpeakerEventGroupBits::TASK_STOPPING) { ESP_LOGV(TAG, "Stopping"); From e742e64fb044fb1c8d919886c466c4a49188aec8 Mon Sep 17 00:00:00 2001 From: Keith Burzinski Date: Wed, 30 Sep 2026 12:33:17 -0500 Subject: [PATCH 091/133] [serial_proxy] Never let a client write stall the main loop past a fixed budget (#19923) --- .../components/serial_proxy/serial_proxy.cpp | 40 +++++++++++++++++++ .../components/serial_proxy/serial_proxy.h | 17 ++++++++ esphome/components/uart/__init__.py | 6 +++ esphome/components/uart/uart_component.h | 5 +++ .../uart/uart_component_esp8266.cpp | 6 +++ .../components/uart/uart_component_esp8266.h | 1 + .../uart/uart_component_esp_idf.cpp | 26 +++++++++--- .../components/uart/uart_component_esp_idf.h | 6 +++ tests/components/uart/test.esp32-idf.yaml | 1 + 9 files changed, 103 insertions(+), 5 deletions(-) diff --git a/esphome/components/serial_proxy/serial_proxy.cpp b/esphome/components/serial_proxy/serial_proxy.cpp index 129745c1c91..729ec40256a 100644 --- a/esphome/components/serial_proxy/serial_proxy.cpp +++ b/esphome/components/serial_proxy/serial_proxy.cpp @@ -2,8 +2,10 @@ #ifdef USE_SERIAL_PROXY +#include "esphome/core/application.h" #include "esphome/core/log.h" +#include #include #include "esphome/core/util.h" @@ -16,6 +18,9 @@ namespace esphome::serial_proxy { static const char *const TAG = "serial_proxy"; +uint32_t SerialProxy::stall_loop_time = 0; +uint32_t SerialProxy::stall_spent_ms = 0; + void SerialProxy::setup() { // Set up modem control pins if configured if (this->rts_pin_ != nullptr) { @@ -293,6 +298,35 @@ void SerialProxy::write_from_client(api::APIConnection *api_connection, const ui #endif if (data == nullptr || len == 0) return; + // Whatever the driver cannot buffer stalls the main loop for its wire time. At high baud + // rates that is brief and losing nothing is worth it; at low ones it would trip the + // watchdog, so cap the stall and drop the rest. The cap covers the whole loop pass: + // several writes can arrive in one, and each alone might stay under it. + const size_t free = this->parent_->available_for_write(); + bool trimmed = false; + if (len > free) { + const uint32_t loop_time = App.get_loop_component_start_time(); + if (loop_time != stall_loop_time) { + stall_loop_time = loop_time; + stall_spent_ms = 0; + } + const uint32_t stall_ms = this->wire_time_ms_(len - free); + trimmed = stall_spent_ms + stall_ms > SERIAL_PROXY_MAX_WRITE_STALL_MS; + if (trimmed && !this->trim_warned_) { + ESP_LOGW(TAG, + "TX buffer full on serial proxy [%" PRIu32 "]: dropping %zu of %zu bytes (would stall %" PRIu32 + " ms at %" PRIu32 " baud); raise the UART tx_buffer_size or pace writes", + this->instance_index_, len - free, len, stall_ms, this->parent_->get_baud_rate()); + } + if (trimmed) { + len = free; + } else { + stall_spent_ms += stall_ms; + } + } + this->trim_warned_ = trimmed; + if (len == 0) + return; this->write_array(data, len); #ifdef USE_SERIAL_PROXY_TAP @@ -303,6 +337,12 @@ void SerialProxy::write_from_client(api::APIConnection *api_connection, const ui #endif } +uint32_t SerialProxy::wire_time_ms_(size_t bytes) const { + const uint32_t bits_per_byte = 1 + this->parent_->get_data_bits() + this->parent_->get_stop_bits() + + (this->parent_->get_parity() != uart::UART_CONFIG_PARITY_NONE ? 1 : 0); + return static_cast(bytes) * bits_per_byte * 1000 / std::max(this->parent_->get_baud_rate(), 1); +} + SerialProxyResult SerialProxy::set_modem_pins(api::APIConnection *api_connection, uint32_t line_states) { #ifdef USE_API if (!this->is_subscriber_(api_connection)) { diff --git a/esphome/components/serial_proxy/serial_proxy.h b/esphome/components/serial_proxy/serial_proxy.h index e3f4264cfa5..df775d4f630 100644 --- a/esphome/components/serial_proxy/serial_proxy.h +++ b/esphome/components/serial_proxy/serial_proxy.h @@ -53,6 +53,11 @@ enum class SerialProxyResult : uint8_t { /// Maximum bytes to read from UART in a single loop iteration inline constexpr size_t SERIAL_PROXY_MAX_READ_SIZE = 256; +/// Longest main-loop stall client writes may cause per loop pass, shared by every instance; +/// bytes the UART cannot buffer within it are dropped. Well under the shortest watchdog +/// timeout, since the API hands the proxies up to ten writes in one pass. +inline constexpr uint32_t SERIAL_PROXY_MAX_WRITE_STALL_MS = 1000; + #ifdef USE_SERIAL_PROXY_TAP /// Observes a port's traffic without owning it, and may inject bytes of its own. /// @@ -204,6 +209,9 @@ class SerialProxy final : public uart::UARTDevice, public Component { bool is_subscriber_(api::APIConnection *api_connection) const { return this->api_connection_ == api_connection; } #endif + /// Time the wire needs for the given number of bytes at the current framing + uint32_t wire_time_ms_(size_t bytes) const; + #ifdef USE_SERIAL_PROXY_TAP /// Return the port to RAW when a subscriber goes away, so the mode never outlives it void reset_mode_(); @@ -221,6 +229,12 @@ class SerialProxy final : public uart::UARTDevice, public Component { /// Instance index for identifying this proxy in API messages uint32_t instance_index_{0}; + /// Stall spent by writes in the current loop pass, keyed by the pass's cached start time. + /// Static on purpose: there is one main loop, and writes to different ports that arrive in + /// the same pass all stall it, so the budget is one per device rather than one per port + static uint32_t stall_loop_time; + static uint32_t stall_spent_ms; + /// Subscribed API client (only one allowed at a time) api::APIConnection *api_connection_{nullptr}; @@ -248,6 +262,9 @@ class SerialProxy final : public uart::UARTDevice, public Component { bool rts_state_{false}; bool dtr_state_{false}; + /// Set while writes are being trimmed, so a client streaming into a slow port warns once + bool trim_warned_{false}; + #ifdef USE_SERIAL_PROXY_TAP SerialProxyTap *tap_{nullptr}; #endif diff --git a/esphome/components/uart/__init__.py b/esphome/components/uart/__init__.py index 419659598cd..598e3df168a 100644 --- a/esphome/components/uart/__init__.py +++ b/esphome/components/uart/__init__.py @@ -46,6 +46,7 @@ from esphome.const import ( CONF_SEQUENCE, CONF_TIMEOUT, CONF_TRIGGER_ID, + CONF_TX_BUFFER_SIZE, CONF_TX_PIN, CONF_UART_ID, PLATFORM_HOST, @@ -297,6 +298,9 @@ CONFIG_SCHEMA = cv.All( ), cv.Optional(CONF_PORT): cv.All(validate_port, cv.only_on(PLATFORM_HOST)), cv.Optional(CONF_RX_BUFFER_SIZE, default=256): cv.validate_bytes, + cv.Optional(CONF_TX_BUFFER_SIZE): cv.All( + cv.only_on_esp32, cv.validate_bytes, cv.int_range(min=129) + ), cv.Optional(CONF_RX_FULL_THRESHOLD): cv.All( cv.only_on_esp32, cv.validate_bytes, cv.int_range(min=1, max=120) ), @@ -391,6 +395,8 @@ async def to_code(config): cg.add(var.set_rx_timeout(config[CONF_RX_TIMEOUT])) if CONF_FLUSH_TIMEOUT in config: cg.add(var.set_flush_timeout(config[CONF_FLUSH_TIMEOUT])) + if (tx_buffer_size := config.get(CONF_TX_BUFFER_SIZE)) is not None: + cg.add(var.set_tx_buffer_size(tx_buffer_size)) # The member already defaults to UART_SCLK_DEFAULT, so only emit a real choice if (clock_source := config.get(CONF_CLOCK_SOURCE, "DEFAULT")) != "DEFAULT": cg.add(var.set_clock_source(UART_CLOCK_SOURCES[clock_source])) diff --git a/esphome/components/uart/uart_component.h b/esphome/components/uart/uart_component.h index 3e525317917..4269ef0b16f 100644 --- a/esphome/components/uart/uart_component.h +++ b/esphome/components/uart/uart_component.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include "esphome/core/defines.h" #include "esphome/core/component.h" @@ -81,6 +82,10 @@ class UARTComponent { // @return Number of available bytes. virtual size_t available() = 0; + // Returns how many bytes write_array() accepts right now without blocking. + // Platforms that cannot tell return SIZE_MAX: write_array() takes everything and may block. + virtual size_t available_for_write() { return SIZE_MAX; } + // Pure virtual method to block until all bytes have been written to the UART bus. // @return UARTFlushResult indicating whether the flush was confirmed, timed out, failed, or assumed successful. virtual UARTFlushResult flush() = 0; diff --git a/esphome/components/uart/uart_component_esp8266.cpp b/esphome/components/uart/uart_component_esp8266.cpp index 1d492fce476..0f6d7f6577c 100644 --- a/esphome/components/uart/uart_component_esp8266.cpp +++ b/esphome/components/uart/uart_component_esp8266.cpp @@ -221,6 +221,12 @@ size_t ESP8266UartComponent::available() { return this->sw_serial_->available(); } } +size_t ESP8266UartComponent::available_for_write() { + if (this->hw_serial_ != nullptr) { + return this->hw_serial_->availableForWrite(); + } + return SIZE_MAX; // software serial bit-bangs each byte synchronously; there is no buffer to fill +} UARTFlushResult ESP8266UartComponent::flush() { ESP_LOGVV(TAG, " Flushing"); if (this->hw_serial_ != nullptr) { diff --git a/esphome/components/uart/uart_component_esp8266.h b/esphome/components/uart/uart_component_esp8266.h index 54bcad39933..0098e752b2d 100644 --- a/esphome/components/uart/uart_component_esp8266.h +++ b/esphome/components/uart/uart_component_esp8266.h @@ -58,6 +58,7 @@ class ESP8266UartComponent final : public UARTComponent, public Component { bool read_array(uint8_t *data, size_t len) override; size_t available() override; + size_t available_for_write() override; UARTFlushResult flush() override; uint32_t get_config(); diff --git a/esphome/components/uart/uart_component_esp_idf.cpp b/esphome/components/uart/uart_component_esp_idf.cpp index a052c6015e7..5d90fc1c8a6 100644 --- a/esphome/components/uart/uart_component_esp_idf.cpp +++ b/esphome/components/uart/uart_component_esp_idf.cpp @@ -7,6 +7,7 @@ #include "esphome/core/log.h" #include "esphome/core/gpio.h" #include "driver/gpio.h" +#include "hal/uart_ll.h" #include "esp_private/gpio.h" #include "soc/gpio_num.h" #include "soc/soc_caps.h" @@ -162,11 +163,11 @@ void IDFUARTComponent::load_settings(bool dump_config) { } err = uart_driver_install(this->uart_num_, // UART number this->rx_buffer_size_, // RX ring buffer size - 0, // TX ring buffer size. If zero, driver will not use a TX buffer and TX function will - // block task until all data has been sent out - 0, // event queue size/depth - nullptr, // event queue - 0 // Flags used to allocate the interrupt + this->tx_buffer_size_, // TX ring buffer size; 0 makes uart_write_bytes() block until + // the FIFO has taken everything + 0, // event queue size/depth + nullptr, // event queue + 0 // Flags used to allocate the interrupt ); if (err != ESP_OK) { ESP_LOGW(TAG, "uart_driver_install failed: %s", esp_err_to_name(err)); @@ -356,6 +357,9 @@ void IDFUARTComponent::dump_config() { " RX Timeout: %u", this->rx_buffer_size_, this->rx_full_threshold_, this->rx_timeout_); } + if (this->tx_buffer_size_ > 0) { + ESP_LOGCONFIG(TAG, " TX Buffer Size: %zu", this->tx_buffer_size_); + } if (this->flush_timeout_ms_ > 0) { ESP_LOGCONFIG(TAG, " Flush Timeout: %" PRIu32 " ms", this->flush_timeout_ms_); } @@ -396,6 +400,18 @@ void IDFUARTComponent::set_rx_timeout(size_t rx_timeout) { this->rx_timeout_ = rx_timeout; } +size_t IDFUARTComponent::available_for_write() { + if (this->uart_num_ == UART_NUM_MAX || !uart_is_driver_installed(this->uart_num_)) + return 0; + if (this->tx_buffer_size_ == 0) { + return uart_ll_get_txfifo_len(UART_LL_GET_HW(this->uart_num_)); + } + // The driver's figure already deducts its ring item headers, so a write of this size fits + size_t free = 0; + uart_get_tx_buffer_free_size(this->uart_num_, &free); + return free; +} + void IDFUARTComponent::write_array(const uint8_t *data, size_t len) { int32_t write_len = uart_write_bytes(this->uart_num_, data, len); if (write_len != (int32_t) len) { diff --git a/esphome/components/uart/uart_component_esp_idf.h b/esphome/components/uart/uart_component_esp_idf.h index 8684937b030..7c93b74fbdd 100644 --- a/esphome/components/uart/uart_component_esp_idf.h +++ b/esphome/components/uart/uart_component_esp_idf.h @@ -33,10 +33,15 @@ class IDFUARTComponent final : public UARTComponent, public Component { bool read_array(uint8_t *data, size_t len) override; size_t available() override; + size_t available_for_write() override; UARTFlushResult flush() override; void set_flush_timeout(uint32_t flush_timeout_ms) override { this->flush_timeout_ms_ = flush_timeout_ms; } + /// TX ring buffer size for the driver; 0 leaves TX unbuffered so write_array() blocks until + /// the FIFO has taken everything. + void set_tx_buffer_size(size_t tx_buffer_size) { this->tx_buffer_size_ = tx_buffer_size; } + void set_clock_source(uart_sclk_t clock_source) { this->clock_source_ = static_cast(clock_source); } uint8_t get_hw_serial_number() { return this->uart_num_; } @@ -109,6 +114,7 @@ class IDFUARTComponent final : public UARTComponent, public Component { uint8_t peek_byte_{0}; uint8_t clock_source_{UART_SCLK_DEFAULT}; ///< uart_sclk_t stored in a byte; the IDF values are all small. uint32_t flush_timeout_ms_{0}; ///< 0 means wait indefinitely (portMAX_DELAY). + size_t tx_buffer_size_{0}; #ifdef USE_UART_WAKE_LOOP_ON_RX // ISR callback for UART RX data notification — wakes the main loop directly. diff --git a/tests/components/uart/test.esp32-idf.yaml b/tests/components/uart/test.esp32-idf.yaml index 333576e1c83..9550de911b7 100644 --- a/tests/components/uart/test.esp32-idf.yaml +++ b/tests/components/uart/test.esp32-idf.yaml @@ -20,6 +20,7 @@ uart: baud_rate: 9600 data_bits: 8 rx_buffer_size: 512 + tx_buffer_size: 512 rx_full_threshold: 10 rx_timeout: 1 parity: EVEN From 1d69ddc09ea90db465890ee31e1a3471aedcdc4a Mon Sep 17 00:00:00 2001 From: Remco van Essen Date: Wed, 30 Sep 2026 20:19:23 +0200 Subject: [PATCH 092/133] [tas58xx] Add TAS58xx audio amplifier (TAS5805M) (#19595) --- CODEOWNERS | 1 + esphome/components/tas58xx/__init__.py | 1 + esphome/components/tas58xx/audio_dac.py | 176 +++++++++ esphome/components/tas58xx/binary_sensor.py | 51 +++ esphome/components/tas58xx/model_tas5805m.cpp | 84 +++++ esphome/components/tas58xx/tas58xx.cpp | 337 ++++++++++++++++++ esphome/components/tas58xx/tas58xx.h | 140 ++++++++ tests/components/tas58xx/common-speaker.yaml | 7 + tests/components/tas58xx/common.yaml | 61 ++++ tests/components/tas58xx/test.esp32-idf.yaml | 5 + .../components/tas58xx/test.esp8266-ard.yaml | 3 + tests/components/tas58xx/test.rp2040-ard.yaml | 3 + .../tas58xx/validate.esp32-idf.yaml | 11 + 13 files changed, 880 insertions(+) create mode 100644 esphome/components/tas58xx/__init__.py create mode 100644 esphome/components/tas58xx/audio_dac.py create mode 100644 esphome/components/tas58xx/binary_sensor.py create mode 100644 esphome/components/tas58xx/model_tas5805m.cpp create mode 100644 esphome/components/tas58xx/tas58xx.cpp create mode 100644 esphome/components/tas58xx/tas58xx.h create mode 100644 tests/components/tas58xx/common-speaker.yaml create mode 100644 tests/components/tas58xx/common.yaml create mode 100644 tests/components/tas58xx/test.esp32-idf.yaml create mode 100644 tests/components/tas58xx/test.esp8266-ard.yaml create mode 100644 tests/components/tas58xx/test.rp2040-ard.yaml create mode 100644 tests/components/tas58xx/validate.esp32-idf.yaml diff --git a/CODEOWNERS b/CODEOWNERS index ff2b6eb3275..e82158f390f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -554,6 +554,7 @@ esphome/components/syslog/* @clydebarrow esphome/components/systa_bus/* @Mat931 esphome/components/t6615/* @tylermenezes esphome/components/tas2780/* @remcom +esphome/components/tas58xx/* @mrtoy-me @remcom esphome/components/tc74/* @sethgirvan esphome/components/tca9548a/* @andreashergert1984 esphome/components/tca9555/* @mobrembski diff --git a/esphome/components/tas58xx/__init__.py b/esphome/components/tas58xx/__init__.py new file mode 100644 index 00000000000..b87036984dc --- /dev/null +++ b/esphome/components/tas58xx/__init__.py @@ -0,0 +1 @@ +CODEOWNERS = ["@mrtoy-me", "@remcom"] diff --git a/esphome/components/tas58xx/audio_dac.py b/esphome/components/tas58xx/audio_dac.py new file mode 100644 index 00000000000..18a60a66455 --- /dev/null +++ b/esphome/components/tas58xx/audio_dac.py @@ -0,0 +1,176 @@ +from dataclasses import dataclass +import logging + +from esphome import automation, pins +from esphome.automation import maybe_simple_id +import esphome.codegen as cg +from esphome.components import i2c +from esphome.components.audio_dac import AudioDac +import esphome.config_validation as cv +from esphome.const import CONF_ENABLE_PIN, CONF_ID, CONF_MODEL +from esphome.cpp_generator import MockObj +from esphome.types import ConfigType + +_LOGGER = logging.getLogger(__name__) + +DEPENDENCIES = ["i2c"] + +CONF_ANALOG_GAIN = "analog_gain" +CONF_DAC_MODE = "dac_mode" +CONF_IGNORE_ENABLE_PIN_WARNING = "ignore_enable_pin_warning" +CONF_MIXER_MODE = "mixer_mode" +CONF_VOLUME_MIN_DB = "volume_min_db" +CONF_VOLUME_MAX_DB = "volume_max_db" +CONF_TAS58XX_ID = "tas58xx_id" + +tas58xx_ns = cg.esphome_ns.namespace("tas58xx") +TAS58xx = tas58xx_ns.class_("TAS58xx", AudioDac, cg.PollingComponent, i2c.I2CDevice) + +DacMode = tas58xx_ns.enum("DacMode") +DAC_MODES = { + "btl": DacMode.DAC_MODE_BTL, + "pbtl": DacMode.DAC_MODE_PBTL, +} + +MixerMode = tas58xx_ns.enum("MixerMode") +MIXER_MODES = { + "stereo": MixerMode.MIXER_MODE_STEREO, + "stereo_inverse": MixerMode.MIXER_MODE_STEREO_INVERSE, + "mono": MixerMode.MIXER_MODE_MONO, + "left": MixerMode.MIXER_MODE_LEFT, + "right": MixerMode.MIXER_MODE_RIGHT, +} + + +@dataclass(frozen=True) +class Model: + """Limits of one model. The C++ side of a model is its ModelInfo constant.""" + + model_info: MockObj + analog_gain_min_db: float + volume_min_db: float + volume_max_db: float + default_address: int + + +MODELS: dict[str, Model] = { + "tas5805m": Model( + model_info=tas58xx_ns.TAS5805M_MODEL, + analog_gain_min_db=-15.5, + volume_min_db=-103.0, + volume_max_db=24.0, + default_address=0x2D, + ), +} + + +def _analog_gain_validator(model: Model): + range_validator = cv.All( + cv.decibel, cv.float_range(min=model.analog_gain_min_db, max=0.0) + ) + + def validator(value: float) -> float: + value = range_validator(value) + if value * 2 != int(value * 2): + raise cv.Invalid("analog_gain must be a multiple of 0.5 dB") + return value + + return validator + + +def _model_schema(model: Model) -> cv.Schema: + volume_validator = cv.All( + cv.decibel, + cv.float_range(min=model.volume_min_db, max=model.volume_max_db), + ) + return ( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(TAS58xx), + cv.Optional(CONF_ENABLE_PIN): pins.gpio_output_pin_schema, + cv.Optional(CONF_IGNORE_ENABLE_PIN_WARNING, default=False): cv.boolean, + cv.Optional( + CONF_ANALOG_GAIN, default=model.analog_gain_min_db + ): _analog_gain_validator(model), + cv.Optional(CONF_DAC_MODE, default="btl"): cv.enum( + DAC_MODES, lower=True + ), + cv.Optional(CONF_MIXER_MODE, default="stereo"): cv.enum( + MIXER_MODES, lower=True + ), + cv.Optional( + CONF_VOLUME_MIN_DB, default=model.volume_min_db + ): volume_validator, + cv.Optional( + CONF_VOLUME_MAX_DB, default=model.volume_max_db + ): volume_validator, + } + ) + .extend(cv.polling_component_schema("1s")) + .extend(i2c.i2c_device_schema(model.default_address)) + ) + + +def _validate_config(config: ConfigType) -> ConfigType: + if config[CONF_VOLUME_MIN_DB] >= config[CONF_VOLUME_MAX_DB]: + raise cv.Invalid(f"{CONF_VOLUME_MIN_DB} must be less than {CONF_VOLUME_MAX_DB}") + if config[CONF_DAC_MODE] == "pbtl" and config[CONF_MIXER_MODE] in ( + "stereo", + "stereo_inverse", + ): + raise cv.Invalid( + f"{CONF_DAC_MODE} 'pbtl' drives a single speaker; use {CONF_MIXER_MODE} 'mono', 'left' or 'right'" + ) + if CONF_ENABLE_PIN in config: + if config[CONF_IGNORE_ENABLE_PIN_WARNING]: + raise cv.Invalid( + f"{CONF_IGNORE_ENABLE_PIN_WARNING} only applies when {CONF_ENABLE_PIN} is not set" + ) + elif not config[CONF_IGNORE_ENABLE_PIN_WARNING]: + # Without PDN high the device does not answer on I2C, and setup only reports an I2C failure + _LOGGER.warning( + "%s: %s not configured - if PDN (power down) is not hardwired high then add %s. " + "Set %s: true to hide this warning", + config[CONF_ID], + CONF_ENABLE_PIN, + CONF_ENABLE_PIN, + CONF_IGNORE_ENABLE_PIN_WARNING, + ) + return config + + +CONFIG_SCHEMA = cv.All( + cv.typed_schema( + {name: _model_schema(model) for name, model in MODELS.items()}, + key=CONF_MODEL, + lower=True, + ), + _validate_config, +) + + +TAS58XX_ACTION_SCHEMA = maybe_simple_id({cv.GenerateID(): cv.use_id(TAS58xx)}) + +for _name, _call in ( + ("tas58xx.activate", "activate()"), + ("tas58xx.deactivate", "deactivate()"), +): + automation.register_apply_action( + _name, TAS58XX_ACTION_SCHEMA, automation.ApplyCall(_call) + ) + + +async def to_code(config: ConfigType) -> None: + model = MODELS[config[CONF_MODEL]] + var = cg.new_Pvariable(config[CONF_ID], cg.RawExpression(f"&{model.model_info}")) + await cg.register_component(var, config) + await i2c.register_i2c_device(var, config) + + cg.add(var.set_analog_gain(config[CONF_ANALOG_GAIN])) + cg.add(var.set_dac_mode(config[CONF_DAC_MODE])) + cg.add(var.set_mixer_mode(config[CONF_MIXER_MODE])) + cg.add(var.set_volume_min_db(config[CONF_VOLUME_MIN_DB])) + cg.add(var.set_volume_max_db(config[CONF_VOLUME_MAX_DB])) + if enable_pin_config := config.get(CONF_ENABLE_PIN): + enable_pin = await cg.gpio_pin_expression(enable_pin_config) + cg.add(var.set_enable_pin(enable_pin)) diff --git a/esphome/components/tas58xx/binary_sensor.py b/esphome/components/tas58xx/binary_sensor.py new file mode 100644 index 00000000000..8a5bb03dd7b --- /dev/null +++ b/esphome/components/tas58xx/binary_sensor.py @@ -0,0 +1,51 @@ +import esphome.codegen as cg +from esphome.components import binary_sensor +import esphome.config_validation as cv +from esphome.const import DEVICE_CLASS_PROBLEM, ENTITY_CATEGORY_DIAGNOSTIC +from esphome.types import ConfigType + +from .audio_dac import CONF_TAS58XX_ID, TAS58xx, tas58xx_ns + +CONF_HAVE_FAULT = "have_fault" + +# Each name matches a FaultSensor value in tas58xx.h +FAULT_SENSORS = ( + "left_channel_dc_fault", + "right_channel_dc_fault", + "left_channel_over_current", + "right_channel_over_current", + "otp_crc_check", + "bq_write_failed", + "clock_fault", + "pvdd_over_voltage", + "pvdd_under_voltage", + "over_temp_shutdown", + "over_temp_warning", +) + +FaultSensor = tas58xx_ns.enum("FaultSensor") + +_FAULT_SCHEMA = binary_sensor.binary_sensor_schema( + device_class=DEVICE_CLASS_PROBLEM, + entity_category=ENTITY_CATEGORY_DIAGNOSTIC, +) + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(CONF_TAS58XX_ID): cv.use_id(TAS58xx), + cv.Optional(CONF_HAVE_FAULT): _FAULT_SCHEMA, + **{cv.Optional(key): _FAULT_SCHEMA for key in FAULT_SENSORS}, + } +) + + +async def to_code(config: ConfigType) -> None: + parent = await cg.get_variable(config[CONF_TAS58XX_ID]) + if sensor_config := config.get(CONF_HAVE_FAULT): + sens = await binary_sensor.new_binary_sensor(sensor_config) + cg.add(parent.set_have_fault_binary_sensor(sens)) + for key in FAULT_SENSORS: + if sensor_config := config.get(key): + sens = await binary_sensor.new_binary_sensor(sensor_config) + fault = getattr(FaultSensor, f"FAULT_SENSOR_{key.upper()}") + cg.add(parent.set_fault_binary_sensor(fault, sens)) diff --git a/esphome/components/tas58xx/model_tas5805m.cpp b/esphome/components/tas58xx/model_tas5805m.cpp new file mode 100644 index 00000000000..e5eb5a837e1 --- /dev/null +++ b/esphome/components/tas58xx/model_tas5805m.cpp @@ -0,0 +1,84 @@ +#include "tas58xx.h" + +#include "esphome/core/hal.h" + +namespace esphome::tas58xx { + +// Remainder of the startup sequence from TI PurePath Console, run after the reset. Register 0x00 selects the page. +// Registers 0x46, 0x7D, 0x7E and page 1 register 0x51 are not documented in the datasheet. +static const uint8_t STARTUP_SEQUENCE[][2] PROGMEM = { + {0x03, 0x00}, // DEVICE_CTRL_2: deep sleep + {0x46, 0x01}, + {0x03, 0x02}, // DEVICE_CTRL_2: Hi-Z + // The I2C address is latched at power up, after which the ADR pin can report faults + {0x61, 0x0B}, // ADR_PIN_CONFIG: FAULTZ + {0x60, 0x01}, // ADR_PIN_CTRL: output + {0x7D, 0x11}, + {0x7E, 0xFF}, + {0x00, 0x01}, + {0x51, 0x05}, + {0x00, 0x00}, +}; + +static const LogString *model_name() { return LOG_STR("TAS5805M"); } + +// An if chain rather than a switch: a switch table would land in rodata, which is RAM on ESP8266. +static const LogString *fault_name(uint8_t index) { + if (index == 0) + return LOG_STR("Right channel over current"); + if (index == 1) + return LOG_STR("Left channel over current"); + if (index == 2) + return LOG_STR("Right channel DC fault"); + if (index == 3) + return LOG_STR("Left channel DC fault"); + if (index == 8) + return LOG_STR("PVDD under voltage"); + if (index == 9) + return LOG_STR("PVDD over voltage"); + if (index == 10) + return LOG_STR("Clock fault"); + if (index == 14) + return LOG_STR("BQ write failed"); + if (index == 15) + return LOG_STR("OTP CRC check error"); + if (index == 16) + return LOG_STR("Over temperature shutdown"); + if (index == 26) + return LOG_STR("Over temperature warning"); + return LOG_STR("Unknown fault"); +} + +const ModelInfo TAS5805M_MODEL = { + .name = model_name, + .startup_sequence = STARTUP_SEQUENCE, + .startup_sequence_length = sizeof(STARTUP_SEQUENCE) / sizeof(STARTUP_SEQUENCE[0]), + .mixer_book = 0x8C, + .mixer_page = 0x29, + .mixer_register = 0x18, + // The clock fault is left out of the log and have_fault: it is set whenever the I2S clock stops, which is normal + .fault_error_mask = 0x0001C30F, + .fault_warning_mask = 0x04000000, + // DC and over current faults keep the output off until cleared (datasheet 7.5.3.3.1, 7.5.3.3.2). They are not + // cleared automatically: a DC fault re-trips only after 570 ms, so a clear on every poll would pass DC to the + // speaker. + .fault_output_off_mask = 0x0000000F, + .fault_latched_mask = 0x0401C70F, + .fault_name = fault_name, + .fault_sensor_bits = + { + 3, // FAULT_SENSOR_LEFT_CHANNEL_DC_FAULT + 2, // FAULT_SENSOR_RIGHT_CHANNEL_DC_FAULT + 1, // FAULT_SENSOR_LEFT_CHANNEL_OVER_CURRENT + 0, // FAULT_SENSOR_RIGHT_CHANNEL_OVER_CURRENT + 15, // FAULT_SENSOR_OTP_CRC_CHECK + 14, // FAULT_SENSOR_BQ_WRITE_FAILED + 10, // FAULT_SENSOR_CLOCK_FAULT + 9, // FAULT_SENSOR_PVDD_OVER_VOLTAGE + 8, // FAULT_SENSOR_PVDD_UNDER_VOLTAGE + 16, // FAULT_SENSOR_OVER_TEMP_SHUTDOWN + 26, // FAULT_SENSOR_OVER_TEMP_WARNING + }, +}; + +} // namespace esphome::tas58xx diff --git a/esphome/components/tas58xx/tas58xx.cpp b/esphome/components/tas58xx/tas58xx.cpp new file mode 100644 index 00000000000..458150a1254 --- /dev/null +++ b/esphome/components/tas58xx/tas58xx.cpp @@ -0,0 +1,337 @@ +#include "tas58xx.h" + +#include + +#include "esphome/core/hal.h" +#include "esphome/core/helpers.h" +#include "esphome/core/log.h" + +namespace esphome::tas58xx { + +static const char *const TAG = "tas58xx"; + +static constexpr uint8_t TAS58XX_PAGE_SELECT = 0x00; // Page Select, in every book +static constexpr uint8_t TAS58XX_BOOK_SELECT = 0x7F; // Book Select, on page 0 of every book +static constexpr uint8_t TAS58XX_BOOK_CONTROL = 0x00; +static constexpr uint8_t TAS58XX_PAGE_0 = 0x00; + +/* BOOK 0x00, PAGE 0x00 */ +static constexpr uint8_t TAS58XX_RESET_CTRL = 0x01; +static constexpr uint8_t TAS58XX_RESET_CTRL_ALL = 0x11; // Reset DSP/control port and registers +static constexpr uint8_t TAS58XX_DEVICE_CTRL_1 = 0x02; +static constexpr uint8_t TAS58XX_DEVICE_CTRL_1_PBTL = (1 << 2); +static constexpr uint8_t TAS58XX_DEVICE_CTRL_2 = 0x03; +static constexpr uint8_t TAS58XX_DEVICE_CTRL_2_MUTE = (1 << 3); +static constexpr uint8_t TAS58XX_CTRL_STATE_MASK = 0x03; // DEVICE_CTRL_2 and POWER_STATE +static constexpr uint8_t TAS58XX_CTRL_STATE_DEEP_SLEEP = 0x00; +static constexpr uint8_t TAS58XX_CTRL_STATE_SLEEP = 0x01; +static constexpr uint8_t TAS58XX_CTRL_STATE_HIZ = 0x02; +static constexpr uint8_t TAS58XX_CTRL_STATE_PLAY = 0x03; +static constexpr uint8_t TAS58XX_CTRL_STATE_UNKNOWN = 0xFF; // Not a device value, forces the next update to act +static constexpr uint8_t TAS58XX_DIG_VOL = 0x4C; // 0x00 = +24 dB to 0xFE = -103 dB in 0.5 dB steps, 0xFF = mute +static constexpr uint8_t TAS58XX_DIG_VOL_0DB = 0x30; +static constexpr uint8_t TAS58XX_DIG_VOL_MINUS_103DB = 0xFE; +static constexpr uint8_t TAS58XX_AGAIN = 0x54; // 0x00 = 0 dB to 0x1F = -15.5 dB in 0.5 dB steps +static constexpr uint8_t TAS58XX_AGAIN_MINUS_15_5DB = 0x1F; +static constexpr uint8_t TAS58XX_POWER_STATE = 0x68; +static constexpr uint8_t TAS58XX_CHAN_FAULT = 0x70; // GLOBAL_FAULT1, GLOBAL_FAULT2 and OT_WARNING follow +static constexpr uint8_t TAS58XX_FAULT_REGISTER_COUNT = 4; +static constexpr uint8_t TAS58XX_FAULT_CLEAR = 0x78; +static constexpr uint8_t TAS58XX_FAULT_CLEAR_ANALOG = 0x80; + +/* Input mixer, four 9.23 fixed point big endian coefficients at a location that differs per model */ +static constexpr uint8_t TAS58XX_MIXER_COEFFICIENT_SIZE = 4; +// Only the second byte differs between mute, -6 dB and 0 dB +static constexpr uint8_t TAS58XX_MIXER_MUTE = 0x00; +static constexpr uint8_t TAS58XX_MIXER_MINUS_6DB = 0x40; +static constexpr uint8_t TAS58XX_MIXER_0DB = 0x80; + +static constexpr uint32_t PDN_LOW_MS = 1; +static constexpr uint32_t PDN_TO_I2C_MS = 5; // Minimum time from PDN high to I2C access +static constexpr uint32_t RESET_SETTLE_MS = 5; + +static const LogString *power_state_name(uint8_t state) { + if (state == TAS58XX_CTRL_STATE_DEEP_SLEEP) + return LOG_STR("Deep sleep"); + if (state == TAS58XX_CTRL_STATE_SLEEP) + return LOG_STR("Sleep"); + if (state == TAS58XX_CTRL_STATE_HIZ) + return LOG_STR("Hi-Z"); + return LOG_STR("Play"); +} + +void TAS58xx::setup() { + if (this->enable_pin_ != nullptr) { + this->enable_pin_->setup(); + this->enable_pin_->digital_write(false); + delay(PDN_LOW_MS); + this->enable_pin_->digital_write(true); + delay(PDN_TO_I2C_MS); + } + if (!this->init_()) { + this->mark_failed(); + } +} + +bool TAS58xx::select_book_page_(uint8_t book, uint8_t page) { + // The book can only be changed from page 0 + return this->write_byte(TAS58XX_PAGE_SELECT, TAS58XX_PAGE_0) && this->write_byte(TAS58XX_BOOK_SELECT, book) && + this->write_byte(TAS58XX_PAGE_SELECT, page); +} + +bool TAS58xx::init_() { + // Header of every TI PurePath Console export: select book 0 in case the MCU restarted without a PDN toggle, + // silence the output with Hi-Z, reset the DSP and the control registers, then return to Hi-Z + if (!this->select_book_page_(TAS58XX_BOOK_CONTROL, TAS58XX_PAGE_0) || + !this->write_byte(TAS58XX_DEVICE_CTRL_2, TAS58XX_CTRL_STATE_HIZ) || + !this->write_byte(TAS58XX_RESET_CTRL, TAS58XX_RESET_CTRL_ALL) || + !this->write_byte(TAS58XX_DEVICE_CTRL_2, TAS58XX_CTRL_STATE_HIZ)) { + ESP_LOGE(TAG, "I2C write failed during reset"); + return false; + } + delay(RESET_SETTLE_MS); + + for (uint8_t i = 0; i < this->model_->startup_sequence_length; i++) { + const uint8_t *entry = this->model_->startup_sequence[i]; + if (!this->write_byte(progmem_read_byte(&entry[0]), progmem_read_byte(&entry[1]))) { + ESP_LOGE(TAG, "I2C write failed during init"); + return false; + } + } + + // Setters are public, so keep the value inside the register range + const uint8_t again = + static_cast(lroundf(clamp(-this->analog_gain_db_ * 2.0f, 0.0f, float{TAS58XX_AGAIN_MINUS_15_5DB}))); + if (!this->write_byte(TAS58XX_DEVICE_CTRL_1, this->dac_mode_ == DAC_MODE_PBTL ? TAS58XX_DEVICE_CTRL_1_PBTL : 0) || + !this->write_byte(TAS58XX_AGAIN, again) || !this->write_volume_() || + !this->write_ctrl_state_(TAS58XX_CTRL_STATE_PLAY, this->is_muted_) || + !this->write_byte(TAS58XX_FAULT_CLEAR, TAS58XX_FAULT_CLEAR_ANALOG)) { + ESP_LOGE(TAG, "I2C write failed during init"); + return false; + } + // The mixer is written once the I2S clocks run, see on_audio_started() and update() + this->power_state_ = TAS58XX_CTRL_STATE_UNKNOWN; + this->mixer_written_ = false; + return true; +} + +void TAS58xx::activate() { + if (this->is_failed()) + return; + ESP_LOGD(TAG, "[0x%02X] Activating", this->address_); + // A failed mixer write can leave the device in another book + if (!this->select_book_page_(TAS58XX_BOOK_CONTROL, TAS58XX_PAGE_0)) { + ESP_LOGE(TAG, "Failed to select the control port"); + return; + } + // Also the way to restart the output after a DC or over current fault + if (!this->write_byte(TAS58XX_FAULT_CLEAR, TAS58XX_FAULT_CLEAR_ANALOG)) { + ESP_LOGW(TAG, "Failed to clear faults"); + } + const bool muted = this->is_muted_; + // Leaving deep sleep needs this sequence to reset the internal state machine, see datasheet 7.4.5 + if (this->ctrl_state_ == TAS58XX_CTRL_STATE_DEEP_SLEEP && + !(this->write_ctrl_state_(TAS58XX_CTRL_STATE_HIZ, muted) && + this->write_ctrl_state_(TAS58XX_CTRL_STATE_DEEP_SLEEP, muted) && + this->write_ctrl_state_(TAS58XX_CTRL_STATE_HIZ, muted))) { + return; + } + this->write_ctrl_state_(TAS58XX_CTRL_STATE_PLAY, muted); +} + +void TAS58xx::deactivate() { + if (this->is_failed()) + return; + ESP_LOGD(TAG, "[0x%02X] Deactivating", this->address_); + if (!this->select_book_page_(TAS58XX_BOOK_CONTROL, TAS58XX_PAGE_0)) { + ESP_LOGE(TAG, "Failed to select the control port"); + return; + } + // The DSP, and with it the mixer, stays active in deep sleep + this->write_ctrl_state_(TAS58XX_CTRL_STATE_DEEP_SLEEP, this->is_muted_); +} + +bool TAS58xx::write_ctrl_state_(uint8_t state, bool muted) { + if (!this->write_byte(TAS58XX_DEVICE_CTRL_2, state | (muted ? TAS58XX_DEVICE_CTRL_2_MUTE : 0))) { + ESP_LOGE(TAG, "Failed to write DEVICE_CTRL_2"); + return false; + } + this->ctrl_state_ = state; + return true; +} + +// Returns false if the fault registers could not be read +bool TAS58xx::read_faults_() { + uint8_t fault_registers[TAS58XX_FAULT_REGISTER_COUNT]; + if (!this->read_bytes(TAS58XX_CHAN_FAULT, fault_registers, sizeof(fault_registers))) + return false; + uint32_t faults = 0; + for (uint8_t reg = 0; reg < TAS58XX_FAULT_REGISTER_COUNT; reg++) + faults |= uint32_t{fault_registers[reg]} << (reg * 8); + const ModelInfo &model = *this->model_; + const uint32_t active = faults & (model.fault_error_mask | model.fault_warning_mask); + + // Clearing makes a lasting condition latch again on every poll, so only log changes + const uint32_t changed = active ^ this->logged_faults_; + for (uint8_t index = 0; index < 32; index++) { + const uint32_t mask = uint32_t{1} << index; + if (!(changed & mask)) + continue; + const LogString *name = model.fault_name(index); + if (!(active & mask)) { + ESP_LOGI(TAG, "[0x%02X] %s cleared", this->address_, LOG_STR_ARG(name)); + } else if (model.fault_error_mask & mask) { + ESP_LOGE(TAG, "[0x%02X] %s", this->address_, LOG_STR_ARG(name)); + } else { + ESP_LOGW(TAG, "[0x%02X] %s", this->address_, LOG_STR_ARG(name)); + } + } + if (changed & active & model.fault_output_off_mask) { + ESP_LOGW(TAG, "[0x%02X] Output stays off: fix the cause, then call tas58xx.activate or power cycle the amplifier", + this->address_); + } + this->logged_faults_ = active; + +#ifdef USE_BINARY_SENSOR + if (this->have_fault_binary_sensor_ != nullptr) + this->have_fault_binary_sensor_->publish_state((active & model.fault_error_mask) != 0); + for (uint8_t fault = 0; fault < FAULT_SENSOR_COUNT; fault++) { + if (this->fault_binary_sensors_[fault] != nullptr) + this->fault_binary_sensors_[fault]->publish_state(faults & (uint32_t{1} << model.fault_sensor_bits[fault])); + } +#endif + + // Latched fault bits stay set after the condition is gone and only report it; clear those so the bits, and the + // sensors, follow the current state. The clear register resets all faults at once, so hold off while the output + // is off. + if ((faults & model.fault_latched_mask) && !(active & model.fault_output_off_mask) && + !this->write_byte(TAS58XX_FAULT_CLEAR, TAS58XX_FAULT_CLEAR_ANALOG)) { + ESP_LOGW(TAG, "Failed to clear faults"); + } + return true; +} + +void TAS58xx::on_audio_started() { + // DSP coefficients can only be written with a running I2S clock (datasheet 7.5.3.1) and are kept when it stops + if (this->is_failed() || this->mixer_written_) + return; + if (!this->write_mixer_()) { + ESP_LOGW(TAG, "Failed to write mixer"); + } +} + +void TAS58xx::update() { + uint8_t power_state; + if (!this->read_faults_() || !this->read_byte(TAS58XX_POWER_STATE, &power_state)) { + this->status_set_warning(LOG_STR("Failed to read status")); + return; + } + this->status_clear_warning(); + + power_state &= TAS58XX_CTRL_STATE_MASK; + if (power_state != this->power_state_) { + ESP_LOGD(TAG, "[0x%02X] Power state: %s", this->address_, LOG_STR_ARG(power_state_name(power_state))); + this->power_state_ = power_state; + } + // The device only plays with a running I2S clock. Fallback for speakers that do not call on_audio_started(); + // a failed write is retried on the next update. + if (power_state == TAS58XX_CTRL_STATE_PLAY && !this->mixer_written_ && !this->write_mixer_()) { + ESP_LOGW(TAG, "Failed to write mixer"); + } +} + +void TAS58xx::dump_config() { + const LogString *mixer_mode; + if (this->mixer_mode_ == MIXER_MODE_STEREO_INVERSE) { + mixer_mode = LOG_STR("Stereo inverse"); + } else if (this->mixer_mode_ == MIXER_MODE_MONO) { + mixer_mode = LOG_STR("Mono"); + } else if (this->mixer_mode_ == MIXER_MODE_LEFT) { + mixer_mode = LOG_STR("Left"); + } else if (this->mixer_mode_ == MIXER_MODE_RIGHT) { + mixer_mode = LOG_STR("Right"); + } else { + mixer_mode = LOG_STR("Stereo"); + } + ESP_LOGCONFIG(TAG, + "Audio Amplifier:\n" + " Model: %s", + LOG_STR_ARG(this->model_->name())); + LOG_I2C_DEVICE(this); + LOG_PIN(" Enable Pin: ", this->enable_pin_); + LOG_UPDATE_INTERVAL(this); + ESP_LOGCONFIG(TAG, + " Analog Gain: %.1f dB\n" + " DAC Mode: %s\n" + " Mixer Mode: %s\n" + " Volume Range: %.1f dB - %.1f dB", + this->analog_gain_db_, + this->dac_mode_ == DAC_MODE_PBTL ? LOG_STR_LITERAL("PBTL") : LOG_STR_LITERAL("BTL"), + LOG_STR_ARG(mixer_mode), this->volume_min_db_, this->volume_max_db_); +#ifdef USE_BINARY_SENSOR + LOG_BINARY_SENSOR(" ", "Any Fault", this->have_fault_binary_sensor_); + for (auto *sensor : this->fault_binary_sensors_) { + LOG_BINARY_SENSOR(" ", "Fault", sensor); + } +#endif +} + +bool TAS58xx::set_mute_(bool muted) { + if (!this->write_ctrl_state_(this->ctrl_state_, muted)) + return false; + this->is_muted_ = muted; + return true; +} + +bool TAS58xx::set_volume(float volume) { + float previous = this->volume_; + this->volume_ = clamp(volume, 0.0f, 1.0f); + if (!this->write_volume_()) { + this->volume_ = previous; + return false; + } + return true; +} + +bool TAS58xx::write_volume_() { + // volume 0.0 maps to volume_min_db_, which is only close to silence at -103 dB + const float volume_db = std::lerp(this->volume_min_db_, this->volume_max_db_, this->volume_); + const uint8_t dig_vol = static_cast( + lroundf(clamp(TAS58XX_DIG_VOL_0DB - volume_db * 2.0f, 0.0f, float{TAS58XX_DIG_VOL_MINUS_103DB}))); + ESP_LOGV(TAG, "Setting volume to 0x%02X", dig_vol); + return this->write_byte(TAS58XX_DIG_VOL, dig_vol); +} + +bool TAS58xx::write_mixer_() { + uint8_t left_to_left = TAS58XX_MIXER_0DB; + uint8_t right_to_left = TAS58XX_MIXER_MUTE; + uint8_t left_to_right = TAS58XX_MIXER_MUTE; + uint8_t right_to_right = TAS58XX_MIXER_0DB; + if (this->mixer_mode_ == MIXER_MODE_STEREO_INVERSE) { + left_to_left = TAS58XX_MIXER_MUTE; + right_to_left = TAS58XX_MIXER_0DB; + left_to_right = TAS58XX_MIXER_0DB; + right_to_right = TAS58XX_MIXER_MUTE; + } else if (this->mixer_mode_ == MIXER_MODE_MONO) { + left_to_left = TAS58XX_MIXER_MINUS_6DB; + right_to_left = TAS58XX_MIXER_MINUS_6DB; + left_to_right = TAS58XX_MIXER_MINUS_6DB; + right_to_right = TAS58XX_MIXER_MINUS_6DB; + } else if (this->mixer_mode_ == MIXER_MODE_LEFT) { + left_to_right = TAS58XX_MIXER_0DB; + right_to_right = TAS58XX_MIXER_MUTE; + } else if (this->mixer_mode_ == MIXER_MODE_RIGHT) { + left_to_left = TAS58XX_MIXER_MUTE; + right_to_left = TAS58XX_MIXER_0DB; + } + const uint8_t coefficients[4 * TAS58XX_MIXER_COEFFICIENT_SIZE] = {0, left_to_left, 0, 0, 0, right_to_left, 0, 0, + 0, left_to_right, 0, 0, 0, right_to_right, 0, 0}; + bool ok = this->select_book_page_(this->model_->mixer_book, this->model_->mixer_page) && + this->write_bytes(this->model_->mixer_register, coefficients, sizeof(coefficients)); + // Always return to the control port, even after a failed write + ok = this->select_book_page_(TAS58XX_BOOK_CONTROL, TAS58XX_PAGE_0) && ok; + this->mixer_written_ = ok; + return ok; +} + +} // namespace esphome::tas58xx diff --git a/esphome/components/tas58xx/tas58xx.h b/esphome/components/tas58xx/tas58xx.h new file mode 100644 index 00000000000..b1e9a531ddc --- /dev/null +++ b/esphome/components/tas58xx/tas58xx.h @@ -0,0 +1,140 @@ +#pragma once + +#include + +#include "esphome/components/audio_dac/audio_dac.h" +#include "esphome/components/i2c/i2c.h" +#include "esphome/core/component.h" +#include "esphome/core/hal.h" +#include "esphome/core/log.h" + +#ifdef USE_BINARY_SENSOR +#include "esphome/components/binary_sensor/binary_sensor.h" +#endif + +namespace esphome::tas58xx { + +enum DacMode : uint8_t { + DAC_MODE_BTL = 0, // Bridge tied load, two speakers + DAC_MODE_PBTL = 1, // Parallel bridge tied load, one speaker +}; + +enum MixerMode : uint8_t { + MIXER_MODE_STEREO = 0, + MIXER_MODE_STEREO_INVERSE, + MIXER_MODE_MONO, + MIXER_MODE_LEFT, + MIXER_MODE_RIGHT, +}; + +/// Fault binary sensors that map to a single fault bit. The Python FAULT_SENSORS list uses the same names. +enum FaultSensor : uint8_t { + FAULT_SENSOR_LEFT_CHANNEL_DC_FAULT = 0, + FAULT_SENSOR_RIGHT_CHANNEL_DC_FAULT, + FAULT_SENSOR_LEFT_CHANNEL_OVER_CURRENT, + FAULT_SENSOR_RIGHT_CHANNEL_OVER_CURRENT, + FAULT_SENSOR_OTP_CRC_CHECK, + FAULT_SENSOR_BQ_WRITE_FAILED, + FAULT_SENSOR_CLOCK_FAULT, + FAULT_SENSOR_PVDD_OVER_VOLTAGE, + FAULT_SENSOR_PVDD_UNDER_VOLTAGE, + FAULT_SENSOR_OVER_TEMP_SHUTDOWN, + FAULT_SENSOR_OVER_TEMP_WARNING, + FAULT_SENSOR_COUNT, +}; + +/// Everything that differs between models of the family. One constant instance exists per model, see +/// model_*.cpp, and each TAS58xx instance points to the one for its model. +/// +/// Fault bits are packed into a 32 bit word: CHAN_FAULT, GLOBAL_FAULT1, GLOBAL_FAULT2 and OT_WARNING, one byte +/// each, from low to high. A bit index is register * 8 + bit. +struct ModelInfo { + const LogString *(*name)(); + /// Remainder of the startup sequence, run after the reset, as {register, value} pairs in PROGMEM + const uint8_t (*startup_sequence)[2]; + uint8_t startup_sequence_length; + /// Location of the four 9.23 fixed point input mixer coefficients: LEFT_TO_LEFT, RIGHT_TO_LEFT, LEFT_TO_RIGHT, + /// RIGHT_TO_RIGHT + uint8_t mixer_book; + uint8_t mixer_page; + uint8_t mixer_register; + /// Faults logged as errors and reported by have_fault + uint32_t fault_error_mask; + /// Faults logged as warnings + uint32_t fault_warning_mask; + /// Faults that keep the output off until cleared by activate() + uint32_t fault_output_off_mask; + /// Faults that stay set after the condition is gone and are cleared after each read + uint32_t fault_latched_mask; + const LogString *(*fault_name)(uint8_t index); + /// Bit index for each FaultSensor + uint8_t fault_sensor_bits[FAULT_SENSOR_COUNT]; +}; + +extern const ModelInfo TAS5805M_MODEL; + +class TAS58xx : public audio_dac::AudioDac, public PollingComponent, public i2c::I2CDevice { + public: + explicit TAS58xx(const ModelInfo *model) : model_(model) {} + + void setup() override; + void dump_config() override; + float get_setup_priority() const override { return setup_priority::IO; } + void update() override; + + /// Clear faults, leave deep sleep and switch to play; the device waits in Hi-Z until an I2S clock is present. + /// This is also how the output is restarted after a DC or over current fault. + void activate(); + /// Switch to deep sleep, the lowest power state that keeps I2C and the DSP running. + void deactivate(); + + bool set_mute_off() override { return this->set_mute_(false); } + bool set_mute_on() override { return this->set_mute_(true); } + bool set_volume(float volume) override; + + bool is_muted() override { return this->is_muted_; } + float volume() override { return this->volume_; } + + void on_audio_started() override; + + void set_enable_pin(GPIOPin *enable_pin) { this->enable_pin_ = enable_pin; } + void set_analog_gain(float analog_gain_db) { this->analog_gain_db_ = analog_gain_db; } + void set_dac_mode(DacMode dac_mode) { this->dac_mode_ = dac_mode; } + void set_mixer_mode(MixerMode mixer_mode) { this->mixer_mode_ = mixer_mode; } + void set_volume_min_db(float volume_min_db) { this->volume_min_db_ = volume_min_db; } + void set_volume_max_db(float volume_max_db) { this->volume_max_db_ = volume_max_db; } + +#ifdef USE_BINARY_SENSOR + SUB_BINARY_SENSOR(have_fault) + void set_fault_binary_sensor(FaultSensor fault, binary_sensor::BinarySensor *sensor) { + this->fault_binary_sensors_[fault] = sensor; + } +#endif + + protected: + bool select_book_page_(uint8_t book, uint8_t page); + bool init_(); + bool write_ctrl_state_(uint8_t state, bool muted); + bool set_mute_(bool muted); + bool write_volume_(); + bool write_mixer_(); + bool read_faults_(); + + const ModelInfo *model_; + GPIOPin *enable_pin_{nullptr}; +#ifdef USE_BINARY_SENSOR + std::array fault_binary_sensors_{}; +#endif + float volume_{0}; + float analog_gain_db_{-15.5f}; + float volume_min_db_{-103.0f}; + float volume_max_db_{24.0f}; + uint32_t logged_faults_{0}; // Fault bits as last logged, packed like the fault masks + DacMode dac_mode_{DAC_MODE_BTL}; + MixerMode mixer_mode_{MIXER_MODE_STEREO}; + uint8_t ctrl_state_{0}; + uint8_t power_state_{0xFF}; // Last POWER_STATE seen by update(), 0xFF until the first read + bool mixer_written_{false}; // Mixer written since the last reset +}; + +} // namespace esphome::tas58xx diff --git a/tests/components/tas58xx/common-speaker.yaml b/tests/components/tas58xx/common-speaker.yaml new file mode 100644 index 00000000000..c8ad45c880e --- /dev/null +++ b/tests/components/tas58xx/common-speaker.yaml @@ -0,0 +1,7 @@ +speaker: + - platform: i2s_audio + id: tas58xx_speaker + i2s_audio_id: i2s_audio_bus + audio_dac: tas58xx_amp + dac_type: external + i2s_dout_pin: GPIO13 diff --git a/tests/components/tas58xx/common.yaml b/tests/components/tas58xx/common.yaml new file mode 100644 index 00000000000..e12ddf75671 --- /dev/null +++ b/tests/components/tas58xx/common.yaml @@ -0,0 +1,61 @@ +audio_dac: + - platform: tas58xx + model: tas5805m + id: tas58xx_amp + i2c_id: i2c_bus + address: 0x2D + enable_pin: GPIO12 + analog_gain: -15.5dB + dac_mode: btl + mixer_mode: stereo + volume_min_db: -60dB + volume_max_db: 0dB + update_interval: 1s + - platform: tas58xx + model: TAS5805M + id: tas58xx_amp_2 + i2c_id: i2c_bus + address: 0x2C + ignore_enable_pin_warning: true + +esphome: + on_boot: + - tas58xx.deactivate: tas58xx_amp + - tas58xx.activate: + id: tas58xx_amp + - tas58xx.activate: tas58xx_amp_2 + +binary_sensor: + - platform: tas58xx + tas58xx_id: tas58xx_amp + have_fault: + name: Any Fault + left_channel_dc_fault: + name: Left Channel DC Fault + right_channel_dc_fault: + name: Right Channel DC Fault + left_channel_over_current: + name: Left Channel Over Current + right_channel_over_current: + name: Right Channel Over Current + otp_crc_check: + name: OTP CRC Check + bq_write_failed: + name: BQ Write Failed + clock_fault: + name: Clock Fault + pvdd_over_voltage: + name: PVDD Over Voltage + pvdd_under_voltage: + name: PVDD Under Voltage + over_temp_shutdown: + name: Over Temperature Shutdown + over_temp_warning: + name: Over Temperature Warning + + - platform: tas58xx + tas58xx_id: tas58xx_amp_2 + have_fault: + name: Second Amp Any Fault + left_channel_dc_fault: + name: Second Amp Left Channel DC Fault diff --git a/tests/components/tas58xx/test.esp32-idf.yaml b/tests/components/tas58xx/test.esp32-idf.yaml new file mode 100644 index 00000000000..a868a2ac03a --- /dev/null +++ b/tests/components/tas58xx/test.esp32-idf.yaml @@ -0,0 +1,5 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/esp32-idf.yaml + i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-idf.yaml + tas58xx: !include common.yaml + tas58xx_speaker: !include common-speaker.yaml diff --git a/tests/components/tas58xx/test.esp8266-ard.yaml b/tests/components/tas58xx/test.esp8266-ard.yaml new file mode 100644 index 00000000000..4f95cb4fd08 --- /dev/null +++ b/tests/components/tas58xx/test.esp8266-ard.yaml @@ -0,0 +1,3 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/esp8266-ard.yaml + tas58xx: !include common.yaml diff --git a/tests/components/tas58xx/test.rp2040-ard.yaml b/tests/components/tas58xx/test.rp2040-ard.yaml new file mode 100644 index 00000000000..5fcf8e93570 --- /dev/null +++ b/tests/components/tas58xx/test.rp2040-ard.yaml @@ -0,0 +1,3 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/rp2040-ard.yaml + tas58xx: !include common.yaml diff --git a/tests/components/tas58xx/validate.esp32-idf.yaml b/tests/components/tas58xx/validate.esp32-idf.yaml new file mode 100644 index 00000000000..9a94137bbb2 --- /dev/null +++ b/tests/components/tas58xx/validate.esp32-idf.yaml @@ -0,0 +1,11 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/esp32-idf.yaml + +audio_dac: + - platform: tas58xx + model: tas5805m + id: tas58xx_pbtl + i2c_id: i2c_bus + dac_mode: PBTL + mixer_mode: mono + analog_gain: -3.5dB From ea5832a410ced951c96b17794ce039a0265e6d42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Sep 2026 22:21:01 +0200 Subject: [PATCH 093/133] Bump pytest-mock from 3.15.1 to 3.16.0 (#19952) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 7f6d75f9470..3ec864a3405 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -8,7 +8,7 @@ yamlrocks==0.6.1 # used by script/sync_dependency_versions.py # Unit tests pytest==9.1.1 pytest-cov==7.1.0 -pytest-mock==3.15.1 +pytest-mock==3.16.0 pytest-asyncio==1.4.0 pytest-xdist==3.8.0 asyncmock==0.4.2 From 35474aae3d3bef713acd0977f7f00ae98b3e7480 Mon Sep 17 00:00:00 2001 From: Nate Clark Date: Wed, 30 Sep 2026 16:24:22 -0400 Subject: [PATCH 094/133] [mqtt] Stop disabling the mbedTLS hardware MPI accelerator for TLS (#19958) Co-authored-by: Claude Opus 5.5 (1M context) --- esphome/components/mqtt/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/esphome/components/mqtt/__init__.py b/esphome/components/mqtt/__init__.py index b6badb4ef96..9418614346f 100644 --- a/esphome/components/mqtt/__init__.py +++ b/esphome/components/mqtt/__init__.py @@ -3,7 +3,6 @@ import esphome.codegen as cg from esphome.components import logger, socket from esphome.components.esp32 import ( add_idf_component, - add_idf_sdkconfig_option, idf_version, include_builtin_idf_component, ) @@ -452,10 +451,6 @@ async def to_code(config): cg.add(var.set_cl_certificate(config[CONF_CLIENT_CERTIFICATE])) cg.add(var.set_cl_key(config[CONF_CLIENT_CERTIFICATE_KEY])) - # prevent error -0x428e - # See https://github.com/espressif/esp-idf/issues/139 - add_idf_sdkconfig_option("CONFIG_MBEDTLS_HARDWARE_MPI", False) - if CONF_IDF_SEND_ASYNC in config and config[CONF_IDF_SEND_ASYNC]: cg.add_define("USE_MQTT_IDF_ENQUEUE") # end esp-idf From cb47dfcf0e44e0ce060f5851f6d42000f37c3970 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Thu, 1 Oct 2026 07:22:41 +1000 Subject: [PATCH 095/133] [light] Fix white flash on turn-on during an effect, start strobe on its first color (#19916) Co-authored-by: Claude Opus 5.5 --- esphome/components/light/base_light_effects.h | 5 + esphome/components/light/light_call.cpp | 33 +++- esphome/components/light/light_call.h | 1 + .../fixtures/light_repeat_effect.yaml | 49 ++++++ tests/integration/test_light_repeat_effect.py | 152 ++++++++++++++++++ 5 files changed, 232 insertions(+), 8 deletions(-) create mode 100644 tests/integration/fixtures/light_repeat_effect.yaml create mode 100644 tests/integration/test_light_repeat_effect.py diff --git a/esphome/components/light/base_light_effects.h b/esphome/components/light/base_light_effects.h index ba3fba6c12d..e32dbde9521 100644 --- a/esphome/components/light/base_light_effects.h +++ b/esphome/components/light/base_light_effects.h @@ -163,6 +163,11 @@ struct StrobeLightEffectColor { class StrobeLightEffect : public LightEffect { public: explicit StrobeLightEffect(const char *name) : LightEffect(name) {} + void start() override { + // Place the cycle at the end of the last color, so the first apply() switches straight to the first color + this->at_color_ = this->colors_.size() - 1; + this->last_switch_ = millis() - this->colors_.back().duration; + } void apply() override { const uint32_t now = millis(); if (now - this->last_switch_ < this->colors_[this->at_color_].duration) diff --git a/esphome/components/light/light_call.cpp b/esphome/components/light/light_call.cpp index f540d2f31f1..7e54fccebca 100644 --- a/esphome/components/light/light_call.cpp +++ b/esphome/components/light/light_call.cpp @@ -156,11 +156,13 @@ void LightCall::perform() { ESP_LOGV(TAG, " Effect: '%.*s'", (int) effect_s.size(), effect_s.c_str()); } - this->parent_->start_effect_(this->effect_); + if (this->effect_ != this->parent_->active_effect_index_) { + this->parent_->start_effect_(this->effect_); - // Also set light color values when starting an effect - // For example to turn off the light - this->parent_->set_immediately_(v, true); + // Also set light color values when starting an effect + // For example to turn off the light + this->parent_->set_immediately_(v, true); + } } else { // INSTANT CHANGE this->parent_->set_immediately_(v, publish); @@ -193,10 +195,9 @@ LightColorValues LightCall::validate_() { auto *name = this->parent_->get_name().c_str(); auto traits = this->parent_->get_traits(); -#ifdef USE_LIGHT_RESUME_EFFECT // Snapshot before the adjustments below add flags of their own + const bool sets_values = (this->flags_ & VALUE_FLAGS_MASK) != 0; const bool plain_turn_on = this->has_state() && this->state_ && (this->flags_ & ~STATE_ONLY_FLAGS_MASK) == 0; -#endif // USE_LIGHT_RESUME_EFFECT // Color mode check if (this->has_color_mode() && !traits.supports_color_mode(this->color_mode_)) { @@ -348,9 +349,25 @@ LightColorValues LightCall::validate_() { } #endif // USE_LIGHT_RESUME_EFFECT - // If effect is already active, remove effect start + // A plain turn-on of a lit light keeps a running effect as it is. + // Effects' own calls don't publish, so they are not caught here. + if (plain_turn_on && this->get_publish_() && this->parent_->remote_values.is_on() && + this->parent_->active_effect_index_ != 0) { + this->effect_ = this->parent_->active_effect_index_; + this->set_flag_(FLAG_HAS_EFFECT); + } + + // If effect is already active, remove effect start. When a lit light that stays on gets no new values or flash, + // keep the flag and drop any transition, which has nothing to fade to; perform() then leaves the running effect + // undisturbed. A call that turns the light off keeps no flag, so the turn-off block below stops the effect. + // has_brightness() catches the brightness added above to make the turn-on visible. if (this->has_effect_() && this->effect_ == this->parent_->active_effect_index_) { - this->clear_flag_(FLAG_HAS_EFFECT); + if (sets_values || this->has_brightness() || this->has_flash_() || this->effect_ == 0 || + !this->parent_->remote_values.is_on() || !v.is_on()) { + this->clear_flag_(FLAG_HAS_EFFECT); + } else { + this->clear_flag_(FLAG_HAS_TRANSITION); + } } // validate effect index diff --git a/esphome/components/light/light_call.h b/esphome/components/light/light_call.h index c9f6af7c916..a226690853e 100644 --- a/esphome/components/light/light_call.h +++ b/esphome/components/light/light_call.h @@ -217,6 +217,7 @@ class LightCall { static constexpr uint16_t CLAMP_FLAGS_MASK = 0x00FFu; // bits 0-7 // Flags a plain turn-on may carry; any other flag means the caller asked for something specific static constexpr uint16_t STATE_ONLY_FLAGS_MASK = FLAG_HAS_STATE | FLAG_PUBLISH | FLAG_SAVE; + static constexpr uint16_t VALUE_FLAGS_MASK = CLAMP_FLAGS_MASK | FLAG_HAS_COLOR_TEMPERATURE | FLAG_HAS_COLOR_MODE; inline bool has_transition_() { return (this->flags_ & FLAG_HAS_TRANSITION) != 0; } inline bool has_flash_() { return (this->flags_ & FLAG_HAS_FLASH) != 0; } diff --git a/tests/integration/fixtures/light_repeat_effect.yaml b/tests/integration/fixtures/light_repeat_effect.yaml new file mode 100644 index 00000000000..85dc346f2c0 --- /dev/null +++ b/tests/integration/fixtures/light_repeat_effect.yaml @@ -0,0 +1,49 @@ +esphome: + name: light-repeat-effect +host: +api: # Port will be automatically injected +logger: + level: DEBUG + +output: + - platform: template + id: red_output + type: float + write_action: + - logger.log: + format: "RED_OUTPUT:%.4f" + args: [state] + - platform: template + id: green_output + type: float + write_action: + - logger.log: + format: "GREEN_OUTPUT:%.4f" + args: [state] + - platform: template + id: blue_output + type: float + write_action: + - logger.log: + format: "BLUE_OUTPUT:%.4f" + args: [state] + +light: + - platform: rgb + name: "Test RGB Light" + id: test_rgb_light + red: red_output + green: green_output + blue: blue_output + effects: + - strobe: + name: "Slow Strobe" + colors: + - red: 100% + green: 0% + blue: 0% + duration: 1s + - red: 0% + green: 0% + blue: 100% + duration: 1s diff --git a/tests/integration/test_light_repeat_effect.py b/tests/integration/test_light_repeat_effect.py new file mode 100644 index 00000000000..e8e4cf8e12d --- /dev/null +++ b/tests/integration/test_light_repeat_effect.py @@ -0,0 +1,152 @@ +"""Integration test for starting a strobe effect and for turn-ons sent while it runs. + +Starting the strobe used to leave the last published colour (white here) on the output until +the first color's duration had passed since boot, then skip to the second color. + +A turn-on naming the effect that is already running, or a plain turn-on of a lit light, used +to get the default transition (or the requested one). That faded the output towards white, which the strobe never +updates, until the next strobe step snapped it back. +""" + +from __future__ import annotations + +import asyncio +import re +from typing import Any + +from aioesphomeapi import EntityState, LightState +import pytest + +from .state_utils import InitialStateHelper +from .types import APIClientConnectedFactory, RunCompiledFunction + + +@pytest.mark.asyncio +async def test_light_repeat_effect( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """The strobe starts on its first color, and turn-ons while it runs don't fade to white.""" + output_pattern = re.compile(r"(RED|GREEN|BLUE)_OUTPUT:([\d.]+)") + outputs: dict[str, list[float]] = {"RED": [], "GREEN": [], "BLUE": []} + green = outputs["GREEN"] + blue = outputs["BLUE"] + warnings: list[str] = [] + + def on_log_line(line: str) -> None: + if match := output_pattern.search(line): + outputs[match.group(1)].append(float(match.group(2))) + elif "effect cannot be used with" in line: + warnings.append(line) + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + entities, _ = await client.list_entities_services() + light = next(e for e in entities if e.object_id == "test_rgb_light") + + state_futures: dict[int, asyncio.Future[LightState]] = {} + + def on_state(state: EntityState) -> None: + if isinstance(state, LightState) and state.key in state_futures: + future = state_futures[state.key] + if not future.done(): + future.set_result(state) + + initial_state_helper = InitialStateHelper(entities) + client.subscribe_states(initial_state_helper.on_state_wrapper(on_state)) + await initial_state_helper.wait_for_initial_states() + + async def send_and_wait(timeout: float = 5.0, **kwargs: Any) -> LightState: + """Send a light command and wait for the matching state response.""" + state_futures[light.key] = asyncio.get_running_loop().create_future() + client.light_command(key=light.key, **kwargs) + return await asyncio.wait_for(state_futures[light.key], timeout=timeout) + + # Plain turn-on leaves the published colour at the white default + state = await send_and_wait(state=True, transition_length=0.0) + assert state.state is True + + green.clear() + blue.clear() + state = await send_and_wait(state=True, effect="Slow Strobe") + assert state.effect == "Slow Strobe" + await asyncio.sleep(0.3) + + # The first color is red; white shows green, and the second color is blue + assert green and blue, "No output observed after starting the strobe" + assert max(green) == pytest.approx(0.0, abs=0.01), ( + f"Starting the strobe showed white: green={green}" + ) + assert max(blue) == pytest.approx(0.0, abs=0.01), ( + f"Starting the strobe skipped its first color: blue={blue}" + ) + + for description, command in ( + ("Repeating the running effect", {"effect": "Slow Strobe"}), + ( + "Repeating the running effect with a transition", + {"effect": "Slow Strobe", "transition_length": 2.0}, + ), + ("A plain turn-on", {}), + ): + green.clear() + state = await send_and_wait(state=True, **command) + assert state.effect == "Slow Strobe", f"{description} changed the effect" + # Cover the rest of the current strobe step and the start of the next one + await asyncio.sleep(1.0) + + assert green, "No green output observed while the strobe was running" + assert max(green) == pytest.approx(0.0, abs=0.01), ( + f"{description} faded the output towards white: {green}" + ) + + assert not warnings, f"Unexpected warnings: {warnings}" + + # A flash requested with the running effect still flashes, here to white + green.clear() + state = await send_and_wait(state=True, effect="Slow Strobe", flash_length=0.5) + assert not warnings, f"Unexpected warnings: {warnings}" + await asyncio.sleep(0.2) + assert max(green) == pytest.approx(1.0, abs=0.01), ( + f"The flash was dropped: green={green}" + ) + await asyncio.sleep(0.5) + + # Brightness 0 with no state turns the light off but leaves the effect + # running; naming that effect must still turn the light on + state = await send_and_wait(brightness=0.0) + assert state.state is False + state = await send_and_wait(state=True, effect="Slow Strobe") + assert state.state is True, "Turn-on with the running effect was dropped" + assert state.brightness == pytest.approx(1.0) + assert state.effect == "Slow Strobe" + + # A lit light at brightness 0 is made visible by a turn-on naming the effect + state = await send_and_wait(state=True, brightness=0.0) + assert state.state is True + assert state.brightness == pytest.approx(0.0) + state = await send_and_wait(state=True, effect="Slow Strobe") + assert state.brightness == pytest.approx(1.0), ( + "Turn-on with the running effect did not make the light visible" + ) + assert state.effect == "Slow Strobe" + + # Turning the light off while naming the running effect must stop the effect, not just + # publish the light as off and leave the effect driving the outputs + warnings.clear() + state = await send_and_wait(state=False, effect="Slow Strobe") + assert state.state is False + # Let the default turn-off transition finish, then watch a full strobe cycle + await asyncio.sleep(1.5) + for values in outputs.values(): + values.clear() + await asyncio.sleep(2.2) + assert max((v for values in outputs.values() for v in values), default=0.0) == ( + pytest.approx(0.0, abs=0.01) + ), f"The effect kept driving the outputs after turn-off: {outputs}" + assert not warnings, f"Unexpected warnings: {warnings}" + + client.light_command(key=light.key, effect="None") From 568c1ae494a00d3e49c5cd3d0b69d01aaf0c6027 Mon Sep 17 00:00:00 2001 From: luar123 <49960470+luar123@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:45:50 +0200 Subject: [PATCH 096/133] [zigbee] Implement switch support for esp32 (#17579) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- esphome/components/zigbee/__init__.py | 92 ++++++++++++------ esphome/components/zigbee/const_esp32.py | 4 + .../zigbee/zigbee_attribute_esp32.h | 22 +++++ esphome/components/zigbee/zigbee_ep_esp32.py | 49 ++++++++++ esphome/components/zigbee/zigbee_esp32.cpp | 57 +++++++++++ esphome/components/zigbee/zigbee_esp32.h | 18 +++- esphome/components/zigbee/zigbee_esp32.py | 20 ++++ .../components/zigbee/zigbee_event_esp32.h | 97 +++++++++++++++++++ .../components/zigbee/zigbee_helpers_esp32.c | 14 +++ esphome/components/zigbee/zigbee_zephyr.py | 6 +- tests/components/zigbee/common_esp32.yaml | 11 +++ 11 files changed, 349 insertions(+), 41 deletions(-) create mode 100644 esphome/components/zigbee/zigbee_event_esp32.h diff --git a/esphome/components/zigbee/__init__.py b/esphome/components/zigbee/__init__.py index e9fef59f353..02f05ef8a2e 100644 --- a/esphome/components/zigbee/__init__.py +++ b/esphome/components/zigbee/__init__.py @@ -43,6 +43,7 @@ from .zigbee_esp32 import ( final_validate_esp32, validate_binary_sensor_esp32, validate_sensor_esp32, + validate_switch_esp32, zigbee_require_vfs_select, ) from .zigbee_zephyr import ( @@ -67,34 +68,58 @@ def _check_report_deprecation(value: str) -> str: return value -BASE_SCHEMA = cv.Schema( - { - cv.Optional(CONF_REPORT): cv.All( - cv.requires_component("zigbee"), - cv.requires_component("esp32"), - _check_report_deprecation, - cv.enum(REPORT, lower=True), - ), - cv.OnlyWith(CONF_CLUSTER, ["esp32", "zigbee"], default="generic"): cv.All( - cv.requires_component("zigbee"), - cv.requires_component("esp32"), - cv.one_of(*["generic", "device_class"], lower=True), - ), - cv.Optional(CONF_ENDPOINT): cv.All( - cv.requires_component("zigbee"), - cv.requires_component("esp32"), - cv.int_range(1, CONF_MAX_EP_NUMBER), - ), - cv.Optional(CONF_USE_DEVICE_TYPE): cv.All( - cv.requires_component("zigbee"), - cv.requires_component("esp32"), - cv.boolean, - ), - } +def _get_base_schema(cluster_options: list[str] | None = None) -> cv.Schema: + schema = cv.Schema( + { + cv.Optional(CONF_REPORT): cv.All( + cv.requires_component("zigbee"), + cv.requires_component("esp32"), + _check_report_deprecation, + cv.enum(REPORT, lower=True), + ), + cv.Optional(CONF_ENDPOINT): cv.All( + cv.requires_component("zigbee"), + cv.requires_component("esp32"), + cv.int_range(1, CONF_MAX_EP_NUMBER), + ), + cv.Optional(CONF_USE_DEVICE_TYPE): cv.All( + cv.requires_component("zigbee"), + cv.requires_component("esp32"), + cv.boolean, + ), + } + ) + if cluster_options: + schema = cv.Schema( + { + cv.OnlyWith( + CONF_CLUSTER, ["esp32", "zigbee"], default=cluster_options[0] + ): cv.All( + cv.requires_component("zigbee"), + cv.requires_component("esp32"), + cv.one_of(*cluster_options, lower=True), + ), + } + ).extend(schema) + return schema + + +# set BASE_SCHEMA for CI and backwards compatibility +BASE_SCHEMA = _get_base_schema() + +BINARY_SENSOR_SCHEMA = ( + cv.Schema({}) + .extend(_get_base_schema(["generic", "device_class"])) + .extend(zephyr_binary_sensor) +) +SENSOR_SCHEMA = ( + cv.Schema({}) + .extend(_get_base_schema(["generic", "device_class"])) + .extend(zephyr_sensor) +) +SWITCH_SCHEMA = ( + cv.Schema({}).extend(_get_base_schema(["generic", "on_off"])).extend(zephyr_switch) ) -BINARY_SENSOR_SCHEMA = cv.Schema({}).extend(BASE_SCHEMA).extend(zephyr_binary_sensor) -SENSOR_SCHEMA = cv.Schema({}).extend(BASE_SCHEMA).extend(zephyr_sensor) -SWITCH_SCHEMA = cv.Schema({}).extend(zephyr_switch) NUMBER_SCHEMA = cv.Schema({}).extend(zephyr_number) @@ -226,12 +251,15 @@ async def setup_sensor(entity: cg.MockObj, config: ConfigType) -> None: async def setup_switch(entity: cg.MockObj, config: ConfigType) -> None: - if not config.get(CONF_ZIGBEE_ID) or config.get(CONF_INTERNAL): + if "zigbee" not in CORE.loaded_integrations or config.get(CONF_INTERNAL): return if CORE.using_zephyr: - from .zigbee_zephyr import zephyr_setup_switch - - await zephyr_setup_switch(entity, config) + if not config.get(CONF_ZIGBEE_ID): + return + from .zigbee_zephyr import add_switch + else: + from .zigbee_esp32 import add_component as add_switch + CORE.add_job(add_switch, entity, config) async def setup_number( @@ -284,7 +312,7 @@ def validate_switch(config: ConfigType) -> ConfigType: if "zigbee" not in CORE.loaded_integrations or config.get(CONF_INTERNAL): return config if CORE.is_esp32: - return config + return validate_switch_esp32(config) return consume_endpoint(config) diff --git a/esphome/components/zigbee/const_esp32.py b/esphome/components/zigbee/const_esp32.py index 32ae3e37348..b8122d14417 100644 --- a/esphome/components/zigbee/const_esp32.py +++ b/esphome/components/zigbee/const_esp32.py @@ -14,7 +14,9 @@ KEY_ZIGBEE_EP_NO_NUM = "zigbee_ep_no_num" KEY_ZIGBEE_FIRST_EP_CL = "zigbee_first_ep_cl" DEVICE_ID = { + "ON_OFF_OUTPUT": cg.RawExpression("EZB_ZHA_ON_OFF_OUTPUT_DEVICE_ID"), "RANGE_EXTENDER": cg.RawExpression("EZB_ZHA_RANGE_EXTENDER_DEVICE_ID"), + "MAINS_POWER_OUTLET": cg.RawExpression("EZB_ZHA_MAINS_POWER_OUTLET_DEVICE_ID"), "SIMPLE_SENSOR": cg.RawExpression("EZB_ZHA_SIMPLE_SENSOR_DEVICE_ID"), "LIGHT_SENSOR": cg.RawExpression("EZB_ZHA_LIGHT_SENSOR_DEVICE_ID"), "OCCUPANCY_SENSOR": cg.RawExpression("EZB_ZHA_OCCUPANCY_SENSOR_DEVICE_ID"), @@ -26,8 +28,10 @@ DEVICE_ID = { cluster_id = cg.esphome_ns.enum("ezb_zcl_cluster_id_e") CLUSTER_ID = { "BASIC": cluster_id.EZB_ZCL_CLUSTER_ID_BASIC, + "ON_OFF": cluster_id.EZB_ZCL_CLUSTER_ID_ON_OFF, "TIME": cluster_id.EZB_ZCL_CLUSTER_ID_TIME, "BINARY_INPUT": cluster_id.EZB_ZCL_CLUSTER_ID_BINARY_INPUT, + "BINARY_OUTPUT": cluster_id.EZB_ZCL_CLUSTER_ID_BINARY_OUTPUT, "ANALOG_INPUT": cluster_id.EZB_ZCL_CLUSTER_ID_ANALOG_INPUT, "ILLUMINANCE_MEASUREMENT": cluster_id.EZB_ZCL_CLUSTER_ID_ILLUMINANCE_MEASUREMENT, "TEMPERATURE_MEASUREMENT": cluster_id.EZB_ZCL_CLUSTER_ID_TEMPERATURE_MEASUREMENT, diff --git a/esphome/components/zigbee/zigbee_attribute_esp32.h b/esphome/components/zigbee/zigbee_attribute_esp32.h index 47f460406c9..2680fc2203e 100644 --- a/esphome/components/zigbee/zigbee_attribute_esp32.h +++ b/esphome/components/zigbee/zigbee_attribute_esp32.h @@ -20,6 +20,9 @@ #ifdef USE_BINARY_SENSOR #include "esphome/components/binary_sensor/binary_sensor.h" #endif +#ifdef USE_SWITCH +#include "esphome/components/switch/switch.h" +#endif namespace esphome::zigbee { @@ -47,12 +50,23 @@ class ZigbeeAttribute final : public Component { template void set_attr(const T &value); uint8_t attr_type() { return attr_type_; } void set_report(ZigbeeReportT report); + + template void add_on_value_callback(F &&callback) { on_value_callback_.add(std::forward(callback)); } + void on_value(ezb_zcl_attribute_t attribute) { + if (attribute.data.type == this->attr_type() && attribute.data.value) { + this->on_value_callback_.call(attribute); + } + } + #ifdef USE_SENSOR template void connect(sensor::Sensor *sensor); template void connect(sensor::Sensor *sensor, F &&f); #endif #ifdef USE_BINARY_SENSOR template void connect(binary_sensor::BinarySensor *sensor); +#endif +#ifdef USE_SWITCH + template void connect(switch_::Switch *device); #endif bool report_enabled = false; @@ -71,6 +85,7 @@ class ZigbeeAttribute final : public Component { bool set_attr_requested_{false}; bool report_requested_{false}; bool force_report_{false}; + LazyCallbackManager on_value_callback_{}; template T scale_value_(float value); template T invalid_value_(); }; @@ -135,6 +150,13 @@ template void ZigbeeAttribute::connect(binary_sensor::BinarySensor * sensor->add_on_state_callback([this](bool value) { this->set_attr((T) (this->scale_ * value)); }); } #endif +#ifdef USE_SWITCH +template void ZigbeeAttribute::connect(switch_::Switch *device) { + this->add_on_value_callback( + [device](ezb_zcl_attribute_t attribute) { device->control(*(T *) attribute.data.value); }); + device->add_on_state_callback([this](bool value) { this->set_attr((T) (this->scale_ * value)); }); +} +#endif } // namespace esphome::zigbee diff --git a/esphome/components/zigbee/zigbee_ep_esp32.py b/esphome/components/zigbee/zigbee_ep_esp32.py index c45f5800459..b0f3d17bd69 100644 --- a/esphome/components/zigbee/zigbee_ep_esp32.py +++ b/esphome/components/zigbee/zigbee_ep_esp32.py @@ -108,6 +108,35 @@ BINARY_INPUT_EP = { ], } +BINARY_OUTPUT_EP = { + CONF_CLUSTERS: [ + { + CONF_ID: "BINARY_OUTPUT", + ROLE: "SERVER", + CONF_ATTRIBUTES: [ + { + CONF_ATTRIBUTE_ID: 0x55, + CONF_TYPE: "BOOL", + CONF_REPORT: cv.enum(REPORT, lower=True)("default"), + CONNECT: True, + }, + { + CONF_ATTRIBUTE_ID: 0x51, + CONF_TYPE: "BOOL", + }, + { + CONF_ATTRIBUTE_ID: 0x6F, + CONF_TYPE: "MAP8", + }, + { + CONF_ATTRIBUTE_ID: 0x1C, + CONF_TYPE: "STRING", + }, + ], + }, + ], +} + def _pressure_ep(device_type: bool = False) -> dict[str, Any]: ep = { @@ -300,6 +329,26 @@ BINARY_SENSOR_EP_CONFIGS: dict[str, dict[str, Any]] = { }, } +SWITCH_EP_CONFIGS: dict[str, dict[str, Any]] = { + "on_off": { + DEVICE_TYPE: "ON_OFF_OUTPUT", + CONF_CLUSTERS: [ + { + CONF_ID: "ON_OFF", + ROLE: "SERVER", + CONF_ATTRIBUTES: [ + { + CONF_ATTRIBUTE_ID: 0x0, + CONF_TYPE: "BOOL", + CONF_REPORT: cv.enum(REPORT, lower=True)("default"), + CONNECT: True, + }, + ], + }, + ], + }, +} + def _get_next_ep_num(eps: list[int]) -> int: try: diff --git a/esphome/components/zigbee/zigbee_esp32.cpp b/esphome/components/zigbee/zigbee_esp32.cpp index c00b13cffd8..c6597ebfd69 100644 --- a/esphome/components/zigbee/zigbee_esp32.cpp +++ b/esphome/components/zigbee/zigbee_esp32.cpp @@ -154,6 +154,31 @@ bool ZigbeeComponent::app_signal_handler(const ezb_app_signal_t *app_signal) { return true; } +void load_zb_event(ZBEvent *event, ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute) { + event->load_set_attr_value_event(info, attribute); +} + +template void enqueue_zb_event(Args... args) { + // Allocate an event from the pool + ZBEvent *event = global_zigbee->zb_event_pool_.allocate(); + if (event == nullptr) { + // No events available - queue is full or we're out of memory + global_zigbee->zb_events_.increment_dropped_count(); + return; + } + + // Load new event data (replaces previous event) + load_zb_event(event, args...); + + // Push the event to the queue + global_zigbee->zb_events_.push(event); + // Push always succeeds because we're the only producer and the pool ensures we never exceed queue size + global_zigbee->enable_loop_soon_any_context(); +} + +// Explicit template instantiations for the friend function +template void enqueue_zb_event(ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute); + static void zb_attribute_handler(ezb_zcl_set_attr_value_message_t *message) { ESP_RETURN_ON_FALSE(message, , TAG, "Empty message"); ESP_RETURN_ON_FALSE(message->info.status == EZB_ZCL_STATUS_SUCCESS, , TAG, "Received message: error status(%d)", @@ -161,6 +186,7 @@ static void zb_attribute_handler(ezb_zcl_set_attr_value_message_t *message) { ESP_LOGD(TAG, "ZCL SetAttributeValue message for endpoint(%d) cluster(0x%04x) %s with status(0x%02x)", message->info.dst_ep, message->info.cluster_id, message->info.cluster_role == EZB_ZCL_CLUSTER_SERVER ? "server" : "client", message->info.status); + enqueue_zb_event(message->info, message->in.attribute); } static void zb_action_handler(ezb_zcl_core_action_callback_id_t callback_id, void *message) { @@ -180,6 +206,13 @@ static void zb_action_handler(ezb_zcl_core_action_callback_id_t callback_id, voi } } +void ZigbeeComponent::handle_attribute_(ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute) { + auto it = this->attributes_.find({info.dst_ep, info.cluster_id, info.cluster_role, attribute.id}); + if (it != this->attributes_.end()) { + it->second->on_value(attribute); + } +} + void ZigbeeComponent::create_default_cluster(uint8_t endpoint_id, uint16_t device_id) { ezb_af_ep_desc_t ep_desc = esphome_zb_zha_default_ep_desc_create(endpoint_id, device_id, this->basic_cluster_data_.power_source); @@ -349,6 +382,30 @@ void ZigbeeComponent::setup() { } void ZigbeeComponent::loop() { + // Process all pending events + ZBEvent *event = this->zb_events_.pop(); + while (event != nullptr) { + // Handle the event + switch (event->callback_id_) { + case EZB_ZCL_CORE_SET_ATTR_VALUE_CB_ID: + this->handle_attribute_(event->event_.set_attr.info, event->event_.set_attr.attribute); + break; + default: + ESP_LOGW(TAG, "Received event with unhandled callback id: 0x%x", static_cast(event->callback_id_)); + break; + } + + // Free the event back to the pool + this->zb_event_pool_.release(event); + // Get the next event + event = this->zb_events_.pop(); + } + // Log dropped events periodically + uint16_t dropped = this->zb_events_.get_and_reset_dropped_count(); + if (dropped > 0) { + ESP_LOGW(TAG, "Dropped %u Zigbee events due to buffer overflow", dropped); + } + if (!this->start_reported_ && this->started_) { this->start_cb_.call(); this->start_reported_ = true; diff --git a/esphome/components/zigbee/zigbee_esp32.h b/esphome/components/zigbee/zigbee_esp32.h index e84ed9817cd..093ae959390 100644 --- a/esphome/components/zigbee/zigbee_esp32.h +++ b/esphome/components/zigbee/zigbee_esp32.h @@ -12,11 +12,10 @@ #include "ezbee/zha.h" #include "esphome/core/automation.h" #include "esphome/core/component.h" +#include "esphome/core/lock_free_queue.h" +#include "esphome/core/event_pool.h" #include "zigbee_helpers_esp32.h" - -#ifdef USE_BINARY_SENSOR -#include "esphome/components/binary_sensor/binary_sensor.h" -#endif +#include "zigbee_event_esp32.h" namespace esphome::zigbee { @@ -24,6 +23,7 @@ namespace esphome::zigbee { static const uint16_t ED_KEEP_ALIVE = 3000; /* 3000 millisecond */ static const uint8_t MAX_CHILDREN = 10; static const uint32_t EZB_PRIMARY_CHANNEL_MASK = 0x07FFF800U; /* channels 11-26 */ +static constexpr uint8_t MAX_ZB_QUEUE_SIZE = 32; #define EZB_DEFAULT_RADIO_CONFIG() \ { .radio_mode = ESP_ZIGBEE_RADIO_MODE_NATIVE, } @@ -88,6 +88,7 @@ class ZigbeeComponent final : public Component { template void add_attr_(ZigbeeAttribute *attr, uint8_t endpoint_id, uint16_t cluster_id, uint8_t role, uint16_t attr_id, T *value_p); + void handle_attribute_(ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute); // attributes_ will be used during operation in zigbee callbacks to update the attribute values and trigger // automations // key tuple could be replaced by single 64 (48) bit int with bit fields for endpoint, cluster, role and attr_id @@ -95,6 +96,15 @@ class ZigbeeComponent final : public Component { std::vector attr_values_; ezb_af_device_desc_t dev_desc_ = ezb_af_create_device_desc(); CallbackManager join_cb_{}; + template friend void enqueue_zb_event(Args... args); + esphome::LockFreeQueue zb_events_; + // Pool sized to queue capacity (SIZE-1) because LockFreeQueue is a ring + // buffer that holds N-1 elements (one slot distinguishes full from empty). + // This guarantees allocate() returns nullptr before push() can fail, which: + // 1. Prevents leaking a pool slot (the Nth allocate succeeds but push fails) + // 2. Avoids needing release() on the producer path after a failed push(), + // preserving the SPSC contract on the pool's internal free list + esphome::EventPool zb_event_pool_; LazyCallbackManager start_cb_{}; bool start_reported_{false}; std::atomic started_ = false; diff --git a/esphome/components/zigbee/zigbee_esp32.py b/esphome/components/zigbee/zigbee_esp32.py index d5e8ebbf5ca..f48bf9ef0be 100644 --- a/esphome/components/zigbee/zigbee_esp32.py +++ b/esphome/components/zigbee/zigbee_esp32.py @@ -27,6 +27,7 @@ from esphome.const import ( CONF_UNIT_OF_MEASUREMENT, CONF_VALUE, CONF_WIFI, + DEVICE_CLASS_OUTLET, ) from esphome.core import CORE from esphome.coroutine import CoroPriority, coroutine_with_priority @@ -67,8 +68,10 @@ from .const_esp32 import ( from .zigbee_ep_esp32 import ( ANALOG_INPUT_EP, BINARY_INPUT_EP, + BINARY_OUTPUT_EP, BINARY_SENSOR_EP_CONFIGS, SENSOR_EP_CONFIGS, + SWITCH_EP_CONFIGS, add_ep, create_ep, ) @@ -275,6 +278,23 @@ def validate_binary_sensor_esp32(config: ConfigType) -> ConfigType: return config +def validate_switch_esp32(config: ConfigType) -> ConfigType: + if config[CONF_CLUSTER] == "on_off": + ep = copy.deepcopy(SWITCH_EP_CONFIGS["on_off"]) + if config.get(CONF_DEVICE_CLASS) == DEVICE_CLASS_OUTLET: + ep[DEVICE_TYPE] = "MAINS_POWER_OUTLET" + else: + if config.get(CONF_DEVICE_CLASS) == DEVICE_CLASS_OUTLET: + _LOGGER.warning( + "'device_class: outlet' has no effect with 'cluster: generic', " + "use 'cluster: on_off' to expose a MAINS_POWER_OUTLET device type" + ) + ep = copy.deepcopy(BINARY_OUTPUT_EP) + setup_attributes(config, ep[CONF_CLUSTERS]) + add_ep(ep, config.get(CONF_ENDPOINT), config.get(CONF_USE_DEVICE_TYPE)) + return config + + def zigbee_require_vfs_select(config: ConfigType) -> ConfigType: """Register VFS select requirement during config validation.""" # Zigbee uses esp_vfs_eventfd which requires VFS select support diff --git a/esphome/components/zigbee/zigbee_event_esp32.h b/esphome/components/zigbee/zigbee_event_esp32.h new file mode 100644 index 00000000000..b8b3e97b4cf --- /dev/null +++ b/esphome/components/zigbee/zigbee_event_esp32.h @@ -0,0 +1,97 @@ +#pragma once + +#include "esphome/core/defines.h" +#ifdef USE_ESP32 +#ifdef USE_ZIGBEE + +#include // for offsetof +#include // for memcpy +#include "esp_zigbee.h" +#include "ezbee/zha.h" + +namespace esphome::zigbee { + +class ZBEvent { + public: + // IMPORTANT: We MUST copy all values because the pointer from ESP-IDF + // is only valid during the callback execution. Since ZB events are processed + // asynchronously in the main loop, we store our own copy inline to ensure + // the data remains valid until the event is processed. + ZBEvent(ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute) + : event_{}, callback_id_(EZB_ZCL_CORE_SET_ATTR_VALUE_CB_ID) { + this->init_set_attr_value_data_(info, attribute); + } + + // Destructor to clean up heap allocations + ~ZBEvent() { this->release(); } + + // Default constructor for pre-allocation in pool + ZBEvent() : event_{}, callback_id_(EZB_ZCL_CORE_CB_ID_END) {} + + // Invoked on return to EventPool - clean up any heap-allocated data + void release() { + // Free any allocated memory within the event + switch (this->callback_id_) { + case EZB_ZCL_CORE_SET_ATTR_VALUE_CB_ID: + if (!this->event_.set_attr.is_inline && this->event_.set_attr.data.heap_data != nullptr) { + delete[] this->event_.set_attr.data.heap_data; + this->event_.set_attr.data.heap_data = nullptr; + } + break; + default: + break; + } + } + + // Load new event data for reuse (replaces previous event data) + // Note: release() is NOT called here because EventPool::release() already + // calls event->release() before returning to the free list. Every event + // from allocate() is already in a clean state. + void load_set_attr_value_event(ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute) { + this->callback_id_ = EZB_ZCL_CORE_SET_ATTR_VALUE_CB_ID; + this->init_set_attr_value_data_(info, attribute); + } + + // Disable copy to prevent double-delete + ZBEvent(const ZBEvent &) = delete; + ZBEvent &operator=(const ZBEvent &) = delete; + + union { + // NOLINTNEXTLINE(readability-identifier-naming) + struct set_attr_event { + ezb_zcl_message_info_t info; + ezb_zcl_attribute_t attribute; + union { + uint8_t *heap_data; + uint8_t inline_data[4]; // For small data types (<= 32 bit) + } data; + bool is_inline; + } set_attr; + } event_; + + ezb_zcl_core_action_callback_id_t callback_id_; + + private: + void init_set_attr_value_data_(ezb_zcl_message_info_t info, ezb_zcl_attribute_t attribute) { + this->event_.set_attr.info = info; + this->event_.set_attr.attribute = attribute; + // get attribute.data.value with correct type + if (attribute.data.value != nullptr) { + // Copy the attribute value to avoid dangling pointer issues + size_t value_size = ezb_zcl_get_attr_value_size(attribute.data.type, attribute.data.value); + if (value_size > 4) { + this->event_.set_attr.data.heap_data = new uint8_t[value_size]; + memcpy(this->event_.set_attr.data.heap_data, attribute.data.value, value_size); + this->event_.set_attr.attribute.data.value = this->event_.set_attr.data.heap_data; + this->event_.set_attr.is_inline = false; + } else { + memcpy(this->event_.set_attr.data.inline_data, attribute.data.value, value_size); + this->event_.set_attr.attribute.data.value = this->event_.set_attr.data.inline_data; + this->event_.set_attr.is_inline = true; + } + } + } +}; +} // namespace esphome::zigbee +#endif // USE_ZIGBEE +#endif // USE_ESP32 diff --git a/esphome/components/zigbee/zigbee_helpers_esp32.c b/esphome/components/zigbee/zigbee_helpers_esp32.c index 53f2c31e8d4..22f014f3ef4 100644 --- a/esphome/components/zigbee/zigbee_helpers_esp32.c +++ b/esphome/components/zigbee/zigbee_helpers_esp32.c @@ -27,6 +27,12 @@ ezb_err_t esphome_zb_add_or_update_cluster(uint16_t cluster_id, ezb_af_ep_desc_t ezb_af_ep_desc_t esphome_zb_zha_default_ep_desc_create(uint8_t ep_id, uint16_t device_id, uint8_t power_source) { ezb_af_ep_desc_t ep_desc; switch (device_id) { + case EZB_ZHA_MAINS_POWER_OUTLET_DEVICE_ID: { + ezb_zha_mains_power_outlet_config_t config = EZB_ZHA_MAINS_POWER_OUTLET_CONFIG(); + config.basic_cfg.power_source = power_source; + ep_desc = ezb_zha_create_mains_power_outlet(ep_id, &config); + break; + } case EZB_ZHA_LIGHT_SENSOR_DEVICE_ID: { ezb_zha_light_sensor_config_t config = EZB_ZHA_LIGHT_SENSOR_CONFIG(); config.basic_cfg.power_source = power_source; @@ -60,12 +66,16 @@ ezb_zcl_cluster_desc_t esphome_zb_default_cluster_dscr_create(uint16_t cluster_i return ezb_zcl_basic_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_IDENTIFY: return ezb_zcl_identify_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_ON_OFF: + return ezb_zcl_on_off_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_TIME: return ezb_zcl_time_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_ANALOG_INPUT: return ezb_zcl_analog_input_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_BINARY_INPUT: return ezb_zcl_binary_input_create_cluster_desc(NULL, role_mask); + case EZB_ZCL_CLUSTER_ID_BINARY_OUTPUT: + return ezb_zcl_binary_output_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_ILLUMINANCE_MEASUREMENT: return ezb_zcl_illuminance_measurement_create_cluster_desc(NULL, role_mask); case EZB_ZCL_CLUSTER_ID_TEMPERATURE_MEASUREMENT: @@ -97,12 +107,16 @@ ezb_err_t esphome_zb_cluster_add_attr(uint16_t cluster_id, ezb_zcl_cluster_desc_ return ezb_zcl_basic_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_IDENTIFY: return ezb_zcl_identify_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_ON_OFF: + return ezb_zcl_on_off_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_TIME: return ezb_zcl_time_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_ANALOG_INPUT: return ezb_zcl_analog_input_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_BINARY_INPUT: return ezb_zcl_binary_input_cluster_desc_add_attr(cluster_desc, attr_id, value_p); + case EZB_ZCL_CLUSTER_ID_BINARY_OUTPUT: + return ezb_zcl_binary_output_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_ILLUMINANCE_MEASUREMENT: return ezb_zcl_illuminance_measurement_cluster_desc_add_attr(cluster_desc, attr_id, value_p); case EZB_ZCL_CLUSTER_ID_TEMPERATURE_MEASUREMENT: diff --git a/esphome/components/zigbee/zigbee_zephyr.py b/esphome/components/zigbee/zigbee_zephyr.py index 588305da300..3f863140e69 100644 --- a/esphome/components/zigbee/zigbee_zephyr.py +++ b/esphome/components/zigbee/zigbee_zephyr.py @@ -314,10 +314,6 @@ async def _ctx_to_code(config: ConfigType) -> None: cg.add(cg.RawExpression("ZB_AF_REGISTER_DEVICE_CTX(&zb_device_ctx)")) -async def zephyr_setup_switch(entity: cg.MockObj, config: ConfigType) -> None: - CORE.add_job(_add_switch, entity, config) - - async def zephyr_setup_number( entity: cg.MockObj, config: ConfigType, @@ -425,7 +421,7 @@ async def add_sensor(entity: cg.MockObj, config: ConfigType) -> None: ) -async def _add_switch(entity: cg.MockObj, config: ConfigType) -> None: +async def add_switch(entity: cg.MockObj, config: ConfigType) -> None: await _add_zigbee_ep( entity, config, diff --git a/tests/components/zigbee/common_esp32.yaml b/tests/components/zigbee/common_esp32.yaml index 706da1f10c1..3d19180c064 100644 --- a/tests/components/zigbee/common_esp32.yaml +++ b/tests/components/zigbee/common_esp32.yaml @@ -45,6 +45,17 @@ sensor: device_class: volume_flow_rate cluster: device_class +switch: + - platform: template + name: "Template Switch 1" + optimistic: true + cluster: on_off + - platform: template + name: "Template Switch 2" + optimistic: true + cluster: on_off + device_class: outlet + zigbee: model: zigbee_test router: false From 6f5d7475f01013080d05ec4fec58603503a19dc7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 00:50:40 +0200 Subject: [PATCH 097/133] [esp8266] Derive the native image header flash size from the linker script (#19961) --- esphome/build_gen/arduino8266.py | 31 +++++++++- esphome/core/config.py | 14 +++-- .../unit_tests/build_gen/test_arduino8266.py | 56 ++++++++++++++++++- tests/unit_tests/core/test_config.py | 13 +++++ 4 files changed, 106 insertions(+), 8 deletions(-) diff --git a/esphome/build_gen/arduino8266.py b/esphome/build_gen/arduino8266.py index fd198b2e0c7..5b14af5a1d7 100644 --- a/esphome/build_gen/arduino8266.py +++ b/esphome/build_gen/arduino8266.py @@ -1065,10 +1065,10 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool: " rspfile_content = $in_newline", " description = LINK $out", "rule elf2bin", - # --flash_size deliberately stays board-derived, as under - # PlatformIO (which reads upload.maximum_size, not the ldscript). + # --flash_size follows PlatformIO: ldscript filename first, then + # upload.maximum_size, so the header always matches the layout. # -W: the framework's own elf2bin.py trips SyntaxWarning on 3.12+. - f" command = $python -W ignore::SyntaxWarning {_q(framework / 'tools' / 'elf2bin.py')} --eboot {_q(framework / 'bootloaders' / 'eboot' / 'eboot.elf')} --app $in --flash_mode {flash_mode} --flash_freq {_FLASH_FREQ_MHZ} --flash_size {_flash_size_str(BOARDS[board][KEY_FLASH_SIZE])} --path {_q(toolchain_bin)} --out $out", + f" command = $python -W ignore::SyntaxWarning {_q(framework / 'tools' / 'elf2bin.py')} --eboot {_q(framework / 'bootloaders' / 'eboot' / 'eboot.elf')} --app $in --flash_mode {flash_mode} --flash_freq {_FLASH_FREQ_MHZ} --flash_size {_elf2bin_flash_size(board, flash_ld_name)} --path {_q(toolchain_bin)} --out $out", " description = BIN $out", "rule copy", " command = $python $buildtool copy $in $out", @@ -1189,3 +1189,28 @@ def _flash_size_str(flash_size: int) -> str: """Flash size argument for elf2bin (e.g. ``4M``, ``512K``).""" mb = 1024 * 1024 return f"{flash_size // mb}M" if flash_size >= mb else f"{flash_size // 1024}K" + + +# Same pattern PlatformIO's _get_flash_size applies to the ldscript path +_LD_FLASH_SIZE_RE = re.compile(r"\.flash\.(\d+[mk]).*\.ld") +# The framework elf2bin.py's --flash_size choices +_ELF2BIN_FLASH_SIZES = frozenset({"256K", "512K", "1M", "2M", "4M", "8M", "16M"}) + + +def _elf2bin_flash_size(board: str, flash_ld_name: str) -> str: + """Image-header flash size as PlatformIO derives it: ldscript filename, + else board_upload.maximum_size, else the board table. The SDK clamps the + chip to the header size at boot, so a header smaller than the linked + layout breaks OTA writes on the running device.""" + if match := _LD_FLASH_SIZE_RE.search(flash_ld_name): + token = match.group(1) + base = 1024 if token[-1] == "k" else 1024 * 1024 + return _flash_size_str(int(token[:-1]) * base) + if max_size := _pio_option("board_upload.maximum_size", ""): + if ( + not max_size.isdigit() + or (size := _flash_size_str(int(max_size))) not in _ELF2BIN_FLASH_SIZES + ): + raise EsphomeError(f"Invalid board_upload.maximum_size value {max_size!r}") + return size + return _flash_size_str(BOARDS[board][KEY_FLASH_SIZE]) diff --git a/esphome/core/config.py b/esphome/core/config.py index bb3f03d3663..281d4304101 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -549,7 +549,9 @@ def _add_library_str(lib: str) -> None: # platformio_options keys the native ESP8266 Arduino generator (a later PR # in this chain) will honor; its ignored-option warning will consume the same # list so the two cannot drift -NATIVE_ARDUINO_PIO_OPTIONS = frozenset({"board_build.f_cpu", "board_build.ldscript"}) +NATIVE_ARDUINO_PIO_OPTIONS = frozenset( + {"board_build.f_cpu", "board_build.ldscript", "board_upload.maximum_size"} +) # The full set that survives into CORE.platformio_options under the native # arduino toolchain: lib_ignore is the only specially-translated key below # that is stored rather than translated away. Consumed by the esp8266 native @@ -609,9 +611,13 @@ async def _add_platformio_options(pio_options: dict[str, str | list[str]]) -> No # through to the ignored-option warning). Other native # toolchains have no equivalent and fall through too. cg.add_platformio_option(key, vals[-1]) - elif key != "upload_speed": - # upload_speed needs no handling: it is read from the raw - # config at upload time (upload_using_esptool) + elif key != "upload_speed" and not ( + key == "board_upload.flash_size" and CORE.using_toolchain_arduino + ): + # upload_speed is read from the raw config at upload time. + # board_upload.flash_size is dropped silently on arduino: + # PlatformIO's esp8266 builder never reads it either, and + # published configs (Athom) commonly set it. _LOGGER.warning( "esphome->platformio_options->%s is ignored when building with " "the native '%s' toolchain", diff --git a/tests/unit_tests/build_gen/test_arduino8266.py b/tests/unit_tests/build_gen/test_arduino8266.py index 4a0355fd010..066746d36d8 100644 --- a/tests/unit_tests/build_gen/test_arduino8266.py +++ b/tests/unit_tests/build_gen/test_arduino8266.py @@ -21,12 +21,18 @@ from esphome.arduino8266.framework import InstalledPaths, toolchain_tool from esphome.build_gen import arduino8266 from esphome.build_gen.arduino8266 import ( _defines_flags, + _elf2bin_flash_size, _flag_defines, _flash_size_str, _resolve_build_config, get_flash_ld_path, ) -from esphome.components.esp8266.boards import BOARDS, ESP8266_BOARD_BUILD +from esphome.components.esp8266.boards import ( + BOARDS, + ESP8266_BOARD_BUILD, + KEY_FLASH_SIZE, + board_ld_script, +) from esphome.components.esp8266.build_surgery import RATETABLE_RULE from esphome.components.esp8266.const import KEY_BOARD, KEY_ESP8266, KEY_SCANF_FLOAT import esphome.config_validation as cv @@ -747,6 +753,54 @@ def test_flash_size_str() -> None: assert _flash_size_str(512 * 1024) == "512K" +def test_elf2bin_flash_size() -> None: + """The image-header size follows the ldscript filename like PlatformIO, + falling back to board_upload.maximum_size and then the board table.""" + assert _elf2bin_flash_size("esp8285", "eagle.flash.2m.ld") == "2M" + assert _elf2bin_flash_size("esp01", "eagle.flash.512k.ld") == "512K" + assert _elf2bin_flash_size("nodemcuv2", "eagle.flash.4m1m.ld") == "4M" + # The testing-mode prefix still matches (search, not match) + assert _elf2bin_flash_size("esp8285", "testing_eagle.flash.2m.ld") == "2M" + # Custom ldscript name: board_upload.maximum_size wins over the board + CORE.platformio_options["board_upload.maximum_size"] = "2097152" + assert _elf2bin_flash_size("esp8285", "custom.ld") == "2M" + del CORE.platformio_options["board_upload.maximum_size"] + assert _elf2bin_flash_size("esp8285", "custom.ld") == "1M" + + +@pytest.mark.parametrize("bad", ["2MB", "3145728", "-1"]) +def test_elf2bin_flash_size_rejects_bad_maximum_size(bad: str) -> None: + """A non-numeric or unsupported board_upload.maximum_size fails by name + instead of a ValueError or a late elf2bin choices error.""" + CORE.platformio_options["board_upload.maximum_size"] = bad + with pytest.raises(EsphomeError, match="board_upload.maximum_size"): + _elf2bin_flash_size("esp8285", "custom.ld") + + +def test_elf2bin_flash_size_default_matches_board_table() -> None: + """Without an ldscript override, every board's own ldscript parses to + the board-table size, so the emitted --flash_size is unchanged.""" + for board, entry in BOARDS.items(): + assert _elf2bin_flash_size(board, board_ld_script(entry)) == _flash_size_str( + entry[KEY_FLASH_SIZE] + ), board + + +def test_write_project_flash_size_follows_ldscript_override( + tmp_path: Path, +) -> None: + """An ldscript overriding the board's flash size drives the image header + too (the Athom shape: esp8285 with eagle.flash.2m.ld). A 1M header over + a 2M layout clamps the chip below the OTA scratch area and bricks OTA.""" + paths = _make_framework(tmp_path) + (paths.framework / "variants" / "esp8285").mkdir() + CORE.data[KEY_ESP8266][KEY_BOARD] = "esp8285" + CORE.platformio_options["board_build.ldscript"] = "eagle.flash.2m.ld" + content = _write_ninja(paths) + assert "--flash_size 2M" in content + assert "eagle.flash.2m.ld" in content + + def test_write_project_testing_mode(tmp_path: Path) -> None: paths = _make_framework(tmp_path) CORE.testing_mode = True diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index 189c024b3a7..cfb9651e6b2 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -1341,6 +1341,8 @@ async def test_add_platformio_options_native_idf( "lib_ignore": "libsodium", "upload_speed": "115200", "board_build.f_flash": "80000000L", + # Silently dropped on arduino only; warns here + "board_upload.flash_size": "2MB", } ) @@ -1351,6 +1353,9 @@ async def test_add_platformio_options_native_idf( # nothing else lands in platformio_options on the native toolchain. assert CORE.platformio_options == {"lib_ignore": ["libsodium"]} assert "esphome->platformio_options->board_build.f_flash is ignored" in caplog.text + assert ( + "esphome->platformio_options->board_upload.flash_size is ignored" in caplog.text + ) assert "upload_speed" not in caplog.text # build_flags has a first-class esphome equivalent, so it is deprecated. # lib_deps/lib_ignore are kept as valid platformio_options (no warning). @@ -1464,13 +1469,21 @@ async def test_add_platformio_options_native_arduino( "board_build.ldscript": ["eagle.flash.2m.ld", "eagle.flash.4m2m.ld"], "board_build.filesystem": "littlefs", "upload_speed": "115200", + # The Athom shape: maximum_size is the elf2bin fallback, + # flash_size is dropped silently (PlatformIO never reads it) + "board_upload.maximum_size": "2097152", + "board_upload.flash_size": "2MB", } ) assert CORE.platformio_options["board_build.f_cpu"] == "160000000L" assert CORE.platformio_options["board_build.ldscript"] == "eagle.flash.4m2m.ld" + assert CORE.platformio_options["board_upload.maximum_size"] == "2097152" + assert "board_upload.flash_size" not in CORE.platformio_options assert "board_build.f_cpu is ignored" not in caplog.text assert "board_build.ldscript is ignored" not in caplog.text + assert "board_upload.maximum_size is ignored" not in caplog.text + assert "board_upload.flash_size is ignored" not in caplog.text assert ( "esphome->platformio_options->board_build.filesystem is ignored" in caplog.text ) From 94ed17815c64d4469a1b801aa7b2d652788d810d Mon Sep 17 00:00:00 2001 From: mtmatteo79 <74114415+mtmatteo79@users.noreply.github.com> Date: Thu, 1 Oct 2026 01:55:39 +0200 Subject: [PATCH 098/133] [bthome_mithermometer] Decode 0.1C temperature and 1% humidity objects (#19956) --- .../bthome_mithermometer/bthome_ble.cpp | 15 +++++ .../sensor/bthome_objects_test.cpp | 66 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 tests/components/bthome_mithermometer/sensor/bthome_objects_test.cpp diff --git a/esphome/components/bthome_mithermometer/bthome_ble.cpp b/esphome/components/bthome_mithermometer/bthome_ble.cpp index 1ebabea0a33..51f5d866a49 100644 --- a/esphome/components/bthome_mithermometer/bthome_ble.cpp +++ b/esphome/components/bthome_mithermometer/bthome_ble.cpp @@ -422,6 +422,21 @@ bool BTHomeMiThermometer::handle_service_data_(const ble_device_base::ServiceDat } break; } + case 0x45: { // temperature (0.1C), e.g. Shelly BLU H&T + if (this->temperature_ != nullptr) { + const int16_t raw = encode_uint16(value[1], value[0]); + this->temperature_->publish_state(raw * 0.1f); + reported = true; + } + break; + } + case 0x2E: { // humidity (uint8, 1%), e.g. Shelly BLU H&T + if (this->humidity_ != nullptr) { + this->humidity_->publish_state(value[0]); + reported = true; + } + break; + } default: break; } diff --git a/tests/components/bthome_mithermometer/sensor/bthome_objects_test.cpp b/tests/components/bthome_mithermometer/sensor/bthome_objects_test.cpp new file mode 100644 index 00000000000..e3903212be8 --- /dev/null +++ b/tests/components/bthome_mithermometer/sensor/bthome_objects_test.cpp @@ -0,0 +1,66 @@ +#include + +#include +#include +#include + +#include "esphome/components/bthome_mithermometer/bthome_ble.h" + +namespace esphome::bthome_mithermometer::testing { + +namespace { + +constexpr uint64_t SENSOR_ADDRESS = 0xA4C1384E1678ULL; + +// Unencrypted BTHome v2 service data (UUID 0xFCD2) sent from SENSOR_ADDRESS. +ble_device_base::ESPBTDevice advert(std::initializer_list service_data) { + std::vector adv = {static_cast(service_data.size() + 3), 0x16, 0xD2, 0xFC}; + adv.insert(adv.end(), service_data.begin(), service_data.end()); + uint8_t mac[6]; + for (size_t i = 0; i < 6; i++) + mac[i] = static_cast(SENSOR_ADDRESS >> (i * 8)); + ble_device_base::ESPBTDevice device; + device.from_scan_result(mac, -60, 0, adv.data(), static_cast(adv.size())); + return device; +} + +struct Harness { + Harness() { + this->thermometer.set_address(SENSOR_ADDRESS); + this->thermometer.set_temperature(&this->temperature); + this->thermometer.set_humidity(&this->humidity); + this->thermometer.set_battery_level(&this->battery_level); + } + + BTHomeMiThermometer thermometer; + sensor::Sensor temperature, humidity, battery_level; +}; + +} // namespace + +// Shelly BLU H&T: packet id, battery 90 %, humidity 55 % (0x2E), temperature 22.5 C (0x45) +TEST(BTHomeMiThermometerObjects, DecodesShellyStyleTemperatureAndHumidity) { + Harness h; + ASSERT_TRUE(h.thermometer.parse_device(advert({0x40, 0x00, 0x01, 0x01, 0x5A, 0x2E, 0x37, 0x45, 0xE1, 0x00}))); + EXPECT_NEAR(h.temperature.state, 22.5f, 0.001f); + EXPECT_FLOAT_EQ(h.humidity.state, 55.0f); + EXPECT_FLOAT_EQ(h.battery_level.state, 90.0f); +} + +TEST(BTHomeMiThermometerObjects, DecodesNegativeTenthDegreeTemperature) { + Harness h; + // -5.3 C = -53 = 0xFFCB + ASSERT_TRUE(h.thermometer.parse_device(advert({0x40, 0x45, 0xCB, 0xFF}))); + EXPECT_NEAR(h.temperature.state, -5.3f, 0.001f); + EXPECT_FALSE(h.humidity.has_state()); +} + +// PVVX firmware: temperature 22.50 C (0x02) and humidity 45.67 % (0x03) keep working +TEST(BTHomeMiThermometerObjects, StillDecodesHundredthDegreeObjects) { + Harness h; + ASSERT_TRUE(h.thermometer.parse_device(advert({0x40, 0x02, 0xCA, 0x08, 0x03, 0xD7, 0x11}))); + EXPECT_NEAR(h.temperature.state, 22.5f, 0.001f); + EXPECT_NEAR(h.humidity.state, 45.67f, 0.001f); +} + +} // namespace esphome::bthome_mithermometer::testing From 5a11f8d7c3b0728e526d0040f6c6ff53b5ea21e2 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 04:58:06 +0000 Subject: [PATCH 099/133] Bump bundled esphome-device-builder to 1.18.0 (#19968) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4c82b8eba9a..e68ea3ee72e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.17.2 +RUN uv pip install --no-cache-dir esphome-device-builder==1.18.0 RUN \ platformio settings set enable_telemetry No \ From 3dc608c7721cdc27e9d6957362b2a111aaf56f9f Mon Sep 17 00:00:00 2001 From: Panda <735591750@QQ.COM> Date: Thu, 1 Oct 2026 19:56:32 +0800 Subject: [PATCH 100/133] [core] Parse ESPHOME_SERIAL_LOGGING_RESET as a boolean (#19963) --- esphome/__main__.py | 4 ++-- tests/unit_tests/test_main.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index 49fc7a3020f..e4213e8edb9 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2343,7 +2343,7 @@ def parse_args(argv): "-r", action="store_true", help="Reset the device before starting serial logs.", - default=os.getenv("ESPHOME_SERIAL_LOGGING_RESET"), + default=get_bool_env("ESPHOME_SERIAL_LOGGING_RESET"), ) _add_states_args(parser_logs) @@ -2384,7 +2384,7 @@ def parse_args(argv): "-r", action="store_true", help="Reset the device before starting serial logs.", - default=os.getenv("ESPHOME_SERIAL_LOGGING_RESET"), + default=get_bool_env("ESPHOME_SERIAL_LOGGING_RESET"), ) parser_run.add_argument( "--ota-platform", diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 837643417d5..633ac1b3e16 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -6437,6 +6437,30 @@ def test_parse_args_logs_states() -> None: assert args.states is True +@pytest.mark.parametrize("command", ["logs", "run"]) +@pytest.mark.parametrize( + ("env_value", "expected"), + [(None, False), ("false", False), ("0", False), ("true", True), ("1", True)], +) +def test_parse_args_serial_logging_reset_env( + monkeypatch: pytest.MonkeyPatch, + command: str, + env_value: str | None, + expected: bool, +) -> None: + """The serial reset environment default must be a boolean for both commands.""" + if env_value is None: + monkeypatch.delenv("ESPHOME_SERIAL_LOGGING_RESET", raising=False) + else: + monkeypatch.setenv("ESPHOME_SERIAL_LOGGING_RESET", env_value) + + args = parse_args(["esphome", command, "device.yaml"]) + assert args.reset is expected + if not expected: + args = parse_args(["esphome", command, "--reset", "device.yaml"]) + assert args.reset is True + + def test_parse_args_argcomplete_only_runs_when_completing() -> None: """Only import and invoke argcomplete when _ARGCOMPLETE is set. From 7e32770c5a927ac73301b8c74503fbdb1ac4c6a6 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 14:37:42 +0200 Subject: [PATCH 101/133] [ci] Pin SOURCE_DATE_EPOCH in the idf.py equivalence check (#19970) --- script/check_idf_py_equivalence.py | 4 ++++ tests/script/test_check_idf_py_equivalence.py | 22 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/script/check_idf_py_equivalence.py b/script/check_idf_py_equivalence.py index fa349e223a4..ff7c58a5a60 100755 --- a/script/check_idf_py_equivalence.py +++ b/script/check_idf_py_equivalence.py @@ -15,6 +15,7 @@ from __future__ import annotations import argparse import hashlib import json +import os from pathlib import Path import re import subprocess @@ -106,6 +107,9 @@ def check(build_path: Path) -> list[str]: (build_path / "build" / "project_description.json").read_text(encoding="utf-8") ) name, version = _setup_core(build_path, description) + # ESP-IDF's openthread stamps the configure time into its compile flags; + # pin it before the env is cached so both configures get the same value. + os.environ["SOURCE_DATE_EPOCH"] = "0" env = toolchain._get_idf_env(version) python = toolchain._get_idf_tool("python") idf_py = toolchain._get_idf_path(version) / "tools" / "idf.py" diff --git a/tests/script/test_check_idf_py_equivalence.py b/tests/script/test_check_idf_py_equivalence.py index f8135772d0c..6ded9a3a815 100644 --- a/tests/script/test_check_idf_py_equivalence.py +++ b/tests/script/test_check_idf_py_equivalence.py @@ -2,6 +2,7 @@ from collections.abc import Callable, Iterator import json +import os from pathlib import Path import subprocess import sys @@ -55,22 +56,32 @@ def _run_check( side_effect: Callable[[list[str]], None] = lambda cmd: None, rc: int = 0, esphome_rcs: tuple[int, int] = (0, 0), + envs: list[dict[str, str]] | None = None, ) -> tuple[list[str], list[list[str]]]: - """Run check() with idf.py replaced by ``side_effect``; return problems, calls.""" + """Run check() with idf.py replaced by ``side_effect``; return problems, calls. + + ``envs`` collects the env each idf.py call receives. + """ calls: list[list[str]] = [] def run(cmd: list[str], **kwargs: object) -> subprocess.CompletedProcess: calls.append(cmd) + if envs is not None: + envs.append(kwargs["env"]) side_effect(cmd) return subprocess.CompletedProcess(cmd, rc, "out\n", "err\n") with ( - patch.object(toolchain, "_get_idf_env", return_value={}), + # Snapshot at call time like the real cached env. + patch.object( + toolchain, "_get_idf_env", side_effect=lambda *_: dict(os.environ) + ), patch.object(toolchain, "_get_idf_tool", return_value="/py"), patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]), patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]), patch.object(guard.subprocess, "run", side_effect=run), + patch.dict(os.environ), ): return guard.check(tree), calls @@ -86,6 +97,13 @@ def test_check_passes_when_idf_py_changes_nothing(tmp_path: Path) -> None: ] +def test_check_pins_source_date_epoch(tmp_path: Path) -> None: + """ESP-IDF's openthread bakes the configure time into its compile flags.""" + envs: list[dict[str, str]] = [] + _run_check(_make_tree(tmp_path), envs=envs) + assert [env.get("SOURCE_DATE_EPOCH") for env in envs] == ["0", "0"] + + def test_check_reports_changed_files_and_rebuilt_outputs(tmp_path: Path) -> None: tree = _make_tree(tmp_path) build = tree / "build" From 5a3e95b161f887ddc7010737955333e788e24bd9 Mon Sep 17 00:00:00 2001 From: tomaszduda23 Date: Thu, 1 Oct 2026 14:41:18 +0200 Subject: [PATCH 102/133] [nrf52] Add nRF Connect SDK 3.4.0 / Zephyr 4.x support (#19863) Co-authored-by: Claude Sonnet 4.6 --- esphome/components/logger/logger_zephyr.cpp | 11 +- esphome/components/nrf52/__init__.py | 34 +++-- esphome/components/nrf52/framework.py | 91 +++++++++--- tests/unit_tests/components/nrf52/test_pch.py | 2 + .../components/nrf52/test_run_compile_hex.py | 130 ++++++++++++++++++ tests/unit_tests/test_nrf52_framework.py | 96 +++++++++++++ tests/unit_tests/test_nrf52_toolchain.py | 16 +++ 7 files changed, 346 insertions(+), 34 deletions(-) create mode 100644 tests/unit_tests/components/nrf52/test_run_compile_hex.py diff --git a/esphome/components/logger/logger_zephyr.cpp b/esphome/components/logger/logger_zephyr.cpp index 69ef6167da5..396728ceaf0 100644 --- a/esphome/components/logger/logger_zephyr.cpp +++ b/esphome/components/logger/logger_zephyr.cpp @@ -20,6 +20,13 @@ __attribute__((weak)) void print_coredump() {} namespace esphome::logger { +// Zephyr 3.7 renamed z_arch_esf_t to struct arch_esf; the old name was later removed. +#if KERNEL_VERSION_NUMBER >= 0x030700 +using FatalErrorEsf = ::arch_esf; +#else +using FatalErrorEsf = z_arch_esf_t; +#endif + __attribute__((section(".noinit"))) struct { uint32_t magic; uint32_t reason; @@ -168,7 +175,7 @@ void Logger::dump_crash_() { } } -void k_sys_fatal_error_handler(unsigned int reason, const z_arch_esf_t *esf) { +void k_sys_fatal_error_handler(unsigned int reason, const FatalErrorEsf *esf) { crash_buf.magic = App.get_config_hash(); crash_buf.reason = reason; if (esf) { @@ -192,7 +199,7 @@ void k_sys_fatal_error_handler(unsigned int reason, const z_arch_esf_t *esf) { extern "C" { -void k_sys_fatal_error_handler(unsigned int reason, const z_arch_esf_t *esf) { +void k_sys_fatal_error_handler(unsigned int reason, const esphome::logger::FatalErrorEsf *esf) { esphome::logger::k_sys_fatal_error_handler(reason, esf); } } diff --git a/esphome/components/nrf52/__init__.py b/esphome/components/nrf52/__init__.py index 8e91cff0497..5ebf5b747f7 100644 --- a/esphome/components/nrf52/__init__.py +++ b/esphome/components/nrf52/__init__.py @@ -438,7 +438,8 @@ async def to_code(config: ConfigType) -> None: zephyr_add_prj_conf("SERIAL", False, False) # disable stuff to make image smaller by default - zephyr_add_prj_conf("NCS_BOOT_BANNER", False, False) + if framework_ver >= cv.Version(2, 9, 2): + zephyr_add_prj_conf("NCS_BOOT_BANNER", False, False) zephyr_add_prj_conf("BOOT_BANNER", False, False) zephyr_add_prj_conf("PRINTK", False, False) zephyr_add_prj_conf("CONSOLE", False, False) @@ -495,8 +496,8 @@ def get_download_types(storage_json: StorageJSON) -> list[dict[str, str]]: types = [] UF2_PATH = "zephyr/zephyr.uf2" DFU_PATH = "firmware.zip" - HEX_PATH = "zephyr/zephyr.hex" # SDK 2.6.1, only generated when OTA is disabled - HEX_MERGED_PATH = "zephyr/merged.hex" # SDK 2.9.2, always generated + HEX_PATH = "zephyr/zephyr.hex" # SDK 2.6.1 without OTA, SDK 3.4.0+ + HEX_MERGED_PATH = "zephyr/merged.hex" # SDK 2.9.2 to 3.3.x, always generated APP_IMAGE_PATH = "zephyr/app_update.bin" build_dir = Path(storage_json.firmware_bin_path).parent if (build_dir / UF2_PATH).is_file(): @@ -968,6 +969,11 @@ def run_compile(args, config: ConfigType) -> bool: _LOGGER.info("Build inputs changed, cleaning %s", build_dir) rmtree(build_dir) + # SDK 3.4.0+ no longer generates merged.hex; drop one left by an older SDK + # build so it is never packaged or offered for download. + for stale_hex in (build_dir / "merged.hex", build_dir / "zephyr" / "merged.hex"): + stale_hex.unlink(missing_ok=True) + if pch_on: pch.log_pch_in_use() # Zephyr turns ccache on by itself when it is installed @@ -1012,13 +1018,18 @@ def run_compile(args, config: ConfigType) -> bool: west_out = zephyr_dir / "zephyr" _copy_if_exists(west_out / "zephyr.uf2", zephyr_dir / "zephyr.uf2") _copy_if_exists(west_out / "zephyr.signed.bin", zephyr_dir / "app_update.bin") + _copy_if_exists(west_out / "zephyr.hex", zephyr_dir / "zephyr.hex") _copy_if_exists(build_dir / "merged.hex", zephyr_dir / "merged.hex") - # For Adafruit bootloader builds, regenerate the UF2 from merged.hex, - # whose records carry the correct flash addresses. The build's own - # zephyr.uf2 uses the board's default offset, which is wrong in some cases. - merged_hex = zephyr_dir / "merged.hex" - if bootloader in _UF2_FAMILY_IDS and merged_hex.is_file(): + # For Adafruit bootloader builds, regenerate the UF2 from a hex file. + # merged.hex carries the correct flash addresses; SDK 3.4.0+ no longer + # generates it, so use zephyr.hex there. Chosen by version so a merged.hex + # left by an older SDK build is never picked. + if framework_ver >= cv.Version(3, 4, 0): + hex_file = zephyr_dir / "zephyr.hex" + else: + hex_file = zephyr_dir / "merged.hex" + if bootloader in _UF2_FAMILY_IDS and hex_file.is_file(): # Drop the build's own wrong-offset UF2 so it isn't shipped alongside. app_uf2 = west_out / "zephyr.uf2" if app_uf2.is_file(): @@ -1035,12 +1046,12 @@ def run_compile(args, config: ConfigType) -> bool: "-c", "-o", str(zephyr_dir / "zephyr.uf2"), - str(merged_hex), + str(hex_file), ], env=env, stream_output=True, ): - raise EsphomeError("Failed to generate UF2 from merged hex") + raise EsphomeError(f"Failed to generate UF2 from {hex_file.name}") if bootloader in ( BOOTLOADER_ADAFRUIT, @@ -1048,9 +1059,6 @@ def run_compile(args, config: ConfigType) -> bool: BOOTLOADER_ADAFRUIT_NRF52_SD140_V6, BOOTLOADER_ADAFRUIT_NRF52_SD140_V7, ): - # no fallback is needed for adafruit case. merged merged.hex is always generated. - # get_download_types needs fallback for mcuboot (non adafruit) - hex_file = zephyr_dir / "merged.hex" dfu_package = build_dir / "firmware.zip" genpkg_cmd = [ str(paths["python_executable"]), diff --git a/esphome/components/nrf52/framework.py b/esphome/components/nrf52/framework.py index 48af47e8ffc..58c87619bf3 100644 --- a/esphome/components/nrf52/framework.py +++ b/esphome/components/nrf52/framework.py @@ -28,6 +28,26 @@ _LOGGER = logging.getLogger(__name__) _REQUIREMENTS = Path(__file__).parent / "requirements.txt" TOOLCHAIN_VERSION = "0.17.4" +# Zephyr SDK used by nRF Connect SDK 3.4.0 and newer. +_TOOLCHAIN_VERSION_NCS_3_4_0 = "1.0.1" +_TOOLCHAIN_VERSIONS = (TOOLCHAIN_VERSION, _TOOLCHAIN_VERSION_NCS_3_4_0) + + +def _uses_sdk_ng_1_toolchain() -> bool: + """True when the framework needs Zephyr SDK 1.0+. + + SDK 1.0 moved the GNU toolchain under gnu/ and renamed its archive + to toolchain_gnu_*. + """ + return CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] >= cv.Version(3, 4, 0) + + +def _get_toolchain_version() -> str: + """Return the Zephyr SDK toolchain version for the current framework.""" + if _uses_sdk_ng_1_toolchain(): + return _TOOLCHAIN_VERSION_NCS_3_4_0 + return TOOLCHAIN_VERSION + # Packages the PlatformIO toolchain's Zephyr build script needs beyond west # (which comes from requirements.txt). Keep the pin in sync with @@ -40,6 +60,20 @@ SDK_NG_TOOLCHAIN_MIRRORS = str_to_lst_of_str( "https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v{VERSION}/toolchain_{sysname}-{machine}_arm-zephyr-eabi.{extension}", ) ) +_SDK_NG_TOOLCHAIN_GNU_MIRRORS = str_to_lst_of_str( + os.environ.get( + "ESPHOME_SDK_NG_TOOLCHAIN_GNU_MIRRORS", + "https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v{VERSION}/toolchain_gnu_{sysname}-{machine}_arm-zephyr-eabi.{extension}", + ) +) + + +def _get_sdk_ng_toolchain_mirrors() -> list[str]: + """Return toolchain mirror URLs for the current framework version.""" + if _uses_sdk_ng_1_toolchain(): + return _SDK_NG_TOOLCHAIN_GNU_MIRRORS + return SDK_NG_TOOLCHAIN_MIRRORS + # Minimal SDK provides cmake discovery files (Zephyr-sdkConfig.cmake) and # host tools (dtc etc.) required by the Zephyr cmake build system. @@ -85,15 +119,22 @@ def _get_toolchain_path(version: str) -> Path: return get_sdk_nrf_tools_path() / "toolchains" / version -def toolchain_tool(name: str) -> Path: - """Path to one of the pinned Zephyr SDK's tools (objdump, readelf, ...). +def _get_arm_toolchain_path() -> Path: + """The arm-zephyr-eabi directory inside the pinned Zephyr SDK. - The single owner of the ``arm-zephyr-eabi/bin/arm-zephyr-eabi-`` - layout and the Windows suffix. + The single owner of the SDK 0.x (arm-zephyr-eabi/) and SDK 1.0+ + (gnu/arm-zephyr-eabi/) layouts. """ + toolchain_root = _get_toolchain_path(_get_toolchain_version()) + if _uses_sdk_ng_1_toolchain(): + return toolchain_root / "gnu" / "arm-zephyr-eabi" + return toolchain_root / "arm-zephyr-eabi" + + +def toolchain_tool(name: str) -> Path: + """Path to one of the pinned Zephyr SDK's tools (objdump, readelf, ...).""" suffix = ".exe" if os.name == "nt" else "" - bin_path = _get_toolchain_path(TOOLCHAIN_VERSION) / "arm-zephyr-eabi" / "bin" - return bin_path / f"arm-zephyr-eabi-{name}{suffix}" + return _get_arm_toolchain_path() / "bin" / f"arm-zephyr-eabi-{name}{suffix}" _SITECUSTOMIZE = """\ @@ -169,7 +210,7 @@ def get_build_env() -> dict: # "Zephyr-sdk_DIR" environment hint proved unreliable here: containerized # non-root builds failed to locate the SDK with it, while # ZEPHYR_SDK_INSTALL_DIR fixed the same invocation. - env["ZEPHYR_SDK_INSTALL_DIR"] = str(_get_toolchain_path(TOOLCHAIN_VERSION)) + env["ZEPHYR_SDK_INSTALL_DIR"] = str(_get_toolchain_path(_get_toolchain_version())) return env @@ -561,36 +602,41 @@ def _check_and_install(version: str) -> None: raise EsphomeError(f"Install Zephyr requirements for {version} failure") zephyr_sentinel.touch() - # Shared by every SDK version; locked only while missing - if not (_get_toolchain_path(TOOLCHAIN_VERSION) / ".ready").exists(): - with _install_lock(f"toolchain-{TOOLCHAIN_VERSION}"): + # Shared by every SDK version that uses the same toolchain; locked only + # while missing + toolchain_version = _get_toolchain_version() + if not (_get_toolchain_path(toolchain_version) / ".ready").exists(): + with _install_lock(f"toolchain-{toolchain_version}"): _install_toolchain() def _install_toolchain() -> None: - toolchains_dir = _get_toolchain_path(TOOLCHAIN_VERSION) + toolchain_version = _get_toolchain_version() + toolchains_dir = _get_toolchain_path(toolchain_version) sentinel = toolchains_dir / ".ready" if not sentinel.exists(): - rmdir(toolchains_dir, msg=f"Clean up {TOOLCHAIN_VERSION} toolchain environment") + rmdir(toolchains_dir, msg=f"Clean up {toolchain_version} toolchain environment") sysname, machine, extension = _get_toolchain_platform_info() substitutions = { - "VERSION": TOOLCHAIN_VERSION, + "VERSION": toolchain_version, "sysname": sysname, "machine": machine, "extension": extension, } # Downloaded next to the destination (not a temp file) so an # interrupted download's .part file resumes on the next run. + # SDK 1.0+ Zephyr-sdkConfig.cmake looks for the toolchain in + # gnu/arm-zephyr-eabi/; extraction strips the archive's single root. for mirrors, extract_dir, what, slug in ( (SDK_NG_MINIMAL_MIRRORS, toolchains_dir, "Zephyr SDK minimal", "minimal"), ( - SDK_NG_TOOLCHAIN_MIRRORS, - toolchains_dir / "arm-zephyr-eabi", + _get_sdk_ng_toolchain_mirrors(), + _get_arm_toolchain_path(), "toolchain", "toolchain", ), ): - _LOGGER.info("Downloading %s %s ...", TOOLCHAIN_VERSION, what) + _LOGGER.info("Downloading %s %s ...", toolchain_version, what) download_and_extract( mirrors, substitutions, @@ -598,10 +644,17 @@ def _install_toolchain() -> None: extract_dir, progress_header="Extracting", ) - # Best-effort prune of resume leftovers, including a previous - # TOOLCHAIN_VERSION's orphans; the SDK archives are hundreds of MB. - # A locked file must not discard the just-completed install. + # Best-effort prune of resume leftovers, including orphans of retired + # toolchain versions; the SDK archives are hundreds of MB. The other + # toolchain still in use may be downloading under its own lock, so its + # leftovers are kept. A locked file must not discard the just-completed + # install. + other_versions = tuple( + f"{v}." for v in _TOOLCHAIN_VERSIONS if v != toolchain_version + ) for leftover in toolchains_dir.parent.glob("*.archive.part*"): + if leftover.name.startswith(other_versions): + continue try: leftover.unlink() except OSError as err: diff --git a/tests/unit_tests/components/nrf52/test_pch.py b/tests/unit_tests/components/nrf52/test_pch.py index 5af55586c33..0dcac3cad4a 100644 --- a/tests/unit_tests/components/nrf52/test_pch.py +++ b/tests/unit_tests/components/nrf52/test_pch.py @@ -7,6 +7,7 @@ import pytest from esphome.components import nrf52 from esphome.components.zephyr.const import KEY_BOARD +import esphome.config_validation as cv from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION, Toolchain from esphome.core import CORE, EsphomeError @@ -110,6 +111,7 @@ def run_cmd(tmp_path: Path) -> Mock: CORE.build_path = tmp_path / "build" CORE.name = "livingroom" CORE.toolchain = Toolchain.SDK_NRF + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: cv.Version(3, 2, 0)} with ( patch.object(nrf52, "check_and_install"), patch.object(nrf52, "_generate_cmake_lists", return_value=False), diff --git a/tests/unit_tests/components/nrf52/test_run_compile_hex.py b/tests/unit_tests/components/nrf52/test_run_compile_hex.py new file mode 100644 index 00000000000..f86a29a9d24 --- /dev/null +++ b/tests/unit_tests/components/nrf52/test_run_compile_hex.py @@ -0,0 +1,130 @@ +"""nrf52 sdk-nrf build: the hex file used for the Adafruit UF2 and DFU package.""" + +from collections.abc import Iterator +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest + +from esphome.components import nrf52 +from esphome.components.nrf52.const import BOOTLOADER_ADAFRUIT_NRF52_SD140_V7 +from esphome.components.zephyr.const import KEY_BOARD, KEY_BOOTLOADER +import esphome.config_validation as cv +from esphome.const import KEY_CORE, KEY_FRAMEWORK_VERSION, Toolchain +from esphome.core import CORE + + +@pytest.fixture +def build_dir(tmp_path: Path) -> Path: + CORE.config_path = tmp_path / "test.yaml" + CORE.build_path = tmp_path / "build" + CORE.name = "livingroom" + CORE.toolchain = Toolchain.SDK_NRF + return CORE.relative_pioenvs_path(CORE.name) + + +@pytest.fixture +def run_cmd(tmp_path: Path) -> Iterator[Mock]: + with ( + patch.object(nrf52, "check_and_install"), + patch.object(nrf52, "_generate_cmake_lists", return_value=False), + patch.object( + nrf52, + "get_build_paths", + return_value={"python_executable": "python3", "framework_path": tmp_path}, + ), + patch.object(nrf52, "get_build_env", return_value={}), + patch.object( + nrf52, + "zephyr_data", + return_value={ + KEY_BOARD: "board", + KEY_BOOTLOADER: BOOTLOADER_ADAFRUIT_NRF52_SD140_V7, + }, + ), + patch.object(nrf52.pch, "pch_enabled", return_value=False), + patch.object(nrf52, "run_command_ok", return_value=True) as run, + ): + yield run + + +def _uf2_hex(run_cmd: Mock) -> str: + cmd = next(c.args[0] for c in run_cmd.call_args_list if "-f" in c.args[0]) + return cmd[-1] + + +def _dfu_hex(run_cmd: Mock) -> str: + cmd = next(c.args[0] for c in run_cmd.call_args_list if "genpkg" in c.args[0]) + return cmd[cmd.index("--application") + 1] + + +def _build( + build_dir: Path, + version: cv.Version, + outputs: list[str], + run_cmd: Mock, + leftovers: tuple[str, ...] = (), +) -> None: + """Run run_compile; *outputs* appear when west builds, *leftovers* before.""" + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: version} + (build_dir / "zephyr" / "zephyr").mkdir(parents=True) + # Without a CMake cache run_compile wipes the build dir first + for name in ["CMakeCache.txt", *leftovers]: + (build_dir / name).write_text("") + + def west_build(cmd: list[str], **kwargs: object) -> bool: + if "west" in cmd: + for name in outputs: + (build_dir / name).write_text("") + return True + + run_cmd.side_effect = west_build + nrf52.run_compile(None, {}) + + +def test_uf2_and_dfu_use_merged_hex(build_dir: Path, run_cmd: Mock) -> None: + _build( + build_dir, + cv.Version(2, 9, 2), + ["merged.hex", "zephyr/zephyr/zephyr.hex"], + run_cmd, + ) + merged = str(build_dir / "zephyr" / "merged.hex") + assert _uf2_hex(run_cmd) == merged + assert _dfu_hex(run_cmd) == merged + + +def test_uf2_and_dfu_fall_back_to_zephyr_hex(build_dir: Path, run_cmd: Mock) -> None: + """The nRF Connect SDK 3.4.0 build produces no merged.hex.""" + _build(build_dir, cv.Version(3, 4, 0), ["zephyr/zephyr/zephyr.hex"], run_cmd) + app_hex = build_dir / "zephyr" / "zephyr.hex" + assert app_hex.is_file() + assert _uf2_hex(run_cmd) == str(app_hex) + assert _dfu_hex(run_cmd) == str(app_hex) + + +def test_older_sdk_does_not_fall_back_to_zephyr_hex( + build_dir: Path, run_cmd: Mock +) -> None: + """Before SDK 3.4.0 a missing merged.hex must not be replaced by zephyr.hex.""" + _build(build_dir, cv.Version(3, 2, 0), ["zephyr/zephyr/zephyr.hex"], run_cmd) + assert not any("-f" in c.args[0] for c in run_cmd.call_args_list) + assert _dfu_hex(run_cmd) == str(build_dir / "zephyr" / "merged.hex") + + +def test_stale_merged_hex_is_removed_before_building( + build_dir: Path, run_cmd: Mock +) -> None: + """A merged.hex left by an older SDK build must not be packaged.""" + _build( + build_dir, + cv.Version(3, 4, 0), + ["zephyr/zephyr/zephyr.hex"], + run_cmd, + leftovers=("merged.hex", "zephyr/merged.hex"), + ) + assert not (build_dir / "merged.hex").exists() + assert not (build_dir / "zephyr" / "merged.hex").exists() + app_hex = str(build_dir / "zephyr" / "zephyr.hex") + assert _uf2_hex(run_cmd) == app_hex + assert _dfu_hex(run_cmd) == app_hex diff --git a/tests/unit_tests/test_nrf52_framework.py b/tests/unit_tests/test_nrf52_framework.py index d6b27bdbef2..17d583d1393 100644 --- a/tests/unit_tests/test_nrf52_framework.py +++ b/tests/unit_tests/test_nrf52_framework.py @@ -20,6 +20,7 @@ from esphome.components.nrf52.framework import ( _get_penv_site_packages, _get_platformio_penv_path, _get_toolchain_platform_info, + _install_toolchain, _needs_venv_rebuild, _wanted_west_projects, check_and_install, @@ -808,6 +809,63 @@ class TestCheckAndInstall: assert substitutions["machine"] == "x86_64" assert substitutions["extension"] == "tar.xz" + def test_toolchain_download_uses_gnu_url_for_sdk_3_4_0( + self, + tmp_path: Path, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """For nRF Connect SDK >= 3.4.0 the toolchain archive name includes 'toolchain_gnu_'.""" + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse("3.4.0")} + sdk_version = "3.4.0" + tools = get_sdk_nrf_tools_path() + python_env = tools / "penvs" / f"v{sdk_version}" + framework = tools / "frameworks" / f"v{sdk_version}" + toolchain_dir = tools / "toolchains" / "1.0.1" + for d in (python_env, framework, toolchain_dir): + d.mkdir(parents=True, exist_ok=True) + (framework / "zephyr" / "scripts").mkdir(parents=True, exist_ok=True) + (framework / "zephyr" / "scripts" / "requirements.txt").touch() + _mark_venv_ready(python_env) + (framework / ".ready").touch() + + check_and_install() + + # Two download calls: minimal SDK first, toolchain second + toolchain_call = mock_nrf52_ops.download_from_mirrors.call_args_list[1] + mirrors = toolchain_call.args[0] + assert all("toolchain_gnu_" in m for m in mirrors) + + def test_toolchain_extracts_under_gnu_for_sdk_3_4_0( + self, + tmp_path: Path, + mock_nrf52_ops: SimpleNamespace, + ) -> None: + """SDK 1.0+ toolchain archive must land in gnu/arm-zephyr-eabi/. + + Zephyr-sdkConfig.cmake validates the toolchain at gnu/arm-zephyr-eabi/ + in SDK 1.0+; if the archive is extracted to arm-zephyr-eabi/ instead, + cmake reports the package as not found. + """ + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse("3.4.0")} + sdk_version = "3.4.0" + tools = get_sdk_nrf_tools_path() + python_env = tools / "penvs" / f"v{sdk_version}" + framework = tools / "frameworks" / f"v{sdk_version}" + toolchain_dir = tools / "toolchains" / "1.0.1" + for d in (python_env, framework, toolchain_dir): + d.mkdir(parents=True, exist_ok=True) + (framework / "zephyr" / "scripts").mkdir(parents=True, exist_ok=True) + (framework / "zephyr" / "scripts" / "requirements.txt").touch() + _mark_venv_ready(python_env) + (framework / ".ready").touch() + + check_and_install() + + # Two extract calls: minimal SDK first (to toolchain root), toolchain second + extract_calls = mock_nrf52_ops.archive_extract_all.call_args_list + _, toolchain_extract_dir = extract_calls[1].args[:2] + assert toolchain_extract_dir == toolchain_dir / "gnu" / "arm-zephyr-eabi" + # --------------------------------------------------------------------------- # setup_platformio_python_env tests @@ -1040,6 +1098,44 @@ def test_get_build_env( assert env["SOME_PREEXISTING_VAR"] == "kept" +def test_get_build_env_sdk_3_4_0_uses_toolchain_root( + setup_core: Path, +) -> None: + """For NCS >= 3.4.0, ZEPHYR_SDK_INSTALL_DIR still points at the toolchain root.""" + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse("3.4.0")} + + env = get_build_env() + + tools = get_sdk_nrf_tools_path() + assert env["ZEPHYR_SDK_INSTALL_DIR"] == str(tools / "toolchains" / "1.0.1") + assert "Zephyr-sdk_DIR" not in env + + +def test_install_toolchain_keeps_other_live_toolchain_leftovers( + setup_core: Path, +) -> None: + """Pruning must not delete a partial download of the other toolchain in use.""" + CORE.data[KEY_CORE] = {KEY_FRAMEWORK_VERSION: Version.parse("3.4.0")} + toolchains = get_sdk_nrf_tools_path() / "toolchains" + toolchains.mkdir(parents=True) + own = toolchains / "1.0.1.toolchain.archive.part" + other = toolchains / f"{TOOLCHAIN_VERSION}.toolchain.archive.part" + retired = toolchains / "0.16.8.toolchain.archive.part" + for leftover in (own, other, retired): + leftover.write_text("") + + with patch( + "esphome.components.nrf52.framework.download_and_extract", + side_effect=lambda *args, **kwargs: args[3].mkdir(parents=True), + ): + _install_toolchain() + + assert not own.exists() + assert other.exists() + assert not retired.exists() + assert (toolchains / "1.0.1" / ".ready").exists() + + # --------------------------------------------------------------------------- # get_sdk_nrf_tools_path tests # --------------------------------------------------------------------------- diff --git a/tests/unit_tests/test_nrf52_toolchain.py b/tests/unit_tests/test_nrf52_toolchain.py index ee8f6eaf499..b09540bb804 100644 --- a/tests/unit_tests/test_nrf52_toolchain.py +++ b/tests/unit_tests/test_nrf52_toolchain.py @@ -24,6 +24,7 @@ def _set_sdk_version(version: cv.Version) -> None: def test_binutils_come_from_the_pinned_zephyr_sdk() -> None: + _set_sdk_version(cv.Version(2, 9, 0)) bin_path = ( get_sdk_nrf_tools_path() / "toolchains" @@ -37,6 +38,21 @@ def test_binutils_come_from_the_pinned_zephyr_sdk() -> None: assert toolchain.get_readelf_path() == bin_path / f"arm-zephyr-eabi-readelf{suffix}" +def test_binutils_use_new_toolchain_for_sdk_3_4_0() -> None: + _set_sdk_version(cv.Version(3, 4, 0)) + bin_path = ( + get_sdk_nrf_tools_path() + / "toolchains" + / "1.0.1" + / "gnu" + / "arm-zephyr-eabi" + / "bin" + ) + suffix = ".exe" if os.name == "nt" else "" + assert toolchain.get_objdump_path() == bin_path / f"arm-zephyr-eabi-objdump{suffix}" + assert toolchain.get_readelf_path() == bin_path / f"arm-zephyr-eabi-readelf{suffix}" + + def test_elf_uses_the_nested_layout_from_sdk_2_9_2(nrf52_build: Path) -> None: _set_sdk_version(cv.Version(2, 9, 2)) # A flat image left by an older SDK must not be picked over the current From 2accef437ff94503d418779acbaf63f4782b89ab Mon Sep 17 00:00:00 2001 From: Jeff Brown Date: Thu, 1 Oct 2026 05:46:49 -0700 Subject: [PATCH 103/133] [pmsa003i] Fix spurious component failures and warnings on power up (#19933) Co-authored-by: J. Nick Koston --- esphome/components/pmsa003i/pmsa003i.cpp | 40 ++++++++++++++++-------- esphome/components/pmsa003i/pmsa003i.h | 1 + 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/esphome/components/pmsa003i/pmsa003i.cpp b/esphome/components/pmsa003i/pmsa003i.cpp index 0b5c72a94d2..f20b93fff80 100644 --- a/esphome/components/pmsa003i/pmsa003i.cpp +++ b/esphome/components/pmsa003i/pmsa003i.cpp @@ -15,24 +15,38 @@ static const uint8_t CHECKSUM_START_INDEX = COUNT_DATA_BYTES - 2; static const uint8_t COUNT_16_BIT_VALUES = (COUNT_PAYLOAD_LENGTH_BYTES + COUNT_PAYLOAD_BYTES) / 2; static const uint8_t START_CHARACTER_1 = 0x42; static const uint8_t START_CHARACTER_2 = 0x4D; -static const uint8_t READ_DATA_RETRY_COUNT = 3; + +// Timeout for determining when the device is ready for use, in milliseconds. +// The PMSA003I typically takes 2.3 seconds to perform its first measurement after a cold power up and +// I2C requests performed during that time will be NACKed. Use a slightly longer timeout to tolerate +// timing variation. Note that although we consider the device ready as soon as it responds to I2C +// requests, it may take 30 seconds or more for the data to stabilize according to the datasheet. +static const uint32_t READY_TIMEOUT_MS = 3000; + +// Poll interval for determining when the device is ready for use, in milliseconds. +static const uint32_t READY_POLL_INTERVAL_MS = 100; void PMSA003IComponent::setup() { - PM25AQIData data; - bool successful_read = this->read_data_(&data); + // Stop polling until the device is actually ready to prevent spurious I2C warnings during premature updates. + this->stop_poller(); + this->poll_until_ready_or_timeout_(millis()); +} - if (!successful_read) { - for (uint8_t i = 0; i < READ_DATA_RETRY_COUNT; i++) { - successful_read = this->read_data_(&data); - if (successful_read) { - break; - } - } +void PMSA003IComponent::poll_until_ready_or_timeout_(uint32_t start_time) { + // Check whether the device is responding and identifies itself as expected. + // We don't care about the actual sensor readings or the packet CRC in this case. + uint8_t buffer[2]; + if (this->read(buffer, sizeof(buffer)) == i2c::ERROR_OK && buffer[0] == START_CHARACTER_1 && + buffer[1] == START_CHARACTER_2) { + ESP_LOGD(TAG, "PMSA003I is ready"); + this->start_poller(); + return; } - if (!successful_read) { - this->mark_failed(); - return; + if (millis() - start_time < READY_TIMEOUT_MS) { + this->set_timeout(READY_POLL_INTERVAL_MS, [this, start_time]() { this->poll_until_ready_or_timeout_(start_time); }); + } else { + this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL)); } } diff --git a/esphome/components/pmsa003i/pmsa003i.h b/esphome/components/pmsa003i/pmsa003i.h index 908b073be1e..6eccde36659 100644 --- a/esphome/components/pmsa003i/pmsa003i.h +++ b/esphome/components/pmsa003i/pmsa003i.h @@ -46,6 +46,7 @@ class PMSA003IComponent final : public PollingComponent, public i2c::I2CDevice { void set_pmc_10_0_sensor(sensor::Sensor *pmc_10_0) { this->pmc_10_0_sensor_ = pmc_10_0; } protected: + void poll_until_ready_or_timeout_(uint32_t start_time); bool read_data_(PM25AQIData *data); bool standard_units_; From 56a7a1116210b2c936a7d7015016b16f12505d3c Mon Sep 17 00:00:00 2001 From: rwrozelle Date: Thu, 1 Oct 2026 08:48:31 -0400 Subject: [PATCH 104/133] [openthread] rewrite openthread esp.cpp to use new esp functions (#16334) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: Claude Sonnet 4.6 Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston --- esphome/components/openthread/openthread.cpp | 44 ++-- esphome/components/openthread/openthread.h | 7 +- .../components/openthread/openthread_esp.cpp | 196 +++++++----------- .../openthread/openthread_zephyr.cpp | 5 +- 4 files changed, 105 insertions(+), 147 deletions(-) diff --git a/esphome/components/openthread/openthread.cpp b/esphome/components/openthread/openthread.cpp index c788b1f968d..7d98522cd23 100644 --- a/esphome/components/openthread/openthread.cpp +++ b/esphome/components/openthread/openthread.cpp @@ -230,23 +230,27 @@ void *OpenThreadSrpComponent::pool_alloc_(size_t size) { bool OpenThreadComponent::teardown() { switch (this->teardown_stage_) { case TeardownStage::TEARDOWN_STAGE_NOT_STARTED: { - auto lock = InstanceLock::try_acquire(100); - if (!lock) { - // Try again on next teardown loop - ESP_LOGV(TAG, "Failed to acquire OpenThread lock during teardown"); - return false; - } - // Start tearing down - this->teardown_stage_ = TeardownStage::TEARDOWN_STAGE_STOP_IN_PROCESS; - ESP_LOGV(TAG, "Clear SRP"); - otInstance *instance = lock.get_instance(); - otSrpClientClearHostAndServices(instance); - otSrpClientBuffersFreeAllServices(instance); - if (otThreadSetEnabled(instance, false) != OT_ERROR_NONE) { - ESP_LOGW(TAG, "Failed to disable Thread during teardown"); - } - if (otIp6SetEnabled(instance, false) != OT_ERROR_NONE) { - ESP_LOGW(TAG, "Failed to disable IPv6 during teardown"); + { + auto lock = InstanceLock::try_acquire(100); + // The OT task may still be starting up; stay pending and retry on + // the next call rather than giving up after a single failed attempt. + if (!lock) { + ESP_LOGV(TAG, "Failed to acquire OpenThread lock during teardown"); + return false; + } + this->teardown_stage_ = TeardownStage::TEARDOWN_STAGE_STOP_IN_PROCESS; + ESP_LOGV(TAG, "Clear SRP"); + otInstance *instance = lock.get_instance(); + otSrpClientClearHostAndServices(instance); + otSrpClientBuffersFreeAllServices(instance); + if (otThreadSetEnabled(instance, false) != OT_ERROR_NONE) { + ESP_LOGW(TAG, "Failed to disable Thread during teardown"); + } + if (otIp6SetEnabled(instance, false) != OT_ERROR_NONE) { + ESP_LOGW(TAG, "Failed to disable IPv6 during teardown"); + } + // Release the lock before stopping -- openthread_stop_() (esp_openthread_stop() on + // ESP32) acquires it internally, and the lock is not recursive. } // Stop OpenThread global_openthread_component = nullptr; @@ -254,11 +258,11 @@ bool OpenThreadComponent::teardown() { int error = this->openthread_stop_(); if (error != 0) { ESP_LOGW(TAG, "Failed attempt to stop OpenThread %d", error); - this->teardown_stage_ = TeardownStage::TEARDOWN_STAGE_COMPLETED; } } break; case TeardownStage::TEARDOWN_STAGE_STOP_IN_PROCESS: - // Waiting on OpenThread stop + // Unreachable today; teardown is synchronous on both platforms. Kept for a future + // graceful-exit path, or a platform whose teardown() cannot be made synchronous. break; case TeardownStage::TEARDOWN_STAGE_COMPLETED: ESP_LOGV(TAG, "OpenThreadComponent Teardown Complete"); @@ -286,7 +290,7 @@ void OpenThreadComponent::apply_poll_period(uint32_t poll_period) { #if CONFIG_OPENTHREAD_MTD this->set_poll_period(poll_period); if (!this->is_lock_initialized()) { - // The action may run before the stack is up, e.g. from a restore mode; ot_main applies the stored value. + // The action may run before the stack is up, e.g. from a restore mode; setup() applies the stored value. ESP_LOGD(TAG, "Not (yet) ready to apply"); return; } diff --git a/esphome/components/openthread/openthread.h b/esphome/components/openthread/openthread.h index 8bccae7f85e..8599304c89a 100644 --- a/esphome/components/openthread/openthread.h +++ b/esphome/components/openthread/openthread.h @@ -35,11 +35,10 @@ class OpenThreadComponent final : public Component { float get_setup_priority() const override { return setup_priority::WIFI; } bool is_connected() const { return this->connected_; } - /// Returns true once esp_openthread_init() has completed and the OT lock is usable. + /// Returns true once esp_openthread_start() has completed and the OT lock is usable. bool is_lock_initialized() const { return this->lock_initialized_; } network::IPAddresses get_ip_addresses(); std::optional get_omr_address(); - void ot_main(); void on_factory_reset(std::function callback); void defer_factory_reset_external_callback(); @@ -60,7 +59,6 @@ class OpenThreadComponent final : public Component { protected: /** Apply Link Mode settings (incl poll period). * Callers running outside the OpenThread task must hold InstanceLock. - * ot_main() runs on the OpenThread task itself and must not acquire the lock. */ void apply_linkmode_(otInstance *instance); @@ -73,7 +71,8 @@ class OpenThreadComponent final : public Component { #endif std::optional output_power_{}; std::atomic lock_initialized_{false}; - std::atomic teardown_stage_{TeardownStage::TEARDOWN_STAGE_NOT_STARTED}; + // Only ever written from teardown(), on the main task -- no atomic needed. + TeardownStage teardown_stage_{TeardownStage::TEARDOWN_STAGE_NOT_STARTED}; std::atomic connected_{false}; private: diff --git a/esphome/components/openthread/openthread_esp.cpp b/esphome/components/openthread/openthread_esp.cpp index 881bbea3c99..526fc961f2f 100644 --- a/esphome/components/openthread/openthread_esp.cpp +++ b/esphome/components/openthread/openthread_esp.cpp @@ -1,6 +1,5 @@ #include "esphome/core/defines.h" #if defined(USE_OPENTHREAD) && defined(USE_ESP32) -#include #include "openthread.h" #include "esp_log.h" @@ -13,14 +12,9 @@ #include "esphome/core/log.h" #include "esp_err.h" -#include "esp_event.h" #include "esp_netif.h" -#include "esp_netif_types.h" -#include "esp_openthread_cli.h" #include "esp_openthread_netif_glue.h" #include "esp_vfs_eventfd.h" -#include "freertos/FreeRTOS.h" -#include "freertos/task.h" #include "nvs_flash.h" static const char *const TAG = "openthread"; @@ -39,140 +33,94 @@ void OpenThreadComponent::setup() { ESP_ERROR_CHECK(nvs_flash_init()); ESP_ERROR_CHECK(esp_vfs_eventfd_register(&eventfd_config)); - xTaskCreate( - [](void *arg) { - static_cast(arg)->ot_main(); - vTaskDelete(nullptr); - }, - "ot_main", 10240, this, 5, nullptr); -} + esp_openthread_config_t config = {.netif_config = ESP_NETIF_DEFAULT_OPENTHREAD(), + .platform_config = { + .radio_config = + { + .radio_mode = RADIO_MODE_NATIVE, + .radio_uart_config = {}, + }, + .host_config = + { + // There is a conflict between esphome's logger which also + // claims the usb serial jtag device. + // .host_connection_mode = HOST_CONNECTION_MODE_CLI_USB, + // .host_usb_config = USB_SERIAL_JTAG_DRIVER_CONFIG_DEFAULT(), + }, + .port_config = + { + .storage_partition_name = "nvs", + .netif_queue_size = 10, + .task_queue_size = 10, + }, + }}; -static esp_netif_t *init_openthread_netif(const esp_openthread_platform_config_t *config) { - esp_netif_config_t cfg = ESP_NETIF_DEFAULT_OPENTHREAD(); - esp_netif_t *netif = esp_netif_new(&cfg); - assert(netif != nullptr); - ESP_ERROR_CHECK(esp_netif_attach(netif, esp_openthread_netif_glue_init(config))); - - return netif; -} - -void OpenThreadComponent::ot_main() { - esp_openthread_platform_config_t config = { - .radio_config = - { - .radio_mode = RADIO_MODE_NATIVE, - .radio_uart_config = {}, - }, - .host_config = - { - // There is a conflict between esphome's logger which also - // claims the usb serial jtag device. - // .host_connection_mode = HOST_CONNECTION_MODE_CLI_USB, - // .host_usb_config = USB_SERIAL_JTAG_DRIVER_CONFIG_DEFAULT(), - }, - .port_config = - { - .storage_partition_name = "nvs", - .netif_queue_size = 10, - .task_queue_size = 10, - }, - }; - - // Initialize the OpenThread stack - // otLoggingSetLevel(OT_LOG_LEVEL_DEBG); - ESP_ERROR_CHECK(esp_openthread_init(&config)); + ESP_ERROR_CHECK(esp_openthread_start(&config)); // Mark lock as initialized so InstanceLock callers know it's safe to acquire. - // Must be set after esp_openthread_init() which creates the internal semaphore. + // Must be set after esp_openthread_start() which creates the internal semaphore. this->lock_initialized_ = true; // Fetch OT instance once to avoid repeated call into OT stack otInstance *instance = esp_openthread_get_instance(); + { + InstanceLock lock = InstanceLock::acquire(); -#if CONFIG_OPENTHREAD_STATE_INDICATOR_ENABLE - ESP_ERROR_CHECK(esp_openthread_state_indicator_init(instance)); -#endif + this->apply_linkmode_(instance); -#if CONFIG_OPENTHREAD_LOG_LEVEL_DYNAMIC - // The OpenThread log level directly matches ESP log level - (void) otLoggingSetLevel(CONFIG_LOG_DEFAULT_LEVEL); -#endif - // Initialize the OpenThread cli -#if CONFIG_OPENTHREAD_CLI - esp_openthread_cli_init(); -#endif - - esp_netif_t *openthread_netif; - // Initialize the esp_netif bindings - openthread_netif = init_openthread_netif(&config); - esp_netif_set_default_netif(openthread_netif); - -#if CONFIG_OPENTHREAD_CLI_ESP_EXTENSION - esp_cli_custom_command_init(); -#endif // CONFIG_OPENTHREAD_CLI_ESP_EXTENSION - - ESP_LOGD(TAG, "Thread Version: %" PRIu16, otThreadGetVersion()); - - this->apply_linkmode_(instance); - - if (this->output_power_.has_value()) { - if (const auto err = otPlatRadioSetTransmitPower(instance, *this->output_power_); err != OT_ERROR_NONE) { - ESP_LOGE(TAG, "Failed to set power: %s", otThreadErrorToString(err)); + if (this->output_power_.has_value()) { + if (const auto err = otPlatRadioSetTransmitPower(instance, *this->output_power_); err != OT_ERROR_NONE) { + ESP_LOGE(TAG, "Failed to set power: %s", otThreadErrorToString(err)); + } } - } - - // Run the main loop -#if CONFIG_OPENTHREAD_CLI - esp_openthread_cli_create_task(); -#endif - ESP_LOGI(TAG, "Activating dataset..."); - otOperationalDatasetTlvs dataset = {}; + ESP_LOGI(TAG, "Activating dataset..."); + otOperationalDatasetTlvs dataset = {}; #ifndef USE_OPENTHREAD_FORCE_DATASET - // Check if openthread has a valid dataset from a previous execution - otError error = otDatasetGetActiveTlvs(instance, &dataset); - if (error != OT_ERROR_NONE) { - // Make sure the length is 0 so we fallback to the configuration - dataset.mLength = 0; - } else { - ESP_LOGI(TAG, "Found existing dataset, ignoring config (force_dataset: true to override)"); - } + // Check if openthread has a valid dataset from a previous execution + otError error = otDatasetGetActiveTlvs(instance, &dataset); + if (error != OT_ERROR_NONE) { + // Make sure the length is 0 so we fallback to the configuration + dataset.mLength = 0; + } else { + ESP_LOGI(TAG, "Found existing dataset, ignoring config (force_dataset: true to override)"); + } #endif #ifdef USE_OPENTHREAD_TLVS - if (dataset.mLength == 0) { - // If we didn't have an active dataset, and we have tlvs, parse it and pass it to esp_openthread_auto_start - size_t len = (sizeof(USE_OPENTHREAD_TLVS) - 1) / 2; - if (len > sizeof(dataset.mTlvs)) { - ESP_LOGW(TAG, "TLV buffer too small, truncating"); - len = sizeof(dataset.mTlvs); + if (dataset.mLength == 0) { + // If we didn't have an active dataset, and we have tlvs, parse it and pass it to esp_openthread_auto_start + size_t len = (sizeof(USE_OPENTHREAD_TLVS) - 1) / 2; + if (len > sizeof(dataset.mTlvs)) { + ESP_LOGW(TAG, "TLV buffer too small, truncating"); + len = sizeof(dataset.mTlvs); + } + parse_hex(USE_OPENTHREAD_TLVS, sizeof(USE_OPENTHREAD_TLVS) - 1, dataset.mTlvs, len); + dataset.mLength = len; } - parse_hex(USE_OPENTHREAD_TLVS, sizeof(USE_OPENTHREAD_TLVS) - 1, dataset.mTlvs, len); - dataset.mLength = len; - } #endif - // Pass the existing dataset, or NULL which will use the preprocessor definitions - ESP_ERROR_CHECK(esp_openthread_auto_start(dataset.mLength > 0 ? &dataset : nullptr)); + // Pass the existing dataset, or NULL which will use the preprocessor definitions + ESP_ERROR_CHECK(esp_openthread_auto_start(dataset.mLength > 0 ? &dataset : nullptr)); - // Register state change callback to update connected_ reactively instead of polling - otError ot_err = otSetStateChangedCallback(instance, OpenThreadComponent::on_state_changed, this); - if (ot_err != OT_ERROR_NONE) { - ESP_LOGW(TAG, "Failed to register state change callback: %d", ot_err); + // Register state change callback to update connected_ reactively instead of polling + otError ot_err = otSetStateChangedCallback(instance, OpenThreadComponent::on_state_changed, this); + if (ot_err != OT_ERROR_NONE) { + ESP_LOGW(TAG, "Failed to register state change callback: %d", ot_err); + } } - esp_openthread_launch_mainloop(); - - // Clean up - reset lock flag before deinit destroys the semaphore - this->lock_initialized_ = false; - esp_openthread_deinit(); - esp_openthread_netif_glue_deinit(); - esp_netif_destroy(openthread_netif); - - esp_vfs_eventfd_unregister(); - this->teardown_stage_ = TeardownStage::TEARDOWN_STAGE_COMPLETED; - vTaskDelete(NULL); + ESP_LOGD(TAG, "Thread Version: %" PRIu16, otThreadGetVersion()); } -int OpenThreadComponent::openthread_stop_() { return esp_openthread_mainloop_exit(); } +int OpenThreadComponent::openthread_stop_() { + // Clean up - reset lock flag before deinit destroys the semaphore + this->lock_initialized_ = false; + int error = esp_openthread_stop(); + // Mark complete even on failure: we're already mid-shutdown/reboot, so there's no + // recovery path to retry into -- leaving the stage stuck would only burn the full + // teardown timeout for no benefit. + this->teardown_stage_ = TeardownStage::TEARDOWN_STAGE_COMPLETED; + return error; +} network::IPAddresses OpenThreadComponent::get_ip_addresses() { network::IPAddresses addresses; @@ -199,8 +147,14 @@ InstanceLock InstanceLock::try_acquire(int delay) { } InstanceLock InstanceLock::acquire() { - // Wait for the lock to be created by ot_main() before attempting to acquire it. - // esp_openthread_lock_acquire() will assert-crash if called before esp_openthread_init(). + // teardown() clears global_openthread_component before the stack fully stops; a caller + // racing teardown would otherwise dereference a null pointer below. + if (global_openthread_component == nullptr) { + ESP_LOGE(TAG, "OpenThread component torn down, cannot acquire instance lock"); + abort(); + } + // Wait for the lock to be created before attempting to acquire it. + // esp_openthread_lock_acquire() will assert-crash if called before esp_openthread_start(). constexpr uint32_t lock_init_timeout_ms = 10000; uint32_t start = millis(); while (!global_openthread_component->is_lock_initialized()) { diff --git a/esphome/components/openthread/openthread_zephyr.cpp b/esphome/components/openthread/openthread_zephyr.cpp index cacb4c01226..0f474fc040c 100644 --- a/esphome/components/openthread/openthread_zephyr.cpp +++ b/esphome/components/openthread/openthread_zephyr.cpp @@ -83,9 +83,10 @@ void OpenThreadComponent::setup() { } openthread_state_changed_cb_register(context, &ot_state_changed_cb); openthread_start(context); -} -void OpenThreadComponent::ot_main() {} + InstanceLock lock = InstanceLock::acquire(); + this->apply_linkmode_(lock.get_instance()); +} otInstance *OpenThreadComponent::get_openthread_instance_() { return openthread_get_default_instance(); } From 826571518acd50e3194fa390f629f3ac58fedd79 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:15:31 -0500 Subject: [PATCH 105/133] [noise] Generate the responder ephemeral key ahead of the handshake (#19000) --- esphome/components/api/__init__.py | 2 + esphome/components/api/api_connection.h | 6 ++ esphome/components/api/api_server.cpp | 23 ++++++- esphome/components/api/api_server.h | 5 +- esphome/components/noise/__init__.py | 5 ++ esphome/components/noise/noise.cpp | 34 ++++++++++ esphome/components/noise/noise.h | 18 +++++ esphome/components/noise/noise_handshake.cpp | 7 ++ esphome/components/noise/noise_handshake.h | 3 +- esphome/core/defines.h | 1 + tests/components/noise/__init__.py | 8 +++ .../components/noise/test_noise_handshake.cpp | 66 +++++++++++++++++++ 12 files changed, 174 insertions(+), 4 deletions(-) diff --git a/esphome/components/api/__init__.py b/esphome/components/api/__init__.py index 2291dc54f8f..40f520a0575 100644 --- a/esphome/components/api/__init__.py +++ b/esphome/components/api/__init__.py @@ -13,6 +13,7 @@ from esphome.components.logger import request_log_listener from esphome.components.noise import ( # noqa: F401 ENCRYPTION_SCHEMA, decode_encryption_key, + enable_spare_ephemeral, encryption_schema, new_psk_progmem, validate_encryption_key, @@ -613,6 +614,7 @@ async def to_code(config: ConfigType) -> None: # and plaintext disabled. Only a factory reset can remove it. cg.add_define("USE_API_PLAINTEXT") cg.add_define("USE_API_NOISE") + enable_spare_ephemeral() else: cg.add_define("USE_API_PLAINTEXT") diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index f03c4c42e8f..0b8a1dc2b29 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -317,9 +317,15 @@ class APIConnection final : public APIServerConnectionBase { void on_noise_encryption_set_key_request(const NoiseEncryptionSetKeyRequest &msg); #endif + // How long a new connection holds off the spare ephemeral refill + static constexpr uint32_t CONNECT_GRACE_MS = 1000; bool is_authenticated() { return static_cast(this->flags_.connection_state) == ConnectionState::AUTHENTICATED; } + // An older unauthenticated connection is a stale half open client and does not count + bool is_still_connecting(uint32_t now) { + return !this->is_authenticated() && now - this->last_traffic_ < CONNECT_GRACE_MS; + } bool is_connection_setup() { return static_cast(this->flags_.connection_state) == ConnectionState::CONNECTED || this->is_authenticated(); diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 2a89d31a74a..da9ca8bc799 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -162,6 +162,13 @@ void APIServer::loop() { this->accept_new_connections_(); } + const bool connected = network::is_connected(); +#ifdef USE_NOISE_SPARE_EPHEMERAL + if (connected && !noise::has_spare_ephemeral()) { + this->refill_spare_ephemeral_(); + } +#endif + if (this->api_connection_count_ == 0) { // Check reboot timeout - done in loop to avoid scheduler heap churn // (cancelled scheduler items sit in heap memory until their scheduled time). @@ -178,8 +185,7 @@ void APIServer::loop() { } // Process clients and remove disconnected ones in a single pass - // Check network connectivity once for all clients - if (!network::is_connected()) { + if (!connected) { // Network is down - disconnect all clients for (auto &client : this->active_clients()) { client->on_fatal_error(); @@ -207,6 +213,19 @@ void APIServer::loop() { } } +#ifdef USE_NOISE_SPARE_EPHEMERAL +// An OTA handshake is not visible here and just pays the refill it triggered +void APIServer::refill_spare_ephemeral_() { + const uint32_t now = App.get_loop_component_start_time(); + for (auto &client : this->active_clients()) { + if (client->is_still_connecting(now)) { + return; + } + } + noise::prepare_spare_ephemeral(); +} +#endif + void APIServer::remove_client_(uint8_t client_index) { auto &client = this->clients_[client_index]; diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 49cf77ca5af..865cd17156d 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -5,7 +5,7 @@ #include "api_buffer.h" // Must precede clients_ so APIConnection is complete for default_delete (libc++). #include "api_connection.h" -#ifdef USE_API_NOISE +#if defined(USE_API_NOISE) || defined(USE_NOISE_SPARE_EPHEMERAL) // Only present in the build when the noise component is loaded #include "esphome/components/noise/noise.h" #endif @@ -367,6 +367,9 @@ class APIServer final : public Component uint8_t provisioning_source_{0}; #endif +#ifdef USE_NOISE_SPARE_EPHEMERAL + void refill_spare_ephemeral_(); +#endif #ifdef USE_API_NOISE noise::NoiseContext noise_ctx_; #ifndef USE_API_NOISE_PSK_FROM_YAML diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index 47cd4cfc677..c894c147cdf 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -92,6 +92,11 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: return ENCRYPTION_SCHEMA(config) +def enable_spare_ephemeral() -> None: + """Compile the spare ephemeral key slot; the component that refills it calls this.""" + cg.add_define("USE_NOISE_SPARE_EPHEMERAL") + + async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") cg.add_library("esphome/noise-c", "0.1.30") diff --git a/esphome/components/noise/noise.cpp b/esphome/components/noise/noise.cpp index 86ea1ec8747..118bfb234fd 100644 --- a/esphome/components/noise/noise.cpp +++ b/esphome/components/noise/noise.cpp @@ -1,6 +1,7 @@ #include "noise.h" #ifdef USE_NOISE #include "esphome/core/hal.h" +#include "esphome/core/helpers.h" #include "esphome/core/log.h" #include "esphome/core/progmem.h" @@ -8,6 +9,7 @@ #include #include +#include namespace esphome::noise { @@ -21,6 +23,38 @@ void NoiseContext::load_psk(psk_t &out) const { progmem_memcpy(out.data(), this->psk_, out.size()); } +#ifdef USE_NOISE_SPARE_EPHEMERAL +static constexpr size_t PRIVATE_KEY_SIZE = SPARE_EPHEMERAL_KEY_SIZE; +static constexpr size_t PUBLIC_KEY_SIZE = SPARE_EPHEMERAL_KEY_SIZE; +uint8_t spare_ephemeral[SPARE_EPHEMERAL_SIZE]; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) + +void prepare_spare_ephemeral() { + uint8_t *private_key = spare_ephemeral; + uint8_t *public_key = spare_ephemeral + PRIVATE_KEY_SIZE; + // Same steps as noise-c's keygen; the clamp sets the ready bit + if (!random_bytes(private_key, PRIVATE_KEY_SIZE)) { + sodium_memzero(spare_ephemeral, sizeof(spare_ephemeral)); + return; + } + private_key[0] &= 0xF8; + private_key[PRIVATE_KEY_SIZE - 1] = (private_key[PRIVATE_KEY_SIZE - 1] & 0x7F) | 0x40; + if (crypto_scalarmult_curve25519_base(public_key, private_key) != 0) { + sodium_memzero(spare_ephemeral, sizeof(spare_ephemeral)); + } +} + +int consume_spare_ephemeral(NoiseHandshakeState *state) { + if (!has_spare_ephemeral()) { + return 0; + } + // noise-c keeps its own copy, so the slot is wiped either way + int err = noise_handshakestate_set_local_ephemeral(state, spare_ephemeral, PRIVATE_KEY_SIZE, + spare_ephemeral + PRIVATE_KEY_SIZE, PUBLIC_KEY_SIZE); + sodium_memzero(spare_ephemeral, sizeof(spare_ephemeral)); + return err; +} +#endif // USE_NOISE_SPARE_EPHEMERAL + const LogString *noise_err_to_logstr(int err) { if (err == NOISE_ERROR_NO_MEMORY) return LOG_STR("NO_MEMORY"); diff --git a/esphome/components/noise/noise.h b/esphome/components/noise/noise.h index 1033d5423ce..271a829944a 100644 --- a/esphome/components/noise/noise.h +++ b/esphome/components/noise/noise.h @@ -6,6 +6,9 @@ #include #include "esphome/core/log.h" +// Forward decl from +using NoiseHandshakeState = struct NoiseHandshakeState_s; + namespace esphome::noise { using psk_t = std::array; @@ -38,6 +41,21 @@ class NoiseContext { /// Convert a noise error code to a readable error const LogString *noise_err_to_logstr(int err); +#ifdef USE_NOISE_SPARE_EPHEMERAL +// One responder ephemeral key pair made ahead of time (about 60 ms on ESP8266), refilled by the api server +// while idle and consumed by the next noise handshake of any transport; empty means the handshake generates +// its own. Private key then public key; the private key stays in RAM until consumed. +static constexpr size_t SPARE_EPHEMERAL_KEY_SIZE = 32; +static constexpr size_t SPARE_EPHEMERAL_SIZE = 2 * SPARE_EPHEMERAL_KEY_SIZE; +extern uint8_t spare_ephemeral[SPARE_EPHEMERAL_SIZE]; // NOLINT(cppcoreguidelines-avoid-non-const-global-variables) +// The clamp always sets bit 254, so it doubles as the ready flag +inline bool has_spare_ephemeral() { return (spare_ephemeral[SPARE_EPHEMERAL_KEY_SIZE - 1] & 0x40) != 0; } +/// Fill the slot; blocks for the base point multiply +void prepare_spare_ephemeral(); +/// Move the key pair into a handshake that has not started and wipe the slot; 0 unless noise-c rejects it +int consume_spare_ephemeral(NoiseHandshakeState *state); +#endif + // Shared wire format for the noise transports (api and ota): every frame is // FRAME_INDICATOR, a 16-bit big-endian payload length, then the payload. // Handshake payloads start with a status byte; transport payloads end with diff --git a/esphome/components/noise/noise_handshake.cpp b/esphome/components/noise/noise_handshake.cpp index 64846dcb678..bd1afa198d3 100644 --- a/esphome/components/noise/noise_handshake.cpp +++ b/esphome/components/noise/noise_handshake.cpp @@ -57,6 +57,13 @@ int NoiseResponderHandshake::init(const NoiseContext &ctx, const uint8_t *prolog HANDSHAKE_STEP_LOG("noise_handshakestate_set_prologue", err); return this->fail_init_(err); } +#ifdef USE_NOISE_SPARE_EPHEMERAL + err = consume_spare_ephemeral(this->handshake_); + // Not fatal: the handshake generates its own key instead + if (err != 0) { + HANDSHAKE_STEP_LOG("noise_handshakestate_set_local_ephemeral", err); + } +#endif err = noise_handshakestate_start(this->handshake_); if (err != 0) { HANDSHAKE_STEP_LOG("noise_handshakestate_start", err); diff --git a/esphome/components/noise/noise_handshake.h b/esphome/components/noise/noise_handshake.h index bf1aa8cb7fb..ef556a26b00 100644 --- a/esphome/components/noise/noise_handshake.h +++ b/esphome/components/noise/noise_handshake.h @@ -37,7 +37,8 @@ class NoiseResponderHandshake { NoiseResponderHandshake &operator=(const NoiseResponderHandshake &) = delete; /// Create and start the handshake with the context's PSK and the prologue. - /// A repeated call frees the previous handshake state and starts over. + /// A repeated call frees the previous handshake state and starts over. A + /// spare ephemeral key, when one is ready, is used instead of generating. [[nodiscard]] int init(const NoiseContext &ctx, const uint8_t *prologue, size_t prologue_len); /// ACTION_FAILED is the catch-all: returned before init(), after split() /// has released the state, and when noise-c reports a failed handshake. diff --git a/esphome/core/defines.h b/esphome/core/defines.h index f845154389b..88f35dac46b 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -281,6 +281,7 @@ #define USE_IMPROV_NEXT_URL #define USE_MD5 #define USE_NOISE +#define USE_NOISE_SPARE_EPHEMERAL #define USE_SHA256 #ifndef USE_RP2 // no MQTT backend or esp_wireguard library on RP2 #define USE_MQTT diff --git a/tests/components/noise/__init__.py b/tests/components/noise/__init__.py index 60a5740a839..aeae988f442 100644 --- a/tests/components/noise/__init__.py +++ b/tests/components/noise/__init__.py @@ -1,3 +1,4 @@ +import esphome.codegen as cg from tests.testing_helpers import ComponentManifestOverride @@ -5,3 +6,10 @@ def override_manifest(manifest: ComponentManifestOverride) -> None: # to_code must run: it defines USE_NOISE and adds the noise-c library # the component sources under test need. manifest.enable_codegen() + real_to_code = manifest.to_code + + async def to_code_testing(config): + await real_to_code(config) + cg.add_define("USE_NOISE_SPARE_EPHEMERAL") + + manifest.to_code = to_code_testing diff --git a/tests/components/noise/test_noise_handshake.cpp b/tests/components/noise/test_noise_handshake.cpp index f2081f29650..f9548a61f0e 100644 --- a/tests/components/noise/test_noise_handshake.cpp +++ b/tests/components/noise/test_noise_handshake.cpp @@ -157,6 +157,72 @@ TEST(NoiseResponderHandshakeTest, FullHandshakeAndTransportRoundTrip) { noise_cipherstate_free(recv_cipher); } +// One full NNpsk0 handshake; responder_e gets the ephemeral public key the responder put on the wire, read +// before the initiator consumes the buffer in place +static void run_handshake(NoiseResponderHandshake &responder, uint8_t responder_e[SPARE_EPHEMERAL_KEY_SIZE]) { + const psk_t psk = make_psk(7); + ASSERT_EQ(responder.init(ctx_for(psk), PROLOGUE, sizeof(PROLOGUE)), 0); + Initiator initiator(psk, PROLOGUE, sizeof(PROLOGUE)); + uint8_t msg[MAX_HANDSHAKE_SIZE]; + size_t msg_len = initiator.write_message(msg, sizeof(msg)); + ASSERT_EQ(responder.read_message(msg, msg_len), 0); + size_t reply_len = 0; + ASSERT_EQ(responder.write_message(msg, sizeof(msg), reply_len), 0); + ASSERT_GE(reply_len, SPARE_EPHEMERAL_KEY_SIZE); + std::memcpy(responder_e, msg, SPARE_EPHEMERAL_KEY_SIZE); + ASSERT_EQ(initiator.read_message(msg, reply_len), 0); + ASSERT_EQ(responder.action(), Action::ACTION_SPLIT); +} + +TEST(SpareEphemeralTest, EmptySlotLeavesHandshakeToGenerate) { + ASSERT_FALSE(has_spare_ephemeral()); + NoiseResponderHandshake responder; + uint8_t responder_e[SPARE_EPHEMERAL_KEY_SIZE]; + run_handshake(responder, responder_e); + EXPECT_FALSE(has_spare_ephemeral()); +} + +TEST(SpareEphemeralTest, ConsumeHandsTheKeyToANewState) { + prepare_spare_ephemeral(); + ASSERT_TRUE(has_spare_ephemeral()); + const NoiseProtocolId nid = { + .prefix_id = NOISE_PREFIX_STANDARD, + .pattern_id = NOISE_PATTERN_NN, + .modifier_ids = {NOISE_MODIFIER_PSK0}, + .dh_id = NOISE_DH_CURVE25519, + .cipher_id = NOISE_CIPHER_CHACHAPOLY, + .hash_id = NOISE_HASH_SHA256, + .hybrid_id = NOISE_DH_NONE, + }; + NoiseHandshakeState *state = nullptr; + ASSERT_EQ(noise_handshakestate_new_by_id(&state, &nid, NOISE_ROLE_RESPONDER), 0); + const psk_t psk = make_psk(7); + ASSERT_EQ(noise_handshakestate_set_pre_shared_key(state, psk.data(), psk.size()), 0); + ASSERT_EQ(noise_handshakestate_set_prologue(state, PROLOGUE, sizeof(PROLOGUE)), 0); + EXPECT_EQ(consume_spare_ephemeral(state), 0); + EXPECT_FALSE(has_spare_ephemeral()); + noise_handshakestate_free(state); +} + +TEST(SpareEphemeralTest, SlotKeyIsOnTheWireAndConsumedOnce) { + prepare_spare_ephemeral(); + ASSERT_TRUE(has_spare_ephemeral()); + uint8_t expected_pub[SPARE_EPHEMERAL_KEY_SIZE]; + std::memcpy(expected_pub, spare_ephemeral + SPARE_EPHEMERAL_KEY_SIZE, sizeof(expected_pub)); + + NoiseResponderHandshake first; + uint8_t responder_e[SPARE_EPHEMERAL_KEY_SIZE]; + run_handshake(first, responder_e); + // The spare, not a generated key, went out; and it went out once + EXPECT_EQ(std::memcmp(responder_e, expected_pub, sizeof(expected_pub)), 0); + EXPECT_FALSE(has_spare_ephemeral()); + + NoiseResponderHandshake second; + run_handshake(second, responder_e); + EXPECT_NE(std::memcmp(responder_e, expected_pub, sizeof(expected_pub)), 0); + EXPECT_FALSE(has_spare_ephemeral()); +} + TEST(NoiseResponderHandshakeTest, ReInitRestartsHandshake) { // The documented retry shape: a repeated init() frees the previous state // and starts over. The first message under the new key authenticating From 54f6fb0eb2e3e42865ebf3a4b21eddf105e50725 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:25:48 -0500 Subject: [PATCH 106/133] [espidf] Skip ldgen when only app code changes (#18857) --- .github/workflows/ci-docker.yml | 1 + esphome/build_gen/espidf.py | 62 ++++++++- esphome/espidf/toolchain.py | 65 +++++++++- tests/unit_tests/build_gen/test_espidf.py | 47 +++++++ tests/unit_tests/test_espidf_toolchain.py | 148 ++++++++++++++++++++++ 5 files changed, 321 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 2784f135114..5bcdb22cb86 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -219,5 +219,6 @@ jobs: run: | docker run --rm \ -v "${{ github.workspace }}/docker/test_configs:/config" \ + -e ESPHOME_LDGEN_STRICT=1 \ "ghcr.io/esphome/esphome-amd64:${{ needs.check-docker.outputs.tag }}" \ compile "${{ matrix.id }}.yaml" diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 9f48013a4e8..124fa3ddb10 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -20,7 +20,7 @@ from esphome.framework_helpers import ( get_project_cxx_compile_flags, get_project_link_flags, ) -from esphome.helpers import mkdir_p, write_file_if_changed +from esphome.helpers import get_bool_env, mkdir_p, write_file_if_changed _LOGGER = logging.getLogger(__name__) @@ -35,6 +35,46 @@ list(FILTER esphome_cxx_compile_options EXCLUDE REGEX "^-std=") list(APPEND esphome_cxx_compile_options "-std={standard}") idf_build_set_property(CXX_COMPILE_OPTIONS "${{esphome_cxx_compile_options}}")""" +# Drops the app archive from ldgen's inputs so app-only edits skip the +# sections.ld regeneration. Safe: no mapping fragment references it +# (run_compile re-checks each build). Filters only the top-level call; +# the prior definition stays reachable with an underscore prefix. +_LDGEN_OVERRIDE = """\ +if(COMMAND __ldgen_get_lib_deps_of_target) + set_property(GLOBAL PROPERTY ESPHOME_LDGEN_ARMED 1) + function(__ldgen_get_lib_deps_of_target target out_list_var) + if(NOT COMMAND ___ldgen_get_lib_deps_of_target) + message(FATAL_ERROR "ESPHome ldgen override lost the original " + "implementation; set ESPHOME_LDGEN_FULL_DEPS=1 and rebuild.") + endif() + ___ldgen_get_lib_deps_of_target(${target} ${out_list_var}) + if(out_list_var STREQUAL "ldgen_libraries") + set_property(GLOBAL PROPERTY ESPHOME_LDGEN_FILTERED 1) + list(LENGTH ${out_list_var} esphome_ldgen_before) + list(REMOVE_ITEM ${out_list_var} idf::src __idf_src) + list(LENGTH ${out_list_var} esphome_ldgen_after) + if(esphome_ldgen_before EQUAL esphome_ldgen_after) + message(@SEVERITY@ "ESPHome ldgen app archive exclusion matched " + "nothing; app edits will regenerate sections.ld.") + endif() + endif() + set(${out_list_var} "${${out_list_var}}" PARENT_SCOPE) + endfunction() +else() + message(@MISSING@ "ESPHome ldgen override target not found; " + "app edits will regenerate sections.ld.") +endif()""" + +# Runs after project() so the walk has happened; catches the remaining +# silent path where the top-level out-var was renamed. +_LDGEN_OVERRIDE_CHECK = """\ +get_property(esphome_ldgen_armed GLOBAL PROPERTY ESPHOME_LDGEN_ARMED) +get_property(esphome_ldgen_filtered GLOBAL PROPERTY ESPHOME_LDGEN_FILTERED) +if(esphome_ldgen_armed AND NOT esphome_ldgen_filtered) + message(@SEVERITY@ "ESPHome ldgen override never filtered the app " + "archive; app edits will regenerate sections.ld.") +endif()""" + def get_available_components() -> list[str] | None: """List the built-in ESP-IDF components from ``project_description.json``. @@ -125,6 +165,22 @@ def get_project_cmakelists( else "" ) + # Stops the ~3s sections.ld regeneration on app-only edits; see + # _LDGEN_OVERRIDE. ESPHOME_LDGEN_FULL_DEPS=1 restores stock behavior; + # ESPHOME_LDGEN_STRICT=1 (CI) fails the configure when an IDF bump + # breaks the override instead of degrading to stock deps. + if get_bool_env("ESPHOME_LDGEN_FULL_DEPS"): + ldgen_override = "" + ldgen_override_check = "" + else: + strict = get_bool_env("ESPHOME_LDGEN_STRICT") + severity = "FATAL_ERROR" if strict else "WARNING" + missing = "FATAL_ERROR" if strict else "STATUS" + ldgen_override = _LDGEN_OVERRIDE.replace("@SEVERITY@", severity).replace( + "@MISSING@", missing + ) + ldgen_override_check = _LDGEN_OVERRIDE_CHECK.replace("@SEVERITY@", severity) + # CMake variables registered via cg.add_cmake_arg(). Emitted before # include(project.cmake) so values like EXCLUDE_COMPONENTS are already # set when project.cmake seeds the component list, and on minimal @@ -202,6 +258,8 @@ set(EXTRA_COMPONENT_DIRS ${{CMAKE_SOURCE_DIR}}/src) include($ENV{{IDF_PATH}}/tools/cmake/project.cmake) +{ldgen_override} + {cpp_standard_options} {cxx_compile_options} @@ -214,6 +272,8 @@ include($ENV{{IDF_PATH}}/tools/cmake/project.cmake) project({CORE.name}) +{ldgen_override_check} + # Emit per-memory-type JSON size data for ESPHome to read post-build. # json2 stays small; raw dumps every symbol (~2s on a large map) and # this command runs inside the link edge, blocking everything downstream. diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index 4243eff0ba2..d73dbf6b7e5 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -1,6 +1,7 @@ """ESP-IDF direct build API for ESPHome.""" from dataclasses import dataclass, field +import fnmatch import hashlib import json import logging @@ -25,7 +26,7 @@ from esphome.core import CORE, EsphomeError from esphome.espidf import variant_to_idf_target from esphome.espidf.framework import check_esp_idf_install, get_framework_env from esphome.espidf.size_summary import print_summary -from esphome.helpers import add_git_ceiling_directory, write_file +from esphome.helpers import add_git_ceiling_directory, get_bool_env, write_file _LOGGER = logging.getLogger(__name__) @@ -647,6 +648,65 @@ def _patch_memory_segments(): _LOGGER.warning("Could not patch memory segments in %s", memory_ld) +_LDGEN_FRAGMENTS_RE = re.compile(r'--fragments-list\s+"([^"]+)"') +_LDGEN_ARCHIVE_RE = re.compile(r"^\s*archive:\s*(\S+)", re.MULTILINE) + + +def _fragment_maps_app_archive(text: str) -> bool: + """True when an archive: spec selects libsrc.a, the archive of the src + component excluded as idf::src/__idf_src in build_gen/espidf.py. + + The bare * is IDF's stock catch-all; its archive-level entries resolve + in the linker against all link inputs, so it stays safe when the + archive is excluded from ldgen's own inputs. + """ + return any( + value != "*" and fnmatch.fnmatch("libsrc.a", value) + for value in _LDGEN_ARCHIVE_RE.findall(text) + ) + + +def _ldgen_check_skip(msg: str, strict: bool) -> None: + """A skipped fragment check is debug for users, fatal under strict.""" + if strict: + raise EsphomeError(f"ldgen fragment check: {msg} (ESPHOME_LDGEN_STRICT)") + _LOGGER.debug("Skipping ldgen fragment check: %s", msg) + + +def _warn_if_app_archive_mapped() -> None: + """Belt for the ldgen exclusion (see build_gen/espidf.py): warn if any + linker fragment names the app archive, since ldgen would silently skip + remapping it rather than fail. + """ + strict = get_bool_env("ESPHOME_LDGEN_STRICT") + build_ninja = CORE.relative_build_path("build", "build.ninja") + try: + ninja_text = build_ninja.read_text(encoding="utf-8", errors="replace") + except OSError as e: + _ldgen_check_skip(f"could not read {build_ninja}: {e}", strict) + return + match = _LDGEN_FRAGMENTS_RE.search(ninja_text) + if match is None: + _ldgen_check_skip(f"no --fragments-list in {build_ninja}", strict) + return + for fragment in match.group(1).split(";"): + try: + text = Path(fragment).read_text(encoding="utf-8", errors="replace") + except OSError as e: + _ldgen_check_skip(f"could not read {fragment}: {e}", strict) + continue + if _fragment_maps_app_archive(text): + msg = ( + f"Linker fragment {fragment} maps the app archive; its " + "entries may be skipped. Set ESPHOME_LDGEN_FULL_DEPS=1 " + "and rebuild." + ) + if strict: + raise EsphomeError(msg) + _LOGGER.warning("%s", msg) + return + + def run_compile(config, verbose: bool) -> int: """Compile the ESP-IDF project. @@ -677,6 +737,9 @@ def run_compile(config, verbose: bool) -> int: else: _LOGGER.info("Build configuration is up to date") + if not get_bool_env("ESPHOME_LDGEN_FULL_DEPS"): + _warn_if_app_archive_mapped() + # In testing mode, generate the linker script first, patch DRAM/IRAM sizes, # then build. memory.ld is regenerated by ninja during the build phase, # so we must patch after it's generated but before linking (same timing diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index 4a842847962..7d53ee1e5e0 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -163,6 +163,53 @@ def test_has_discovered_components_after_configure(tmp_path: Path) -> None: assert has_discovered_components() +@pytest.mark.parametrize("minimal", [False, True]) +def test_get_project_cmakelists_emits_ldgen_override( + minimal: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + """Both renders override the ldgen dep walker to drop the app archive, + after include(project.cmake) which defines the original.""" + monkeypatch.delenv("ESPHOME_LDGEN_FULL_DEPS", raising=False) + monkeypatch.delenv("ESPHOME_LDGEN_STRICT", raising=False) + content = _render(minimal=minimal) + assert "REMOVE_ITEM ${out_list_var} idf::src __idf_src" in content + # Quoted so spaced elements survive and an empty list stays defined + assert 'set(${out_list_var} "${${out_list_var}}" PARENT_SCOPE)' in content + assert 'message(WARNING "ESPHome ldgen app archive exclusion' in content + assert 'message(STATUS "ESPHome ldgen override target not found' in content + assert 'message(WARNING "ESPHome ldgen override never filtered' in content + assert content.index("tools/cmake/project.cmake") < content.index( + "function(__ldgen_get_lib_deps_of_target" + ) + # The never-filtered check must run after project() has walked the deps + assert content.index("project(test)") < content.index("esphome_ldgen_armed GLOBAL") + + +def test_get_project_cmakelists_ldgen_strict_fails_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ESPHOME_LDGEN_STRICT turns both degradation paths into hard errors so + CI fails right away when an IDF bump breaks the override.""" + monkeypatch.delenv("ESPHOME_LDGEN_FULL_DEPS", raising=False) + monkeypatch.setenv("ESPHOME_LDGEN_STRICT", "1") + content = _render() + assert 'message(FATAL_ERROR "ESPHome ldgen app archive exclusion' in content + assert 'message(FATAL_ERROR "ESPHome ldgen override target not found' in content + assert 'message(FATAL_ERROR "ESPHome ldgen override never filtered' in content + assert "@SEVERITY@" not in content + assert "@MISSING@" not in content + + +def test_get_project_cmakelists_ldgen_full_deps_escape_hatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ESPHOME_LDGEN_FULL_DEPS restores stock ldgen behavior.""" + monkeypatch.setenv("ESPHOME_LDGEN_FULL_DEPS", "true") + content = _render() + assert "__ldgen_get_lib_deps_of_target" not in content + assert "esphome_ldgen_armed" not in content + + def test_get_project_cmakelists_size_command_uses_json2() -> None: """The POST_BUILD size command uses the cheap json2 format, with --ng only on the 1.x tool bundled with IDF < 6.""" diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index c012785e892..1b40460b919 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -630,6 +630,154 @@ def test_component_cache_ignores_corrupt_file(setup_core: Path, tmp_path: Path) assert toolchain.load_cached_builtin_components() is None +@pytest.fixture(autouse=True) +def _clear_ldgen_env(monkeypatch: pytest.MonkeyPatch) -> None: + """Isolate tests from ambient ldgen escape hatch and strict knobs.""" + monkeypatch.delenv("ESPHOME_LDGEN_STRICT", raising=False) + monkeypatch.delenv("ESPHOME_LDGEN_FULL_DEPS", raising=False) + + +def _write_fragments_build_ninja(tmp_path: Path, fragments: list[Path]) -> None: + build_dir = CORE.relative_build_path("build") + build_dir.mkdir(parents=True, exist_ok=True) + frag_list = ";".join(str(f) for f in fragments) + (build_dir / "build.ninja").write_text( + f' COMMAND = python ldgen.py --fragments-list "{frag_list}" --input x\n' + ) + + +def test_warn_if_app_archive_mapped_warns( + setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A fragment naming the app archive, even with trailing text or leading + whitespace, triggers the loud warning.""" + _setup_build(setup_core) + frag = tmp_path / "linker.lf" + frag.write_text("[mapping:evil]\n archive: libsrc.a # app\nentries:\n") + _write_fragments_build_ninja(tmp_path, [frag]) + toolchain._warn_if_app_archive_mapped() + assert "maps the app archive" in caplog.text + + +def test_warn_if_app_archive_mapped_scans_past_unreadable( + setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An unreadable fragment doesn't stop later fragments being checked.""" + _setup_build(setup_core) + frag = tmp_path / "linker.lf" + frag.write_text("[mapping:evil]\narchive: libsrc.a\n") + _write_fragments_build_ninja(tmp_path, [tmp_path / "missing.lf", frag]) + toolchain._warn_if_app_archive_mapped() + assert "maps the app archive" in caplog.text + + +def test_warn_if_app_archive_mapped_strict_no_fragments_list( + setup_core: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Under strict, a build.ninja the check can't parse fails the build.""" + monkeypatch.setenv("ESPHOME_LDGEN_STRICT", "1") + _setup_build(setup_core) + build_dir = CORE.relative_build_path("build") + build_dir.mkdir(parents=True, exist_ok=True) + (build_dir / "build.ninja").write_text("rule CXX\n command = gcc\n") + with pytest.raises(EsphomeError, match="no --fragments-list"): + toolchain._warn_if_app_archive_mapped() + + +def test_warn_if_app_archive_mapped_strict_raises( + setup_core: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Under ESPHOME_LDGEN_STRICT a mapped app archive fails the build.""" + monkeypatch.setenv("ESPHOME_LDGEN_STRICT", "1") + _setup_build(setup_core) + frag = tmp_path / "linker.lf" + frag.write_text("[mapping:evil]\narchive: libsrc.a\n") + _write_fragments_build_ninja(tmp_path, [frag]) + with pytest.raises(EsphomeError, match="maps the app archive"): + toolchain._warn_if_app_archive_mapped() + + +def test_warn_if_app_archive_mapped_glob( + setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A glob archive spec that selects the app archive is also flagged.""" + _setup_build(setup_core) + frag = tmp_path / "linker.lf" + frag.write_text("[mapping:evil]\narchive: lib*\n") + _write_fragments_build_ninja(tmp_path, [frag]) + toolchain._warn_if_app_archive_mapped() + assert "maps the app archive" in caplog.text + + +def test_warn_if_app_archive_mapped_clean( + setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Normal fragments, including IDF's stock archive: * catch-all, + produce no warning.""" + _setup_build(setup_core) + frag = tmp_path / "linker.lf" + frag.write_text( + "[mapping:freertos]\narchive: libfreertos.a\n[mapping:default]\narchive: *\n" + ) + _write_fragments_build_ninja(tmp_path, [frag]) + toolchain._warn_if_app_archive_mapped() + assert "maps the app archive" not in caplog.text + + +def test_warn_if_app_archive_mapped_missing_fragment( + setup_core: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An unreadable fragment file is non-fatal.""" + _setup_build(setup_core) + _write_fragments_build_ninja(tmp_path, [tmp_path / "missing.lf"]) + toolchain._warn_if_app_archive_mapped() + assert "maps the app archive" not in caplog.text + + +def test_warn_if_app_archive_mapped_no_build_ninja(setup_core: Path) -> None: + """No build.ninja yet is a quiet no-op.""" + _setup_build(setup_core) + toolchain._warn_if_app_archive_mapped() + + +def test_warn_if_app_archive_mapped_no_fragments_list(setup_core: Path) -> None: + """A build.ninja without a fragments-list argument is a quiet no-op.""" + _setup_build(setup_core) + build_dir = CORE.relative_build_path("build") + build_dir.mkdir(parents=True, exist_ok=True) + (build_dir / "build.ninja").write_text("rule CXX\n command = gcc\n") + toolchain._warn_if_app_archive_mapped() + + +def test_run_compile_runs_fragment_check(setup_core: Path) -> None: + """The fragment belt runs by default on every compile.""" + _setup_build(setup_core) + + with ( + _up_to_date_compile(), + patch.object(toolchain, "_warn_if_app_archive_mapped") as mock_check, + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + + mock_check.assert_called_once() + + +def test_run_compile_full_deps_skips_fragment_check( + setup_core: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ESPHOME_LDGEN_FULL_DEPS disables the fragment belt with the override.""" + monkeypatch.setenv("ESPHOME_LDGEN_FULL_DEPS", "1") + _setup_build(setup_core) + + with ( + _up_to_date_compile(), + patch.object(toolchain, "_warn_if_app_archive_mapped") as mock_check, + ): + assert toolchain.run_compile({CONF_ESPHOME: {}}, verbose=False) == 0 + + mock_check.assert_not_called() + + @pytest.mark.parametrize("limit", [1, None]) def test_run_compile_passes_compile_process_limit( setup_core: Path, limit: int | None From eb423be9d25a895cbd18e84c99fae3e63fa2ceef Mon Sep 17 00:00:00 2001 From: Bascht74 <66269310+Bascht74@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:36:09 +0200 Subject: [PATCH 107/133] [socket] Add an IPv4 lookup next to set_sockaddr (#19909) Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: J. Nick Koston Co-authored-by: J. Nick Koston --- esphome/components/socket/ipv4_resolve.cpp | 205 ++++++++++++++++++ esphome/components/socket/ipv4_resolve.h | 103 +++++++++ .../socket/test_ipv4_resolve_host.cpp | 71 ++++++ .../ipv4_resolve_test_component/__init__.py | 22 ++ .../ipv4_resolve_test_component.cpp | 46 ++++ .../ipv4_resolve_test_component.h | 12 + .../fixtures/socket_ipv4_resolve.yaml | 17 ++ tests/integration/test_socket_ipv4_resolve.py | 51 +++++ 8 files changed, 527 insertions(+) create mode 100644 esphome/components/socket/ipv4_resolve.cpp create mode 100644 esphome/components/socket/ipv4_resolve.h create mode 100644 tests/components/socket/test_ipv4_resolve_host.cpp create mode 100644 tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py create mode 100644 tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.cpp create mode 100644 tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.h create mode 100644 tests/integration/fixtures/socket_ipv4_resolve.yaml create mode 100644 tests/integration/test_socket_ipv4_resolve.py diff --git a/esphome/components/socket/ipv4_resolve.cpp b/esphome/components/socket/ipv4_resolve.cpp new file mode 100644 index 00000000000..d2316cd064b --- /dev/null +++ b/esphome/components/socket/ipv4_resolve.cpp @@ -0,0 +1,205 @@ +#include "ipv4_resolve.h" + +#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) + +#include "socket.h" +#include "esphome/core/helpers.h" +#include "esphome/core/log.h" + +#include + +#if !defined(USE_HOST) && !defined(USE_ZEPHYR) +#include "lwip/dns.h" +#else +#include +#endif + +namespace esphome::socket { + +static const char *const TAG = "socket"; + +bool Ipv4Resolve::consume_failure() { +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + uint8_t expected = STATE_FAILED; + return this->state_word_.compare_exchange_strong(expected, STATE_IDLE); +#else + if (this->state_word_ != STATE_FAILED) { + return false; + } + this->state_word_ = STATE_IDLE; + return true; +#endif +} + +void Ipv4Resolve::forget() { +#if defined(IPV4_RESOLVE_VOLATILE_EPOCH) + // The generation moves first; RESOLVING stays set until the callback + // drops the stale result, so start() cannot replace that lookup. + this->epoch_ = this->epoch_ + 1; + this->addr_word_ = 0; + if (this->state_word_ != STATE_RESOLVING) { + this->state_word_ = STATE_IDLE; + } +#elif defined(IPV4_RESOLVE_ATOMIC_STATE) + // PUBLISHING stays, so start() cannot queue a second callback + // while this one is storing the address. + uint8_t expected = STATE_RESOLVING; + if (this->state_word_.compare_exchange_strong(expected, STATE_IDLE)) { + this->set_addr_(0); + return; + } + if (this->state_() == STATE_PUBLISHING) { + return; + } + this->set_state_(STATE_IDLE); + this->set_addr_(0); +#else + this->set_state_(STATE_IDLE); + this->set_addr_(0); +#endif +} + +socklen_t Ipv4Resolve::to_sockaddr(struct sockaddr *dest, socklen_t destlen, uint16_t port) const { + if (this->state_() != STATE_RESOLVED || destlen < sizeof(sockaddr_in)) { + return 0; + } + auto *in = reinterpret_cast(dest); + memset(in, 0, sizeof(sockaddr_in)); + in->sin_family = AF_INET; + in->sin_port = htons(port); + in->sin_addr.s_addr = this->addr_(); + return sizeof(sockaddr_in); +} + +#if !defined(USE_HOST) && !defined(USE_ZEPHYR) +#if defined(IPV4_RESOLVE_VOLATILE_EPOCH) +bool Ipv4Resolve::drop_stale_(uint32_t expected) { + if (this->epoch_ == expected) { + return false; + } + this->state_word_ = STATE_IDLE; + this->addr_word_ = 0; + return true; +} +#endif + +void Ipv4Resolve::dns_found(const char *name, const ip_addr_t *addr, void *arg) { + auto *self = static_cast(arg); +#if defined(IPV4_RESOLVE_VOLATILE_EPOCH) + const uint32_t expected = self->pending_epoch_; + if (self->drop_stale_(expected)) { + return; + } +#endif + if (addr != nullptr && IP_IS_V4(addr)) { +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + // Only the callback that wins the exchange owns the lookup. + uint8_t expected = STATE_RESOLVING; + if (!self->state_word_.compare_exchange_strong(expected, STATE_PUBLISHING)) { + return; + } + self->set_addr_(ip4_addr_get_u32(ip_2_ip4(addr))); + expected = STATE_PUBLISHING; + // Lost ownership; the leftover address is gated by to_sockaddr()'s state check. + if (!self->state_word_.compare_exchange_strong(expected, STATE_RESOLVED)) { + return; + } +#elif defined(IPV4_RESOLVE_VOLATILE_EPOCH) + self->addr_word_ = ip4_addr_get_u32(ip_2_ip4(addr)); + self->state_word_ = STATE_RESOLVED; + if (self->drop_stale_(expected)) { + return; + } +#else + // Do not publish over a newer start(). + if (self->state_() != STATE_RESOLVING) { + return; + } + self->set_addr_(ip4_addr_get_u32(ip_2_ip4(addr))); + self->set_state_(STATE_RESOLVED); +#endif + } else { + ESP_LOGW(self->tag_ != nullptr ? self->tag_ : TAG, "DNS failed for %s", name); +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + uint8_t expected = STATE_RESOLVING; + self->state_word_.compare_exchange_strong(expected, STATE_FAILED); +#elif defined(IPV4_RESOLVE_VOLATILE_EPOCH) + self->state_word_ = STATE_FAILED; + if (self->drop_stale_(expected)) { + return; + } +#else + if (self->state_() != STATE_RESOLVING) { + return; + } + self->set_state_(STATE_FAILED); +#endif + } +} +#endif + +void Ipv4Resolve::start(const char *host, uint16_t port, const char *tag) { + const uint8_t state = this->state_(); + if (state == STATE_RESOLVED || state == STATE_RESOLVING || state == STATE_PUBLISHING) { + return; + } + this->set_state_(STATE_IDLE); + this->tag_ = tag; + struct sockaddr_storage literal; + if (set_sockaddr(reinterpret_cast(&literal), sizeof(literal), host, port) != 0) { + if (literal.ss_family == AF_INET) { + auto *in = reinterpret_cast(&literal); + this->set_addr_(in->sin_addr.s_addr); + this->set_state_(STATE_RESOLVED); + return; + } + this->set_state_(STATE_FAILED); + ESP_LOGW(tag, "Not an IPv4 address: %s", host); + return; + } +#if !defined(USE_HOST) && !defined(USE_ZEPHYR) + ip_addr_t cached; + err_t err; + { + LwIPLock lock; +#if defined(IPV4_RESOLVE_VOLATILE_EPOCH) + this->pending_epoch_ = this->epoch_; +#endif + this->set_state_(STATE_RESOLVING); + err = dns_gethostbyname_addrtype(host, &cached, &Ipv4Resolve::dns_found, this, LWIP_DNS_ADDRTYPE_IPV4); + if (err != ERR_INPROGRESS && this->state_() == STATE_RESOLVING) { + this->set_state_(STATE_IDLE); + } + } + if (err == ERR_OK && IP_IS_V4(&cached)) { + this->set_addr_(ip4_addr_get_u32(ip_2_ip4(&cached))); + this->set_state_(STATE_RESOLVED); + return; + } + if (err == ERR_INPROGRESS || this->state_() == STATE_RESOLVED || this->state_() == STATE_PUBLISHING) { + return; + } +#else + struct addrinfo hints {}; + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + struct addrinfo *res = nullptr; + if (getaddrinfo(host, nullptr, &hints, &res) == 0 && res != nullptr) { + auto *in = reinterpret_cast(res->ai_addr); + if (res->ai_family == AF_INET) { + this->set_addr_(in->sin_addr.s_addr); + this->set_state_(STATE_RESOLVED); + } + freeaddrinfo(res); + if (this->ready()) { + return; + } + } +#endif + this->set_state_(STATE_FAILED); + ESP_LOGW(tag, "Could not resolve %s", host); +} + +} // namespace esphome::socket + +#endif diff --git a/esphome/components/socket/ipv4_resolve.h b/esphome/components/socket/ipv4_resolve.h new file mode 100644 index 00000000000..628efb32d9c --- /dev/null +++ b/esphome/components/socket/ipv4_resolve.h @@ -0,0 +1,103 @@ +#pragma once + +#include "headers.h" + +#if defined(USE_SOCKET_IMPL_LWIP_TCP) || defined(USE_SOCKET_IMPL_LWIP_SOCKETS) || defined(USE_SOCKET_IMPL_BSD_SOCKETS) + +#include + +// MULTI_ATOMICS: one atomic state word, races closed by compare_exchange. +// SINGLE: volatile state, the callback never runs beside loop(). +// MULTI_NO_ATOMICS: volatile state plus a generation; BK72xx has no +// compare_exchange and the DNS callback runs on the tcpip thread. +#if defined(ESPHOME_THREAD_MULTI_NO_ATOMICS) +#define IPV4_RESOLVE_VOLATILE_EPOCH +#elif defined(ESPHOME_THREAD_SINGLE) +#define IPV4_RESOLVE_VOLATILE +#else +#define IPV4_RESOLVE_ATOMIC_STATE +#include +#endif + +#if !defined(USE_HOST) && !defined(USE_ZEPHYR) +#include "lwip/ip_addr.h" +#endif + +namespace esphome::socket { + +/// One IPv4 literal or hostname. Must outlive a pending lookup. +class Ipv4Resolve { + public: + static constexpr uint8_t STATE_IDLE = 0; + static constexpr uint8_t STATE_RESOLVING = 1; + static constexpr uint8_t STATE_RESOLVED = 2; + static constexpr uint8_t STATE_FAILED = 3; + // The callback holds this between winning the lookup and storing the address. + static constexpr uint8_t STATE_PUBLISHING = 4; + + /// Drop the stored address so the next start() resolves again. + /// A result already publishing may still land, so ready() can be true + /// right after this; after changing hosts, forget() until ready() is false. + void forget(); + /// Drop a failed lookup so the next start() tries again. + bool consume_failure(); + bool ready() const { return this->state_() == STATE_RESOLVED; } + /// Write the stored address into dest. Returns 0 until ready() is true. + socklen_t to_sockaddr(struct sockaddr *dest, socklen_t destlen, uint16_t port) const; + /// Resolve host; tag names the failure log. On host and Zephyr this + /// blocks in getaddrinfo(). + void start(const char *host, uint16_t port, const char *tag); + + private: +#if !defined(USE_HOST) && !defined(USE_ZEPHYR) + static void dns_found(const char *name, const ip_addr_t *addr, void *arg); +#if defined(IPV4_RESOLVE_VOLATILE_EPOCH) + bool drop_stale_(uint32_t expected); +#endif +#endif + uint8_t state_() const { +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + return this->state_word_.load(); +#else + return this->state_word_; +#endif + } + uint32_t addr_() const { +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + return this->addr_word_.load(); +#else + return this->addr_word_; +#endif + } + void set_state_(uint8_t state) { +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + this->state_word_.store(state); +#else + this->state_word_ = state; +#endif + } + void set_addr_(uint32_t addr) { +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + this->addr_word_.store(addr); +#else + this->addr_word_ = addr; +#endif + } + const char *tag_{nullptr}; +#if defined(IPV4_RESOLVE_ATOMIC_STATE) + std::atomic addr_word_{0}; + std::atomic state_word_{STATE_IDLE}; +#elif defined(IPV4_RESOLVE_VOLATILE_EPOCH) + volatile uint32_t addr_word_{0}; + volatile uint32_t epoch_{0}; + volatile uint32_t pending_epoch_{0}; + volatile uint8_t state_word_{STATE_IDLE}; +#else + volatile uint32_t addr_word_{0}; + volatile uint8_t state_word_{STATE_IDLE}; +#endif +}; + +} // namespace esphome::socket + +#endif diff --git a/tests/components/socket/test_ipv4_resolve_host.cpp b/tests/components/socket/test_ipv4_resolve_host.cpp new file mode 100644 index 00000000000..6381135c405 --- /dev/null +++ b/tests/components/socket/test_ipv4_resolve_host.cpp @@ -0,0 +1,71 @@ +#include + +#include + +#include "esphome/components/socket/ipv4_resolve.h" + +#ifdef USE_HOST + +namespace esphome::socket::testing { + +TEST(Ipv4Resolve, LiteralIsReadyWithoutDns) { + Ipv4Resolve lookup; + lookup.start("192.168.1.1", 1, "test"); + EXPECT_TRUE(lookup.ready()); + + struct sockaddr_storage addr {}; + socklen_t len = lookup.to_sockaddr(reinterpret_cast(&addr), sizeof(addr), 6053); + ASSERT_EQ(len, sizeof(sockaddr_in)); + auto *in = reinterpret_cast(&addr); + EXPECT_EQ(in->sin_family, AF_INET); + EXPECT_EQ(ntohs(in->sin_port), 6053); + EXPECT_EQ(in->sin_addr.s_addr, htonl(0xC0A80101)); +} + +TEST(Ipv4Resolve, ForgetDropsTheLiteral) { + Ipv4Resolve lookup; + lookup.start("10.0.0.5", 80, "test"); + ASSERT_TRUE(lookup.ready()); + lookup.forget(); + EXPECT_FALSE(lookup.ready()); + + struct sockaddr_storage addr {}; + EXPECT_EQ(lookup.to_sockaddr(reinterpret_cast(&addr), sizeof(addr), 80), 0u); +} + +TEST(Ipv4Resolve, Ipv6LiteralIsRejected) { + Ipv4Resolve lookup; + lookup.start("::1", 443, "test"); + EXPECT_FALSE(lookup.ready()); + EXPECT_TRUE(lookup.consume_failure()); +} + +TEST(Ipv4Resolve, ShortBufferWritesNothing) { + Ipv4Resolve lookup; + lookup.start("192.0.2.10", 502, "test"); + ASSERT_TRUE(lookup.ready()); + struct sockaddr_in addr {}; + EXPECT_EQ(lookup.to_sockaddr(reinterpret_cast(&addr), sizeof(addr) - 1, 502), 0u); +} + +TEST(Ipv4Resolve, RetryAfterFailureKeepsTheAddress) { + Ipv4Resolve lookup; + lookup.start("::1", 443, "test"); + EXPECT_FALSE(lookup.ready()); + lookup.start("192.168.1.1", 1, "test"); + EXPECT_TRUE(lookup.ready()); + EXPECT_FALSE(lookup.consume_failure()); + EXPECT_TRUE(lookup.ready()); +} + +TEST(Ipv4Resolve, ForgetDropsAStaleFailure) { + Ipv4Resolve lookup; + lookup.start("::1", 443, "test"); + lookup.forget(); + EXPECT_FALSE(lookup.consume_failure()); + EXPECT_FALSE(lookup.ready()); +} + +} // namespace esphome::socket::testing + +#endif diff --git a/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py new file mode 100644 index 00000000000..3e397d6186d --- /dev/null +++ b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/__init__.py @@ -0,0 +1,22 @@ +import esphome.codegen as cg +import esphome.config_validation as cv +from esphome.const import CONF_ID +from esphome.types import ConfigType + +AUTO_LOAD = ["socket"] + +ipv4_resolve_test_component_ns = cg.esphome_ns.namespace("ipv4_resolve_test_component") +Ipv4ResolveTestComponent = ipv4_resolve_test_component_ns.class_( + "Ipv4ResolveTestComponent", cg.Component +) + +CONFIG_SCHEMA = cv.Schema( + { + cv.GenerateID(): cv.declare_id(Ipv4ResolveTestComponent), + } +).extend(cv.COMPONENT_SCHEMA) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) diff --git a/tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.cpp b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.cpp new file mode 100644 index 00000000000..e4a6e914e89 --- /dev/null +++ b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.cpp @@ -0,0 +1,46 @@ +#include "ipv4_resolve_test_component.h" +#include "esphome/components/socket/ipv4_resolve.h" +#include "esphome/core/log.h" + +#include + +namespace esphome::ipv4_resolve_test_component { + +static const char *const TAG = "ipv4_resolve_test"; + +static bool check_sockaddr(socket::Ipv4Resolve &lookup, uint16_t port, uint32_t expected) { + struct sockaddr_storage addr {}; + socklen_t len = lookup.to_sockaddr(reinterpret_cast(&addr), sizeof(addr), port); + if (len != sizeof(sockaddr_in)) { + return false; + } + auto *in = reinterpret_cast(&addr); + return in->sin_family == AF_INET && ntohs(in->sin_port) == port && in->sin_addr.s_addr == htonl(expected); +} + +void Ipv4ResolveTestComponent::setup() { + ESP_LOGI(TAG, "IPv4 resolve test starting"); + + socket::Ipv4Resolve lookup; + struct sockaddr_storage addr {}; + + lookup.start("192.168.1.1", 1, TAG); + bool ok = lookup.ready() && check_sockaddr(lookup, 6053, 0xC0A80101); + ESP_LOGI(TAG, "Literal resolve: %s", ok ? LOG_STR_LITERAL("PASSED") : LOG_STR_LITERAL("FAILED")); + + lookup.forget(); + ok = !lookup.ready() && lookup.to_sockaddr(reinterpret_cast(&addr), sizeof(addr), 80) == 0; + ESP_LOGI(TAG, "Forget drops address: %s", ok ? LOG_STR_LITERAL("PASSED") : LOG_STR_LITERAL("FAILED")); + + lookup.start("::1", 443, TAG); + ok = !lookup.ready() && lookup.consume_failure(); + ESP_LOGI(TAG, "IPv6 literal rejected: %s", ok ? LOG_STR_LITERAL("PASSED") : LOG_STR_LITERAL("FAILED")); + + lookup.start("localhost", 6053, TAG); + ok = lookup.ready() && check_sockaddr(lookup, 6053, 0x7F000001); + ESP_LOGI(TAG, "Hostname resolve: %s", ok ? LOG_STR_LITERAL("PASSED") : LOG_STR_LITERAL("FAILED")); + + ESP_LOGI(TAG, "IPv4 resolve test complete"); +} + +} // namespace esphome::ipv4_resolve_test_component diff --git a/tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.h b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.h new file mode 100644 index 00000000000..b223f228e5f --- /dev/null +++ b/tests/integration/fixtures/external_components/ipv4_resolve_test_component/ipv4_resolve_test_component.h @@ -0,0 +1,12 @@ +#pragma once + +#include "esphome/core/component.h" + +namespace esphome::ipv4_resolve_test_component { + +class Ipv4ResolveTestComponent : public Component { + public: + void setup() override; +}; + +} // namespace esphome::ipv4_resolve_test_component diff --git a/tests/integration/fixtures/socket_ipv4_resolve.yaml b/tests/integration/fixtures/socket_ipv4_resolve.yaml new file mode 100644 index 00000000000..14931d0918b --- /dev/null +++ b/tests/integration/fixtures/socket_ipv4_resolve.yaml @@ -0,0 +1,17 @@ +esphome: + name: socket-ipv4-resolve-test + +host: + +api: + +logger: + level: INFO + +external_components: + - source: + type: local + path: EXTERNAL_COMPONENT_PATH + components: [ipv4_resolve_test_component] + +ipv4_resolve_test_component: diff --git a/tests/integration/test_socket_ipv4_resolve.py b/tests/integration/test_socket_ipv4_resolve.py new file mode 100644 index 00000000000..5d7bd5a100f --- /dev/null +++ b/tests/integration/test_socket_ipv4_resolve.py @@ -0,0 +1,51 @@ +"""Integration test for the socket Ipv4Resolve helper on host.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from .types import APIClientConnectedFactory, RunCompiledFunction + +CHECKS = ( + "Literal resolve", + "Forget drops address", + "IPv6 literal rejected", + "Hostname resolve", +) + + +@pytest.mark.asyncio +async def test_socket_ipv4_resolve( + yaml_config: str, + run_compiled: RunCompiledFunction, + api_client_connected: APIClientConnectedFactory, +) -> None: + """Exercise Ipv4Resolve literals, forget, failure, and getaddrinfo on host.""" + test_complete = asyncio.Event() + results: dict[str, bool] = {} + + def on_log_line(line: str) -> None: + if "IPv4 resolve test complete" in line: + test_complete.set() + return + for check in CHECKS: + if f"{check}:" in line: + results[check] = "PASSED" in line + + async with ( + run_compiled(yaml_config, line_callback=on_log_line), + api_client_connected() as client, + ): + device_info = await client.device_info() + assert device_info is not None + assert device_info.name == "socket-ipv4-resolve-test" + + try: + await asyncio.wait_for(test_complete.wait(), timeout=10.0) + except TimeoutError: + pytest.fail("IPv4 resolve test timed out") + + for check in CHECKS: + assert results.get(check), f"{check} check failed or never ran" From 9a855b43773d193e318dcd48c4bf836ba46dba2f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:36:37 -0500 Subject: [PATCH 108/133] [ina260] Use the sub_* helpers for optional child entities (#19977) --- esphome/components/ina260/sensor.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/esphome/components/ina260/sensor.py b/esphome/components/ina260/sensor.py index b7b94a248b4..20cf1fee45f 100644 --- a/esphome/components/ina260/sensor.py +++ b/esphome/components/ina260/sensor.py @@ -59,14 +59,7 @@ async def to_code(config: ConfigType) -> None: await cg.register_component(var, config) await i2c.register_i2c_device(var, config) - if CONF_BUS_VOLTAGE in config: - sens = await sensor.new_sensor(config[CONF_BUS_VOLTAGE]) - cg.add(var.set_bus_voltage_sensor(sens)) - - if CONF_CURRENT in config: - sens = await sensor.new_sensor(config[CONF_CURRENT]) - cg.add(var.set_current_sensor(sens)) - - if CONF_POWER in config: - sens = await sensor.new_sensor(config[CONF_POWER]) - cg.add(var.set_power_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_BUS_VOLTAGE, var.set_bus_voltage_sensor) + await sensors(CONF_CURRENT, var.set_current_sensor) + await sensors(CONF_POWER, var.set_power_sensor) From 852fb0c5a5279defb5aae7941590d48a4842eb06 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:37:09 -0500 Subject: [PATCH 109/133] [midea] Use the sub_* helpers for optional child entities (#19976) --- esphome/components/midea/climate.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/esphome/components/midea/climate.py b/esphome/components/midea/climate.py index e7a684babea..da952ab87b0 100644 --- a/esphome/components/midea/climate.py +++ b/esphome/components/midea/climate.py @@ -214,15 +214,10 @@ async def to_code(config): cg.add(var.set_custom_presets(config[CONF_CUSTOM_PRESETS])) if CONF_CUSTOM_FAN_MODES in config: cg.add(var.set_custom_fan_modes(config[CONF_CUSTOM_FAN_MODES])) - if CONF_OUTDOOR_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_OUTDOOR_TEMPERATURE]) - cg.add(var.set_outdoor_temperature_sensor(sens)) - if CONF_POWER_USAGE in config: - sens = await sensor.new_sensor(config[CONF_POWER_USAGE]) - cg.add(var.set_power_sensor(sens)) - if CONF_HUMIDITY_SETPOINT in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY_SETPOINT]) - cg.add(var.set_humidity_setpoint_sensor(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_OUTDOOR_TEMPERATURE, var.set_outdoor_temperature_sensor) + await sensors(CONF_POWER_USAGE, var.set_power_sensor) + await sensors(CONF_HUMIDITY_SETPOINT, var.set_humidity_setpoint_sensor) # MideaUART uses the Arduino WiFi API for the network-notify frame # (WiFi auto-enables Network via dependency mapping). On ESP-IDF the # library talks to esp_wifi directly, so no library entry is needed. From af9c4cb778ff59d9cc4dae1268f39c20fbd94870 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:37:12 -0500 Subject: [PATCH 110/133] [xiaomi_mjyd02yla] Use the sub_* helpers for optional child entities (#19975) --- .../xiaomi_mjyd02yla/binary_sensor.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/esphome/components/xiaomi_mjyd02yla/binary_sensor.py b/esphome/components/xiaomi_mjyd02yla/binary_sensor.py index 5d562a2ca91..be5827c1719 100644 --- a/esphome/components/xiaomi_mjyd02yla/binary_sensor.py +++ b/esphome/components/xiaomi_mjyd02yla/binary_sensor.py @@ -77,15 +77,9 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) cg.add(var.set_bindkey(config[CONF_BINDKEY])) - if CONF_IDLE_TIME in config: - sens = await sensor.new_sensor(config[CONF_IDLE_TIME]) - cg.add(var.set_idle_time(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) - if CONF_ILLUMINANCE in config: - sens = await sensor.new_sensor(config[CONF_ILLUMINANCE]) - cg.add(var.set_illuminance(sens)) - if CONF_LIGHT in config: - sens = await binary_sensor.new_binary_sensor(config[CONF_LIGHT]) - cg.add(var.set_light(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_IDLE_TIME, var.set_idle_time) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) + await sensors(CONF_ILLUMINANCE, var.set_illuminance) + binary_sensors = binary_sensor.sub_binary_sensors(config) + await binary_sensors(CONF_LIGHT, var.set_light) From b4cb164e5d09df85d55689cb73458fea54dd56f7 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:37:39 -0500 Subject: [PATCH 111/133] [xiaomi_jqjcy01ym] Use the sub_* helpers for optional child entities (#19974) --- esphome/components/xiaomi_jqjcy01ym/sensor.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/esphome/components/xiaomi_jqjcy01ym/sensor.py b/esphome/components/xiaomi_jqjcy01ym/sensor.py index b791c40dca6..4bfe9c8f756 100644 --- a/esphome/components/xiaomi_jqjcy01ym/sensor.py +++ b/esphome/components/xiaomi_jqjcy01ym/sensor.py @@ -72,15 +72,8 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_HUMIDITY in config: - sens = await sensor.new_sensor(config[CONF_HUMIDITY]) - cg.add(var.set_humidity(sens)) - if CONF_FORMALDEHYDE in config: - sens = await sensor.new_sensor(config[CONF_FORMALDEHYDE]) - cg.add(var.set_formaldehyde(sens)) - if CONF_BATTERY_LEVEL in config: - sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL]) - cg.add(var.set_battery_level(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_HUMIDITY, var.set_humidity) + await sensors(CONF_FORMALDEHYDE, var.set_formaldehyde) + await sensors(CONF_BATTERY_LEVEL, var.set_battery_level) From 879c5cfc5b33c756f695acc9aa907ec6d10bf71d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 11:37:55 -0500 Subject: [PATCH 112/133] [xiaomi_gcls002] Use the sub_* helpers for optional child entities (#19973) --- esphome/components/xiaomi_gcls002/sensor.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/esphome/components/xiaomi_gcls002/sensor.py b/esphome/components/xiaomi_gcls002/sensor.py index 63ba5da5972..75138970246 100644 --- a/esphome/components/xiaomi_gcls002/sensor.py +++ b/esphome/components/xiaomi_gcls002/sensor.py @@ -71,15 +71,8 @@ async def to_code(config: ConfigType) -> None: cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex)) - if CONF_TEMPERATURE in config: - sens = await sensor.new_sensor(config[CONF_TEMPERATURE]) - cg.add(var.set_temperature(sens)) - if CONF_MOISTURE in config: - sens = await sensor.new_sensor(config[CONF_MOISTURE]) - cg.add(var.set_moisture(sens)) - if CONF_ILLUMINANCE in config: - sens = await sensor.new_sensor(config[CONF_ILLUMINANCE]) - cg.add(var.set_illuminance(sens)) - if CONF_CONDUCTIVITY in config: - sens = await sensor.new_sensor(config[CONF_CONDUCTIVITY]) - cg.add(var.set_conductivity(sens)) + sensors = sensor.sub_sensors(config) + await sensors(CONF_TEMPERATURE, var.set_temperature) + await sensors(CONF_MOISTURE, var.set_moisture) + await sensors(CONF_ILLUMINANCE, var.set_illuminance) + await sensors(CONF_CONDUCTIVITY, var.set_conductivity) From 5e672588189a5b9dbe78bf0204863e0d03edc29a Mon Sep 17 00:00:00 2001 From: Jesse Hills <3060199+jesserockz@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:59:00 +1300 Subject: [PATCH 113/133] [noise] Import noise-c and libsodium as ESP-IDF managed components (#18351) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> Co-authored-by: J. Nick Koston --- esphome/components/esp32/__init__.py | 38 +++- esphome/components/noise/__init__.py | 37 +++- esphome/espidf/clang_tidy.py | 32 +++- esphome/espidf/component.py | 13 +- esphome/idf_component.yml | 11 ++ esphome/platformio/library.py | 14 +- platformio.ini | 5 +- .../components/noise/test_noise_to_code.py | 163 ++++++++++++++++ .../test_esp32_write_idf_component_yml.py | 175 ++++++++++++++++++ tests/unit_tests/test_espidf_clang_tidy.py | 159 +++++++++++++++- tests/unit_tests/test_espidf_component.py | 106 +++++++++++ 11 files changed, 726 insertions(+), 27 deletions(-) create mode 100644 tests/unit_tests/components/noise/test_noise_to_code.py create mode 100644 tests/unit_tests/components/test_esp32_write_idf_component_yml.py diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index c3da73f5107..49a115c9c56 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -322,6 +322,34 @@ ARDUINO_EXCLUDED_IDF_COMPONENTS = ( "joltwallet__littlefs", # LittleFS - ESPHome doesn't use filesystem ) +# Entries arduino-esp32 only declares below the given IDF version; stubbing one past +# it clashes with ESPHome's own managed component of the same short name. +ARDUINO_EXCLUDED_IDF_COMPONENTS_MAX_IDF: dict[str, cv.Version] = { + "espressif__libsodium": cv.Version(6, 0, 0), +} + + +def arduino_bundles_libsodium() -> bool: + """arduino-esp32 ships its own libsodium below IDF 6.0.""" + return ( + CORE.using_arduino + and idf_version() + < ARDUINO_EXCLUDED_IDF_COMPONENTS_MAX_IDF["espressif__libsodium"] + ) + + +def arduino_excluded_idf_components() -> set[str]: + """The arduino-bundled components to stub for this build's IDF version.""" + version = idf_version() + return { + component + for component in ARDUINO_EXCLUDED_IDF_COMPONENTS + if (max_version := ARDUINO_EXCLUDED_IDF_COMPONENTS_MAX_IDF.get(component)) + is None + or version < max_version + } + + # Mapping of Arduino libraries to IDF managed components they require # When an Arduino library is enabled via cg.add_library(), these components # are automatically un-stubbed from ARDUINO_EXCLUDED_IDF_COMPONENTS. @@ -3513,9 +3541,7 @@ def _write_idf_component_yml(): } # Only stub components that are not required by any enabled Arduino library - components_to_stub = ( - set(ARDUINO_EXCLUDED_IDF_COMPONENTS) - required_idf_components - ) + components_to_stub = arduino_excluded_idf_components() - required_idf_components stubs_dir = CORE.relative_build_path("component_stubs") stubs_dir.mkdir(exist_ok=True) @@ -3580,7 +3606,11 @@ def _write_idf_component_yml(): # Don't process arduino libraries if name not in ARDUINO_DISABLED_LIBRARIES ] - for component in generate_idf_components(libraries): + # A library also declared as a managed component is not converted too, or + # IDF sees the same requirement twice; converted components reach it through + # ${ESPHOME_PROJECT_MANAGED_COMPONENTS}. + managed = set(CORE.data[KEY_ESP32].get(KEY_COMPONENTS, {})) + for component in generate_idf_components(libraries, managed=managed): dependencies[component.get_sanitized_name()] = { "override_path": str(component.path) } diff --git a/esphome/components/noise/__init__.py b/esphome/components/noise/__init__.py index c894c147cdf..19ad36612de 100644 --- a/esphome/components/noise/__init__.py +++ b/esphome/components/noise/__init__.py @@ -12,6 +12,11 @@ from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] DOMAIN = "noise" +# Keep in sync with platformio.ini and esphome/idf_component.yml. +# LIBSODIUM_VERSION must match the version noise-c pins in its manifests. +NOISE_C_VERSION = "0.1.30" +LIBSODIUM_VERSION = "1.10021.11" + noise_ns = cg.esphome_ns.namespace("noise") CONFIG_SCHEMA = cv.Schema({}) @@ -92,6 +97,22 @@ def encryption_schema(config: ConfigType | None) -> ConfigType: return ENCRYPTION_SCHEMA(config) +def _use_managed_components() -> bool: + """Whether noise-c and libsodium come from the ESP-IDF component registry. + + Both build as ESP-IDF components, so on ESP32 they skip the PlatformIO library + converter unless arduino-esp32 bundles its own libsodium. Not toolchain + dependent: the PlatformIO toolchain reads the project manifest too, and every + consumer of libsodium must make the same choice or a second copy appears. + """ + if not CORE.is_esp32: + return False + + from esphome.components.esp32 import arduino_bundles_libsodium + + return not arduino_bundles_libsodium() + + def enable_spare_ephemeral() -> None: """Compile the spare ephemeral key slot; the component that refills it calls this.""" cg.add_define("USE_NOISE_SPARE_EPHEMERAL") @@ -99,12 +120,16 @@ def enable_spare_ephemeral() -> None: async def to_code(config: ConfigType) -> None: cg.add_define("USE_NOISE") - cg.add_library("esphome/noise-c", "0.1.30") - # noise-c depends on libsodium, but declaring it here too lets the - # library manager see the full set up front instead of discovering - # libsodium only after noise-c has downloaded, so the two can download - # in parallel. The version must match noise-c's library.json. - cg.add_library("esphome/libsodium", "1.10021.11") + # libsodium is declared next to noise-c so the library manager sees both up front + # and nothing else pulls a second copy; the version must match noise-c's own pin + if _use_managed_components(): + from esphome.components.esp32 import add_idf_component + + add_idf_component(name="esphome/noise-c", ref=NOISE_C_VERSION) + add_idf_component(name="esphome/libsodium", ref=LIBSODIUM_VERSION) + else: + cg.add_library("esphome/noise-c", NOISE_C_VERSION) + cg.add_library("esphome/libsodium", LIBSODIUM_VERSION) # Enable optimized memzero/memcmp in libsodium instead of volatile byte loops cg.add_build_flag("-DHAVE_WEAK_SYMBOLS=1") cg.add_build_flag("-DHAVE_INLINE_ASM=1") diff --git a/esphome/espidf/clang_tidy.py b/esphome/espidf/clang_tidy.py index 3e1a510dee9..623b352e97b 100644 --- a/esphome/espidf/clang_tidy.py +++ b/esphome/espidf/clang_tidy.py @@ -238,6 +238,17 @@ def _parse_lib_deps(platformio_ini: Path, framework: str): return libs +def _esphome_manifest_deps() -> set[str]: + """Names of the managed components declared in ``esphome/idf_component.yml``.""" + import yaml + + esphome_dir = Path(__file__).resolve().parent.parent + manifest = yaml.safe_load( + (esphome_dir / "idf_component.yml").read_text(encoding="utf-8") + ) + return set(manifest.get("dependencies") or {}) + + def _convert_pio_libs( platformio_ini: Path, framework: str ) -> dict[str, dict[str, str]]: @@ -250,12 +261,19 @@ def _convert_pio_libs( The whole library set is resolved as a single batch so a shared transitive dependency (e.g. esphome/libsodium pulled by both noise-c and esp_wireguard) is deduplicated to one component instead of clashing override_path entries. + + Libraries the manifest already provides as managed components are skipped, as + in the real esp32 build; converting them too would give IDF the same requirement + twice. Arduino below IDF 6.0 has them rule-gated off, so there they convert. """ + from esphome.components.esp32 import arduino_bundles_libsodium from esphome.espidf.component import generate_idf_components libraries = _parse_lib_deps(platformio_ini, framework) + # Same rule as noise._use_managed_components and the manifest + managed = set() if arduino_bundles_libsodium() else _esphome_manifest_deps() deps: dict[str, dict[str, str]] = {} - for component in generate_idf_components(libraries): + for component in generate_idf_components(libraries, managed=managed): deps[component.get_sanitized_name()] = {"override_path": str(component.path)} return deps @@ -273,24 +291,18 @@ def _arduino_excluded_stubs(work_dir: Path) -> dict[str, dict]: ethernet) are NOT stubbed -- those are real deps we need, and arduino-esp32 resolves to the same component rather than conflicting. """ - import yaml - from esphome.components.esp32 import ( - ARDUINO_EXCLUDED_IDF_COMPONENTS, _idf_component_dep_name, _idf_component_stub_name, + arduino_excluded_idf_components, ) - esphome_dir = Path(__file__).resolve().parent.parent - base_manifest = yaml.safe_load( - (esphome_dir / "idf_component.yml").read_text(encoding="utf-8") - ) - esphome_deps = set(base_manifest.get("dependencies") or {}) + esphome_deps = _esphome_manifest_deps() stubs_dir = work_dir / "component_stubs" stubs_dir.mkdir(parents=True, exist_ok=True) deps: dict[str, dict] = {} - for component in sorted(ARDUINO_EXCLUDED_IDF_COMPONENTS): + for component in sorted(arduino_excluded_idf_components()): if _idf_component_dep_name(component) in esphome_deps: continue # ESPHome needs this one for real (don't stub it away) stub_path = stubs_dir / _idf_component_stub_name(component) diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index bed999656b3..c53dbe17dba 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -287,12 +287,19 @@ def _emit_idf_component(component: IDFComponent) -> None: ) -def generate_idf_components(libraries: list[Library]) -> list[IDFComponent]: - """Resolve and convert a batch of PlatformIO libraries to IDF components.""" +def generate_idf_components( + libraries: list[Library], managed: set[str] | None = None +) -> list[IDFComponent]: + """Resolve and convert a batch of PlatformIO libraries to IDF components. + + ``managed`` names registry components already in the project manifest; they + are not converted, since IDF rejects the same requirement from two components. + Converted components reach them through ``${ESPHOME_PROJECT_MANAGED_COMPONENTS}``. + """ backend = LibraryBackend( platform=ESP32_PLATFORM, framework=_idf_framework(), emit=_emit_idf_component, cache_key="idf", ) - return convert_libraries(libraries, backend) + return convert_libraries(libraries, backend, provided=managed) diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 1a972896c3b..fc8090a322e 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -106,3 +106,14 @@ dependencies: version: d44c800a9e876a8394caefc2ce4915dd96dac77b rules: - if: "$ESPHOME_ARDUINO_COMPONENT == 1" + # noise (api, ota). Arduino below IDF 6.0 bundles espressif/libsodium, which + # clashes with esphome/libsodium, so those builds use the PlatformIO libraries. + esphome/noise-c: + version: 0.1.30 + rules: + - if: "$ESPHOME_ARDUINO_COMPONENT == 0 || idf_version >= 6.0.0" + # Declared alongside noise-c so the library converter skips esp_wireguard's copy + esphome/libsodium: + version: 1.10021.11 + rules: + - if: "$ESPHOME_ARDUINO_COMPONENT == 0 || idf_version >= 6.0.0" diff --git a/esphome/platformio/library.py b/esphome/platformio/library.py index e5e4aa7245f..12aaae315e0 100644 --- a/esphome/platformio/library.py +++ b/esphome/platformio/library.py @@ -1120,7 +1120,9 @@ def _prefetch_wave( def convert_libraries( - libraries: list[Library], backend: LibraryBackend + libraries: list[Library], + backend: LibraryBackend, + provided: set[str] | None = None, ) -> list[ConvertedLibrary]: """Resolve and convert a batch of PlatformIO libraries for ``backend``. @@ -1141,10 +1143,18 @@ def convert_libraries( ``lib_ignore`` from ``esphome->platformio_options`` excludes libraries by short name (part after the ``/``), matched against both the top-level libraries and every dependency discovered during the graph walk. + + ``provided`` names libraries the toolchain supplies by other means (for ESP-IDF, + managed components from ``add_idf_component``); unlike ``backend.provides`` they + carry an owner. They are excluded like ``lib_ignore`` so nothing is both + converted and managed, which ESP-IDF refuses to build. """ nodes: dict[str, _LibNode] = {} - lib_ignore = lib_ignore_set() + # Folded into one set so every is_lib_ignored() call site honors both. + lib_ignore = lib_ignore_set() | { + name.split("/")[-1].lower() for name in provided or () + } # The generated build files inside the shared cache bake in the dependency # wiring, which lib_ignore changes; salt the cache path so configs with diff --git a/platformio.ini b/platformio.ini index dd2a5724935..00fad1ff660 100644 --- a/platformio.ini +++ b/platformio.ini @@ -45,7 +45,6 @@ lib_deps_base = lib_deps = ${common.lib_deps_base} https://github.com/dudanov/MideaUART.git#eeea6c3e9b4474f067054592b435be1c4e466815 ; midea - esphome/noise-c@0.1.30 ; noise (api, ota) improv/Improv@1.2.7 ; improv_serial / improv_ble kikuchan98/pngle@1.1.0 ; online_image ; Using the repository directly, otherwise ESP-IDF can't use the library @@ -77,6 +76,10 @@ lib_compat_mode = strict extends = common lib_deps = ${common.lib_deps} + ; noise (api, ota) -- on the ESP-IDF framework these come from the + ; component registry instead (see esphome/idf_component.yml), so they + ; are not in [common]. + esphome/noise-c@0.1.30 ; noise (api, ota) SPI ; spi (Arduino built-in) Wire ; i2c (Arduino built-int) heman/AsyncMqttClient-esphome@1.0.0 ; mqtt diff --git a/tests/unit_tests/components/noise/test_noise_to_code.py b/tests/unit_tests/components/noise/test_noise_to_code.py new file mode 100644 index 00000000000..a7e0a45e5c2 --- /dev/null +++ b/tests/unit_tests/components/noise/test_noise_to_code.py @@ -0,0 +1,163 @@ +"""Tests for the noise-c/libsodium library wiring in the noise component. + +Drives the real to_code() so every branch of the managed-versus-converted +decision runs end to end. +""" + +from __future__ import annotations + +import asyncio +import re + +import pytest + +import esphome.codegen as cg +from esphome.components import esp32, noise +import esphome.config_validation as cv +from esphome.const import ( + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, + Framework, + Platform, + Toolchain, +) +from esphome.core import CORE + +DEFAULT_IDF_VERSION = cv.Version(5, 5, 4) + + +def _setup_core( + platform: Platform, + framework: Framework, + toolchain: Toolchain, + idf_version: cv.Version = DEFAULT_IDF_VERSION, +) -> None: + CORE.reset() + CORE.toolchain = toolchain + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: str(platform), + KEY_TARGET_FRAMEWORK: str(framework), + } + if platform == Platform.ESP32: + CORE.data[esp32.KEY_ESP32] = { + esp32.KEY_VARIANT: "ESP32", + esp32.KEY_IDF_VERSION: idf_version, + } + + +def _record_calls( + monkeypatch: pytest.MonkeyPatch, +) -> tuple[list[dict], list[tuple]]: + """Capture both wiring paths so each test can assert one ran and one did not.""" + idf_calls: list[dict] = [] + lib_calls: list[tuple] = [] + monkeypatch.setattr( + esp32, "add_idf_component", lambda **kwargs: idf_calls.append(kwargs) + ) + monkeypatch.setattr( + cg, + "add_library", + lambda name, version, repository=None: lib_calls.append((name, version)), + ) + return idf_calls, lib_calls + + +@pytest.mark.parametrize("toolchain", [Toolchain.ESP_IDF, Toolchain.PLATFORMIO]) +def test_to_code_esp32_idf_uses_managed_idf_components( + toolchain: Toolchain, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ESP32 + ESP-IDF declares both as managed IDF components on either toolchain.""" + _setup_core(Platform.ESP32, Framework.ESP_IDF, toolchain) + idf_calls, lib_calls = _record_calls(monkeypatch) + + asyncio.run(noise.to_code({})) + + assert idf_calls == [ + {"name": "esphome/noise-c", "ref": noise.NOISE_C_VERSION}, + {"name": "esphome/libsodium", "ref": noise.LIBSODIUM_VERSION}, + ] + assert lib_calls == [] + + +def test_to_code_esp32_arduino_below_idf6_uses_add_library( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Arduino below IDF 6.0 bundles espressif/libsodium, so both stay PlatformIO + libraries.""" + _setup_core( + Platform.ESP32, Framework.ARDUINO, Toolchain.ESP_IDF, cv.Version(5, 5, 4) + ) + idf_calls, lib_calls = _record_calls(monkeypatch) + + asyncio.run(noise.to_code({})) + + assert lib_calls == [ + ("esphome/noise-c", noise.NOISE_C_VERSION), + ("esphome/libsodium", noise.LIBSODIUM_VERSION), + ] + assert idf_calls == [] + + +@pytest.mark.parametrize("idf_version", [cv.Version(6, 0, 0), cv.Version(6, 1, 0)]) +def test_to_code_esp32_arduino_idf6_uses_managed_idf_components( + idf_version: cv.Version, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """From IDF 6.0 there is no clash, so Arduino uses managed components too.""" + _setup_core(Platform.ESP32, Framework.ARDUINO, Toolchain.ESP_IDF, idf_version) + idf_calls, lib_calls = _record_calls(monkeypatch) + + asyncio.run(noise.to_code({})) + + assert idf_calls == [ + {"name": "esphome/noise-c", "ref": noise.NOISE_C_VERSION}, + {"name": "esphome/libsodium", "ref": noise.LIBSODIUM_VERSION}, + ] + assert lib_calls == [] + + +def test_to_code_non_esp32_uses_add_library( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Off ESP32 entirely (e.g. host) there are no IDF components at all.""" + _setup_core(Platform.HOST, Framework.NATIVE, Toolchain.PLATFORMIO) + idf_calls, lib_calls = _record_calls(monkeypatch) + + asyncio.run(noise.to_code({})) + + assert lib_calls == [ + ("esphome/noise-c", noise.NOISE_C_VERSION), + ("esphome/libsodium", noise.LIBSODIUM_VERSION), + ] + assert idf_calls == [] + + +def test_versions_match_the_repo_manifests() -> None: + """A bump that misses one of the duplicated pins would ship two libsodium versions.""" + from pathlib import Path + + import yaml + + repo_root = Path(__file__).resolve().parents[4] + manifest = yaml.safe_load( + (repo_root / "esphome" / "idf_component.yml").read_text(encoding="utf-8") + ) + deps = manifest["dependencies"] + + assert deps["esphome/noise-c"]["version"] == noise.NOISE_C_VERSION + assert deps["esphome/libsodium"]["version"] == noise.LIBSODIUM_VERSION + # Both are skipped on Arduino below IDF 6.0, where the PlatformIO library + # path is used instead (see noise._use_managed_components). + for name in ("esphome/noise-c", "esphome/libsodium"): + assert deps[name]["rules"] == [ + {"if": "$ESPHOME_ARDUINO_COMPONENT == 0 || idf_version >= 6.0.0"} + ] + # Every noise-c pin in platformio.ini, not just one of them + pins = re.findall( + r"esphome/noise-c@(\S+)", + (repo_root / "platformio.ini").read_text(encoding="utf-8"), + ) + assert set(pins) == {noise.NOISE_C_VERSION} + assert len(pins) >= 1 diff --git a/tests/unit_tests/components/test_esp32_write_idf_component_yml.py b/tests/unit_tests/components/test_esp32_write_idf_component_yml.py new file mode 100644 index 00000000000..931de627a1a --- /dev/null +++ b/tests/unit_tests/components/test_esp32_write_idf_component_yml.py @@ -0,0 +1,175 @@ +"""Tests for esp32's _write_idf_component_yml() managed-component wiring. + +A library already declared as a managed IDF component must not also be converted, +or ESP-IDF sees the same requirement twice and refuses to build. +""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from esphome.components import esp32 +import esphome.config_validation as cv +from esphome.const import ( + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, + Framework, + Platform, + Toolchain, +) +from esphome.core import CORE + + +def _setup_core(tmp_path: Path) -> None: + CORE.reset() + CORE.name = "testdevice" + CORE.build_path = tmp_path + CORE.toolchain = Toolchain.ESP_IDF + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: str(Platform.ESP32), + KEY_TARGET_FRAMEWORK: str(Framework.ESP_IDF), + } + + +def test_write_idf_component_yml_passes_managed_components( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Names registered via add_idf_component are passed as ``managed`` so the + converter skips them.""" + _setup_core(tmp_path) + CORE.data[esp32.KEY_ESP32] = { + esp32.KEY_COMPONENTS: { + "esphome/noise-c": { + esp32.KEY_REPO: None, + esp32.KEY_REF: "0.1.15", + esp32.KEY_PATH: None, + }, + }, + } + + captured: dict[str, set[str] | None] = {} + + # A converted (non-managed) library the batch still resolves, so the loop + # wiring its override_path into the manifest is exercised for real too. + converted = MagicMock() + converted.get_sanitized_name.return_value = "esphome/other-lib" + converted.path = tmp_path / "pio_components" / "other-lib" + + def fake_generate_idf_components(libraries, managed=None): + captured["managed"] = managed + return [converted] + + monkeypatch.setattr(esp32, "generate_idf_components", fake_generate_idf_components) + + esp32._write_idf_component_yml() + + assert captured["managed"] == {"esphome/noise-c"} + # The managed component itself is still written into the manifest deps + # directly (from KEY_COMPONENTS), just not converted a second time. + yml_path = tmp_path / "src" / "idf_component.yml" + assert yml_path.is_file() + contents = yml_path.read_text(encoding="utf-8") + assert "esphome/noise-c" in contents + assert "0.1.15" in contents + # The converted library the batch DID return is still wired in. + assert "esphome/other-lib" in contents + assert str(converted.path) in contents + + +def test_write_idf_component_yml_empty_managed_when_no_components( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """No add_idf_component calls means an empty managed set, the old behavior.""" + _setup_core(tmp_path) + CORE.data[esp32.KEY_ESP32] = {esp32.KEY_COMPONENTS: {}} + + captured: dict[str, set[str] | None] = {} + + def fake_generate_idf_components(libraries, managed=None): + captured["managed"] = managed + return [] + + monkeypatch.setattr(esp32, "generate_idf_components", fake_generate_idf_components) + + esp32._write_idf_component_yml() + + assert captured["managed"] == set() + + +@pytest.mark.parametrize( + ("version", "libsodium_stubbed"), + [ + (cv.Version(5, 5, 4), True), + (cv.Version(5, 99, 99), True), + (cv.Version(6, 0, 0), False), + (cv.Version(6, 1, 0), False), + ], +) +def test_arduino_excluded_idf_components_depends_on_idf_version( + version: cv.Version, libsodium_stubbed: bool +) -> None: + """espressif/libsodium is stubbed below IDF 6.0 only; unmapped entries always.""" + CORE.reset() + CORE.data[esp32.KEY_ESP32] = {esp32.KEY_IDF_VERSION: version} + + excluded = esp32.arduino_excluded_idf_components() + + assert ("espressif__libsodium" in excluded) is libsodium_stubbed + assert "espressif__cbor" in excluded + + +@pytest.mark.parametrize( + ("framework", "version", "bundled"), + [ + (Framework.ARDUINO, cv.Version(5, 5, 5), True), + (Framework.ARDUINO, cv.Version(6, 0, 0), False), + (Framework.ESP_IDF, cv.Version(5, 5, 5), False), + ], +) +def test_arduino_bundles_libsodium( + framework: Framework, version: cv.Version, bundled: bool, tmp_path: Path +) -> None: + """Only Arduino below IDF 6.0 brings its own libsodium.""" + _setup_core(tmp_path) + CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = str(framework) + CORE.data[esp32.KEY_ESP32] = {esp32.KEY_IDF_VERSION: version} + + assert esp32.arduino_bundles_libsodium() is bundled + + +@pytest.mark.parametrize( + ("version", "libsodium_stubbed"), + [(cv.Version(5, 5, 4), True), (cv.Version(6, 0, 0), False)], +) +def test_write_idf_component_yml_arduino_stubs_follow_idf_version( + version: cv.Version, + libsodium_stubbed: bool, + tmp_path: Path, +) -> None: + """The manifest stubs follow the IDF version: espressif/libsodium below 6.0 only.""" + _setup_core(tmp_path) + CORE.toolchain = Toolchain.PLATFORMIO + CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = str(Framework.ARDUINO) + CORE.data[esp32.KEY_ESP32] = { + esp32.KEY_COMPONENTS: {}, + esp32.KEY_IDF_VERSION: version, + esp32.KEY_ARDUINO_LIBRARIES: set(), + } + + esp32._write_idf_component_yml() + + contents = (tmp_path / "src" / "idf_component.yml").read_text(encoding="utf-8") + assert ("espressif/libsodium" in contents) is libsodium_stubbed + assert "espressif/cbor" in contents + stub_dir = ( + tmp_path + / "component_stubs" + / esp32._idf_component_stub_name("espressif__libsodium") + ) + assert stub_dir.is_dir() is libsodium_stubbed diff --git a/tests/unit_tests/test_espidf_clang_tidy.py b/tests/unit_tests/test_espidf_clang_tidy.py index e0cfec88c85..b0d016b1b43 100644 --- a/tests/unit_tests/test_espidf_clang_tidy.py +++ b/tests/unit_tests/test_espidf_clang_tidy.py @@ -5,12 +5,26 @@ import json import os from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch import pytest +import yaml +from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION +import esphome.config_validation as cv +from esphome.const import KEY_CORE, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM +from esphome.core import CORE from esphome.espidf import clang_tidy -from esphome.espidf.clang_tidy import _Settings, _setup_core, _write_tidy_project +from esphome.espidf.clang_tidy import ( + _arduino_excluded_stubs, + _convert_pio_libs, + _esphome_manifest_deps, + _Settings, + _setup_core, + _write_tidy_project, +) +import esphome.espidf.component as espidf_component REPO_ROOT = Path(__file__).resolve().parents[2] @@ -137,3 +151,146 @@ def test_generate_compile_commands_configures_twice( ) assert result == tmp_path / "build" / "compile_commands.json" assert mock_write.call_args_list[1].args[1] == ["esp_timer", "lwip"] + + +def test_esphome_manifest_deps_reads_repo_manifest() -> None: + """Top-level dependency names, independent of the per-dependency rules.""" + manifest = yaml.safe_load( + (REPO_ROOT / "esphome" / "idf_component.yml").read_text(encoding="utf-8") + ) + + deps = _esphome_manifest_deps() + + assert isinstance(deps, set) + assert "esphome/noise-c" in deps + assert "esphome/libsodium" in deps + # Cross-check against a fresh parse instead of hardcoding the manifest's + # whole key list, so this doesn't need updating whenever a dependency is + # added or removed. + assert deps == set(manifest["dependencies"]) + + +def test_convert_pio_libs_arduino_framework_passes_empty_managed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Below IDF 6.0 the manifest gates noise-c/libsodium off on Arduino, so they + still go through the converter.""" + _set_idf_version(cv.Version(5, 5, 5)) + monkeypatch.setattr(clang_tidy, "_parse_lib_deps", lambda ini, framework: []) + + captured: dict[str, set[str] | None] = {} + + # A converted library the batch resolves, so the loop wiring its + # override_path into the returned deps mapping is exercised for real too. + converted = SimpleNamespace( + get_sanitized_name=lambda: "esphome/other-lib", + path=tmp_path / "other-lib", + ) + + def fake_generate_idf_components(libraries, managed=None): + captured["managed"] = managed + return [converted] + + monkeypatch.setattr( + espidf_component, "generate_idf_components", fake_generate_idf_components + ) + + result = _convert_pio_libs(tmp_path / "platformio.ini", "arduino") + + assert captured["managed"] == set() + assert result == { + "esphome/other-lib": {"override_path": str(tmp_path / "other-lib")} + } + + +def test_convert_pio_libs_arduino_idf_6_passes_manifest_deps( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """From IDF 6.0 the manifest enables them on Arduino too, so the converter + must skip them.""" + _set_idf_version(cv.Version(6, 0, 0)) + monkeypatch.setattr(clang_tidy, "_parse_lib_deps", lambda ini, framework: []) + + captured: dict[str, set[str] | None] = {} + + def fake_generate_idf_components(libraries, managed=None): + captured["managed"] = managed + return [] + + monkeypatch.setattr( + espidf_component, "generate_idf_components", fake_generate_idf_components + ) + + assert _convert_pio_libs(tmp_path / "platformio.ini", "arduino") == {} + assert captured["managed"] == _esphome_manifest_deps() + + +def test_convert_pio_libs_espidf_framework_passes_manifest_deps( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Manifest-provided components are passed as ``managed`` so the converter + skips them.""" + _set_idf_version(cv.Version(5, 5, 5), "espidf") + monkeypatch.setattr(clang_tidy, "_parse_lib_deps", lambda ini, framework: []) + + captured: dict[str, set[str] | None] = {} + + def fake_generate_idf_components(libraries, managed=None): + captured["managed"] = managed + return [] + + monkeypatch.setattr( + espidf_component, "generate_idf_components", fake_generate_idf_components + ) + + result = _convert_pio_libs(tmp_path / "platformio.ini", "espidf") + + assert captured["managed"] == _esphome_manifest_deps() + assert "esphome/noise-c" in captured["managed"] + assert result == {} + + +def _set_idf_version(version: cv.Version, framework: str = "arduino") -> None: + CORE.reset() + CORE.data[KEY_ESP32] = {KEY_IDF_VERSION: version} + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: "esp32", + KEY_TARGET_FRAMEWORK: framework, + } + + +def test_arduino_excluded_stubs_skips_components_esphome_manifest_provides( + tmp_path: Path, +) -> None: + """A component ESPHome's own manifest declares (espressif/lan867x for ethernet) + is not stubbed; one only arduino-esp32 bundles still is.""" + _set_idf_version(cv.Version(5, 5, 4)) + + deps = _arduino_excluded_stubs(tmp_path) + + # lan867x is a real ESPHome dependency (esphome/idf_component.yml), so it + # must be excluded from the stub set. + assert "espressif/lan867x" not in deps + # espressif/libsodium (arduino-esp32's bundled copy) is a different + # package from ESPHome's own esphome/libsodium, so below IDF 6.0, where + # arduino-esp32 still declares it, it is stubbed. + assert "espressif/libsodium" in deps + stub_info = deps["espressif/libsodium"] + assert stub_info["version"] == "*" + stub_path = Path(stub_info["override_path"]) + assert (stub_path / "CMakeLists.txt").is_file() + + +def test_arduino_excluded_stubs_skips_libsodium_from_idf_6(tmp_path: Path) -> None: + """From IDF 6.0 arduino-esp32 drops espressif/libsodium; a stub would clash + with esphome/libsodium.""" + _set_idf_version(cv.Version(6, 0, 0)) + + deps = _arduino_excluded_stubs(tmp_path) + + assert "espressif/libsodium" not in deps + # Other arduino-bundled components are still stubbed. + assert "espressif/cbor" in deps diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index 3789eefc645..67da0f4f6df 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -803,6 +803,112 @@ def test_generate_idf_components_lib_ignore_filters_top_level_and_dependencies( assert download_salts == [hashlib.sha256(b"b,c").hexdigest()[:8]] +def test_generate_idf_components_managed_filters_top_level_and_dependencies( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + esp32_idf_core: None, +) -> None: + # managed (e.g. noise-c/libsodium already declared via add_idf_component) + # must drop B at the top level and C when discovered as a dependency of A, + # exactly like lib_ignore -- neither may be resolved, downloaded, or wired + # into a manifest. + manifests = { + "esphome/A": { + "name": "A", + "dependencies": [ + {"owner": "esphome", "name": "C", "version": "==1.10021.0"} + ], + }, + "esphome/B": {"name": "B"}, + } + + download_salts: list[str] = [] + + def fake_download(self, force=False, salt="", namespace=""): + download_salts.append(salt) + self.path = tmp_path / self.get_sanitized_name().replace("/", "__") + (self.path / "src").mkdir(parents=True, exist_ok=True) + (self.path / "src" / "x.c").write_text("int x;") + (self.path / "library.json").write_text(json.dumps(manifests[self.name])) + + monkeypatch.setattr(IDFComponent, "download", fake_download) + + resolve_calls: list[str] = [] + + def fake_resolve(owner, pkgname, requirements): + resolve_calls.append(pkgname) + return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", None + + monkeypatch.setattr( + esphome.platformio.library, "_resolve_registry_version", fake_resolve + ) + + top = generate_idf_components( + [Library("esphome/A", "1.0.0", None), Library("esphome/B", "1.0.0", None)], + managed={"esphome/B", "esphome/C"}, + ) + + assert [c.name for c in top] == ["esphome/A"] + # Managed libraries were never resolved (and therefore never downloaded). + assert resolve_calls == ["A"] + # The managed dependency is not wired into A's manifest. + assert top[0].dependencies == [] + # managed changes the generated wiring just like lib_ignore, so the cache + # path is salted the same way. + assert download_salts == [hashlib.sha256(b"b,c").hexdigest()[:8]] + + +def test_generate_idf_components_lib_ignore_and_managed_combine_into_salt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + esp32_idf_core: None, +) -> None: + # lib_ignore and managed both contribute to the same exclusion set, so a + # config using both gets a salt reflecting the union of the two sources + # rather than either alone. + manifests = { + "esphome/A": {"name": "A"}, + "esphome/D": {"name": "D"}, + "esphome/E": {"name": "E"}, + } + + download_salts: list[str] = [] + + def fake_download(self, force=False, salt="", namespace=""): + download_salts.append(salt) + self.path = tmp_path / self.get_sanitized_name().replace("/", "__") + (self.path / "src").mkdir(parents=True, exist_ok=True) + (self.path / "src" / "x.c").write_text("int x;") + (self.path / "library.json").write_text(json.dumps(manifests[self.name])) + + monkeypatch.setattr(IDFComponent, "download", fake_download) + + resolve_calls: list[str] = [] + + def fake_resolve(owner, pkgname, requirements): + resolve_calls.append(pkgname) + return owner, pkgname, "1.0.0", f"http://x/{pkgname}.tar.gz", None + + monkeypatch.setattr( + esphome.platformio.library, "_resolve_registry_version", fake_resolve + ) + monkeypatch.setattr(CORE, "platformio_options", {"lib_ignore": ["D"]}) + + top = generate_idf_components( + [ + Library("esphome/A", "1.0.0", None), + Library("esphome/D", "1.0.0", None), + Library("esphome/E", "1.0.0", None), + ], + managed={"esphome/E"}, + ) + + assert [c.name for c in top] == ["esphome/A"] + assert resolve_calls == ["A"] + # The salt reflects BOTH lib_ignore's "D" and managed's "E" together. + assert download_salts == [hashlib.sha256(b"d,e").hexdigest()[:8]] + + def test_generate_idf_components_handles_dependency_cycle( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From afba43160095d97b812770818138e65487029fef Mon Sep 17 00:00:00 2001 From: Remco van Essen Date: Thu, 1 Oct 2026 20:15:13 +0200 Subject: [PATCH 114/133] [resampler] Add resampler microphone platform (#19953) --- esphome/components/resampler/__init__.py | 15 ++ .../resampler/microphone/__init__.py | 77 ++++++++ .../microphone/resampler_microphone.cpp | 174 ++++++++++++++++++ .../microphone/resampler_microphone.h | 62 +++++++ .../components/resampler/speaker/__init__.py | 16 +- .../resampler/common-microphone.yaml | 29 +++ .../resampler/test-microphone.esp32-idf.yaml | 6 + .../test-microphone.esp32-s3-idf.yaml | 6 + 8 files changed, 372 insertions(+), 13 deletions(-) create mode 100644 esphome/components/resampler/microphone/__init__.py create mode 100644 esphome/components/resampler/microphone/resampler_microphone.cpp create mode 100644 esphome/components/resampler/microphone/resampler_microphone.h create mode 100644 tests/components/resampler/common-microphone.yaml create mode 100644 tests/components/resampler/test-microphone.esp32-idf.yaml create mode 100644 tests/components/resampler/test-microphone.esp32-s3-idf.yaml diff --git a/esphome/components/resampler/__init__.py b/esphome/components/resampler/__init__.py index e69de29bb2d..b9b070e0154 100644 --- a/esphome/components/resampler/__init__.py +++ b/esphome/components/resampler/__init__.py @@ -0,0 +1,15 @@ +from typing import Any + +import esphome.codegen as cg +import esphome.config_validation as cv + +resampler_ns = cg.esphome_ns.namespace("resampler") + +CONF_TAPS = "taps" + + +def validate_taps(taps: Any) -> int: + value = cv.int_range(min=16, max=128)(taps) + if value % 4 != 0: + raise cv.Invalid("Number of taps must be divisible by 4") + return value diff --git a/esphome/components/resampler/microphone/__init__.py b/esphome/components/resampler/microphone/__init__.py new file mode 100644 index 00000000000..0a8d0c1ca3a --- /dev/null +++ b/esphome/components/resampler/microphone/__init__.py @@ -0,0 +1,77 @@ +import esphome.codegen as cg +from esphome.components import audio, microphone +import esphome.config_validation as cv +from esphome.const import ( + CONF_BITS_PER_SAMPLE, + CONF_CHANNELS, + CONF_FILTERS, + CONF_ID, + CONF_MICROPHONE, + CONF_SAMPLE_RATE, + PLATFORM_ESP32, +) +from esphome.types import ConfigType + +from .. import CONF_TAPS, resampler_ns, validate_taps + +AUTO_LOAD = ["audio"] +DEPENDENCIES = ["microphone"] + +ResamplerMicrophone = resampler_ns.class_( + "ResamplerMicrophone", cg.Component, microphone.Microphone +) + + +def _set_stream_limits(config: ConfigType) -> ConfigType: + # Only the sample rate changes; the bits and channels are those selected from the source microphone + source = config[CONF_MICROPHONE] + audio.set_stream_limits( + min_bits_per_sample=source[CONF_BITS_PER_SAMPLE], + max_bits_per_sample=source[CONF_BITS_PER_SAMPLE], + min_channels=len(source[CONF_CHANNELS]), + max_channels=len(source[CONF_CHANNELS]), + min_sample_rate=config[CONF_SAMPLE_RATE], + max_sample_rate=config[CONF_SAMPLE_RATE], + )(config) + return config + + +CONFIG_SCHEMA = cv.All( + microphone.MICROPHONE_SCHEMA.extend( + { + cv.GenerateID(): cv.declare_id(ResamplerMicrophone), + cv.Required(CONF_MICROPHONE): microphone.microphone_source_schema( + min_bits_per_sample=16, + max_bits_per_sample=32, + min_channels=1, + max_channels=2, + ), + cv.Optional(CONF_SAMPLE_RATE, default=16000): cv.int_range(8000, 48000), + cv.Optional(CONF_FILTERS, default=16): cv.int_range(min=2, max=1024), + cv.Optional(CONF_TAPS, default=16): validate_taps, + } + ).extend(cv.COMPONENT_SCHEMA), + cv.only_on([PLATFORM_ESP32]), + _set_stream_limits, +) + + +FINAL_VALIDATE_SCHEMA = cv.Schema( + { + cv.Required( + CONF_MICROPHONE + ): microphone.final_validate_microphone_source_schema("resampler"), + }, + extra=cv.ALLOW_EXTRA, +) + + +async def to_code(config: ConfigType) -> None: + mic_source = await microphone.microphone_source_to_code(config[CONF_MICROPHONE]) + var = cg.new_Pvariable(config[CONF_ID], mic_source) + await cg.register_component(var, config) + await microphone.register_microphone(var, config) + + cg.add(var.set_target_sample_rate(config[CONF_SAMPLE_RATE])) + cg.add(var.set_filters(config[CONF_FILTERS])) + cg.add(var.set_taps(config[CONF_TAPS])) diff --git a/esphome/components/resampler/microphone/resampler_microphone.cpp b/esphome/components/resampler/microphone/resampler_microphone.cpp new file mode 100644 index 00000000000..0dcd5233800 --- /dev/null +++ b/esphome/components/resampler/microphone/resampler_microphone.cpp @@ -0,0 +1,174 @@ +#include "resampler_microphone.h" + +#ifdef USE_ESP32 + +#include "esphome/core/helpers.h" +#include "esphome/core/log.h" + +#include + +namespace esphome::resampler { + +static const char *const TAG = "resampler.microphone"; + +// Duration of audio the resampler converts per step; longer source chunks are processed in several steps +static constexpr uint32_t BUFFER_DURATION_MS = 16; + +void ResamplerMicrophone::setup() { + const audio::AudioStreamInfo input_stream_info = this->source_->get_audio_stream_info(); + this->audio_stream_info_ = audio::AudioStreamInfo(input_stream_info.get_bits_per_sample(), + input_stream_info.get_channels(), this->target_sample_rate_); + + // Allocate now for the expected source format; process_audio_ only sets up again if that format changes + if (!this->init_resampler_(input_stream_info)) { + this->mark_failed(); + return; + } + + this->source_->add_data_callback([this](const std::vector &data) { this->process_audio_(data); }); + + this->disable_loop(); +} + +void ResamplerMicrophone::dump_config() { + ESP_LOGCONFIG(TAG, + "Resampler Microphone:\n" + " Target Sample Rate: %" PRIu32 " Hz\n" + " Taps: %u\n" + " Filters: %u", + this->target_sample_rate_, this->taps_, this->filters_); +} + +void ResamplerMicrophone::start() { + if (this->is_failed() || this->active_listeners_ == UINT8_MAX) + return; + ++this->active_listeners_; + this->enable_loop(); +} + +void ResamplerMicrophone::stop() { + if (this->active_listeners_ == 0) + return; + --this->active_listeners_; + this->enable_loop(); +} + +void ResamplerMicrophone::loop() { + if (this->active_listeners_ == 0) { + if (this->state_ != microphone::STATE_STOPPED) { + this->source_->stop(); + this->state_ = microphone::STATE_STOPPED; + } + this->disable_loop(); + return; + } + + switch (this->state_) { + case microphone::STATE_STOPPED: + this->source_->start(); + this->state_ = microphone::STATE_STARTING; + break; + case microphone::STATE_STARTING: + if (this->source_->is_running()) { + this->state_ = microphone::STATE_RUNNING; + } + break; + case microphone::STATE_RUNNING: + // Follow the source if it restarts, e.g. after a driver error + if (!this->source_->is_running()) { + this->state_ = microphone::STATE_STARTING; + } + break; + case microphone::STATE_STOPPING: + break; + } +} + +bool ResamplerMicrophone::init_resampler_(const audio::AudioStreamInfo &input_stream_info) { + this->resampler_.reset(); + this->resampler_ready_ = false; + + if (input_stream_info.get_sample_rate() == this->target_sample_rate_) { + // The source already delivers the target sample rate, so its audio is passed through unchanged + this->input_stream_info_ = input_stream_info; + this->resampler_ready_ = true; + return true; + } + + const audio::AudioStreamInfo output_stream_info(input_stream_info.get_bits_per_sample(), + input_stream_info.get_channels(), this->target_sample_rate_); + + auto resampler = make_unique( + input_stream_info.ms_to_samples(BUFFER_DURATION_MS), output_stream_info.ms_to_samples(BUFFER_DURATION_MS)); + + esp_audio_libs::resampler::ResamplerConfiguration resample_config = { + .source_sample_rate = static_cast(input_stream_info.get_sample_rate()), + .target_sample_rate = static_cast(this->target_sample_rate_), + .source_bits_per_sample = input_stream_info.get_bits_per_sample(), + .target_bits_per_sample = input_stream_info.get_bits_per_sample(), + .channels = input_stream_info.get_channels(), + // Filters out frequencies above the new Nyquist limit when downsampling, to avoid aliasing + .use_pre_or_post_filter = this->target_sample_rate_ < input_stream_info.get_sample_rate(), + .subsample_interpolate = false, // Doubles the CPU load; more filters is a better alternative + .number_of_taps = this->taps_, + .number_of_filters = this->filters_, + }; + + if (!resampler->initialize(resample_config)) { + ESP_LOGE(TAG, "Not enough memory to resample"); + return false; + } + + this->output_buffer_.reserve(output_stream_info.ms_to_bytes(BUFFER_DURATION_MS)); + this->resampler_ = std::move(resampler); + // Only set on success, so a failed set up is retried with the next chunk + this->input_stream_info_ = input_stream_info; + this->resampler_ready_ = true; + return true; +} + +void ResamplerMicrophone::process_audio_(const std::vector &data) { + const audio::AudioStreamInfo input_stream_info = this->source_->get_audio_stream_info(); + if (input_stream_info != this->input_stream_info_) { + this->init_resampler_(input_stream_info); + } + if (!this->resampler_ready_) { + return; + } + + if (this->resampler_ == nullptr) { + this->data_callbacks_.call(data); + return; + } + + const size_t input_bytes_per_frame = input_stream_info.frames_to_bytes(1); + const uint32_t max_input_frames = input_stream_info.ms_to_frames(BUFFER_DURATION_MS); + // Both limits match the sizes the resampler's internal buffers were allocated with in init_resampler_ + const uint32_t max_output_frames = this->audio_stream_info_.ms_to_frames(BUFFER_DURATION_MS); + + const uint8_t *input = data.data(); + uint32_t input_frames = input_stream_info.bytes_to_frames(data.size()); + while (input_frames > 0) { + // Stays within the reserved capacity, so this never reallocates + this->output_buffer_.resize(this->audio_stream_info_.frames_to_bytes(max_output_frames)); + + // The resampler's internal buffers hold at most BUFFER_DURATION_MS of audio, so feed it in steps of that size. + // 0 dB keeps the microphone level that downstream detectors are tuned for; overshoot saturates instead of wrapping. + esp_audio_libs::resampler::ResamplerResults results = this->resampler_->resample( + input, this->output_buffer_.data(), std::min(input_frames, max_input_frames), max_output_frames, 0.0f); + + input += results.frames_used * input_bytes_per_frame; + input_frames -= results.frames_used; + + if (results.frames_generated > 0) { + this->output_buffer_.resize(this->audio_stream_info_.frames_to_bytes(results.frames_generated)); + this->data_callbacks_.call(this->output_buffer_); + } else if (results.frames_used == 0) { + break; // No progress; drop the rest of the chunk instead of spinning + } + } +} + +} // namespace esphome::resampler + +#endif // USE_ESP32 diff --git a/esphome/components/resampler/microphone/resampler_microphone.h b/esphome/components/resampler/microphone/resampler_microphone.h new file mode 100644 index 00000000000..780f22e4294 --- /dev/null +++ b/esphome/components/resampler/microphone/resampler_microphone.h @@ -0,0 +1,62 @@ +#pragma once + +#ifdef USE_ESP32 + +#include "esphome/components/audio/audio.h" +#include "esphome/components/microphone/microphone.h" +#include "esphome/components/microphone/microphone_source.h" + +#include "esphome/core/component.h" + +#include // esp-audio-libs + +#include +#include + +namespace esphome::resampler { + +/// @brief Microphone that converts the audio of a source microphone to a different sample rate. +/// The bits per sample and channels are selected by the source's ``MicrophoneSource``; only the sample rate changes. +/// Resampling runs in the source microphone's data callback, so it needs no task or ring buffer of its own. +class ResamplerMicrophone final : public Component, public microphone::Microphone { + public: + explicit ResamplerMicrophone(microphone::MicrophoneSource *source) : source_(source) {} + + void setup() override; + void loop() override; + void dump_config() override; + + void start() override; + void stop() override; + + void set_target_sample_rate(uint32_t target_sample_rate) { this->target_sample_rate_ = target_sample_rate; } + void set_filters(uint16_t filters) { this->filters_ = filters; } + void set_taps(uint16_t taps) { this->taps_ = taps; } + + protected: + /// @brief Sets up the resampler for the given input format. No resampler is needed if the sample rates match. + /// @return false if the resampler failed to allocate; the audio is then dropped + bool init_resampler_(const audio::AudioStreamInfo &input_stream_info); + + /// @brief Resamples a chunk of source audio and passes it to the data callbacks. Source microphone task only. + void process_audio_(const std::vector &data); + + microphone::MicrophoneSource *source_; + std::unique_ptr resampler_; + // Reused for every chunk so resampling does not allocate + std::vector output_buffer_; + + // Format the resampler is set up for + audio::AudioStreamInfo input_stream_info_; + + uint32_t target_sample_rate_; + uint16_t taps_; + uint16_t filters_; + + uint8_t active_listeners_{0}; + bool resampler_ready_{false}; +}; + +} // namespace esphome::resampler + +#endif // USE_ESP32 diff --git a/esphome/components/resampler/speaker/__init__.py b/esphome/components/resampler/speaker/__init__.py index 7de468cb502..8fa8aeb61c4 100644 --- a/esphome/components/resampler/speaker/__init__.py +++ b/esphome/components/resampler/speaker/__init__.py @@ -1,5 +1,3 @@ -from typing import Any - import esphome.codegen as cg from esphome.components import audio, psram, speaker import esphome.config_validation as cv @@ -17,16 +15,15 @@ from esphome.const import ( from esphome.core.entity_helpers import inherit_property_from from esphome.types import ConfigType +from .. import CONF_TAPS, resampler_ns, validate_taps + AUTO_LOAD = ["audio"] CODEOWNERS = ["@kahrendt"] -resampler_ns = cg.esphome_ns.namespace("resampler") ResamplerSpeaker = resampler_ns.class_( "ResamplerSpeaker", cg.Component, speaker.Speaker ) -CONF_TAPS = "taps" - PASSTHROUGH = "passthrough" @@ -60,13 +57,6 @@ def _validate_audio_compatibility(config: ConfigType) -> None: )(config) -def _validate_taps(taps: Any) -> int: - value = cv.int_range(min=16, max=128)(taps) - if value % 4 != 0: - raise cv.Invalid("Number of taps must be divisible by 4") - return value - - CONFIG_SCHEMA = cv.All( speaker.SPEAKER_SCHEMA.extend( { @@ -80,7 +70,7 @@ CONFIG_SCHEMA = cv.All( ): cv.positive_time_period_milliseconds, cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_FILTERS, default=16): cv.int_range(min=2, max=1024), - cv.Optional(CONF_TAPS, default=16): _validate_taps, + cv.Optional(CONF_TAPS, default=16): validate_taps, } ).extend(cv.COMPONENT_SCHEMA), cv.only_on([PLATFORM_ESP32]), diff --git a/tests/components/resampler/common-microphone.yaml b/tests/components/resampler/common-microphone.yaml new file mode 100644 index 00000000000..123ba87edce --- /dev/null +++ b/tests/components/resampler/common-microphone.yaml @@ -0,0 +1,29 @@ +microphone: + - platform: i2s_audio + id: resampler_i2s_mic_id + i2s_audio_id: i2s_audio_bus + adc_type: external + i2s_din_pin: ${din_pin} + sample_rate: 48000 + bits_per_sample: 32bit + channel: stereo + - platform: resampler + id: resampler_mic_id + microphone: + microphone: resampler_i2s_mic_id + channels: 0 + bits_per_sample: 16 + sample_rate: 16000 + on_data: + - logger.log: + format: "Received %u bytes" + args: [x.size()] + - platform: resampler + id: resampler_mic_stereo_id + microphone: + microphone: resampler_i2s_mic_id + channels: [0, 1] + bits_per_sample: 32 + sample_rate: 44100 + filters: 32 + taps: 32 diff --git a/tests/components/resampler/test-microphone.esp32-idf.yaml b/tests/components/resampler/test-microphone.esp32-idf.yaml new file mode 100644 index 00000000000..9ca0ce7f05a --- /dev/null +++ b/tests/components/resampler/test-microphone.esp32-idf.yaml @@ -0,0 +1,6 @@ +substitutions: + din_pin: GPIO21 + +packages: + i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-idf.yaml + resampler: !include common-microphone.yaml diff --git a/tests/components/resampler/test-microphone.esp32-s3-idf.yaml b/tests/components/resampler/test-microphone.esp32-s3-idf.yaml new file mode 100644 index 00000000000..5ff29672a0f --- /dev/null +++ b/tests/components/resampler/test-microphone.esp32-s3-idf.yaml @@ -0,0 +1,6 @@ +substitutions: + din_pin: GPIO16 + +packages: + i2s_audio: !include ../../test_build_components/common/i2s_audio/esp32-s3-idf.yaml + resampler: !include common-microphone.yaml From 788d2bc6316d8d08eb8e46ead9e713e6fe1e82f9 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:19:30 -0500 Subject: [PATCH 115/133] [ci] Balance component test batches by estimated build time (#19980) Co-authored-by: Jonathan Swoboda <154711427+swoboda1337@users.noreply.github.com> --- script/determine-jobs.py | 5 - script/split_components_for_ci.py | 216 +++++++++++++------ tests/script/test_determine_jobs.py | 19 +- tests/script/test_split_components_for_ci.py | 157 ++++++++++++++ 4 files changed, 318 insertions(+), 79 deletions(-) create mode 100644 tests/script/test_split_components_for_ci.py diff --git a/script/determine-jobs.py b/script/determine-jobs.py index bd127a7a435..9c8f3cd3b4f 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -99,10 +99,6 @@ from split_components_for_ci import create_intelligent_batches # For large PRs (>= 65 files), use split for better parallelization CLANG_TIDY_SPLIT_THRESHOLD = 65 -# Component test batch size (weighted) -# Isolated components count as 10x, groupable components count as 1x -COMPONENT_TEST_BATCH_SIZE = 40 - # Above the threshold, fan out across up to this many jobs, balanced by the # recorded per-file durations. The target is serial junit-time weight per # bucket, not wall time (calibrated with the conftest compile cap); it @@ -1534,7 +1530,6 @@ def main() -> None: batches, _ = create_intelligent_batches( components=changed_components_with_tests, tests_dir=tests_dir, - batch_size=COMPONENT_TEST_BATCH_SIZE, directly_changed=batch_directly_changed, ) # Convert batches to CI matrix entries: the component list plus which diff --git a/script/split_components_for_ci.py b/script/split_components_for_ci.py index 7f06f50f480..cc458d5d664 100755 --- a/script/split_components_for_ci.py +++ b/script/split_components_for_ci.py @@ -13,7 +13,9 @@ from __future__ import annotations import argparse from collections import defaultdict +from dataclasses import dataclass, field import json +import math from pathlib import Path import sys @@ -28,13 +30,34 @@ from script.analyze_component_buses import ( create_grouping_signature, merge_compatible_bus_groups, ) -from script.helpers import get_component_test_files, split_conflicting_groups +from script.helpers import ( + get_component_test_files, + parse_test_filename, + split_conflicting_groups, +) -# Weighting for batch creation -# Isolated components can't be grouped/merged, so they count as 10x -# Groupable components can be merged into single builds, so they count as 1x -ISOLATED_WEIGHT = 10 -GROUPABLE_WEIGHT = 1 +# Estimated CI seconds for one build per test platform, read from batch job +# logs. Absolute accuracy does not matter, only the ratios between platforms. +# A full run takes over an hour, so recording real timings is not practical. +PLATFORM_BUILD_SECONDS = { + "host": 10, + "esp8266-ard": 30, + "esp32-idf": 45, + "rtl87xx-ard": 35, + "ln882x-ard": 50, + "rp2040-ard": 50, + "rp2350-ard": 50, + "bk72xx-ard": 60, +} +NRF52_BUILD_SECONDS = 45 +# Other ESP32 chips and Arduino on ESP32 share less of the ccache +DEFAULT_BUILD_SECONDS = 90 +# Each extra component merged into a grouped build makes it larger +GROUPED_COMPONENT_SECONDS = 5 +# Estimated build seconds per CI runner; sets how many runners are used. +# Approximate: the runner count charges each grouped build once, but a large +# group spreads over several runners, which each pay for that build. +TARGET_BATCH_SECONDS = 600 # Platform used for batching (platform-agnostic batching) # Batches are split across CI runners and each runner tests all platforms @@ -61,10 +84,115 @@ def has_test_files(component_name: str, tests_dir: Path) -> bool: ) +def build_seconds(platform: str) -> int: + """Return the estimated CI seconds for one build on a test platform.""" + if (seconds := PLATFORM_BUILD_SECONDS.get(platform)) is not None: + return seconds + if platform.startswith("nrf52"): + return NRF52_BUILD_SECONDS + return DEFAULT_BUILD_SECONDS + + +@dataclass +class _BatchItem: + """A component and the builds it adds to the batch it lands in.""" + + component: str + # Builds that always run on their own (isolated tests, variants on a + # platform without a base test) + own_seconds: int + # Per (signature, platform) shared build: the seconds of this component's + # files on that platform when no other batch member shares the build + grouped_builds: dict[tuple[str, str], int] = field(default_factory=dict) + + def standalone_seconds(self) -> int: + return self.own_seconds + sum(self.grouped_builds.values()) + + +def _grouped_build_seconds(platform: str, solo_seconds: list[int]) -> int: + """Return the seconds of one (signature, platform) build in a batch. + + test_build_components only groups when two or more members share the + build; a grouped member then skips its variants on that platform. + """ + if len(solo_seconds) <= 1: + return sum(solo_seconds) + return build_seconds(platform) + GROUPED_COMPONENT_SECONDS * (len(solo_seconds) - 1) + + +def _make_item( + tests_dir: Path, component: str, signature: str, is_isolated: bool +) -> _BatchItem: + files_by_platform: dict[str, list[str]] = defaultdict(list) + for test_file in (tests_dir / component).glob("test[.-]*.yaml"): + test_name, platform = parse_test_filename(test_file) + files_by_platform[platform].append(test_name) + own_seconds = 0 + grouped: dict[tuple[str, str], int] = {} + for platform, test_names in files_by_platform.items(): + seconds = build_seconds(platform) * len(test_names) + if is_isolated or "test" not in test_names: + own_seconds += seconds + else: + grouped[(signature, platform)] = seconds + return _BatchItem(component, own_seconds, grouped) + + +class _Batch: + """A CI runner's components and its estimated build seconds.""" + + def __init__(self) -> None: + self.components: list[str] = [] + self.seconds = 0 + self.grouped_builds: dict[tuple[str, str], list[int]] = defaultdict(list) + + def added_seconds(self, item: _BatchItem) -> int: + """Return the seconds item would add; joining an existing build is cheap.""" + added = item.own_seconds + for (signature, platform), solo in item.grouped_builds.items(): + members = self.grouped_builds.get((signature, platform), []) + added += _grouped_build_seconds( + platform, [*members, solo] + ) - _grouped_build_seconds(platform, members) + return added + + def add(self, item: _BatchItem) -> None: + self.seconds += self.added_seconds(item) + for build, solo in item.grouped_builds.items(): + self.grouped_builds[build].append(solo) + self.components.append(item.component) + + +def balance_batches(items: list[_BatchItem], target_seconds: int) -> list[list[str]]: + """Spread items over enough runners to stay near target_seconds each. + + The runner count comes from the total estimate with every grouped build + counted once. Heaviest items go first, each to the runner that ends up + lightest, so grouped components follow the builds they can join. + """ + if not items: + return [] + grouped_builds: dict[tuple[str, str], list[int]] = defaultdict(list) + for item in items: + for build, solo in item.grouped_builds.items(): + grouped_builds[build].append(solo) + total = sum(item.own_seconds for item in items) + sum( + _grouped_build_seconds(platform, solos) + for (_, platform), solos in grouped_builds.items() + ) + count = min(len(items), max(1, math.ceil(total / target_seconds))) + batches = [_Batch() for _ in range(count)] + for item in sorted(items, key=lambda i: (-i.standalone_seconds(), i.component)): + min(batches, key=lambda b: (b.seconds + b.added_seconds(item), b.seconds)).add( + item + ) + return [batch.components for batch in batches if batch.components] + + def create_intelligent_batches( components: list[str], tests_dir: Path, - batch_size: int = 40, + target_seconds: int = TARGET_BATCH_SECONDS, directly_changed: set[str] | None = None, ) -> tuple[list[list[str]], dict[tuple[str, str], list[str]]]: """Create batches optimized for component grouping. @@ -76,7 +204,7 @@ def create_intelligent_batches( Args: components: List of component names to batch tests_dir: Path to tests/components directory - batch_size: Target size for each batch + target_seconds: Estimated build seconds per batch directly_changed: Set of directly changed components (for logging only) Returns: @@ -157,51 +285,17 @@ def create_intelligent_batches( # actually be split into two at build time -- throwing off CI distribution. signature_groups = split_conflicting_groups(signature_groups) - # Create batches by keeping signature groups together - # Components with the same signature stay in the same batches - batches = [] - - # Sort signature groups to prioritize groupable components - # 1. Put "isolated_*" signatures last (can't be grouped with others) - # 2. Sort groupable signatures by size (largest first) - # 3. "no_buses" components CAN be grouped together - def sort_key(item): - (_platform, signature), components = item - is_isolated = signature.startswith(ISOLATED_SIGNATURE_PREFIX) - # Put "isolated_*" last (1), groupable first (0) - # Within each category, sort by size (largest first) - return (is_isolated, -len(components)) - - sorted_groups = sorted(signature_groups.items(), key=sort_key) - - # Strategy: Create batches using weighted sizes - # - Isolated components count as 10x (since they can't be grouped/merged) - # - Groupable components count as 1x (can be merged into single builds) - # - This distributes isolated components across more runners - # - Ensures each runner has a good mix of groupable vs isolated components - - current_batch = [] - current_weight = 0 - - for (_platform, signature), group_components in sorted_groups: - is_isolated = signature.startswith(ISOLATED_SIGNATURE_PREFIX) - weight_per_component = ISOLATED_WEIGHT if is_isolated else GROUPABLE_WEIGHT - - for component in group_components: - # Check if adding this component would exceed the batch size - if current_weight + weight_per_component > batch_size and current_batch: - # Start a new batch - batches.append(current_batch) - current_batch = [] - current_weight = 0 - - # Add component to current batch - current_batch.append(component) - current_weight += weight_per_component - - # Don't forget the last batch - if current_batch: - batches.append(current_batch) + items = [ + _make_item( + tests_dir, + component, + signature, + signature.startswith(ISOLATED_SIGNATURE_PREFIX), + ) + for (_platform, signature), group_components in sorted(signature_groups.items()) + for component in group_components + ] + batches = balance_batches(items, target_seconds) return batches, signature_groups @@ -218,11 +312,11 @@ def main() -> int: help="JSON array of component names", ) parser.add_argument( - "--batch-size", - "-b", + "--target-seconds", + "-t", type=int, - default=40, - help="Target batch size (default: 40, weighted)", + default=TARGET_BATCH_SECONDS, + help=f"Estimated build seconds per batch (default: {TARGET_BATCH_SECONDS})", ) parser.add_argument( "--tests-dir", @@ -268,7 +362,7 @@ def main() -> int: batches, signature_groups = create_intelligent_batches( components=components, tests_dir=args.tests_dir, - batch_size=args.batch_size, + target_seconds=args.target_seconds, directly_changed=directly_changed, ) @@ -378,15 +472,15 @@ def main() -> int: file=sys.stderr, ) - print(f" - Groupable (weight=1): {groupable_count}", file=sys.stderr) - print(f" - Isolated (weight=10): {isolated_count}", file=sys.stderr) + print(f" - Groupable: {groupable_count}", file=sys.stderr) + print(f" - Isolated: {isolated_count}", file=sys.stderr) if actual_components < len(components): print( f"Components skipped (no test files): {len(components) - actual_components}", file=sys.stderr, ) print(f"Number of batches: {len(batches)}", file=sys.stderr) - print(f"Batch size target (weighted): {args.batch_size}", file=sys.stderr) + print(f"Target build seconds per batch: {args.target_seconds}", file=sys.stderr) if len(batches) > 0: print( f"Average components per batch: {actual_components / len(batches):.1f}", diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index cea2a2bfe6f..da688703fc4 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -2594,7 +2594,7 @@ def test_detect_platform_hint_from_filename_case_insensitive( assert result == expected_platform -def test_component_batching_beta_branch_40_per_batch( +def test_component_batching_beta_branch_groups_evenly( tmp_path: Path, mock_determine_integration_tests: Mock, mock_should_run_clang_tidy: Mock, @@ -2604,11 +2604,10 @@ def test_component_batching_beta_branch_40_per_batch( mock_determine_cpp_unit_tests: Mock, capsys: pytest.CaptureFixture[str], ) -> None: - """Test that beta/release branches create batches with 40 actual components each. + """Test that beta/release branches group every component and split evenly. For beta/release branches, all components should be groupable (not isolated), - and each batch should contain 40 actual components with weight 1 each. - This matches the original behavior before consolidation. + so they share one grouped build per runner and spread evenly. """ # Create 120 test components with test files component_names = [f"comp_{i:03d}" for i in range(120)] @@ -2672,15 +2671,9 @@ def test_component_batching_beta_branch_40_per_batch( assert "component_test_batches" in output batches = output["component_test_batches"] - # Should have 3 batches (120 components / 40 per batch = 3) - assert len(batches) == 3, f"Expected 3 batches, got {len(batches)}" - - # Each batch should have approximately 40 components (all weight=1, groupable) - for i, batch in enumerate(batches): - batch_components = batch["components"].split() - assert len(batch_components) == 40, ( - f"Batch {i} should have 40 components, got {len(batch_components)}" - ) + # One grouped esp32-idf build plus 5 s per extra component is 640 s, + # so two runners of 60 components each + assert [len(batch["components"].split()) for batch in batches] == [60, 60] # Verify all 120 components are in batches all_components = [] diff --git a/tests/script/test_split_components_for_ci.py b/tests/script/test_split_components_for_ci.py new file mode 100644 index 00000000000..88a1a17b333 --- /dev/null +++ b/tests/script/test_split_components_for_ci.py @@ -0,0 +1,157 @@ +"""Unit tests for script/split_components_for_ci.py module.""" + +from collections.abc import Generator +from pathlib import Path +import sys +from unittest.mock import patch + +import pytest + +script_dir = str((Path(__file__).parent / ".." / ".." / "script").resolve()) +sys.path.insert(0, script_dir) + +import split_components_for_ci # noqa: E402 + +import script.helpers # noqa: E402 + + +@pytest.fixture +def tests_dir(tmp_path: Path) -> Generator[Path, None, None]: + """Point the helpers at an empty tests/components tree.""" + path = tmp_path / "tests" / "components" + path.mkdir(parents=True) + with patch.object(script.helpers, "root_path", str(tmp_path)): + yield path + + +def _add_component(tests_dir: Path, name: str, files: list[str]) -> None: + comp_dir = tests_dir / name + comp_dir.mkdir() + for file in files: + (comp_dir / file).write_text(f"# {name}\n") + + +def test_build_seconds() -> None: + """Known platforms use the table; nrf52 boards share one estimate.""" + build_seconds = split_components_for_ci.build_seconds + assert build_seconds("host") == 10 + assert build_seconds("esp32-idf") == 45 + assert ( + build_seconds("nrf52-xiao-ble") == split_components_for_ci.NRF52_BUILD_SECONDS + ) + assert ( + build_seconds("esp32-s3-idf") == split_components_for_ci.DEFAULT_BUILD_SECONDS + ) + + +def test_make_item_isolated_and_grouped(tests_dir: Path) -> None: + """Isolated components own every build; groupable ones join shared builds.""" + _add_component( + tests_dir, + "comp", + [ + "test.esp32-idf.yaml", + "test.host.yaml", + "test-extra.esp32-idf.yaml", + "test-only.rp2040-ard.yaml", + "validate.esp32-idf.yaml", + ], + ) + + isolated = split_components_for_ci._make_item( + tests_dir, "comp", "isolated_comp", True + ) + assert isolated.own_seconds == 45 + 10 + 45 + 50 + assert isolated.grouped_builds == {} + + # The rp2040 variant has no base test to group with, so it always runs + grouped = split_components_for_ci._make_item(tests_dir, "comp", "i2c", False) + assert grouped.own_seconds == 50 + assert grouped.grouped_builds == {("i2c", "esp32-idf"): 90, ("i2c", "host"): 10} + + +def test_grouped_component_joins_existing_build() -> None: + """A second member turns a lone build into a group that skips variants.""" + item = split_components_for_ci._BatchItem + batch = split_components_for_ci._Batch() + batch.add(item("a", 0, {("i2c", "esp32-idf"): 90})) + assert batch.seconds == 90 + + other = item("b", 0, {("i2c", "esp32-idf"): 45, ("i2c", "host"): 10}) + grouped = 45 + split_components_for_ci.GROUPED_COMPONENT_SECONDS + assert batch.added_seconds(other) == grouped - 90 + 10 + batch.add(other) + assert batch.seconds == grouped + 10 + + +def test_isolated_components_balance_across_runners(tests_dir: Path) -> None: + """Heavy isolated components spread out and light ones fill the gaps.""" + platforms = ["esp32-idf", "esp8266-ard", "rp2040-ard", "bk72xx-ard"] + for name in ("big_a", "big_b"): + _add_component( + tests_dir, + name, + [f"test.{p}.yaml" for p in platforms] + + [f"test-variant.{p}.yaml" for p in platforms], + ) + for name in ("small_a", "small_b"): + _add_component(tests_dir, name, ["test.esp32-idf.yaml"]) + + batches, _ = split_components_for_ci.create_intelligent_batches( + components=["big_a", "big_b", "small_a", "small_b"], + tests_dir=tests_dir, + target_seconds=500, + directly_changed={"big_a", "big_b", "small_a", "small_b"}, + ) + + assert sorted(sorted(batch) for batch in batches) == [ + ["big_a", "small_a"], + ["big_b", "small_b"], + ] + + +def test_groupable_variants_skipped_when_grouped(tests_dir: Path) -> None: + """Grouped members skip their variants, so the group fits one runner.""" + names = [f"comp_{i:02d}" for i in range(12)] + for name in names: + _add_component( + tests_dir, + name, + ["test.esp32-idf.yaml", "test-a.esp32-idf.yaml", "test-b.esp32-idf.yaml"], + ) + + batches, _ = split_components_for_ci.create_intelligent_batches( + components=names, tests_dir=tests_dir, target_seconds=400 + ) + + assert batches == [names] + + +def test_groupable_components_split_evenly(tests_dir: Path) -> None: + """A group too large for one runner spreads evenly.""" + platforms = ["esp32-idf", "esp8266-ard", "rp2040-ard", "bk72xx-ard"] + names = [f"comp_{i:02d}" for i in range(12)] + for name in names: + _add_component(tests_dir, name, [f"test.{p}.yaml" for p in platforms]) + + batches, _ = split_components_for_ci.create_intelligent_batches( + components=names, tests_dir=tests_dir, target_seconds=100 + ) + + assert sorted(len(batch) for batch in batches) == [2, 2, 2, 3, 3] + assert sorted(c for batch in batches for c in batch) == names + + +def test_make_item_reads_tests_dir(tmp_path: Path) -> None: + """Costs come from the given tests_dir, not the repository tree.""" + other = tmp_path / "other" + other.mkdir() + _add_component(other, "comp", ["test.esp32-idf.yaml"]) + + item = split_components_for_ci._make_item(other, "comp", "isolated_comp", True) + assert item.own_seconds == 45 + + +def test_balance_batches_empty() -> None: + """No components means no runners.""" + assert split_components_for_ci.balance_batches([], 600) == [] From 3bae9e4ee2fddbd46711386d3fad917051f8dfcb Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:33:50 -0500 Subject: [PATCH 116/133] [climate] Fix the restore state memset warning on every platform (#19983) --- esphome/components/climate/climate.cpp | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/esphome/components/climate/climate.cpp b/esphome/components/climate/climate.cpp index 494efd8edc4..d08952115bd 100644 --- a/esphome/components/climate/climate.cpp +++ b/esphome/components/climate/climate.cpp @@ -368,19 +368,11 @@ optional Climate::restore_state_() { } void Climate::save_state_(const ClimateTraits &traits) { -#if (defined(USE_ESP32) || defined(USE_ESP8266)) && !defined(CLANG_TIDY) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wclass-memaccess" -#define TEMP_IGNORE_MEMACCESS -#endif ClimateDeviceRestoreState state{}; - // initialize as zero (including padding) to prevent random data on stack triggering erase + // initialize as zero (including padding) to prevent random data on stack triggering erase; + // the void * cast tells GCC the bytewise clear of a non-trivial type is intentional // NOLINTNEXTLINE(bugprone-raw-memory-call-on-non-trivial-type) -- intentional bytewise zero for RTC save - memset(&state, 0, sizeof(ClimateDeviceRestoreState)); -#ifdef TEMP_IGNORE_MEMACCESS -#pragma GCC diagnostic pop -#undef TEMP_IGNORE_MEMACCESS -#endif + memset(static_cast(&state), 0, sizeof(ClimateDeviceRestoreState)); state.mode = this->mode; if (traits.has_feature_flags(CLIMATE_SUPPORTS_TWO_POINT_TARGET_TEMPERATURE | From e9f0a53ec97788b3aa8f5032837aab2ede76df35 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:34:17 -0500 Subject: [PATCH 117/133] [display] Fix the null pointer comparison warning in the display log macro (#19984) --- esphome/components/display/display.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/esphome/components/display/display.h b/esphome/components/display/display.h index bebeba8dcb2..cc75eb70b8f 100644 --- a/esphome/components/display/display.h +++ b/esphome/components/display/display.h @@ -287,13 +287,15 @@ template class DisplayWriter { // Type alias for Display writer - uses optimized DisplayWriter instead of std::function using display_writer_t = DisplayWriter; +// Callers usually pass `this`; checking a copy keeps GCC's -Wnonnull-compare quiet. #define LOG_DISPLAY(prefix, type, obj) \ - if ((obj) != nullptr) { \ + if (auto *log_display_obj = (obj); log_display_obj != nullptr) { \ ESP_LOGCONFIG(TAG, \ prefix type "\n" \ "%s Rotations: %d °\n" \ "%s Dimensions: %dpx x %dpx", \ - prefix, (obj)->rotation_, prefix, (obj)->get_width(), (obj)->get_height()); \ + prefix, log_display_obj->rotation_, prefix, log_display_obj->get_width(), \ + log_display_obj->get_height()); \ } /// Turn the pixel OFF. From 2de4f4cf8151bf7084145e8756b82b2379e6af8d Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:34:43 -0500 Subject: [PATCH 118/133] [speaker] Fix the fall through warning in the media player turn off command (#19987) --- .../components/speaker/media_player/speaker_media_player.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/speaker/media_player/speaker_media_player.cpp b/esphome/components/speaker/media_player/speaker_media_player.cpp index 06525235d47..cac2f6649c8 100644 --- a/esphome/components/speaker/media_player/speaker_media_player.cpp +++ b/esphome/components/speaker/media_player/speaker_media_player.cpp @@ -224,7 +224,7 @@ void SpeakerMediaPlayer::watch_media_commands_() { break; case media_player::MEDIA_PLAYER_COMMAND_TURN_OFF: this->is_turn_off_ = true; - // Intentional Fall-through + [[fallthrough]]; #endif case media_player::MEDIA_PLAYER_COMMAND_STOP: // Pipelines do not stop immediately after calling the stop command, so confirm its stopped before unpausing. From e500ece4d5ca69471d61f50d31f9f15fa4e66965 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:35:01 -0500 Subject: [PATCH 119/133] [uart] Fix a maybe uninitialized warning in the fixed size read (#19988) --- esphome/components/uart/uart.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/esphome/components/uart/uart.h b/esphome/components/uart/uart.h index eda5b72ea82..d1c67733634 100644 --- a/esphome/components/uart/uart.h +++ b/esphome/components/uart/uart.h @@ -37,9 +37,10 @@ class UARTDevice { bool read_array(uint8_t *data, size_t len) { return this->parent_->read_array(data, len); } template optional> read_array() { // NOLINT - std::array res; - if (!this->read_array(res.data(), N)) { - return {}; + // One object returned on both paths; a separate empty optional trips GCC's -Wmaybe-uninitialized. + optional> res(std::in_place); + if (!this->read_array(res->data(), N)) { + res.reset(); } return res; } From 36bcce563ab7f72d2b8864dd1c26bb807c701fbc Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:35:15 -0500 Subject: [PATCH 120/133] [sensor] Fix a double promotion warning in the NTC resistance filter (#19985) --- esphome/components/sensor/filter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esphome/components/sensor/filter.cpp b/esphome/components/sensor/filter.cpp index dbd6f4d34b8..9b9d7c033ae 100644 --- a/esphome/components/sensor/filter.cpp +++ b/esphome/components/sensor/filter.cpp @@ -450,7 +450,7 @@ optional ToNTCResistanceFilter::new_value(float value) { } double k = 273.15; // https://de.wikipedia.org/wiki/Steinhart-Hart-Gleichung#cite_note-stein2_s4-3 - double t = value + k; + double t = static_cast(value) + k; double y = (this->a_ - 1 / (t)) / (2 * this->c_); double x = sqrt(pow(this->b_ / (3 * this->c_), 3) + y * y); double resistance = exp(pow(x - y, 1 / 3.0) - pow(x + y, 1 / 3.0)); From 13463422507ebd7eee7a97403a65d1e15337b8df Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:35:41 -0500 Subject: [PATCH 121/133] [light] Fix a double promotion warning in the light test config (#19989) --- tests/components/light/common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/components/light/common.yaml b/tests/components/light/common.yaml index c40cfd5e3a3..046acef9ce6 100644 --- a/tests/components/light/common.yaml +++ b/tests/components/light/common.yaml @@ -188,7 +188,7 @@ number: then: - light.turn_on: id: test_monochromatic_light - brightness: !lambda "return x / 100.0;" + brightness: !lambda "return x / 100.0f;" output: - platform: template From 5d4b436fc003da8201a52a52f0aa54e181ca3165 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:35:56 -0500 Subject: [PATCH 122/133] [mk2pvrouter] Fix an ignored qualifiers warning in the group length check (#19986) --- esphome/components/mk2pvrouter/mk2pvrouter.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/esphome/components/mk2pvrouter/mk2pvrouter.cpp b/esphome/components/mk2pvrouter/mk2pvrouter.cpp index 0c0476fb11e..f8aa1003bc4 100644 --- a/esphome/components/mk2pvrouter/mk2pvrouter.cpp +++ b/esphome/components/mk2pvrouter/mk2pvrouter.cpp @@ -1,5 +1,6 @@ #include "mk2pvrouter.h" #include "esphome/core/log.h" +#include #include namespace esphome::mk2pvrouter { @@ -48,7 +49,7 @@ uint8_t Mk2PVRouter::calculate_crc_(const char *grp, size_t grp_len) { // Verifies the CRC of a group against its trailing CRC byte. bool Mk2PVRouter::check_crc_(const char *grp, const char *grp_end) { const auto grp_len = grp_end - grp; - if (grp_len < static_cast(CRC_SUFFIX_LEN)) { + if (grp_len < static_cast(CRC_SUFFIX_LEN)) { ESP_LOGE(TAG, "Empty or too short group"); return false; } From 3ed1bc7e2077b666ae36c3c2f9581e458eeff0b0 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:50:46 -0500 Subject: [PATCH 123/133] [esp32_ble_tracker] Guard Bluedroid against stale queued connection requests (#19375) --- .../components/esp32_ble_tracker/__init__.py | 24 ++++++ .../esp32_ble_tracker/bluedroid_stubs.cpp | 48 ++++++++++++ esphome/core/defines.h | 1 + .../test_direct_conn_guard.py | 73 +++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 esphome/components/esp32_ble_tracker/bluedroid_stubs.cpp create mode 100644 tests/component_tests/esp32_ble_tracker/test_direct_conn_guard.py diff --git a/esphome/components/esp32_ble_tracker/__init__.py b/esphome/components/esp32_ble_tracker/__init__.py index 205acc55a53..fc52b92ee7b 100644 --- a/esphome/components/esp32_ble_tracker/__init__.py +++ b/esphome/components/esp32_ble_tracker/__init__.py @@ -436,6 +436,25 @@ async def to_code(config: ConfigType) -> None: cg.add_define("USE_ESP32_BLE_SOFTWARE_COEXISTENCE") +# First tagged release per series with espressif/esp-idf@82e71c1767 (see bluedroid_stubs.cpp). +# A series without an entry keeps the guard until a fixed release is tagged; the guard is +# harmless on fixed sources. The 5.4, 5.5 and 6.1 branches carry the fix but have no tag yet. +DIRECT_CONN_FIX_VERSIONS = { + (5, 2): cv.Version(5, 2, 8), + (5, 3): cv.Version(5, 3, 6), + (6, 0): cv.Version(6, 0, 3), +} +DIRECT_CONN_FIX_ALL_FROM = cv.Version(6, 2, 0) + + +def _needs_direct_conn_guard() -> bool: + ver = idf_version() + if ver >= DIRECT_CONN_FIX_ALL_FROM: + return False + fixed = DIRECT_CONN_FIX_VERSIONS.get((ver.major, ver.minor)) + return fixed is None or ver < fixed + + # This needs to be run as a job with very low priority so that all components have # chance to call register_ble_tracker and register_client before the list is checked # and added to the global defines list. @@ -452,6 +471,11 @@ async def _add_ble_features() -> None: if BLEFeatures.ESP_BT_DEVICE in required_features: cg.add_define("USE_ESP32_BLE_DEVICE") cg.add_define("USE_ESP32_BLE_UUID") + if cg.get_slot_count(CLIENT_COUNT_DEFINE) and _needs_direct_conn_guard(): + # --undefined keeps the wrapper, libsrc.a is scanned before the IDF libraries + cg.add_define("USE_ESP32_BLE_TRACKER_DIRECT_CONN_GUARD") + cg.add_build_flag("-Wl,--wrap=l2cble_init_direct_conn") + cg.add_build_flag("-Wl,--undefined=__wrap_l2cble_init_direct_conn") ESP32_BLE_START_SCAN_ACTION_SCHEMA = cv.Schema( diff --git a/esphome/components/esp32_ble_tracker/bluedroid_stubs.cpp b/esphome/components/esp32_ble_tracker/bluedroid_stubs.cpp new file mode 100644 index 00000000000..5cedc22d069 --- /dev/null +++ b/esphome/components/esp32_ble_tracker/bluedroid_stubs.cpp @@ -0,0 +1,48 @@ +/* + * Bluedroid queues outgoing BLE connections as raw link block pointers and does + * not drop them when the block is released, so btm_send_pending_direct_conn() + * can start a connect on a released block and l2c_link_timeout() later crashes + * on its null timer parameter. Mirrors espressif/esp-idf@82e71c1767; codegen + * only enables it for releases without that commit. + */ + +#include "esphome/core/defines.h" + +#ifdef USE_ESP32_BLE_TRACKER_DIRECT_CONN_GUARD + +#include +#include +#include "esphome/core/log.h" + +namespace esphome::esp32_ble_tracker { +static const char *const TAG = "esp32_ble_tracker"; +} // namespace esphome::esp32_ble_tracker + +static_assert(ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(6, 2, 0), + "ESP-IDF 6.2 and later have the fix, this guard should not be enabled (esphome/esphome#19373)"); + +// NOLINTBEGIN(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) +extern "C" { + +bool __real_l2cble_init_direct_conn(void *p_lcb); +void l2cu_release_lcb(void *p_lcb); + +bool __wrap_l2cble_init_direct_conn(void *p_lcb) { + // in_use is the first member of the private tL2C_LCB (checked ESP-IDF 5.0 to 6.1) + const auto *in_use = static_cast(p_lcb); + if (p_lcb == nullptr || *in_use == 0) { + ESP_LOGW(esphome::esp32_ble_tracker::TAG, "Dropped queued connect on a released link block"); + return false; + } + const bool started = __real_l2cble_init_direct_conn(p_lcb); + // Every failure path releases the block except unknown device, also fixed upstream + if (!started && *in_use != 0) { + l2cu_release_lcb(p_lcb); + } + return started; +} + +} // extern "C" +// NOLINTEND(bugprone-reserved-identifier,cert-dcl37-c,cert-dcl51-cpp,readability-identifier-naming) + +#endif // USE_ESP32_BLE_TRACKER_DIRECT_CONN_GUARD diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 88f35dac46b..9b3252fa05c 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -389,6 +389,7 @@ #define USE_ESP32_BLE_SERVER_ON_CONNECT #define USE_ESP32_BLE_SERVER_ON_DISCONNECT #define USE_ESP32_BLE_TRACKER +#define USE_ESP32_BLE_TRACKER_DIRECT_CONN_GUARD #define USE_BLE_GATT_CLIENT #define ESPHOME_BLE_GATT_CLIENT_COUNT 1 #define ESPHOME_ESP32_BLE_TRACKER_LISTENER_COUNT 1 diff --git a/tests/component_tests/esp32_ble_tracker/test_direct_conn_guard.py b/tests/component_tests/esp32_ble_tracker/test_direct_conn_guard.py new file mode 100644 index 00000000000..d3aa2ba8787 --- /dev/null +++ b/tests/component_tests/esp32_ble_tracker/test_direct_conn_guard.py @@ -0,0 +1,73 @@ +"""The Bluedroid queued connection guard is emitted only for client builds on unfixed ESP-IDF.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +from esphome import config_validation as cv +from esphome.components import esp32_ble_tracker +from esphome.core import CORE + +# Spelled out so a typo in the component's flags fails here +_GUARD_FLAGS = { + "-Wl,--wrap=l2cble_init_direct_conn", + "-Wl,--undefined=__wrap_l2cble_init_direct_conn", +} + + +def _pin_idf(monkeypatch: pytest.MonkeyPatch, idf: str) -> None: + monkeypatch.setattr(esp32_ble_tracker, "idf_version", lambda: cv.Version.parse(idf)) + + +@pytest.mark.parametrize( + ("config_file", "idf", "expected"), + [ + pytest.param("scan_window_raised.yaml", "5.5.5", True, id="client"), + pytest.param("scan_window_raised.yaml", "6.0.3", False, id="client_fixed_idf"), + pytest.param("scan_window_scan_only.yaml", "5.5.5", False, id="scan_only"), + ], +) +def test_guard_only_in_client_builds_on_unfixed_idf( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], + monkeypatch: pytest.MonkeyPatch, + config_file: str, + idf: str, + expected: bool, +) -> None: + _pin_idf(monkeypatch, idf) + generate_main(component_config_path(config_file)) + assert (CORE.build_flags >= _GUARD_FLAGS) is expected + assert CORE.build_flags.isdisjoint(_GUARD_FLAGS) is not expected + defines = {define.name for define in CORE.defines} + assert ("USE_ESP32_BLE_TRACKER_DIRECT_CONN_GUARD" in defines) is expected + + +@pytest.mark.parametrize( + ("idf", "expected"), + [ + ("5.1.6", True), # series that never got the fix + ("5.2.7", True), + ("5.2.8", False), + ("5.3.5", True), + ("5.3.6", False), + ("5.4.4", True), + ("5.4.5", True), # no fixed 5.4, 5.5 or 6.1 tag yet + ("5.5.5", True), + ("5.5.6", True), + ("6.0.2", True), + ("6.0.3", False), + ("6.1.0", True), + ("6.1.1", True), + ("6.2.0", False), + ("7.0.0", False), + ], +) +def test_needs_direct_conn_guard( + monkeypatch: pytest.MonkeyPatch, idf: str, expected: bool +) -> None: + _pin_idf(monkeypatch, idf) + assert esp32_ble_tracker._needs_direct_conn_guard() is expected From 0ed997da926cccfd14ce043d4fc10c1597abc25f Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 13:55:30 -0500 Subject: [PATCH 124/133] [espidf] Add --skip-bootloader for OTA only builds (#19925) --- esphome/__main__.py | 47 +++- esphome/build_gen/espidf.py | 74 ++++++ esphome/components/esp32/__init__.py | 27 ++- esphome/core/__init__.py | 3 + esphome/espidf/toolchain.py | 82 ++++++- script/check_idf_py_equivalence.py | 77 +++++-- tests/script/test_check_idf_py_equivalence.py | 80 ++++++- tests/unit_tests/build_gen/test_espidf.py | 72 ++++++ tests/unit_tests/test_download_types.py | 29 +++ tests/unit_tests/test_espidf_toolchain.py | 218 +++++++++++++++++- tests/unit_tests/test_main.py | 104 +++++++++ 11 files changed, 765 insertions(+), 48 deletions(-) diff --git a/esphome/__main__.py b/esphome/__main__.py index e4213e8edb9..cbeb9ff01ce 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -148,6 +148,7 @@ class ArgsProtocol(Protocol): file: str | None no_logs: bool only_generate: bool + skip_bootloader: bool show_secrets: bool dashboard: bool configuration: str @@ -827,6 +828,14 @@ def write_cpp_file() -> int: def compile_program(args: ArgsProtocol, config: ConfigType) -> int: + if CORE.skip_bootloader and not (CORE.is_esp32 and CORE.using_toolchain_esp_idf): + # Info, not a warning: an orchestrator cannot see YAML toolchain + # overrides, this is its expected no-op, and a full build is safe. + _LOGGER.info( + "--skip-bootloader ignored: only supported on ESP32 with the " + "esp-idf toolchain" + ) + CORE.skip_bootloader = False # Keep this gate here, NOT in config validation: device-builder needs # `esphome config` to keep succeeding with placeholders so onboarding can run. if CONF_WIFI in config: @@ -857,7 +866,10 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int: return rc # Create factory.bin, ota.bin, and firmware.elf copy - toolchain.create_factory_bin() + if not toolchain.create_factory_bin(): + # A build whose factory image could not be produced must not + # exit 0; downloads would serve an image from an older build. + return 1 toolchain.create_ota_bin() toolchain.create_elf_copy() from esphome.build_helpers.idedata import warn_if_idedata_missing @@ -975,8 +987,9 @@ def upload_using_esptool( # and partitions included where the target needs them) image = native.get_factory_firmware_path() if not image.is_file(): + hint = getattr(native, "missing_image_hint", lambda: None)() raise EsphomeError( - f"{image} does not exist; compile the configuration first" + hint or f"{image} does not exist; compile the configuration first" ) flash_images = [FlashImage(path=image, offset="0x0")] else: @@ -1379,6 +1392,12 @@ def _upload_via_native_api( ota_type = espota2.OTA_TYPE_UPDATE_PARTITION_TABLE elif getattr(args, "bootloader", False): check_partition_access("--bootloader") + if ( + getattr(args, "file", None) is None + and (native := native_backend()) + and (hint := getattr(native, "missing_image_hint", lambda: None)()) + ): + raise EsphomeError(hint) binary = CORE.bootloader_bin ota_type = espota2.OTA_TYPE_UPDATE_BOOTLOADER if getattr(args, "file", None) is not None: @@ -1754,6 +1773,18 @@ def command_logs(args: ArgsProtocol, config: ConfigType) -> int | None: def command_run(args: ArgsProtocol, config: ConfigType) -> int | None: + if ( + CORE.skip_bootloader + and CORE.is_esp32 + and CORE.using_toolchain_esp_idf + and any( + get_port_type(device) == PortType.SERIAL for device in (args.device or []) + ) + ): + # Fail before the compile: the result could never flash over serial. + # Elsewhere the flag is ignored, so serial stays fine there. + _LOGGER.error("--skip-bootloader builds cannot be flashed over serial") + return 1 exit_code = write_cpp(config) if exit_code != 0: return exit_code @@ -2139,6 +2170,15 @@ SIMPLE_CONFIG_ACTIONS = [ ] +def _add_skip_bootloader_arg(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--skip-bootloader", + help="Do not build the bootloader or the factory image; " + "the result can only be flashed over OTA.", + action="store_true", + ) + + def _add_states_args(parser: argparse.ArgumentParser) -> None: """Add mutually exclusive ``--states``/``--no-states`` flags to a parser. @@ -2280,6 +2320,7 @@ def parse_args(argv): help="Only generate source code, do not compile.", action="store_true", ) + _add_skip_bootloader_arg(parser_compile) parser_upload = subparsers.add_parser( "upload", @@ -2376,6 +2417,7 @@ def parse_args(argv): parser_run.add_argument( "--no-logs", help="Disable starting logs.", action="store_true" ) + _add_skip_bootloader_arg(parser_run) _add_states_args(parser_run) @@ -2635,6 +2677,7 @@ def run_esphome(argv): CORE.config_path = conf_path CORE.dashboard = args.dashboard + CORE.skip_bootloader = getattr(args, "skip_bootloader", False) if args.toolchain is not None: # CLI toolchain wins over esp32.toolchain in YAML. CORE.toolchain = args.toolchain diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 124fa3ddb10..3838d076026 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -3,6 +3,7 @@ import json import logging from pathlib import Path +import re import textwrap from esphome.build_helpers import pch @@ -120,6 +121,78 @@ def _cmake_quote(value: str) -> str: return f'"{escaped}"' +# CONFIG_APP_BUILD_BOOTLOADER is hidden and force-selected, so it can only be +# cleared at the CMake level (the same state IDF's RAM-app build type uses). +# The macro is IDF's __build_process_project_includes plus two added lines; +# the flag is ignored and the bootloader builds as usual if IDF changes it. +IDF_BOOTLOADER_OVERRIDE = """\ +# ESPHome bootloader skip switch; see esphome/espidf/toolchain.py. +if(ESPHOME_SKIP_BOOTLOADER) + macro(__build_process_project_includes) + idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE) + include(${sdkconfig_cmake}) + set(CONFIG_APP_BUILD_BOOTLOADER "") + # bt's CMakeLists reads the lowercase idf_target that the (now + # skipped) bootloader project_include leaks; keep it defined, or + # its empty TARGET_SRC_NAME sends file(GLOB_RECURSE) across /. + idf_build_get_property(idf_target IDF_TARGET) + idf_build_get_property(build_properties __BUILD_PROPERTIES) + foreach(build_property ${build_properties}) + idf_build_get_property(val ${build_property}) + set(${build_property} "${val}") + endforeach() + idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS) + foreach(component_target ${build_component_targets}) + __component_get_property(dir ${component_target} COMPONENT_DIR) + __component_get_property(_name ${component_target} COMPONENT_NAME) + set(COMPONENT_NAME ${_name}) + set(COMPONENT_DIR ${dir}) + set(COMPONENT_PATH ${dir}) + if(EXISTS ${COMPONENT_DIR}/project_include.cmake) + include(${COMPONENT_DIR}/project_include.cmake) + endif() + endforeach() + endmacro() +endif() +""" + +# The lines the override adds to IDF's macro; idf_macro_matches() below +# strips them before comparing with the live macro. +BOOTLOADER_OVERRIDE_ADDED_LINES = ( + 'set(CONFIG_APP_BUILD_BOOTLOADER "")', + "idf_build_get_property(idf_target IDF_TARGET)", +) + +_MACRO = re.compile( + r"macro\(__build_process_project_includes\)(.*?)endmacro\(\)", re.DOTALL +) + + +def _normalized_macro(text: str) -> list[str] | None: + """The macro body as comment-free, whitespace-collapsed lines.""" + if (match := _MACRO.search(text)) is None: + return None + return [ + re.sub(r"\s+", " ", line) + for raw in match.group(1).splitlines() + if (line := raw.split("#", 1)[0].strip()) + ] + + +_EXPECTED_MACRO = [ + line + for line in _normalized_macro(IDF_BOOTLOADER_OVERRIDE) + if line not in BOOTLOADER_OVERRIDE_ADDED_LINES +] + + +def idf_macro_matches(idf_path: Path) -> bool: + """Whether IDF's macro still matches the copy the override replays.""" + build_cmake = idf_path / "tools" / "cmake" / "build.cmake" + live = _normalized_macro(build_cmake.read_text(encoding="utf-8")) + return live == _EXPECTED_MACRO + + def get_project_cmakelists( minimal: bool = False, builtin_components: list[str] | None = None ) -> str: @@ -258,6 +331,7 @@ set(EXTRA_COMPONENT_DIRS ${{CMAKE_SOURCE_DIR}}/src) include($ENV{{IDF_PATH}}/tools/cmake/project.cmake) +{IDF_BOOTLOADER_OVERRIDE} {ldgen_override} {cpp_standard_options} diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 49a115c9c56..6a693eff89d 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -670,20 +670,29 @@ def get_download_types(storage_json): # No recorded firmware path means nothing was built; no downloads. if storage_json.firmware_bin_path is None: return [] - return [ - { - "title": "Factory format (Previously Modern)", - "description": "For use with ESPHome Web and other tools.", - "file": "firmware.factory.bin", - "download": f"{storage_json.name}.factory.bin", - }, + from esphome.espidf.toolchain import tree_skips_bootloader + + types = [] + # A --skip-bootloader tree deliberately has no factory image; an + # unreadable tree (PlatformIO, capability probes) reads as full. + if not tree_skips_bootloader(Path(storage_json.firmware_bin_path).parent): + types.append( + { + "title": "Factory format (Previously Modern)", + "description": "For use with ESPHome Web and other tools.", + "file": "firmware.factory.bin", + "download": f"{storage_json.name}.factory.bin", + } + ) + types.append( { "title": "OTA format (Previously Legacy)", "description": "For OTA updating a device.", "file": "firmware.ota.bin", "download": f"{storage_json.name}.ota.bin", - }, - ] + } + ) + return types def only_on_variant(*, supported=None, unsupported=None, msg_prefix="This feature"): diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index fa3d6bb64ed..30c03720665 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -589,6 +589,8 @@ class EsphomeCore: self.vscode = False # True if running in testing mode (disables validation checks for grouped testing) self.testing_mode = False + # True if this build skips the bootloader and factory image (OTA only) + self.skip_bootloader = False # The name of the node self.name: str | None = None # The friendly name of the node @@ -692,6 +694,7 @@ class EsphomeCore: from esphome.pins import PIN_SCHEMA_REGISTRY self.dashboard = False + self.skip_bootloader = False self.name = None self.friendly_name = None self.area = None diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index d73dbf6b7e5..ffd80825159 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -26,11 +26,14 @@ from esphome.core import CORE, EsphomeError from esphome.espidf import variant_to_idf_target from esphome.espidf.framework import check_esp_idf_install, get_framework_env from esphome.espidf.size_summary import print_summary -from esphome.helpers import add_git_ceiling_directory, get_bool_env, write_file +from esphome.helpers import add_git_ceiling_directory, get_bool_env, rmtree, write_file _LOGGER = logging.getLogger(__name__) DOMAIN = "espidf_toolchain" +# The -D that tells the generated CMakeLists to skip the in-tree bootloader +# build; also read back from CMakeCache.txt to identify a tree. +SKIP_BOOTLOADER_DEFINE = "ESPHOME_SKIP_BOOTLOADER" @dataclass @@ -38,6 +41,7 @@ class _CacheData: paths: dict[str, tuple] = field(default_factory=dict) env: dict[str, dict[str, str]] = field(default_factory=dict) cmake_output: dict[Path, str] = field(default_factory=dict) + skip_bootloader: bool | None = None cmake_tools: dict[Path, dict[str, Path]] = field(default_factory=dict) @@ -285,6 +289,48 @@ def _cache_entries_changed() -> bool: return any(cache.get(k) != v for k, v in _configure_defines().items()) +def tree_skips_bootloader(build_dir: Path) -> bool: + """Whether a configured tree was set up to skip the bootloader build. + + Total: an unreadable tree reads as the stock full build. + """ + try: + cache = _parse_cmakecache(build_dir / "CMakeCache.txt") + except (OSError, ValueError) as err: + _LOGGER.debug("Cannot read %s, assuming a full build: %s", build_dir, err) + return False + return cache.get(SKIP_BOOTLOADER_DEFINE) == "1" + + +def _skip_bootloader() -> bool: + """Whether this tree should not build a bootloader at all; per-run memo.""" + cache = _cache() + if cache.skip_bootloader is None: + cache.skip_bootloader = _compute_skip_bootloader() + return cache.skip_bootloader + + +def _compute_skip_bootloader() -> bool: + if not CORE.skip_bootloader: + return False + from esphome.build_gen.espidf import idf_macro_matches + + if not idf_macro_matches(_get_idf_path()): + _LOGGER.warning( + "--skip-bootloader ignored: IDF changed its bootloader macro; " + "building the bootloader" + ) + return False + return True + + +def missing_image_hint() -> str | None: + """Why an expected flash image is absent, for upload error messages.""" + if tree_skips_bootloader(_build_dir()): + return "this build was compiled with --skip-bootloader; recompile without it" + return None + + def _configure_defines() -> dict[str, str]: """Every ``-D`` idf.py passes to cmake, in its order.""" return { @@ -292,6 +338,9 @@ def _configure_defines() -> dict[str, str]: "PYTHON": _get_idf_tool("python"), "ESP_PLATFORM": "1", **_cache_entries(), + # ESPHome's own switch; idf.py never passes it and cmake keeps the + # cached value, so idf.py runs against the tree stay in the same mode. + SKIP_BOOTLOADER_DEFINE: "1" if _skip_bootloader() else "0", } @@ -312,6 +361,14 @@ def run_reconfigure(verbose: bool = False) -> int: """Run the CMake configure, with the arguments idf.py uses.""" build_dir = _build_dir() build_dir.mkdir(parents=True, exist_ok=True) + if _skip_bootloader() and not tree_skips_bootloader(build_dir): + # Flipping into skip mode: full-mode leftovers are stale for + # OTA --bootloader and downloads, and a partial cleanup would + # poison the flip back (deleted byproducts never regenerate). + for stale in ("bootloader", "bootloader-prefix"): + if (path := build_dir / stale).is_dir(): + rmtree(path) + get_factory_firmware_path().unlink(missing_ok=True) cmd = [_get_idf_tool("cmake"), "-G", "Ninja"] if _idf_py().binary_dir_arg: cmd += ["-B", str(build_dir)] @@ -861,6 +918,20 @@ def get_idedata() -> dict | None: def create_factory_bin() -> bool: """Create factory.bin by merging bootloader, partition table, and app.""" build_dir = CORE.relative_build_path("build") + if tree_skips_bootloader(build_dir): + # Nothing to merge, and nothing stale: the flip into skip mode + # already removed the factory image and the sub-build. + _LOGGER.info("Bootloader skipped; no factory image") + return True + if _merge_factory_bin(build_dir): + return True + # Never leave an image that does not match this build. + get_factory_firmware_path().unlink(missing_ok=True) + return False + + +def _merge_factory_bin(build_dir: Path) -> bool: + """Run the esptool merge for a full-build tree.""" flasher_args_path = build_dir / "flasher_args.json" if not flasher_args_path.is_file(): @@ -883,10 +954,11 @@ def create_factory_bin() -> bool: flash_data.get("flash_files", {}).items(), key=lambda kv: int(kv[0], 16) ): file_path = build_dir / fname - if file_path.is_file(): - sections.extend([addr, str(file_path)]) - else: - _LOGGER.warning("Flash file not found: %s", file_path) + if not file_path.is_file(): + # A partial factory image would not boot; never write one. + _LOGGER.error("Flash file not found: %s", file_path) + return False + sections.extend([addr, str(file_path)]) if not sections: _LOGGER.warning("No flash sections found") diff --git a/script/check_idf_py_equivalence.py b/script/check_idf_py_equivalence.py index ff7c58a5a60..c0aa3c0d5e5 100755 --- a/script/check_idf_py_equivalence.py +++ b/script/check_idf_py_equivalence.py @@ -30,14 +30,31 @@ WATCHED = ( "build/compile_commands.json", "build/project_description.json", "build/config/sdkconfig.h", - "build/bootloader/bootloader.bin", ) -# Ninja logs whose outputs mean real work when their recorded mtime changes. -# The top level re-logs the bootloader step's byproducts on every build, so -# the bootloader is judged by its own sub-build log instead. +# Only a tree that builds the bootloader has one to watch. +BOOTLOADER_BIN = "build/bootloader/bootloader.bin" +OVERRIDE_INEFFECTIVE = ( + "skip-mode tree built a bootloader; the IDF_BOOTLOADER_OVERRIDE macro in " + "esphome/build_gen/espidf.py is not taking effect" +) +# Ninja logs whose outputs mean real work when their recorded mtime changes; +# the bootloader is judged by its own sub-build log when one exists. TOP_NINJA_LOG = "build/.ninja_log" -NINJA_LOGS = (TOP_NINJA_LOG, "build/bootloader/.ninja_log") + + +def _ninja_logs(skip_bootloader: bool) -> list[str]: + """A missing sub-build log stays an error in the mode that requires one.""" + logs = [TOP_NINJA_LOG] + if not skip_bootloader: + logs.append("build/bootloader/.ninja_log") + return logs + + BOOTLOADER_BYPRODUCT = re.compile(r"(^|/build/)bootloader/") +MACRO_CHANGED = ( + "IDF changed __build_process_project_includes; update " + "IDF_BOOTLOADER_OVERRIDE in esphome/build_gen/espidf.py" +) WORK_SUFFIXES = (".obj", ".o", ".a", ".elf", ".map", ".bin", ".ld") DEFAULT_GLOB = "tests/test_build_components/build/.esphome/build/*" @@ -46,19 +63,24 @@ def _digest(path: Path) -> str | None: return hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None -def watched(name: str) -> list[str]: - """Files that change if idf.py configures or builds differently.""" - return [*WATCHED, f"build/{name}.elf", f"build/{name}.bin"] +def watched(name: str, skip_bootloader: bool) -> list[str]: + """The files idf.py must leave untouched for this tree's mode.""" + files = [*WATCHED, f"build/{name}.elf", f"build/{name}.bin"] + if not skip_bootloader: + files.append(BOOTLOADER_BIN) + return files -def _snapshot(build_path: Path, name: str) -> dict[str, str | None]: - return {f: _digest(build_path / f) for f in watched(name)} +def _snapshot( + build_path: Path, name: str, skip_bootloader: bool +) -> dict[str, str | None]: + return {f: _digest(build_path / f) for f in watched(name, skip_bootloader)} -def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]: +def _ninja_mtimes(build_path: Path, logs: list[str]) -> dict[tuple[str, str], str]: """(log, output) -> recorded mtime; compaction-safe, unlike a line count.""" mtimes = {} - for name in NINJA_LOGS: + for name in logs: log = build_path / name lines = log.read_text(encoding="utf-8").splitlines() if log.is_file() else [] for fields in (line.split("\t") for line in lines if not line.startswith("#")): @@ -69,10 +91,12 @@ def _ninja_mtimes(build_path: Path) -> dict[tuple[str, str], str]: return mtimes -def _log_problems(build_path: Path, mtimes: dict[tuple[str, str], str]) -> list[str]: +def _log_problems( + build_path: Path, mtimes: dict[tuple[str, str], str], logs: list[str] +) -> list[str]: """A missing or unparsable ninja log would otherwise compare as unchanged.""" problems = [] - for log in NINJA_LOGS: + for log in logs: if not (build_path / log).is_file(): problems.append(f"missing {log}") elif not any(k[0] == log and k[1].endswith(WORK_SUFFIXES) for k in mtimes): @@ -101,12 +125,22 @@ def _setup_core(build_path: Path, description: dict) -> tuple[str, str]: def check(build_path: Path) -> list[str]: """Return the problems found in one build tree.""" # pylint: disable=protected-access + from esphome.build_gen.espidf import idf_macro_matches + from esphome.core import CORE from esphome.espidf import toolchain description = json.loads( (build_path / "build" / "project_description.json").read_text(encoding="utf-8") ) name, version = _setup_core(build_path, description) + # Reconfiguring must not flip the tree's bootloader mode: the check + # validates the shape the build produced, not this process's flags. + skip_bootloader = toolchain.tree_skips_bootloader(build_path / "build") + CORE.skip_bootloader = skip_bootloader + # A prior tree's memoized decision must not leak into this one. + toolchain._cache().skip_bootloader = None + if not idf_macro_matches(toolchain._get_idf_path(version)): + return [MACRO_CHANGED] # ESP-IDF's openthread stamps the configure time into its compile flags; # pin it before the env is cached so both configures get the same value. os.environ["SOURCE_DATE_EPOCH"] = "0" @@ -122,11 +156,14 @@ def check(build_path: Path) -> list[str]: return [f"ESPHome's CMake configure failed with exit code {rc}"] if (rc := toolchain._run_ninja("all", verbose=False, jobs=None)) != 0: return [f"ESPHome's ninja build failed with exit code {rc}"] - before = _snapshot(build_path, name) - mtimes_before = _ninja_mtimes(build_path) + before = _snapshot(build_path, name, skip_bootloader) + logs = _ninja_logs(skip_bootloader) + mtimes_before = _ninja_mtimes(build_path, logs) # A moved or renamed output would otherwise compare as "unchanged". problems = [f"missing {f}" for f, digest in before.items() if digest is None] - if problems := problems + _log_problems(build_path, mtimes_before): + if skip_bootloader and (build_path / BOOTLOADER_BIN).is_file(): + problems.append(OVERRIDE_INEFFECTIVE) + if problems := problems + _log_problems(build_path, mtimes_before, logs): return problems for action in ("reconfigure", "build"): result = subprocess.run( @@ -139,10 +176,10 @@ def check(build_path: Path) -> list[str]: ) if result.returncode != 0: return [f"idf.py {action} failed:\n{result.stdout}{result.stderr}"] - after = _snapshot(build_path, name) - mtimes_after = _ninja_mtimes(build_path) + after = _snapshot(build_path, name, skip_bootloader) + mtimes_after = _ninja_mtimes(build_path, logs) problems = [f"idf.py changed {f}" for f in before if before[f] != after[f]] - problems += _log_problems(build_path, mtimes_after) + problems += _log_problems(build_path, mtimes_after, logs) for key in sorted(mtimes_before.keys() | mtimes_after.keys()): log, out = key if not out.endswith(WORK_SUFFIXES): diff --git a/tests/script/test_check_idf_py_equivalence.py b/tests/script/test_check_idf_py_equivalence.py index 6ded9a3a815..42dbdf4dc7d 100644 --- a/tests/script/test_check_idf_py_equivalence.py +++ b/tests/script/test_check_idf_py_equivalence.py @@ -14,6 +14,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent.parent / "script")) import check_idf_py_equivalence as guard # noqa: E402 +from esphome.build_gen import espidf as build_gen # noqa: E402 from esphome.core import CORE # noqa: E402 from esphome.espidf import toolchain # noqa: E402 @@ -25,12 +26,26 @@ def _reset_core() -> Iterator[None]: CORE.reset() -def _make_tree(tmp_path: Path) -> Path: +BOOTLOADER_LOG = "build/bootloader/.ninja_log" +ALL_LOGS = (guard.TOP_NINJA_LOG, BOOTLOADER_LOG) + + +def _make_tree(tmp_path: Path, skip_bootloader: bool = False) -> Path: + """A fake build tree; stock shape by default, or the skip shape + (define set to 1, no bootloader bin, no sub-build).""" tree = tmp_path / "config" / ".esphome" / "build" / "dev" build = tree / "build" - for name in (*guard.watched("dev"), *guard.NINJA_LOGS): + files = [ + *guard.watched("dev", skip_bootloader), + *guard._ninja_logs(skip_bootloader), + ] + for name in files: (tree / name).parent.mkdir(parents=True, exist_ok=True) (tree / name).write_bytes(b"x") + define = "1" if skip_bootloader else "0" + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED={define}\n" + ) (build / "project_description.json").write_text( json.dumps( { @@ -44,9 +59,10 @@ def _make_tree(tmp_path: Path) -> Path: "# ninja log v7\n1\t2\t10\tesp-idf/a.obj\t0\n" "1\t2\t10\tbootloader/bootloader.bin\t0\n" ) - (build / "bootloader" / ".ninja_log").write_text( - "# ninja log v7\n1\t2\t10\tbootloader.elf\t0\n" - ) + if not skip_bootloader: + (build / "bootloader" / ".ninja_log").write_text( + "# ninja log v7\n1\t2\t10\tbootloader.elf\t0\n" + ) (tree / "sdkconfig.dev").write_text("") return tree @@ -56,6 +72,7 @@ def _run_check( side_effect: Callable[[list[str]], None] = lambda cmd: None, rc: int = 0, esphome_rcs: tuple[int, int] = (0, 0), + macro_matches: bool = True, envs: list[dict[str, str]] | None = None, ) -> tuple[list[str], list[list[str]]]: """Run check() with idf.py replaced by ``side_effect``; return problems, calls. @@ -80,6 +97,7 @@ def _run_check( patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), patch.object(toolchain, "run_reconfigure", return_value=esphome_rcs[0]), patch.object(toolchain, "_run_ninja", return_value=esphome_rcs[1]), + patch.object(build_gen, "idf_macro_matches", return_value=macro_matches), patch.object(guard.subprocess, "run", side_effect=run), patch.dict(os.environ), ): @@ -168,9 +186,7 @@ def test_check_stops_when_idf_py_fails(tmp_path: Path) -> None: assert len(calls) == 1 -@pytest.mark.parametrize( - "remove", ["build/build.ninja", "build/dev.bin", *guard.NINJA_LOGS] -) +@pytest.mark.parametrize("remove", ["build/build.ninja", "build/dev.bin", *ALL_LOGS]) def test_check_fails_when_an_input_is_missing(tmp_path: Path, remove: str) -> None: """A moved or renamed output must not compare as unchanged.""" tree = _make_tree(tmp_path) @@ -197,7 +213,7 @@ def test_check_stops_when_the_esphome_baseline_fails( assert calls == [] -@pytest.mark.parametrize("log", guard.NINJA_LOGS) +@pytest.mark.parametrize("log", ALL_LOGS) def test_check_fails_when_a_ninja_log_has_no_entries(tmp_path: Path, log: str) -> None: """A log format change must not leave the rebuild check with nothing to compare.""" tree = _make_tree(tmp_path) @@ -223,6 +239,52 @@ def test_main_rejects_a_path_that_is_not_a_tree( mock_check.assert_not_called() +def test_check_resets_the_skip_memo_per_tree(tmp_path: Path) -> None: + """A second tree must not inherit the first tree's memoized mode.""" + tree = _make_tree(tmp_path) + CORE.skip_bootloader = True + toolchain._cache().skip_bootloader = True # leftover from a prior tree + problems, _ = _run_check(tree) + assert problems == [] + assert toolchain._skip_bootloader() is False + + +def test_check_accepts_a_skip_bootloader_tree(tmp_path: Path) -> None: + """No bootloader bin or sub-build is the skip shape, not missing input.""" + tree = _make_tree(tmp_path, skip_bootloader=True) + problems, calls = _run_check(tree) + assert problems == [] + assert len(calls) == 2 + # The baseline reconfigure must not flip the tree's mode. + assert CORE.skip_bootloader is True + + +def test_check_flags_an_ineffective_override(tmp_path: Path) -> None: + """A skip-mode tree that still built a bootloader must fail CI.""" + tree = _make_tree(tmp_path, skip_bootloader=True) + (tree / guard.BOOTLOADER_BIN).parent.mkdir(parents=True) + (tree / guard.BOOTLOADER_BIN).write_bytes(b"x") + problems, _ = _run_check(tree) + assert problems == [guard.OVERRIDE_INEFFECTIVE] + + +def test_check_requires_the_sub_log_on_a_stock_tree(tmp_path: Path) -> None: + """The mode comes from the define, so a vanished sub-build stays an error.""" + tree = _make_tree(tmp_path) + (tree / BOOTLOADER_LOG).unlink() + problems, calls = _run_check(tree) + assert problems == [f"missing {BOOTLOADER_LOG}"] + assert calls == [] + + +def test_check_fails_loudly_when_the_idf_macro_changed(tmp_path: Path) -> None: + """An IDF bump that rewrites the overridden macro must fail CI.""" + tree = _make_tree(tmp_path) + problems, calls = _run_check(tree, macro_matches=False) + assert problems == [guard.MACRO_CHANGED] + assert calls == [] + + def test_main_without_build_trees( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: diff --git a/tests/unit_tests/build_gen/test_espidf.py b/tests/unit_tests/build_gen/test_espidf.py index 7d53ee1e5e0..7ed6201b6aa 100644 --- a/tests/unit_tests/build_gen/test_espidf.py +++ b/tests/unit_tests/build_gen/test_espidf.py @@ -670,3 +670,75 @@ def test_pch_checksum_disabled_writes_nothing( _make_pch_project(tmp_path) write_pch_checksum() assert not CORE.relative_build_path(_PCH_SUM_PATH).exists() + + +# The macro body as shipped in tools/cmake/build.cmake; byte identical in +# IDF 5.5.5 and 6.1.0, so one fixture covers both supported versions. +IDF_BUILD_CMAKE = """\ +some_other_cmake() + +macro(__build_process_project_includes) + # Include the sdkconfig cmake file, since the following operations require + # knowledge of config values. + idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE) + include(${sdkconfig_cmake}) + + # Make each build property available as a read-only variable + idf_build_get_property(build_properties __BUILD_PROPERTIES) + foreach(build_property ${build_properties}) + idf_build_get_property(val ${build_property}) + set(${build_property} "${val}") + endforeach() + + idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS) + + # Include each component's project_include.cmake + foreach(component_target ${build_component_targets}) + __component_get_property(dir ${component_target} COMPONENT_DIR) + __component_get_property(_name ${component_target} COMPONENT_NAME) + set(COMPONENT_NAME ${_name}) + set(COMPONENT_DIR ${dir}) + set(COMPONENT_PATH ${dir}) # this is deprecated, users are encouraged to use COMPONENT_DIR; + # retained for compatibility + if(EXISTS ${COMPONENT_DIR}/project_include.cmake) + include(${COMPONENT_DIR}/project_include.cmake) + endif() + endforeach() +endmacro() +""" + + +def _write_idf_build_cmake(tmp_path: Path, text: str = IDF_BUILD_CMAKE) -> Path: + idf = tmp_path / "idf" + (idf / "tools" / "cmake").mkdir(parents=True) + (idf / "tools" / "cmake" / "build.cmake").write_text(text) + return idf + + +def test_normalized_macro_strips_comments_and_whitespace() -> None: + from esphome.build_gen.espidf import _normalized_macro + + text = "macro(__build_process_project_includes)\n a( b ) # tail\n\n # only\n c(d)\nendmacro()" + assert _normalized_macro(text) == ["a( b )", "c(d)"] + + +def test_normalized_macro_none_without_macro() -> None: + from esphome.build_gen.espidf import _normalized_macro + + assert _normalized_macro("nothing here") is None + + +def test_idf_macro_matches_the_shipped_body(tmp_path: Path) -> None: + """The override's embedded copy must equal what build.cmake ships.""" + from esphome.build_gen.espidf import idf_macro_matches + + assert idf_macro_matches(_write_idf_build_cmake(tmp_path)) is True + + +def test_idf_macro_mismatch_detected(tmp_path: Path) -> None: + from esphome.build_gen.espidf import idf_macro_matches + + changed = IDF_BUILD_CMAKE.replace( + "include(${sdkconfig_cmake})", "include(${sdkconfig_cmake} NEW_ARG)" + ) + assert idf_macro_matches(_write_idf_build_cmake(tmp_path, changed)) is False diff --git a/tests/unit_tests/test_download_types.py b/tests/unit_tests/test_download_types.py index 0d664b8db66..0f5aae444d0 100644 --- a/tests/unit_tests/test_download_types.py +++ b/tests/unit_tests/test_download_types.py @@ -52,6 +52,35 @@ def test_recorded_firmware_path_yields_downloads(platform: str, tmp_path: Path) ) +def test_esp32_skip_bootloader_tree_offers_no_factory_entry(tmp_path: Path) -> None: + """A --skip-bootloader tree has no factory image; do not offer one. + + An unreadable tree (PlatformIO, capability probes passing a bare + path) reads as a full build and keeps the entry. + """ + from esphome.espidf.toolchain import SKIP_BOOTLOADER_DEFINE + + storage = _wizard_storage() + storage.firmware_bin_path = tmp_path / "firmware.bin" + + files = [entry["file"] for entry in _download_types("esp32", storage)] + assert files == ["firmware.factory.bin", "firmware.ota.bin"] + + cache = tmp_path / "CMakeCache.txt" + cache.write_text(f"{SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n") + files = [entry["file"] for entry in _download_types("esp32", storage)] + assert files == ["firmware.ota.bin"] + + # A capability probe passes the path as a plain string. + storage.firmware_bin_path = str(tmp_path / "firmware.bin") + files = [entry["file"] for entry in _download_types("esp32", storage)] + assert files == ["firmware.ota.bin"] + + cache.write_text(f"{SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n") + files = [entry["file"] for entry in _download_types("esp32", storage)] + assert files == ["firmware.factory.bin", "firmware.ota.bin"] + + def _nrf52_files(tmp_path: Path, *built: str) -> list[str]: """The files nrf52 offers for a build directory holding *built*.""" (tmp_path / "zephyr").mkdir() diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 1b40460b919..63d15a694ca 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -8,11 +8,16 @@ import json import os from pathlib import Path import subprocess -from unittest.mock import call, patch +from unittest.mock import MagicMock, call, patch import pytest -from esphome.components.esp32.const import KEY_ESP32, KEY_IDF_VERSION, KEY_VARIANT +from esphome.components.esp32.const import ( + KEY_ESP32, + KEY_FLASH_SIZE, + KEY_IDF_VERSION, + KEY_VARIANT, +) import esphome.config_validation as cv from esphome.const import ( CONF_COMPILE_PROCESS_LIMIT, @@ -893,6 +898,7 @@ def test_run_reconfigure_cmake_argv_matches_idf_py(setup_core: Path) -> None: "-DESP_PLATFORM=1", f"-DSDKCONFIG={sdkconfig}", "-DCCACHE_ENABLE=0", + "-DESPHOME_SKIP_BOOTLOADER=0", project, ] kwargs = mock_run.call_args.kwargs @@ -927,6 +933,7 @@ def test_run_reconfigure_cmake_argv_matches_idf6_py( "-DPYTHON=/tools/python", "-DESP_PLATFORM=1", f"-DCCACHE_ENABLE={expected}", + "-DESPHOME_SKIP_BOOTLOADER=0", str(build_dir.parent), ] @@ -938,7 +945,8 @@ def test_run_reconfigure_without_sdkconfig_or_filter(setup_core: Path) -> None: assert toolchain.run_reconfigure(verbose=True) == 0 cmd = mock_run.call_args.args[0] assert not any(arg.startswith("-DSDKCONFIG=") for arg in cmd) - assert cmd[-2] == "-DCCACHE_ENABLE=0" + assert cmd[-3] == "-DCCACHE_ENABLE=0" + assert cmd[-2] == "-DESPHOME_SKIP_BOOTLOADER=0" assert mock_run.call_args.kwargs["filter_lines"] is None @@ -1005,6 +1013,7 @@ _CONFIGURED = ( "PYTHON_DEPS_CHECKED:UNINITIALIZED=1\n" "PYTHON:UNINITIALIZED=/tools/python\n" "ESP_PLATFORM:UNINITIALIZED=1\n" + "ESPHOME_SKIP_BOOTLOADER:UNINITIALIZED=0\n" ) @@ -1256,3 +1265,206 @@ def test_get_cmake_cache_value_reads_the_configured_cache(setup_core: Path) -> N with patch.object(toolchain, "_get_cmake_output", return_value=output): assert toolchain.get_cmake_cache_value("ESPHOME_PCH") == "OFF" assert toolchain.get_cmake_cache_value("ESPHOME_MISSING") is None + + +def test_skip_bootloader_requires_flag_and_matching_macro( + setup_core: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The switch is explicit, and an IDF macro change wins over it.""" + from esphome.build_gen import espidf as build_gen + + assert toolchain._skip_bootloader() is False + CORE.skip_bootloader = True + toolchain._cache().skip_bootloader = None # decision is memoized per run + with ( + patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), + patch.object(build_gen, "idf_macro_matches", return_value=True), + ): + assert toolchain._skip_bootloader() is True + toolchain._cache().skip_bootloader = None + with ( + patch.object(toolchain, "_get_idf_path", return_value=Path("/idf")), + patch.object(build_gen, "idf_macro_matches", return_value=False) as mock_match, + caplog.at_level("WARNING"), + ): + assert toolchain._skip_bootloader() is False + assert toolchain._skip_bootloader() is False + mock_match.assert_called_once() # the memo also dedupes the warning + assert "--skip-bootloader ignored" in caplog.text + + +def test_configure_defines_follow_skip_bootloader(setup_core: Path) -> None: + with ( + patch.object(toolchain, "_get_idf_tool", return_value="/tools/python"), + patch.object(toolchain, "_cache_entries", return_value={}), + ): + with patch.object(toolchain, "_skip_bootloader", return_value=True): + assert ( + toolchain._configure_defines()[toolchain.SKIP_BOOTLOADER_DEFINE] == "1" + ) + with patch.object(toolchain, "_skip_bootloader", return_value=False): + assert ( + toolchain._configure_defines()[toolchain.SKIP_BOOTLOADER_DEFINE] == "0" + ) + + +def test_tree_skips_bootloader_reads_the_define(tmp_path: Path) -> None: + build = tmp_path / "build" + build.mkdir() + assert toolchain.tree_skips_bootloader(build) is False # not configured + cache = build / "CMakeCache.txt" + cache.write_text(f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n") + assert toolchain.tree_skips_bootloader(build) is True + cache.write_text(f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n") + assert toolchain.tree_skips_bootloader(build) is False + + +def test_tree_skips_bootloader_unreadable_cache_reads_full( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Unreadable trees fall back to the safe full build, with a trace.""" + build = tmp_path / "build" + (build / "CMakeCache.txt").mkdir(parents=True) # read raises OSError + with caplog.at_level("DEBUG"): + assert toolchain.tree_skips_bootloader(build) is False + assert "assuming a full build" in caplog.text + + +def test_create_factory_bin_merges_the_listed_flash_files( + setup_core: Path, +) -> None: + """Every listed file lands in the esptool merge argv, address sorted.""" + _setup_build(setup_core) + CORE.data[KEY_ESP32][KEY_FLASH_SIZE] = "4MB" + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n" + ) + (build / "app.bin").write_bytes(b"app") + (build / "boot.bin").write_bytes(b"boot") + (build / "flasher_args.json").write_text( + '{"flash_files": {"0x10000": "app.bin", "0x0": "boot.bin"},' + ' "extra_esptool_args": {"chip": "esp32"}}' + ) + with ( + patch.object(toolchain, "_get_idf_env", return_value={}), + patch.object(toolchain, "_get_idf_tool", return_value="/py"), + patch.object( + toolchain.subprocess, "run", return_value=MagicMock(returncode=0) + ) as mock_run, + ): + assert toolchain.create_factory_bin() is True + argv = mock_run.call_args.args[0] + boot = argv.index("0x0") + assert argv[boot + 1].endswith("boot.bin") + assert argv[boot + 2] == "0x10000" + + +def test_create_factory_bin_fails_on_a_missing_listed_flash_file( + setup_core: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A partial factory image would not boot; never write one.""" + _setup_build(setup_core) + CORE.data[KEY_ESP32][KEY_FLASH_SIZE] = "4MB" + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n" + ) + (build / "flasher_args.json").write_text('{"flash_files": {"0x0": "missing.bin"}}') + stale = toolchain.get_factory_firmware_path() + stale.write_bytes(b"old") + with patch.object(toolchain.subprocess, "run") as mock_run: + assert toolchain.create_factory_bin() is False + mock_run.assert_not_called() + assert "Flash file not found" in caplog.text + # The image from an earlier build must not be served as this one. + assert not stale.exists() + + +def test_create_factory_bin_skip_mode_creates_nothing( + setup_core: Path, caplog: pytest.LogCaptureFixture +) -> None: + """The flip into skip mode already cleaned up; the merge just no-ops.""" + _setup_build(setup_core) + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n" + ) + with ( + patch.object(toolchain.subprocess, "run") as mock_run, + caplog.at_level("INFO"), + ): + assert toolchain.create_factory_bin() is True + mock_run.assert_not_called() + assert "no factory image" in caplog.text + + +def test_run_reconfigure_flip_into_skip_mode_cleans_up(setup_core: Path) -> None: + """Full-mode leftovers are stale for OTA --bootloader and downloads, and + a partial cleanup would poison the flip back to full mode.""" + _setup_build(setup_core) + build = CORE.relative_build_path("build") + (build / "bootloader").mkdir(parents=True) + (build / "bootloader" / "bootloader.bin").write_bytes(b"old") + # bootloader-prefix deliberately absent: cleanup skips what is not there. + stale = toolchain.get_factory_firmware_path() + stale.parent.mkdir(parents=True, exist_ok=True) + stale.write_bytes(b"old") + with ( + patch.object(toolchain, "_skip_bootloader", return_value=True), + patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"), + patch.object(toolchain, "_tool_env", return_value={}), + patch.object(toolchain, "run_build_tool", return_value=0), + patch.object(toolchain, "_idf_py") as mock_idf_py, + ): + mock_idf_py.return_value.binary_dir_arg = False + assert toolchain.run_reconfigure() == 0 + assert not (build / "bootloader").exists() + assert not stale.exists() + + +def test_run_reconfigure_skip_steady_state_cleans_nothing(setup_core: Path) -> None: + """Cleanup belongs to the flip; a reconfigure of a skip tree touches nothing.""" + _setup_build(setup_core) + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n" + ) + marker = build / "bootloader" + marker.mkdir() + with ( + patch.object(toolchain, "_skip_bootloader", return_value=True), + patch.object(toolchain, "_get_idf_tool", side_effect=lambda n: f"/tools/{n}"), + patch.object(toolchain, "_tool_env", return_value={}), + patch.object(toolchain, "run_build_tool", return_value=0), + patch.object(toolchain, "_idf_py") as mock_idf_py, + ): + mock_idf_py.return_value.binary_dir_arg = False + assert toolchain.run_reconfigure() == 0 + assert marker.exists() + + +def test_create_factory_bin_full_mode_needs_flasher_args(setup_core: Path) -> None: + """Past the skip gate, a full tree without flasher_args fails as before.""" + _setup_build(setup_core) + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=0\n" + ) + assert toolchain.create_factory_bin() is False + + +def test_missing_image_hint_names_the_flag(setup_core: Path) -> None: + _setup_build(setup_core) + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + assert toolchain.missing_image_hint() is None # stock tree + (build / "CMakeCache.txt").write_text( + f"{toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n" + ) + assert "--skip-bootloader" in toolchain.missing_image_hint() diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 633ac1b3e16..e46b61cfd57 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -1548,6 +1548,7 @@ class MockArgs: partition_table: bool = False bootloader: bool = False states: bool | None = None + device: list[str] | None = None def test_upload_program_serial_esp32( @@ -5357,6 +5358,28 @@ def _setup_build_info_test( return build_info_path, firmware_path +def test_compile_program_warns_and_ignores_skip_bootloader_elsewhere( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A full build is always safe; orchestrators cannot see YAML overrides.""" + setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test_device") + CORE.skip_bootloader = True + + config: dict[str, Any] = {CONF_ESPHOME: {CONF_NAME: "test_device"}} + with ( + patch( + "esphome.components.esp8266.check_rosetta", + side_effect=EsphomeError("stop here"), + ), + pytest.raises(EsphomeError, match="stop here"), + caplog.at_level("INFO"), + ): + compile_program(MockArgs(), config) + + assert "--skip-bootloader ignored" in caplog.text + assert CORE.skip_bootloader is False + + def test_compile_program_esp8266_runs_rosetta_check(tmp_path: Path) -> None: """Test that compile_program runs the Rosetta preflight for ESP8266 targets.""" setup_core(platform=PLATFORM_ESP8266, tmp_path=tmp_path, name="test_device") @@ -6846,6 +6869,70 @@ def test_upload_using_esptool_arduino_toolchain( assert cmd_list[firmware_offset_idx + 1] == str(factory) +def test_upload_using_esptool_skip_bootloader_tree_names_the_flag( + tmp_path: Path, +) -> None: + """A serial flash needs the factory image the skip flag did not build.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path, name="test") + CORE.data[KEY_ESP32] = {KEY_VARIANT: "ESP32"} + CORE.toolchain = Toolchain.ESP_IDF + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + from esphome.espidf import toolchain as espidf_toolchain + + (build / "CMakeCache.txt").write_text( + f"{espidf_toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n" + ) + config = {CONF_ESPHOME: {"platformio_options": {}}} + with pytest.raises(EsphomeError, match="compiled with --skip-bootloader"): + upload_using_esptool(config, "/dev/ttyUSB0", None, None) + + +def test_command_run_rejects_serial_device_with_skip_bootloader( + tmp_path: Path, +) -> None: + """The compile could never be flashed over serial; fail before it runs.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path, name="test") + CORE.skip_bootloader = True + CORE.toolchain = Toolchain.ESP_IDF + args = MockArgs(device=["/dev/ttyUSB0"]) + with patch("esphome.__main__.write_cpp") as mock_write: + assert command_run(args, {}) == 1 + mock_write.assert_not_called() + + +def test_upload_program_ota_bootloader_skip_tree_names_the_flag( + mock_run_ota: Mock, + mock_get_port_type: Mock, + tmp_path: Path, +) -> None: + """OTA --bootloader on a skip tree errors before picking the binary.""" + setup_core(platform=PLATFORM_ESP32, tmp_path=tmp_path, name="test") + CORE.data[KEY_ESP32] = {KEY_VARIANT: "ESP32"} + CORE.toolchain = Toolchain.ESP_IDF + build = CORE.relative_build_path("build") + build.mkdir(parents=True) + from esphome.espidf import toolchain as espidf_toolchain + + (build / "CMakeCache.txt").write_text( + f"{espidf_toolchain.SKIP_BOOTLOADER_DEFINE}:UNINITIALIZED=1\n" + ) + mock_get_port_type.return_value = "NETWORK" + config = { + CONF_OTA: [ + { + CONF_PLATFORM: CONF_ESPHOME, + CONF_PORT: 3232, + "allow_partition_access": True, + } + ] + } + args = MockArgs(bootloader=True) + with pytest.raises(EsphomeError, match="compiled with --skip-bootloader"): + upload_program(config, args, ["192.168.1.100"]) + mock_run_ota.assert_not_called() + + @pytest.mark.parametrize( ("toolchain", "pio_project_written"), [ @@ -7073,6 +7160,23 @@ def test_compile_program_espidf_idedata_success_is_silent( assert "idedata" not in caplog.text +def test_compile_program_espidf_failed_factory_bin_fails_the_build() -> None: + """A compile whose factory image could not be produced must not exit 0.""" + CORE.toolchain = Toolchain.ESP_IDF + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: "esp32", + KEY_TARGET_FRAMEWORK: "esp-idf", + } + with ( + patch("esphome.espidf.toolchain.run_compile", return_value=0), + patch("esphome.espidf.toolchain.create_factory_bin", return_value=False), + patch("esphome.espidf.toolchain.create_ota_bin") as mock_ota, + patch("esphome.__main__._check_and_emit_build_info"), + ): + assert compile_program(MagicMock(), {}) == 1 + mock_ota.assert_not_called() + + def test_compile_program_espidf_idedata_none_warns( caplog: pytest.LogCaptureFixture, ) -> None: From bfb7e76b2e3bbc6db9374dd786e9aa92f82f13d6 Mon Sep 17 00:00:00 2001 From: Clyde Stubbs <2366188+clydebarrow@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:02:58 +1000 Subject: [PATCH 125/133] [light] Add `color` option to strobe and other color state (#19874) Co-authored-by: Claude Sonnet 5 Co-authored-by: J. Nick Koston --- esphome/components/light/effects.py | 132 +++++++++++------- .../addressable_color_wipe_color_test.yaml | 22 +++ .../test_addressable_color_wipe_color.py | 40 ++++++ .../light/test_effect_color.py | 69 +++++++++ .../components/fastled_clockless/common.yaml | 2 + tests/components/light/common.yaml | 4 + 6 files changed, 222 insertions(+), 47 deletions(-) create mode 100644 tests/component_tests/light/config/addressable_color_wipe_color_test.yaml create mode 100644 tests/component_tests/light/test_addressable_color_wipe_color.py create mode 100644 tests/component_tests/light/test_effect_color.py diff --git a/esphome/components/light/effects.py b/esphome/components/light/effects.py index b12ec189b9e..dc6266b2fd2 100644 --- a/esphome/components/light/effects.py +++ b/esphome/components/light/effects.py @@ -31,9 +31,10 @@ from esphome.const import ( ) from esphome.cpp_generator import MockObjClass from esphome.schema_extractors import SCHEMA_EXTRACT, schema_extractor +from esphome.types import ConfigType from esphome.util import Registry -from .automation import validate_light_state +from .automation import COLOR_SCHEMA, color_to_rgb, validate_light_state from .types import ( COLOR_MODES, AddressableColorWipeEffect, @@ -88,6 +89,28 @@ MONOCHROMATIC_EFFECTS = [] RGB_EFFECTS = [] ADDRESSABLE_EFFECTS = [] + +def _with_color_option(schema: cv.Schema, *default_keys: str) -> cv.Schema: + """Add the `color:` option to a per-item effect color schema. + + `color_to_rgb` must run before the keys are defaulted to full level below, so a + `color:` can populate `default_keys` (red/green/blue and, where the effect has + one, color_brightness) and a plain channel list still defaults them to full + level. Centralizing both steps here means a new effect schema only has to call + this once instead of repeating the ordering rule itself. + """ + schema = schema.extend(COLOR_SCHEMA) + schema.add_extra(color_to_rgb) + + def _default_to_full_level(config: ConfigType) -> ConfigType: + for key in default_keys: + config.setdefault(key, 1.0) + return config + + schema.add_extra(_default_to_full_level) + return schema + + EFFECTS_REGISTRY = Registry() @@ -267,6 +290,30 @@ async def random_effect_to_code(config, effect_id): return effect +STROBE_COLOR_SCHEMA = cv.Schema( + { + cv.Optional(CONF_STATE, default=True): validate_light_state, + cv.Optional(CONF_BRIGHTNESS, default=1.0): cv.percentage, + cv.Optional(CONF_COLOR_MODE): cv.enum(COLOR_MODES, upper=True, space="_"), + cv.Optional(CONF_COLOR_BRIGHTNESS): cv.percentage, + cv.Optional(CONF_RED): cv.percentage, + cv.Optional(CONF_GREEN): cv.percentage, + cv.Optional(CONF_BLUE): cv.percentage, + cv.Optional(CONF_WHITE, default=1.0): cv.percentage, + cv.Optional(CONF_COLOR_TEMPERATURE): cv.color_temperature, + cv.Optional(CONF_COLD_WHITE, default=1.0): cv.percentage, + cv.Optional(CONF_WARM_WHITE, default=1.0): cv.percentage, + cv.Required(CONF_DURATION): cv.positive_time_period_milliseconds, + cv.Optional( + CONF_TRANSITION_LENGTH, default="0s" + ): cv.positive_time_period_milliseconds, + } +) +STROBE_COLOR_SCHEMA = _with_color_option( + STROBE_COLOR_SCHEMA, CONF_COLOR_BRIGHTNESS, CONF_RED, CONF_GREEN, CONF_BLUE +) + + @register_binary_effect( "strobe", StrobeLightEffect, @@ -280,29 +327,7 @@ async def random_effect_to_code(config, effect_id): ], ): cv.All( cv.ensure_list( - cv.Schema( - { - cv.Optional(CONF_STATE, default=True): validate_light_state, - cv.Optional(CONF_BRIGHTNESS, default=1.0): cv.percentage, - cv.Optional(CONF_COLOR_MODE): cv.enum( - COLOR_MODES, upper=True, space="_" - ), - cv.Optional(CONF_COLOR_BRIGHTNESS, default=1.0): cv.percentage, - cv.Optional(CONF_RED, default=1.0): cv.percentage, - cv.Optional(CONF_GREEN, default=1.0): cv.percentage, - cv.Optional(CONF_BLUE, default=1.0): cv.percentage, - cv.Optional(CONF_WHITE, default=1.0): cv.percentage, - cv.Optional(CONF_COLOR_TEMPERATURE): cv.color_temperature, - cv.Optional(CONF_COLD_WHITE, default=1.0): cv.percentage, - cv.Optional(CONF_WARM_WHITE, default=1.0): cv.percentage, - cv.Required( - CONF_DURATION - ): cv.positive_time_period_milliseconds, - cv.Optional( - CONF_TRANSITION_LENGTH, default="0s" - ): cv.positive_time_period_milliseconds, - } - ), + STROBE_COLOR_SCHEMA, cv.has_at_least_one_key( CONF_STATE, CONF_BRIGHTNESS, @@ -406,6 +431,27 @@ async def addressable_rainbow_effect_to_code(config, effect_id): return var +ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA = cv.Schema( + { + cv.Optional(CONF_COLOR_BRIGHTNESS): cv.percentage, + cv.Optional(CONF_RED): cv.percentage, + cv.Optional(CONF_GREEN): cv.percentage, + cv.Optional(CONF_BLUE): cv.percentage, + cv.Optional(CONF_WHITE, default=1.0): cv.percentage, + cv.Optional(CONF_RANDOM, default=False): cv.boolean, + cv.Required(CONF_NUM_LEDS): cv.All(cv.uint32_t, cv.Range(min=1)), + cv.Optional(CONF_GRADIENT, default=False): cv.boolean, + } +) +ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA = _with_color_option( + ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA, + CONF_COLOR_BRIGHTNESS, + CONF_RED, + CONF_GREEN, + CONF_BLUE, +) + + @register_addressable_effect( "addressable_color_wipe", AddressableColorWipeEffect, @@ -413,17 +459,7 @@ async def addressable_rainbow_effect_to_code(config, effect_id): { cv.Optional( CONF_COLORS, default=[{CONF_NUM_LEDS: 1, CONF_RANDOM: True}] - ): cv.ensure_list( - { - cv.Optional(CONF_RED, default=1.0): cv.percentage, - cv.Optional(CONF_GREEN, default=1.0): cv.percentage, - cv.Optional(CONF_BLUE, default=1.0): cv.percentage, - cv.Optional(CONF_WHITE, default=1.0): cv.percentage, - cv.Optional(CONF_RANDOM, default=False): cv.boolean, - cv.Required(CONF_NUM_LEDS): cv.All(cv.uint32_t, cv.Range(min=1)), - cv.Optional(CONF_GRADIENT, default=False): cv.boolean, - } - ), + ): cv.ensure_list(ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA), cv.Optional( CONF_ADD_LED_INTERVAL, default="0.1s" ): cv.positive_time_period_milliseconds, @@ -434,19 +470,21 @@ async def addressable_color_wipe_effect_to_code(config, effect_id): var = cg.new_Pvariable(effect_id, config[CONF_NAME]) cg.add(var.set_add_led_interval(config[CONF_ADD_LED_INTERVAL])) cg.add(var.set_reverse(config[CONF_REVERSE])) - colors = [ - cg.StructInitializer( - AddressableColorWipeEffectColor, - ("r", int(round(color[CONF_RED] * 255))), - ("g", int(round(color[CONF_GREEN] * 255))), - ("b", int(round(color[CONF_BLUE] * 255))), - ("w", int(round(color[CONF_WHITE] * 255))), - ("random", color[CONF_RANDOM]), - ("num_leds", color[CONF_NUM_LEDS]), - ("gradient", color[CONF_GRADIENT]), + colors = [] + for color in config.get(CONF_COLORS, []): + color_brightness = color[CONF_COLOR_BRIGHTNESS] + colors.append( + cg.StructInitializer( + AddressableColorWipeEffectColor, + ("r", int(round(color[CONF_RED] * color_brightness * 255))), + ("g", int(round(color[CONF_GREEN] * color_brightness * 255))), + ("b", int(round(color[CONF_BLUE] * color_brightness * 255))), + ("w", int(round(color[CONF_WHITE] * 255))), + ("random", color[CONF_RANDOM]), + ("num_leds", color[CONF_NUM_LEDS]), + ("gradient", color[CONF_GRADIENT]), + ) ) - for color in config.get(CONF_COLORS, []) - ] cg.add(var.set_colors(colors)) return var diff --git a/tests/component_tests/light/config/addressable_color_wipe_color_test.yaml b/tests/component_tests/light/config/addressable_color_wipe_color_test.yaml new file mode 100644 index 00000000000..fc1e80688fe --- /dev/null +++ b/tests/component_tests/light/config/addressable_color_wipe_color_test.yaml @@ -0,0 +1,22 @@ +esphome: + name: test-color-wipe-color + +esp32: + board: esp32dev + framework: + type: arduino + +light: + - platform: fastled_clockless + id: wipe_light + chipset: WS2811 + pin: GPIO13 + num_leds: 10 + effects: + - addressable_color_wipe: + name: Wipe + colors: + - color: darkred + num_leds: 1 + - red: 50% + num_leds: 1 diff --git a/tests/component_tests/light/test_addressable_color_wipe_color.py b/tests/component_tests/light/test_addressable_color_wipe_color.py new file mode 100644 index 00000000000..f1dd44a32c0 --- /dev/null +++ b/tests/component_tests/light/test_addressable_color_wipe_color.py @@ -0,0 +1,40 @@ +"""Regression test: addressable_color_wipe's `color:` must scale by color_brightness. + +The per-item schema tests in test_effect_color.py only check the validated config +dict; they would still pass if addressable_color_wipe_effect_to_code dropped +color_brightness on the floor (as it originally did), since a dark color's red/green/ +blue are already peak-normalized to 1.0 by the time codegen sees them. Generate the +actual C++ and check the AddressableColorWipeEffectColor byte values instead, so a +regression in the scaling multiplication itself is caught. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + + +@pytest.fixture +def main_cpp( + generate_main: Callable[[str | Path], str], + component_config_path: Callable[[str], Path], +) -> str: + return generate_main( + component_config_path("addressable_color_wipe_color_test.yaml") + ) + + +def test_dark_color_name_is_scaled_by_color_brightness(main_cpp: str) -> None: + # color: darkred (0x8B0000) normalizes to r=1.0 but color_brightness=0x8B/0xFF, + # so the byte value must come back down to 0x8B (139), not 255. + assert ".r = 139," in main_cpp + assert ".g = 0," in main_cpp + assert ".b = 0," in main_cpp + + +def test_explicit_channel_is_unaffected(main_cpp: str) -> None: + # red: 50% with no color: or color_brightness: set must still scale by the + # default color_brightness of 1.0, i.e. come out as plain 50% of 255. + assert ".r = 128," in main_cpp diff --git a/tests/component_tests/light/test_effect_color.py b/tests/component_tests/light/test_effect_color.py new file mode 100644 index 00000000000..52e4e440dd4 --- /dev/null +++ b/tests/component_tests/light/test_effect_color.py @@ -0,0 +1,69 @@ +"""Tests for the `color` option in per-item effect color lists.""" + +from __future__ import annotations + +import pytest + +from esphome import config_validation as cv +from esphome.components.light.effects import ( + ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA, + STROBE_COLOR_SCHEMA, +) +from esphome.const import CONF_BLUE, CONF_COLOR_BRIGHTNESS, CONF_GREEN, CONF_RED + + +def test_strobe_plain_defaults_full_level() -> None: + result = STROBE_COLOR_SCHEMA({"duration": "1s"}) + assert (result[CONF_RED], result[CONF_GREEN], result[CONF_BLUE]) == (1.0, 1.0, 1.0) + assert result[CONF_COLOR_BRIGHTNESS] == 1.0 + + +def test_strobe_color_name() -> None: + result = STROBE_COLOR_SCHEMA({"duration": "1s", "color": "darkred"}) + assert "color" not in result + assert (result[CONF_RED], result[CONF_GREEN], result[CONF_BLUE]) == (1.0, 0.0, 0.0) + assert result[CONF_COLOR_BRIGHTNESS] == pytest.approx(0x8B / 0xFF) + + +def test_strobe_hex_color() -> None: + result = STROBE_COLOR_SCHEMA({"duration": "1s", "color": "0x223344"}) + assert result[CONF_BLUE] == 1.0 + assert result[CONF_COLOR_BRIGHTNESS] == pytest.approx(0x44 / 0xFF) + + +def test_strobe_explicit_channels_unaffected() -> None: + result = STROBE_COLOR_SCHEMA({"duration": "1s", "red": "50%"}) + assert result[CONF_RED] == 0.5 + assert (result[CONF_GREEN], result[CONF_BLUE]) == (1.0, 1.0) + assert result[CONF_COLOR_BRIGHTNESS] == 1.0 + + +def test_strobe_color_conflicts_with_rgb() -> None: + with pytest.raises(cv.Invalid, match="cannot be used with"): + STROBE_COLOR_SCHEMA({"duration": "1s", "color": "red", "red": "10%"}) + + +def test_addressable_color_wipe_plain_defaults_full_level() -> None: + result = ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA({"num_leds": 1}) + assert (result[CONF_RED], result[CONF_GREEN], result[CONF_BLUE]) == (1.0, 1.0, 1.0) + assert result[CONF_COLOR_BRIGHTNESS] == 1.0 + + +def test_addressable_color_wipe_color_name() -> None: + result = ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA({"num_leds": 1, "color": "darkred"}) + assert "color" not in result + assert (result[CONF_RED], result[CONF_GREEN], result[CONF_BLUE]) == (1.0, 0.0, 0.0) + assert result[CONF_COLOR_BRIGHTNESS] == pytest.approx(0x8B / 0xFF) + + +def test_addressable_color_wipe_explicit_channels_unaffected() -> None: + result = ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA({"num_leds": 1, "red": "50%"}) + assert result[CONF_RED] == 0.5 + assert result[CONF_COLOR_BRIGHTNESS] == 1.0 + + +def test_addressable_color_wipe_color_conflicts_with_rgb() -> None: + with pytest.raises(cv.Invalid, match="cannot be used with"): + ADDRESSABLE_COLOR_WIPE_COLOR_SCHEMA( + {"num_leds": 1, "color": "blue", "blue": "10%"} + ) diff --git a/tests/components/fastled_clockless/common.yaml b/tests/components/fastled_clockless/common.yaml index a7ce7ed2803..98bc65f4ddb 100644 --- a/tests/components/fastled_clockless/common.yaml +++ b/tests/components/fastled_clockless/common.yaml @@ -21,6 +21,8 @@ light: green: 0% blue: 0% num_leds: 1 + - color: darkred + num_leds: 1 add_led_interval: 100ms reverse: false - addressable_scan: diff --git a/tests/components/light/common.yaml b/tests/components/light/common.yaml index 046acef9ce6..3cf2c592234 100644 --- a/tests/components/light/common.yaml +++ b/tests/components/light/common.yaml @@ -429,6 +429,10 @@ light: blue: 0% duration: 1s transition_length: 500ms + - state: true + color: darkred + duration: 1s + transition_length: 500ms - platform: partition id: test_addressable_transition name: Addressable Transition Test From 7962b52d2a9aa93921f3cda6082472b272b2b594 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 14:05:15 -0500 Subject: [PATCH 126/133] [ci] Skip the bootloader in the component batch build jobs (#19931) --- .github/workflows/ci.yml | 9 ++- script/test_build_components.py | 27 ++++++++ tests/script/test_test_build_components.py | 81 ++++++++++++++++++++++ 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9507c59381..a7ed0350630 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1225,7 +1225,14 @@ jobs: if [ -n "$compile_csv" ]; then # Run compilation with grouping and isolation - python3 script/test_build_components.py -e compile -c "$compile_csv" -f --isolate "$directly_changed_csv" + # The bootloader has no ESPHome code and these builds never + # flash; the check_idf_py batch keeps the full build so the + # native sub-build and the equivalence check stay covered. + skip_flag="--skip-bootloader" + if [[ "${{ matrix.batch.check_idf_py }}" == "true" ]]; then + skip_flag="" + fi + python3 script/test_build_components.py -e compile -c "$compile_csv" -f --isolate "$directly_changed_csv" $skip_flag if [[ "${{ matrix.batch.check_idf_py }}" == "true" ]]; then # The real idf.py must find nothing to configure or build in a # tree built above; catches drift on ESP-IDF bumps. diff --git a/script/test_build_components.py b/script/test_build_components.py index d3dfd360763..c955391a619 100755 --- a/script/test_build_components.py +++ b/script/test_build_components.py @@ -333,6 +333,12 @@ def extract_platform_with_version(base_file: Path) -> str: return base_file.stem.replace("build_components_base.", "") +def _wants_skip_bootloader(skip: bool, command: str, platform: str) -> bool: + """Every esp32 target: idf variants, and esp32-ard, whose Arduino + core builds as an ESP-IDF component under the native toolchain.""" + return skip and command == "compile" and platform.startswith("esp32") + + def run_esphome_test( component: str, test_file: Path, @@ -344,6 +350,7 @@ def run_esphome_test( continue_on_fail: bool, use_testing_mode: bool = False, toolchain: str | None = None, + skip_bootloader: bool = False, ) -> TestResult: """Run esphome test for a single component. @@ -408,6 +415,8 @@ def run_esphome_test( # Add command cmd.append(esphome_command) + if _wants_skip_bootloader(skip_bootloader, esphome_command, platform): + cmd.append("--skip-bootloader") # Add config file cmd.append(str(output_file)) @@ -469,6 +478,7 @@ def run_grouped_test( esphome_command: str, continue_on_fail: bool, toolchain: str | None = None, + skip_bootloader: bool = False, ) -> TestResult: """Run esphome test for a group of components with shared bus configs. @@ -554,6 +564,8 @@ def run_grouped_test( # Add command cmd.append(esphome_command) + if _wants_skip_bootloader(skip_bootloader, esphome_command, platform): + cmd.append("--skip-bootloader") cmd.append(str(output_file)) @@ -614,6 +626,7 @@ def run_grouped_component_tests( continue_on_fail: bool, additional_isolated: set[str] | None = None, toolchain: str | None = None, + skip_bootloader: bool = False, ) -> tuple[set[tuple[str, str]], list[TestResult]]: """Run grouped component tests. @@ -956,6 +969,7 @@ def run_grouped_component_tests( esphome_command=esphome_command, continue_on_fail=continue_on_fail, toolchain=toolchain, + skip_bootloader=skip_bootloader, ) # Mark all components as tested @@ -980,6 +994,7 @@ def run_individual_component_test( tested_components: set[tuple[str, str]], test_results: list[TestResult], toolchain: str | None = None, + skip_bootloader: bool = False, ) -> None: """Run an individual component test if not already tested in a group. @@ -1014,6 +1029,7 @@ def run_individual_component_test( esphome_command=esphome_command, continue_on_fail=continue_on_fail, toolchain=toolchain, + skip_bootloader=skip_bootloader, ) test_results.append(test_result) @@ -1027,6 +1043,7 @@ def test_components( isolated_components: set[str] | None = None, base_only: bool = False, toolchain: str | None = None, + skip_bootloader: bool = False, fail_on_no_tests: bool = False, ) -> int: """Test components with optional intelligent grouping. @@ -1127,6 +1144,7 @@ def test_components( continue_on_fail=continue_on_fail, additional_isolated=isolated_components, toolchain=toolchain, + skip_bootloader=skip_bootloader, ) test_results.extend(grouped_results) @@ -1156,6 +1174,7 @@ def test_components( tested_components=tested_components, test_results=test_results, toolchain=toolchain, + skip_bootloader=skip_bootloader, ) else: # Platform-specific test @@ -1189,6 +1208,7 @@ def test_components( tested_components=tested_components, test_results=test_results, toolchain=toolchain, + skip_bootloader=skip_bootloader, ) silent: list[str] = [] @@ -1294,6 +1314,12 @@ def main() -> int: "--toolchain", help="Select toolchain for compiling.", ) + parser.add_argument( + "--skip-bootloader", + action="store_true", + help="Pass --skip-bootloader to esphome compile; component builds " + "never flash, and the bootloader is covered by the toolchain jobs", + ) parser.add_argument( "--fail-on-no-tests", action="store_true", @@ -1321,6 +1347,7 @@ def main() -> int: fail_on_no_tests=args.fail_on_no_tests, base_only=args.base_only, toolchain=args.toolchain, + skip_bootloader=args.skip_bootloader, ) diff --git a/tests/script/test_test_build_components.py b/tests/script/test_test_build_components.py index 1d21e5d943e..cf15472888e 100644 --- a/tests/script/test_test_build_components.py +++ b/tests/script/test_test_build_components.py @@ -94,6 +94,87 @@ def test_run_esphome_test_wraps_output_in_group( assert out.index("::group::") < out.index("> [foo]") < out.index("::endgroup::") +@pytest.mark.parametrize( + ("platform", "command", "skip", "expects_flag"), + [ + ("esp32-idf", "compile", True, True), + ("esp32-s3-idf", "compile", True, True), + ("esp32-ard", "compile", True, True), + ("esp32-idf", "compile", False, False), + ("esp32-idf", "config", True, False), + ("esp8266-ard", "compile", True, False), + ], + ids=[ + "idf-skip", + "idf-variant", + "arduino-as-idf-component", + "idf-off", + "config", + "esp8266", + ], +) +def test_run_esphome_test_skip_bootloader_argv( + _ci: None, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + platform: str, + command: str, + skip: bool, + expects_flag: bool, +) -> None: + """The flag lands on esp32 family compiles only.""" + seen: list[list[str]] = [] + + def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: + seen.append(cmd) + return _FakeCompleted(0) + + monkeypatch.setattr(tbc.subprocess, "run", fake_run) + repo_root = Path(tbc.__file__).parent.parent + test_file = repo_root / "tests" / "components" / "foo" / f"test.{platform}.yaml" + tbc.run_esphome_test( + component="foo", + test_file=test_file, + platform=platform, + platform_with_version=platform, + base_file=_make_base_file(tmp_path), + build_dir=tmp_path, + esphome_command=command, + continue_on_fail=True, + skip_bootloader=skip, + ) + assert ("--skip-bootloader" in seen[0]) is expects_flag + + +def test_run_grouped_test_skip_bootloader_argv( + _ci: None, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """The grouped command site adds the flag the same way.""" + seen: list[list[str]] = [] + + def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: + seen.append(cmd) + return _FakeCompleted(0) + + monkeypatch.setattr(tbc.subprocess, "run", fake_run) + repo_root = Path(tbc.__file__).parent.parent + tests_dir = repo_root / "tests" / "components" + tbc.run_grouped_test( + components=["gpio"], + platform="esp32-idf", + platform_with_version="esp32-idf", + base_file=_make_base_file(tmp_path), + build_dir=tmp_path, + tests_dir=tests_dir, + esphome_command="compile", + continue_on_fail=True, + skip_bootloader=True, + ) + assert "--skip-bootloader" in seen[0] + + def test_run_esphome_test_closes_group_before_failure_report( _ci: None, monkeypatch: pytest.MonkeyPatch, From f7b4ea9f0e138f252d24c924d950e9a3623184dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:31:37 -0500 Subject: [PATCH 127/133] Bump filelock from 4.0.4 to 4.0.5 (#19991) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2a0b7b31f47..940871224ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ requests==2.34.2 py7zr==1.1.3 platformdirs==4.12.0 # native esp-idf toolchain global cache dir ninja==1.13.2 # native host and esp8266 arduino toolchain build driver -filelock==4.0.4 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg +filelock==4.0.5 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg # esp-idf >= 5.0 requires this pyparsing >= 3.3.3 From 96c61208497b2964c1573c617fd1de3666f9b367 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:32:05 -0500 Subject: [PATCH 128/133] Bump zeroconf from 0.151.3 to 0.151.5 (#19992) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 940871224ec..f05f092904f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ esptool==5.4.0 click==8.5.0 aioesphomeapi==46.6.0 aiohappyeyeballs==2.7.1 # Happy Eyeballs for requests downloads; already pulled in by aioesphomeapi -zeroconf==0.151.3 +zeroconf==0.151.5 puremagic==2.2.0 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import From 8ac184228dde87642b61330036073d93adee3900 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:32:17 -0500 Subject: [PATCH 129/133] Bump platformdirs from 4.12.0 to 4.12.1 (#19994) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f05f092904f..32931b650e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ bleak==3.0.2 smpclient==7.2.0 requests==2.34.2 py7zr==1.1.3 -platformdirs==4.12.0 # native esp-idf toolchain global cache dir +platformdirs==4.12.1 # native esp-idf toolchain global cache dir ninja==1.13.2 # native host and esp8266 arduino toolchain build driver filelock==4.0.5 # inter-process locks (PlatformIO cache heal, git clone cache); >=3.32 for FileLock(fallback_to_soft=...), older versions silently drop the kwarg From d49741dc10345014c68c3dc9c4c3c7a93f68560b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:48:16 -0500 Subject: [PATCH 130/133] Bump prek from 0.5.3 to 0.5.4 (#19993) Signed-off-by: dependabot[bot] --- requirements_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_test.txt b/requirements_test.txt index 3ec864a3405..a66e05ba746 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -2,7 +2,7 @@ pylint==4.0.9 flake8==7.4.1 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py ruff==0.16.9 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py pyupgrade==3.21.2 # .pre-commit-config.yaml rev synced by script/sync_dependency_versions.py -prek==0.5.3 # .github/workflows/ci.yml reads this pin +prek==0.5.4 # .github/workflows/ci.yml reads this pin yamlrocks==0.6.1 # used by script/sync_dependency_versions.py # Unit tests From 2d37c6f58b78df01dd348e48a21ae5700af5add1 Mon Sep 17 00:00:00 2001 From: "esphome[bot]" <115708604+esphome[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:51:43 -0500 Subject: [PATCH 131/133] Bump bundled esphome-device-builder to 1.19.0 (#19995) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e68ea3ee72e..8d3a5d2cc50 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ RUN \ -r /requirements.txt # Install the ESPHome Device Builder dashboard. -RUN uv pip install --no-cache-dir esphome-device-builder==1.18.0 +RUN uv pip install --no-cache-dir esphome-device-builder==1.19.0 RUN \ platformio settings set enable_telemetry No \ From 4075c15bea92e1b2fc9d8a4af639f01e567e8235 Mon Sep 17 00:00:00 2001 From: Jeff Brown Date: Thu, 1 Oct 2026 13:17:48 -0700 Subject: [PATCH 132/133] [stcc4] Add component for the Sensirion STCC4 CO2 sensor (#19434) Co-authored-by: will Co-authored-by: J. Nick Koston --- CODEOWNERS | 1 + .../sensirion_common/i2c_sensirion.cpp | 5 +- .../sensirion_common/i2c_sensirion.h | 7 +- esphome/components/stcc4/__init__.py | 0 esphome/components/stcc4/sensor.py | 141 +++++++++ esphome/components/stcc4/stcc4.cpp | 294 ++++++++++++++++++ esphome/components/stcc4/stcc4.h | 55 ++++ tests/components/stcc4/common.yaml | 36 +++ tests/components/stcc4/test.esp32-idf.yaml | 4 + tests/components/stcc4/test.esp8266-ard.yaml | 4 + tests/components/stcc4/test.rp2040-ard.yaml | 4 + 11 files changed, 548 insertions(+), 3 deletions(-) create mode 100644 esphome/components/stcc4/__init__.py create mode 100644 esphome/components/stcc4/sensor.py create mode 100644 esphome/components/stcc4/stcc4.cpp create mode 100644 esphome/components/stcc4/stcc4.h create mode 100644 tests/components/stcc4/common.yaml create mode 100644 tests/components/stcc4/test.esp32-idf.yaml create mode 100644 tests/components/stcc4/test.esp8266-ard.yaml create mode 100644 tests/components/stcc4/test.rp2040-ard.yaml diff --git a/CODEOWNERS b/CODEOWNERS index e82158f390f..12c0a54ac17 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -541,6 +541,7 @@ esphome/components/st7735/* @SenexCrenshaw esphome/components/st7789v/* @kbx81 esphome/components/st7920/* @marsjan155 esphome/components/statsd/* @Links2004 +esphome/components/stcc4/* @j9brown esphome/components/stts22h/* @B48D81EFCC esphome/components/substitutions/* @esphome/core esphome/components/sun/* @OttoWinter diff --git a/esphome/components/sensirion_common/i2c_sensirion.cpp b/esphome/components/sensirion_common/i2c_sensirion.cpp index f6ff4711d40..3325c277722 100644 --- a/esphome/components/sensirion_common/i2c_sensirion.cpp +++ b/esphome/components/sensirion_common/i2c_sensirion.cpp @@ -65,14 +65,15 @@ bool SensirionI2CDevice::write_command_(uint16_t command, CommandLen command_len } bool SensirionI2CDevice::get_register_(uint16_t reg, CommandLen command_len, uint16_t *data, const uint8_t len, - const uint8_t delay_ms) { + const uint8_t delay_ms, uint8_t sensirion_options) { if (!this->write_command_(reg, command_len, nullptr, 0)) { ESP_LOGE(TAG, "Write failed: reg=0x%X (%d) err=%d,", reg, command_len, this->last_error_); return false; } delay(delay_ms); bool result = this->read_data(data, len); - if (!result) { + if (!result && + (this->last_error_ != i2c::ERROR_NOT_ACKNOWLEDGED || !(sensirion_options & SENSIRION_OPTION_READ_MAY_NACK))) { ESP_LOGE(TAG, "Read failed: reg=0x%X err=%d,", reg, this->last_error_); } return result; diff --git a/esphome/components/sensirion_common/i2c_sensirion.h b/esphome/components/sensirion_common/i2c_sensirion.h index 558fbdbb129..1b1cd2ba15a 100644 --- a/esphome/components/sensirion_common/i2c_sensirion.h +++ b/esphome/components/sensirion_common/i2c_sensirion.h @@ -16,6 +16,9 @@ namespace esphome::sensirion_common { */ static const uint8_t CRC_POLYNOMIAL = 0x31; // default for Sensirion +/// When reading a register and the device reports NACK because the value is not ready yet, don't log it as an error. +static constexpr uint8_t SENSIRION_OPTION_READ_MAY_NACK = 1u << 0; + class SensirionI2CDevice : public i2c::I2CDevice { public: enum CommandLen : uint8_t { ADDR_8_BIT = 1, ADDR_16_BIT = 2 }; @@ -139,9 +142,11 @@ class SensirionI2CDevice : public i2c::I2CDevice { * @param data pointer to raw result * @param len number of words to read * @param delay milliseconds to to wait between sending the I2C command and reading the result + * @param sensirion_options options for the request * @return true if reading succeeded */ - bool get_register_(uint16_t reg, CommandLen command_len, uint16_t *data, uint8_t len, uint8_t delay); + bool get_register_(uint16_t reg, CommandLen command_len, uint16_t *data, uint8_t len, uint8_t delay, + uint8_t sensirion_options = 0); /** last error code from I2C operation */ diff --git a/esphome/components/stcc4/__init__.py b/esphome/components/stcc4/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/esphome/components/stcc4/sensor.py b/esphome/components/stcc4/sensor.py new file mode 100644 index 00000000000..212fe6bb489 --- /dev/null +++ b/esphome/components/stcc4/sensor.py @@ -0,0 +1,141 @@ +import esphome.codegen as cg +from esphome.components import i2c, sensirion_common, sensor +from esphome.components.const import CONF_HUMIDITY_SOURCE +import esphome.config_validation as cv +from esphome.const import ( + CONF_AMBIENT_PRESSURE_COMPENSATION, + CONF_AMBIENT_PRESSURE_COMPENSATION_SOURCE, + CONF_CO2, + CONF_HUMIDITY, + CONF_ID, + CONF_MEASUREMENT_MODE, + CONF_TEMPERATURE, + CONF_TEMPERATURE_SOURCE, + CONF_UPDATE_INTERVAL, + DEVICE_CLASS_CARBON_DIOXIDE, + DEVICE_CLASS_HUMIDITY, + DEVICE_CLASS_TEMPERATURE, + ICON_MOLECULE_CO2, + ICON_THERMOMETER, + ICON_WATER_PERCENT, + STATE_CLASS_MEASUREMENT, + UNIT_CELSIUS, + UNIT_PARTS_PER_MILLION, + UNIT_PERCENT, +) +from esphome.types import ConfigType + +CODEOWNERS = ["@j9brown"] +DEPENDENCIES = ["i2c"] +AUTO_LOAD = ["sensirion_common"] + +stcc4_ns = cg.esphome_ns.namespace("stcc4") +STCC4Component = stcc4_ns.class_( + "STCC4Component", cg.PollingComponent, sensirion_common.SensirionI2CDevice +) + +MeasurementMode = stcc4_ns.enum("MeasurementMode", is_class=True) + +MEASUREMENT_MODE_OPTIONS = { + "continuous": MeasurementMode.CONTINUOUS, + "single_shot": MeasurementMode.SINGLE_SHOT, +} + + +def validate_config(config: ConfigType) -> ConfigType: + if config[CONF_MEASUREMENT_MODE] == "continuous": + if CONF_UPDATE_INTERVAL in config: + raise cv.Invalid( + "update_interval must not be specified in continuous measurement mode" + ) + elif CONF_UPDATE_INTERVAL not in config: + config[CONF_UPDATE_INTERVAL] = cv.update_interval("60s") + return config + + +CONFIG_SCHEMA = ( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(STCC4Component), + cv.Optional(CONF_CO2): sensor.sensor_schema( + unit_of_measurement=UNIT_PARTS_PER_MILLION, + icon=ICON_MOLECULE_CO2, + accuracy_decimals=0, + device_class=DEVICE_CLASS_CARBON_DIOXIDE, + state_class=STATE_CLASS_MEASUREMENT, + ), + cv.Optional(CONF_TEMPERATURE): sensor.sensor_schema( + unit_of_measurement=UNIT_CELSIUS, + icon=ICON_THERMOMETER, + accuracy_decimals=2, + device_class=DEVICE_CLASS_TEMPERATURE, + state_class=STATE_CLASS_MEASUREMENT, + ), + cv.Optional(CONF_HUMIDITY): sensor.sensor_schema( + unit_of_measurement=UNIT_PERCENT, + icon=ICON_WATER_PERCENT, + accuracy_decimals=2, + device_class=DEVICE_CLASS_HUMIDITY, + state_class=STATE_CLASS_MEASUREMENT, + ), + cv.Inclusive(CONF_TEMPERATURE_SOURCE, "rht_compensation"): cv.use_id( + sensor.Sensor + ), + cv.Inclusive(CONF_HUMIDITY_SOURCE, "rht_compensation"): cv.use_id( + sensor.Sensor + ), + cv.Exclusive( + CONF_AMBIENT_PRESSURE_COMPENSATION, "ambient_pressure_compensation" + ): cv.All(cv.pressure, cv.float_range(min=0.4, max=1.1)), + cv.Exclusive( + CONF_AMBIENT_PRESSURE_COMPENSATION_SOURCE, + "ambient_pressure_compensation", + ): cv.use_id(sensor.Sensor), + cv.Optional(CONF_MEASUREMENT_MODE, default="continuous"): cv.enum( + MEASUREMENT_MODE_OPTIONS, lower=True + ), + cv.Optional(CONF_UPDATE_INTERVAL): cv.update_interval, + } + ) + .extend(cv.COMPONENT_SCHEMA) + .extend(i2c.i2c_device_schema(0x64)) + .add_extra(validate_config) +) + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + await i2c.register_i2c_device(var, config) + + sensors = sensor.sub_sensors(config) + await sensors(CONF_CO2, var.set_co2_sensor) + await sensors(CONF_TEMPERATURE, var.set_temperature_sensor) + await sensors(CONF_HUMIDITY, var.set_humidity_sensor) + + if (temperature_source := config.get(CONF_TEMPERATURE_SOURCE)) is not None: + sens = await cg.get_variable(temperature_source) + cg.add(var.set_temperature_source(sens)) + + if (humidity_source := config.get(CONF_HUMIDITY_SOURCE)) is not None: + sens = await cg.get_variable(humidity_source) + cg.add(var.set_humidity_source(sens)) + + if ( + ambient_pressure_compensation := config.get(CONF_AMBIENT_PRESSURE_COMPENSATION) + ) is not None: + cg.add( + var.set_ambient_pressure_compensation( + ambient_pressure_compensation * 1000 # convert bar to hPa + ) + ) + + if ( + ambient_pressure_compensation_source := config.get( + CONF_AMBIENT_PRESSURE_COMPENSATION_SOURCE + ) + ) is not None: + sens = await cg.get_variable(ambient_pressure_compensation_source) + cg.add(var.set_ambient_pressure_source(sens)) + + cg.add(var.set_measurement_mode(config[CONF_MEASUREMENT_MODE])) diff --git a/esphome/components/stcc4/stcc4.cpp b/esphome/components/stcc4/stcc4.cpp new file mode 100644 index 00000000000..efc7a2b8ee7 --- /dev/null +++ b/esphome/components/stcc4/stcc4.cpp @@ -0,0 +1,294 @@ +#include + +#include "stcc4.h" +#include "esphome/core/hal.h" +#include "esphome/core/log.h" + +namespace esphome::stcc4 { + +static const char *const TAG = "stcc4"; + +// I2C Commands +static constexpr uint16_t STCC4_CMD_START_CONTINUOUS_MEASUREMENT = 0x218b; +static constexpr uint16_t STCC4_CMD_STOP_CONTINUOUS_MEASUREMENT = 0x3f86; +static constexpr uint16_t STCC4_CMD_MEASURE_SINGLE_SHOT = 0x219d; +static constexpr uint16_t STCC4_CMD_READ_MEASUREMENT = 0xec05; +static constexpr uint16_t STCC4_CMD_GET_PRODUCT_ID = 0x365b; +static constexpr uint16_t STCC4_CMD_SET_RHT_COMPENSATION = 0xe000; +static constexpr uint16_t STCC4_CMD_SET_PRESSURE_COMPENSATION = 0xe016; + +// Exit sleep is an 8-bit command (single byte 0x00), not 16-bit +static constexpr uint8_t STCC4_CMD_EXIT_SLEEP_MODE = 0x00; + +static constexpr uint32_t STCC4_PRODUCT_ID = 0x0901018a; + +// Timeout for determining when the device is ready for use, in milliseconds. +// While waiting for the previous measurement to finish, the device will NACK all I2C requests +// and it can take up to 1200 ms for the operation to complete according to the datasheet. +static constexpr uint32_t READY_TIMEOUT_MS = 1200; + +// Poll interval for determining when the device is ready for use, in milliseconds. +static constexpr uint32_t READY_POLL_INTERVAL_MS = 100; + +// Convert units the the device's representation according to the datasheet. +constexpr uint16_t temperature_in_c_to_ticks(float temperature_in_c) { + return uint16_t((std::clamp(temperature_in_c, -45.f, 130.f) + 45.f) * 65535.f / 175.f); +} + +constexpr float temperature_in_ticks_to_c(uint16_t temperature_in_ticks) { + return temperature_in_ticks * 175.f / 65535.f - 45.f; +} + +constexpr uint16_t humidity_in_percent_to_ticks(float humidity_in_percent) { + return uint16_t((std::clamp(humidity_in_percent, 0.f, 100.f) + 6.f) * 65535.f / 125.f); +} + +constexpr float humidity_in_ticks_to_percent(uint16_t humidity_in_ticks) { + return humidity_in_ticks * 125.f / 65535.f - 6.f; +} + +constexpr uint16_t pressure_in_hpa_to_pa_2(float pressure_in_hpa) { + return uint16_t(std::clamp(pressure_in_hpa, 400.f, 1100.f) * 50.f); +} + +constexpr float pressure_in_pa_2_to_hpa(uint16_t pressure_in_pa_2) { return pressure_in_pa_2 / 50.f; } + +void STCC4Component::setup() { + this->stop_poller(); // not ready yet + + // Wait 100 ms after power up before attempting to communicate with the sensor + this->set_timeout(100, [this]() { + // Send exit sleep mode command (8-bit, NACK expected), wait 5 ms to exit sleep + this->write_command(STCC4_CMD_EXIT_SLEEP_MODE); + this->set_timeout(5, [this]() { this->poll_until_ready_for_setup_or_timeout_(millis()); }); + }); +} + +void STCC4Component::poll_until_ready_for_setup_or_timeout_(uint32_t start_time) { + // Stop continuous measurements in case they were previously running + // The device may NACK this request if it is not ready to communicate yet + if (this->write_command(STCC4_CMD_STOP_CONTINUOUS_MEASUREMENT)) { + // Read product ID to verify communication (6 words: 2 for product_id + 4 for serial) + // The device may NACK this request if it is not ready to communicate yet + uint16_t raw_product_id[6]; + if (this->get_register(STCC4_CMD_GET_PRODUCT_ID, raw_product_id, 6, 1)) { + uint32_t product_id = (uint32_t(raw_product_id[0]) << 16) | raw_product_id[1]; + uint64_t serial_number = (uint64_t(raw_product_id[2]) << 48) | (uint64_t(raw_product_id[3]) << 32) | + (uint64_t(raw_product_id[4]) << 16) | raw_product_id[5]; + ESP_LOGD(TAG, "Product ID: 0x%08" PRIX32 ", Serial: 0x%016" PRIX64, product_id, serial_number); + if (product_id != STCC4_PRODUCT_ID) { + ESP_LOGE(TAG, "Unsupported product ID"); + this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL)); + return; + } + + // Set static ambient pressure compensation if configured + if (this->ambient_pressure_in_pa_2_ != 0) { + if (!this->write_ambient_pressure_compensation_(this->ambient_pressure_in_pa_2_)) { + this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL)); + return; + } + } + + // Apply the dynamic compensation sources' current values, if configured + if (this->temperature_source_ != nullptr && this->humidity_source_ != nullptr) { + this->update_rht_compensation_from_source_(); + } + if (this->ambient_pressure_source_ != nullptr) { + this->update_ambient_pressure_compensation_from_source_(); + } + + if (this->measurement_mode_ == MeasurementMode::SINGLE_SHOT) { + this->start_poller(); + this->finish_setup_(); + return; + } + + // Start continuous measurement + if (!this->write_command(STCC4_CMD_START_CONTINUOUS_MEASUREMENT)) { + ESP_LOGE(TAG, "Failed to start continuous measurement"); + this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL)); + return; + } + this->schedule_continuous_update_(false); + this->finish_setup_(); + return; + } + } + + if (millis() - start_time < READY_TIMEOUT_MS) { + ESP_LOGVV(TAG, "Retry sync"); + this->set_timeout(READY_POLL_INTERVAL_MS, + [this, start_time]() { this->poll_until_ready_for_setup_or_timeout_(start_time); }); + return; + } + + ESP_LOGE(TAG, "Failed to stop continuous measurements and read product ID"); + this->mark_failed(LOG_STR(ESP_LOG_MSG_COMM_FAIL)); +} + +void STCC4Component::finish_setup_() { + this->ready_ = true; + // Follow the sources only once measuring started, so a failed setup stops writing to the device + if (this->temperature_source_ != nullptr && this->humidity_source_ != nullptr) { + this->temperature_source_->add_on_state_callback([this](float) { this->update_rht_compensation_from_source_(); }); + this->humidity_source_->add_on_state_callback([this](float) { this->update_rht_compensation_from_source_(); }); + } + if (this->ambient_pressure_source_ != nullptr) { + this->ambient_pressure_source_->add_on_state_callback( + [this](float) { this->update_ambient_pressure_compensation_from_source_(); }); + } +} + +void STCC4Component::dump_config() { + ESP_LOGCONFIG(TAG, "STCC4:"); + LOG_I2C_DEVICE(this); + if (this->is_failed()) { + ESP_LOGW(TAG, ESP_LOG_MSG_COMM_FAIL); + } + ESP_LOGCONFIG(TAG, " Measurement mode: %s", + this->measurement_mode_ == MeasurementMode::CONTINUOUS ? LOG_STR_LITERAL("Continuous (1s)") + : LOG_STR_LITERAL("Single shot")); + if (this->ambient_pressure_source_ != nullptr) { + ESP_LOGCONFIG(TAG, " Dynamic ambient pressure compensation using '%s'", + this->ambient_pressure_source_->get_name().c_str()); + } else if (this->ambient_pressure_in_pa_2_ != 0) { + ESP_LOGCONFIG(TAG, " Ambient pressure compensation: %f hPa", + pressure_in_pa_2_to_hpa(this->ambient_pressure_in_pa_2_)); + } + if (this->temperature_source_ != nullptr) { + ESP_LOGCONFIG(TAG, " Temperature compensation using '%s'", this->temperature_source_->get_name().c_str()); + } + if (this->humidity_source_ != nullptr) { + ESP_LOGCONFIG(TAG, " Humidity compensation using '%s'", this->humidity_source_->get_name().c_str()); + } + LOG_UPDATE_INTERVAL(this); + LOG_SENSOR(" ", "CO2", this->co2_sensor_); + LOG_SENSOR(" ", "Temperature", this->temperature_sensor_); + LOG_SENSOR(" ", "Humidity", this->humidity_sensor_); +} + +void STCC4Component::update() { + if (!this->ready_ || this->measurement_mode_ != MeasurementMode::SINGLE_SHOT) + return; + + // Perform single-shot measurement, wait 500 ms for the measurement to be ready + if (!this->write_command(STCC4_CMD_MEASURE_SINGLE_SHOT)) { + ESP_LOGW(TAG, "Failed to start single shot measurement"); + this->status_set_warning(); + return; + } + this->set_timeout(500, [this]() { + if (this->read_measurement_(0)) { + this->status_clear_warning(); + } else { + ESP_LOGW(TAG, "Failed to read measurement data"); + this->status_set_warning(); + } + }); +} + +void STCC4Component::schedule_continuous_update_(bool retry_for_clock_drift) { + // In continuous measurement mode, the STCC4 produces a sample every 1000 ms according to its + // internal clock. The datasheet recommends retrying 150 ms after a failed read to compensate + // for clock drift between the host and the device. + this->set_timeout(retry_for_clock_drift ? 150 : 1000, [this, retry_for_clock_drift]() { + if (this->read_measurement_(retry_for_clock_drift ? 0 : sensirion_common::SENSIRION_OPTION_READ_MAY_NACK)) { + this->status_clear_warning(); + this->schedule_continuous_update_(false); + } else if (!retry_for_clock_drift) { + this->schedule_continuous_update_(true); + } else { + this->status_set_warning(); + this->schedule_continuous_update_(false); + } + }); +} + +bool STCC4Component::read_measurement_(uint8_t sensirion_options) { + // Read measurement data: 4 words (CO2, temperature, humidity, status) + uint16_t raw_data[4]; + if (!this->get_register_(STCC4_CMD_READ_MEASUREMENT, ADDR_16_BIT, raw_data, 4, 1, sensirion_options)) { + return false; + } + + // CO2 value is in ppm as int16 (ignore negative values during warm-up) + const int16_t co2_raw = int16_t(raw_data[0]); + if (this->co2_sensor_ != nullptr && co2_raw >= 0) { + this->co2_sensor_->publish_state(co2_raw); + } + + if (this->temperature_sensor_ != nullptr) { + this->temperature_sensor_->publish_state(temperature_in_ticks_to_c(raw_data[1])); + } + + if (this->humidity_sensor_ != nullptr) { + this->humidity_sensor_->publish_state(humidity_in_ticks_to_percent(raw_data[2])); + } + return true; +} + +void STCC4Component::update_rht_compensation_from_source_() { + const float temperature_in_c = this->temperature_source_->state; + const float humidity_in_percent = this->humidity_source_->state; + if (std::isnan(temperature_in_c) || std::isnan(humidity_in_percent)) + return; + + const uint16_t temperature_in_ticks = temperature_in_c_to_ticks(temperature_in_c); + const uint16_t humidity_in_ticks = humidity_in_percent_to_ticks(humidity_in_percent); + if ((this->temperature_in_ticks_ != temperature_in_ticks || this->humidity_in_ticks_ != humidity_in_ticks) && + this->write_rht_compensation_(temperature_in_ticks, humidity_in_ticks)) { + this->temperature_in_ticks_ = temperature_in_ticks; + this->humidity_in_ticks_ = humidity_in_ticks; + } +} + +void STCC4Component::set_ambient_pressure_compensation(float pressure_in_hpa) { + this->ambient_pressure_in_pa_2_ = pressure_in_hpa_to_pa_2(pressure_in_hpa); +} + +void STCC4Component::update_ambient_pressure_compensation_from_source_() { + const float pressure_in_hpa = this->ambient_pressure_source_->state; + if (std::isnan(pressure_in_hpa)) + return; + + if (pressure_in_hpa < 100.f || pressure_in_hpa > 10000.f) { + // Some pressure sensors report values in Pa instead of hPa and there's no way to check at compile time. + // Warn if the value seems far outside of the expected range. + if (!this->ambient_pressure_unit_warning_logged_) { + this->ambient_pressure_unit_warning_logged_ = true; + ESP_LOGW(TAG, "Ambient pressure compensation sensor might have incompatible units: got %f hPa", pressure_in_hpa); + } + return; // skip this update + } else { + this->ambient_pressure_unit_warning_logged_ = false; + } + + const uint16_t ambient_pressure_in_pa_2 = pressure_in_hpa_to_pa_2(pressure_in_hpa); + if (this->ambient_pressure_in_pa_2_ != ambient_pressure_in_pa_2 && + this->write_ambient_pressure_compensation_(ambient_pressure_in_pa_2)) { + this->ambient_pressure_in_pa_2_ = ambient_pressure_in_pa_2; + } +} + +bool STCC4Component::write_rht_compensation_(uint16_t temperature_in_ticks, uint16_t humidity_in_ticks) { + ESP_LOGVV(TAG, "Set RHT compensation: %f °C, %f %%RH", temperature_in_ticks_to_c(temperature_in_ticks), + humidity_in_ticks_to_percent(humidity_in_ticks)); + const uint16_t data[2] = {temperature_in_ticks, humidity_in_ticks}; + if (!this->write_command(STCC4_CMD_SET_RHT_COMPENSATION, data, 2)) { + ESP_LOGE(TAG, "Failed to set RHT compensation"); + return false; + } + return true; +} + +bool STCC4Component::write_ambient_pressure_compensation_(uint16_t pressure_in_pa_2) { + ESP_LOGVV(TAG, "Set pressure compensation: %f hPa", pressure_in_pa_2_to_hpa(pressure_in_pa_2)); + if (!this->write_command(STCC4_CMD_SET_PRESSURE_COMPENSATION, pressure_in_pa_2)) { + ESP_LOGE(TAG, "Failed to set ambient pressure compensation"); + return false; + } + return true; +} + +} // namespace esphome::stcc4 diff --git a/esphome/components/stcc4/stcc4.h b/esphome/components/stcc4/stcc4.h new file mode 100644 index 00000000000..ad25130ec73 --- /dev/null +++ b/esphome/components/stcc4/stcc4.h @@ -0,0 +1,55 @@ +#pragma once + +#include "esphome/core/component.h" +#include "esphome/components/sensor/sensor.h" +#include "esphome/components/sensirion_common/i2c_sensirion.h" + +namespace esphome::stcc4 { + +enum class MeasurementMode : uint8_t { + CONTINUOUS = 0, + SINGLE_SHOT, +}; + +class STCC4Component final : public PollingComponent, public sensirion_common::SensirionI2CDevice { + public: + void setup() override; + void dump_config() override; + void update() override; + + void set_co2_sensor(sensor::Sensor *co2) { this->co2_sensor_ = co2; } + void set_temperature_sensor(sensor::Sensor *temperature) { this->temperature_sensor_ = temperature; } + void set_humidity_sensor(sensor::Sensor *humidity) { this->humidity_sensor_ = humidity; } + void set_temperature_source(sensor::Sensor *temperature) { this->temperature_source_ = temperature; } + void set_humidity_source(sensor::Sensor *humidity) { this->humidity_source_ = humidity; } + void set_ambient_pressure_compensation(float pressure_in_hpa); + void set_ambient_pressure_source(sensor::Sensor *pressure) { this->ambient_pressure_source_ = pressure; } + void set_measurement_mode(MeasurementMode mode) { this->measurement_mode_ = mode; } + + protected: + void poll_until_ready_for_setup_or_timeout_(uint32_t start_time); + void finish_setup_(); + void schedule_continuous_update_(bool retry_for_clock_drift); + bool read_measurement_(uint8_t sensirion_options); + void update_rht_compensation_from_source_(); + void update_ambient_pressure_compensation_from_source_(); + bool write_rht_compensation_(uint16_t temperature_in_ticks, uint16_t humidity_in_ticks); + bool write_ambient_pressure_compensation_(uint16_t pressure_in_pa_2); + + sensor::Sensor *co2_sensor_{nullptr}; + sensor::Sensor *temperature_sensor_{nullptr}; + sensor::Sensor *humidity_sensor_{nullptr}; + sensor::Sensor *temperature_source_{nullptr}; + sensor::Sensor *humidity_source_{nullptr}; + sensor::Sensor *ambient_pressure_source_{nullptr}; + + MeasurementMode measurement_mode_{MeasurementMode::CONTINUOUS}; + + uint16_t temperature_in_ticks_{0}; + uint16_t humidity_in_ticks_{0}; + uint16_t ambient_pressure_in_pa_2_{0}; + bool ambient_pressure_unit_warning_logged_{false}; + bool ready_{false}; +}; + +} // namespace esphome::stcc4 diff --git a/tests/components/stcc4/common.yaml b/tests/components/stcc4/common.yaml new file mode 100644 index 00000000000..e87b7192269 --- /dev/null +++ b/tests/components/stcc4/common.yaml @@ -0,0 +1,36 @@ +sensor: + # SHT4x for external RHT compensation (boards without onboard SHT4x) + - platform: sht4x + i2c_id: i2c_bus + temperature: + name: SHT4x Temperature + id: sht4x_temperature + humidity: + name: SHT4x Humidity + id: sht4x_humidity + - platform: template + id: ambient_pressure + unit_of_measurement: "hPa" + lambda: 'return 1013;' + + # STCC4 CO2 sensor with onboard temperature/humidity and external compensation + - platform: stcc4 + i2c_id: i2c_bus + id: stcc4_sensor_single_shot + co2: + name: STCC4 CO2 + temperature: + name: STCC4 Temperature + humidity: + name: STCC4 Humidity + temperature_source: sht4x_temperature + humidity_source: sht4x_humidity + measurement_mode: single_shot + ambient_pressure_compensation: 1013 mbar + update_interval: 60s + - platform: stcc4 + i2c_id: i2c_bus + address: 0x65 + id: stcc4_sensor_continuous + measurement_mode: continuous + ambient_pressure_compensation_source: ambient_pressure diff --git a/tests/components/stcc4/test.esp32-idf.yaml b/tests/components/stcc4/test.esp32-idf.yaml new file mode 100644 index 00000000000..b47e39c3898 --- /dev/null +++ b/tests/components/stcc4/test.esp32-idf.yaml @@ -0,0 +1,4 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/esp32-idf.yaml + +<<: !include common.yaml diff --git a/tests/components/stcc4/test.esp8266-ard.yaml b/tests/components/stcc4/test.esp8266-ard.yaml new file mode 100644 index 00000000000..4a98b9388ab --- /dev/null +++ b/tests/components/stcc4/test.esp8266-ard.yaml @@ -0,0 +1,4 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/esp8266-ard.yaml + +<<: !include common.yaml diff --git a/tests/components/stcc4/test.rp2040-ard.yaml b/tests/components/stcc4/test.rp2040-ard.yaml new file mode 100644 index 00000000000..319a7c71a65 --- /dev/null +++ b/tests/components/stcc4/test.rp2040-ard.yaml @@ -0,0 +1,4 @@ +packages: + i2c: !include ../../test_build_components/common/i2c/rp2040-ard.yaml + +<<: !include common.yaml From dff793a9493a13ed44dd6f166a3a36a2661bf46a Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Thu, 1 Oct 2026 15:20:27 -0500 Subject: [PATCH 133/133] [espidf] Mirror registry components locally so configures skip the registry (#19949) --- esphome/espidf/component_mirror.py | 431 ++++++++++ esphome/espidf/toolchain.py | 38 +- esphome/framework_helpers.py | 6 +- .../test_espidf_component_mirror.py | 801 ++++++++++++++++++ tests/unit_tests/test_espidf_toolchain.py | 99 +++ tests/unit_tests/test_framework_helpers.py | 16 +- tests/unit_tests/test_lazy_imports.py | 4 +- 7 files changed, 1381 insertions(+), 14 deletions(-) create mode 100644 esphome/espidf/component_mirror.py create mode 100644 tests/unit_tests/test_espidf_component_mirror.py diff --git a/esphome/espidf/component_mirror.py b/esphome/espidf/component_mirror.py new file mode 100644 index 00000000000..a6bbf435fad --- /dev/null +++ b/esphome/espidf/component_mirror.py @@ -0,0 +1,431 @@ +"""Local mirror of IDF component-registry packages. + +The component manager contacts the registry on every cmake configure, even +with an unchanged ``dependencies.lock``, but checks +``IDF_COMPONENT_LOCAL_STORAGE_URL`` mirrors first and stops on a hit. +Mirroring the pinned components with its own ``registry sync`` removes all +registry traffic and makes builds work offline. +""" + +from __future__ import annotations + +from contextlib import suppress +from dataclasses import dataclass, field +import json +import logging +import os +from pathlib import Path +import subprocess +from typing import TYPE_CHECKING, NamedTuple +import zipfile + +from esphome.build_helpers.tools_cache import IDF_TOOLS_CACHE, tools_cache_path +from esphome.core import EsphomeError +from esphome.framework_helpers import rename_with_retry +from esphome.helpers import rmtree, write_file + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Mapping + +_LOGGER = logging.getLogger(__name__) + +# Bump the name if the manager's storage layout ever changes incompatibly; +# the stale directory is removed with the tools cache on clean-all. +_MIRROR_DIR_NAME = "component_mirror" + +_ENV_LOCAL_STORAGE_URL = "IDF_COMPONENT_LOCAL_STORAGE_URL" +_ENV_CHECK_NEW_VERSION = "IDF_COMPONENT_CHECK_NEW_VERSION" + +_DEFAULT_REGISTRY_URL = "https://components.espressif.com" +# Everything a missing, unreadable or wrongly-shaped index file can raise. +_BAD_INDEX_ERRORS = (OSError, ValueError, TypeError, KeyError, AttributeError) +_SYNC_LOCK_NAME = ".sync.lock" +_STAGING_DIR_NAME = ".staging" +_SYNC_TIMEOUT_S = 120 + + +class ServiceDep(NamedTuple): + """A registry ("service") dependency pinned to an exact version.""" + + namespace: str + name: str + version: str + + @property + def spec(self) -> str: + return f"{self.namespace}/{self.name}=={self.version}" + + +def get_mirror_path() -> Path: + """The machine-global mirror directory.""" + return tools_cache_path(*IDF_TOOLS_CACHE) / _MIRROR_DIR_NAME + + +def component_mirror_env() -> dict[str, str]: + """Environment additions that serve the mirror to the component manager. + + A user local storage list keeps precedence; the new-version check (an + extra solve whose answer exact pins make moot) is off unless user-set. + """ + mirror = get_mirror_path() + try: + mirror.mkdir(parents=True, exist_ok=True) + except OSError as err: + # Offline support is quietly lost otherwise; say so. + _LOGGER.warning("Component mirror unavailable at %s: %s", mirror, err) + return {} + _LOGGER.info("Serving IDF components from the local mirror at %s", mirror) + local_storage = mirror.as_uri() + if user_value := os.environ.get(_ENV_LOCAL_STORAGE_URL): + local_storage = f"{user_value};{local_storage}" + env = {_ENV_LOCAL_STORAGE_URL: local_storage} + if _ENV_CHECK_NEW_VERSION not in os.environ: + env[_ENV_CHECK_NEW_VERSION] = "0" + return env + + +def _load_yaml_dict(path: Path) -> dict | None: + """Read a small YAML mapping; missing file or bad content is None.""" + # Deferred: keeps pyyaml off the serial upload fast path. + import yaml + + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError: + return None + except (OSError, UnicodeDecodeError) as err: + _LOGGER.warning("Could not read %s: %s", path, err) + return None + try: + data = yaml.safe_load(text) + except yaml.YAMLError as err: + _LOGGER.warning("Could not parse %s: %s", path, err) + return None + return data if isinstance(data, dict) else None + + +def _iter_deps(path: Path) -> Iterator[tuple[str, str, dict]]: + """(namespace, name, entry) for each namespaced dependency in a YAML file.""" + data = _load_yaml_dict(path) + deps = data.get("dependencies") if data else None + if not isinstance(deps, dict): + return + for key, entry in deps.items(): + if isinstance(key, str) and "/" in key and isinstance(entry, dict): + # The registry stores lowercase paths; match a mixed-case key. + namespace, _, name = key.lower().partition("/") + yield namespace, name, entry + + +def parse_lock_service_deps(lock_path: Path) -> list[ServiceDep]: + """Pinned default-registry dependencies from a dependencies.lock. + + Git, local and idf sources cannot be mirrored; non-default registries + are left to the manager. + """ + deps: list[ServiceDep] = [] + for namespace, name, entry in _iter_deps(lock_path): + source = entry.get("source") + if not isinstance(source, dict) or source.get("type") != "service": + continue + registry_url = str(source.get("registry_url") or _DEFAULT_REGISTRY_URL) + if registry_url.rstrip("/") != _DEFAULT_REGISTRY_URL: + _LOGGER.debug("Not mirroring %s/%s: non-default registry", namespace, name) + continue + if isinstance(version := entry.get("version"), str): + deps.append(ServiceDep(namespace, name, version)) + return deps + + +def parse_manifest_service_deps(manifest_path: Path) -> list[ServiceDep]: + """Exactly-pinned registry dependencies from an idf_component.yml. + + The manifest exists before the first configure, so a fresh solve can + install from the mirror; range specs are left to the solver. + """ + deps: list[ServiceDep] = [] + for namespace, name, entry in _iter_deps(manifest_path): + if entry.keys() != {"version"} or not isinstance( + version := entry["version"], str + ): + continue + # The YAML shorthand keeps its operator: "==1.2.3" is an exact pin. + version = version.removeprefix("==") + # Exact versions start with a digit ("1.12.0", "1.3.3~1"); range + # operators are prefixes and wildcards contain "*". + if version[:1].isdigit() and "*" not in version: + deps.append(ServiceDep(namespace, name, version)) + return deps + + +def parse_manifest_ranged_specs(manifest_path: Path) -> list[str]: + """Compote specs for the manifest's range-pinned entries. + + Refreshed before every fresh solve, as a range resolves on dev; the + previous candidate stays the offline fallback. + """ + specs: list[str] = [] + for namespace, name, entry in _iter_deps(manifest_path): + if entry.keys() != {"version"} or not isinstance( + version := entry["version"], str + ): + continue + # [:1] alone would match "": the empty string is in every string. + if version.startswith(("^", "~", "<", ">", "!")) or ( + version.startswith("=") and not version.startswith("==") + ): + specs.append(f"{namespace}/{name}{version}") + return specs + + +def project_service_deps(lock_path: Path, manifest_path: Path) -> list[ServiceDep]: + """The mirrorable dependencies of a build; the lock wins, the manifest + covers the build whose lock has not been written yet.""" + deps = parse_lock_service_deps(lock_path) + seen = {(dep.namespace, dep.name) for dep in deps} + for dep in parse_manifest_service_deps(manifest_path): + if (dep.namespace, dep.name) not in seen: + deps.append(dep) + return deps + + +def _mirror_has(mirror: Path, dep: ServiceDep) -> bool: + """Whether the mirror holds the dependency's index entry and files. + + The manager fetches every file the entry names with no registry + fallback once the version is found locally. + """ + json_path = mirror / "components" / dep.namespace / f"{dep.name}.json" + try: + for entry in json.loads(json_path.read_text(encoding="utf-8"))["versions"]: + if entry.get("version") == dep.version: + checksums = entry.get("checksums") + # is_zipfile rejects an archive torn by a crash or an + # old race, so the next sync replaces it (self-heal). + archive = mirror / entry["url"] + if ( + archive.suffix == ".zip" + and archive.is_file() + and not (zipfile.is_zipfile(archive)) + ): + # Deleted, the manager falls back to the registry even + # when the heal sync cannot run; kept, it installs the + # torn file and the configure fails. + with suppress(OSError): + archive.unlink() + return False + return archive.is_file() and ( + not checksums or (mirror / checksums).is_file() + ) + except _BAD_INDEX_ERRORS: + return False + return False + + +def missing_deps(mirror: Path, deps: list[ServiceDep]) -> list[ServiceDep]: + return [dep for dep in deps if not _mirror_has(mirror, dep)] + + +def _read_versions(path: Path) -> list[dict]: + """The version entries of an index; [] when missing or corrupt, so + publishing heals it. An I/O error raises: the content may be valid, + so the caller retries later instead of dropping its versions.""" + try: + entries = json.loads(path.read_text(encoding="utf-8"))["versions"] + return [entry for entry in entries if isinstance(entry, dict)] + except FileNotFoundError: + return [] + except (ValueError, TypeError, KeyError) as err: + _LOGGER.debug("Ignoring the corrupt index %s: %s", path, err) + return [] + + +def _publish_index(src: Path, dst: Path) -> None: + """Publish the staged index merged with the live one, atomically. + + The staged index lists only the versions this sync fetched; the + atomic write keeps the live index intact when it fails, and raises. + """ + staged = json.loads(src.read_text(encoding="utf-8")) + versions = [entry for entry in staged["versions"] if isinstance(entry, dict)] + known = {entry.get("version") for entry in versions} + staged["versions"] = versions + [ + entry for entry in _read_versions(dst) if entry.get("version") not in known + ] + write_file(dst, json.dumps(staged)) + + +def _is_component_index(rel: Path) -> bool: + return ( + rel.parts[0] == "components" and len(rel.parts) == 3 and rel.suffix == ".json" + ) + + +def _promote(staging: Path, mirror: Path) -> None: + """Move the synced files into the mirror, atomically per file. + + Indexes go last, so a concurrent configure never reads a version + entry whose files have not landed yet. + """ + files = sorted( + (path for path in staging.rglob("*") if path.is_file()), + key=lambda path: (_is_component_index(path.relative_to(staging)), path), + ) + for src in files: + rel = src.relative_to(staging) + dst = mirror / rel + dst.parent.mkdir(parents=True, exist_ok=True) + if _is_component_index(rel): + _publish_index(src, dst) + else: + rename_with_retry(src, dst, overwrite=True) + + +@dataclass +class _SyncRun: + """Tools and progress of one registry sync.""" + + python: str + env: dict[str, str] + staging: Path + mirror: Path + synced: int = field(default=0, init=False) + + def _attempt(self, specs: list[str]) -> bool: + """One registry sync invocation; promotes its staging on success.""" + # Raises on an undeletable tree; never promote stale files. + rmtree(self.staging) + cmd = [self.python, "-m", "idf_component_manager", "registry", "sync"] + # latest: the default "all" follows ranged transitive specs and + # syncs every matching version. + cmd += ["--resolution", "latest"] + for spec in specs: + cmd += ["--component", spec] + cmd.append(str(self.staging)) + result = subprocess.run( + cmd, + env=self.env, + capture_output=True, + # Not text=True: the locale codec can raise UnicodeDecodeError. + encoding="utf-8", + errors="replace", + timeout=_SYNC_TIMEOUT_S, + check=False, + ) + if result.returncode != 0: + tail = "\n".join((result.stderr or result.stdout).strip().splitlines()[-5:]) + _LOGGER.warning( + "Could not mirror %s (exit %d):\n%s", + ", ".join(specs), + result.returncode, + tail, + ) + return False + _promote(self.staging, self.mirror) + return True + + def sync(self, specs: list[str]) -> bool: + """The whole set in one invocation, per component after a timeout.""" + try: + # One invocation for the whole set: one manager startup, which + # matters on slow hosts. + if not self._attempt(specs): + return False + self.synced = len(specs) + return True + except subprocess.TimeoutExpired: + if len(specs) == 1: + raise + _LOGGER.warning( + "Mirroring timed out after %d s; retrying one component at a time", + _SYNC_TIMEOUT_S, + ) + # Per component the timeout bounds each one, finished ones stay + # promoted, and the first failure stops the loop. + for spec in specs: + if not self._attempt([spec]): + return False + self.synced += 1 + return True + + +def sync_component_mirror( + lock_path: Path, + manifest_path: Path, + get_python: Callable[[], str], + get_env: Callable[[], Mapping[str, str]], +) -> bool: + """Mirror any of the build's registry components the mirror lacks. + + Returns False on a failed attempt so callers can skip retrying this + run; ``get_python``/``get_env`` are only called when a sync is needed. + """ + mirror = get_mirror_path() + to_sync = missing_deps(mirror, project_service_deps(lock_path, manifest_path)) + # Before a fresh solve, ranged pins re-resolve as they would on dev. + refresh = [] if lock_path.exists() else parse_manifest_ranged_specs(manifest_path) + if not to_sync and not refresh: + return True + try: + python = get_python() + env = dict(get_env()) + mirror.mkdir(parents=True, exist_ok=True) + except (OSError, EsphomeError) as err: + _LOGGER.warning("Could not mirror IDF components: %s", err) + return False + # Staged: the manager writes in place, and a configure in another + # process must never read a truncated file from the live mirror. + staging = mirror / _STAGING_DIR_NAME + # Lazy import, as in git.py: keeps filelock off the CLI startup path. + from filelock import FileLock, Timeout + + lock = FileLock(str(mirror / _SYNC_LOCK_NAME), fallback_to_soft=False) + try: + lock.acquire(blocking=False) + except Timeout: + # Another esphome process is already filling the shared mirror. + _LOGGER.debug("Component mirror sync skipped: already in progress") + return True + except OSError as err: + # A broken cache, not contention; the retry guard should apply. + _LOGGER.warning("Could not lock the component mirror: %s", err) + return False + specs = [dep.spec for dep in to_sync] + refresh + _LOGGER.info("Mirroring %d IDF component(s) for offline builds...", len(specs)) + run = _SyncRun(python, env, staging, mirror) + try: + if not run.sync(specs): + return False + if still := missing_deps(mirror, to_sync): + # A name the registry spells differently syncs clean yet + # covers nothing; retrying would loop forever. + _LOGGER.warning( + "Mirror sync left %d component(s) uncovered: %s", + len(still), + ", ".join(dep.spec for dep in still), + ) + return False + except subprocess.TimeoutExpired: + _LOGGER.warning( + "Mirroring timed out after %d s; kept %d of %d component(s)", + _SYNC_TIMEOUT_S, + run.synced, + len(specs), + ) + return False + except (*_BAD_INDEX_ERRORS, EsphomeError, subprocess.SubprocessError) as err: + # Includes a failed index publish; the live index is intact. + # A shape error is a bug: keep its traceback. + _LOGGER.warning( + "Could not mirror IDF components: %s", + err, + exc_info=isinstance(err, (TypeError, KeyError, AttributeError)), + ) + return False + finally: + # Cleanup only; a leftover tree is removed by the next attempt. + with suppress(OSError): + rmtree(staging) + lock.release() + _LOGGER.info("Mirrored %d IDF component(s) for offline builds", run.synced) + return True diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index ffd80825159..2e8cc0c6329 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -24,6 +24,7 @@ from esphome.const import ( ) from esphome.core import CORE, EsphomeError from esphome.espidf import variant_to_idf_target +from esphome.espidf.component_mirror import component_mirror_env, sync_component_mirror from esphome.espidf.framework import check_esp_idf_install, get_framework_env from esphome.espidf.size_summary import print_summary from esphome.helpers import add_git_ceiling_directory, get_bool_env, rmtree, write_file @@ -43,6 +44,7 @@ class _CacheData: cmake_output: dict[Path, str] = field(default_factory=dict) skip_bootloader: bool | None = None cmake_tools: dict[Path, dict[str, Path]] = field(default_factory=dict) + mirror_sync_failed: bool = False def _cache() -> _CacheData: @@ -110,6 +112,11 @@ def _get_idf_path(version: str | None = None) -> Path | None: return Path(_get_esphome_esp_idf_paths(version)[0]) +def _esphome_manages_idf() -> bool: + """A checkout supplied through IDF_PATH is the user's, not ESPHome's.""" + return "IDF_PATH" not in os.environ + + def _get_idf_env(version: str | None = None) -> dict[str, str]: """Get environment variables needed for ESP-IDF build.""" version = version or _get_core_framework_version() @@ -120,10 +127,12 @@ def _get_idf_env(version: str | None = None) -> dict[str, str]: env_cache[version].pop("PYTHONPATH", None) # Use provided IDF framework if available - if "IDF_PATH" not in os.environ: + if _esphome_manages_idf(): env_cache[version] |= get_framework_env( *_get_esphome_esp_idf_paths(version) ) + # Serve the component manager from the local registry mirror. + env_cache[version] |= component_mirror_env() # Cap git's repo search at the config directory so ESP-IDF's # `git describe` for the app version can't error out on an @@ -369,6 +378,8 @@ def run_reconfigure(verbose: bool = False) -> int: if (path := build_dir / stale).is_dir(): rmtree(path) get_factory_firmware_path().unlink(missing_ok=True) + # First, so the configure (even a first solve) installs from the mirror. + _sync_component_mirror() cmd = [_get_idf_tool("cmake"), "-G", "Ninja"] if _idf_py().binary_dir_arg: cmd += ["-B", str(build_dir)] @@ -387,6 +398,9 @@ def run_reconfigure(verbose: bool = False) -> int: (build_dir / "CMakeCache.txt").unlink(missing_ok=True) _LOGGER.error("CMake configure failed with exit code %d", rc) _print_hints(log_path) + return rc + # Mirror what the solve added to the lock (transitive dependencies). + _sync_component_mirror() return rc @@ -481,6 +495,24 @@ def _print_hints(log_path: Path) -> None: _LOGGER.warning("%s", hints) +def _sync_component_mirror() -> None: + """Best-effort update of the local registry mirror; never fails the build.""" + if not _esphome_manages_idf(): + # _get_idf_env injects no mirror env, so nothing would read a sync. + return + cache = _cache() + if cache.mirror_sync_failed: + return + if not sync_component_mirror( + CORE.relative_build_path("dependencies.lock"), + CORE.relative_build_path("src/idf_component.yml"), + lambda: _get_idf_tool("python"), + _get_idf_env, + ): + # One failed attempt (e.g. offline) is enough per run. + cache.mirror_sync_failed = True + + def _builtin_component_cache_path() -> Path | None: """Cache file for this build's built-in component list. @@ -491,7 +523,7 @@ def _builtin_component_cache_path() -> Path | None: of CONFIG_* options and only gate their sources on them. A checkout supplied through IDF_PATH is not managed by ESPHome and is never cached. """ - if "IDF_PATH" in os.environ: + if not _esphome_manages_idf(): return None target = variant_to_idf_target(CORE.data[KEY_ESP32][KEY_VARIANT]) excluded = CORE.cmake_args.get("EXCLUDE_COMPONENTS", "") @@ -793,6 +825,8 @@ def run_compile(config, verbose: bool) -> int: return rc else: _LOGGER.info("Build configuration is up to date") + # Ninja can still re-run cmake on its own; keep the mirror current. + _sync_component_mirror() if not get_bool_env("ESPHOME_LDGEN_FULL_DEPS"): _warn_if_app_archive_mapped() diff --git a/esphome/framework_helpers.py b/esphome/framework_helpers.py index 86010e30650..0599b531c28 100644 --- a/esphome/framework_helpers.py +++ b/esphome/framework_helpers.py @@ -461,7 +461,7 @@ def _zip_extract_all( report(1) -def _rename_with_retry( +def rename_with_retry( src: Path, dst: Path, attempts: int = 5, overwrite: bool = False ) -> None: """Rename ``src`` to ``dst`` with backoff retries on Windows sharing violations. @@ -561,7 +561,7 @@ def _7z_extract_all( rmtree(dest) else: dest.unlink() - _rename_with_retry(item, dest) + rename_with_retry(item, dest) finally: # staging is created before the try, so it always exists here; the # guard is defensive cleanup and its False branch is unreachable. @@ -1193,7 +1193,7 @@ def download_with_resume( # retries fail, keep the verified part so the next attempt (or # run) only has to redo the rename, not the download. try: - _rename_with_retry(part, dest, overwrite=True) + rename_with_retry(part, dest, overwrite=True) except PermissionError as e: _LOGGER.debug("Could not move %s into place: %s", part, e) last_error = e diff --git a/tests/unit_tests/test_espidf_component_mirror.py b/tests/unit_tests/test_espidf_component_mirror.py new file mode 100644 index 00000000000..fec535393e4 --- /dev/null +++ b/tests/unit_tests/test_espidf_component_mirror.py @@ -0,0 +1,801 @@ +"""Tests for the local IDF component-registry mirror.""" + +from __future__ import annotations + +from collections.abc import Callable +import json +from pathlib import Path +import subprocess +from unittest.mock import MagicMock, patch +import zipfile + +from filelock import FileLock +import pytest + +from esphome.core import EsphomeError +from esphome.espidf import component_mirror + +_LOCK_TEXT = """\ +dependencies: + bblanchon/arduinojson: + component_hash: 5e6aff2bc5bc988b36c8e90c01e637495ee895ad3af93911860c21ec6da79b6a + dependencies: [] + source: + registry_url: https://components.espressif.com/ + type: service + version: 7.4.3 + espressif/mdns: + component_hash: 3ba256ac95e07c274be53cbd73f06cb846c403b61e8fbdf1be57bdb79db7a63e + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com + type: service + version: 1.12.0 + esphome/libsodium: + source: + path: /pio_components/idf/cb5ad5b7/esphome/libsodium + type: local + version: '*' + fastled/fastled: + source: + git: https://github.com/fastled/fastled + type: git + version: 3.7.0 + other/private: + source: + registry_url: https://registry.example.com + type: service + version: 1.0.0 + idf: + source: + type: idf + version: 5.5.5 +direct_dependencies: +- bblanchon/arduinojson +- espressif/mdns +manifest_hash: f4e38c031cbb70d5070c5806fc580cce194b8c9a0b919f52a7e952f7d496dda4 +target: esp32 +version: 2.0.0 +""" + +_MANIFEST_TEXT = """\ +dependencies: + espressif/mdns: + version: 1.12.0 + espressif/esp-tflite-micro: + version: 1.3.3~1 + fastled/fastled: + version: 3.7.0 + git: https://github.com/fastled/fastled + esphome/libsodium: + version: '*' + override_path: /stub/libsodium + espressif/ranged: + version: ^1.2.0 + esphome/shorthand: + version: ==2.1.0 + esphome/unquoted: + version: 2 +""" + +_ARDUINOJSON = component_mirror.ServiceDep("bblanchon", "arduinojson", "7.4.3") +_MDNS = component_mirror.ServiceDep("espressif", "mdns", "1.12.0") +_TFLITE = component_mirror.ServiceDep("espressif", "esp-tflite-micro", "1.3.3~1") +_SHORTHAND = component_mirror.ServiceDep("esphome", "shorthand", "2.1.0") +_MANIFEST_DEPS = [_MDNS, _TFLITE, _SHORTHAND] +_NS_CMP_1 = component_mirror.ServiceDep("ns", "cmp", "1.0.0") +_NS_CMP_2 = component_mirror.ServiceDep("ns", "cmp", "2.0.0") + + +@pytest.fixture(autouse=True) +def _isolate_mirror(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ESPHOME_ESP_IDF_PREFIX", str(tmp_path / "idf_install")) + + +def _write_lock(project_dir: Path, text: str = _LOCK_TEXT) -> Path: + lock = project_dir / "dependencies.lock" + lock.parent.mkdir(parents=True, exist_ok=True) + lock.write_text(text, encoding="utf-8") + return lock + + +def _ns_cmp_lock(version: str) -> str: + return ( + "dependencies:\n" + " ns/cmp:\n" + " source:\n" + " type: service\n" + f" version: {version}\n" + ) + + +def _write_manifest(project_dir: Path, text: str = _MANIFEST_TEXT) -> Path: + manifest = project_dir / "src" / "idf_component.yml" + manifest.parent.mkdir(parents=True, exist_ok=True) + manifest.write_text(text, encoding="utf-8") + return manifest + + +def _add_to_mirror(mirror: Path, dep: component_mirror.ServiceDep) -> None: + """Lay out one component the way `registry sync` does.""" + version_dir = f"components/{dep.namespace}/{dep.name}/{dep.version}" + archive = f"{version_dir}/{dep.namespace}__{dep.name}-v{dep.version}.zip" + checksums = f"{version_dir}/CHECKSUMS.json" + json_path = mirror / "components" / dep.namespace / f"{dep.name}.json" + json_path.parent.mkdir(parents=True, exist_ok=True) + json_path.write_text( + json.dumps( + { + "name": dep.name, + "namespace": dep.namespace, + "versions": [ + {"version": dep.version, "url": archive, "checksums": checksums} + ], + } + ), + encoding="utf-8", + ) + archive_path = mirror / archive + archive_path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(archive_path, "w") as zf: + zf.writestr("idf_component.yml", "") + (mirror / checksums).write_text("{}", encoding="utf-8") + + +# --------------------------------------------------------------------------- +# parse_lock_service_deps / parse_manifest_service_deps / project_service_deps +# --------------------------------------------------------------------------- + + +def test_parse_lock_keeps_only_default_registry_service_deps(tmp_path: Path) -> None: + """git, local, idf and non-default registry sources cannot be mirrored.""" + deps = component_mirror.parse_lock_service_deps(_write_lock(tmp_path)) + assert deps == [_ARDUINOJSON, _MDNS] + + +def test_parse_lock_missing_file(tmp_path: Path) -> None: + assert component_mirror.parse_lock_service_deps(tmp_path / "none.lock") == [] + + +@pytest.mark.parametrize( + "text", + [ + pytest.param("{unbalanced", id="corrupt-yaml"), + pytest.param("[]", id="not-a-mapping"), + pytest.param("dependencies:\n espressif/mdns: not-a-dict\n", id="bad-entry"), + pytest.param(_ns_cmp_lock("1.2"), id="non-string-version"), + ], +) +def test_parse_lock_tolerates_bad_content(tmp_path: Path, text: str) -> None: + assert component_mirror.parse_lock_service_deps(_write_lock(tmp_path, text)) == [] + + +def test_parse_lock_unreadable_file_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """Read trouble other than a missing file says so instead of raising.""" + lock = tmp_path / "dependencies.lock" + lock.mkdir() # read_text raises OSError, not FileNotFoundError + assert component_mirror.parse_lock_service_deps(lock) == [] + assert "Could not read" in caplog.text + + +def test_parse_lock_non_utf8_file_warns( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A non-UTF-8 lock is a warning, never a failed build.""" + lock = tmp_path / "dependencies.lock" + lock.write_bytes(b"dependencies:\n # caf\xe9\n") + assert component_mirror.parse_lock_service_deps(lock) == [] + assert "Could not read" in caplog.text + + +def test_parse_lock_defaults_registry_url(tmp_path: Path) -> None: + """A service entry without registry_url is a default-registry dependency.""" + lock = _write_lock(tmp_path, _ns_cmp_lock("1.0.0")) + assert component_mirror.parse_lock_service_deps(lock) == [_NS_CMP_1] + + +def test_parse_manifest_keeps_only_exact_registry_pins(tmp_path: Path) -> None: + """Only exact pins survive, including the == shorthand prefix.""" + deps = component_mirror.parse_manifest_service_deps(_write_manifest(tmp_path)) + assert deps == _MANIFEST_DEPS + + +def test_parse_manifest_missing_file(tmp_path: Path) -> None: + assert component_mirror.parse_manifest_service_deps(tmp_path / "none.yml") == [] + + +def test_parse_manifest_lowercases_mixed_case_keys(tmp_path: Path) -> None: + """The registry stores lowercase paths; a mixed-case key must match them.""" + manifest = tmp_path / "idf_component.yml" + manifest.write_text("dependencies:\n Espressif/MDNS:\n version: 1.2.0\n") + deps = component_mirror.parse_manifest_service_deps(manifest) + assert deps == [component_mirror.ServiceDep("espressif", "mdns", "1.2.0")] + + +def test_parse_manifest_ranged_specs(tmp_path: Path) -> None: + """Ranges, not exact pins, git/override entries or an empty version, + become refresh specs.""" + manifest = _write_manifest( + tmp_path, _MANIFEST_TEXT + " ns/empty:\n version: ''\n" + ) + specs = component_mirror.parse_manifest_ranged_specs(manifest) + assert specs == ["espressif/ranged^1.2.0"] + + +def test_project_service_deps_merges_lock_and_manifest(tmp_path: Path) -> None: + """The lock wins for a component in both; the manifest fills the rest.""" + lock = _write_lock(tmp_path) + manifest = _write_manifest(tmp_path) + assert component_mirror.project_service_deps(lock, manifest) == [ + _ARDUINOJSON, + _MDNS, + _TFLITE, + _SHORTHAND, + ] + + +def test_project_service_deps_manifest_only(tmp_path: Path) -> None: + """A fresh build has no lock yet; the manifest alone drives the sync.""" + manifest = _write_manifest(tmp_path) + assert ( + component_mirror.project_service_deps(tmp_path / "dependencies.lock", manifest) + == _MANIFEST_DEPS + ) + + +# --------------------------------------------------------------------------- +# missing_deps +# --------------------------------------------------------------------------- + + +def test_missing_deps_covered_and_not(tmp_path: Path) -> None: + mirror = tmp_path / "mirror" + _add_to_mirror(mirror, _MDNS) + deps = [ + _MDNS, + component_mirror.ServiceDep("espressif", "mdns", "1.13.0"), + _ARDUINOJSON, + ] + assert component_mirror.missing_deps(mirror, deps) == deps[1:] + + +@pytest.mark.parametrize( + "damage", + [ + pytest.param( + lambda m: next(m.rglob("*.zip")).unlink(), + id="archive-gone", + ), + pytest.param( + lambda m: (m / "components/espressif/mdns/1.12.0/CHECKSUMS.json").unlink(), + id="checksums-gone", + ), + pytest.param( + lambda m: (m / "components/espressif/mdns.json").write_text("{broken"), + id="corrupt-index", + ), + pytest.param( + lambda m: next(m.rglob("*.zip")).write_bytes(b"PK\x03\x04torn"), + id="truncated-archive", + ), + ], +) +def test_missing_deps_damaged_mirror( + tmp_path: Path, damage: Callable[[Path], object] +) -> None: + """Every file the index references must exist, and a broken index + counts as missing, not covered.""" + mirror = tmp_path / "mirror" + _add_to_mirror(mirror, _MDNS) + damage(mirror) + assert component_mirror.missing_deps(mirror, [_MDNS]) == [_MDNS] + + +def test_missing_deps_deletes_a_torn_archive(tmp_path: Path) -> None: + """A torn archive is removed on detection, so the manager falls back to + the registry even before a heal sync runs.""" + mirror = tmp_path / "mirror" + _add_to_mirror(mirror, _MDNS) + torn = next(mirror.rglob("*.zip")) + torn.write_bytes(b"PK\x03\x04torn") + assert component_mirror.missing_deps(mirror, [_MDNS]) == [_MDNS] + assert not torn.exists() + + +def test_missing_deps_covered_without_checksums_field(tmp_path: Path) -> None: + """An older payload without a checksums field only needs its archive.""" + mirror = tmp_path / "mirror" + _add_to_mirror(mirror, _MDNS) + index = mirror / "components/espressif/mdns.json" + doc = json.loads(index.read_text()) + del doc["versions"][0]["checksums"] + index.write_text(json.dumps(doc)) + assert component_mirror.missing_deps(mirror, [_MDNS]) == [] + + +# --------------------------------------------------------------------------- +# _publish_index / _promote +# --------------------------------------------------------------------------- + + +def _staged_index(tmp_path: Path, versions: list) -> Path: + src = tmp_path / "staged.json" + src.write_text(json.dumps({"versions": versions}), encoding="utf-8") + return src + + +def test_publish_index_first_sync(tmp_path: Path) -> None: + """No live index yet: the staged one is published as is.""" + src = _staged_index(tmp_path, [{"version": "2.0.0"}]) + dst = tmp_path / "live.json" + component_mirror._publish_index(src, dst) + assert json.loads(dst.read_text()) == {"versions": [{"version": "2.0.0"}]} + + +@pytest.mark.parametrize( + "live", ["not json", '{"versions": null}', '{"versions": {"a": 1}}'] +) +def test_publish_index_replaces_a_corrupt_live_index(tmp_path: Path, live: str) -> None: + """An unreadable live index is replaced wholesale; that is the heal.""" + src = _staged_index(tmp_path, [{"version": "2.0.0"}]) + dst = tmp_path / "live.json" + dst.write_text(live) + component_mirror._publish_index(src, dst) + assert json.loads(dst.read_text()) == {"versions": [{"version": "2.0.0"}]} + + +def test_publish_index_merges_and_filters(tmp_path: Path) -> None: + """Unfetched live versions survive; non-dict entries drop, not raise.""" + src = _staged_index(tmp_path, [{"version": "2.0.0"}, "junk"]) + dst = tmp_path / "live.json" + dst.write_text( + json.dumps({"versions": [{"version": "1.0.0"}, {"version": "2.0.0"}, "bad"]}) + ) + component_mirror._publish_index(src, dst) + assert json.loads(dst.read_text()) == { + "versions": [{"version": "2.0.0"}, {"version": "1.0.0"}] + } + + +def test_publish_index_unreadable_live_index_raises(tmp_path: Path) -> None: + """An I/O error is not corruption; raising keeps the live versions and + a later run retries.""" + src = _staged_index(tmp_path, [{"version": "2.0.0"}]) + dst = tmp_path / "live.json" + dst.mkdir() # read_text raises IsADirectoryError, not FileNotFoundError + with pytest.raises(OSError): + component_mirror._publish_index(src, dst) + + +def test_promote_moves_archives_before_indexes(tmp_path: Path) -> None: + """A concurrent reader must never see an index entry without its files.""" + staging = tmp_path / "staging" + _add_to_mirror(staging, _NS_CMP_2) + order: list[str] = [] + real_rename = component_mirror.rename_with_retry + real_write = component_mirror.write_file + + def recording_rename(src: Path, dst: Path, **kwargs) -> None: + order.append(src.name) + real_rename(src, dst, **kwargs) + + def recording_write(path: Path, text: str) -> None: + order.append(path.name) + real_write(path, text) + + with ( + patch.object(component_mirror, "rename_with_retry", recording_rename), + patch.object(component_mirror, "write_file", recording_write), + ): + component_mirror._promote(staging, tmp_path / "mirror") + + assert order[-1] == "cmp.json" + assert set(order[:-1]) == {"CHECKSUMS.json", "ns__cmp-v2.0.0.zip"} + + +# --------------------------------------------------------------------------- +# component_mirror_env +# --------------------------------------------------------------------------- + + +def test_component_mirror_env_sets_file_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("IDF_COMPONENT_LOCAL_STORAGE_URL", raising=False) + monkeypatch.delenv("IDF_COMPONENT_CHECK_NEW_VERSION", raising=False) + env = component_mirror.component_mirror_env() + mirror = component_mirror.get_mirror_path() + assert env["IDF_COMPONENT_LOCAL_STORAGE_URL"] == mirror.as_uri() + assert env["IDF_COMPONENT_CHECK_NEW_VERSION"] == "0" + assert mirror.is_dir() + + +def test_component_mirror_env_preserves_user_values( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A user local storage list keeps precedence; their new-version choice wins.""" + monkeypatch.setenv("IDF_COMPONENT_LOCAL_STORAGE_URL", "file:///user/mirror") + monkeypatch.setenv("IDF_COMPONENT_CHECK_NEW_VERSION", "1") + env = component_mirror.component_mirror_env() + mirror_uri = component_mirror.get_mirror_path().as_uri() + assert env["IDF_COMPONENT_LOCAL_STORAGE_URL"] == f"file:///user/mirror;{mirror_uri}" + assert "IDF_COMPONENT_CHECK_NEW_VERSION" not in env + + +def test_component_mirror_env_unwritable_cache( + caplog: pytest.LogCaptureFixture, +) -> None: + """A read-only cache disables the feature with a warning, not a failure.""" + with patch.object(Path, "mkdir", side_effect=OSError("read-only")): + assert component_mirror.component_mirror_env() == {} + assert "Component mirror unavailable" in caplog.text + + +# --------------------------------------------------------------------------- +# sync_component_mirror +# --------------------------------------------------------------------------- + + +def _run_sync( + project_dir: Path, + *, + returncode: int = 0, + stderr: str = "", + side_effect: Exception | Callable | None = None, + get_python=lambda: "/penv/python", + get_env=lambda: {"PATH": "/penv"}, +) -> tuple[bool, MagicMock]: + with patch.object( + component_mirror.subprocess, + "run", + return_value=subprocess.CompletedProcess([], returncode, "", stderr), + side_effect=side_effect, + ) as mock_run: + ok = component_mirror.sync_component_mirror( + project_dir / "dependencies.lock", + project_dir / "src" / "idf_component.yml", + get_python, + get_env, + ) + return ok, mock_run + + +def _fake_registry_sync(returncode: int = 0): + """A subprocess.run stand-in that lays files out like `registry sync`.""" + + def run(cmd, **kwargs) -> subprocess.CompletedProcess: + _add_to_mirror(Path(cmd[-1]), _NS_CMP_2) + return subprocess.CompletedProcess(cmd, returncode, "", "sync failed") + + return run + + +def _assert_sync_lock_released() -> None: + lock = FileLock(str(component_mirror.get_mirror_path() / ".sync.lock")) + lock.acquire(blocking=False) + lock.release() + + +def test_sync_runs_the_manager_for_missing_deps( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + _write_lock(tmp_path) + ok, mock_run = _run_sync(tmp_path) + # The stub stages nothing, so the post-promote recheck reports failure. + assert not ok + assert "uncovered" in caplog.text + mirror = component_mirror.get_mirror_path() + # The whole set goes in one invocation: one manager startup. + mock_run.assert_called_once() + assert mock_run.call_args.args[0] == [ + "/penv/python", + "-m", + "idf_component_manager", + "registry", + "sync", + "--resolution", + "latest", + "--component", + "bblanchon/arduinojson==7.4.3", + "--component", + "espressif/mdns==1.12.0", + str(mirror / ".staging"), + ] + assert mock_run.call_args.kwargs["env"] == {"PATH": "/penv"} + _assert_sync_lock_released() + + +def test_sync_skips_when_covered(tmp_path: Path) -> None: + """A covered project resolves no environment and runs no subprocess.""" + _write_lock(tmp_path) + mirror = component_mirror.get_mirror_path() + _add_to_mirror(mirror, _MDNS) + _add_to_mirror(mirror, _ARDUINOJSON) + + def _boom() -> str: + raise AssertionError("environment resolved for a covered mirror") + + ok, mock_run = _run_sync(tmp_path, get_python=_boom, get_env=_boom) + assert ok + mock_run.assert_not_called() + + +def test_sync_failure_is_tolerated( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + _write_lock(tmp_path) + ok, mock_run = _run_sync(tmp_path, returncode=1, stderr="boom") + assert not ok + mock_run.assert_called_once() + assert "Could not mirror" in caplog.text + _assert_sync_lock_released() + + +def test_sync_subprocess_errors_are_tolerated( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + _write_lock(tmp_path) + ok, _ = _run_sync(tmp_path, side_effect=OSError("no such file")) + assert not ok + assert "Could not mirror" in caplog.text + _assert_sync_lock_released() + + +def test_sync_refreshes_ranged_specs_only_before_a_fresh_solve( + tmp_path: Path, +) -> None: + """Without a lock the batch also re-resolves ranged pins, as dev would; + with a lock the ranged candidate is left alone.""" + _write_manifest(tmp_path) + ok, mock_run = _run_sync(tmp_path) + assert not ok # the stub stages nothing, so the recheck fails + assert "espressif/ranged^1.2.0" in mock_run.call_args.args[0] + + _write_lock(tmp_path) + ok, mock_run = _run_sync(tmp_path) + assert not ok + assert "espressif/ranged^1.2.0" not in mock_run.call_args.args[0] + + +def test_sync_ranged_refresh_runs_with_everything_else_covered( + tmp_path: Path, +) -> None: + """A covered build still refreshes its ranged pins when the lock is + gone, in one invocation.""" + manifest = _write_manifest(tmp_path) + mirror = component_mirror.get_mirror_path() + for dep in component_mirror.parse_manifest_service_deps(manifest): + _add_to_mirror(mirror, dep) + ok, mock_run = _run_sync(tmp_path) + assert ok + mock_run.assert_called_once() + assert "espressif/ranged^1.2.0" in mock_run.call_args.args[0] + + +def test_sync_timeout_retries_ranged_specs_too(tmp_path: Path) -> None: + """The fallback covers refresh specs, not only the exact pins.""" + _write_manifest( + tmp_path, + "dependencies:\n ns/one:\n version: ^1.0\n ns/two:\n version: ^2.0\n", + ) + effects = [ + subprocess.TimeoutExpired(cmd=[], timeout=120), + subprocess.CompletedProcess([], 0, "", ""), + subprocess.CompletedProcess([], 0, "", ""), + ] + ok, mock_run = _run_sync(tmp_path, side_effect=effects) + assert ok + retried = [call.args[0][-2] for call in mock_run.call_args_list[1:]] + assert retried == ["ns/one^1.0", "ns/two^2.0"] + + +def test_sync_timeout_falls_back_to_one_component_at_a_time( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A timed-out batch retries per component, so the ones that finish + stay promoted and later runs fetch only the rest.""" + _write_lock(tmp_path) + + def slow_sync(cmd, **kwargs): + specs = [arg for arg in cmd if "==" in arg] + if specs != ["bblanchon/arduinojson==7.4.3"]: + # The batch, and later the mdns retry, exceed the timeout. + raise subprocess.TimeoutExpired(cmd=cmd, timeout=120) + namespace, _, rest = specs[0].partition("/") + name, _, version = rest.partition("==") + _add_to_mirror( + Path(cmd[-1]), component_mirror.ServiceDep(namespace, name, version) + ) + return subprocess.CompletedProcess(cmd, 0, "", "") + + ok, mock_run = _run_sync(tmp_path, side_effect=slow_sync) + assert not ok + assert mock_run.call_count == 3 # batch, arduinojson, mdns + assert "retrying one component" in caplog.text + assert "kept 1 of 2" in caplog.text + mirror = component_mirror.get_mirror_path() + assert component_mirror.missing_deps(mirror, [_ARDUINOJSON, _MDNS]) == [_MDNS] + assert not (mirror / ".staging").exists() + _assert_sync_lock_released() + + +def test_sync_timeout_then_per_component_completes(tmp_path: Path) -> None: + """When every component fits on its own, the fallback finishes the job.""" + _write_lock(tmp_path) + + def slow_sync(cmd, **kwargs): + specs = [arg for arg in cmd if "==" in arg] + if len(specs) > 1: + raise subprocess.TimeoutExpired(cmd=cmd, timeout=120) + namespace, _, rest = specs[0].partition("/") + name, _, version = rest.partition("==") + _add_to_mirror( + Path(cmd[-1]), component_mirror.ServiceDep(namespace, name, version) + ) + return subprocess.CompletedProcess(cmd, 0, "", "") + + ok, mock_run = _run_sync(tmp_path, side_effect=slow_sync) + assert ok + assert mock_run.call_count == 3 # batch, then one call per component + mirror = component_mirror.get_mirror_path() + assert component_mirror.missing_deps(mirror, [_ARDUINOJSON, _MDNS]) == [] + + +def test_sync_timeout_then_component_error_stops( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A hard error in the fallback loop stops it like any other failure.""" + _write_lock(tmp_path) + effects = [ + subprocess.TimeoutExpired(cmd=[], timeout=120), + subprocess.CompletedProcess([], 1, "", "boom"), + ] + ok, mock_run = _run_sync(tmp_path, side_effect=effects) + assert not ok + assert mock_run.call_count == 2 + assert "Could not mirror" in caplog.text + _assert_sync_lock_released() + + +def test_sync_unexpected_error_logs_a_traceback( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An operational failure warns in one line; a bug keeps its traceback.""" + _write_lock(tmp_path) + with patch.object(component_mirror, "_promote", side_effect=KeyError("url")): + ok, _ = _run_sync(tmp_path) + assert not ok + record = next(r for r in caplog.records if "Could not mirror" in r.message) + assert record.exc_info + + +def test_sync_single_component_timeout_is_not_retried( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """With one missing component the batch was already the retry; a build + pays the timeout once.""" + _write_lock(tmp_path, _ns_cmp_lock("2.0.0")) + ok, mock_run = _run_sync( + tmp_path, side_effect=subprocess.TimeoutExpired(cmd=[], timeout=120) + ) + assert not ok + mock_run.assert_called_once() + assert "kept 0 of 1" in caplog.text + _assert_sync_lock_released() + + +def test_sync_environment_resolution_failure_is_tolerated( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + _write_lock(tmp_path) + + def _boom() -> str: + raise EsphomeError("python not found") + + ok, mock_run = _run_sync(tmp_path, get_python=_boom) + assert not ok + mock_run.assert_not_called() + assert "Could not mirror" in caplog.text + + +def test_sync_skips_when_another_process_holds_the_lock(tmp_path: Path) -> None: + _write_lock(tmp_path) + mirror = component_mirror.get_mirror_path() + mirror.mkdir(parents=True) + held = FileLock(str(mirror / ".sync.lock")) + held.acquire(blocking=False) + try: + ok, mock_run = _run_sync(tmp_path) + finally: + held.release() + assert ok + mock_run.assert_not_called() + + +def test_sync_ignores_a_leftover_lock_file(tmp_path: Path) -> None: + """A lock file from a dead process does not block: the OS lock is gone.""" + _write_lock(tmp_path) + mirror = component_mirror.get_mirror_path() + mirror.mkdir(parents=True) + (mirror / ".sync.lock").touch() + _, mock_run = _run_sync(tmp_path) + mock_run.assert_called() # the dead lock did not block the sync + + +def test_sync_lock_oserror_is_a_failure( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A broken cache is not contention; the retry guard should apply.""" + _write_lock(tmp_path) + with patch("filelock.FileLock.acquire", side_effect=PermissionError("ro")): + ok, mock_run = _run_sync(tmp_path) + assert not ok + mock_run.assert_not_called() + assert "Could not lock" in caplog.text + + +def test_sync_undeletable_staging_is_a_failure( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """An undeletable staging tree aborts the sync; stale files must + never be promoted.""" + _write_lock(tmp_path) + with patch.object(component_mirror, "rmtree", side_effect=OSError("stuck staging")): + ok, mock_run = _run_sync(tmp_path) + assert not ok + mock_run.assert_not_called() + assert "Could not mirror" in caplog.text + _assert_sync_lock_released() + + +def test_sync_promotes_staged_files_and_merges_the_index(tmp_path: Path) -> None: + """New files land atomically and existing versions survive the merge.""" + mirror = component_mirror.get_mirror_path() + _add_to_mirror(mirror, _NS_CMP_1) + _write_lock(tmp_path, _ns_cmp_lock("2.0.0")) + + ok, _ = _run_sync(tmp_path, side_effect=_fake_registry_sync()) + + assert ok + doc = json.loads((mirror / "components" / "ns" / "cmp.json").read_text()) + assert {entry["version"] for entry in doc["versions"]} == {"1.0.0", "2.0.0"} + assert (mirror / "components/ns/cmp/2.0.0/ns__cmp-v2.0.0.zip").is_file() + assert (mirror / "components/ns/cmp/1.0.0/ns__cmp-v1.0.0.zip").is_file() + assert not (mirror / ".staging").exists() + + +def test_sync_failure_leaves_no_staging_behind(tmp_path: Path) -> None: + """A failed sync promotes nothing and removes its staging directory.""" + _write_lock(tmp_path, _ns_cmp_lock("2.0.0")) + + ok, _ = _run_sync(tmp_path, side_effect=_fake_registry_sync(returncode=1)) + + assert not ok + mirror = component_mirror.get_mirror_path() + assert not (mirror / ".staging").exists() + assert not (mirror / "components" / "ns").exists() + + +def test_sync_failed_index_publish_keeps_the_live_index( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + """A failed publish keeps the live index; a later run retries.""" + mirror = component_mirror.get_mirror_path() + _add_to_mirror(mirror, _NS_CMP_1) + _write_lock(tmp_path, _ns_cmp_lock("2.0.0")) + + with patch.object( + component_mirror, "write_file", side_effect=EsphomeError("disk full") + ): + ok, _ = _run_sync(tmp_path, side_effect=_fake_registry_sync()) + + assert not ok + assert "Could not mirror" in caplog.text + doc = json.loads((mirror / "components" / "ns" / "cmp.json").read_text()) + assert {entry["version"] for entry in doc["versions"]} == {"1.0.0"} + _assert_sync_lock_released() diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py index 63d15a694ca..1aaff7e9ceb 100644 --- a/tests/unit_tests/test_espidf_toolchain.py +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -215,6 +215,38 @@ def test_get_idf_env_sets_git_ceiling_directories(setup_core: Path) -> None: assert str(CORE.config_dir) in env["GIT_CEILING_DIRECTORIES"].split(os.pathsep) +def test_get_idf_env_serves_the_component_mirror(setup_core: Path) -> None: + """The mirror env rides the managed-IDF branch, next to its sync gate.""" + toolchain._cache().env.clear() + with ( + patch.object( + toolchain, + "get_framework_env", + return_value={"PATH": "/penv"}, + ), + patch.object(toolchain, "_get_esphome_esp_idf_paths", return_value=((), {})), + patch.object( + toolchain, + "component_mirror_env", + return_value={"IDF_COMPONENT_LOCAL_STORAGE_URL": "file:///mirror"}, + ), + ): + env = toolchain._get_idf_env(version="5.5.4") + assert env["IDF_COMPONENT_LOCAL_STORAGE_URL"] == "file:///mirror" + + +def test_get_idf_env_user_idf_skips_the_mirror(setup_core: Path) -> None: + """A user-managed IDF gets neither the framework env nor the mirror.""" + toolchain._cache().env.clear() + with ( + patch.dict(os.environ, {"IDF_PATH": str(setup_core)}), + patch.object(toolchain, "component_mirror_env") as mock_env, + ): + env = toolchain._get_idf_env(version="5.5.4") + mock_env.assert_not_called() + assert "IDF_COMPONENT_LOCAL_STORAGE_URL" not in env + + def test_get_idf_env_pops_inherited_pythonpath(setup_core: Path) -> None: """A PYTHONPATH from the parent environment must not reach idf.py. @@ -362,6 +394,73 @@ def test_run_compile_discovery_without_cmakecache(setup_core: Path) -> None: assert not CORE.relative_build_path("build/CMakeCache.txt").exists() +@pytest.mark.parametrize( + ("cmake_rc", "expected"), + [(0, ["sync", "cmake", "sync"]), (1, ["sync", "cmake"])], +) +def test_run_reconfigure_syncs_mirror_around_the_configure( + setup_core: Path, cmake_rc: int, expected: list[str] +) -> None: + """Every configure path syncs: before, so the solve installs from the + mirror; after a success, so solver-added lock entries are mirrored.""" + _setup_build(setup_core) + calls: list[str] = [] + + with ( + _fake_tools() as mock_run, + patch.object( + toolchain, + "_sync_component_mirror", + side_effect=lambda: calls.append("sync"), + ), + # The real hint printer resolves an IDF install (or downloads one). + patch.object(toolchain, "_print_hints"), + ): + mock_run.side_effect = lambda *a, **k: calls.append("cmake") or cmake_rc + assert toolchain.run_reconfigure() == cmake_rc + + assert calls == expected + + +def test_run_compile_syncs_mirror_when_up_to_date(setup_core: Path) -> None: + """No reconfigure still refreshes the mirror once for ninja-driven + cmake re-runs.""" + _setup_build(setup_core) + config = {CONF_ESPHOME: {}} + + with ( + _up_to_date_compile(), + patch.object(toolchain, "_sync_component_mirror") as mock_sync, + ): + assert toolchain.run_compile(config, verbose=False) == 0 + + mock_sync.assert_called_once() + + +def test_sync_component_mirror_skips_user_idf( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A user-managed IDF never gets the mirror env, so a sync is unread.""" + monkeypatch.setenv("IDF_PATH", "/opt/esp-idf") + with patch.object(toolchain, "sync_component_mirror") as mock_sync: + toolchain._sync_component_mirror() + mock_sync.assert_not_called() + + +def test_sync_component_mirror_failure_not_retried( + setup_core: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """One failed attempt (e.g. offline) must not stall every later call.""" + monkeypatch.delenv("IDF_PATH", raising=False) + _setup_build(setup_core) + with patch.object( + toolchain, "sync_component_mirror", return_value=False + ) as mock_sync: + toolchain._sync_component_mirror() + toolchain._sync_component_mirror() + mock_sync.assert_called_once() + + def test_run_compile_reconfigures_after_full_write_outside_testing_mode( setup_core: Path, ) -> None: diff --git a/tests/unit_tests/test_framework_helpers.py b/tests/unit_tests/test_framework_helpers.py index f3b182073f2..f63838a8e27 100644 --- a/tests/unit_tests/test_framework_helpers.py +++ b/tests/unit_tests/test_framework_helpers.py @@ -27,7 +27,6 @@ from esphome.framework_helpers import ( _7z_extract_all, _BatchDownloadProgress, _detect_archive_root, - _rename_with_retry, _tar_extract_all, _zip_extract_all, archive_extract_all, @@ -40,6 +39,7 @@ from esphome.framework_helpers import ( get_project_link_flags, get_python_env_executable_path, get_system_python_path, + rename_with_retry, rmdir, run_batch_downloads, run_command, @@ -1013,7 +1013,7 @@ class TestDownloadWithResume: with ( patch("requests.get", return_value=_mock_response(b"data")) as mock_get, patch( - "esphome.framework_helpers._rename_with_retry", + "esphome.framework_helpers.rename_with_retry", side_effect=[PermissionError("locked"), None], ) as rename, ): @@ -1030,7 +1030,7 @@ class TestDownloadWithResume: with ( patch("requests.get", return_value=_mock_response(b"data")), patch( - "esphome.framework_helpers._rename_with_retry", + "esphome.framework_helpers.rename_with_retry", side_effect=PermissionError("locked"), ), pytest.raises(EsphomeError, match="after 1 attempts"), @@ -2019,7 +2019,7 @@ class TestZipExtractAllBranches: # --------------------------------------------------------------------------- -# _rename_with_retry +# rename_with_retry # --------------------------------------------------------------------------- @@ -2028,7 +2028,7 @@ class TestRenameWithRetry: src = tmp_path / "src.txt" src.write_text("data") dst = tmp_path / "dst.txt" - _rename_with_retry(src, dst) + rename_with_retry(src, dst) assert dst.read_text() == "data" assert not src.exists() @@ -2050,7 +2050,7 @@ class TestRenameWithRetry: patch.object(Path, "rename", flaky_rename), patch("esphome.framework_helpers.time.sleep"), ): - _rename_with_retry(src, dst, attempts=3) + rename_with_retry(src, dst, attempts=3) assert dst.read_text() == "data" def test_raises_after_all_attempts_fail(self, tmp_path: Path) -> None: @@ -2062,14 +2062,14 @@ class TestRenameWithRetry: patch("esphome.framework_helpers.time.sleep"), pytest.raises(PermissionError), ): - _rename_with_retry(src, dst, attempts=3) + rename_with_retry(src, dst, attempts=3) def test_attempts_zero_is_noop(self, tmp_path: Path) -> None: """Zero attempts means the for-loop body never runs; src is untouched.""" src = tmp_path / "src.txt" src.write_text("data") dst = tmp_path / "dst.txt" - _rename_with_retry(src, dst, attempts=0) + rename_with_retry(src, dst, attempts=0) assert src.exists() assert not dst.exists() diff --git a/tests/unit_tests/test_lazy_imports.py b/tests/unit_tests/test_lazy_imports.py index 1eeddd80eac..28ca2394eb3 100644 --- a/tests/unit_tests/test_lazy_imports.py +++ b/tests/unit_tests/test_lazy_imports.py @@ -192,7 +192,9 @@ def test_native_upload_fast_path_does_not_import_heavy_modules( fixture_path, probe_env, "native_upload_fast_path.py", - extra=("esphome.components.esp8266",), + # The cache-hit modules guard the espidf toolchain import chain: + # a top-level yaml import there would tax every upload subprocess. + extra=("esphome.components.esp8266", *CACHE_HIT_HEAVY_MODULES), ) assert not leaked, ( f"native upload_using_esptool pulls in heavy modules: {leaked}. "