diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml index 52d72544d3..2081264b91 100644 --- a/.github/actions/build-image/action.yaml +++ b/.github/actions/build-image/action.yaml @@ -47,7 +47,7 @@ runs: - name: Build and push to ghcr by digest id: build-ghcr - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 env: DOCKER_BUILD_SUMMARY: false DOCKER_BUILD_RECORD_UPLOAD: false @@ -73,7 +73,7 @@ runs: - name: Build and push to dockerhub by digest id: build-dockerhub - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 env: DOCKER_BUILD_SUMMARY: false DOCKER_BUILD_RECORD_UPLOAD: false diff --git a/.github/actions/restore-python/action.yml b/.github/actions/restore-python/action.yml index 751f9ecf58..03b4803860 100644 --- a/.github/actions/restore-python/action.yml +++ b/.github/actions/restore-python/action.yml @@ -35,6 +35,10 @@ runs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true + # Pin uv version so the action does not have to fetch the + # manifest from raw.githubusercontent.com on every cache + # miss; that fetch flakes on Windows runners. + version: "0.11.15" - name: Create Python virtual environment if: steps.cache-venv.outputs.cache-hit != 'true' && runner.os != 'Windows' shell: bash diff --git a/.github/workflows/ci-api-proto.yml b/.github/workflows/ci-api-proto.yml index 1dc0ccb7fe..675bbe9d2c 100644 --- a/.github/workflows/ci-api-proto.yml +++ b/.github/workflows/ci-api-proto.yml @@ -32,6 +32,10 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true + # Pin uv version so the action does not have to fetch the + # manifest from raw.githubusercontent.com on every cache + # miss; that fetch flakes on Windows runners. + version: "0.11.15" - name: Install apt dependencies run: | diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 3fd17888c7..89fbec5420 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -48,7 +48,7 @@ jobs: with: python-version: "3.11" - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Set TAG run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbbb06c86c..53516db913 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,14 +6,6 @@ on: branches: [dev, beta, release] pull_request: - paths: - - "**" - - "!.github/workflows/*.yml" - - "!.github/actions/build-image/*" - - ".github/workflows/ci.yml" - - "!.yamllint" - - "!.github/dependabot.yml" - - "!docker/**" merge_group: permissions: @@ -60,6 +52,10 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true + # Pin uv version so the action does not have to fetch the + # manifest from raw.githubusercontent.com on every cache + # miss; that fetch flakes on Windows runners. + version: "0.11.15" - name: Create Python virtual environment if: steps.cache-venv.outputs.cache-hit != 'true' run: | @@ -97,6 +93,8 @@ jobs: runs-on: ubuntu-24.04 needs: - common + - determine-jobs + if: needs.determine-jobs.outputs.core-ci == 'true' steps: - name: Check out code from GitHub uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -175,6 +173,10 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true + # Pin uv version so the action does not have to fetch the + # manifest from raw.githubusercontent.com on every cache + # miss; that fetch flakes on Windows runners. + version: "0.11.15" - name: Install device-builder + esphome from PR # Install device-builder with its esphome + test extras # first so its pinned versions of pytest/etc. land, then @@ -215,6 +217,8 @@ jobs: runs-on: ${{ matrix.os }} needs: - common + - determine-jobs + if: needs.determine-jobs.outputs.core-ci == 'true' steps: - name: Check out code from GitHub uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -253,6 +257,7 @@ jobs: needs: - common outputs: + core-ci: ${{ steps.determine.outputs.core-ci }} integration-tests: ${{ steps.determine.outputs.integration-tests }} integration-test-buckets: ${{ steps.determine.outputs.integration-test-buckets }} clang-tidy: ${{ steps.determine.outputs.clang-tidy }} @@ -306,6 +311,7 @@ jobs: echo "$output" | jq # Extract individual fields + echo "core-ci=$(echo "$output" | jq -r '.core_ci')" >> $GITHUB_OUTPUT echo "integration-tests=$(echo "$output" | jq -r '.integration_tests')" >> $GITHUB_OUTPUT echo "integration-test-buckets=$(echo "$output" | jq -c '.integration_test_buckets')" >> $GITHUB_OUTPUT echo "clang-tidy=$(echo "$output" | jq -r '.clang_tidy')" >> $GITHUB_OUTPUT @@ -365,6 +371,10 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true + # Pin uv version so the action does not have to fetch the + # manifest from raw.githubusercontent.com on every cache + # miss; that fetch flakes on Windows runners. + version: "0.11.15" - name: Create Python virtual environment if: steps.cache-venv.outputs.cache-hit != 'true' run: | @@ -957,7 +967,8 @@ jobs: runs-on: ubuntu-latest needs: - common - if: github.event_name == 'pull_request' && !startsWith(github.base_ref, 'beta') && !startsWith(github.base_ref, 'release') + - determine-jobs + if: github.event_name == 'pull_request' && !startsWith(github.base_ref, 'beta') && !startsWith(github.base_ref, 'release') && needs.determine-jobs.outputs.core-ci == 'true' steps: - name: Check out code from GitHub uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index fbef0f5157..dfc0e08bfa 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,7 +56,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 + uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -84,6 +84,6 @@ jobs: exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 + uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9799f882db..344bd416c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -99,15 +99,15 @@ jobs: python-version: "3.11" - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Log in to docker hub - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to the GitHub container registry - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -178,17 +178,17 @@ jobs: merge-multiple: true - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Log in to docker hub if: matrix.registry == 'dockerhub' - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to the GitHub container registry if: matrix.registry == 'ghcr' - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 2e57093bbb..7003f6c482 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Stale - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 + uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 with: debug-only: ${{ github.ref != 'refs/heads/dev' }} # Dry-run when not run on dev branch remove-stale-when-updated: true diff --git a/.github/workflows/sync-device-classes.yml b/.github/workflows/sync-device-classes.yml index 23a63c5d8a..84be3c8e22 100644 --- a/.github/workflows/sync-device-classes.yml +++ b/.github/workflows/sync-device-classes.yml @@ -50,6 +50,10 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true + # Pin uv version so the action does not have to fetch the + # manifest from raw.githubusercontent.com on every cache + # miss; that fetch flakes on Windows runners. + version: "0.11.15" - name: Install Home Assistant run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da5fb94d5e..0470a948f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.15.12 + rev: v0.15.14 hooks: # Run the linter. - id: ruff diff --git a/esphome/__main__.py b/esphome/__main__.py index 16a05ad552..07bbd89358 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -2449,7 +2449,10 @@ def run_esphome(argv): # Skipped when -s overrides are passed, since the cache was written # against the previous substitution set. config: ConfigType | None = None - if args.command in ("upload", "logs") and not command_line_substitutions: + cache_eligible = ( + args.command in ("upload", "logs") and not command_line_substitutions + ) + if cache_eligible: from esphome.compiled_config import load_compiled_config config = load_compiled_config(conf_path) @@ -2464,6 +2467,16 @@ def run_esphome(argv): command_line_substitutions, skip_external_update=skip_external, ) + # Refresh the cache so the next upload/logs hits the fast path + # instead of re-running read_config. Skip when the storage + # sidecar is absent (no compile has run): the cache would + # never be loaded back, so writing secrets to disk is wasted. + if cache_eligible and config is not None: + from esphome.compiled_config import save_compiled_config + from esphome.storage_json import ext_storage_path + + if ext_storage_path(conf_path.name).exists(): + save_compiled_config(config) if config is None: return 2 CORE.config = config diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index b6f4aa2141..f2bf3752fa 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1,5 +1,6 @@ #include "api_connection.h" #ifdef USE_API +#include "api_connection_buffer.h" // for encode_to_buffer / get_batch_delay_ms_ inlines #ifdef USE_API_NOISE #include "api_frame_helper_noise.h" #endif diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index 4165b7f3a2..804cd9ddd1 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -11,7 +11,8 @@ #endif #include "api_pb2.h" #include "api_pb2_service.h" -#include "api_server.h" +#include "list_entities.h" +#include "subscribe_state.h" #include "esphome/core/application.h" #include "esphome/core/component.h" #ifdef USE_ESP32_CRASH_HANDLER @@ -36,6 +37,9 @@ class ComponentIterator; namespace esphome::api { +// Forward-declared to break the api_server.h cycle; full-type inlines are in api_connection_buffer.h. +class APIServer; + // Keepalive timeout in milliseconds static constexpr uint32_t KEEPALIVE_TIMEOUT_MS = 60000; // Maximum number of entities to process in a single batch during initial state/info sending @@ -411,44 +415,10 @@ class APIConnection final : public APIServerConnectionBase { // Non-template buffer management for send_message bool send_message_(uint32_t payload_size, uint8_t message_type, MessageEncodeFn encode_fn, const void *msg); - // Core batch encoding logic. Computes header size, checks fit, resizes buffer, encodes. - // ALWAYS_INLINE so the compiler can devirtualize encode_fn at hot call sites. - static inline uint16_t ESPHOME_ALWAYS_INLINE encode_to_buffer(uint32_t calculated_size, MessageEncodeFn encode_fn, - const void *msg, APIConnection *conn, - uint32_t remaining_size) { -#ifdef HAS_PROTO_MESSAGE_DUMP - if (conn->flags_.log_only_mode) { - auto *proto_msg = static_cast(msg); - DumpBuffer dump_buf; - conn->log_send_message_(proto_msg->message_name(), proto_msg->dump_to(dump_buf)); - return 1; - } -#endif - const uint8_t footer_size = conn->helper_->frame_footer_size(); - - // First message uses max padding (already in buffer), subsequent use exact header size - size_t to_add; - if (conn->flags_.batch_first_message) { - conn->flags_.batch_first_message = false; - conn->batch_header_size_ = conn->helper_->frame_header_padding(); - to_add = calculated_size; - } else { - conn->batch_header_size_ = conn->helper_->frame_header_size(calculated_size, conn->batch_message_type_); - to_add = calculated_size + conn->batch_header_size_ + footer_size; - } - - // Check if it fits (using actual header size, not max padding) - uint16_t total_calculated_size = calculated_size + conn->batch_header_size_ + footer_size; - if (total_calculated_size > remaining_size) - return 0; - - auto &shared_buf = conn->parent_->get_shared_buffer_ref(); - shared_buf.resize(shared_buf.size() + to_add); - ProtoWriteBuffer buffer{&shared_buf, shared_buf.size() - calculated_size}; - encode_fn(msg, buffer PROTO_ENCODE_DEBUG_INIT(&shared_buf)); - - return total_calculated_size; - } + // Core batch encoding logic. ALWAYS_INLINE so encode_fn devirtualizes at hot call sites. + // Defined in api_connection_buffer.h (needs APIServer complete). + static uint16_t ESPHOME_ALWAYS_INLINE encode_to_buffer(uint32_t calculated_size, MessageEncodeFn encode_fn, + const void *msg, APIConnection *conn, uint32_t remaining_size); // Noinline version of encode_to_buffer for cold paths (entity info, zero-payload messages). // All cold callers share this single copy instead of each getting an ALWAYS_INLINE expansion. @@ -792,7 +762,8 @@ class APIConnection final : public APIServerConnectionBase { // Read by process_batch_multi_ to pass into MessageInfo. uint8_t batch_header_size_{0}; - uint32_t get_batch_delay_ms_() const { return this->parent_->get_batch_delay(); } + // Defined in api_connection_buffer.h (needs APIServer complete). + uint32_t get_batch_delay_ms_() const; // Message will use 8 more bytes than the minimum size, and typical // MTU is 1500. Sometimes users will see as low as 1460 MTU. // If its IPv6 the header is 40 bytes, and if its IPv4 diff --git a/esphome/components/api/api_connection_buffer.h b/esphome/components/api/api_connection_buffer.h new file mode 100644 index 0000000000..1dd8a162e4 --- /dev/null +++ b/esphome/components/api/api_connection_buffer.h @@ -0,0 +1,54 @@ +#pragma once + +#include "esphome/core/defines.h" +#ifdef USE_API + +// Inline APIConnection methods that need APIServer complete. Include this +// instead of api_connection.h when calling encode_to_buffer or get_batch_delay_ms_. + +#include "api_connection.h" +#include "api_server.h" + +namespace esphome::api { + +inline uint16_t ESPHOME_ALWAYS_INLINE APIConnection::encode_to_buffer(uint32_t calculated_size, + MessageEncodeFn encode_fn, const void *msg, + APIConnection *conn, uint32_t remaining_size) { +#ifdef HAS_PROTO_MESSAGE_DUMP + if (conn->flags_.log_only_mode) { + auto *proto_msg = static_cast(msg); + DumpBuffer dump_buf; + conn->log_send_message_(proto_msg->message_name(), proto_msg->dump_to(dump_buf)); + return 1; + } +#endif + const uint8_t footer_size = conn->helper_->frame_footer_size(); + + // First message uses max padding (already in buffer), subsequent use exact header size + size_t to_add; + if (conn->flags_.batch_first_message) { + conn->flags_.batch_first_message = false; + conn->batch_header_size_ = conn->helper_->frame_header_padding(); + to_add = calculated_size; + } else { + conn->batch_header_size_ = conn->helper_->frame_header_size(calculated_size, conn->batch_message_type_); + to_add = calculated_size + conn->batch_header_size_ + footer_size; + } + + // Check if it fits (using actual header size, not max padding) + uint16_t total_calculated_size = calculated_size + conn->batch_header_size_ + footer_size; + if (total_calculated_size > remaining_size) + return 0; + + auto &shared_buf = conn->parent_->get_shared_buffer_ref(); + shared_buf.resize(shared_buf.size() + to_add); + ProtoWriteBuffer buffer{&shared_buf, shared_buf.size() - calculated_size}; + encode_fn(msg, buffer PROTO_ENCODE_DEBUG_INIT(&shared_buf)); + + return total_calculated_size; +} + +inline uint32_t APIConnection::get_batch_delay_ms_() const { return this->parent_->get_batch_delay(); } + +} // namespace esphome::api +#endif diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 6c26c4e187..c30bd2e612 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -30,11 +30,6 @@ APIServer *global_api_server = nullptr; // NOLINT(cppcoreguidelines-avoid-non-c APIServer::APIServer() { global_api_server = this; } -// Custom deleter defined here so `delete` sees the complete APIConnection type. -// This prevents libc++ from emitting an "incomplete type" error when other -// translation units only have the forward declaration of APIConnection. -void APIServer::APIConnectionDeleter::operator()(APIConnection *p) const { delete p; } - void APIServer::socket_failed_(const LogString *msg) { ESP_LOGW(TAG, "Socket %s: errno %d", LOG_STR_ARG(msg), errno); this->destroy_socket_(); diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 6b575e536d..fbc8115091 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -3,6 +3,8 @@ #include "esphome/core/defines.h" #ifdef USE_API #include "api_buffer.h" +// Must precede clients_ so APIConnection is complete for default_delete (libc++). +#include "api_connection.h" #include "api_noise_context.h" #include "api_pb2.h" #include "api_pb2_service.h" @@ -12,8 +14,6 @@ #include "esphome/core/controller.h" #include "esphome/core/log.h" #include "esphome/core/string_ref.h" -#include "list_entities.h" -#include "subscribe_state.h" #ifdef USE_LOGGER #include "esphome/components/logger/logger.h" #endif @@ -191,15 +191,9 @@ class APIServer final : public Component, bool is_connected_with_state_subscription() const; // Range-for view over the populated slice [0, api_connection_count_). Read-only with respect - // to ownership — callers get `const unique_ptr&` so they can invoke non-const methods on the + // to ownership; callers get `const unique_ptr&` so they can invoke non-const methods on the // APIConnection but cannot reset/move the slot and break the count invariant. - // Custom deleter is defined out-of-line in api_server.cpp so libc++ does not - // eagerly instantiate `delete static_cast(p)` here, where - // only the forward declaration of APIConnection is visible (incomplete type). - struct APIConnectionDeleter { - void operator()(APIConnection *p) const; - }; - using APIConnectionPtr = std::unique_ptr; + using APIConnectionPtr = std::unique_ptr; class ActiveClientsView { const APIConnectionPtr *begin_; const APIConnectionPtr *end_; diff --git a/esphome/components/audio/__init__.py b/esphome/components/audio/__init__.py index 13b379ba3a..c9775ab601 100644 --- a/esphome/components/audio/__init__.py +++ b/esphome/components/audio/__init__.py @@ -335,7 +335,7 @@ async def to_code(config): add_idf_component( name="esphome/esp-audio-libs", - ref="3.0.0", + ref="3.1.0", ) data = _get_data() diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index c3461f9c51..ca30aab943 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -1,5 +1,6 @@ #include "bluetooth_proxy.h" +#include "esphome/components/api/api_server.h" #include "esphome/core/log.h" #include "esphome/core/macros.h" #include "esphome/core/application.h" diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index e9b0f1fd0a..24312d64ad 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -113,6 +113,7 @@ ARDUINO_FRAMEWORK_NAME = "framework-arduinoespressif32" ARDUINO_FRAMEWORK_PKG = f"pioarduino/{ARDUINO_FRAMEWORK_NAME}" ARDUINO_LIBS_NAME = f"{ARDUINO_FRAMEWORK_NAME}-libs" ARDUINO_LIBS_PKG = f"pioarduino/{ARDUINO_LIBS_NAME}" +ARDUINO_ESP32_COMPONENT_NAME = "espressif/arduino-esp32" LOG_LEVELS_IDF = [ "NONE", @@ -792,19 +793,15 @@ PLATFORM_VERSION_LOOKUP = { } -def _check_pio_versions(config): - config = config.copy() - value = config[CONF_FRAMEWORK] +def _resolve_framework_version(value: ConfigType) -> cv.Version: + """Resolve a named or raw framework version and validate the minimum. + Normalises value[CONF_VERSION] to its string form and returns the parsed + cv.Version. Shared between the PIO and esp-idf toolchain paths; toolchain- + specific concerns (source defaults, platform_version) live in the per- + toolchain functions. + """ if value[CONF_VERSION] in PLATFORM_VERSION_LOOKUP: - if CONF_SOURCE in value or CONF_PLATFORM_VERSION in value: - raise cv.Invalid( - "Version needs to be explicitly set when a custom source or platform_version is used." - ) - - platform_lookup = PLATFORM_VERSION_LOOKUP[value[CONF_VERSION]] - value[CONF_PLATFORM_VERSION] = _parse_pio_platform_version(str(platform_lookup)) - if value[CONF_TYPE] == FRAMEWORK_ARDUINO: version = ARDUINO_FRAMEWORK_VERSION_LOOKUP[value[CONF_VERSION]] else: @@ -817,7 +814,38 @@ def _check_pio_versions(config): if value[CONF_TYPE] == FRAMEWORK_ARDUINO: if version < cv.Version(3, 0, 0): raise cv.Invalid("Only Arduino 3.0+ is supported.") - recommended_version = ARDUINO_FRAMEWORK_VERSION_LOOKUP["recommended"] + recommended = ARDUINO_FRAMEWORK_VERSION_LOOKUP["recommended"] + else: + if version < cv.Version(5, 0, 0): + raise cv.Invalid("Only ESP-IDF 5.0+ is supported.") + recommended = ESP_IDF_FRAMEWORK_VERSION_LOOKUP["recommended"] + + if version != recommended: + _LOGGER.warning( + "The selected framework version is not the recommended one. " + "If there are connectivity or build issues please remove the manual version." + ) + + return version + + +def _check_pio_versions(config: ConfigType) -> ConfigType: + config = config.copy() + value = config[CONF_FRAMEWORK] + + is_named_version = value[CONF_VERSION] in PLATFORM_VERSION_LOOKUP + if is_named_version and (CONF_SOURCE in value or CONF_PLATFORM_VERSION in value): + raise cv.Invalid( + "Version needs to be explicitly set when a custom source or platform_version is used." + ) + if is_named_version: + value[CONF_PLATFORM_VERSION] = _parse_pio_platform_version( + str(PLATFORM_VERSION_LOOKUP[value[CONF_VERSION]]) + ) + + version = _resolve_framework_version(value) + + if value[CONF_TYPE] == FRAMEWORK_ARDUINO: platform_lookup = ARDUINO_PLATFORM_VERSION_LOOKUP.get(version) value[CONF_SOURCE] = value.get( CONF_SOURCE, _format_framework_arduino_version(version) @@ -825,9 +853,6 @@ def _check_pio_versions(config): if _is_framework_url(value[CONF_SOURCE]): value[CONF_SOURCE] = f"{ARDUINO_FRAMEWORK_PKG}@{value[CONF_SOURCE]}" else: - if version < cv.Version(5, 0, 0): - raise cv.Invalid("Only ESP-IDF 5.0+ is supported.") - recommended_version = ESP_IDF_FRAMEWORK_VERSION_LOOKUP["recommended"] platform_lookup = ESP_IDF_PLATFORM_VERSION_LOOKUP.get(version) value[CONF_SOURCE] = value.get( CONF_SOURCE, @@ -843,12 +868,6 @@ def _check_pio_versions(config): ) value[CONF_PLATFORM_VERSION] = _parse_pio_platform_version(str(platform_lookup)) - if version != recommended_version: - _LOGGER.warning( - "The selected framework version is not the recommended one. " - "If there are connectivity or build issues please remove the manual version." - ) - if value[CONF_PLATFORM_VERSION] != _parse_pio_platform_version( str(PLATFORM_VERSION_LOOKUP["recommended"]) ): @@ -860,19 +879,26 @@ def _check_pio_versions(config): return config -def _check_esp_idf_versions(config): - config = _check_pio_versions(config) +def _check_esp_idf_versions(config: ConfigType) -> ConfigType: + config = config.copy() value = config[CONF_FRAMEWORK] - # Remove unwanted keys if present - for key in (CONF_SOURCE, CONF_PLATFORM_VERSION): - value.pop(key, None) + # platform_version is a PlatformIO concept; drop it if a user carried it + # over from a PIO-style config. CONF_SOURCE, on the other hand, is kept: + # it lets a user override the framework tarball URL under the esp-idf + # toolchain (the espidf framework downloader consults it). + value.pop(CONF_PLATFORM_VERSION, None) - # Official ESP-IDF frameworks don't use extra - version = cv.Version.parse(value[CONF_VERSION]) - version = cv.Version(version.major, version.minor, version.patch) + version = _resolve_framework_version(value) - value[CONF_VERSION] = str(version) + if CONF_SOURCE in value: + _LOGGER.warning( + "A custom framework source is set. " + "If there are connectivity or build issues please remove the manual source." + ) + + # Official ESP-IDF frameworks don't use the 'extra' semver component. + value[CONF_VERSION] = str(cv.Version(version.major, version.minor, version.patch)) return config @@ -1718,6 +1744,31 @@ async def _add_yaml_idf_components(components: list[ConfigType]): ) +@coroutine_with_priority(CoroPriority.FINAL - 1) +async def _finalize_arduino_aware_flags(): + """Build flags that depend on whether arduino-esp32 is linked in. + + Scheduler runs lower priority values later, so ``FINAL - 1`` fires + after every ``FINAL`` job (incl. ``_add_yaml_idf_components``) -- + by then ``KEY_COMPONENTS`` is fully populated. + + - Skip our esp_panic_handler wrap when Arduino is linked; Arduino + wraps the same symbol and the linker errors on the duplicate. + - Define USE_ARDUINO in the hybrid esp-idf+arduino-esp32-component + case so ESPHome's ``#ifdef USE_ARDUINO`` paths light up. The + framework=arduino branch already adds it inline in to_code. + """ + arduino_linked = ( + CORE.using_arduino + or ARDUINO_ESP32_COMPONENT_NAME in CORE.data[KEY_ESP32][KEY_COMPONENTS] + ) + if not arduino_linked: + cg.add_build_flag("-Wl,--wrap=esp_panic_handler") + cg.add_define("USE_ESP32_CRASH_HANDLER") + elif not CORE.using_arduino: + cg.add_build_flag("-DUSE_ARDUINO") + + async def to_code(config): framework_ver: cv.Version = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] conf = config[CONF_FRAMEWORK] @@ -1770,6 +1821,7 @@ async def to_code(config): cg.add_build_flag("-Wno-error=overloaded-virtual") cg.add_build_flag("-Wno-error=reorder") cg.add_build_flag("-Wno-error=volatile") + cg.add_build_flag("-Wno-error=cpp") # -Wno- (not -Wno-error=): suppress entirely, too noisy on C++ aggregates cg.add_build_flag("-Wno-missing-field-initializers") @@ -1777,11 +1829,8 @@ async def to_code(config): cg.add_build_flag("-DUSE_ESP32") cg.add_define("USE_NATIVE_64BIT_TIME") cg.add_build_flag("-Wl,-z,noexecstack") - # Arduino already wraps esp_panic_handler for its own backtrace handler, - # so only add our wrap when using ESP-IDF framework to avoid linker conflicts. - if conf[CONF_TYPE] == FRAMEWORK_ESP_IDF: - cg.add_build_flag("-Wl,--wrap=esp_panic_handler") - cg.add_define("USE_ESP32_CRASH_HANDLER") + # Deferred so KEY_COMPONENTS is fully populated -- see the coroutine. + CORE.add_job(_finalize_arduino_aware_flags) cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) variant = config[CONF_VARIANT] cg.add_build_flag(f"-DUSE_ESP32_VARIANT_{variant}") @@ -1962,7 +2011,7 @@ async def to_code(config): add_idf_sdkconfig_option("CONFIG_HEAP_PLACE_FUNCTION_INTO_FLASH", True) # Setup watchdog - add_idf_sdkconfig_option("CONFIG_ESP_TASK_WDT", True) + add_idf_sdkconfig_option("CONFIG_ESP_TASK_WDT_INIT", True) add_idf_sdkconfig_option("CONFIG_ESP_TASK_WDT_PANIC", True) add_idf_sdkconfig_option("CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0", False) add_idf_sdkconfig_option("CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU1", False) @@ -2004,7 +2053,8 @@ async def to_code(config): if not advanced[CONF_ENABLE_LWIP_MDNS_QUERIES]: add_idf_sdkconfig_option("CONFIG_LWIP_DNS_SUPPORT_MDNS_QUERIES", False) if not advanced[CONF_ENABLE_LWIP_BRIDGE_INTERFACE]: - add_idf_sdkconfig_option("CONFIG_LWIP_BRIDGEIF_MAX_PORTS", 0) + # Kconfig range is [1,63]; 0 gets clamped to the default. + add_idf_sdkconfig_option("CONFIG_LWIP_BRIDGEIF_MAX_PORTS", 1) _configure_lwip_max_sockets(conf) @@ -2096,7 +2146,6 @@ async def to_code(config): for key, flag in ASSERTION_LEVELS.items(): add_idf_sdkconfig_option(flag, assertion_level == key) - add_idf_sdkconfig_option("CONFIG_COMPILER_OPTIMIZATION_DEFAULT", False) compiler_optimization = advanced[CONF_COMPILER_OPTIMIZATION] for key, flag in COMPILER_OPTIMIZATIONS.items(): add_idf_sdkconfig_option(flag, compiler_optimization == key) @@ -2251,7 +2300,8 @@ async def to_code(config): add_idf_sdkconfig_option("CONFIG_FATFS_VOLUME_COUNT", 2) elif advanced[CONF_DISABLE_FATFS]: add_idf_sdkconfig_option("CONFIG_FATFS_LFN_NONE", True) - add_idf_sdkconfig_option("CONFIG_FATFS_VOLUME_COUNT", 0) + # Kconfig range is [1,10]; 0 gets clamped to the default. + add_idf_sdkconfig_option("CONFIG_FATFS_VOLUME_COUNT", 1) for name, value in conf[CONF_SDKCONFIG_OPTIONS].items(): add_idf_sdkconfig_option(name, RawSdkconfigValue(value)) @@ -2488,9 +2538,8 @@ def _write_sdkconfig(): def _platformio_library_to_dependency(library: Library) -> tuple[str, dict[str, str]]: dependency: dict[str, str] = {} - name, version, path = generate_idf_component(library) + name, _version, path = generate_idf_component(library) dependency["override_path"] = str(path) - dependency["version"] = version return name, dependency @@ -2542,7 +2591,7 @@ def _write_idf_component_yml(): if CORE.using_toolchain_esp_idf: add_idf_component( - name="espressif/arduino-esp32", + name=ARDUINO_ESP32_COMPONENT_NAME, ref=str(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]), ) diff --git a/esphome/components/esp8266/hal.cpp b/esphome/components/esp8266/hal.cpp index e8f472dc8a..3501c51859 100644 --- a/esphome/components/esp8266/hal.cpp +++ b/esphome/components/esp8266/hal.cpp @@ -5,6 +5,7 @@ #include #include +#include extern "C" { #include @@ -71,23 +72,22 @@ uint32_t IRAM_ATTR HOT millis() { return result; } -// Poll-based delay that avoids ::delay() — Arduino's __delay has an intra-object -// call to the original millis() that --wrap can't intercept, so calling ::delay() -// would keep the slow Arduino millis body alive in IRAM. optimistic_yield still -// enters esp_schedule()/esp_suspend_within_cont() via yield(), so SDK tasks and -// WiFi run correctly. Theoretically less power-efficient than Arduino's -// os_timer-based delay() for long waits, but nearly all ESPHome delays are short -// (sensor/I²C/SPI settling in the 1–100 ms range) where the difference is -// negligible. +// Delegate to Arduino's 1-arg esp_delay(), which uses os_timer + esp_suspend to +// suspend the cont task for `ms` milliseconds without polling millis(). This +// matches pre-2026.5.0 behavior (when esphome::delay() forwarded to ::delay()) +// and lets the SDK run freely while we wait, which timing-sensitive +// interrupt-driven code (e.g. ESP8266 software-serial RX in components like +// fingerprint_grow) depends on. The poll-based busy-wait that this replaced +// rarely yielded inside short waits like delay(1), starving WiFi/SDK tasks and +// extending interrupt latency. Unlike ::delay(), esp_delay()'s 1-arg form does +// not call millis(), so the slow Arduino millis() body is not pulled into IRAM +// by this path (the --wrap=millis goal of #15662 is preserved). void HOT delay(uint32_t ms) { if (ms == 0) { optimistic_yield(1000); return; } - uint32_t start = millis(); - while (millis() - start < ms) { - optimistic_yield(1000); - } + esp_delay(ms); } void arch_restart() { diff --git a/esphome/components/homeassistant/sensor/homeassistant_sensor.cpp b/esphome/components/homeassistant/sensor/homeassistant_sensor.cpp index 112795a4ff..b79a56953a 100644 --- a/esphome/components/homeassistant/sensor/homeassistant_sensor.cpp +++ b/esphome/components/homeassistant/sensor/homeassistant_sensor.cpp @@ -17,9 +17,9 @@ void HomeassistantSensor::setup() { } if (this->attribute_ != nullptr) { - ESP_LOGD(TAG, "'%s::%s': Got attribute state %.2f", this->entity_id_, this->attribute_, *val); + ESP_LOGV(TAG, "'%s::%s': Got attribute state %.2f", this->entity_id_, this->attribute_, *val); } else { - ESP_LOGD(TAG, "'%s': Got state %.2f", this->entity_id_, *val); + ESP_LOGV(TAG, "'%s': Got state %.2f", this->entity_id_, *val); } this->publish_state(*val); }); diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 680ca069c0..691f68e912 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -2,6 +2,7 @@ #ifdef USE_ESP32 +#include #include #include "esphome/components/audio/audio.h" @@ -299,6 +300,15 @@ void I2SAudioSpeakerBase::stop_i2s_driver_() { i2s_channel_disable(this->tx_handle_); i2s_del_channel(this->tx_handle_); this->tx_handle_ = nullptr; + + // i2s_del_channel() leaves dout wired to this port's data-out signal in the GPIO matrix: it only + // clears an internal reservation mask, never the esp_rom_gpio_connect_out_signal() routing that + // setup installed. If another speaker reuses this port (shared bus), its audio still reaches our + // dout. Detach the pin and drive it low so a stale output stops driving downstream hardware: a + // SPDIF optical transmitter would otherwise stay lit, and an analog DAC would emit noise. + gpio_reset_pin(this->dout_pin_); + gpio_set_direction(this->dout_pin_, GPIO_MODE_OUTPUT); + gpio_set_level(this->dout_pin_, 0); } this->parent_->unlock(); } diff --git a/esphome/components/libretiny/patch_linker.py.script b/esphome/components/libretiny/patch_linker.py.script index 282a31d3f2..3a8a4787ed 100644 --- a/esphome/components/libretiny/patch_linker.py.script +++ b/esphome/components/libretiny/patch_linker.py.script @@ -13,7 +13,9 @@ import subprocess # - RTL8710B: hal.h uses section(".image2.ram.text"); stock linker consumes it. # - RTL8720C: hal.h uses section(".sram.text"); stock linker consumes it. # - LN882H: stock linker has no glob for ".sram.text", so we inject -# KEEP(*(.sram.text*)) into ".flash_copysection" (> RAM0 AT> FLASH). +# KEEP(*(.sram.text*)) into ".flash_copysection" (> RAM0 AT> FLASH) +# immediately after KEEP(*(.vectors)), so the vector table stays at +# __copysection_ram0_start (0x20000000) for correct Cortex-M4 VTOR alignment. # # All families also get a post-link summary showing where IRAM_ATTR landed. @@ -27,7 +29,11 @@ _KEEP_LINE = ( "__esphome_sram_text_end = .; " + _MARKER + "\n" ) -_LN_COPY = re.compile(r"(\.flash_copysection\s*:\s*\{\s*\n)") +# Inject after KEEP(*(.vectors)) so the vector table stays at +# __copysection_ram0_start (0x20000000). Cortex-M4 VTOR requires a 512-byte- +# aligned address; injecting before the vectors would push them to an +# unaligned offset and mis-route every IRQ handler. +_LN_COPY = re.compile(r"(KEEP\(\*\(\.vectors\)\)[^\n]*\n)") def _detect(env): @@ -56,7 +62,7 @@ KNOWN_VARIANTS = frozenset({ def _inject_keep(host_section): - """Return a patcher that injects _KEEP_LINE at the top of `host_section`.""" + """Return a patcher that injects _KEEP_LINE after `host_section` match.""" def patch(content): if _MARKER in content: return content diff --git a/esphome/components/light/__init__.py b/esphome/components/light/__init__.py index 9540c64486..68d9f85af2 100644 --- a/esphome/components/light/__init__.py +++ b/esphome/components/light/__init__.py @@ -86,10 +86,22 @@ class EffectRef: component_path: list[str | int] # path_context when the action was validated +@dataclass +class EffectCycleRef: + """A pending light.effect.next/previous action to validate. + + Records that the referenced light needs at least one effect configured. + """ + + light_id: ID + component_path: list[str | int] + + @dataclass class LightData: gamma_tables: dict = field(default_factory=dict) # gamma_value -> fwd_arr effect_refs: list[EffectRef] = field(default_factory=list) + effect_cycle_refs: list[EffectCycleRef] = field(default_factory=list) def _get_data() -> LightData: @@ -160,13 +172,15 @@ def _final_validate(config: ConfigType) -> ConfigType: this never runs — but the ID validator will catch the missing light ID separately. """ data = _get_data() - if not data.effect_refs: + if not data.effect_refs and not data.effect_cycle_refs: return config - # Drain the list so we only validate once even though + # Drain the lists so we only validate once even though # FINAL_VALIDATE_SCHEMA runs for each light platform instance. refs = data.effect_refs data.effect_refs = [] + cycle_refs = data.effect_cycle_refs + data.effect_cycle_refs = [] fconf = fv.full_config.get() @@ -188,6 +202,21 @@ def _final_validate(config: ConfigType) -> ConfigType: path=[cv.ROOT_CONFIG_PATH] + ref.component_path, ) + for ref in cycle_refs: + try: + light_path = fconf.get_path_for_id(ref.light_id)[:-1] + light_config = fconf.get_config_for_path(light_path) + except KeyError: + continue + + if not light_config.get(CONF_EFFECTS): + raise cv.FinalExternalInvalid( + f"Light '{ref.light_id}' has no effects configured, but a " + f"'light.effect.next' or 'light.effect.previous' action " + f"references it. Add at least one effect to the light.", + path=[cv.ROOT_CONFIG_PATH] + ref.component_path, + ) + return config diff --git a/esphome/components/light/automation.h b/esphome/components/light/automation.h index 993d4a2ea6..260414f033 100644 --- a/esphome/components/light/automation.h +++ b/esphome/components/light/automation.h @@ -104,6 +104,47 @@ template class DimRelativeAction : pub transition_length_{}; }; +// Cycle through the light's configured effects. `Forward` selects direction +// at compile time so the chosen branch is the only one that gets instantiated +// per action site. `include_none` is runtime so a single set of templates +// covers both the "wrap through None" and "skip None" variants. +template class LightEffectCycleAction : public Action { + public: + explicit LightEffectCycleAction(LightState *parent) : parent_(parent) {} + + void set_include_none(bool include_none) { this->include_none_ = include_none; } + + void play(const Ts &...) override { + size_t count = this->parent_->get_effect_count(); + if (count == 0) { + return; + } + uint32_t current = this->parent_->get_current_effect_index(); + uint32_t next; + if (this->include_none_) { + uint32_t total = static_cast(count) + 1; + if constexpr (Forward) { + next = (current + 1) % total; + } else { + next = (current + total - 1) % total; + } + } else { + if constexpr (Forward) { + next = (current % static_cast(count)) + 1; + } else { + next = (current <= 1) ? static_cast(count) : current - 1; + } + } + auto call = this->parent_->turn_on(); + call.set_effect(next); + call.perform(); + } + + protected: + LightState *parent_; + bool include_none_{false}; +}; + template class LightIsOnCondition : public Condition { public: explicit LightIsOnCondition(LightState *state) : state_(state) {} diff --git a/esphome/components/light/automation.py b/esphome/components/light/automation.py index cef774af38..7eaba9b117 100644 --- a/esphome/components/light/automation.py +++ b/esphome/components/light/automation.py @@ -26,8 +26,8 @@ from esphome.const import ( CONF_WARM_WHITE, CONF_WHITE, ) -from esphome.core import CORE, EsphomeError, Lambda -from esphome.cpp_generator import LambdaExpression +from esphome.core import CORE, ID, EsphomeError, Lambda +from esphome.cpp_generator import LambdaExpression, MockObj, TemplateArgsType from esphome.types import ConfigType from .types import ( @@ -39,12 +39,15 @@ from .types import ( DimRelativeAction, LightCall, LightControlAction, + LightEffectCycleAction, LightIsOffCondition, LightIsOnCondition, LightState, ToggleAction, ) +CONF_INCLUDE_NONE = "include_none" + @automation.register_action( "light.toggle", @@ -253,6 +256,75 @@ async def light_control_to_code(config, action_id, template_arg, args): return cg.new_Pvariable(action_id, template_arg, paren, apply_lambda) +def _record_effect_cycle_ref(config: ConfigType) -> ConfigType: + """Record a cycle-action reference for later validation against the target light.""" + from . import EffectCycleRef, _get_data + + _get_data().effect_cycle_refs.append( + EffectCycleRef( + light_id=config[CONF_ID], + component_path=path_context.get(), + ) + ) + return config + + +LIGHT_EFFECT_CYCLE_ACTION_BASE_SCHEMA = cv.Schema( + { + cv.Required(CONF_ID): cv.use_id(LightState), + cv.Optional(CONF_INCLUDE_NONE, default=False): cv.boolean, + } +) +LIGHT_EFFECT_CYCLE_ACTION_BASE_SCHEMA.add_extra(_record_effect_cycle_ref) + +LIGHT_EFFECT_CYCLE_ACTION_SCHEMA = automation.maybe_simple_id( + LIGHT_EFFECT_CYCLE_ACTION_BASE_SCHEMA +) + + +@automation.register_action( + "light.effect.next", + LightEffectCycleAction, + LIGHT_EFFECT_CYCLE_ACTION_SCHEMA, + synchronous=True, +) +async def light_effect_next_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + return await _light_effect_cycle_to_code(config, action_id, template_arg, True) + + +@automation.register_action( + "light.effect.previous", + LightEffectCycleAction, + LIGHT_EFFECT_CYCLE_ACTION_SCHEMA, + synchronous=True, +) +async def light_effect_previous_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + return await _light_effect_cycle_to_code(config, action_id, template_arg, False) + + +async def _light_effect_cycle_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + forward: bool, +) -> MockObj: + paren = await cg.get_variable(config[CONF_ID]) + cycle_template_arg = cg.TemplateArguments(forward, *template_arg) + var = cg.new_Pvariable(action_id, cycle_template_arg, paren) + cg.add(var.set_include_none(config[CONF_INCLUDE_NONE])) + return var + + CONF_RELATIVE_BRIGHTNESS = "relative_brightness" LIGHT_DIM_RELATIVE_ACTION_SCHEMA = cv.Schema( { diff --git a/esphome/components/light/types.py b/esphome/components/light/types.py index 534dcd2194..c7385cbee3 100644 --- a/esphome/components/light/types.py +++ b/esphome/components/light/types.py @@ -39,6 +39,7 @@ LIMIT_MODES = { # Actions ToggleAction = light_ns.class_("ToggleAction", automation.Action) LightControlAction = light_ns.class_("LightControlAction", automation.Action) +LightEffectCycleAction = light_ns.class_("LightEffectCycleAction", automation.Action) DimRelativeAction = light_ns.class_("DimRelativeAction", automation.Action) AddressableSet = light_ns.class_("AddressableSet", automation.Action) LightIsOnCondition = light_ns.class_("LightIsOnCondition", automation.Condition) diff --git a/esphome/components/lvgl/schemas.py b/esphome/components/lvgl/schemas.py index 553e0f7398..58ef88d6a8 100644 --- a/esphome/components/lvgl/schemas.py +++ b/esphome/components/lvgl/schemas.py @@ -1,4 +1,5 @@ from collections.abc import Callable +from typing import Any from esphome import config_validation as cv from esphome.automation import Trigger, validate_automation @@ -534,7 +535,16 @@ def strip_defaults(schema: cv.Schema): return cv.Schema({cv.Optional(k): v for k, v in schema.schema.items()}) -def container_schema(widget_type: WidgetType, extras=None): +# Keyed by (id(widget_type), id(extras)); strong refs in the value keep both +# alive so id() can't be recycled. +_CONTAINER_SCHEMA_CACHE: dict[ + tuple[int, int], tuple[Any, Any, Callable[[Any], Any]] +] = {} + + +def container_schema( + widget_type: WidgetType, extras: Any = None +) -> Callable[[Any], Any]: """ Create a schema for a container widget of a given type. All obj properties are available, plus the extras passed in, plus any defined for the specific widget being specified. @@ -542,19 +552,31 @@ def container_schema(widget_type: WidgetType, extras=None): :param extras: Additional options to be made available, e.g. layout properties for children :return: The schema for this type of widget. """ - schema = obj_schema(widget_type).extend( - {cv.GenerateID(): cv.declare_id(widget_type.w_type)} - ) - if extras: - schema = schema.extend(extras) - # Delayed evaluation for recursion + cache_key = (id(widget_type), id(extras)) + cached = _CONTAINER_SCHEMA_CACHE.get(cache_key) + if cached is not None: + cached_widget_type, cached_extras, cached_validator = cached + if cached_widget_type is widget_type and cached_extras is extras: + return cached_validator - schema = schema.extend(widget_type.schema) + cached_schema: cv.Schema | None = None - def validator(value): + def get_schema() -> cv.Schema: + nonlocal cached_schema + if cached_schema is None: + schema = obj_schema(widget_type).extend( + {cv.GenerateID(): cv.declare_id(widget_type.w_type)} + ) + if extras: + schema = schema.extend(extras) + cached_schema = schema.extend(widget_type.schema) + return cached_schema + + def validator(value: Any) -> Any: value = value or {} - return append_layout_schema(schema, value)(value) + return append_layout_schema(get_schema(), value)(value) + _CONTAINER_SCHEMA_CACHE[cache_key] = (widget_type, extras, validator) return validator diff --git a/esphome/components/lvgl/widgets/__init__.py b/esphome/components/lvgl/widgets/__init__.py index ab1c61ff88..400f7c709b 100644 --- a/esphome/components/lvgl/widgets/__init__.py +++ b/esphome/components/lvgl/widgets/__init__.py @@ -1,4 +1,6 @@ +from collections.abc import Callable import sys +from typing import Any from esphome import codegen as cg, config_validation as cv from esphome.automation import register_action @@ -15,6 +17,7 @@ from esphome.const import ( from esphome.core import ID, EsphomeError, TimePeriod from esphome.coroutine import FakeAwaitable from esphome.cpp_generator import MockObj +from esphome.schema_extractors import EnableSchemaExtraction from esphome.types import Expression from ..defines import ( @@ -73,6 +76,34 @@ from ..types import ( EVENT_LAMB = "event_lamb__" +def _build_update_schema(widget_type: "WidgetType") -> Schema: + # Local import: ..schemas imports WidgetType from this module. + from ..schemas import base_update_schema + + return base_update_schema(widget_type, widget_type.parts).extend( + widget_type.modify_schema + ) + + +def _update_action_schema( + widget_type: "WidgetType", +) -> Schema | Callable[[Any], Any]: + # Eager when extracting so build_language_schema.py sees the mapping; + # lazy otherwise to skip ~200 ms of import-time voluptuous work. + if EnableSchemaExtraction: + return _build_update_schema(widget_type) + + cached: Schema | None = None + + def validator(value: Any) -> Any: + nonlocal cached + if cached is None: + cached = _build_update_schema(widget_type) + return cached(value) + + return validator + + class WidgetType: """ Describes a type of Widget, e.g. "bar" or "line" @@ -113,18 +144,17 @@ class WidgetType: # Local import to avoid circular import from ..automation import update_to_code - from ..schemas import WIDGET_TYPES, base_update_schema + from ..schemas import WIDGET_TYPES if not is_mock: if self.name in WIDGET_TYPES: raise EsphomeError(f"Duplicate definition of widget type '{self.name}'") WIDGET_TYPES[self.name] = self - # Register the update action automatically, adding widget-specific properties register_action( f"lvgl.{self.name}.update", ObjUpdateAction, - base_update_schema(self, self.parts).extend(self.modify_schema), + _update_action_schema(self), synchronous=True, )(update_to_code) diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 36f13f7d07..b670bd3c4d 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -206,7 +206,7 @@ async def to_code(config: ConfigType) -> None: ) # sendspin-cpp library - esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.6.0") + esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.6.1") cg.add_define("USE_SENDSPIN", True) # for MDNS diff --git a/esphome/components/sx126x/sx126x.cpp b/esphome/components/sx126x/sx126x.cpp index 6e6857fadb..83afeac50a 100644 --- a/esphome/components/sx126x/sx126x.cpp +++ b/esphome/components/sx126x/sx126x.cpp @@ -30,8 +30,8 @@ static constexpr uint8_t OCP_140MA = 0x38; // 140 mA max current static constexpr float LOW_DATA_RATE_OPTIMIZE_THRESHOLD = 16.38f; // 16.38 ms uint8_t SX126x::read_fifo_(uint8_t offset, std::vector &packet) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->transfer_byte(RADIO_READ_BUFFER); this->transfer_byte(offset); uint8_t status = this->transfer_byte(0x00); @@ -43,8 +43,8 @@ uint8_t SX126x::read_fifo_(uint8_t offset, std::vector &packet) { } void SX126x::write_fifo_(uint8_t offset, const std::vector &packet) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->transfer_byte(RADIO_WRITE_BUFFER); this->transfer_byte(offset); for (const uint8_t &byte : packet) { @@ -55,8 +55,8 @@ void SX126x::write_fifo_(uint8_t offset, const std::vector &packet) { } uint8_t SX126x::read_opcode_(uint8_t opcode, uint8_t *data, uint8_t size) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->transfer_byte(opcode); uint8_t status = this->transfer_byte(0x00); for (int32_t i = 0; i < size; i++) { @@ -67,8 +67,8 @@ uint8_t SX126x::read_opcode_(uint8_t opcode, uint8_t *data, uint8_t size) { } void SX126x::write_opcode_(uint8_t opcode, uint8_t *data, uint8_t size) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->transfer_byte(opcode); for (int32_t i = 0; i < size; i++) { this->transfer_byte(data[i]); @@ -78,8 +78,8 @@ void SX126x::write_opcode_(uint8_t opcode, uint8_t *data, uint8_t size) { } void SX126x::read_register_(uint16_t reg, uint8_t *data, uint8_t size) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->write_byte(RADIO_READ_REGISTER); this->write_byte((reg >> 8) & 0xFF); this->write_byte((reg >> 0) & 0xFF); @@ -91,8 +91,8 @@ void SX126x::read_register_(uint16_t reg, uint8_t *data, uint8_t size) { } void SX126x::write_register_(uint16_t reg, uint8_t *data, uint8_t size) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->write_byte(RADIO_WRITE_REGISTER); this->write_byte((reg >> 8) & 0xFF); this->write_byte((reg >> 0) & 0xFF); diff --git a/esphome/components/tuya/tuya.cpp b/esphome/components/tuya/tuya.cpp index fd14844908..3058d82cc4 100644 --- a/esphome/components/tuya/tuya.cpp +++ b/esphome/components/tuya/tuya.cpp @@ -206,15 +206,17 @@ void Tuya::handle_command_(uint8_t command, uint8_t version, const uint8_t *buff if (this->status_pin_reported_ != -1) { this->init_state_ = TuyaInitState::INIT_DATAPOINT; this->send_empty_command_(TuyaCommandType::DATAPOINT_QUERY); - bool is_pin_equals = - this->status_pin_ != nullptr && this->status_pin_->get_pin() == this->status_pin_reported_; - // Configure status pin toggling (if reported and configured) or WIFI_STATE periodic send - if (!is_pin_equals) { - ESP_LOGW(TAG, "Supplied status_pin does not equals the reported pin %i. Using supplied pin anyway.", + if (this->status_pin_ != nullptr) { + if (this->status_pin_->get_pin() != this->status_pin_reported_) { + ESP_LOGW(TAG, "Supplied status_pin does not equal the reported pin %i. Using supplied pin anyway.", + this->status_pin_reported_); + } + ESP_LOGV(TAG, "Configured status pin %i", this->status_pin_->get_pin()); + this->set_interval("wifi", 1000, [this] { this->set_status_pin_(); }); + } else { + ESP_LOGW(TAG, "MCU reported status_pin %i but no status_pin was configured; running in limited mode.", this->status_pin_reported_); } - ESP_LOGV(TAG, "Configured status pin %i", this->status_pin_->get_pin()); - this->set_interval("wifi", 1000, [this] { this->set_status_pin_(); }); } else { this->init_state_ = TuyaInitState::INIT_WIFI; ESP_LOGV(TAG, "Configured WIFI_STATE periodic send"); diff --git a/esphome/components/uart/__init__.py b/esphome/components/uart/__init__.py index 7075228743..4ea32e26a3 100644 --- a/esphome/components/uart/__init__.py +++ b/esphome/components/uart/__init__.py @@ -513,10 +513,11 @@ async def uart_write_to_code(config, action_id, template_arg, args): @coroutine_with_priority(CoroPriority.FINAL) async def final_step(): """Final code generation step to configure optional UART features.""" - if CORE.is_esp32 and CORE.has_networking: - # Wake-on-RX is essentially free on ESP32 (just an ISR function pointer - # registration) — enable by default to reduce RX buffer overflow risk - # by waking the main loop immediately when data arrives. + if (CORE.is_esp32 or CORE.is_esp8266) and CORE.has_networking: + # Wake-on-RX is essentially free (just an ISR function pointer + # registration on ESP32, an inline flag set on ESP8266 software + # serial) — enable by default to reduce RX buffer overflow risk by + # waking the main loop immediately when data arrives. cg.add_define("USE_UART_WAKE_LOOP_ON_RX") diff --git a/esphome/components/uart/uart_component_esp8266.cpp b/esphome/components/uart/uart_component_esp8266.cpp index 0ea7930760..fc1509f737 100644 --- a/esphome/components/uart/uart_component_esp8266.cpp +++ b/esphome/components/uart/uart_component_esp8266.cpp @@ -4,6 +4,9 @@ #include "esphome/core/defines.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#ifdef USE_UART_WAKE_LOOP_ON_RX +#include "esphome/core/wake.h" +#endif #ifdef USE_LOGGER #include "esphome/components/logger/logger.h" @@ -149,7 +152,11 @@ void ESP8266UartComponent::dump_config() { if (this->hw_serial_ != nullptr) { ESP_LOGCONFIG(TAG, " Using hardware serial interface."); } else { - ESP_LOGCONFIG(TAG, " Using software serial"); + ESP_LOGCONFIG(TAG, " Using software serial" +#ifdef USE_UART_WAKE_LOOP_ON_RX + "\n Wake on data RX: ENABLED" +#endif + ); } this->check_logger_conflict(); } @@ -266,6 +273,12 @@ void IRAM_ATTR ESP8266SoftwareSerial::gpio_intr(ESP8266SoftwareSerial *arg) { arg->rx_in_pos_ = (arg->rx_in_pos_ + 1) % arg->rx_buffer_size_; // Clear RX pin so that the interrupt doesn't re-trigger right away again. arg->rx_pin_.clear_interrupt(); +#ifdef USE_UART_WAKE_LOOP_ON_RX + // Wake the main loop so the consuming component drains the byte promptly + // instead of waiting for the next loop_interval_ tick. Important for timing + // sensitive setups that poll read() in a tight loop (e.g. fingerprint_grow). + wake_loop_isrsafe(); +#endif } void IRAM_ATTR HOT ESP8266SoftwareSerial::write_byte(uint8_t data) { if (this->gpio_tx_pin_ == nullptr) { diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index e13d5668af..580d7f6477 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -5,7 +5,7 @@ import math import os from pathlib import Path import re -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from esphome.const import ( CONF_COMMENT, @@ -569,6 +569,12 @@ class EsphomeCore: self.build_path: Path | None = None # The validated configuration, this is None until the config has been validated self.config: ConfigType | None = None + # YAML frontmatter loaded from user YAML files. Frontmatter is a leading + # YAML document separated by `---` from the actual configuration. It is + # ignored by config validation and code generation, but kept here so it + # can be inspected by callers (tooling, future features). Keyed by the + # resolved Path of the source file. + self.frontmatter: dict[Path, Any] = {} # The pending tasks in the task queue (mostly for C++ generation) # This is a priority queue (with heapq) # Each item is a tuple of form: (-priority, unique number, task) @@ -634,6 +640,7 @@ class EsphomeCore: self.config_path = None self.build_path = None self.config = None + self.frontmatter = {} self.event_loop = _FakeEventLoop() self.task_counter = 0 self.variables = {} diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index b9202fb6bf..a452a3f34a 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -93,7 +93,7 @@ class URLSource(Source): class GitSource(Source): - def __init__(self, url: str, ref: str): + def __init__(self, url: str, ref: str | None): self.url = url self.ref = ref @@ -109,7 +109,7 @@ class GitSource(Source): return path def __str__(self): - return f"{self.url}#{self.ref}" + return f"{self.url}#{self.ref}" if self.ref else self.url class InvalidIDFComponent(Exception): @@ -154,41 +154,6 @@ class IDFComponent: self.path = self.source.download(self.get_sanitized_name(), force=force) -def _sanitize_version(version: str) -> str: - """ - Sanitize a version string by removing common requirement prefixes or a leading v. - - Args: - version: Version string to clean. - - Returns: - Cleaned version string without common requirement symbols. - """ - version = version.strip() - - prefixes = ( - "^", - "~=", - "~", - ">=", - "<=", - "==", - "!=", - ">", - "<", - "=", - "v", - "V", - ) - - for p in prefixes: - if version.startswith(p): - version = version[len(p) :] - break - - return version.strip() - - def _get_package_from_pio_registry( username: str | None, pkgname: str, requirements: str ) -> tuple[str, str, str | None, str | None]: @@ -387,7 +352,6 @@ def _convert_library_to_component(library: Library) -> IDFComponent: IDFComponent: The resolved component with name, version, and URL Raises: - ValueError: If a repository URL is missing a reference (#) RuntimeError: If no artifact can be found for the library """ name = None @@ -396,20 +360,25 @@ def _convert_library_to_component(library: Library) -> IDFComponent: # Repository is provided directly if library.repository: - # Parse repository URL to extract name and version + # Parse repository URL: path becomes the component name, fragment + # (if any) becomes the git ref stored on GitSource. A missing + # fragment is fine -- clone_or_update leaves the depth-1 clone on + # the remote's default branch, matching PIO's lib_deps behavior + # and external_components handling. split_result = urlsplit(library.repository) - if not split_result.fragment.strip(): - raise ValueError(f"Missing ref in URL {library.repository}") # Sanitize name name = str(split_result.path).strip("/") name = name.removesuffix(".git") - # Sanitize version - version = _sanitize_version(split_result.fragment) + # IDF Component Manager only accepts "*", a 40-char commit hash, or + # semver here. The actual git ref is preserved in GitSource.ref; + # override_path makes this field cosmetic at build time. + version = "*" repository = urlunsplit(split_result._replace(fragment="")) - source = GitSource(str(repository), split_result.fragment) + ref = split_result.fragment.strip() or None + source = GitSource(str(repository), ref) # Version is provided - resolve using PlatformIO registry elif library.version: @@ -619,9 +588,6 @@ def generate_idf_component_yml(component: IDFComponent) -> str: if description: data["description"] = description - # Do not use the version from library.json/library.properties; it may be incorrect. - data["version"] = component.version - repository = component.data.get("repository", {}).get("url", None) if repository: data["repository"] = repository @@ -631,20 +597,11 @@ def generate_idf_component_yml(component: IDFComponent) -> str: if "dependencies" not in data: data["dependencies"] = {} - # Add this dependency to dependencies - dep = {} - dep["version"] = dependency.version - - # Should use dependency.path as override path - try: - dep["override_path"] = str(dependency.path) - except RuntimeError as e: - # No local path: only a GitSource can substitute its URL. - if not isinstance(dependency.source, GitSource): - raise e - dep["git"] = dependency.source.url - - data["dependencies"][dependency.get_sanitized_name()] = dep + # Every dependency goes through _generate_idf_component → + # component.download() before this runs, so .path is always set. + data["dependencies"][dependency.get_sanitized_name()] = { + "override_path": str(dependency.path), + } return yaml_util.dump(data) @@ -699,6 +656,26 @@ def _process_dependencies(component: IDFComponent): if not dependencies: return + # PIO's library.json accepts both the list-of-dicts form and the + # shorthand dict form ``{"owner/Name": "version_spec"}``. Normalize + # the dict form so the loop below sees a uniform list. Iterating a + # dict gives string keys, which would silently fail the + # ``"name" in dependency`` substring check and skip every entry. + if isinstance(dependencies, dict): + normalized = [] + for raw_name, spec in dependencies.items(): + if "/" in raw_name: + owner, pkgname = raw_name.split("/", 1) + else: + owner, pkgname = None, raw_name + entry = {"name": pkgname, "owner": owner} + if isinstance(spec, dict): + entry.update(spec) + else: + entry["version"] = spec + normalized.append(entry) + dependencies = normalized + _LOGGER.info("Processing %s@%s component dependencies...", name, version) for dependency in dependencies: # Validate dependency structure diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 8996ff1e02..079c97cc98 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -7,6 +7,7 @@ import json import logging import os from pathlib import Path +import platform import shutil import subprocess import sys @@ -17,7 +18,7 @@ import requests from esphome.config_validation import Version from esphome.core import CORE -from esphome.helpers import ProgressBar, get_str_env, rmtree +from esphome.helpers import ProgressBar, get_str_env, rmtree, write_file_if_changed PathType = str | os.PathLike @@ -549,11 +550,11 @@ def _tar_extract_all( if not (mode & stat.S_IXUSR): mode &= ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) mode |= stat.S_IRUSR | stat.S_IWUSR - elif member.isdir() or member.issym(): - # Ignore mode for directories & symlinks - mode = None - else: - # Block special files + elif not (member.isdir() or member.issym()): + # Block special files. Directories and symlinks keep + # their masked-original mode — passing None here would + # crash tarfile.extract on Python <3.12 (its chmod + # path calls os.chmod unconditionally). continue member.mode = mode @@ -783,12 +784,109 @@ def download_from_mirrors( return None +def _write_idf_version_txt(framework_path: Path, version: str) -> None: + """Write /version.txt if missing. + + IDF's build.cmake picks the version it embeds in the firmware (and + stamps onto the bootloader) in this order: ``${IDF_PATH}/version.txt`` + if present, else ``git describe`` against IDF_PATH, else the + ``IDF_VERSION_MAJOR/MINOR/PATCH`` triplet from ``tools/cmake/version.cmake``. + On a clean esphome-libs tarball ``.git`` is fully stripped, so + git_describe returns ``HEAD-HASH-NOTFOUND`` (falsy) and the triplet + wins -- correct by luck. But a *partial* ``.git`` (e.g. a custom + framework.source pointed at a real git URL where build artifacts + mark the tree dirty) makes git_describe return ``-dirty``, + which is what then gets baked into the bootloader. Dropping + version.txt forces the right answer regardless. + """ + version_txt = framework_path / "version.txt" + if version_txt.exists(): + return + try: + version_txt.write_text(f"v{version}\n", encoding="utf-8") + except OSError as e: + _LOGGER.warning( + "Could not write %s (%s); bootloader version string may be incorrect.", + version_txt, + e, + ) + + +# Backport of espressif/esp-idf#18272: every ESPHome-supported IDF release +# through v6.0 ships a tools.json whose ninja 1.12.1 entry has no +# ``linux-arm64`` source. ``idf_tools.py`` then either fails to find a +# matching binary or grabs the x86_64 one, which can't execute on +# aarch64. cmake is already populated across the same release range; we +# only need to inject ninja. Values lifted verbatim from the IDF v6.0.1 +# tools.json where the fix landed natively. +_NINJA_ARM64_BACKPORT: dict[str, dict[str, str | int]] = { + "1.12.1": { + "rename_dist": "ninja-linux-arm64-v1.12.1.zip", + "sha256": "5c25c6570b0155e95fce5918cb95f1ad9870df5768653afe128db822301a05a1", + "size": 121787, + "url": "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux-aarch64.zip", + }, +} + + +def _patch_tools_json_for_linux_arm64(framework_path: Path) -> None: + """Inject ninja linux-arm64 entries into the framework's tools.json on aarch64. + + Idempotent: a tools.json that already has the entry, or a host that + isn't aarch64, is a no-op. Applied unconditionally on every install + check so a build dir extracted before the backport got fixed up + without forcing a clean. + """ + if platform.machine() != "aarch64": + return + + tools_json = framework_path / "tools" / "tools.json" + if not tools_json.is_file(): + return + + try: + with open(tools_json, encoding="utf-8") as f: + data = json.load(f) + except (json.JSONDecodeError, OSError) as e: + _LOGGER.warning( + "Could not parse %s for linux-arm64 backport (%s); " + "skipping. A clean reinstall of the framework directory " + "may be needed.", + tools_json, + e, + ) + return + + changed = False + for tool in data.get("tools", []): + if tool.get("name") != "ninja": + continue + for ver in tool.get("versions", []): + entry = _NINJA_ARM64_BACKPORT.get(ver.get("name")) + if entry is None or ver.get("linux-arm64"): + continue + ver["linux-arm64"] = entry + changed = True + + if changed: + # write_file_if_changed stages a tempfile in the destination dir + # and atomically replaces — safe against mid-write interruption + # and concurrent invocations. + write_file_if_changed(tools_json, json.dumps(data, indent=2) + "\n") + _LOGGER.info( + "Patched %s to add ninja linux-arm64 download " + "(espressif/esp-idf#18272 backport).", + tools_json, + ) + + def _check_esphome_idf_framework_install( version: str, targets: list[str], tools: list[str], force: bool = False, env: dict[str, str] | None = None, + source_url: str | None = None, ) -> tuple[Path, bool]: """ Check and install ESP-IDF framework. @@ -799,6 +897,11 @@ def _check_esphome_idf_framework_install( tools: list of tools to install force: If True, force reinstallation env: Optional dictionary of environment variables to set + source_url: Optional override URL for the framework tarball. Supports + the same ``{VERSION}`` / ``{MAJOR}`` / ``{MINOR}`` / ``{PATCH}`` / + ``{EXTRA}`` substitutions as ESPHOME_IDF_FRAMEWORK_MIRRORS. When + set, it replaces the default mirror list — no implicit fallback, + so a misspelled URL fails loudly. Returns: tuple of (framework_path, install_flag) @@ -820,6 +923,10 @@ def _check_esphome_idf_framework_install( env_stamp_file = framework_path / ESPHOME_STAMP_FILE idf_tools_path = framework_path / "tools" / "idf_tools.py" _LOGGER.info("Checking ESP-IDF %s framework ...", version) + # Logged every invocation (not just on install) so the user can verify the + # override. A changed URL needs ``esphome clean`` to force a re-download. + if source_url: + _LOGGER.info("Using framework source override: %s", source_url) # 2. Download and extract the framework if not already extracted. # The marker is written last after extraction succeeds, so its presence @@ -847,14 +954,23 @@ def _check_esphome_idf_framework_install( except ValueError: pass - download_from_mirrors( - ESPHOME_IDF_FRAMEWORK_MIRRORS, substitutions, tmp.file - ) + mirrors = [source_url] if source_url else ESPHOME_IDF_FRAMEWORK_MIRRORS + download_from_mirrors(mirrors, substitutions, tmp.file) _LOGGER.info("Extracting ESP-IDF %s framework ...", version) archive_extract_all(tmp.file, framework_path, progress_header="Extracting") extracted_marker.touch() + # Idempotent post-extract patch: written every invocation so a build + # dir extracted before this fix gets the file too, without forcing a + # clean. Skips when version.txt already exists. + _write_idf_version_txt(framework_path, version) + + # Apply the ninja linux-arm64 backport on every invocation, not just on + # fresh extracts — idempotent and cheap, and lets a build dir carrying + # a pre-patch tools.json get fixed up without forcing a clean. + _patch_tools_json_for_linux_arm64(framework_path) + # 3. Check if the framework tools are the same and correctly installed if not install: install = True @@ -1011,6 +1127,7 @@ def check_esp_idf_install( tools: list[str] | None = None, features: list[str] | None = None, force: bool = False, + source_url: str | None = None, ) -> tuple[Path, Path]: """ Check and install ESP-IDF framework and Python environment. @@ -1021,6 +1138,10 @@ def check_esp_idf_install( tools: list of tools to install features: Features to install force: If True, force reinstallation + source_url: Optional override URL for the framework tarball. When + set, it replaces the default mirror list (no fallback). Forwarded + to ``_check_esphome_idf_framework_install``; supports the same URL + substitutions. Returns: tuple of (framework_path, python_env_path) @@ -1043,7 +1164,7 @@ def check_esp_idf_install( # 1) Framework framework_path, installed = _check_esphome_idf_framework_install( - version, targets, tools, force=force, env=env + version, targets, tools, force=force, env=env, source_url=source_url ) features = features or ESPHOME_IDF_DEFAULT_FEATURES diff --git a/esphome/espidf/runner.py b/esphome/espidf/runner.py index 65df37c7b2..da3f77cdd3 100644 --- a/esphome/espidf/runner.py +++ b/esphome/espidf/runner.py @@ -66,6 +66,12 @@ FILTER_IDF_LINES: list[str] = [ # Drop the blank line rich emits after the note so the build log # doesn't end with an orphan gap before ESPHome's own status lines. r"\s*$", + # ESP-IDF shells out to ``git rev-parse`` to embed a commit hash; + # esphome-libs strips ``.git`` from the tarball so those probes fail + # noisily without affecting the build. + r"-- git rev-parse returned ", + r"fatal: not a git repository", + r"Stopping at filesystem boundary", ] diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index e0bc5bb393..ef28575caa 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -10,6 +10,7 @@ import shutil import subprocess from esphome.components.esp32.const import KEY_ESP32, KEY_FLASH_SIZE, KEY_IDF_VERSION +from esphome.const import CONF_FRAMEWORK, CONF_SOURCE from esphome.core import CORE, EsphomeError from esphome.espidf.framework import check_esp_idf_install, get_framework_env from esphome.espidf.size_summary import print_summary @@ -37,13 +38,27 @@ def _get_core_framework_version(): return str(CORE.data[KEY_ESP32][KEY_IDF_VERSION]) +def _get_framework_source_override() -> str | None: + """Return the user-supplied esp32.framework.source override, if any. + + The override lets a user point the IDF tarball download at a custom URL + (mirror, fork, local server). Substitutions like ``{VERSION}`` / + ``{MAJOR}`` etc. work the same as in the default mirror list. + """ + if CORE.config is None: + return None + return CORE.config.get(KEY_ESP32, {}).get(CONF_FRAMEWORK, {}).get(CONF_SOURCE) + + def _get_esphome_esp_idf_paths( version: str | None = None, ) -> tuple[os.PathLike, os.PathLike]: version = version or _get_core_framework_version() paths = _cache().paths if version not in paths: - paths[version] = check_esp_idf_install(version) + paths[version] = check_esp_idf_install( + version, source_url=_get_framework_source_override() + ) return paths[version] diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 42d0d5de6b..6bc166ff44 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -2,7 +2,7 @@ dependencies: bblanchon/arduinojson: version: "7.4.2" esphome/esp-audio-libs: - version: 3.0.0 + version: 3.1.0 esphome/esp-micro-speech-features: version: 1.2.3 esphome/micro-decoder: @@ -100,6 +100,6 @@ dependencies: esp32async/asynctcp: version: 3.4.91 sendspin/sendspin-cpp: - version: 0.6.0 + version: 0.6.1 lvgl/lvgl: version: 9.5.0 diff --git a/esphome/storage_json.py b/esphome/storage_json.py index e481827080..7f8885ba5f 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -14,6 +14,7 @@ from esphome.const import ( KEY_CORE, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM, + Toolchain, ) from esphome.core import CORE from esphome.helpers import write_file_if_changed @@ -98,6 +99,7 @@ class StorageJSON: no_mdns: bool, framework: str | None = None, core_platform: str | None = None, + toolchain: str | None = None, ) -> None: # Version of the storage JSON schema assert storage_version is None or isinstance(storage_version, int) @@ -134,6 +136,8 @@ class StorageJSON: self.framework = framework # The core platform of this firmware. Like "esp32", "rp2040", "host" etc. self.core_platform = core_platform + # The toolchain used for the build ("platformio" / "esp-idf") + self.toolchain = toolchain def as_dict(self): return { @@ -153,6 +157,7 @@ class StorageJSON: "no_mdns": self.no_mdns, "framework": self.framework, "core_platform": self.core_platform, + "toolchain": self.toolchain, } def to_json(self): @@ -189,6 +194,7 @@ class StorageJSON: ), framework=esph.target_framework, core_platform=esph.target_platform, + toolchain=esph.toolchain.value if esph.toolchain is not None else None, ) @staticmethod @@ -236,6 +242,7 @@ class StorageJSON: no_mdns = storage.get("no_mdns", False) framework = storage.get("framework") core_platform = storage.get("core_platform") + toolchain = storage.get("toolchain") return StorageJSON( storage_version, name, @@ -253,6 +260,7 @@ class StorageJSON: no_mdns, framework, core_platform, + toolchain, ) @staticmethod @@ -273,6 +281,18 @@ class StorageJSON: """ CORE.name = self.name CORE.build_path = self.build_path + # Restore toolchain so upload/logs picks the right firmware_bin path. + # An unknown value (corrupt sidecar, or written by a newer ESPHome) + # just leaves CORE.toolchain None — the fallback then picks PlatformIO. + if self.toolchain and CORE.toolchain is None: + try: + CORE.toolchain = Toolchain(self.toolchain) + except ValueError: + _LOGGER.debug( + "Ignoring unknown toolchain %r from %s", + self.toolchain, + storage_path(), + ) target_platform = self.core_platform or self.target_platform.lower() CORE.data[KEY_CORE] = { KEY_TARGET_PLATFORM: target_platform, diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index b56d024418..9a36ad089c 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -768,10 +768,35 @@ def _load_yaml_internal_with_type( content: TextIOWrapper, yaml_loader: Callable[[Path], dict[str, Any]], ) -> Any: - """Load a YAML file.""" + """Load a YAML file. + + Supports an optional leading YAML frontmatter document: when the file + contains two YAML documents separated by ``---``, the first document is + treated as metadata and stored in :attr:`CORE.frontmatter` keyed by the + resolved file path, while the second document is returned as the actual + configuration. Frontmatter is ignored by config validation and code + generation. + """ loader = loader_type(content, fname, yaml_loader) try: - return loader.get_single_data() or OrderedDict() + documents: list[Any] = [] + while loader.check_data(): + documents.append(loader.get_data()) + if len(documents) > 2: + raise EsphomeError( + f"YAML file '{fname}' contains {len(documents)} documents but " + f"at most two are supported (an optional frontmatter document " + f"followed by the configuration)." + ) + if len(documents) == 2: + frontmatter = documents[0] + config = documents[1] + if frontmatter is not None: + CORE.frontmatter[Path(fname).resolve()] = frontmatter + return config if config is not None else OrderedDict() + if len(documents) == 1: + return documents[0] or OrderedDict() + return OrderedDict() except yaml.YAMLError as exc: raise EsphomeError(exc) from exc finally: diff --git a/requirements.txt b/requirements.txt index e3de4a134c..178e05497f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,8 +12,8 @@ platformio==6.1.19 esptool==5.2.0 click==8.3.3 esphome-dashboard==20260425.0 -aioesphomeapi==45.0.3 -zeroconf==0.149.7 +aioesphomeapi==45.0.4 +zeroconf==0.149.16 puremagic==1.30 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import diff --git a/requirements_test.txt b/requirements_test.txt index ea4941a882..102a9cae6e 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.0.5 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating -ruff==0.15.13 # also change in .pre-commit-config.yaml when updating +ruff==0.15.14 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating pre-commit @@ -16,7 +16,7 @@ hypothesis==6.92.1 # CodSpeed benchmarks under tests/benchmarks/python/ # (skipped via pytest.importorskip when missing -- only required for the # benchmarks job in .github/workflows/ci.yml) -pytest-codspeed==5.0.2 +pytest-codspeed==5.0.3 # Used by the import-time regression check (.github/workflows/ci.yml → import-time job) importtime-waterfall==1.0.0 diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 3259fb5836..ef2175eb79 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -5,6 +5,7 @@ This script is a centralized way to determine which CI jobs need to run based on what files have changed. It outputs JSON with the following structure: { + "core_ci": true/false, "integration_tests": true/false, "integration_test_buckets": [{"name": "1/3", "tests": ["tests/integration/test_foo.py", ...]}, ...], "clang_tidy": true/false, @@ -22,6 +23,11 @@ what files have changed. It outputs JSON with the following structure: } The CI workflow uses this information to: +- Gate the unconditional jobs (ci-custom, pytest, pre-commit-ci-lite) via core_ci; + false when a pull_request only touches CI-irrelevant meta paths (other workflow + files, .github/actions/build-image/*, .yamllint, .github/dependabot.yml, docker/**) + so workflow-only PRs satisfy the required CI Status check without running the + unconditional jobs. Always true on non-pull_request events and under --force-all. - Skip or run integration tests - Skip or run clang-tidy (and whether to do a full scan) - Skip or run clang-format @@ -712,6 +718,69 @@ def should_run_benchmarks(branch: str | None = None) -> bool: return any(get_component_from_path(f) in benchmarked_components for f in files) +# Files / path patterns whose changes alone don't warrant running the +# unconditional CI jobs (`ci-custom`, `pytest`, `pre-commit-ci-lite`). +# Single source of truth for what we treat as "CI-irrelevant" on +# pull_request events; ci.yml used to encode this in its own +# `pull_request.paths` filter, but that hid the required `CI Status` +# check on PRs that only touched these files (dependabot Action bumps, +# dependabot.yml edits, docker/ changes, etc.) and forced admin +# force-merges. +# +# ci.yml itself is deliberately *not* ignored — editing the CI workflow +# must still run CI. Workflows that have their own dedicated triggers +# (codeql.yml, ci-docker.yml, ...) are matched via the +# `.github/workflows/*.yml` prefix below and exclude ci.yml explicitly. +CI_IRRELEVANT_EXACT_FILES = frozenset( + { + ".yamllint", + ".github/dependabot.yml", + } +) + + +def _is_ci_irrelevant_path(path: str) -> bool: + """Whether a single changed path is irrelevant to the unconditional CI jobs.""" + if path in CI_IRRELEVANT_EXACT_FILES: + return True + # docker/** — all descendants + if path.startswith("docker/"): + return True + # .github/workflows/*.yml — top-level workflow files other than ci.yml + # (ci.yml itself must still trigger full CI when edited). + if path.startswith(".github/workflows/") and path.endswith(".yml"): + if path == ".github/workflows/ci.yml": + return False + if "/" not in path[len(".github/workflows/") :]: + return True + # .github/actions/build-image/* — direct children only, matches the + # single-star glob the workflow used to encode. + if path.startswith(".github/actions/build-image/"): + rest = path[len(".github/actions/build-image/") :] + if rest and "/" not in rest: + return True + return False + + +def should_run_core_ci(branch: str | None = None) -> bool: + """Determine if the unconditional CI jobs (ci-custom/pytest/pre-commit-ci-lite) should run. + + Returns False only when every changed file is in the CI-irrelevant set + above (see ``_is_ci_irrelevant_path``). Empty diffs return True so we + never accidentally skip CI when the diff probe fails. + + Args: + branch: Branch to compare against. If None, uses default. + + Returns: + True if the unconditional CI jobs should run, False otherwise. + """ + files = changed_files(branch) + if not files: + return True + return any(not _is_ci_irrelevant_path(f) for f in files) + + def _any_changed_file_endswith(branch: str | None, extensions: tuple[str, ...]) -> bool: """Check if a changed file ends with any of the specified extensions.""" return any(file.endswith(extensions) for file in changed_files(branch)) @@ -1075,6 +1144,16 @@ def main() -> None: args = parser.parse_args() # Determine what should run + # core_ci gates the unconditional jobs in ci.yml (ci-custom, pytest, + # pre-commit-ci-lite). Non-pull_request events (push to dev/beta/release + # and merge_group) always run them so behavior like venv-cache saves on + # push to dev is preserved. + event_name = os.environ.get("GITHUB_EVENT_NAME", "") + run_core_ci = ( + True + if args.force_all or event_name != "pull_request" + else should_run_core_ci(args.branch) + ) if args.force_all: integration_run_all, integration_test_files = True, [] run_clang_tidy = True @@ -1255,6 +1334,7 @@ def main() -> None: component_test_batches = [] output: dict[str, Any] = { + "core_ci": run_core_ci, "integration_tests": run_integration, "integration_test_buckets": integration_test_buckets, "clang_tidy": run_clang_tidy, diff --git a/tests/component_tests/light/test_effect_validation.py b/tests/component_tests/light/test_effect_validation.py index 579e92c62a..aab9072cc8 100644 --- a/tests/component_tests/light/test_effect_validation.py +++ b/tests/component_tests/light/test_effect_validation.py @@ -9,13 +9,17 @@ import pytest from esphome import config_validation as cv from esphome.components.light import ( + EffectCycleRef, EffectRef, _final_validate, _get_data, available_effects_str, find_effect_index, ) -from esphome.components.light.automation import _record_effect_ref +from esphome.components.light.automation import ( + _record_effect_cycle_ref, + _record_effect_ref, +) from esphome.config import Config, path_context from esphome.const import CONF_EFFECT, CONF_EFFECTS, CONF_ID, CONF_NAME from esphome.core import ID, Lambda @@ -215,6 +219,111 @@ def test_final_validate_drains_refs() -> None: fv.full_config.reset(token) +# --- _final_validate: EffectCycleRef --- + + +def _setup_cycle_final_validate( + cycle_refs: list[EffectCycleRef], + light_configs: list[ConfigType], + declare_ids: list[tuple[ID, list[str | int]]], +) -> Token: + """Set up CORE.data and fv.full_config for EffectCycleRef final_validate tests.""" + data = _get_data() + data.effect_cycle_refs = cycle_refs + + full_conf = Config() + full_conf["light"] = light_configs + for id_, path in declare_ids: + full_conf.declare_ids.append((id_, path)) + + return fv.full_config.set(full_conf) + + +def test_final_validate_cycle_accepts_light_with_effects() -> None: + """Cycle ref against a light with effects should not raise.""" + light_id = ID("led1", is_declaration=True) + token = _setup_cycle_final_validate( + cycle_refs=[ + EffectCycleRef(light_id=light_id, component_path=["esphome"]), + ], + light_configs=[{CONF_ID: light_id, CONF_EFFECTS: _make_effects("Fast Pulse")}], + declare_ids=[(light_id, ["light", 0, CONF_ID])], + ) + try: + _final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_final_validate_cycle_rejects_light_without_effects_key() -> None: + """Cycle ref against a light with no CONF_EFFECTS key should raise.""" + light_id = ID("led1", is_declaration=True) + token = _setup_cycle_final_validate( + cycle_refs=[ + EffectCycleRef(light_id=light_id, component_path=["esphome"]), + ], + light_configs=[{CONF_ID: light_id}], + declare_ids=[(light_id, ["light", 0, CONF_ID])], + ) + try: + with pytest.raises(cv.FinalExternalInvalid, match="no effects configured"): + _final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_final_validate_cycle_rejects_light_with_empty_effects() -> None: + """Cycle ref against a light with empty effects list should raise.""" + light_id = ID("led1", is_declaration=True) + token = _setup_cycle_final_validate( + cycle_refs=[ + EffectCycleRef(light_id=light_id, component_path=["esphome"]), + ], + light_configs=[{CONF_ID: light_id, CONF_EFFECTS: []}], + declare_ids=[(light_id, ["light", 0, CONF_ID])], + ) + try: + with pytest.raises(cv.FinalExternalInvalid, match="no effects configured"): + _final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_final_validate_cycle_unknown_light_id_skipped() -> None: + """Cycle refs to unknown light IDs should be silently skipped.""" + data = _get_data() + data.effect_cycle_refs = [ + EffectCycleRef( + light_id=ID("nonexistent", is_declaration=True), + component_path=["esphome"], + ) + ] + + full_conf = Config() + token = fv.full_config.set(full_conf) + try: + _final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_final_validate_drains_cycle_refs() -> None: + """Cycle refs should be drained after validation to avoid redundant runs.""" + light_id = ID("led1", is_declaration=True) + token = _setup_cycle_final_validate( + cycle_refs=[ + EffectCycleRef(light_id=light_id, component_path=["esphome"]), + ], + light_configs=[{CONF_ID: light_id, CONF_EFFECTS: _make_effects("Fast Pulse")}], + declare_ids=[(light_id, ["light", 0, CONF_ID])], + ) + try: + _final_validate({}) + assert _get_data().effect_cycle_refs == [] + finally: + fv.full_config.reset(token) + + # --- _record_effect_ref --- @@ -278,3 +387,19 @@ def test_record_effect_ref_skips_no_effect_key() -> None: config: ConfigType = {CONF_ID: ID("led1", is_declaration=True)} _record_effect_ref(config) assert _get_data().effect_refs == [] + + +# --- _record_effect_cycle_ref --- + + +@pytest.mark.usefixtures("_path_ctx") +def test_record_effect_cycle_ref() -> None: + """Cycle-action config should be recorded with light_id and path.""" + light_id = ID("led1", is_declaration=True) + config: ConfigType = {CONF_ID: light_id} + result = _record_effect_cycle_ref(config) + assert result is config + data = _get_data() + assert len(data.effect_cycle_refs) == 1 + assert data.effect_cycle_refs[0].light_id is light_id + assert data.effect_cycle_refs[0].component_path == ["esphome"] diff --git a/tests/component_tests/lvgl/test_container_schema_cache.py b/tests/component_tests/lvgl/test_container_schema_cache.py new file mode 100644 index 0000000000..39e623d720 --- /dev/null +++ b/tests/component_tests/lvgl/test_container_schema_cache.py @@ -0,0 +1,87 @@ +"""Tests for container_schema() memoization and lazy build.""" + +from __future__ import annotations + +from collections.abc import Generator +from unittest.mock import patch + +import pytest + +from esphome import config_validation as cv +import esphome.components.lvgl # noqa: F401 +from esphome.components.lvgl import schemas as lvgl_schemas +from esphome.components.lvgl.schemas import WIDGET_TYPES, container_schema + + +@pytest.fixture(autouse=True) +def _clear_container_schema_cache() -> Generator[None]: + cache = getattr(lvgl_schemas, "_CONTAINER_SCHEMA_CACHE", None) + if cache is not None: + cache.clear() + yield + if cache is not None: + cache.clear() + + +def _widget_type(name: str = "obj"): + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def test_same_args_return_same_validator() -> None: + wt = _widget_type("obj") + assert container_schema(wt) is container_schema(wt) + + +def test_extras_none_vs_truthy_get_different_validators() -> None: + wt = _widget_type("obj") + no_extras = container_schema(wt) + extras = {cv.Optional("custom_extra"): cv.string} + assert no_extras is not container_schema(wt, extras) + + +def test_different_widget_types_get_different_validators() -> None: + assert container_schema(_widget_type("obj")) is not container_schema( + _widget_type("label") + ) + + +def test_schema_build_is_deferred_until_first_validation() -> None: + wt = _widget_type("obj") + with patch.object( + lvgl_schemas, "obj_schema", wraps=lvgl_schemas.obj_schema + ) as obj_schema_mock: + validator = container_schema(wt) + assert obj_schema_mock.call_count == 0 + validator({}) + assert obj_schema_mock.call_count == 1 + validator({}) + assert obj_schema_mock.call_count == 1 + + +def test_cached_validator_produces_equivalent_output() -> None: + wt = _widget_type("obj") + cached = container_schema(wt) + cached_result = cached({}) + lvgl_schemas._CONTAINER_SCHEMA_CACHE.clear() + reference = container_schema(wt) + assert cached is not reference + assert cached_result == reference({}) + + +def test_id_recycling_is_caught_by_identity_guard() -> None: + wt = _widget_type("obj") + real_extras = {cv.Optional("a"): cv.int_} + validator_a = container_schema(wt, real_extras) + + cache_key = (id(wt), id(real_extras)) + cached_entry = lvgl_schemas._CONTAINER_SCHEMA_CACHE[cache_key] + sentinel = {cv.Optional("a"): cv.int_} + lvgl_schemas._CONTAINER_SCHEMA_CACHE[cache_key] = ( + cached_entry[0], + sentinel, + cached_entry[2], + ) + + assert container_schema(wt, real_extras) is not validator_a diff --git a/tests/component_tests/lvgl/test_update_action_lazy.py b/tests/component_tests/lvgl/test_update_action_lazy.py new file mode 100644 index 0000000000..7fcdc149cf --- /dev/null +++ b/tests/component_tests/lvgl/test_update_action_lazy.py @@ -0,0 +1,53 @@ +"""Tests for lvgl..update lazy schema build.""" + +from __future__ import annotations + +from unittest.mock import patch + +from esphome.automation import ACTION_REGISTRY +import esphome.components.lvgl # noqa: F401 +from esphome.components.lvgl.schemas import WIDGET_TYPES +from esphome.components.lvgl.widgets import _update_action_schema +from esphome.config_validation import Schema + + +def _widget_type(name: str = "obj"): + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def test_registry_entry_uses_lazy_validator() -> None: + entry = ACTION_REGISTRY["lvgl.label.update"] + assert callable(entry.raw_schema) + assert not isinstance(entry.raw_schema, Schema) + + +def test_lazy_validator_defers_build_until_first_call() -> None: + wt = _widget_type("label") + with patch( + "esphome.components.lvgl.widgets._build_update_schema", + wraps=lambda w: Schema({}), + ) as build_mock: + validator = _update_action_schema(wt) + assert build_mock.call_count == 0 + validator({}) + assert build_mock.call_count == 1 + validator({}) + assert build_mock.call_count == 1 + + +def test_eager_build_when_schema_extraction_enabled() -> None: + wt = _widget_type("label") + with patch("esphome.components.lvgl.widgets.EnableSchemaExtraction", True): + result = _update_action_schema(wt) + assert isinstance(result, Schema) + + +def test_lazy_and_eager_produce_equivalent_validation() -> None: + wt = _widget_type("label") + with patch("esphome.components.lvgl.widgets.EnableSchemaExtraction", True): + eager = _update_action_schema(wt) + lazy = _update_action_schema(wt) + sample = {"id": "label_id"} + assert lazy(sample) == eager(sample) diff --git a/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml b/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml index 6c27bd35d0..df5b0123b5 100644 --- a/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml +++ b/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml @@ -1,6 +1,8 @@ <<: !include common.yaml esp32_ble_tracker: + +esp32_ble: max_connections: 9 bluetooth_proxy: diff --git a/tests/components/light/common.yaml b/tests/components/light/common.yaml index 044a8144fa..cd9b27768e 100644 --- a/tests/components/light/common.yaml +++ b/tests/components/light/common.yaml @@ -103,6 +103,16 @@ esphome: - light.turn_on: id: test_monochromatic_light effect: !lambda 'return iteration > 1 ? "Strobe" : "none";' + # Cycle through configured effects (skip "None") + - light.effect.next: test_monochromatic_light + - light.effect.previous: test_monochromatic_light + # Cycle through effects including "None" + - light.effect.next: + id: test_monochromatic_light + include_none: true + - light.effect.previous: + id: test_monochromatic_light + include_none: true - light.dim_relative: id: test_monochromatic_light relative_brightness: 5% diff --git a/tests/dashboard/test_web_server.py b/tests/dashboard/test_web_server.py index 626aea0216..0ee841e68c 100644 --- a/tests/dashboard/test_web_server.py +++ b/tests/dashboard/test_web_server.py @@ -1503,13 +1503,18 @@ async def test_websocket_refresh_command( ) -> None: """Test WebSocket refresh command triggers dashboard update.""" with patch("esphome.dashboard.web_server.DASHBOARD_SUBSCRIBER") as mock_subscriber: - mock_subscriber.request_refresh = Mock() + # Signal an asyncio.Event when request_refresh is invoked so the + # test can deterministically wait for the server-side handler to run + # instead of relying on a fixed sleep (flaky on Windows CI under load). + called = asyncio.Event() + mock_subscriber.request_refresh = Mock(side_effect=called.set) # Send refresh command await websocket_client.write_message(json.dumps({"event": "refresh"})) - # Give it a moment to process - await asyncio.sleep(0.01) + # Wait for the server to process the message and invoke request_refresh + async with asyncio.timeout(5): + await called.wait() # Verify request_refresh was called mock_subscriber.request_refresh.assert_called_once() diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 202ae9030f..7bb9fe2543 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -775,6 +775,88 @@ def test_should_run_import_time_with_branch() -> None: mock_changed.assert_called_once_with("release") +@pytest.mark.parametrize( + ("path", "expected_result"), + [ + # Exact-file matches in the CI-irrelevant set. + (".yamllint", True), + (".github/dependabot.yml", True), + # Other top-level workflow files are irrelevant; ci.yml itself is not. + (".github/workflows/codeql.yml", True), + (".github/workflows/release.yml", True), + (".github/workflows/ci.yml", False), + # Nested files under workflows/ are not matched by the single-star glob. + (".github/workflows/matchers/gcc.json", False), + # build-image action: direct children only (single-star glob). + (".github/actions/build-image/action.yml", True), + (".github/actions/build-image/nested/file.yml", False), + # Other actions are CI-relevant. + (".github/actions/restore-python/action.yml", False), + # docker/** covers everything under docker/. + ("docker/Dockerfile", True), + ("docker/scripts/run.sh", True), + # Regular source files are CI-relevant. + ("esphome/__main__.py", False), + ("esphome/components/wifi/wifi_component.cpp", False), + ("README.md", False), + ("tests/script/test_determine_jobs.py", False), + ], +) +def test_is_ci_irrelevant_path(path: str, expected_result: bool) -> None: + """Test _is_ci_irrelevant_path mirrors the historic ci.yml path filter.""" + assert determine_jobs._is_ci_irrelevant_path(path) == expected_result + + +@pytest.mark.parametrize( + ("changed_files", "expected_result"), + [ + # Empty diffs default to True — don't accidentally skip CI on a + # broken probe. + ([], True), + # Any CI-relevant file flips the result to True. + (["esphome/__main__.py"], True), + (["esphome/components/wifi/wifi_component.cpp"], True), + (["README.md"], True), + # All-irrelevant diffs return False. + ([".github/workflows/codeql.yml"], False), + ( + [".github/workflows/codeql.yml", ".github/workflows/release.yml"], + False, + ), + ([".yamllint"], False), + ([".github/dependabot.yml"], False), + (["docker/Dockerfile"], False), + ( + [ + ".github/workflows/codeql.yml", + ".github/dependabot.yml", + "docker/Dockerfile", + ], + False, + ), + # Mixed diffs always trigger CI. + ( + [".github/workflows/codeql.yml", "esphome/__main__.py"], + True, + ), + # ci.yml itself is treated as CI-relevant. + ([".github/workflows/ci.yml"], True), + ], +) +def test_should_run_core_ci(changed_files: list[str], expected_result: bool) -> None: + """Test should_run_core_ci function.""" + with patch.object(determine_jobs, "changed_files", return_value=changed_files): + assert determine_jobs.should_run_core_ci() == expected_result + + +def test_should_run_core_ci_with_branch() -> None: + """Test should_run_core_ci passes the branch through to changed_files.""" + with patch.object(determine_jobs, "changed_files") as mock_changed: + mock_changed.return_value = [] + determine_jobs.should_run_core_ci("release") + mock_changed.assert_called_once_with("release") + + @pytest.mark.parametrize( ("changed_files", "expected_result"), [ diff --git a/tests/test_build_components/build_components_base.esp32-c6-idf.yaml b/tests/test_build_components/build_components_base.esp32-c6-idf.yaml index 9dbc465ca2..4105481dc5 100644 --- a/tests/test_build_components/build_components_base.esp32-c6-idf.yaml +++ b/tests/test_build_components/build_components_base.esp32-c6-idf.yaml @@ -4,6 +4,7 @@ esphome: esp32: board: esp32-c6-devkitc-1 + flash_size: 8MB framework: type: esp-idf diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index 8c9cfa8101..e12107152b 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -253,6 +253,106 @@ def test_run_esphome_upload_and_logs_fall_back_when_no_cache( mock_read.assert_called_once() +def test_run_esphome_upload_does_not_refresh_cache_without_sidecar( + tmp_path: Path, +) -> None: + """Without a StorageJSON sidecar (no compile has run), the fallback + skips the cache write -- load_compiled_config requires the sidecar, + so writing the rendered (secret-resolved) YAML would be inert and + leak secrets to disk for nothing.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + with ( + patch( + "esphome.__main__.read_config", + return_value={"esphome": {"name": "lite_test"}}, + ), + patch("esphome.compiled_config.save_compiled_config") as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"upload": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "upload", str(yaml_path)]) + + mock_save.assert_not_called() + + +@pytest.mark.parametrize("command", ["upload", "logs"]) +def test_run_esphome_upload_and_logs_refresh_cache_on_fallback( + tmp_path: Path, command: str +) -> None: + """A stale-cache fallback rewrites the cache so the next call hits + the fast path. Without this, every upload/logs after a YAML edit + pays for read_config() until the next compile rewrites the cache.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + storage_dir = tmp_path / ".esphome" / "storage" + _write_storage(storage_dir / "lite_test.yaml.json") + cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml") + _set_cache_mtime(cache, yaml_path, offset=-60) # stale + + fresh_config = {"esphome": {"name": "lite_test"}, "logger": {}} + + with ( + patch("esphome.__main__.read_config", return_value=fresh_config), + patch( + "esphome.compiled_config.save_compiled_config", wraps=save_compiled_config + ) as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {command: lambda args, config: 0}, + ), + ): + assert run_esphome(["esphome", command, str(yaml_path)]) == 0 + + mock_save.assert_called_once_with(fresh_config) + # mtime is now newer than the source YAML, so a follow-up call hits + # the fast path instead of repeating read_config. + assert cache.stat().st_mtime >= yaml_path.stat().st_mtime + + +def test_run_esphome_upload_with_substitution_does_not_refresh_cache( + fresh_cache_files: Path, +) -> None: + """`-s` substitutions skip the cache on both read and write -- saving + here would clobber the cache with a substitution-specific config.""" + with ( + patch("esphome.__main__.read_config", return_value={"esphome": {}}), + patch("esphome.compiled_config.save_compiled_config") as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"upload": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "-s", "var", "val", "upload", str(fresh_cache_files)]) + + mock_save.assert_not_called() + + +def test_run_esphome_compile_does_not_refresh_cache_via_fallback( + fresh_cache_files: Path, +) -> None: + """Compile writes the cache through update_storage_json, not via the + upload/logs fallback path -- the fallback save would skip the + storage_should_clean check.""" + with ( + patch("esphome.__main__.read_config", return_value={"esphome": {}}), + patch("esphome.compiled_config.save_compiled_config") as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"compile": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "compile", str(fresh_cache_files)]) + + mock_save.assert_not_called() + + def test_run_esphome_upload_with_substitution_skips_cache( fresh_cache_files: Path, ) -> None: diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index 8977b05d23..7d6c861ffd 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -203,7 +203,7 @@ def test_generate_idf_component_yml_basic(tmp_component): tmp_component.data = {"description": "test", "repository": {"url": "http://aaa"}} result = generate_idf_component_yml(tmp_component) - assert result == "description: test\nversion: 1.0.0\nrepository: http://aaa\n" + assert result == "description: test\nrepository: http://aaa\n" def test_generate_idf_component_yml_with_dependencies(tmp_component, tmp_path): @@ -217,18 +217,16 @@ def test_generate_idf_component_yml_with_dependencies(tmp_component, tmp_path): assert ( result - == f"""version: 1.0.0 -dependencies: + == f"""dependencies: dep: - version: '1.0' override_path: {dep.path} """ ) -def test_generate_idf_component_yml_missing_path_reraises(tmp_component): - # A dep without a path and without a recognised source should re-raise - # the underlying RuntimeError instead of silently producing a bad manifest. +def test_generate_idf_component_yml_missing_path_raises(tmp_component): + # A dep without a path is a contract violation — every dep is expected + # to have been downloaded before YAML generation. Raise loudly. dep = IDFComponent("foo/bar", "1.0", source=None) tmp_component.dependencies = [dep] @@ -422,15 +420,37 @@ def test_convert_library_with_repository(): result = _convert_library_to_component(lib) assert result.name == "foo/bar" - assert result.version == "1.2.3" + assert result.version == "*" assert isinstance(result.source, GitSource) + assert result.source.ref == "v1.2.3" -def test_convert_library_missing_ref(): +def test_convert_library_with_branch_ref(): + lib = Library("name", None, "https://github.com/foo/bar.git#some-branch") + + result = _convert_library_to_component(lib) + + assert result.name == "foo/bar" + assert result.version == "*" + assert isinstance(result.source, GitSource) + assert result.source.ref == "some-branch" + + +def test_convert_library_missing_ref_uses_default_branch(): + """A bare URL with no #ref clones the remote's default branch. + + Matches PIO's lib_deps behavior and external_components handling -- + git.clone_or_update with ref=None leaves the depth-1 clone on + whatever branch the remote HEAD points at. + """ lib = Library("name", None, "https://github.com/foo/bar.git") - with pytest.raises(ValueError): - _convert_library_to_component(lib) + result = _convert_library_to_component(lib) + + assert result.name == "foo/bar" + assert result.version == "*" + assert isinstance(result.source, GitSource) + assert result.source.ref is None def test_convert_library_registry(monkeypatch): @@ -485,3 +505,113 @@ def test_process_dependencies_skips_invalid(tmp_component): _process_dependencies(tmp_component) assert tmp_component.dependencies == [] + + +def test_process_dependencies_dict_form(tmp_component, monkeypatch): + """PIO library.json shorthand ``{"owner/Name": "version"}`` is honored. + + Iterating a dict gives string keys, which would silently fail the + ``"name" in dependency`` substring check. Normalize to list-of-dicts + first so the dict form (used by e.g. tesla-ble for its nanopb dep) + is treated the same as the verbose list form. + """ + captured: list[Library] = [] + + def fake_generate(library): + captured.append(library) + return IDFComponent( + library.name, library.version, source=URLSource("http://dummy.com") + ) + + tmp_component.data = { + "dependencies": { + "nanopb/Nanopb": "^0.4.91", + "BareName": "1.2.3", + } + } + monkeypatch.setattr( + esphome.espidf.component, "_generate_idf_component", fake_generate + ) + monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None) + + _process_dependencies(tmp_component) + + assert len(tmp_component.dependencies) == 2 + names = sorted(lib.name for lib in captured) + versions = sorted(lib.version for lib in captured) + assert names == ["BareName", "nanopb/Nanopb"] + assert versions == ["1.2.3", "^0.4.91"] + + +def test_process_dependencies_dict_form_with_url_value(tmp_component, monkeypatch): + """A dict-value that's a URL gets routed to ``repository`` like the list form.""" + captured: list[Library] = [] + + def fake_generate(library): + captured.append(library) + return IDFComponent(library.name, "*", source=URLSource("http://dummy.com")) + + tmp_component.data = { + "dependencies": { + "foo/Bar": "https://github.com/foo/bar.git#main", + } + } + monkeypatch.setattr( + esphome.espidf.component, "_generate_idf_component", fake_generate + ) + monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None) + + _process_dependencies(tmp_component) + + assert len(captured) == 1 + assert captured[0].name == "foo/Bar" + assert captured[0].version is None + assert captured[0].repository == "https://github.com/foo/bar.git#main" + + +def test_process_dependencies_dict_form_with_nested_spec(tmp_component, monkeypatch): + """A dict-value that's itself a dict is merged into the entry. + + PIO's library.json allows ``{"owner/Name": {"version": "...", ...}}`` + for entries that need fields beyond just a version (platforms, + frameworks, etc.). The extra fields flow into _check_library_data + via the entry merge. + """ + captured: list[Library] = [] + checked: list[dict] = [] + + def fake_generate(library): + captured.append(library) + return IDFComponent( + library.name, library.version, source=URLSource("http://dummy.com") + ) + + tmp_component.data = { + "dependencies": { + "nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"}, + } + } + monkeypatch.setattr( + esphome.espidf.component, "_generate_idf_component", fake_generate + ) + monkeypatch.setattr( + esphome.espidf.component, + "_check_library_data", + checked.append, + ) + + _process_dependencies(tmp_component) + + assert len(captured) == 1 + assert captured[0].name == "nanopb/Nanopb" + assert captured[0].version == "^0.4.91" + # Extra spec fields reach _check_library_data so platform/framework + # gating still applies. + assert checked == [ + { + "name": "Nanopb", + "owner": "nanopb", + "version": "^0.4.91", + "platforms": "espidf", + } + ] diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py new file mode 100644 index 0000000000..adc8bfce63 --- /dev/null +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -0,0 +1,58 @@ +"""Tests for esphome.espidf.toolchain helpers.""" + +# pylint: disable=protected-access + +from unittest.mock import patch + +from esphome.const import CONF_FRAMEWORK, CONF_SOURCE +from esphome.core import CORE +from esphome.espidf import toolchain + + +def test_get_framework_source_override_no_config(): + """When CORE.config hasn't been set, no override is returned.""" + CORE.config = None + assert toolchain._get_framework_source_override() is None + + +def test_get_framework_source_override_no_esp32_section(): + """A config without an esp32 section yields no override.""" + CORE.config = {} + assert toolchain._get_framework_source_override() is None + + +def test_get_framework_source_override_no_framework_source(): + """An esp32 section without framework.source yields no override.""" + CORE.config = {"esp32": {CONF_FRAMEWORK: {}}} + assert toolchain._get_framework_source_override() is None + + +def test_get_framework_source_override_returns_value(): + """A user-supplied framework source is returned verbatim.""" + url = "https://example.com/esp-idf-v{VERSION}.tar.xz" + CORE.config = {"esp32": {CONF_FRAMEWORK: {CONF_SOURCE: url}}} + assert toolchain._get_framework_source_override() == url + + +def test_get_esphome_esp_idf_paths_forwards_source_override(): + """_get_esphome_esp_idf_paths threads the override into check_esp_idf_install.""" + url = "https://my-mirror/esp-idf-v{VERSION}.tar.xz" + CORE.config = {"esp32": {CONF_FRAMEWORK: {CONF_SOURCE: url}}} + # Hit a fresh cache key so check_esp_idf_install is actually called. + toolchain._cache().paths.clear() + with patch.object( + toolchain, "check_esp_idf_install", return_value=("/fw", "/penv") + ) as mock_install: + toolchain._get_esphome_esp_idf_paths("5.5.4") + mock_install.assert_called_once_with("5.5.4", source_url=url) + + +def test_get_esphome_esp_idf_paths_no_override(): + """When no source override is configured, source_url=None is passed.""" + CORE.config = {} + toolchain._cache().paths.clear() + with patch.object( + toolchain, "check_esp_idf_install", return_value=("/fw", "/penv") + ) as mock_install: + toolchain._get_esphome_esp_idf_paths("5.5.4") + mock_install.assert_called_once_with("5.5.4", source_url=None) diff --git a/tests/unit_tests/test_storage_json.py b/tests/unit_tests/test_storage_json.py index a3a38960e7..ea37492cf4 100644 --- a/tests/unit_tests/test_storage_json.py +++ b/tests/unit_tests/test_storage_json.py @@ -9,7 +9,7 @@ from unittest.mock import MagicMock, Mock, patch import pytest from esphome import storage_json -from esphome.const import CONF_DISABLED, CONF_MDNS +from esphome.const import CONF_DISABLED, CONF_MDNS, Toolchain from esphome.core import CORE @@ -308,6 +308,7 @@ def test_storage_json_from_esphome_core(setup_core: Path) -> None: mock_core.loaded_platforms = {"sensor"} mock_core.config = {CONF_MDNS: {CONF_DISABLED: True}} mock_core.target_framework = "esp-idf" + mock_core.toolchain = Toolchain.ESP_IDF with patch("esphome.components.esp32.get_esp32_variant") as mock_variant: mock_variant.return_value = "ESP32-C3" @@ -327,6 +328,7 @@ def test_storage_json_from_esphome_core(setup_core: Path) -> None: assert result.no_mdns is True assert result.framework == "esp-idf" assert result.core_platform == "esp32" + assert result.toolchain == "esp-idf" def test_storage_json_from_esphome_core_mdns_enabled(setup_core: Path) -> None: @@ -345,10 +347,12 @@ def test_storage_json_from_esphome_core_mdns_enabled(setup_core: Path) -> None: mock_core.loaded_platforms = set() mock_core.config = {} # No MDNS config means enabled mock_core.target_framework = "arduino" + mock_core.toolchain = None result = storage_json.StorageJSON.from_esphome_core(mock_core, old=None) assert result.no_mdns is False + assert result.toolchain is None def test_storage_json_load_valid_file(tmp_path: Path) -> None: @@ -470,6 +474,73 @@ def test_storage_json_equality() -> None: assert storage1 != "not a storage object" +def _make_storage_with_toolchain( + toolchain: str | None, +) -> storage_json.StorageJSON: + return storage_json.StorageJSON( + storage_version=1, + name="dev", + friendly_name=None, + comment=None, + esphome_version="2024.1.0", + src_version=1, + address="dev.local", + web_port=None, + target_platform="ESP32", + build_path=Path("/build"), + firmware_bin_path=Path("/build/firmware.bin"), + loaded_integrations=set(), + loaded_platforms=set(), + no_mdns=False, + framework="esp-idf", + core_platform="esp32", + toolchain=toolchain, + ) + + +def test_storage_json_toolchain_round_trip(setup_core: Path) -> None: + """Sidecar toolchain survives save -> load -> apply_to_core.""" + storage = _make_storage_with_toolchain("esp-idf") + path = setup_core / "storage.json" + path.write_text(storage.to_json()) + + # Serialization key is stable -- device-builder relies on it. + assert json.loads(path.read_text())["toolchain"] == "esp-idf" + + loaded = storage_json.StorageJSON.load(path) + assert loaded is not None + assert loaded.toolchain == "esp-idf" + + CORE.toolchain = None + with patch("esphome.components.esp32.get_esp32_variant"): + loaded.apply_to_core() + assert CORE.toolchain == Toolchain.ESP_IDF + + +def test_storage_json_apply_to_core_preserves_cli_toolchain( + setup_core: Path, +) -> None: + """A CLI-set CORE.toolchain wins over the sidecar value.""" + loaded = _make_storage_with_toolchain("esp-idf") + + CORE.toolchain = Toolchain.PLATFORMIO + with patch("esphome.components.esp32.get_esp32_variant"): + loaded.apply_to_core() + assert CORE.toolchain == Toolchain.PLATFORMIO + + +def test_storage_json_apply_to_core_ignores_unknown_toolchain( + setup_core: Path, +) -> None: + """Unknown enum values (corrupt sidecar / newer ESPHome) fall through to None.""" + loaded = _make_storage_with_toolchain("gcc") + + CORE.toolchain = None + with patch("esphome.components.esp32.get_esp32_variant"): + loaded.apply_to_core() + assert CORE.toolchain is None + + def test_esphome_storage_json_as_dict() -> None: """Test EsphomeStorageJSON.as_dict returns correct dictionary.""" storage = storage_json.EsphomeStorageJSON( diff --git a/tests/unit_tests/test_yaml_util.py b/tests/unit_tests/test_yaml_util.py index e97a188be4..de70a5307d 100644 --- a/tests/unit_tests/test_yaml_util.py +++ b/tests/unit_tests/test_yaml_util.py @@ -34,6 +34,14 @@ def clear_secrets_cache() -> None: yaml_util._SECRET_CACHE.clear() +@pytest.fixture(autouse=True) +def clear_core_frontmatter() -> None: + """Reset CORE.frontmatter between tests.""" + core.CORE.frontmatter = {} + yield + core.CORE.frontmatter = {} + + def test_include_with_vars(fixture_path: Path) -> None: yaml_file = fixture_path / "yaml_util" / "includetest.yaml" @@ -1182,3 +1190,153 @@ def test_track_yaml_loads_records_resolved_paths(tmp_path: Path) -> None: with track_yaml_loads() as loaded: yaml_util.load_yaml(link) assert target.resolve() in loaded + + +# --------------------------------------------------------------------------- +# YAML frontmatter +# --------------------------------------------------------------------------- + + +def test_frontmatter_parsed_and_stored_on_core(tmp_path: Path) -> None: + """A leading `---`-separated YAML document is stored as frontmatter and + stripped from the returned config.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text( + "author: Jesse\nlabels: [office, climate]\n---\nesphome:\n name: my_node\n" + ) + + config = yaml_util.load_yaml(yaml_file) + + # Config does not contain frontmatter keys + assert "author" not in config + assert "labels" not in config + assert config["esphome"]["name"] == "my_node" + + # Frontmatter is stored on CORE keyed by resolved path + frontmatter = core.CORE.frontmatter[yaml_file.resolve()] + assert frontmatter["author"] == "Jesse" + assert frontmatter["labels"] == ["office", "climate"] + + +def test_frontmatter_absent_when_single_document(tmp_path: Path) -> None: + """A YAML file with a single document does not populate CORE.frontmatter.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text("esphome:\n name: my_node\n") + + yaml_util.load_yaml(yaml_file) + assert yaml_file.resolve() not in core.CORE.frontmatter + + +def test_frontmatter_absent_when_leading_doc_separator(tmp_path: Path) -> None: + """A leading `---` with no content above it is just a document start marker, + not frontmatter, and must not populate CORE.frontmatter.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text("---\nesphome:\n name: my_node\n") + + config = yaml_util.load_yaml(yaml_file) + assert config["esphome"]["name"] == "my_node" + assert yaml_file.resolve() not in core.CORE.frontmatter + + +def test_frontmatter_supports_arbitrary_keys(tmp_path: Path) -> None: + """Frontmatter keys are not validated — any structure is accepted.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text( + "any_key: any_value\n" + "nested:\n" + " count: 42\n" + " items:\n" + " - a\n" + " - b\n" + "---\n" + "esphome:\n" + " name: t\n" + ) + + yaml_util.load_yaml(yaml_file) + frontmatter = core.CORE.frontmatter[yaml_file.resolve()] + assert frontmatter["any_key"] == "any_value" + assert frontmatter["nested"]["count"] == 42 + assert frontmatter["nested"]["items"] == ["a", "b"] + + +def test_frontmatter_supports_deeply_nested_paths(tmp_path: Path) -> None: + """Frontmatter preserves deeply nested dict/list structures intact.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text( + "device:\n" + " metadata:\n" + " location:\n" + " building: HQ\n" + " floor: 3\n" + " room:\n" + " number: 302\n" + " occupants:\n" + " - name: Jesse\n" + " role:\n" + " title: maintainer\n" + " since: 2021\n" + " - name: Alice\n" + " role:\n" + " title: contributor\n" + " since: 2024\n" + "---\n" + "esphome:\n" + " name: t\n" + ) + + yaml_util.load_yaml(yaml_file) + fm = core.CORE.frontmatter[yaml_file.resolve()] + room = fm["device"]["metadata"]["location"]["room"] + assert room["number"] == 302 + assert room["occupants"][0]["name"] == "Jesse" + assert room["occupants"][0]["role"]["title"] == "maintainer" + assert room["occupants"][0]["role"]["since"] == 2021 + assert room["occupants"][1]["role"]["title"] == "contributor" + + +def test_frontmatter_more_than_two_documents_raises(tmp_path: Path) -> None: + """Three or more YAML documents is unsupported and must raise.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text("a: 1\n---\nb: 2\n---\nc: 3\n") + + with pytest.raises(EsphomeError, match="at most two are supported"): + yaml_util.load_yaml(yaml_file) + + +def test_frontmatter_empty_frontmatter_doc_not_stored(tmp_path: Path) -> None: + """An empty (null) frontmatter document is treated as no frontmatter.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text("---\n---\nesphome:\n name: t\n") + + config = yaml_util.load_yaml(yaml_file) + assert config["esphome"]["name"] == "t" + assert yaml_file.resolve() not in core.CORE.frontmatter + + +def test_frontmatter_empty_config_doc(tmp_path: Path) -> None: + """An empty config document after a frontmatter document yields an empty config.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text("only: frontmatter\n---\n") + + config = yaml_util.load_yaml(yaml_file) + assert config == {} + assert core.CORE.frontmatter[yaml_file.resolve()]["only"] == "frontmatter" + + +def test_frontmatter_included_file_stored(tmp_path: Path) -> None: + """Frontmatter on an !include'd file is also captured on CORE, keyed by + that file's resolved path.""" + inc = tmp_path / "child.yaml" + inc.write_text("child_meta: hello\n---\nchild_key: value\n") + main = tmp_path / "main.yaml" + main.write_text("esphome:\n name: t\nchild: !include child.yaml\n") + + config = yaml_util.load_yaml(main) + # !include is deferred; force resolution so the child file actually loads + force_load_include_files(config) + assert config["child"].load()["child_key"] == "value" + # Main file has no frontmatter + assert main.resolve() not in core.CORE.frontmatter + # Included file's frontmatter is captured + assert core.CORE.frontmatter[inc.resolve()]["child_meta"] == "hello"