diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml index 52d72544d3..2081264b91 100644 --- a/.github/actions/build-image/action.yaml +++ b/.github/actions/build-image/action.yaml @@ -47,7 +47,7 @@ runs: - name: Build and push to ghcr by digest id: build-ghcr - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 env: DOCKER_BUILD_SUMMARY: false DOCKER_BUILD_RECORD_UPLOAD: false @@ -73,7 +73,7 @@ runs: - name: Build and push to dockerhub by digest id: build-dockerhub - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 env: DOCKER_BUILD_SUMMARY: false DOCKER_BUILD_RECORD_UPLOAD: false diff --git a/.github/actions/restore-python/action.yml b/.github/actions/restore-python/action.yml index 751f9ecf58..03b4803860 100644 --- a/.github/actions/restore-python/action.yml +++ b/.github/actions/restore-python/action.yml @@ -35,6 +35,10 @@ runs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true + # Pin uv version so the action does not have to fetch the + # manifest from raw.githubusercontent.com on every cache + # miss; that fetch flakes on Windows runners. + version: "0.11.15" - name: Create Python virtual environment if: steps.cache-venv.outputs.cache-hit != 'true' && runner.os != 'Windows' shell: bash diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 528e69c478..e87939f824 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,7 @@ updates: directory: "/" schedule: interval: daily + open-pull-requests-limit: 10 ignore: # Hypotehsis is only used for testing and is updated quite often - dependency-name: hypothesis diff --git a/.github/workflows/ci-api-proto.yml b/.github/workflows/ci-api-proto.yml index 1dc0ccb7fe..675bbe9d2c 100644 --- a/.github/workflows/ci-api-proto.yml +++ b/.github/workflows/ci-api-proto.yml @@ -32,6 +32,10 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true + # Pin uv version so the action does not have to fetch the + # manifest from raw.githubusercontent.com on every cache + # miss; that fetch flakes on Windows runners. + version: "0.11.15" - name: Install apt dependencies run: | diff --git a/.github/workflows/ci-docker.yml b/.github/workflows/ci-docker.yml index 3fd17888c7..89fbec5420 100644 --- a/.github/workflows/ci-docker.yml +++ b/.github/workflows/ci-docker.yml @@ -48,7 +48,7 @@ jobs: with: python-version: "3.11" - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Set TAG run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de21456841..53516db913 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,14 +6,6 @@ on: branches: [dev, beta, release] pull_request: - paths: - - "**" - - "!.github/workflows/*.yml" - - "!.github/actions/build-image/*" - - ".github/workflows/ci.yml" - - "!.yamllint" - - "!.github/dependabot.yml" - - "!docker/**" merge_group: permissions: @@ -60,6 +52,10 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true + # Pin uv version so the action does not have to fetch the + # manifest from raw.githubusercontent.com on every cache + # miss; that fetch flakes on Windows runners. + version: "0.11.15" - name: Create Python virtual environment if: steps.cache-venv.outputs.cache-hit != 'true' run: | @@ -97,6 +93,8 @@ jobs: runs-on: ubuntu-24.04 needs: - common + - determine-jobs + if: needs.determine-jobs.outputs.core-ci == 'true' steps: - name: Check out code from GitHub uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -175,6 +173,10 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true + # Pin uv version so the action does not have to fetch the + # manifest from raw.githubusercontent.com on every cache + # miss; that fetch flakes on Windows runners. + version: "0.11.15" - name: Install device-builder + esphome from PR # Install device-builder with its esphome + test extras # first so its pinned versions of pytest/etc. land, then @@ -215,6 +217,8 @@ jobs: runs-on: ${{ matrix.os }} needs: - common + - determine-jobs + if: needs.determine-jobs.outputs.core-ci == 'true' steps: - name: Check out code from GitHub uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -237,7 +241,7 @@ jobs: . venv/bin/activate pytest -vv --cov-report=xml --tb=native --durations=30 -n auto tests --ignore=tests/integration/ - name: Upload coverage to Codecov - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ secrets.CODECOV_TOKEN }} - name: Save Python virtual environment cache @@ -253,6 +257,7 @@ jobs: needs: - common outputs: + core-ci: ${{ steps.determine.outputs.core-ci }} integration-tests: ${{ steps.determine.outputs.integration-tests }} integration-test-buckets: ${{ steps.determine.outputs.integration-test-buckets }} clang-tidy: ${{ steps.determine.outputs.clang-tidy }} @@ -306,6 +311,7 @@ jobs: echo "$output" | jq # Extract individual fields + echo "core-ci=$(echo "$output" | jq -r '.core_ci')" >> $GITHUB_OUTPUT echo "integration-tests=$(echo "$output" | jq -r '.integration_tests')" >> $GITHUB_OUTPUT echo "integration-test-buckets=$(echo "$output" | jq -c '.integration_test_buckets')" >> $GITHUB_OUTPUT echo "clang-tidy=$(echo "$output" | jq -r '.clang_tidy')" >> $GITHUB_OUTPUT @@ -365,6 +371,10 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true + # Pin uv version so the action does not have to fetch the + # manifest from raw.githubusercontent.com on every cache + # miss; that fetch flakes on Windows runners. + version: "0.11.15" - name: Create Python virtual environment if: steps.cache-venv.outputs.cache-hit != 'true' run: | @@ -957,7 +967,8 @@ jobs: runs-on: ubuntu-latest needs: - common - if: github.event_name == 'pull_request' && !startsWith(github.base_ref, 'beta') && !startsWith(github.base_ref, 'release') + - determine-jobs + if: github.event_name == 'pull_request' && !startsWith(github.base_ref, 'beta') && !startsWith(github.base_ref, 'release') && needs.determine-jobs.outputs.core-ci == 'true' steps: - name: Check out code from GitHub uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0a4dd9a92d..dfc0e08bfa 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,7 +56,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4 + uses: github/codeql-action/init@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0 with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} @@ -84,6 +84,6 @@ jobs: exit 1 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4 + uses: github/codeql-action/analyze@7211b7c8077ea37d8641b6271f6a365a22a5fbfa # v4.36.0 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c1086c858c..344bd416c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -99,15 +99,15 @@ jobs: python-version: "3.11" - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Log in to docker hub - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to the GitHub container registry - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -178,17 +178,17 @@ jobs: merge-multiple: true - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 - name: Log in to docker hub if: matrix.registry == 'dockerhub' - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to the GitHub container registry if: matrix.registry == 'ghcr' - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 with: registry: ghcr.io username: ${{ github.actor }} @@ -212,74 +212,6 @@ jobs: docker buildx imagetools create $(jq -Rcnr 'inputs | . / "," | map("-t " + .) | join(" ")' <<< "${{ steps.tags.outputs.tags}}") \ $(printf '${{ steps.tags.outputs.image }}@sha256:%s ' *) - deploy-ha-addon-repo: - if: github.repository == 'esphome/esphome' && needs.init.outputs.branch_build == 'false' - runs-on: ubuntu-latest - needs: - - init - - deploy-manifest - steps: - - name: Generate a token - id: generate-token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.ESPHOME_GITHUB_APP_CLIENT_ID }} - private-key: ${{ secrets.ESPHOME_GITHUB_APP_PRIVATE_KEY }} - owner: esphome - repositories: home-assistant-addon - permission-actions: write # actions.createWorkflowDispatch on the target repo (only API call made with this token) - - - name: Trigger Workflow - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ steps.generate-token.outputs.token }} - script: | - let description = "ESPHome"; - if (context.eventName == "release") { - description = ${{ toJSON(github.event.release.body) }}; - } - github.rest.actions.createWorkflowDispatch({ - owner: "esphome", - repo: "home-assistant-addon", - workflow_id: "bump-version.yml", - ref: "main", - inputs: { - version: "${{ needs.init.outputs.tag }}", - content: description - } - }) - - deploy-esphome-schema: - if: github.repository == 'esphome/esphome' && needs.init.outputs.branch_build == 'false' - runs-on: ubuntu-latest - needs: [init] - environment: ${{ needs.init.outputs.deploy_env }} - steps: - - name: Generate a token - id: generate-token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.ESPHOME_GITHUB_APP_CLIENT_ID }} - private-key: ${{ secrets.ESPHOME_GITHUB_APP_PRIVATE_KEY }} - owner: esphome - repositories: esphome-schema - permission-actions: write # actions.createWorkflowDispatch on the target repo (only API call made with this token) - - - name: Trigger Workflow - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ steps.generate-token.outputs.token }} - script: | - github.rest.actions.createWorkflowDispatch({ - owner: "esphome", - repo: "esphome-schema", - workflow_id: "generate-schemas.yml", - ref: "main", - inputs: { - version: "${{ needs.init.outputs.tag }}", - } - }) - version-notifier: if: github.repository == 'esphome/esphome' && needs.init.outputs.branch_build == 'false' runs-on: ubuntu-latest @@ -302,7 +234,7 @@ jobs: with: github-token: ${{ steps.generate-token.outputs.token }} script: | - github.rest.actions.createWorkflowDispatch({ + await github.rest.actions.createWorkflowDispatch({ owner: "esphome", repo: "version-notifier", workflow_id: "notify.yml", diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 2e57093bbb..7003f6c482 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Stale - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 + uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 with: debug-only: ${{ github.ref != 'refs/heads/dev' }} # Dry-run when not run on dev branch remove-stale-when-updated: true diff --git a/.github/workflows/sync-device-classes.yml b/.github/workflows/sync-device-classes.yml index 23a63c5d8a..84be3c8e22 100644 --- a/.github/workflows/sync-device-classes.yml +++ b/.github/workflows/sync-device-classes.yml @@ -50,6 +50,10 @@ jobs: uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: enable-cache: true + # Pin uv version so the action does not have to fetch the + # manifest from raw.githubusercontent.com on every cache + # miss; that fetch flakes on Windows runners. + version: "0.11.15" - name: Install Home Assistant run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da5fb94d5e..0470a948f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.15.12 + rev: v0.15.14 hooks: # Run the linter. - id: ruff diff --git a/CODEOWNERS b/CODEOWNERS index f9a4d811ca..089b5b82ea 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -417,6 +417,7 @@ esphome/components/restart/* @esphome/core esphome/components/rf_bridge/* @jesserockz esphome/components/rgbct/* @jesserockz esphome/components/ring_buffer/* @kahrendt +esphome/components/router/speaker/* @kahrendt esphome/components/rp2040/* @jesserockz esphome/components/rp2040_ble/* @bdraco esphome/components/rp2040_pio_led_strip/* @Papa-DMan diff --git a/esphome/__main__.py b/esphome/__main__.py index d733534a5c..5f281ce832 100644 --- a/esphome/__main__.py +++ b/esphome/__main__.py @@ -50,6 +50,7 @@ from esphome.const import ( CONF_TOPIC, CONF_USERNAME, CONF_WEB_SERVER, + CONF_WIFI, ENV_NOGITIGNORE, KEY_CORE, KEY_TARGET_PLATFORM, @@ -607,7 +608,7 @@ def run_miniterm(config: ConfigType, port: str, args) -> int: try: module = importlib.import_module("esphome.components." + CORE.target_platform) - process_stacktrace = getattr(module, "process_stacktrace") + process_stacktrace = module.process_stacktrace except (AttributeError, ImportError): _LOGGER.info( 'Stacktrace analysis is unavailable: no compatible analyzer found for target platform "%s".', @@ -733,6 +734,13 @@ def write_cpp_file() -> int: def compile_program(args: ArgsProtocol, config: ConfigType) -> int: + # Keep this gate here, NOT in config validation: device-builder needs + # `esphome config` to keep succeeding with placeholders so onboarding can run. + if CONF_WIFI in config: + from esphome.components.wifi import check_placeholder_credentials + + check_placeholder_credentials(config) + # NOTE: "Build path:" format is parsed by script/ci_memory_impact_extract.py # If you change this format, update the regex in that script as well _LOGGER.info("Compiling app... Build path: %s", CORE.build_path) @@ -786,7 +794,7 @@ def _check_and_emit_build_info() -> None: # Read build_info from JSON try: - with open(build_info_json_path, encoding="utf-8") as f: + with build_info_json_path.open(encoding="utf-8") as f: build_info = json.load(f) except (OSError, json.JSONDecodeError) as e: _LOGGER.debug("Failed to read build_info: %s", e) @@ -1048,7 +1056,7 @@ def _wait_for_serial_port( def _port_found() -> bool: if port is not None: if os.name == "posix": - return os.path.exists(port) + return Path(port).exists() return any(p.path == port for p in get_serial_ports()) ports = get_serial_ports() if known_ports is not None: @@ -1093,7 +1101,7 @@ def upload_program( host = devices[0] try: module = importlib.import_module("esphome.components." + CORE.target_platform) - if getattr(module, "upload_program")(config, args, host): + if module.upload_program(config, args, host): return 0, host except AttributeError: pass @@ -1345,7 +1353,7 @@ def _validate_bootloader_binary(binary: Path) -> None: def show_logs(config: ConfigType, args: ArgsProtocol, devices: list[str]) -> int | None: try: module = importlib.import_module("esphome.components." + CORE.target_platform) - if getattr(module, "show_logs")(config, args, devices): + if module.show_logs(config, args, devices): return 0 except AttributeError: pass @@ -2441,7 +2449,10 @@ def run_esphome(argv): # Skipped when -s overrides are passed, since the cache was written # against the previous substitution set. config: ConfigType | None = None - if args.command in ("upload", "logs") and not command_line_substitutions: + cache_eligible = ( + args.command in ("upload", "logs") and not command_line_substitutions + ) + if cache_eligible: from esphome.compiled_config import load_compiled_config config = load_compiled_config(conf_path) @@ -2456,6 +2467,16 @@ def run_esphome(argv): command_line_substitutions, skip_external_update=skip_external, ) + # Refresh the cache so the next upload/logs hits the fast path + # instead of re-running read_config. Skip when the storage + # sidecar is absent (no compile has run): the cache would + # never be loaded back, so writing secrets to disk is wasted. + if cache_eligible and config is not None: + from esphome.compiled_config import save_compiled_config + from esphome.storage_json import ext_storage_path + + if ext_storage_path(conf_path.name).exists(): + save_compiled_config(config) if config is None: return 2 CORE.config = config diff --git a/esphome/analyze_memory/cli.py b/esphome/analyze_memory/cli.py index 8f1f39e1d6..4fbceb7e5e 100644 --- a/esphome/analyze_memory/cli.py +++ b/esphome/analyze_memory/cli.py @@ -6,6 +6,7 @@ from collections import defaultdict from collections.abc import Callable import heapq from operator import itemgetter +from pathlib import Path import sys from typing import TYPE_CHECKING @@ -509,7 +510,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append( f"{_COMPONENT_CORE} Symbols > {self.SYMBOL_SIZE_THRESHOLD} B ({len(large_core_symbols)} symbols):" ) - for i, (symbol, demangled, size) in enumerate(large_core_symbols): + for i, (_symbol, demangled, size) in enumerate(large_core_symbols): # Core symbols only track (symbol, demangled, size) without section info, # so we don't show section labels here lines.append( @@ -601,7 +602,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append( f"{comp_name} Symbols > {self.SYMBOL_SIZE_THRESHOLD} B & storage ({len(large_symbols)} symbols):" ) - for i, (symbol, demangled, size, section) in enumerate(large_symbols): + for i, (_symbol, demangled, size, section) in enumerate(large_symbols): lines.append( f"{i + 1}. {self._format_symbol_with_section(demangled, size, section)}" ) @@ -640,7 +641,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): lines.append( f" Symbols > {self.RAM_SYMBOL_SIZE_THRESHOLD} B ({len(large_ram_syms)}):" ) - for symbol, demangled, size, section in large_ram_syms[:10]: + for _symbol, demangled, size, section in large_ram_syms[:10]: # Format section label consistently by stripping leading dot section_label = section.lstrip(".") if section else "" display_name = _format_pstorage_name(demangled) @@ -699,7 +700,7 @@ class MemoryAnalyzerCLI(MemoryAnalyzer): content = "\n".join(lines) if output_file: - with open(output_file, "w", encoding="utf-8") as f: + with Path(output_file).open("w", encoding="utf-8") as f: f.write(content) else: print(content) @@ -737,7 +738,6 @@ def main(): # Load build directory import json - from pathlib import Path from esphome.platformio.toolchain import IDEData @@ -785,7 +785,7 @@ def main(): if not idedata_path.exists(): continue try: - with open(idedata_path, encoding="utf-8") as f: + with idedata_path.open(encoding="utf-8") as f: raw_data = json.load(f) idedata = IDEData(raw_data) print(f"Loaded idedata from: {idedata_path}", file=sys.stderr) diff --git a/esphome/analyze_memory/demangle.py b/esphome/analyze_memory/demangle.py index 8999108b51..7dbd6d4f63 100644 --- a/esphome/analyze_memory/demangle.py +++ b/esphome/analyze_memory/demangle.py @@ -154,7 +154,7 @@ def batch_demangle( failed_count = 0 for original, stripped, prefix, demangled in zip( - symbols, symbols_stripped, symbols_prefixes, demangled_lines + symbols, symbols_stripped, symbols_prefixes, demangled_lines, strict=True ): # Add back any prefix that was removed demangled = _restore_symbol_prefix(prefix, stripped, demangled) diff --git a/esphome/analyze_memory/toolchain.py b/esphome/analyze_memory/toolchain.py index 3a8a5f7be4..a724d52f25 100644 --- a/esphome/analyze_memory/toolchain.py +++ b/esphome/analyze_memory/toolchain.py @@ -3,7 +3,6 @@ from __future__ import annotations import logging -import os from pathlib import Path import subprocess from typing import TYPE_CHECKING @@ -37,7 +36,7 @@ def _find_in_platformio_packages(tool_name: str) -> str | None: Full path to the tool or None if not found """ # Get PlatformIO packages directory - platformio_home = Path(os.path.expanduser("~/.platformio/packages")) + platformio_home = Path("~/.platformio/packages").expanduser() if not platformio_home.exists(): return None diff --git a/esphome/build_gen/espidf.py b/esphome/build_gen/espidf.py index 96f84ebbd1..0b50f72382 100644 --- a/esphome/build_gen/espidf.py +++ b/esphome/build_gen/espidf.py @@ -24,7 +24,7 @@ def get_available_components() -> list[str] | None: return None try: - with open(project_desc, encoding="utf-8") as f: + with project_desc.open(encoding="utf-8") as f: data = json.load(f) component_info = data.get("build_component_info", {}) diff --git a/esphome/bundle.py b/esphome/bundle.py index 4537cbce9d..d38f68ebfd 100644 --- a/esphome/bundle.py +++ b/esphome/bundle.py @@ -412,7 +412,7 @@ class ConfigBundleCreator: @staticmethod def _add_to_tar(tar: tarfile.TarFile, bf: BundleFile) -> None: """Add a BundleFile to the tar archive with deterministic metadata.""" - with open(bf.source, "rb") as f: + with bf.source.open("rb") as f: _add_bytes_to_tar(tar, bf.path, f.read()) diff --git a/esphome/components/api/api_connection.cpp b/esphome/components/api/api_connection.cpp index df2380898b..7453da3771 100644 --- a/esphome/components/api/api_connection.cpp +++ b/esphome/components/api/api_connection.cpp @@ -1,5 +1,6 @@ #include "api_connection.h" #ifdef USE_API +#include "api_connection_buffer.h" // for encode_to_buffer / get_batch_delay_ms_ inlines #ifdef USE_API_NOISE #include "api_frame_helper_noise.h" #endif @@ -1242,7 +1243,7 @@ void APIConnection::try_send_store_yaml_() { void APIConnection::on_get_time_response(const GetTimeResponse &value) { if (homeassistant::global_homeassistant_time != nullptr) { homeassistant::global_homeassistant_time->set_epoch_time(value.epoch_seconds); -#ifdef USE_TIME_TIMEZONE +#if defined(USE_HOMEASSISTANT_TIMEZONE) && defined(USE_TIME_TIMEZONE) if (!value.timezone.empty()) { // Check if the sender provided pre-parsed timezone data. // If std_offset is non-zero or DST rules are present, the parsed data was populated. diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index ca24cbd9a4..965ce65533 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -11,7 +11,8 @@ #endif #include "api_pb2.h" #include "api_pb2_service.h" -#include "api_server.h" +#include "list_entities.h" +#include "subscribe_state.h" #include "esphome/core/application.h" #include "esphome/core/component.h" #ifdef USE_ESP32_CRASH_HANDLER @@ -36,6 +37,9 @@ class ComponentIterator; namespace esphome::api { +// Forward-declared to break the api_server.h cycle; full-type inlines are in api_connection_buffer.h. +class APIServer; + // Keepalive timeout in milliseconds static constexpr uint32_t KEEPALIVE_TIMEOUT_MS = 60000; // Maximum number of entities to process in a single batch during initial state/info sending @@ -420,44 +424,10 @@ class APIConnection final : public APIServerConnectionBase { // Non-template buffer management for send_message bool send_message_(uint32_t payload_size, uint8_t message_type, MessageEncodeFn encode_fn, const void *msg); - // Core batch encoding logic. Computes header size, checks fit, resizes buffer, encodes. - // ALWAYS_INLINE so the compiler can devirtualize encode_fn at hot call sites. - static inline uint16_t ESPHOME_ALWAYS_INLINE encode_to_buffer(uint32_t calculated_size, MessageEncodeFn encode_fn, - const void *msg, APIConnection *conn, - uint32_t remaining_size) { -#ifdef HAS_PROTO_MESSAGE_DUMP - if (conn->flags_.log_only_mode) { - auto *proto_msg = static_cast(msg); - DumpBuffer dump_buf; - conn->log_send_message_(proto_msg->message_name(), proto_msg->dump_to(dump_buf)); - return 1; - } -#endif - const uint8_t footer_size = conn->helper_->frame_footer_size(); - - // First message uses max padding (already in buffer), subsequent use exact header size - size_t to_add; - if (conn->flags_.batch_first_message) { - conn->flags_.batch_first_message = false; - conn->batch_header_size_ = conn->helper_->frame_header_padding(); - to_add = calculated_size; - } else { - conn->batch_header_size_ = conn->helper_->frame_header_size(calculated_size, conn->batch_message_type_); - to_add = calculated_size + conn->batch_header_size_ + footer_size; - } - - // Check if it fits (using actual header size, not max padding) - uint16_t total_calculated_size = calculated_size + conn->batch_header_size_ + footer_size; - if (total_calculated_size > remaining_size) - return 0; - - auto &shared_buf = conn->parent_->get_shared_buffer_ref(); - shared_buf.resize(shared_buf.size() + to_add); - ProtoWriteBuffer buffer{&shared_buf, shared_buf.size() - calculated_size}; - encode_fn(msg, buffer PROTO_ENCODE_DEBUG_INIT(&shared_buf)); - - return total_calculated_size; - } + // Core batch encoding logic. ALWAYS_INLINE so encode_fn devirtualizes at hot call sites. + // Defined in api_connection_buffer.h (needs APIServer complete). + static uint16_t ESPHOME_ALWAYS_INLINE encode_to_buffer(uint32_t calculated_size, MessageEncodeFn encode_fn, + const void *msg, APIConnection *conn, uint32_t remaining_size); // Noinline version of encode_to_buffer for cold paths (entity info, zero-payload messages). // All cold callers share this single copy instead of each getting an ALWAYS_INLINE expansion. @@ -801,7 +771,8 @@ class APIConnection final : public APIServerConnectionBase { // Read by process_batch_multi_ to pass into MessageInfo. uint8_t batch_header_size_{0}; - uint32_t get_batch_delay_ms_() const { return this->parent_->get_batch_delay(); } + // Defined in api_connection_buffer.h (needs APIServer complete). + uint32_t get_batch_delay_ms_() const; // Message will use 8 more bytes than the minimum size, and typical // MTU is 1500. Sometimes users will see as low as 1460 MTU. // If its IPv6 the header is 40 bytes, and if its IPv4 diff --git a/esphome/components/api/api_connection_buffer.h b/esphome/components/api/api_connection_buffer.h new file mode 100644 index 0000000000..1dd8a162e4 --- /dev/null +++ b/esphome/components/api/api_connection_buffer.h @@ -0,0 +1,54 @@ +#pragma once + +#include "esphome/core/defines.h" +#ifdef USE_API + +// Inline APIConnection methods that need APIServer complete. Include this +// instead of api_connection.h when calling encode_to_buffer or get_batch_delay_ms_. + +#include "api_connection.h" +#include "api_server.h" + +namespace esphome::api { + +inline uint16_t ESPHOME_ALWAYS_INLINE APIConnection::encode_to_buffer(uint32_t calculated_size, + MessageEncodeFn encode_fn, const void *msg, + APIConnection *conn, uint32_t remaining_size) { +#ifdef HAS_PROTO_MESSAGE_DUMP + if (conn->flags_.log_only_mode) { + auto *proto_msg = static_cast(msg); + DumpBuffer dump_buf; + conn->log_send_message_(proto_msg->message_name(), proto_msg->dump_to(dump_buf)); + return 1; + } +#endif + const uint8_t footer_size = conn->helper_->frame_footer_size(); + + // First message uses max padding (already in buffer), subsequent use exact header size + size_t to_add; + if (conn->flags_.batch_first_message) { + conn->flags_.batch_first_message = false; + conn->batch_header_size_ = conn->helper_->frame_header_padding(); + to_add = calculated_size; + } else { + conn->batch_header_size_ = conn->helper_->frame_header_size(calculated_size, conn->batch_message_type_); + to_add = calculated_size + conn->batch_header_size_ + footer_size; + } + + // Check if it fits (using actual header size, not max padding) + uint16_t total_calculated_size = calculated_size + conn->batch_header_size_ + footer_size; + if (total_calculated_size > remaining_size) + return 0; + + auto &shared_buf = conn->parent_->get_shared_buffer_ref(); + shared_buf.resize(shared_buf.size() + to_add); + ProtoWriteBuffer buffer{&shared_buf, shared_buf.size() - calculated_size}; + encode_fn(msg, buffer PROTO_ENCODE_DEBUG_INIT(&shared_buf)); + + return total_calculated_size; +} + +inline uint32_t APIConnection::get_batch_delay_ms_() const { return this->parent_->get_batch_delay(); } + +} // namespace esphome::api +#endif diff --git a/esphome/components/api/api_server.cpp b/esphome/components/api/api_server.cpp index 6c26c4e187..031fa342c1 100644 --- a/esphome/components/api/api_server.cpp +++ b/esphome/components/api/api_server.cpp @@ -1,6 +1,7 @@ #include "api_server.h" #ifdef USE_API #include +#include #include "api_connection.h" #include "esphome/components/network/util.h" #include "esphome/core/application.h" @@ -30,11 +31,6 @@ APIServer *global_api_server = nullptr; // NOLINT(cppcoreguidelines-avoid-non-c APIServer::APIServer() { global_api_server = this; } -// Custom deleter defined here so `delete` sees the complete APIConnection type. -// This prevents libc++ from emitting an "incomplete type" error when other -// translation units only have the forward declaration of APIConnection. -void APIServer::APIConnectionDeleter::operator()(APIConnection *p) const { delete p; } - void APIServer::socket_failed_(const LogString *msg) { ESP_LOGW(TAG, "Socket %s: errno %d", LOG_STR_ARG(msg), errno); this->destroy_socket_(); @@ -682,7 +678,7 @@ uint32_t APIServer::register_active_action_call(uint32_t client_call_id, APIConn // Schedule automatic cleanup after timeout (client will have given up by then) // Uses numeric ID overload to avoid heap allocation from str_sprintf this->set_timeout(action_call_id, USE_API_ACTION_CALL_TIMEOUT_MS, [this, action_call_id]() { - ESP_LOGD(TAG, "Action call %u timed out", action_call_id); + ESP_LOGD(TAG, "Action call %" PRIu32 " timed out", action_call_id); this->unregister_active_action_call(action_call_id); }); @@ -726,7 +722,7 @@ void APIServer::send_action_response(uint32_t action_call_id, bool success, Stri return; } } - ESP_LOGW(TAG, "Cannot send response: no active call found for action_call_id %u", action_call_id); + ESP_LOGW(TAG, "Cannot send response: no active call found for action_call_id %" PRIu32, action_call_id); } #ifdef USE_API_USER_DEFINED_ACTION_RESPONSES_JSON void APIServer::send_action_response(uint32_t action_call_id, bool success, StringRef error_message, @@ -738,7 +734,7 @@ void APIServer::send_action_response(uint32_t action_call_id, bool success, Stri return; } } - ESP_LOGW(TAG, "Cannot send response: no active call found for action_call_id %u", action_call_id); + ESP_LOGW(TAG, "Cannot send response: no active call found for action_call_id %" PRIu32, action_call_id); } #endif // USE_API_USER_DEFINED_ACTION_RESPONSES_JSON #endif // USE_API_USER_DEFINED_ACTION_RESPONSES diff --git a/esphome/components/api/api_server.h b/esphome/components/api/api_server.h index 6b575e536d..fbc8115091 100644 --- a/esphome/components/api/api_server.h +++ b/esphome/components/api/api_server.h @@ -3,6 +3,8 @@ #include "esphome/core/defines.h" #ifdef USE_API #include "api_buffer.h" +// Must precede clients_ so APIConnection is complete for default_delete (libc++). +#include "api_connection.h" #include "api_noise_context.h" #include "api_pb2.h" #include "api_pb2_service.h" @@ -12,8 +14,6 @@ #include "esphome/core/controller.h" #include "esphome/core/log.h" #include "esphome/core/string_ref.h" -#include "list_entities.h" -#include "subscribe_state.h" #ifdef USE_LOGGER #include "esphome/components/logger/logger.h" #endif @@ -191,15 +191,9 @@ class APIServer final : public Component, bool is_connected_with_state_subscription() const; // Range-for view over the populated slice [0, api_connection_count_). Read-only with respect - // to ownership — callers get `const unique_ptr&` so they can invoke non-const methods on the + // to ownership; callers get `const unique_ptr&` so they can invoke non-const methods on the // APIConnection but cannot reset/move the slot and break the count invariant. - // Custom deleter is defined out-of-line in api_server.cpp so libc++ does not - // eagerly instantiate `delete static_cast(p)` here, where - // only the forward declaration of APIConnection is visible (incomplete type). - struct APIConnectionDeleter { - void operator()(APIConnection *p) const; - }; - using APIConnectionPtr = std::unique_ptr; + using APIConnectionPtr = std::unique_ptr; class ActiveClientsView { const APIConnectionPtr *begin_; const APIConnectionPtr *end_; diff --git a/esphome/components/api/client.py b/esphome/components/api/client.py index d6150fbd29..327973a605 100644 --- a/esphome/components/api/client.py +++ b/esphome/components/api/client.py @@ -101,13 +101,14 @@ async def async_run_logs( client_info=f"ESPHome Logs {__version__}", noise_psk=noise_psk, addresses=addresses, # Pass all addresses for automatic retry + provide_time=False, ) # Try platform-specific stacktrace handler first, fall back to generic platform_process_stacktrace = None try: module = importlib.import_module("esphome.components." + CORE.target_platform) - platform_process_stacktrace = getattr(module, "process_stacktrace") + platform_process_stacktrace = module.process_stacktrace except (AttributeError, ImportError): _LOGGER.info( 'Stacktrace analysis is unavailable: no compatible analyzer found for target platform "%s".', diff --git a/esphome/components/as5600/__init__.py b/esphome/components/as5600/__init__.py index 444306cec3..c05e556376 100644 --- a/esphome/components/as5600/__init__.py +++ b/esphome/components/as5600/__init__.py @@ -100,7 +100,7 @@ def position(min=-MAX_POSITION, max=MAX_POSITION): if isinstance(value, str) and value.endswith("%"): value = percent_to_position(value) - if isinstance(value, str) and (value.endswith("°") or value.endswith("deg")): + if isinstance(value, str) and value.endswith(("°", "deg")): return angle_to_position( value, min=round(min * POSITION_TO_ANGLE), diff --git a/esphome/components/audio/__init__.py b/esphome/components/audio/__init__.py index 13b379ba3a..c9775ab601 100644 --- a/esphome/components/audio/__init__.py +++ b/esphome/components/audio/__init__.py @@ -335,7 +335,7 @@ async def to_code(config): add_idf_component( name="esphome/esp-audio-libs", - ref="3.0.0", + ref="3.1.0", ) data = _get_data() diff --git a/esphome/components/audio/audio_decoder.cpp b/esphome/components/audio/audio_decoder.cpp index d4ff59fc36..f709c23fb6 100644 --- a/esphome/components/audio/audio_decoder.cpp +++ b/esphome/components/audio/audio_decoder.cpp @@ -9,9 +9,12 @@ namespace esphome::audio { static const char *const TAG = "audio.decoder"; -static const uint32_t DECODING_TIMEOUT_MS = 50; // The decode function will yield after this duration static const uint32_t READ_WRITE_TIMEOUT_MS = 20; // Timeout for transferring audio data +// Max consecutive decode iterations that consume input but produce no output; e.g., skipping a large metadata block, +// before yielding and returning. +static const uint8_t MAX_NO_OUTPUT_ITERATIONS = 32; + static const uint32_t MAX_POTENTIALLY_FAILED_COUNT = 10; AudioDecoder::AudioDecoder(size_t input_buffer_size, size_t output_buffer_size) @@ -20,11 +23,13 @@ AudioDecoder::AudioDecoder(size_t input_buffer_size, size_t output_buffer_size) } esp_err_t AudioDecoder::add_source(std::weak_ptr &input_ring_buffer) { - auto source = AudioSourceTransferBuffer::create(this->input_buffer_size_); + // Zero-copy source reading directly from the ring buffer's internal storage. Raw file data is byte + // aligned, so no frame alignment is required. + auto source = RingBufferAudioSource::create(input_ring_buffer.lock(), this->input_buffer_size_); if (source == nullptr) { - return ESP_ERR_NO_MEM; + // create() only returns nullptr for invalid arguments (expired ring buffer or zero buffer size) + return ESP_ERR_INVALID_ARG; } - source->set_source(input_ring_buffer); this->input_buffer_ = std::move(source); return ESP_OK; } @@ -141,13 +146,7 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { } FileDecoderState state = FileDecoderState::MORE_TO_PROCESS; - - uint32_t decoding_start = millis(); - - bool first_loop_iteration = true; - - size_t bytes_processed = 0; - size_t bytes_available_before_processing = 0; + uint8_t no_output_iterations = 0; while (state == FileDecoderState::MORE_TO_PROCESS) { // Transfer decoded out @@ -161,45 +160,39 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { this->playback_ms_ += this->audio_stream_info_.value().frames_to_milliseconds_with_remainder(&this->accumulated_frames_written_); } + + if ((bytes_written > 0) && (this->output_transfer_buffer_->available() == 0)) { + // All decoded audio has been flushed to the sink; return so the caller can react to stop/pause before + // decoding the next batch + return AudioDecoderState::DECODING; + } } else { // If paused, block to avoid wasting CPU resources delay(READ_WRITE_TIMEOUT_MS); } - // Verify there is enough space to store more decoded audio and that the function hasn't been running too long - if ((this->output_transfer_buffer_->free() < this->free_buffer_required_) || - (millis() - decoding_start > DECODING_TIMEOUT_MS)) { + if (this->output_transfer_buffer_->available() > 0) { + // Output transfer buffer indicates backpressure, return so caller can handle other events; + // e.g., stop/pause, before trying again return AudioDecoderState::DECODING; } - // Decode more audio - - // Never shift the input buffer; every decoder buffers internally and consumes only what it processed. - size_t bytes_read = this->input_buffer_->fill(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), false); - - if (!first_loop_iteration && (this->input_buffer_->available() < bytes_processed)) { - // Less data is available than what was processed in last iteration, so don't attempt to decode. - // This attempts to avoid the decoder from consistently trying to decode an incomplete frame. The transfer buffer - // will shift the remaining data to the start and copy more from the source the next time the decode function is - // called - break; + // Reaching here means no decoded output is pending (any would have returned above). Bounds long no-output + // stretches; e.g., skipping a large metadata block, so a source that keeps the ring buffer full can't spin this + // loop without yielding and trip the watchdog. The delay yields allowing other tasks to feed the watchdog and + // the return keeps stop/pause responsive. + if (++no_output_iterations >= MAX_NO_OUTPUT_ITERATIONS) { + delay(1); + return AudioDecoderState::DECODING; } - bytes_available_before_processing = this->input_buffer_->available(); + // Expose the next chunk of file data. Every decoder buffers internally and consumes only what it + // processed, so the source does not need to accumulate or stitch chunks across fill() calls. + this->input_buffer_->fill(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), false); - if ((this->potentially_failed_count_ > 0) && (bytes_read == 0)) { - // Failed to decode in last attempt and there is no new data + const size_t available_before_decode = this->input_buffer_->available(); - if ((this->input_buffer_->free() == 0) && first_loop_iteration) { - // The input buffer is full (or read-only, e.g. const flash source). Since it previously failed on the exact - // same data, we can never recover. For const sources this is correct: the entire file is already available, so - // a decode failure is genuine, not a transient out-of-data condition. - state = FileDecoderState::FAILED; - } else { - // Attempt to get more data next time - state = FileDecoderState::IDLE; - } - } else if (this->input_buffer_->available() == 0) { + if (available_before_decode == 0) { // No data to decode, attempt to get more data next time state = FileDecoderState::IDLE; } else { @@ -231,9 +224,6 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { } } - first_loop_iteration = false; - bytes_processed = bytes_available_before_processing - this->input_buffer_->available(); - if (state == FileDecoderState::POTENTIALLY_FAILED) { ++this->potentially_failed_count_; } else if (state == FileDecoderState::END_OF_FILE) { @@ -241,7 +231,16 @@ AudioDecoderState AudioDecoder::decode(bool stop_gracefully) { } else if (state == FileDecoderState::FAILED) { return AudioDecoderState::FAILED; } else if (state == FileDecoderState::MORE_TO_PROCESS) { - this->potentially_failed_count_ = 0; + // Reset the failsafe only when the iteration made forward progress: input was consumed or output was + // produced (output_transfer_buffer_ is drained empty above, so any available bytes are new). A + // MORE_TO_PROCESS that neither consumes input nor produces output means the decoder is stalled; count it + // toward the failsafe so a stuck stream eventually surfaces as FAILED instead of looping forever. + if ((this->input_buffer_->available() < available_before_decode) || + (this->output_transfer_buffer_->available() > 0)) { + this->potentially_failed_count_ = 0; + } else { + ++this->potentially_failed_count_; + } } } return AudioDecoderState::DECODING; diff --git a/esphome/components/audio/audio_decoder.h b/esphome/components/audio/audio_decoder.h index c34ebbc613..e772b7eb5f 100644 --- a/esphome/components/audio/audio_decoder.h +++ b/esphome/components/audio/audio_decoder.h @@ -61,15 +61,16 @@ class AudioDecoder { */ public: /// @brief Allocates the output transfer buffer and stores the input buffer size for later use by add_source() - /// @param input_buffer_size Size of the input transfer buffer in bytes. + /// @param input_buffer_size Soft cap on the bytes a ring buffer source exposes per fill, in bytes. /// @param output_buffer_size Size of the output transfer buffer in bytes. AudioDecoder(size_t input_buffer_size, size_t output_buffer_size); ~AudioDecoder() = default; - /// @brief Adds a source ring buffer for raw file data. Takes ownership of the ring buffer in a shared_ptr. - /// @param input_ring_buffer weak_ptr of a shared_ptr of the sink ring buffer to transfer ownership - /// @return ESP_OK if successsful, ESP_ERR_NO_MEM if the transfer buffer wasn't allocated + /// @brief Adds a source ring buffer for raw file data. Shares ownership of the ring buffer via a shared_ptr. + /// The decoder reads directly from the ring buffer's internal storage with a zero-copy RingBufferAudioSource. + /// @param input_ring_buffer weak_ptr of the source ring buffer to read from + /// @return ESP_OK if successful, ESP_ERR_INVALID_ARG if the ring buffer is expired or the buffer size is zero esp_err_t add_source(std::weak_ptr &input_ring_buffer); /// @brief Adds a sink ring buffer for decoded audio. Takes ownership of the ring buffer in a shared_ptr. diff --git a/esphome/components/audio/audio_resampler.cpp b/esphome/components/audio/audio_resampler.cpp index c04cc881f5..bef62ce190 100644 --- a/esphome/components/audio/audio_resampler.cpp +++ b/esphome/components/audio/audio_resampler.cpp @@ -12,16 +12,17 @@ static const uint32_t READ_WRITE_TIMEOUT_MS = 20; AudioResampler::AudioResampler(size_t input_buffer_size, size_t output_buffer_size) : input_buffer_size_(input_buffer_size), output_buffer_size_(output_buffer_size) { - this->input_transfer_buffer_ = AudioSourceTransferBuffer::create(input_buffer_size); this->output_transfer_buffer_ = AudioSinkTransferBuffer::create(output_buffer_size); } esp_err_t AudioResampler::add_source(std::weak_ptr &input_ring_buffer) { - if (this->input_transfer_buffer_ != nullptr) { - this->input_transfer_buffer_->set_source(input_ring_buffer); - return ESP_OK; + // The zero-copy RingBufferAudioSource is created lazily on the first resample() call, once both the ring + // buffer (stored here) and the input stream info (set by start()) are available, in either order. + this->source_ring_buffer_ = input_ring_buffer.lock(); + if (this->source_ring_buffer_ == nullptr) { + return ESP_ERR_INVALID_STATE; } - return ESP_ERR_NO_MEM; + return ESP_OK; } esp_err_t AudioResampler::add_sink(std::weak_ptr &output_ring_buffer) { @@ -47,7 +48,7 @@ esp_err_t AudioResampler::start(AudioStreamInfo &input_stream_info, AudioStreamI this->input_stream_info_ = input_stream_info; this->output_stream_info_ = output_stream_info; - if ((this->input_transfer_buffer_ == nullptr) || (this->output_transfer_buffer_ == nullptr)) { + if (this->output_transfer_buffer_ == nullptr) { return ESP_ERR_NO_MEM; } @@ -56,6 +57,13 @@ esp_err_t AudioResampler::start(AudioStreamInfo &input_stream_info, AudioStreamI return ESP_ERR_NOT_SUPPORTED; } + // Reject frame sizes that can't be used as the zero-copy source's alignment up front, where the caller checks + // the return code. The lazy create() in resample() keeps its own guard since it runs before the uint8_t cast. + const size_t bytes_per_frame = this->input_stream_info_.frames_to_bytes(1); + if ((bytes_per_frame == 0) || (bytes_per_frame > RingBufferAudioSource::MAX_ALIGNMENT_BYTES)) { + return ESP_ERR_NOT_SUPPORTED; + } + if ((input_stream_info.get_sample_rate() != output_stream_info.get_sample_rate()) || (input_stream_info.get_bits_per_sample() != output_stream_info.get_bits_per_sample())) { this->resampler_ = make_unique( @@ -87,8 +95,27 @@ esp_err_t AudioResampler::start(AudioStreamInfo &input_stream_info, AudioStreamI } AudioResamplerState AudioResampler::resample(bool stop_gracefully, int32_t *ms_differential) { + if (this->audio_source_ == nullptr) { + // Lazily create the zero-copy source on first use. Frame-aligned reads ensure multi-channel frames are + // never split across the ring buffer's wrap boundary. + const size_t bytes_per_frame = this->input_stream_info_.frames_to_bytes(1); + if ((bytes_per_frame == 0) || (bytes_per_frame > RingBufferAudioSource::MAX_ALIGNMENT_BYTES)) { + // Stream info is unset or the frame is too large to use as an alignment; the uint8_t cast below would + // truncate it and could yield a source that tears frames. + return AudioResamplerState::FAILED; + } + // Pass the shared_ptr by copy so a failed create() leaves source_ring_buffer_ intact; release our + // reference only after the source has taken ownership. + this->audio_source_ = RingBufferAudioSource::create(this->source_ring_buffer_, this->input_buffer_size_, + static_cast(bytes_per_frame)); + if (this->audio_source_ == nullptr) { + return AudioResamplerState::FAILED; + } + this->source_ring_buffer_.reset(); + } + if (stop_gracefully) { - if (!this->input_transfer_buffer_->has_buffered_data() && (this->output_transfer_buffer_->available() == 0)) { + if (!this->audio_source_->has_buffered_data() && (this->output_transfer_buffer_->available() == 0)) { return AudioResamplerState::FINISHED; } } @@ -102,9 +129,11 @@ AudioResamplerState AudioResampler::resample(bool stop_gracefully, int32_t *ms_d delay(READ_WRITE_TIMEOUT_MS); } - this->input_transfer_buffer_->transfer_data_from_source(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS)); + // Expose a chunk of the ring buffer's internal storage. pre_shift is ignored by RingBufferAudioSource + // (there is no intermediate transfer buffer to compact). + this->audio_source_->fill(pdMS_TO_TICKS(READ_WRITE_TIMEOUT_MS), false); - if (this->input_transfer_buffer_->available() == 0) { + if (this->audio_source_->available() == 0) { // No samples available to process return AudioResamplerState::RESAMPLING; } @@ -112,17 +141,17 @@ AudioResamplerState AudioResampler::resample(bool stop_gracefully, int32_t *ms_d const size_t bytes_free = this->output_transfer_buffer_->free(); const uint32_t frames_free = this->output_stream_info_.bytes_to_frames(bytes_free); - const size_t bytes_available = this->input_transfer_buffer_->available(); + const size_t bytes_available = this->audio_source_->available(); const uint32_t frames_available = this->input_stream_info_.bytes_to_frames(bytes_available); if ((this->input_stream_info_.get_sample_rate() != this->output_stream_info_.get_sample_rate()) || (this->input_stream_info_.get_bits_per_sample() != this->output_stream_info_.get_bits_per_sample())) { // Adjust gain by -3 dB to avoid clipping due to the resampling process esp_audio_libs::resampler::ResamplerResults results = - this->resampler_->resample(this->input_transfer_buffer_->get_buffer_start(), - this->output_transfer_buffer_->get_buffer_end(), frames_available, frames_free, -3); + this->resampler_->resample(this->audio_source_->data(), this->output_transfer_buffer_->get_buffer_end(), + frames_available, frames_free, -3); - this->input_transfer_buffer_->decrease_buffer_length(this->input_stream_info_.frames_to_bytes(results.frames_used)); + this->audio_source_->consume(this->input_stream_info_.frames_to_bytes(results.frames_used)); this->output_transfer_buffer_->increase_buffer_length( this->output_stream_info_.frames_to_bytes(results.frames_generated)); @@ -146,10 +175,10 @@ AudioResamplerState AudioResampler::resample(bool stop_gracefully, int32_t *ms_d const size_t bytes_to_transfer = std::min(this->output_stream_info_.frames_to_bytes(frames_free), this->input_stream_info_.frames_to_bytes(frames_available)); - std::memcpy((void *) this->output_transfer_buffer_->get_buffer_end(), - (void *) this->input_transfer_buffer_->get_buffer_start(), bytes_to_transfer); + std::memcpy((void *) this->output_transfer_buffer_->get_buffer_end(), (const void *) this->audio_source_->data(), + bytes_to_transfer); - this->input_transfer_buffer_->decrease_buffer_length(bytes_to_transfer); + this->audio_source_->consume(bytes_to_transfer); this->output_transfer_buffer_->increase_buffer_length(bytes_to_transfer); } diff --git a/esphome/components/audio/audio_resampler.h b/esphome/components/audio/audio_resampler.h index 575ad13692..c09070c0ce 100644 --- a/esphome/components/audio/audio_resampler.h +++ b/esphome/components/audio/audio_resampler.h @@ -22,7 +22,7 @@ namespace esphome::audio { enum class AudioResamplerState : uint8_t { RESAMPLING, // More data is available to resample FINISHED, // All file data has been resampled and transferred - FAILED, // Unused state included for consistency among Audio classes + FAILED, // Failed to allocate the audio source }; class AudioResampler { @@ -32,14 +32,16 @@ class AudioResampler { * component). Also supports converting bits per sample. */ public: - /// @brief Allocates the input and output transfer buffers - /// @param input_buffer_size Size of the input transfer buffer in bytes. + /// @brief Allocates the output transfer buffer. The input source is created later in resample(). + /// @param input_buffer_size Max bytes exposed per fill() call on the zero-copy input source. /// @param output_buffer_size Size of the output transfer buffer in bytes. AudioResampler(size_t input_buffer_size, size_t output_buffer_size); - /// @brief Adds a source ring buffer for audio data. Takes ownership of the ring buffer in a shared_ptr. - /// @param input_ring_buffer weak_ptr of a shared_ptr of the sink ring buffer to transfer ownership - /// @return ESP_OK if successsful, ESP_ERR_NO_MEM if the transfer buffer wasn't allocated + /// @brief Sets the ring buffer the audio is read from and takes shared ownership of it. The zero-copy + /// RingBufferAudioSource that reads directly from its internal storage is created lazily on the first + /// resample() call, so add_source() and start() may be called in any order. + /// @param input_ring_buffer weak_ptr of a shared_ptr of the source ring buffer to transfer ownership + /// @return ESP_OK if successful, ESP_ERR_INVALID_STATE if the ring buffer is no longer alive esp_err_t add_source(std::weak_ptr &input_ring_buffer); /// @brief Adds a sink ring buffer for resampled audio. Takes ownership of the ring buffer in a shared_ptr. @@ -78,7 +80,8 @@ class AudioResampler { void set_pause_output_state(bool pause_state) { this->pause_output_ = pause_state; } protected: - std::unique_ptr input_transfer_buffer_; + std::shared_ptr source_ring_buffer_; + std::unique_ptr audio_source_; std::unique_ptr output_transfer_buffer_; size_t input_buffer_size_; diff --git a/esphome/components/audio/audio_transfer_buffer.cpp b/esphome/components/audio/audio_transfer_buffer.cpp index d9ce8060e2..a611549e58 100644 --- a/esphome/components/audio/audio_transfer_buffer.cpp +++ b/esphome/components/audio/audio_transfer_buffer.cpp @@ -252,6 +252,22 @@ void RingBufferAudioSource::consume(size_t bytes) { } } +void RingBufferAudioSource::clear_buffered_data() { + // Release the held item before reset() so the source no longer references memory the reset will reclaim. + if (this->acquired_item_ != nullptr) { + this->ring_buffer_->receive_release(this->acquired_item_); + this->acquired_item_ = nullptr; + } + this->current_data_ = nullptr; + this->current_available_ = 0; + this->queued_data_ = nullptr; + this->queued_length_ = 0; + this->item_trailing_ptr_ = nullptr; + this->item_trailing_length_ = 0; + this->splice_length_ = 0; + this->ring_buffer_->reset(); +} + bool RingBufferAudioSource::has_buffered_data() const { // splice_length_ is deliberately not considered here. It holds an incomplete frame whose completion // bytes must still arrive through the ring buffer, which ring_buffer_->available() already reports. diff --git a/esphome/components/audio/audio_transfer_buffer.h b/esphome/components/audio/audio_transfer_buffer.h index b713326141..074684f068 100644 --- a/esphome/components/audio/audio_transfer_buffer.h +++ b/esphome/components/audio/audio_transfer_buffer.h @@ -250,6 +250,10 @@ class RingBufferAudioSource : public AudioReadableBuffer { /// exposure stays in place and fill() returns 0 until it is fully consumed. size_t fill(TickType_t ticks_to_wait, bool pre_shift) override; + /// @brief Discards all buffered audio: releases any held ring buffer item, clears the source's in-flight + /// state, and resets the underlying ring buffer. Must be invoked from the ring buffer's consumer thread. + void clear_buffered_data(); + /// @brief Returns a mutable pointer to the currently exposed audio data. /// The pointer may reference the ring buffer's internal storage or, when exposing a stitched frame /// across a wrap boundary, an internal splice buffer. In either case mutations are safe but data diff --git a/esphome/components/audio_file/__init__.py b/esphome/components/audio_file/__init__.py index 23c90e9b76..53193c8008 100644 --- a/esphome/components/audio_file/__init__.py +++ b/esphome/components/audio_file/__init__.py @@ -72,7 +72,7 @@ def _file_schema(value: ConfigType | str) -> ConfigType: def _validate_file_shorthand(value: str) -> ConfigType: value = cv.string_strict(value) - if value.startswith("http://") or value.startswith("https://"): + if value.startswith(("http://", "https://")): return _file_schema( { CONF_TYPE: TYPE_WEB, @@ -98,7 +98,7 @@ def read_audio_file_and_type(file_config: ConfigType) -> tuple[bytes, MockObj]: else: raise cv.Invalid("Unsupported file source") - with open(path, "rb") as f: + with path.open("rb") as f: data = f.read() try: diff --git a/esphome/components/audio_file/media_source/__init__.py b/esphome/components/audio_file/media_source/__init__.py index 635a51b610..0710582813 100644 --- a/esphome/components/audio_file/media_source/__init__.py +++ b/esphome/components/audio_file/media_source/__init__.py @@ -1,7 +1,5 @@ -from typing import Any - import esphome.codegen as cg -from esphome.components import audio, esp32, media_source, psram +from esphome.components import audio, media_source, psram import esphome.config_validation as cv from esphome.const import CONF_ID, CONF_TASK_STACK_IN_PSRAM from esphome.types import ConfigType @@ -21,19 +19,13 @@ def _request_micro_decoder(config: ConfigType) -> ConfigType: return config -def _validate_task_stack_in_psram(value: Any) -> bool: - if value := cv.boolean(value): - return cv.requires_component(psram.DOMAIN)(value) - return value - - CONFIG_SCHEMA = cv.All( media_source.media_source_schema( AudioFileMediaSource, ) .extend( { - cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, } ) .extend(cv.COMPONENT_SCHEMA), @@ -49,6 +41,4 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() diff --git a/esphome/components/audio_http/media_source.py b/esphome/components/audio_http/media_source.py index 519d8df698..e8acbc81af 100644 --- a/esphome/components/audio_http/media_source.py +++ b/esphome/components/audio_http/media_source.py @@ -1,7 +1,5 @@ -from typing import Any - import esphome.codegen as cg -from esphome.components import audio, esp32, media_source, psram +from esphome.components import audio, media_source, psram import esphome.config_validation as cv from esphome.const import CONF_BUFFER_SIZE, CONF_ID, CONF_TASK_STACK_IN_PSRAM from esphome.types import ConfigType @@ -20,14 +18,6 @@ def _request_micro_decoder(config: ConfigType) -> ConfigType: return config -def _validate_task_stack_in_psram(value: Any) -> bool: - # Only require the psram component when actually enabling PSRAM stacks; validating - # the boolean first means `false` doesn't trigger the requires_component check. - if value := cv.boolean(value): - return cv.requires_component(psram.DOMAIN)(value) - return value - - CONFIG_SCHEMA = cv.All( media_source.media_source_schema( AudioHTTPMediaSource, @@ -37,7 +27,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_BUFFER_SIZE, default=50000): cv.int_range( min=5000, max=1000000 ), - cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, } ) .extend(cv.COMPONENT_SCHEMA), @@ -53,7 +43,5 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() cg.add(var.set_buffer_size(config[CONF_BUFFER_SIZE])) diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp index 21573f0184..7ba9e61e19 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.cpp @@ -135,12 +135,26 @@ void BluetoothConnection::loop() { // - For V3_WITH_CACHE: Services are never sent, disable after INIT state // - For V3_WITHOUT_CACHE: Disable only after service discovery is complete // (send_service_ == DONE_SENDING_SERVICES, which is only set after services are sent) - if (this->state() != espbt::ClientState::INIT && (this->connection_type_ == espbt::ConnectionType::V3_WITH_CACHE || - this->send_service_ == DONE_SENDING_SERVICES)) { + // Never disable while DISCONNECTING — BLEClientBase::loop() needs to keep running so the + // 10s safety timeout can force IDLE if CLOSE_EVT is never delivered. + if (this->state() != espbt::ClientState::INIT && this->state() != espbt::ClientState::DISCONNECTING && + (this->connection_type_ == espbt::ConnectionType::V3_WITH_CACHE || + this->send_service_ == DONE_SENDING_SERVICES)) { this->disable_loop(); } } +void BluetoothConnection::on_disconnect_complete(esp_err_t reason) { + // Called from both the CLOSE_EVT handler and the DISCONNECTING safety timeout in the + // base class. Free the proxy slot, notify the API client, and reset send_service_. + // address_ may already be 0 if reset_connection_ ran earlier on this teardown. + if (this->address_ == 0) { + return; + } + ESP_LOGD(TAG, "[%d] [%s] Close, reason=0x%02x, freeing slot", this->connection_index_, this->address_str_, reason); + this->reset_connection_(reason); +} + void BluetoothConnection::reset_connection_(esp_err_t reason) { // Send disconnection notification this->proxy_->send_device_connection(this->address_, false, 0, reason); @@ -372,14 +386,6 @@ bool BluetoothConnection::gattc_event_handler(esp_gattc_cb_event_t event, esp_ga this->proxy_->send_device_connection(this->address_, false, 0, param->disconnect.reason); break; } - case ESP_GATTC_CLOSE_EVT: { - ESP_LOGD(TAG, "[%d] [%s] Close, reason=0x%02x, freeing slot", this->connection_index_, this->address_str_, - param->close.reason); - // Now the GATT connection is fully closed and controller resources are freed - // Safe to mark the connection slot as available - this->reset_connection_(param->close.reason); - break; - } case ESP_GATTC_OPEN_EVT: { if (param->open.status != ESP_GATT_OK && param->open.status != ESP_GATT_ALREADY_OPEN) { this->reset_connection_(param->open.status); diff --git a/esphome/components/bluetooth_proxy/bluetooth_connection.h b/esphome/components/bluetooth_proxy/bluetooth_connection.h index b50ea2d6a2..e5600f6af4 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_connection.h +++ b/esphome/components/bluetooth_proxy/bluetooth_connection.h @@ -33,6 +33,8 @@ class BluetoothConnection final : public esp32_ble_client::BLEClientBase { protected: friend class BluetoothProxy; + void on_disconnect_complete(esp_err_t reason) override; + bool supports_efficient_uuids_() const; void send_service_for_discovery_(); void reset_connection_(esp_err_t reason); diff --git a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp index c3461f9c51..ca30aab943 100644 --- a/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp +++ b/esphome/components/bluetooth_proxy/bluetooth_proxy.cpp @@ -1,5 +1,6 @@ #include "bluetooth_proxy.h" +#include "esphome/components/api/api_server.h" #include "esphome/core/log.h" #include "esphome/core/macros.h" #include "esphome/core/application.h" diff --git a/esphome/components/bme68x_bsec2/__init__.py b/esphome/components/bme68x_bsec2/__init__.py index 5083d283ef..62cd9e2e36 100644 --- a/esphome/components/bme68x_bsec2/__init__.py +++ b/esphome/components/bme68x_bsec2/__init__.py @@ -169,7 +169,7 @@ async def to_code_base(config): path = _compute_local_file_path(_compute_url(config)) try: - with open(path, encoding="utf-8") as f: + with path.open(encoding="utf-8") as f: bsec2_iaq_config = f.read() except Exception as e: raise core.EsphomeError( diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index e9b0f1fd0a..e3bff8f934 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -46,7 +46,7 @@ from esphome.const import ( Toolchain, __version__, ) -from esphome.core import CORE, HexInt, Library +from esphome.core import CORE, EsphomeError, HexInt, Library from esphome.core.config import BOARD_MAX_LENGTH from esphome.coroutine import CoroPriority, coroutine_with_priority from esphome.espidf.component import generate_idf_component @@ -113,6 +113,7 @@ ARDUINO_FRAMEWORK_NAME = "framework-arduinoespressif32" ARDUINO_FRAMEWORK_PKG = f"pioarduino/{ARDUINO_FRAMEWORK_NAME}" ARDUINO_LIBS_NAME = f"{ARDUINO_FRAMEWORK_NAME}-libs" ARDUINO_LIBS_PKG = f"pioarduino/{ARDUINO_LIBS_NAME}" +ARDUINO_ESP32_COMPONENT_NAME = "espressif/arduino-esp32" LOG_LEVELS_IDF = [ "NONE", @@ -792,19 +793,15 @@ PLATFORM_VERSION_LOOKUP = { } -def _check_pio_versions(config): - config = config.copy() - value = config[CONF_FRAMEWORK] +def _resolve_framework_version(value: ConfigType) -> cv.Version: + """Resolve a named or raw framework version and validate the minimum. + Normalises value[CONF_VERSION] to its string form and returns the parsed + cv.Version. Shared between the PIO and esp-idf toolchain paths; toolchain- + specific concerns (source defaults, platform_version) live in the per- + toolchain functions. + """ if value[CONF_VERSION] in PLATFORM_VERSION_LOOKUP: - if CONF_SOURCE in value or CONF_PLATFORM_VERSION in value: - raise cv.Invalid( - "Version needs to be explicitly set when a custom source or platform_version is used." - ) - - platform_lookup = PLATFORM_VERSION_LOOKUP[value[CONF_VERSION]] - value[CONF_PLATFORM_VERSION] = _parse_pio_platform_version(str(platform_lookup)) - if value[CONF_TYPE] == FRAMEWORK_ARDUINO: version = ARDUINO_FRAMEWORK_VERSION_LOOKUP[value[CONF_VERSION]] else: @@ -817,7 +814,38 @@ def _check_pio_versions(config): if value[CONF_TYPE] == FRAMEWORK_ARDUINO: if version < cv.Version(3, 0, 0): raise cv.Invalid("Only Arduino 3.0+ is supported.") - recommended_version = ARDUINO_FRAMEWORK_VERSION_LOOKUP["recommended"] + recommended = ARDUINO_FRAMEWORK_VERSION_LOOKUP["recommended"] + else: + if version < cv.Version(5, 0, 0): + raise cv.Invalid("Only ESP-IDF 5.0+ is supported.") + recommended = ESP_IDF_FRAMEWORK_VERSION_LOOKUP["recommended"] + + if version != recommended: + _LOGGER.warning( + "The selected framework version is not the recommended one. " + "If there are connectivity or build issues please remove the manual version." + ) + + return version + + +def _check_pio_versions(config: ConfigType) -> ConfigType: + config = config.copy() + value = config[CONF_FRAMEWORK] + + is_named_version = value[CONF_VERSION] in PLATFORM_VERSION_LOOKUP + if is_named_version and (CONF_SOURCE in value or CONF_PLATFORM_VERSION in value): + raise cv.Invalid( + "Version needs to be explicitly set when a custom source or platform_version is used." + ) + if is_named_version: + value[CONF_PLATFORM_VERSION] = _parse_pio_platform_version( + str(PLATFORM_VERSION_LOOKUP[value[CONF_VERSION]]) + ) + + version = _resolve_framework_version(value) + + if value[CONF_TYPE] == FRAMEWORK_ARDUINO: platform_lookup = ARDUINO_PLATFORM_VERSION_LOOKUP.get(version) value[CONF_SOURCE] = value.get( CONF_SOURCE, _format_framework_arduino_version(version) @@ -825,9 +853,6 @@ def _check_pio_versions(config): if _is_framework_url(value[CONF_SOURCE]): value[CONF_SOURCE] = f"{ARDUINO_FRAMEWORK_PKG}@{value[CONF_SOURCE]}" else: - if version < cv.Version(5, 0, 0): - raise cv.Invalid("Only ESP-IDF 5.0+ is supported.") - recommended_version = ESP_IDF_FRAMEWORK_VERSION_LOOKUP["recommended"] platform_lookup = ESP_IDF_PLATFORM_VERSION_LOOKUP.get(version) value[CONF_SOURCE] = value.get( CONF_SOURCE, @@ -843,12 +868,6 @@ def _check_pio_versions(config): ) value[CONF_PLATFORM_VERSION] = _parse_pio_platform_version(str(platform_lookup)) - if version != recommended_version: - _LOGGER.warning( - "The selected framework version is not the recommended one. " - "If there are connectivity or build issues please remove the manual version." - ) - if value[CONF_PLATFORM_VERSION] != _parse_pio_platform_version( str(PLATFORM_VERSION_LOOKUP["recommended"]) ): @@ -860,19 +879,26 @@ def _check_pio_versions(config): return config -def _check_esp_idf_versions(config): - config = _check_pio_versions(config) +def _check_esp_idf_versions(config: ConfigType) -> ConfigType: + config = config.copy() value = config[CONF_FRAMEWORK] - # Remove unwanted keys if present - for key in (CONF_SOURCE, CONF_PLATFORM_VERSION): - value.pop(key, None) + # platform_version is a PlatformIO concept; drop it if a user carried it + # over from a PIO-style config. CONF_SOURCE, on the other hand, is kept: + # it lets a user override the framework tarball URL under the esp-idf + # toolchain (the espidf framework downloader consults it). + value.pop(CONF_PLATFORM_VERSION, None) - # Official ESP-IDF frameworks don't use extra - version = cv.Version.parse(value[CONF_VERSION]) - version = cv.Version(version.major, version.minor, version.patch) + version = _resolve_framework_version(value) - value[CONF_VERSION] = str(version) + if CONF_SOURCE in value: + _LOGGER.warning( + "A custom framework source is set. " + "If there are connectivity or build issues please remove the manual source." + ) + + # Official ESP-IDF frameworks don't use the 'extra' semver component. + value[CONF_VERSION] = str(cv.Version(version.major, version.minor, version.patch)) return config @@ -1718,6 +1744,31 @@ async def _add_yaml_idf_components(components: list[ConfigType]): ) +@coroutine_with_priority(CoroPriority.FINAL - 1) +async def _finalize_arduino_aware_flags(): + """Build flags that depend on whether arduino-esp32 is linked in. + + Scheduler runs lower priority values later, so ``FINAL - 1`` fires + after every ``FINAL`` job (incl. ``_add_yaml_idf_components``) -- + by then ``KEY_COMPONENTS`` is fully populated. + + - Skip our esp_panic_handler wrap when Arduino is linked; Arduino + wraps the same symbol and the linker errors on the duplicate. + - Define USE_ARDUINO in the hybrid esp-idf+arduino-esp32-component + case so ESPHome's ``#ifdef USE_ARDUINO`` paths light up. The + framework=arduino branch already adds it inline in to_code. + """ + arduino_linked = ( + CORE.using_arduino + or ARDUINO_ESP32_COMPONENT_NAME in CORE.data[KEY_ESP32][KEY_COMPONENTS] + ) + if not arduino_linked: + cg.add_build_flag("-Wl,--wrap=esp_panic_handler") + cg.add_define("USE_ESP32_CRASH_HANDLER") + elif not CORE.using_arduino: + cg.add_build_flag("-DUSE_ARDUINO") + + async def to_code(config): framework_ver: cv.Version = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION] conf = config[CONF_FRAMEWORK] @@ -1765,11 +1816,12 @@ async def to_code(config): Path(__file__).parent / "iram_fix.py.script", ) else: - cg.add_build_flag("-Wno-error=format") - cg.add_build_flag("-Wno-error=maybe-uninitialized") - cg.add_build_flag("-Wno-error=overloaded-virtual") - cg.add_build_flag("-Wno-error=reorder") - cg.add_build_flag("-Wno-error=volatile") + # Demote IDF's blanket -Werror to warnings so third-party libs + # and user lambdas don't need a -Wno-error= per warning. + # The sdkconfig knob disables IDF's rewrite to -Werror=all (which + # can't be globally undone); -Wno-error then handles the demotion. + add_idf_sdkconfig_option("CONFIG_COMPILER_DISABLE_DEFAULT_ERRORS", False) + cg.add_build_flag("-Wno-error") # -Wno- (not -Wno-error=): suppress entirely, too noisy on C++ aggregates cg.add_build_flag("-Wno-missing-field-initializers") @@ -1777,11 +1829,8 @@ async def to_code(config): cg.add_build_flag("-DUSE_ESP32") cg.add_define("USE_NATIVE_64BIT_TIME") cg.add_build_flag("-Wl,-z,noexecstack") - # Arduino already wraps esp_panic_handler for its own backtrace handler, - # so only add our wrap when using ESP-IDF framework to avoid linker conflicts. - if conf[CONF_TYPE] == FRAMEWORK_ESP_IDF: - cg.add_build_flag("-Wl,--wrap=esp_panic_handler") - cg.add_define("USE_ESP32_CRASH_HANDLER") + # Deferred so KEY_COMPONENTS is fully populated -- see the coroutine. + CORE.add_job(_finalize_arduino_aware_flags) cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) variant = config[CONF_VARIANT] cg.add_build_flag(f"-DUSE_ESP32_VARIANT_{variant}") @@ -1962,7 +2011,7 @@ async def to_code(config): add_idf_sdkconfig_option("CONFIG_HEAP_PLACE_FUNCTION_INTO_FLASH", True) # Setup watchdog - add_idf_sdkconfig_option("CONFIG_ESP_TASK_WDT", True) + add_idf_sdkconfig_option("CONFIG_ESP_TASK_WDT_INIT", True) add_idf_sdkconfig_option("CONFIG_ESP_TASK_WDT_PANIC", True) add_idf_sdkconfig_option("CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0", False) add_idf_sdkconfig_option("CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU1", False) @@ -2004,7 +2053,8 @@ async def to_code(config): if not advanced[CONF_ENABLE_LWIP_MDNS_QUERIES]: add_idf_sdkconfig_option("CONFIG_LWIP_DNS_SUPPORT_MDNS_QUERIES", False) if not advanced[CONF_ENABLE_LWIP_BRIDGE_INTERFACE]: - add_idf_sdkconfig_option("CONFIG_LWIP_BRIDGEIF_MAX_PORTS", 0) + # Kconfig range is [1,63]; 0 gets clamped to the default. + add_idf_sdkconfig_option("CONFIG_LWIP_BRIDGEIF_MAX_PORTS", 1) _configure_lwip_max_sockets(conf) @@ -2096,7 +2146,6 @@ async def to_code(config): for key, flag in ASSERTION_LEVELS.items(): add_idf_sdkconfig_option(flag, assertion_level == key) - add_idf_sdkconfig_option("CONFIG_COMPILER_OPTIMIZATION_DEFAULT", False) compiler_optimization = advanced[CONF_COMPILER_OPTIMIZATION] for key, flag in COMPILER_OPTIMIZATIONS.items(): add_idf_sdkconfig_option(flag, compiler_optimization == key) @@ -2251,7 +2300,8 @@ async def to_code(config): add_idf_sdkconfig_option("CONFIG_FATFS_VOLUME_COUNT", 2) elif advanced[CONF_DISABLE_FATFS]: add_idf_sdkconfig_option("CONFIG_FATFS_LFN_NONE", True) - add_idf_sdkconfig_option("CONFIG_FATFS_VOLUME_COUNT", 0) + # Kconfig range is [1,10]; 0 gets clamped to the default. + add_idf_sdkconfig_option("CONFIG_FATFS_VOLUME_COUNT", 1) for name, value in conf[CONF_SDKCONFIG_OPTIONS].items(): add_idf_sdkconfig_option(name, RawSdkconfigValue(value)) @@ -2488,9 +2538,8 @@ def _write_sdkconfig(): def _platformio_library_to_dependency(library: Library) -> tuple[str, dict[str, str]]: dependency: dict[str, str] = {} - name, version, path = generate_idf_component(library) + name, _version, path = generate_idf_component(library) dependency["override_path"] = str(path) - dependency["version"] = version return name, dependency @@ -2542,7 +2591,7 @@ def _write_idf_component_yml(): if CORE.using_toolchain_esp_idf: add_idf_component( - name="espressif/arduino-esp32", + name=ARDUINO_ESP32_COMPONENT_NAME, ref=str(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]), ) @@ -2609,13 +2658,29 @@ def copy_files(): def _decode_pc(config, addr): - from esphome.platformio import toolchain + # _decode_pc runs from the api log processor's asyncio callback, which + # only catches EsphomeError. Any other exception escaping here tears down + # the protocol and triggers an infinite reconnect/replay loop. Convert + # toolchain-resolution errors (e.g. missing build dir / cmake cache) into + # EsphomeError so the caller can disable decoding cleanly. + if CORE.using_toolchain_esp_idf: + from esphome.espidf import toolchain as idf_toolchain - idedata = toolchain.get_idedata(config) - if not idedata.addr2line_path or not idedata.firmware_elf_path: + try: + addr2line_path = idf_toolchain.get_addr2line_path() + firmware_elf_path = idf_toolchain.get_elf_path() + except RuntimeError as err: + raise EsphomeError(f"ESP-IDF toolchain not available: {err}") from err + else: + from esphome.platformio import toolchain + + idedata = toolchain.get_idedata(config) + addr2line_path = idedata.addr2line_path + firmware_elf_path = idedata.firmware_elf_path + if not addr2line_path or not firmware_elf_path: _LOGGER.debug("decode_pc no addr2line") return - command = [idedata.addr2line_path, "-pfiaC", "-e", idedata.firmware_elf_path, addr] + command = [str(addr2line_path), "-pfiaC", "-e", str(firmware_elf_path), addr] try: translation = subprocess.check_output(command, close_fds=False).decode().strip() except Exception: # pylint: disable=broad-except diff --git a/esphome/components/esp32_ble_client/ble_client_base.cpp b/esphome/components/esp32_ble_client/ble_client_base.cpp index 7f0f2c624d..3fb9632e9a 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.cpp +++ b/esphome/components/esp32_ble_client/ble_client_base.cpp @@ -72,6 +72,7 @@ void BLEClientBase::loop() { // never delivered CLOSE_EVT/DISCONNECT_EVT, services would leak without this call. this->release_services(); this->set_idle_(); + this->on_disconnect_complete(ESP_GATT_CONN_TIMEOUT); } } @@ -418,6 +419,7 @@ bool BLEClientBase::gattc_event_handler(esp_gattc_cb_event_t event, esp_gatt_if_ this->log_gattc_lifecycle_event_("CLOSE"); this->release_services(); this->set_idle_(); + this->on_disconnect_complete(param->close.reason); break; } case ESP_GATTC_SEARCH_RES_EVT: { diff --git a/esphome/components/esp32_ble_client/ble_client_base.h b/esphome/components/esp32_ble_client/ble_client_base.h index 4e0b22cc29..0291a4b993 100644 --- a/esphome/components/esp32_ble_client/ble_client_base.h +++ b/esphome/components/esp32_ble_client/ble_client_base.h @@ -140,6 +140,12 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { void log_gattc_warning_(const char *operation, esp_err_t err); void log_connection_params_(const char *param_type); void handle_connection_result_(esp_err_t ret); + /// Hook called once a connection has been fully torn down (after release_services() and + /// set_idle_()), from both the CLOSE_EVT handler and the DISCONNECTING safety timeout. + /// Subclasses with extra per-connection accounting (e.g. bluetooth_proxy slot state) + /// override this to release that state. `reason` is the controller reason code, or + /// ESP_GATT_CONN_TIMEOUT for the safety-timeout path. + virtual void on_disconnect_complete(esp_err_t reason) {} /// Transition to IDLE and reset conn_id — call when the connection is fully dead. void set_idle_() { this->set_state(espbt::ClientState::IDLE); @@ -149,6 +155,10 @@ class BLEClientBase : public espbt::ESPBTClient, public Component { void set_disconnecting_() { this->disconnecting_started_ = millis(); this->set_state(espbt::ClientState::DISCONNECTING); + // BluetoothConnection::loop() disables the component loop after service discovery + // completes, so the DISCONNECTING timeout check in loop() would never run if CLOSE_EVT + // gets lost. Re-enable the loop so the 10s safety timeout can force IDLE. + this->enable_loop(); } // Compact error logging helpers to reduce flash usage void log_error_(const char *message); diff --git a/esphome/components/esp32_ble_server/ble_characteristic.cpp b/esphome/components/esp32_ble_server/ble_characteristic.cpp index cc519846be..4d364b4655 100644 --- a/esphome/components/esp32_ble_server/ble_characteristic.cpp +++ b/esphome/components/esp32_ble_server/ble_characteristic.cpp @@ -196,42 +196,35 @@ void BLECharacteristic::gatts_event_handler(esp_gatts_cb_event_t event, esp_gatt (*this->on_read_callback_)(param->read.conn_id); } - uint16_t max_offset = 22; - + // Use the client-supplied offset for long reads; short reads always start at 0. + // The Bluedroid stack truncates ATT_READ_RSP / ATT_READ_BLOB_RSP to MTU-1, so we + // just provide as much data as we have from the requested offset and let the stack + // handle framing. The client issues subsequent blob reads with increasing offsets + // until it has received the whole value. + const uint16_t offset = param->read.is_long ? param->read.offset : 0; + esp_gatt_status_t status = ESP_GATT_OK; esp_gatt_rsp_t response; - if (param->read.is_long) { - if (this->value_read_offset_ >= this->value_.size()) { - response.attr_value.len = 0; - response.attr_value.offset = this->value_read_offset_; - this->value_read_offset_ = 0; - } else if (this->value_.size() - this->value_read_offset_ < max_offset) { - // Last message in the chain - response.attr_value.len = this->value_.size() - this->value_read_offset_; - response.attr_value.offset = this->value_read_offset_; - memcpy(response.attr_value.value, this->value_.data() + response.attr_value.offset, response.attr_value.len); - this->value_read_offset_ = 0; - } else { - response.attr_value.len = max_offset; - response.attr_value.offset = this->value_read_offset_; - memcpy(response.attr_value.value, this->value_.data() + response.attr_value.offset, response.attr_value.len); - this->value_read_offset_ += max_offset; - } + response.attr_value.offset = offset; + + if (offset > this->value_.size()) { + status = ESP_GATT_INVALID_OFFSET; + response.attr_value.len = 0; } else { - response.attr_value.offset = 0; - if (this->value_.size() + 1 > max_offset) { - response.attr_value.len = max_offset; - this->value_read_offset_ = max_offset; - } else { - response.attr_value.len = this->value_.size(); + size_t remaining = this->value_.size() - offset; + if (remaining > ESP_GATT_MAX_ATTR_LEN) { + ESP_LOGW(TAG, "Characteristic length %u exceeds buffer size of %u, truncating", + static_cast(remaining), ESP_GATT_MAX_ATTR_LEN); + remaining = ESP_GATT_MAX_ATTR_LEN; } - memcpy(response.attr_value.value, this->value_.data(), response.attr_value.len); + response.attr_value.len = remaining; + memcpy(response.attr_value.value, this->value_.data() + offset, remaining); } response.attr_value.handle = this->handle_; response.attr_value.auth_req = ESP_GATT_AUTH_REQ_NONE; esp_err_t err = - esp_ble_gatts_send_response(gatts_if, param->read.conn_id, param->read.trans_id, ESP_GATT_OK, &response); + esp_ble_gatts_send_response(gatts_if, param->read.conn_id, param->read.trans_id, status, &response); if (err != ESP_OK) { ESP_LOGE(TAG, "esp_ble_gatts_send_response failed: %d", err); } diff --git a/esphome/components/esp32_ble_server/ble_characteristic.h b/esphome/components/esp32_ble_server/ble_characteristic.h index 94c7495cbd..933177a399 100644 --- a/esphome/components/esp32_ble_server/ble_characteristic.h +++ b/esphome/components/esp32_ble_server/ble_characteristic.h @@ -79,7 +79,6 @@ class BLECharacteristic { esp_gatt_char_prop_t properties_; uint16_t handle_{0xFFFF}; - uint16_t value_read_offset_{0}; std::vector value_; std::vector descriptors_; diff --git a/esphome/components/esp32_hosted/__init__.py b/esphome/components/esp32_hosted/__init__.py index 71d1fd3ac1..94e20ea6c9 100644 --- a/esphome/components/esp32_hosted/__init__.py +++ b/esphome/components/esp32_hosted/__init__.py @@ -3,6 +3,7 @@ from pathlib import Path from esphome import pins from esphome.components import esp32 +from esphome.components.const import CONF_USE_PSRAM import esphome.config_validation as cv from esphome.const import ( CONF_CLK_PIN, @@ -39,6 +40,7 @@ BASE_SCHEMA = cv.Schema( cv.Required(CONF_VARIANT): cv.one_of(*esp32.VARIANTS, upper=True), cv.Required(CONF_ACTIVE_HIGH): cv.boolean, cv.Required(CONF_RESET_PIN): pins.internal_gpio_output_pin_number, + cv.Optional(CONF_USE_PSRAM, default=False): cv.boolean, } ) @@ -242,6 +244,12 @@ async def to_code(config): else: _configure_spi(config) + # Place the transport mempool in PSRAM. Required on memory-tight host + # configurations (e.g. P4 with a large LVGL UI) where the internal-RAM + # mempool allocation fails at boot with `sdio_mempool_create` assert. + if config[CONF_USE_PSRAM]: + esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_MEMPOOL_PREFER_SPIRAM", True) + # Library versions idf_ver = esp32.idf_version() os.environ["ESP_IDF_VERSION"] = f"{idf_ver.major}.{idf_ver.minor}" @@ -249,7 +257,7 @@ async def to_code(config): esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="1.5.1") esp32.add_idf_component(name="espressif/wifi_remote_over_eppp", ref="0.3.2") esp32.add_idf_component(name="espressif/eppp_link", ref="1.1.5") - esp32.add_idf_component(name="espressif/esp_hosted", ref="2.12.7") + esp32.add_idf_component(name="espressif/esp_hosted", ref="2.12.8") else: esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="0.13.0") esp32.add_idf_component(name="espressif/eppp_link", ref="0.2.0") diff --git a/esphome/components/esp32_hosted/update/__init__.py b/esphome/components/esp32_hosted/update/__init__.py index b258a26b08..202df21ab5 100644 --- a/esphome/components/esp32_hosted/update/__init__.py +++ b/esphome/components/esp32_hosted/update/__init__.py @@ -75,7 +75,7 @@ def _validate_firmware(config: dict[str, Any]) -> None: return path = CORE.relative_config_path(config[CONF_PATH]) - with open(path, "rb") as f: + with path.open("rb") as f: firmware_data = f.read() calculated = hashlib.sha256(firmware_data).hexdigest() expected = config[CONF_SHA256].lower() @@ -93,7 +93,7 @@ async def to_code(config: dict[str, Any]) -> None: if config[CONF_TYPE] == TYPE_EMBEDDED: path = config[CONF_PATH] - with open(CORE.relative_config_path(path), "rb") as f: + with CORE.relative_config_path(path).open("rb") as f: firmware_data = f.read() rhs = [HexInt(x) for x in firmware_data] arr_id = ID(f"{config[CONF_ID]}_data", is_declaration=True, type=cg.uint8) diff --git a/esphome/components/esp32_hosted/update/esp32_hosted_update.cpp b/esphome/components/esp32_hosted/update/esp32_hosted_update.cpp index 7f3ba77895..70fa41b312 100644 --- a/esphome/components/esp32_hosted/update/esp32_hosted_update.cpp +++ b/esphome/components/esp32_hosted/update/esp32_hosted_update.cpp @@ -121,7 +121,7 @@ void Esp32HostedUpdate::setup() { } } else { ESP_LOGW(TAG, "Invalid app description magic word: 0x%08" PRIx32 " (expected 0x%08" PRIx32 ")", - app_desc->magic_word, ESP_APP_DESC_MAGIC_WORD); + app_desc->magic_word, static_cast(ESP_APP_DESC_MAGIC_WORD)); this->state_ = update::UPDATE_STATE_NO_UPDATE; } } else { diff --git a/esphome/components/esp8266/hal.cpp b/esphome/components/esp8266/hal.cpp index e8f472dc8a..3501c51859 100644 --- a/esphome/components/esp8266/hal.cpp +++ b/esphome/components/esp8266/hal.cpp @@ -5,6 +5,7 @@ #include #include +#include extern "C" { #include @@ -71,23 +72,22 @@ uint32_t IRAM_ATTR HOT millis() { return result; } -// Poll-based delay that avoids ::delay() — Arduino's __delay has an intra-object -// call to the original millis() that --wrap can't intercept, so calling ::delay() -// would keep the slow Arduino millis body alive in IRAM. optimistic_yield still -// enters esp_schedule()/esp_suspend_within_cont() via yield(), so SDK tasks and -// WiFi run correctly. Theoretically less power-efficient than Arduino's -// os_timer-based delay() for long waits, but nearly all ESPHome delays are short -// (sensor/I²C/SPI settling in the 1–100 ms range) where the difference is -// negligible. +// Delegate to Arduino's 1-arg esp_delay(), which uses os_timer + esp_suspend to +// suspend the cont task for `ms` milliseconds without polling millis(). This +// matches pre-2026.5.0 behavior (when esphome::delay() forwarded to ::delay()) +// and lets the SDK run freely while we wait, which timing-sensitive +// interrupt-driven code (e.g. ESP8266 software-serial RX in components like +// fingerprint_grow) depends on. The poll-based busy-wait that this replaced +// rarely yielded inside short waits like delay(1), starving WiFi/SDK tasks and +// extending interrupt latency. Unlike ::delay(), esp_delay()'s 1-arg form does +// not call millis(), so the slow Arduino millis() body is not pulled into IRAM +// by this path (the --wrap=millis goal of #15662 is preserved). void HOT delay(uint32_t ms) { if (ms == 0) { optimistic_yield(1000); return; } - uint32_t start = millis(); - while (millis() - start < ms) { - optimistic_yield(1000); - } + esp_delay(ms); } void arch_restart() { diff --git a/esphome/components/esp8266/hal.h b/esphome/components/esp8266/hal.h index effa9c9371..f3b33da692 100644 --- a/esphome/components/esp8266/hal.h +++ b/esphome/components/esp8266/hal.h @@ -58,6 +58,12 @@ __attribute__((always_inline)) inline const char *progmem_read_ptr(const char *c __attribute__((always_inline)) inline uint16_t progmem_read_uint16(const uint16_t *addr) { return pgm_read_word(addr); // NOLINT } +// Bulk PROGMEM copy: routes to the SDK's aligned-flash `memcpy_P` so callers +// don't have to drop to a byte-by-byte `progmem_read_byte` loop, which on +// ESP8266 is ~4x as many flash accesses as the bulk path. +__attribute__((always_inline)) inline void progmem_memcpy(void *dst, const void *src, size_t len) { + memcpy_P(dst, src, len); // NOLINT +} // NOLINTNEXTLINE(readability-identifier-naming) __attribute__((always_inline)) inline void delayMicroseconds(uint32_t us) { delay_microseconds_safe(us); } diff --git a/esphome/components/espnow/__init__.py b/esphome/components/espnow/__init__.py index 7861c0affa..13f278d3bc 100644 --- a/esphome/components/espnow/__init__.py +++ b/esphome/components/espnow/__init__.py @@ -17,7 +17,7 @@ from esphome.core import HexInt from esphome.types import ConfigType CODEOWNERS = ["@jesserockz"] - +AUTO_LOAD = ["network"] byte_vector = cg.std_vector.template(cg.uint8) peer_address_t = cg.std_ns.class_("array").template(cg.uint8, 6) diff --git a/esphome/components/espnow/espnow_component.cpp b/esphome/components/espnow/espnow_component.cpp index 91d44394e8..403e6f4944 100644 --- a/esphome/components/espnow/espnow_component.cpp +++ b/esphome/components/espnow/espnow_component.cpp @@ -149,12 +149,6 @@ bool ESPNowComponent::is_wifi_enabled() { } void ESPNowComponent::setup() { -#ifndef USE_WIFI - // Initialize LwIP stack for wake_loop_threadsafe() socket support - // When WiFi component is present, it handles esp_netif_init() - ESP_ERROR_CHECK(esp_netif_init()); -#endif - if (this->enable_on_boot_) { this->enable_(); } else { @@ -174,8 +168,6 @@ void ESPNowComponent::enable() { void ESPNowComponent::enable_() { if (!this->is_wifi_enabled()) { - esp_event_loop_create_default(); - wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT(); ESP_ERROR_CHECK(esp_wifi_init(&cfg)); diff --git a/esphome/components/ethernet/ethernet_component_esp32.cpp b/esphome/components/ethernet/ethernet_component_esp32.cpp index d4585bf100..6481c8c1f4 100644 --- a/esphome/components/ethernet/ethernet_component_esp32.cpp +++ b/esphome/components/ethernet/ethernet_component_esp32.cpp @@ -5,6 +5,7 @@ #include "esphome/core/application.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#include "w5500_custom_spi.h" #include #include @@ -163,11 +164,7 @@ void EthernetComponent::setup() { err = spi_bus_initialize(host, &buscfg, SPI_DMA_CH_AUTO); ESPHL_ERROR_CHECK(err, "SPI bus initialize error"); #endif - - err = esp_netif_init(); - ESPHL_ERROR_CHECK(err, "ETH netif init error"); - err = esp_event_loop_create_default(); - ESPHL_ERROR_CHECK(err, "ETH event loop error"); + // Network interface setup handled by network component esp_netif_config_t cfg = ESP_NETIF_DEFAULT_ETH(); this->eth_netif_ = esp_netif_new(&cfg); @@ -207,6 +204,10 @@ void EthernetComponent::setup() { #ifdef USE_ETHERNET_SPI_POLLING_SUPPORT w5500_config.poll_period_ms = this->polling_interval_; #endif + // Install the custom SPI driver that offloads the bulk RX/TX frame transfers off the busy-wait + // path. w5500_config (and the devcfg it references) outlives esp_eth_mac_new_w5500() below, which + // runs the driver's init(). + install_w5500_async_spi(w5500_config); #elif defined(USE_ETHERNET_DM9051) dm9051_config.int_gpio_num = this->interrupt_pin_; #ifdef USE_ETHERNET_SPI_POLLING_SUPPORT diff --git a/esphome/components/ethernet/w5500_custom_spi.cpp b/esphome/components/ethernet/w5500_custom_spi.cpp new file mode 100644 index 0000000000..ed4f149738 --- /dev/null +++ b/esphome/components/ethernet/w5500_custom_spi.cpp @@ -0,0 +1,118 @@ +#include "w5500_custom_spi.h" + +#if defined(USE_ESP32) && defined(USE_ETHERNET_W5500) + +#include +#include +#include +#include +#include + +namespace esphome::ethernet { + +namespace { + +// Per-device context returned by init() and handed back to read/write/deinit. +struct W5500CustomSpiContext { + spi_device_handle_t handle; + SemaphoreHandle_t lock; +}; + +// Transfers up to the ESP32 SPI hardware FIFO size (64 bytes) stay on the polling path; larger +// transfers (the frame payloads) use the blocking, DMA-backed transmit. +constexpr uint32_t W5500_SPI_BULK_THRESHOLD = 64; +constexpr uint32_t W5500_SPI_LOCK_TIMEOUT_MS = 50; + +void *w5500_custom_spi_init(const void *spi_config) { + const auto *config = static_cast(spi_config); + auto *ctx = new (std::nothrow) W5500CustomSpiContext{}; + if (ctx == nullptr) { + return nullptr; + } + // The W5500 SPI frame carries the 16-bit address in the command phase and the 8-bit control + // byte in the address phase; mirror what the stock driver configures. + spi_device_interface_config_t devcfg = *config->spi_devcfg; + devcfg.command_bits = 16; + devcfg.address_bits = 8; + if (spi_bus_add_device(config->spi_host_id, &devcfg, &ctx->handle) != ESP_OK) { + delete ctx; + return nullptr; + } + ctx->lock = xSemaphoreCreateMutex(); + if (ctx->lock == nullptr) { + spi_bus_remove_device(ctx->handle); + delete ctx; + return nullptr; + } + return ctx; +} + +esp_err_t w5500_custom_spi_deinit(void *spi_ctx) { + auto *ctx = static_cast(spi_ctx); + spi_bus_remove_device(ctx->handle); + vSemaphoreDelete(ctx->lock); + delete ctx; + return ESP_OK; +} + +// Runs one transaction under the device lock, choosing the polling vs blocking transmit by size. +// Bulk payloads (> FIFO size) block so the calling task sleeps while DMA runs; small register +// accesses stay on the cheaper polling path. Used by both read and write. +esp_err_t w5500_custom_spi_transfer(W5500CustomSpiContext *ctx, spi_transaction_t *trans, uint32_t len) { + if (xSemaphoreTake(ctx->lock, pdMS_TO_TICKS(W5500_SPI_LOCK_TIMEOUT_MS)) != pdTRUE) { + return ESP_ERR_TIMEOUT; + } + esp_err_t ret; + if (len > W5500_SPI_BULK_THRESHOLD) { + ret = spi_device_transmit(ctx->handle, trans); + } else { + ret = spi_device_polling_transmit(ctx->handle, trans); + } + xSemaphoreGive(ctx->lock); + return ret; +} + +esp_err_t w5500_custom_spi_write(void *spi_ctx, uint32_t cmd, uint32_t addr, const void *data, uint32_t len) { + auto *ctx = static_cast(spi_ctx); + spi_transaction_t trans = {}; + trans.cmd = static_cast(cmd); + trans.addr = addr; + trans.length = 8 * len; + trans.tx_buffer = data; + return w5500_custom_spi_transfer(ctx, &trans, len); +} + +esp_err_t w5500_custom_spi_read(void *spi_ctx, uint32_t cmd, uint32_t addr, void *data, uint32_t len) { + auto *ctx = static_cast(spi_ctx); + spi_transaction_t trans = {}; + // Reads of <= 4 bytes use the transaction's inline RX buffer to avoid 4-byte boundary + // overwrites of adjacent registers (same guard the stock driver uses). + const bool use_rxdata = len <= 4; + trans.flags = use_rxdata ? SPI_TRANS_USE_RXDATA : 0; + trans.cmd = static_cast(cmd); + trans.addr = addr; + trans.length = 8 * len; + trans.rx_buffer = data; + esp_err_t ret = w5500_custom_spi_transfer(ctx, &trans, len); + if (use_rxdata && (ret == ESP_OK)) { + memcpy(data, trans.rx_data, len); + } + return ret; +} + +} // namespace + +void install_w5500_async_spi(eth_w5500_config_t &config) { + // Point the custom driver's config at the W5500 config itself; init() reads spi_host_id and + // spi_devcfg back out of it. The self-reference is valid because both the config and the + // spi_devcfg it points at outlive the esp_eth_mac_new_w5500() call that runs init(). + config.custom_spi_driver.config = &config; + config.custom_spi_driver.init = w5500_custom_spi_init; + config.custom_spi_driver.deinit = w5500_custom_spi_deinit; + config.custom_spi_driver.read = w5500_custom_spi_read; + config.custom_spi_driver.write = w5500_custom_spi_write; +} + +} // namespace esphome::ethernet + +#endif // USE_ESP32 && USE_ETHERNET_W5500 diff --git a/esphome/components/ethernet/w5500_custom_spi.h b/esphome/components/ethernet/w5500_custom_spi.h new file mode 100644 index 0000000000..8756a149af --- /dev/null +++ b/esphome/components/ethernet/w5500_custom_spi.h @@ -0,0 +1,35 @@ +#pragma once + +#include "esphome/core/defines.h" + +#if defined(USE_ESP32) && defined(USE_ETHERNET_W5500) + +#include +// IDF 6.0 moved the per-chip SPI MAC drivers to the Espressif Component Registry; eth_w5500_config_t +// is no longer reachable through esp_eth.h and needs the explicit header. +#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(6, 0, 0) +#include +#else +#include +#endif + +namespace esphome::ethernet { + +// Installs a custom W5500 SPI driver that offloads the bulk frame transfers off the busy-wait path. +// +// The stock W5500 driver runs every SPI transfer through spi_device_polling_transmit(), which +// busy-waits the CPU for the whole transfer. The frame payload (one large read per received frame, +// one large write per transmitted frame) is by far the biggest transfer, so the RX task and the TX +// caller each spin for hundreds of microseconds per frame. This driver sends payload transfers +// through the blocking, interrupt-driven spi_device_transmit() instead, so the calling task sleeps +// while DMA moves the bytes. Small register accesses stay on the polling path, where the busy-wait +// is cheaper than an interrupt round-trip. +// +// Must be called before esp_eth_mac_new_w5500(). The driver reads spi_host_id and spi_devcfg back +// out of `config` in its init() callback, so `config` (and the spi_devcfg it points at) must stay +// alive until esp_eth_mac_new_w5500() returns. +void install_w5500_async_spi(eth_w5500_config_t &config); + +} // namespace esphome::ethernet + +#endif // USE_ESP32 && USE_ETHERNET_W5500 diff --git a/esphome/components/external_components/__init__.py b/esphome/components/external_components/__init__.py index 6eb577e5ad..c892ec1112 100644 --- a/esphome/components/external_components/__init__.py +++ b/esphome/components/external_components/__init__.py @@ -81,7 +81,7 @@ def _process_single_config(config: dict[str, Any]) -> None: elif conf[CONF_TYPE] == TYPE_LOCAL: components_dir = Path(CORE.relative_config_path(conf[CONF_PATH])) else: - raise NotImplementedError() + raise NotImplementedError if config[CONF_COMPONENTS] == "all": num_components = len(list(components_dir.glob("*/__init__.py"))) diff --git a/esphome/components/fingerprint_grow/fingerprint_grow.cpp b/esphome/components/fingerprint_grow/fingerprint_grow.cpp index 3f57789034..b38d42191b 100644 --- a/esphome/components/fingerprint_grow/fingerprint_grow.cpp +++ b/esphome/components/fingerprint_grow/fingerprint_grow.cpp @@ -206,6 +206,7 @@ uint8_t FingerprintGrowComponent::save_fingerprint_() { break; case ENROLL_MISMATCH: ESP_LOGE(TAG, "Scans do not match"); + [[fallthrough]]; default: return this->data_[0]; } diff --git a/esphome/components/font/__init__.py b/esphome/components/font/__init__.py index a10c45a9d7..7510f2f8b6 100644 --- a/esphome/components/font/__init__.py +++ b/esphome/components/font/__init__.py @@ -401,7 +401,7 @@ def validate_file_shorthand(value): data[CONF_WEIGHT] = weight[1:] return font_file_schema(data) - if value.startswith("http://") or value.startswith("https://"): + if value.startswith(("http://", "https://")): return font_file_schema( { CONF_TYPE: TYPE_WEB, @@ -563,13 +563,13 @@ async def to_code(config): point_set.update(flatten(config[CONF_GLYPHS])) # Create the codepoint to font file map base_font = FONT_CACHE[config[CONF_FILE]] - point_font_map: dict[str, Face] = {c: base_font for c in point_set} + point_font_map: dict[str, Face] = dict.fromkeys(point_set, base_font) # process extras, updating the map and extending the codepoint list for extra in config[CONF_EXTRAS]: extra_points = flatten(extra[CONF_GLYPHS]) point_set.update(extra_points) extra_font = FONT_CACHE[extra[CONF_FILE]] - point_font_map.update({c: extra_font for c in extra_points}) + point_font_map.update(dict.fromkeys(extra_points, extra_font)) codepoints = list(point_set) codepoints.sort(key=functools.cmp_to_key(glyph_comparator)) @@ -594,7 +594,9 @@ async def to_code(config): x.height, ] for (x, y) in zip( - glyph_args, list(accumulate([len(x.bitmap_data) for x in glyph_args])) + glyph_args, + list(accumulate([len(x.bitmap_data) for x in glyph_args])), + strict=True, ) ] diff --git a/esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.cpp b/esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.cpp index 835dc4aac0..24d3529fb4 100644 --- a/esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.cpp +++ b/esphome/components/ft5x06/touchscreen/ft5x06_touchscreen.cpp @@ -15,6 +15,16 @@ void FT5x06Touchscreen::setup() { this->attach_interrupt_(this->interrupt_pin_, gpio::INTERRUPT_FALLING_EDGE); } + // reading the chip registers to get max x/y does not seem to work. + if (this->display_ != nullptr) { + if (this->x_raw_max_ == this->x_raw_min_) { + this->x_raw_max_ = this->display_->get_native_width(); + } + if (this->y_raw_max_ == this->y_raw_min_) { + this->y_raw_max_ = this->display_->get_native_height(); + } + } + // wait 200ms after reset. this->set_timeout(200, [this] { this->continue_setup_(); }); } @@ -39,15 +49,6 @@ void FT5x06Touchscreen::continue_setup_() { this->mark_failed(); return; } - // reading the chip registers to get max x/y does not seem to work. - if (this->display_ != nullptr) { - if (this->x_raw_max_ == this->x_raw_min_) { - this->x_raw_max_ = this->display_->get_native_width(); - } - if (this->y_raw_max_ == this->y_raw_min_) { - this->y_raw_max_ = this->display_->get_native_height(); - } - } } void FT5x06Touchscreen::update_touches() { @@ -71,7 +72,7 @@ void FT5x06Touchscreen::update_touches() { uint16_t x = encode_uint16(data[i][0] & 0x0F, data[i][1]); uint16_t y = encode_uint16(data[i][2] & 0xF, data[i][3]); - ESP_LOGD(TAG, "Read %X status, id: %d, pos %d/%d", status, id, x, y); + ESP_LOGV(TAG, "Read %X status, id: %d, pos %d/%d", status, id, x, y); if (status == 0 || status == 2) { this->add_raw_touch_position_(id, x, y); } diff --git a/esphome/components/homeassistant/sensor/homeassistant_sensor.cpp b/esphome/components/homeassistant/sensor/homeassistant_sensor.cpp index 112795a4ff..b79a56953a 100644 --- a/esphome/components/homeassistant/sensor/homeassistant_sensor.cpp +++ b/esphome/components/homeassistant/sensor/homeassistant_sensor.cpp @@ -17,9 +17,9 @@ void HomeassistantSensor::setup() { } if (this->attribute_ != nullptr) { - ESP_LOGD(TAG, "'%s::%s': Got attribute state %.2f", this->entity_id_, this->attribute_, *val); + ESP_LOGV(TAG, "'%s::%s': Got attribute state %.2f", this->entity_id_, this->attribute_, *val); } else { - ESP_LOGD(TAG, "'%s': Got state %.2f", this->entity_id_, *val); + ESP_LOGV(TAG, "'%s': Got state %.2f", this->entity_id_, *val); } this->publish_state(*val); }); diff --git a/esphome/components/homeassistant/time/__init__.py b/esphome/components/homeassistant/time/__init__.py index 62cb96a25a..05ca86a26e 100644 --- a/esphome/components/homeassistant/time/__init__.py +++ b/esphome/components/homeassistant/time/__init__.py @@ -1,7 +1,7 @@ import esphome.codegen as cg from esphome.components import time as time_ import esphome.config_validation as cv -from esphome.const import CONF_ID +from esphome.const import CONF_ID, CONF_TIMEZONE from .. import homeassistant_ns @@ -21,3 +21,5 @@ async def to_code(config): await time_.register_time(var, config) await cg.register_component(var, config) cg.add_define("USE_HOMEASSISTANT_TIME") + if CONF_TIMEZONE not in config: + cg.add_define("USE_HOMEASSISTANT_TIMEZONE") diff --git a/esphome/components/http_request/__init__.py b/esphome/components/http_request/__init__.py index 90879c459e..fd033dac7f 100644 --- a/esphome/components/http_request/__init__.py +++ b/esphome/components/http_request/__init__.py @@ -1,3 +1,5 @@ +from pathlib import Path + from esphome import automation import esphome.codegen as cg from esphome.components import esp32 @@ -63,7 +65,7 @@ CONF_JSON = "json" def validate_url(value): value = cv.url(value) - if value.startswith("http://") or value.startswith("https://"): + if value.startswith(("http://", "https://")): return value raise cv.Invalid("URL must start with 'http://' or 'https://'") @@ -174,7 +176,7 @@ async def to_code(config): if config.get(CONF_VERIFY_SSL): if ca_cert_path := config.get(CONF_CA_CERTIFICATE_PATH): - with open(ca_cert_path, encoding="utf-8") as f: + with Path(ca_cert_path).open(encoding="utf-8") as f: ca_cert_content = f.read() cg.add(var.set_ca_certificate(ca_cert_content)) else: diff --git a/esphome/components/i2s_audio/speaker/__init__.py b/esphome/components/i2s_audio/speaker/__init__.py index 759cc40ca9..8215d8b518 100644 --- a/esphome/components/i2s_audio/speaker/__init__.py +++ b/esphome/components/i2s_audio/speaker/__init__.py @@ -89,10 +89,10 @@ def _set_num_channels_from_config(config): def _set_stream_limits(config): if config.get(CONF_SPDIF_MODE, False): - # SPDIF mode: fixed to 16-bit stereo at configured sample rate + # SPDIF mode: 16/24/32-bit audio and stereo at configured sample rate audio.set_stream_limits( min_bits_per_sample=16, - max_bits_per_sample=16, + max_bits_per_sample=32, min_channels=2, max_channels=2, min_sample_rate=config.get(CONF_SAMPLE_RATE), @@ -213,9 +213,6 @@ def _final_validate(config): ) if config[CONF_CHANNEL] != CONF_STEREO: raise cv.Invalid("SPDIF mode only supports stereo channel configuration") - # bits_per_sample is converted to float by the schema - if config[CONF_BITS_PER_SAMPLE] != 16: - raise cv.Invalid("SPDIF mode only supports 16 bits per sample") if not config[CONF_USE_APLL]: raise cv.Invalid( "SPDIF mode requires 'use_apll: true' for accurate clock generation" diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_spdif.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_spdif.cpp index 8f67562a77..989bcf2977 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_spdif.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_spdif.cpp @@ -138,21 +138,21 @@ void I2SAudioSpeakerSPDIF::run_speaker_task() { // Reset lockstep records queue so it starts paired with the (also-reset) i2s_event_queue_. xQueueReset(this->write_records_queue_); - const uint32_t dma_buffers_duration_ms = DMA_BUFFER_DURATION_MS * SPDIF_DMA_BUFFERS_COUNT; - // Ensure ring buffer duration is at least the duration of all DMA buffers - const uint32_t ring_buffer_duration = std::max(dma_buffers_duration_ms, this->buffer_duration_ms_); - // The DMA buffers may have more bits per sample, so calculate buffer sizes based on the input audio stream info const size_t bytes_per_frame = this->current_stream_info_.frames_to_bytes(1); - // Round the ring buffer size down to a multiple of bytes_per_frame so the wrap boundary stays frame-aligned and - // avoids unnecessary single-frame splices. - const size_t ring_buffer_size = - (this->current_stream_info_.ms_to_bytes(ring_buffer_duration) / bytes_per_frame) * bytes_per_frame; - // For SPDIF mode, one DMA buffer = one SPDIF block = 192 PCM frames + // For SPDIF mode, one DMA buffer = one SPDIF block = 192 PCM frames (~4 ms at 48 kHz), + // not the ~15 ms a standard I2S DMA buffer holds. Derive the DMA floor from actual block size. const uint32_t frames_to_fill_single_dma_buffer = SPDIF_BLOCK_SAMPLES; const size_t bytes_to_fill_single_dma_buffer = this->current_stream_info_.frames_to_bytes(frames_to_fill_single_dma_buffer); + const size_t dma_buffers_floor_bytes = bytes_to_fill_single_dma_buffer * SPDIF_DMA_BUFFERS_COUNT; + + // Round the ring buffer size down to a multiple of bytes_per_frame so the wrap boundary stays frame-aligned and + // avoids unnecessary single-frame splices. Ensure it is at least large enough to cover all DMA buffers. + const size_t requested_ring_buffer_bytes = + (this->current_stream_info_.ms_to_bytes(this->buffer_duration_ms_) / bytes_per_frame) * bytes_per_frame; + const size_t ring_buffer_size = std::max(dma_buffers_floor_bytes, requested_ring_buffer_bytes); bool successful_setup = false; std::unique_ptr audio_source; @@ -177,7 +177,8 @@ void I2SAudioSpeakerSPDIF::run_speaker_task() { // on_sent events drain in lockstep without crediting any audio frames. this->spdif_encoder_->set_preload_mode(true); for (size_t i = 0; i < SPDIF_DMA_BUFFERS_COUNT; i++) { - esp_err_t preload_err = this->spdif_encoder_->flush_with_silence(pdMS_TO_TICKS(DMA_BUFFER_DURATION_MS)); + // i2s_channel_preload_data is non-blocking (returns immediately when the preload buffer fills), so no wait. + esp_err_t preload_err = this->spdif_encoder_->flush_with_silence(0); if (preload_err != ESP_OK) { break; // DMA preload buffer full or error } @@ -410,8 +411,9 @@ esp_err_t I2SAudioSpeakerSPDIF::start_i2s_driver(audio::AudioStreamInfo &audio_s this->sample_rate_, audio_stream_info.get_sample_rate()); return ESP_ERR_NOT_SUPPORTED; } - if (audio_stream_info.get_bits_per_sample() != 16) { - ESP_LOGE(TAG, "Only supports 16 bits per sample"); + const uint8_t bits_per_sample = audio_stream_info.get_bits_per_sample(); + if (bits_per_sample != 16 && bits_per_sample != 24 && bits_per_sample != 32) { + ESP_LOGE(TAG, "Only supports 16, 24, or 32 bits per sample (got %u)", (unsigned) bits_per_sample); return ESP_ERR_NOT_SUPPORTED; } if (audio_stream_info.get_channels() != 2) { @@ -419,11 +421,8 @@ esp_err_t I2SAudioSpeakerSPDIF::start_i2s_driver(audio::AudioStreamInfo &audio_s return ESP_ERR_NOT_SUPPORTED; } - if (this->slot_bit_width_ != I2S_SLOT_BIT_WIDTH_AUTO && - (i2s_slot_bit_width_t) audio_stream_info.get_bits_per_sample() > this->slot_bit_width_) { - ESP_LOGE(TAG, "Stream bits per sample must be less than or equal to the speaker's configuration"); - return ESP_ERR_NOT_SUPPORTED; - } + // Tell the encoder what input width to expect. 32-bit input is truncated to 24-bit on the wire. + this->spdif_encoder_->set_bytes_per_sample(bits_per_sample / 8); if (!this->parent_->try_lock()) { ESP_LOGE(TAG, "Parent bus is busy"); diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp index 680ca069c0..691f68e912 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.cpp @@ -2,6 +2,7 @@ #ifdef USE_ESP32 +#include #include #include "esphome/components/audio/audio.h" @@ -299,6 +300,15 @@ void I2SAudioSpeakerBase::stop_i2s_driver_() { i2s_channel_disable(this->tx_handle_); i2s_del_channel(this->tx_handle_); this->tx_handle_ = nullptr; + + // i2s_del_channel() leaves dout wired to this port's data-out signal in the GPIO matrix: it only + // clears an internal reservation mask, never the esp_rom_gpio_connect_out_signal() routing that + // setup installed. If another speaker reuses this port (shared bus), its audio still reaches our + // dout. Detach the pin and drive it low so a stale output stops driving downstream hardware: a + // SPDIF optical transmitter would otherwise stay lit, and an analog DAC would emit noise. + gpio_reset_pin(this->dout_pin_); + gpio_set_direction(this->dout_pin_, GPIO_MODE_OUTPUT); + gpio_set_level(this->dout_pin_, 0); } this->parent_->unlock(); } diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h index 20bb05e322..34792bdbea 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker.h @@ -19,7 +19,6 @@ namespace esphome::i2s_audio { // Shared constants used by both standard and SPDIF speaker implementations -static constexpr uint32_t DMA_BUFFER_DURATION_MS = 15; static constexpr size_t TASK_STACK_SIZE = 4096; static constexpr ssize_t TASK_PRIORITY = 19; diff --git a/esphome/components/i2s_audio/speaker/i2s_audio_speaker_standard.cpp b/esphome/components/i2s_audio/speaker/i2s_audio_speaker_standard.cpp index e69601e87a..ffe901504d 100644 --- a/esphome/components/i2s_audio/speaker/i2s_audio_speaker_standard.cpp +++ b/esphome/components/i2s_audio/speaker/i2s_audio_speaker_standard.cpp @@ -16,6 +16,7 @@ namespace esphome::i2s_audio { static const char *const TAG = "i2s_audio.speaker.std"; +static constexpr uint32_t DMA_BUFFER_DURATION_MS = 15; static constexpr size_t DMA_BUFFERS_COUNT = 4; // Sized to comfortably absorb scheduling jitter: at most DMA_BUFFERS_COUNT events can be in flight, // doubled so that a transient backlog never overruns the queue (which would desync the lockstep diff --git a/esphome/components/i2s_audio/speaker/spdif_encoder.cpp b/esphome/components/i2s_audio/speaker/spdif_encoder.cpp index 42a72346cc..30146e0a70 100644 --- a/esphome/components/i2s_audio/speaker/spdif_encoder.cpp +++ b/esphome/components/i2s_audio/speaker/spdif_encoder.cpp @@ -17,7 +17,7 @@ static constexpr uint8_t PREAMBLE_M = 0x1d; // Left channel (not block start) static constexpr uint8_t PREAMBLE_W = 0x1b; // Right channel // BMC encoding of 4 zero bits starting at phase HIGH: 00_11_00_11 = 0x33 -// Since both aux nibbles (bits 4-7, 8-11) are zero for 16-bit audio and phase is preserved, both are 0x33. +// Used as a constant in the 16-bit subframe path, where bits 4-11 are always zero. static constexpr uint32_t BMC_ZERO_NIBBLE = 0x33; // Constexpr BMC encoder for compile-time LUT generation. @@ -36,21 +36,43 @@ static constexpr uint16_t bmc_lut_encode(uint32_t data, uint8_t num_bits) { return bmc; } -// 4-bit BMC lookup table: 16 entries (16 bytes in flash) -// Index: 4-bit data value (0-15), always phase=true start +// Compile-time parity helper (constexpr-friendly, runs only at LUT build time). +static constexpr uint32_t bmc_lut_parity(uint32_t value, uint32_t num_bits) { + uint32_t p = 0; + for (uint32_t b = 0; b < num_bits; b++) + p ^= (value >> b) & 1u; + return p; +} + +// Combined BMC + phase-delta lookup tables. +// Each entry packs the BMC pattern (lower bits, phase=high start) together with +// a phase-mask delta in bits 16-31 (0xFFFF if the input has odd parity, else 0). +// XORing the delta into the running phase mask propagates parity across chunks +// without an explicit popcount. + +// 4-bit BMC lookup table: 16 entries x uint32_t = 64 bytes in flash. +// Bits 0-7 : 8-bit BMC pattern (phase=high start) +// Bits 16-31 : phase-mask delta (0xFFFFu if odd parity, else 0) static constexpr auto BMC_LUT_4 = [] { - std::array t{}; - for (uint32_t i = 0; i < 16; i++) - t[i] = static_cast(bmc_lut_encode(i, 4)); + std::array t{}; + for (uint32_t i = 0; i < 16; i++) { + uint32_t bmc = bmc_lut_encode(i, 4); + uint32_t delta = bmc_lut_parity(i, 4) ? 0xFFFF0000u : 0u; + t[i] = bmc | delta; + } return t; }(); -// 8-bit BMC lookup table: 256 entries (512 bytes in flash) -// Index: 8-bit data value (0-255), always phase=true start +// 8-bit BMC lookup table: 256 entries x uint32_t = 1024 bytes in flash. +// Bits 0-15 : 16-bit BMC pattern (phase=high start) +// Bits 16-31 : phase-mask delta (0xFFFFu if odd parity, else 0) static constexpr auto BMC_LUT_8 = [] { - std::array t{}; - for (uint32_t i = 0; i < 256; i++) - t[i] = bmc_lut_encode(i, 8); + std::array t{}; + for (uint32_t i = 0; i < 256; i++) { + uint32_t bmc = bmc_lut_encode(i, 8); + uint32_t delta = bmc_lut_parity(i, 8) ? 0xFFFF0000u : 0u; + t[i] = bmc | delta; + } return t; }(); @@ -63,7 +85,7 @@ bool SPDIFEncoder::setup() { } ESP_LOGV(TAG, "Buffer allocated (%zu bytes)", SPDIF_BLOCK_SIZE_BYTES); - // Build initial channel status block with default sample rate + // Build initial channel status block with default sample rate and width this->build_channel_status_(); this->reset(); @@ -73,7 +95,7 @@ bool SPDIFEncoder::setup() { void SPDIFEncoder::reset() { this->spdif_block_ptr_ = this->spdif_block_buf_.get(); this->frame_in_block_ = 0; - this->is_left_channel_ = true; + this->block_buf_is_silence_block_ = false; } void SPDIFEncoder::set_sample_rate(uint32_t sample_rate) { @@ -84,31 +106,27 @@ void SPDIFEncoder::set_sample_rate(uint32_t sample_rate) { } } +void SPDIFEncoder::set_bytes_per_sample(uint8_t bytes_per_sample) { + if (bytes_per_sample != 2 && bytes_per_sample != 3 && bytes_per_sample != 4) { + ESP_LOGE(TAG, "Unsupported bytes per sample: %u", (unsigned) bytes_per_sample); + return; + } + if (this->bytes_per_sample_ != bytes_per_sample) { + this->bytes_per_sample_ = bytes_per_sample; + this->build_channel_status_(); + // Discard any partial block built at the previous width so we never mix widths on the wire. + this->reset(); + ESP_LOGD(TAG, "Input width set to %u-bit", (unsigned) bytes_per_sample * 8); + } +} + void SPDIFEncoder::build_channel_status_() { // IEC 60958-3 Consumer Channel Status Block (192 bits = 24 bytes) - // Transmitted LSB-first within each byte, one bit per frame via C bit - // - // Byte 0: Control bits - // Bit 0: 0 = Consumer format (not professional AES3) - // Bit 1: 0 = PCM audio (not non-audio data like AC3) - // Bit 2: 0 = No copyright assertion - // Bits 3-5: 000 = No pre-emphasis - // Bits 6-7: 00 = Mode 0 (basic consumer format) - // - // Byte 1: Category code (0x00 = general, 0x01 = CD, etc.) - // - // Byte 2: Source/channel numbers - // Bits 0-3: Source number (0 = unspecified) - // Bits 4-7: Channel number (0 = unspecified) - // - // Byte 3: Sample frequency and clock accuracy - // Bits 0-3: Sample frequency code - // Bits 4-5: Clock accuracy (00 = Level II, ±1000 ppm, appropriate for ESP32) - // Bits 6-7: Reserved (0) - // - // Bytes 4-23: Reserved (zeros for basic compliance) + // Transmitted LSB-first within each byte, one bit per frame via C bit. + + // Any cached silence block was built for the previous channel status; it is now stale. + this->block_buf_is_silence_block_ = false; - // Clear all bytes first this->channel_status_.fill(0); // Byte 0: Consumer, PCM audio, no copyright, no pre-emphasis, Mode 0 @@ -140,132 +158,148 @@ void SPDIFEncoder::build_channel_status_() { // Byte 3: freq_code in bits 0-3, clock accuracy (00) in bits 4-5 this->channel_status_[3] = freq_code; // Clock accuracy bits 4-5 are already 0 - // Bytes 4-23 remain zero (word length not specified, no original sample freq, etc.) + // Byte 4: Word length encoding (IEC 60958-3 consumer) + // bit 0: max length flag (0 = max 20 bits, 1 = max 24 bits) + // bits 1-3: word length code relative to the max + // For our supported widths: + // 16-bit (max 20): 0b0010 = 0x02 -- "16 bits, max 20" + // 24-bit (max 24): 0b1101 = 0x0D -- "24 bits, max 24" + // 32-bit input is truncated to 24-bit on the wire, so use the 24-bit code. + uint8_t word_length_code; + switch (this->bytes_per_sample_) { + case 2: + word_length_code = 0x02; + break; + case 3: // Shared case + case 4: + word_length_code = 0x0D; + break; + default: + word_length_code = 0x00; // not specified + break; + } + this->channel_status_[4] = word_length_code; } -HOT void SPDIFEncoder::encode_sample_(const uint8_t *pcm_sample) { - // ============================================================================ - // Build raw 32-bit subframe (IEC 60958 format) - // ============================================================================ - // Bit layout: - // Bits 0-3: Preamble (handled separately, not in raw_subframe) - // Bits 4-7: Auxiliary audio data (zeros for 16-bit audio) - // Bits 8-11: Audio LSB extension (zeros for 16-bit audio) - // Bits 12-27: 16-bit audio sample (MSB-aligned in 20-bit audio field) - // Bit 28: V (Validity) - 0 = valid audio - // Bit 29: U (User data) - 0 - // Bit 30: C (Channel status) - from channel status block - // Bit 31: P (Parity) - even parity over bits 4-31 - // ============================================================================ +// Extract the C bit for the given frame from channel_status_ and shift it into bit 30 +// so it can be OR'd directly into a raw subframe. +ESPHOME_ALWAYS_INLINE static inline uint32_t c_bit_for_frame(const std::array &channel_status, + uint32_t frame) { + return static_cast((channel_status[frame >> 3] >> (frame & 7)) & 1u) << 30; +} - // Place 16-bit audio sample at bits 12-27 (little-endian input: [0]=LSB, [1]=MSB) - uint32_t raw_subframe = (static_cast(pcm_sample[1]) << 20) | (static_cast(pcm_sample[0]) << 12); +// ============================================================================ +// IEC 60958 subframe bit layout +// ============================================================================ +// Bits 0-3: Preamble (handled separately, not in raw_subframe) +// Bits 4-7: Auxiliary audio data / 24-bit audio LSB +// Bits 8-11: Audio LSB extension (zero for 16-bit, low nibble of audio for 24-bit) +// Bits 12-27: Audio sample (16 high bits in 16-bit mode, mid 16 bits in 24-bit mode) +// Bit 28: V (Validity) - 0 = valid audio +// Bit 29: U (User data) - 0 +// Bit 30: C (Channel status) - from channel status block +// Bit 31: P (Parity) - even parity over bits 4-31 +// ============================================================================ - // V = 0 (valid audio), U = 0 (no user data) - // C = channel status bit for current frame (same bit used for both L and R subframes) - bool c_bit = this->get_channel_status_bit_(this->frame_in_block_); - if (c_bit) { - raw_subframe |= (1U << 30); +// Build a raw IEC 60958 subframe from PCM little-endian input of width Bps bytes. +// Caller is responsible for OR-ing in the C bit and parity. +template ESPHOME_ALWAYS_INLINE static inline uint32_t build_raw_subframe(const uint8_t *pcm_sample) { + static_assert(Bps == 2 || Bps == 3 || Bps == 4, "Unsupported bytes per sample"); + if constexpr (Bps == 2) { + // 16-bit input: MSB-aligned in the 20-bit audio field, bits 12-27. + return (static_cast(pcm_sample[1]) << 20) | (static_cast(pcm_sample[0]) << 12); + } else if constexpr (Bps == 3) { + // 24-bit input: full 24-bit audio field, bits 4-27. + return (static_cast(pcm_sample[2]) << 20) | (static_cast(pcm_sample[1]) << 12) | + (static_cast(pcm_sample[0]) << 4); + } else { // Bps == 4 + // 32-bit input truncated to 24-bit: drop the lowest byte. + return (static_cast(pcm_sample[3]) << 20) | (static_cast(pcm_sample[2]) << 12) | + (static_cast(pcm_sample[1]) << 4); } +} - // Calculate even parity over bits 4-30 - // This ensures consistent BMC ending phase regardless of audio content - uint32_t bits_4_30 = (raw_subframe >> 4) & 0x07FFFFFF; // 27 bits (4-30) - uint32_t ones_count = __builtin_popcount(bits_4_30); - uint32_t parity = ones_count & 1; // 1 if odd count, 0 if even - raw_subframe |= parity << 31; // Set P bit to make total even +// BMC-encode a subframe and write the two output uint32 words to dst. Caller passes +// raw_subframe with the C bit set (bit 30) and the P bit cleared (bit 31 = 0). P is +// derived from the cumulative parity-mask delta of the per-byte LUT lookups. +// +// I2S halfword swap means word[0] transmits as: bits 24-31, 16-23, 8-15, 0-7. +// word[1] transmits as: bits 16-31, 0-15. Within each halfword, MSB-first. +// All preambles end at phase HIGH, so phase=true at the start of bit 4. +// +// P-bit derivation: BMC_LUT_*'s upper half encodes the parity of the input chunk. Each +// chunk's parity delta is shifted down (`lut >> 16`) into a phase_mask that lives in the +// low 16 bits, so the same value can also be XORed against subsequent BMC patterns to +// invert phase. XOR'ing those deltas through all chunks (with bit 31 = 0) yields the +// parity of bits 4-30 in the low bits of phase_mask -- the required value of the P bit +// for even total parity. The BMC of bit 31 lives in bit 0 of the high-byte BMC output +// (i = 7 maps to position (8-1-7)*2 = 0); flipping the source bit flips only the lower +// BMC bit (= phase XOR bit), so applying P is `bmc_24_31 ^= phase_mask & 1u`. +template +ESPHOME_ALWAYS_INLINE static inline void bmc_encode_subframe(uint32_t raw_subframe, uint8_t preamble, uint32_t *dst) { + if constexpr (Bps == 2) { + // 16-bit path: bits 4-11 are zero, encoded inline as BMC_ZERO_NIBBLE constants. + // Eight zero source bits with start phase=HIGH end at phase=HIGH (popcount of zeros is even), + // so encoding of bits 12-15 starts at phase=true. Zeros contribute 0 to parity. + uint32_t nibble = (raw_subframe >> 12) & 0xF; + uint32_t lut_n = BMC_LUT_4[nibble]; + uint32_t bmc_12_15 = lut_n & 0xFFu; + uint32_t phase_mask = lut_n >> 16; // 0xFFFFu if odd parity, else 0 - // ============================================================================ - // Select preamble based on position in block and channel - // ============================================================================ - // B = block start (left channel, frame 0 of 192-frame block) - // M = left channel (frames 1-191) - // W = right channel (all frames) - uint8_t preamble; - if (this->is_left_channel_) { - preamble = (this->frame_in_block_ == 0) ? PREAMBLE_B : PREAMBLE_M; + uint32_t byte_mid = (raw_subframe >> 16) & 0xFF; + uint32_t lut_m = BMC_LUT_8[byte_mid]; + uint32_t bmc_16_23 = (lut_m & 0xFFFFu) ^ phase_mask; + phase_mask ^= lut_m >> 16; + + uint32_t byte_hi = (raw_subframe >> 24) & 0xFF; // bit 7 (= P) is 0 by precondition + uint32_t lut_h = BMC_LUT_8[byte_hi]; + uint32_t bmc_24_31 = (lut_h & 0xFFFFu) ^ phase_mask; + phase_mask ^= lut_h >> 16; + // phase_mask now reflects parity of bits 4-30. Apply P by flipping bit 0 of bmc_24_31. + bmc_24_31 ^= phase_mask & 1u; + + dst[0] = bmc_12_15 | (BMC_ZERO_NIBBLE << 8) | (BMC_ZERO_NIBBLE << 16) | (static_cast(preamble) << 24); + dst[1] = bmc_24_31 | (bmc_16_23 << 16); } else { - preamble = PREAMBLE_W; + // 24-bit (and 32-bit truncated) path: bits 4-11 are live audio. + uint32_t byte_lo = (raw_subframe >> 4) & 0xFF; + uint32_t lut_l = BMC_LUT_8[byte_lo]; + uint32_t bmc_4_11 = lut_l & 0xFFFFu; + uint32_t phase_mask = lut_l >> 16; // 0xFFFFu if odd parity, else 0 + + uint32_t nibble = (raw_subframe >> 12) & 0xF; + uint32_t lut_n = BMC_LUT_4[nibble]; + uint32_t bmc_12_15 = (lut_n & 0xFFu) ^ (phase_mask & 0xFFu); + phase_mask ^= lut_n >> 16; + + uint32_t byte_mid = (raw_subframe >> 16) & 0xFF; + uint32_t lut_m = BMC_LUT_8[byte_mid]; + uint32_t bmc_16_23 = (lut_m & 0xFFFFu) ^ phase_mask; + phase_mask ^= lut_m >> 16; + + uint32_t byte_hi = (raw_subframe >> 24) & 0xFF; // bit 7 (= P) is 0 by precondition + uint32_t lut_h = BMC_LUT_8[byte_hi]; + uint32_t bmc_24_31 = (lut_h & 0xFFFFu) ^ phase_mask; + phase_mask ^= lut_h >> 16; + bmc_24_31 ^= phase_mask & 1u; + + // word[0]: bits 24-31 = preamble, bits 8-23 = bmc(4-11), bits 0-7 = bmc(12-15) + // word[1]: bits 16-31 = bmc(16-23), bits 0-15 = bmc(24-31) + dst[0] = bmc_12_15 | (bmc_4_11 << 8) | (static_cast(preamble) << 24); + dst[1] = bmc_24_31 | (bmc_16_23 << 16); } +} - // ============================================================================ - // BMC encode the data portion (bits 4-31) using lookup tables - // ============================================================================ - // The I2S uses 16-bit halfword swap: bits 16-31 transmit before bits 0-15. - // This applies to BOTH word[0] and word[1]. - // - // word[0] transmission order: [16-23] → [24-31] → [0-7] → [8-15] - // For correct S/PDIF subframe order (preamble → aux → audio): - // - bits 16-23: preamble (8 BMC bits) - // - bits 24-31: BMC(subframe bits 4-7) - first aux nibble - // - bits 0-7: BMC(subframe bits 8-11) - second aux nibble - // - bits 8-15: BMC(subframe bits 12-15) - audio low nibble - // - // word[1] transmission order: [16-31] → [0-15] - // For correct S/PDIF subframe order: - // - bits 16-31: BMC(subframe bits 16-23) - audio mid byte - // - bits 0-15: BMC(subframe bits 24-31) - audio high nibble + VUCP - // ============================================================================ - - // All preambles end at phase HIGH. Bits 4-11 are always zero for 16-bit audio; - // two zero nibbles flip phase 8 times total → back to HIGH. - // So bits 12-15 always start encoding at phase=true. - - // Bits 12-15: 4-bit LUT lookup (always phase=true start) - uint32_t nibble = (raw_subframe >> 12) & 0xF; - uint32_t bmc_12_15 = BMC_LUT_4[nibble]; - - // Phase tracking via branchless XOR mask: - // - 0x0000 means phase=true (use LUT value directly) - // - 0xFFFF means phase=false (complement LUT value) - // End phase = start XOR (popcount & 1) since zero-bits flip phase, - // and for even bit widths: #zeros parity == popcount parity. - uint32_t phase_mask = -(__builtin_popcount(nibble) & 1u) & 0xFFFF; - - // Bits 16-23: 8-bit LUT lookup with phase correction - uint32_t byte_mid = (raw_subframe >> 16) & 0xFF; - uint32_t bmc_16_23 = BMC_LUT_8[byte_mid] ^ phase_mask; - phase_mask ^= -(__builtin_popcount(byte_mid) & 1u) & 0xFFFF; - - // Bits 24-31: 8-bit LUT lookup with phase correction - uint32_t byte_hi = (raw_subframe >> 24) & 0xFF; - uint32_t bmc_24_31 = BMC_LUT_8[byte_hi] ^ phase_mask; - - // ============================================================================ - // Combine with correct positioning for I2S transmission - // ============================================================================ - // I2S with halfword swap: transmits bits 16-31, then bits 0-15. - // Within each halfword, MSB (highest bit) is transmitted first. - // - // For upper halfword (bits 16-31): bit 31 → bit 16 - // For lower halfword (bits 0-15): bit 15 → bit 0 - // - // Desired S/PDIF order: preamble → bmc_4_7 → bmc_8_11 → bmc_12_15 - // - // word[0] layout for correct transmission: - // bits 24-31: preamble (transmitted 1st, as MSB of upper halfword) - // bits 16-23: BMC_ZERO_NIBBLE (transmitted 2nd, aux bits 4-7) - // bits 8-15: BMC_ZERO_NIBBLE (transmitted 3rd, aux bits 8-11) - // bits 0-7: bmc_12_15 (transmitted 4th, audio low nibble) - // - // word[1] layout: - // bits 16-31: bmc_16_23 (transmitted 5th) - // bits 0-15: bmc_24_31 (transmitted 6th) - this->spdif_block_ptr_[0] = - bmc_12_15 | (BMC_ZERO_NIBBLE << 8) | (BMC_ZERO_NIBBLE << 16) | (static_cast(preamble) << 24); - this->spdif_block_ptr_[1] = bmc_24_31 | (bmc_16_23 << 16); - this->spdif_block_ptr_ += 2; - - // ============================================================================ - // Update position tracking - // ============================================================================ - if (!this->is_left_channel_) { - // Completed a stereo frame, advance frame counter - if (++this->frame_in_block_ >= SPDIF_BLOCK_SAMPLES) { - this->frame_in_block_ = 0; - } +template void SPDIFEncoder::encode_silence_frame_() { + static constexpr uint8_t SILENCE[4] = {0, 0, 0, 0}; + uint32_t raw = build_raw_subframe(SILENCE) | c_bit_for_frame(this->channel_status_, this->frame_in_block_); + uint8_t preamble_l = (this->frame_in_block_ == 0) ? PREAMBLE_B : PREAMBLE_M; + bmc_encode_subframe(raw, preamble_l, this->spdif_block_ptr_); + bmc_encode_subframe(raw, PREAMBLE_W, this->spdif_block_ptr_ + 2); + this->spdif_block_ptr_ += 4; + if (++this->frame_in_block_ >= SPDIF_BLOCK_SAMPLES) { + this->frame_in_block_ = 0; } - this->is_left_channel_ = !this->is_left_channel_; } esp_err_t SPDIFEncoder::send_block_(TickType_t ticks_to_wait) { @@ -295,79 +329,162 @@ esp_err_t SPDIFEncoder::send_block_(TickType_t ticks_to_wait) { return err; } -size_t SPDIFEncoder::get_pending_pcm_bytes() const { - if (this->spdif_block_ptr_ == nullptr || this->spdif_block_buf_ == nullptr) { - return 0; +template +HOT esp_err_t SPDIFEncoder::write_typed_(const uint8_t *src, size_t size, TickType_t ticks_to_wait, + uint32_t *blocks_sent, size_t *bytes_consumed) { + const uint8_t *pcm_data = src; + const uint8_t *const pcm_end = src + size; + uint32_t block_count = 0; + + // Hot state lives in locals so the compiler can keep it in registers across the + // per-frame encoding work; byte writes through block_ptr may alias the member fields, + // which would block register allocation if the encoding read them directly from this->*. + uint32_t *block_ptr = this->spdif_block_ptr_; + uint32_t *const block_buf = this->spdif_block_buf_.get(); + uint32_t *const block_end = block_buf + SPDIF_BLOCK_SIZE_U32; + uint32_t frame = this->frame_in_block_; + const std::array &channel_status = this->channel_status_; + + auto save_state = [&]() { + this->spdif_block_ptr_ = block_ptr; + this->frame_in_block_ = static_cast(frame); + }; + + auto report_out_params = [&]() { + if (blocks_sent != nullptr) + *blocks_sent = block_count; + if (bytes_consumed != nullptr) + *bytes_consumed = pcm_data - src; + }; + + // Send a completed block if the buffer is full, propagating any error. + // send_block_ resets this->spdif_block_ptr_ to block_buf on success and leaves it + // unchanged on error -- mirror both behaviors in our local block_ptr. + auto maybe_send = [&]() -> esp_err_t { + if (block_ptr >= block_end) { + esp_err_t err = this->send_block_(ticks_to_wait); + if (err != ESP_OK) { + save_state(); + report_out_params(); + return err; + } + block_ptr = block_buf; + ++block_count; + } + return ESP_OK; + }; + + // Hot path: encode L+R pairs in two peeled sub-loops. Frame 0 carries the only + // buffer-full check and uses PREAMBLE_B (a block fills exactly when frame wraps from + // 191 back to 0). Frames 1..191 use PREAMBLE_M and need no buffer-full check or + // preamble branch. The encoding body is inlined here so block_ptr lives in a register + // for the duration of the loop. + while (pcm_data + 2 * Bps <= pcm_end) { + if (frame == 0) { + esp_err_t err = maybe_send(); + if (err != ESP_OK) + return err; + + uint32_t c_bit = c_bit_for_frame(channel_status, 0); + uint32_t raw_l = build_raw_subframe(pcm_data) | c_bit; + uint32_t raw_r = build_raw_subframe(pcm_data + Bps) | c_bit; + bmc_encode_subframe(raw_l, PREAMBLE_B, block_ptr); + bmc_encode_subframe(raw_r, PREAMBLE_W, block_ptr + 2); + block_ptr += 4; + frame = 1; + pcm_data += 2 * Bps; + } + + // The inner loop runs until min(SPDIF_BLOCK_SAMPLES, frame + input_frames). The + // input-size bound is folded into end_frame so a single `frame < end_frame` test + // governs termination. + uint32_t input_frames = static_cast(pcm_end - pcm_data) / (2u * Bps); + uint32_t end_frame = SPDIF_BLOCK_SAMPLES; + if (frame + input_frames < end_frame) + end_frame = frame + input_frames; + + while (frame < end_frame) { + uint32_t c_bit = c_bit_for_frame(channel_status, frame); + uint32_t raw_l = build_raw_subframe(pcm_data) | c_bit; + uint32_t raw_r = build_raw_subframe(pcm_data + Bps) | c_bit; + bmc_encode_subframe(raw_l, PREAMBLE_M, block_ptr); + bmc_encode_subframe(raw_r, PREAMBLE_W, block_ptr + 2); + block_ptr += 4; + ++frame; + pcm_data += 2 * Bps; + } + if (frame >= SPDIF_BLOCK_SAMPLES) + frame = 0; } - // Each PCM sample (2 bytes) produces 2 uint32_t values in the SPDIF buffer - // So pending uint32s / 2 = pending samples, and each sample is 2 bytes - size_t pending_uint32s = this->spdif_block_ptr_ - this->spdif_block_buf_.get(); - size_t pending_samples = pending_uint32s / 2; - return pending_samples * 2; // 2 bytes per sample + + // Send any complete block that was just finished. + if (block_ptr >= block_end) { + esp_err_t err = this->send_block_(ticks_to_wait); + if (err != ESP_OK) { + save_state(); + report_out_params(); + return err; + } + block_ptr = block_buf; + ++block_count; + } + + save_state(); + report_out_params(); + return ESP_OK; } HOT esp_err_t SPDIFEncoder::write(const uint8_t *src, size_t size, TickType_t ticks_to_wait, uint32_t *blocks_sent, size_t *bytes_consumed) { - const uint8_t *pcm_data = src; - const uint8_t *pcm_end = src + size; - uint32_t block_count = 0; + if (size > 0) { + // Real PCM is about to be encoded into the buffer, so it is no longer a full-silence block. + this->block_buf_is_silence_block_ = false; + } + switch (this->bytes_per_sample_) { + case 2: + return this->write_typed_<2>(src, size, ticks_to_wait, blocks_sent, bytes_consumed); + case 3: + return this->write_typed_<3>(src, size, ticks_to_wait, blocks_sent, bytes_consumed); + case 4: + return this->write_typed_<4>(src, size, ticks_to_wait, blocks_sent, bytes_consumed); + default: + return ESP_ERR_INVALID_STATE; + } +} - while (pcm_data < pcm_end) { - // Check if there's a pending complete block from a previous failed send - if (this->spdif_block_ptr_ >= &this->spdif_block_buf_[SPDIF_BLOCK_SIZE_U32]) { - esp_err_t err = this->send_block_(ticks_to_wait); - if (err != ESP_OK) { - if (blocks_sent != nullptr) { - *blocks_sent = block_count; - } - if (bytes_consumed != nullptr) { - *bytes_consumed = pcm_data - src; - } - return err; - } - ++block_count; +template esp_err_t SPDIFEncoder::flush_with_silence_typed_(TickType_t ticks_to_wait) { + // If a complete block is already pending (from a previous failed send), emit just that block. + // Otherwise pad the partial block with silence (or generate a full silence block if empty) and + // send. Always emits exactly one block on success. + if (this->spdif_block_ptr_ < &this->spdif_block_buf_[SPDIF_BLOCK_SIZE_U32]) { + const bool was_empty = (this->spdif_block_ptr_ == this->spdif_block_buf_.get()); + // Continuous-silence idle case: a full silence block is byte-identical every time for the + // active channel status, so when the buffer already holds one, re-send it as-is. + if (was_empty && this->block_buf_is_silence_block_) { + return this->send_block_(ticks_to_wait); } - - // Encode one 16-bit sample - this->encode_sample_(pcm_data); - pcm_data += 2; - } - - // Send any complete block that was just finished - if (this->spdif_block_ptr_ >= &this->spdif_block_buf_[SPDIF_BLOCK_SIZE_U32]) { - esp_err_t err = this->send_block_(ticks_to_wait); - if (err != ESP_OK) { - if (blocks_sent != nullptr) { - *blocks_sent = block_count; - } - if (bytes_consumed != nullptr) { - *bytes_consumed = pcm_data - src; - } - return err; + // Pad with silence frames at the configured width. + while (this->spdif_block_ptr_ < &this->spdif_block_buf_[SPDIF_BLOCK_SIZE_U32]) { + this->encode_silence_frame_(); } - ++block_count; + // The buffer is a reusable full-silence block only if it was built entirely from silence; a + // partial real-audio block padded out with silence is not. + this->block_buf_is_silence_block_ = was_empty; } - - if (blocks_sent != nullptr) { - *blocks_sent = block_count; - } - if (bytes_consumed != nullptr) { - *bytes_consumed = size; - } - return ESP_OK; + return this->send_block_(ticks_to_wait); } esp_err_t SPDIFEncoder::flush_with_silence(TickType_t ticks_to_wait) { - // If a complete block is already pending (from a previous failed send), emit just that block. - // Otherwise pad the partial block with silence (or generate a full silence block if empty) - // and send. Always emits exactly one block on success. - if (this->spdif_block_ptr_ < &this->spdif_block_buf_[SPDIF_BLOCK_SIZE_U32]) { - static const uint8_t SILENCE[2] = {0, 0}; - while (this->spdif_block_ptr_ < &this->spdif_block_buf_[SPDIF_BLOCK_SIZE_U32]) { - this->encode_sample_(SILENCE); - } + switch (this->bytes_per_sample_) { + case 2: + return this->flush_with_silence_typed_<2>(ticks_to_wait); + case 3: + return this->flush_with_silence_typed_<3>(ticks_to_wait); + case 4: + return this->flush_with_silence_typed_<4>(ticks_to_wait); + default: + return ESP_ERR_INVALID_STATE; } - return this->send_block_(ticks_to_wait); } } // namespace esphome::i2s_audio diff --git a/esphome/components/i2s_audio/speaker/spdif_encoder.h b/esphome/components/i2s_audio/speaker/spdif_encoder.h index 8c5e068841..9e23a858f7 100644 --- a/esphome/components/i2s_audio/speaker/spdif_encoder.h +++ b/esphome/components/i2s_audio/speaker/spdif_encoder.h @@ -24,8 +24,6 @@ static constexpr uint16_t SPDIF_BLOCK_SIZE_BYTES = SPDIF_BLOCK_SAMPLES * (EMULAT static constexpr uint32_t SPDIF_BLOCK_SIZE_U32 = SPDIF_BLOCK_SIZE_BYTES / sizeof(uint32_t); // 3072 bytes / 4 = 768 // I2S frame count for one SPDIF block (for new driver where frame = 8 bytes for 32-bit stereo) static constexpr uint32_t SPDIF_BLOCK_I2S_FRAMES = SPDIF_BLOCK_SIZE_BYTES / 8; // 3072 / 8 = 384 frames -// PCM bytes needed for one complete SPDIF block (192 stereo frames * 2 bytes per sample * 2 channels) -static constexpr uint16_t SPDIF_PCM_BYTES_PER_BLOCK = SPDIF_BLOCK_SAMPLES * 2 * 2; // = 768 bytes /// Callback signature for block completion (raw function pointer for minimal overhead) /// @param user_ctx User context pointer passed during callback registration @@ -64,8 +62,16 @@ class SPDIFEncoder { /// @brief Check if currently in preload mode bool is_preload_mode() const { return this->preload_mode_; } + /// @brief Set input PCM width: 2 = 16-bit, 3 = 24-bit, 4 = 32-bit (truncated to 24-bit on the wire). + /// Must be called before write() if input width changes from the default (16-bit). Triggers a + /// channel-status rebuild to reflect the new word length. + void set_bytes_per_sample(uint8_t bytes_per_sample); + + /// @brief Get the configured input PCM width in bytes per sample + uint8_t get_bytes_per_sample() const { return this->bytes_per_sample_; } + /// @brief Convert PCM audio data to SPDIF BMC encoded data - /// @param src Source PCM audio data (16-bit stereo) + /// @param src Source PCM audio data (stereo, width matches set_bytes_per_sample) /// @param size Size of source data in bytes /// @param ticks_to_wait Timeout for blocking writes /// @param blocks_sent Optional pointer to receive the number of complete SPDIF blocks sent @@ -74,17 +80,6 @@ class SPDIFEncoder { esp_err_t write(const uint8_t *src, size_t size, TickType_t ticks_to_wait, uint32_t *blocks_sent = nullptr, size_t *bytes_consumed = nullptr); - /// @brief Get the number of PCM bytes currently pending in the partial block buffer - /// @return Number of pending PCM bytes (0 to SPDIF_PCM_BYTES_PER_BLOCK - 1) - size_t get_pending_pcm_bytes() const; - - /// @brief Get the number of PCM frames currently pending in the partial block buffer - /// @return Number of pending PCM frames (0 to SPDIF_BLOCK_SAMPLES - 1) - uint32_t get_pending_frames() const { return this->get_pending_pcm_bytes() / 4; } - - /// @brief Check if there is a partial block pending - bool has_pending_data() const { return this->spdif_block_ptr_ != this->spdif_block_buf_.get(); } - /// @brief Emit one complete SPDIF block: pad any pending partial block with silence and send, /// or send a full silence block if nothing is pending. Always produces exactly one block on success. /// @param ticks_to_wait Timeout for blocking writes @@ -95,7 +90,7 @@ class SPDIFEncoder { void reset(); /// @brief Set the sample rate for Channel Status Block encoding - /// @param sample_rate Sample rate in Hz (e.g., 44100, 48000, 96000) + /// @param sample_rate Sample rate in Hz (e.g., 44100, 48000) /// Call this before writing audio data to ensure correct channel status. void set_sample_rate(uint32_t sample_rate); @@ -103,8 +98,19 @@ class SPDIFEncoder { uint32_t get_sample_rate() const { return this->sample_rate_; } protected: - /// @brief Encode a single 16-bit PCM sample into the current block position - HOT void encode_sample_(const uint8_t *pcm_sample); + /// @brief Encode a single stereo silence frame at the current block position. + /// @note Used only by flush_with_silence_typed_ to pad; the hot write path inlines the + /// encoding body directly into write_typed_ to keep block_ptr / frame_in_block_ in registers. + template void encode_silence_frame_(); + + /// @brief Templated write loop. Called from the public write() via runtime dispatch on bytes_per_sample_. + template + HOT esp_err_t write_typed_(const uint8_t *src, size_t size, TickType_t ticks_to_wait, uint32_t *blocks_sent, + size_t *bytes_consumed); + + /// @brief Templated flush-with-silence. Pads the pending block with zeros at the configured width + /// (or builds a full silence block when nothing is pending) and sends it. Always emits one block. + template esp_err_t flush_with_silence_typed_(TickType_t ticks_to_wait); /// @brief Send the completed block via the appropriate callback esp_err_t send_block_(TickType_t ticks_to_wait); @@ -112,15 +118,6 @@ class SPDIFEncoder { /// @brief Build the channel status block from current configuration void build_channel_status_(); - /// @brief Get the channel status bit for a specific frame - /// @param frame Frame number (0-191) - /// @return The C bit value for this frame - ESPHOME_ALWAYS_INLINE inline bool get_channel_status_bit_(uint8_t frame) const { - // Channel status is 192 bits transmitted over 192 frames - // Bit N is transmitted in frame N, LSB-first within each byte - return (this->channel_status_[frame >> 3] >> (frame & 7)) & 1; - } - // Member ordering optimized to minimize padding (largest alignment first) // 4-byte aligned members (pointers and uint32_t) @@ -133,9 +130,13 @@ class SPDIFEncoder { uint32_t sample_rate_{48000}; // Sample rate for Channel Status Block encoding // 1-byte aligned members (grouped together to avoid internal padding) - uint8_t frame_in_block_{0}; // 0-191, tracks stereo frame position within block - bool is_left_channel_{true}; // Alternates L/R for stereo samples - bool preload_mode_{false}; // Whether to use preload callback vs write callback + uint8_t bytes_per_sample_{2}; // Input PCM width: 2/3/4 (16/24/32-bit). 32-bit truncates to 24-bit on the wire. + uint8_t frame_in_block_{0}; // 0-191, tracks stereo frame position within block + bool preload_mode_{false}; // Whether to use preload callback vs write callback + // True when spdif_block_buf_ currently holds a complete full-silence block valid for the active + // channel status. A full silence block is deterministic for a given sample rate and word length, + // so when this is set flush_with_silence() can re-send the buffer verbatim instead of re-encoding. + bool block_buf_is_silence_block_{false}; // Channel Status Block (192 bits = 24 bytes, transmitted over 192 frames) // Placed last since std::array has 1-byte alignment diff --git a/esphome/components/image/__init__.py b/esphome/components/image/__init__.py index 365554f7d2..5f8e5ca132 100644 --- a/esphome/components/image/__init__.py +++ b/esphome/components/image/__init__.py @@ -395,7 +395,7 @@ def download_image(value): def is_svg_file(file): if not file: return False - with open(file, "rb") as f: + with Path(file).open("rb") as f: return " RAM0 AT> FLASH), after KEEP(*(.vectors)) +// so the Cortex-M4 vector table stays 512-byte-aligned for VTOR. // // BK72xx (all variants) are left as a no-op: their SDK wraps flash // operations in GLOBAL_INT_DISABLE() which masks FIQ + IRQ at the CPU for @@ -26,13 +34,7 @@ // layer. #if defined(USE_BK72XX) #define IRAM_ATTR -#elif defined(USE_LIBRETINY_VARIANT_RTL8710B) -// Stock linker consumes *(.image2.ram.text*) into .ram_image2.text (> BD_RAM). -#define IRAM_ATTR __attribute__((noinline, section(".image2.ram.text"))) #else -// RTL8720C: stock linker consumes *(.sram.text*) into .ram.code_text. -// LN882H: patch_linker.py.script injects *(.sram.text*) into -// .flash_copysection (> RAM0 AT> FLASH). #define IRAM_ATTR __attribute__((noinline, section(".sram.text"))) #endif #define PROGMEM diff --git a/esphome/components/libretiny/patch_linker.py.script b/esphome/components/libretiny/patch_linker.py.script index 282a31d3f2..dfeaaa57d1 100644 --- a/esphome/components/libretiny/patch_linker.py.script +++ b/esphome/components/libretiny/patch_linker.py.script @@ -6,14 +6,22 @@ import re import subprocess # ESPHome marks ISR code IRAM_ATTR, which on LibreTiny maps to a per-family -# section routed into RAM-executable memory (see esphome/core/hal.h). +# section routed into RAM-executable memory (see esphome/core/hal.h). The +# input section name is always .sram.text; only the output section it lands +# in differs per family. # # This script is NOT loaded on BK72xx (IRAM_ATTR is a no-op there; the SDK # masks FIQ+IRQ around flash writes). On the remaining families: -# - RTL8710B: hal.h uses section(".image2.ram.text"); stock linker consumes it. -# - RTL8720C: hal.h uses section(".sram.text"); stock linker consumes it. +# - RTL8720C: stock linker consumes *(.sram.text*) into .ram.code_text. +# - RTL8710B: stock linker has KEEP(*(.image2.ram.text*)) in .ram_image2.text, +# but ltchiptool's AmebaZ elf2bin (soc/ambz/binary.py) does NOT list +# .ram_image2.text in sections_ram, so code there is silently dropped from +# the flashed image. Inject KEEP(*(.sram.text*)) at the top of +# .ram_image2.data (which IS extracted) instead. # - LN882H: stock linker has no glob for ".sram.text", so we inject -# KEEP(*(.sram.text*)) into ".flash_copysection" (> RAM0 AT> FLASH). +# KEEP(*(.sram.text*)) into ".flash_copysection" (> RAM0 AT> FLASH) +# immediately after KEEP(*(.vectors)), so the vector table stays at +# __copysection_ram0_start (0x20000000) for correct Cortex-M4 VTOR alignment. # # All families also get a post-link summary showing where IRAM_ATTR landed. @@ -27,7 +35,25 @@ _KEEP_LINE = ( "__esphome_sram_text_end = .; " + _MARKER + "\n" ) -_LN_COPY = re.compile(r"(\.flash_copysection\s*:\s*\{\s*\n)") +# Inject after KEEP(*(.vectors)) so the vector table stays at +# __copysection_ram0_start (0x20000000). Cortex-M4 VTOR requires a 512-byte- +# aligned address; injecting before the vectors would push them to an +# unaligned offset and mis-route every IRQ handler. +_LN_COPY = re.compile(r"(KEEP\(\*\(\.vectors\)\)[^\n]*\n)") +# Inject at the top of .ram_image2.data, before __data_start__ so our code +# does not fall inside the data range markers. .ram_image2.data is one of the +# sections ltchiptool's AmebaZ elf2bin extracts; BD_RAM is rwx so the code is +# executable. AmbZ has no C runtime .data copy loop (the bootloader loads +# image2 into BD_RAM whole) so the inline code is not clobbered after boot. +# +# The regex is intentionally strict (no attribute / ALIGN between the section +# name and the opening brace, brace on its own line). If a future AmbZ SDK +# linker template changes this format, _pre_link raises RuntimeError on the +# unpatched .ld file(s), and the RTL8710B CI compile job in +# tests/test_build_components fails on the PR, surfacing the mismatch loudly +# rather than silently shipping a binary with IRAM_ATTR code dropped from +# one or both OTA slots. +_AMBZ_DATA = re.compile(r"(\.ram_image2\.data\s*:\s*\n?\s*\{\s*\n)") def _detect(env): @@ -56,7 +82,7 @@ KNOWN_VARIANTS = frozenset({ def _inject_keep(host_section): - """Return a patcher that injects _KEEP_LINE at the top of `host_section`.""" + """Return a patcher that injects _KEEP_LINE after `host_section` match.""" def patch(content): if _MARKER in content: return content @@ -65,12 +91,11 @@ def _inject_keep(host_section): # Variants not listed here intentionally have no .ld patcher: -# - RTL8710B: hal.h uses section(".image2.ram.text") which the stock linker -# already routes into .ram_image2.text (> BD_RAM). -# - RTL8720C: stock linker already consumes *(.sram.text*). +# - RTL8720C: stock linker already consumes *(.sram.text*) into .ram.code_text. # - BK72xx (all): SDK masks FIQ+IRQ around flash writes, IRAM_ATTR is no-op. _PATCHERS_BY_VARIANT = { "LN882H": (_inject_keep(_LN_COPY),), + "RTL8710B": (_inject_keep(_AMBZ_DATA),), } @@ -81,13 +106,14 @@ def _patchers_for(variant): def _pre_link(target, source, env): build_dir = env.subst("$BUILD_DIR") ld_files = [f for f in os.listdir(build_dir) if f.endswith(".ld")] - patched = 0 + patched = [] + unpatched = [] for name in ld_files: path = os.path.join(build_dir, name) with open(path, "r", encoding="utf-8") as fh: original = fh.read() if _MARKER in original: - patched += 1 + patched.append(name) continue content = original for fn in _patchers: @@ -96,7 +122,9 @@ def _pre_link(target, source, env): with open(path, "w", encoding="utf-8") as fh: fh.write(content) print("ESPHome: patched {} for IRAM_ATTR placement".format(name)) - patched += 1 + patched.append(name) + else: + unpatched.append(name) if not patched: raise RuntimeError( "ESPHome: no .ld in {} was patched for IRAM_ATTR. Update the " @@ -104,6 +132,20 @@ def _pre_link(target, source, env): build_dir ) ) + # Every .ld in the build must be patched. RTL8710B generates one .ld per + # OTA slot (xip1, xip2); if only one matches, the unpatched slot would + # ship with IRAM_ATTR code dropped to zeros and brick the device on the + # boot after an OTA into that slot. + if unpatched: + raise RuntimeError( + "ESPHome: {} of {} .ld file(s) in {} were not patched for " + "IRAM_ATTR: {}. The regex in patch_linker.py.script " + "(_PATCHERS_BY_VARIANT[{!r}]) matched the others but not " + "these. Update the regex to cover all linker scripts.".format( + len(unpatched), len(ld_files), build_dir, + ", ".join(unpatched), _variant, + ) + ) # Substrings matched against demangled names as a fallback on RTL8720C, diff --git a/esphome/components/light/__init__.py b/esphome/components/light/__init__.py index 9540c64486..68d9f85af2 100644 --- a/esphome/components/light/__init__.py +++ b/esphome/components/light/__init__.py @@ -86,10 +86,22 @@ class EffectRef: component_path: list[str | int] # path_context when the action was validated +@dataclass +class EffectCycleRef: + """A pending light.effect.next/previous action to validate. + + Records that the referenced light needs at least one effect configured. + """ + + light_id: ID + component_path: list[str | int] + + @dataclass class LightData: gamma_tables: dict = field(default_factory=dict) # gamma_value -> fwd_arr effect_refs: list[EffectRef] = field(default_factory=list) + effect_cycle_refs: list[EffectCycleRef] = field(default_factory=list) def _get_data() -> LightData: @@ -160,13 +172,15 @@ def _final_validate(config: ConfigType) -> ConfigType: this never runs — but the ID validator will catch the missing light ID separately. """ data = _get_data() - if not data.effect_refs: + if not data.effect_refs and not data.effect_cycle_refs: return config - # Drain the list so we only validate once even though + # Drain the lists so we only validate once even though # FINAL_VALIDATE_SCHEMA runs for each light platform instance. refs = data.effect_refs data.effect_refs = [] + cycle_refs = data.effect_cycle_refs + data.effect_cycle_refs = [] fconf = fv.full_config.get() @@ -188,6 +202,21 @@ def _final_validate(config: ConfigType) -> ConfigType: path=[cv.ROOT_CONFIG_PATH] + ref.component_path, ) + for ref in cycle_refs: + try: + light_path = fconf.get_path_for_id(ref.light_id)[:-1] + light_config = fconf.get_config_for_path(light_path) + except KeyError: + continue + + if not light_config.get(CONF_EFFECTS): + raise cv.FinalExternalInvalid( + f"Light '{ref.light_id}' has no effects configured, but a " + f"'light.effect.next' or 'light.effect.previous' action " + f"references it. Add at least one effect to the light.", + path=[cv.ROOT_CONFIG_PATH] + ref.component_path, + ) + return config diff --git a/esphome/components/light/automation.h b/esphome/components/light/automation.h index 993d4a2ea6..260414f033 100644 --- a/esphome/components/light/automation.h +++ b/esphome/components/light/automation.h @@ -104,6 +104,47 @@ template class DimRelativeAction : pub transition_length_{}; }; +// Cycle through the light's configured effects. `Forward` selects direction +// at compile time so the chosen branch is the only one that gets instantiated +// per action site. `include_none` is runtime so a single set of templates +// covers both the "wrap through None" and "skip None" variants. +template class LightEffectCycleAction : public Action { + public: + explicit LightEffectCycleAction(LightState *parent) : parent_(parent) {} + + void set_include_none(bool include_none) { this->include_none_ = include_none; } + + void play(const Ts &...) override { + size_t count = this->parent_->get_effect_count(); + if (count == 0) { + return; + } + uint32_t current = this->parent_->get_current_effect_index(); + uint32_t next; + if (this->include_none_) { + uint32_t total = static_cast(count) + 1; + if constexpr (Forward) { + next = (current + 1) % total; + } else { + next = (current + total - 1) % total; + } + } else { + if constexpr (Forward) { + next = (current % static_cast(count)) + 1; + } else { + next = (current <= 1) ? static_cast(count) : current - 1; + } + } + auto call = this->parent_->turn_on(); + call.set_effect(next); + call.perform(); + } + + protected: + LightState *parent_; + bool include_none_{false}; +}; + template class LightIsOnCondition : public Condition { public: explicit LightIsOnCondition(LightState *state) : state_(state) {} diff --git a/esphome/components/light/automation.py b/esphome/components/light/automation.py index cef774af38..7eaba9b117 100644 --- a/esphome/components/light/automation.py +++ b/esphome/components/light/automation.py @@ -26,8 +26,8 @@ from esphome.const import ( CONF_WARM_WHITE, CONF_WHITE, ) -from esphome.core import CORE, EsphomeError, Lambda -from esphome.cpp_generator import LambdaExpression +from esphome.core import CORE, ID, EsphomeError, Lambda +from esphome.cpp_generator import LambdaExpression, MockObj, TemplateArgsType from esphome.types import ConfigType from .types import ( @@ -39,12 +39,15 @@ from .types import ( DimRelativeAction, LightCall, LightControlAction, + LightEffectCycleAction, LightIsOffCondition, LightIsOnCondition, LightState, ToggleAction, ) +CONF_INCLUDE_NONE = "include_none" + @automation.register_action( "light.toggle", @@ -253,6 +256,75 @@ async def light_control_to_code(config, action_id, template_arg, args): return cg.new_Pvariable(action_id, template_arg, paren, apply_lambda) +def _record_effect_cycle_ref(config: ConfigType) -> ConfigType: + """Record a cycle-action reference for later validation against the target light.""" + from . import EffectCycleRef, _get_data + + _get_data().effect_cycle_refs.append( + EffectCycleRef( + light_id=config[CONF_ID], + component_path=path_context.get(), + ) + ) + return config + + +LIGHT_EFFECT_CYCLE_ACTION_BASE_SCHEMA = cv.Schema( + { + cv.Required(CONF_ID): cv.use_id(LightState), + cv.Optional(CONF_INCLUDE_NONE, default=False): cv.boolean, + } +) +LIGHT_EFFECT_CYCLE_ACTION_BASE_SCHEMA.add_extra(_record_effect_cycle_ref) + +LIGHT_EFFECT_CYCLE_ACTION_SCHEMA = automation.maybe_simple_id( + LIGHT_EFFECT_CYCLE_ACTION_BASE_SCHEMA +) + + +@automation.register_action( + "light.effect.next", + LightEffectCycleAction, + LIGHT_EFFECT_CYCLE_ACTION_SCHEMA, + synchronous=True, +) +async def light_effect_next_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + return await _light_effect_cycle_to_code(config, action_id, template_arg, True) + + +@automation.register_action( + "light.effect.previous", + LightEffectCycleAction, + LIGHT_EFFECT_CYCLE_ACTION_SCHEMA, + synchronous=True, +) +async def light_effect_previous_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + return await _light_effect_cycle_to_code(config, action_id, template_arg, False) + + +async def _light_effect_cycle_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + forward: bool, +) -> MockObj: + paren = await cg.get_variable(config[CONF_ID]) + cycle_template_arg = cg.TemplateArguments(forward, *template_arg) + var = cg.new_Pvariable(action_id, cycle_template_arg, paren) + cg.add(var.set_include_none(config[CONF_INCLUDE_NONE])) + return var + + CONF_RELATIVE_BRIGHTNESS = "relative_brightness" LIGHT_DIM_RELATIVE_ACTION_SCHEMA = cv.Schema( { diff --git a/esphome/components/light/types.py b/esphome/components/light/types.py index 534dcd2194..c7385cbee3 100644 --- a/esphome/components/light/types.py +++ b/esphome/components/light/types.py @@ -39,6 +39,7 @@ LIMIT_MODES = { # Actions ToggleAction = light_ns.class_("ToggleAction", automation.Action) LightControlAction = light_ns.class_("LightControlAction", automation.Action) +LightEffectCycleAction = light_ns.class_("LightEffectCycleAction", automation.Action) DimRelativeAction = light_ns.class_("DimRelativeAction", automation.Action) AddressableSet = light_ns.class_("AddressableSet", automation.Action) LightIsOnCondition = light_ns.class_("LightIsOnCondition", automation.Condition) diff --git a/esphome/components/lvgl/__init__.py b/esphome/components/lvgl/__init__.py index 91b101cd25..6e005f897e 100644 --- a/esphome/components/lvgl/__init__.py +++ b/esphome/components/lvgl/__init__.py @@ -1,3 +1,4 @@ +import functools import importlib from pathlib import Path import pkgutil @@ -55,6 +56,7 @@ from .automation import layers_to_code, lvgl_update from .defines import ( CONF_ALIGN_TO_LAMBDA_ID, LOGGER, + add_lv_use, get_focused_widgets, get_lv_images_used, get_refreshed_widgets, @@ -71,13 +73,14 @@ from .keypads import KEYPADS_CONFIG, keypads_to_code from .lv_validation import lv_bool from .lvcode import LvContext, LvglComponent, lv_event_t_ptr, lvgl_static from .schemas import ( + BASE_PROPS, DISP_BG_SCHEMA, FULL_STYLE_SCHEMA, STYLE_REMAP, WIDGET_TYPES, any_widget_schema, container_schema, - obj_schema, + obj_dict, ) from .styles import styles_to_code, theme_to_code from .touchscreens import touchscreen_schema, touchscreens_to_code @@ -100,6 +103,7 @@ from .widgets import ( get_screen_active, set_obj_properties, ) +from .widgets.img import CONF_IMAGE # Import only what we actually use directly in this file from .widgets.msgbox import MSGBOX_SCHEMA, msgboxes_to_code @@ -170,7 +174,7 @@ def generate_lv_conf_h(): if clashes: LOGGER.warning( "Some defines are set both by ESPHome build flags and by LVGL configuration which may lead to unexpected behavior: %s", - sorted(list(clashes)), + sorted(clashes), ) unused_defines = all_defines - lv_defines.keys() - defines_from_flags @@ -433,6 +437,8 @@ async def to_code(configs): # This must be done after all widgets are created styles_used = df.get_styles_used() + if any(BASE_PROPS.get(x) is lvalid.lv_image for x in styles_used): + add_lv_use(CONF_IMAGE) for use in df.get_lv_uses(): df.add_define(f"LV_USE_{use.upper()}") cg.add_define(f"USE_LVGL_{use.upper()}") @@ -513,16 +519,32 @@ def add_hello_world(config): return config -def _theme_schema(value): +@functools.cache +def _build_theme_schema( + widget_types: tuple[tuple[str, widgets.WidgetType], ...], +) -> cv.Schema: + # The theme schema is value-independent: it depends only on the set of + # registered widget types. Key the cache on a snapshot of WIDGET_TYPES so + # that an external component registering a new widget after the first + # validation (legal per any_widget_schema's lazy-evaluation contract) + # produces a fresh tuple, a cache miss, and a rebuilt schema -- the cache + # self-heals instead of stale-rejecting valid themes. See obj_dict() in + # schemas.py for why chained .extend() is avoided here. return cv.Schema( { cv.Optional(df.CONF_DARK_MODE, default=False): cv.boolean, **{ - cv.Optional(name): obj_schema(w).extend(FULL_STYLE_SCHEMA) - for name, w in WIDGET_TYPES.items() + cv.Optional(name): cv.Schema( + {**obj_dict(w), **FULL_STYLE_SCHEMA.schema} + ) + for name, w in widget_types }, } - )(value) + ) + + +def _theme_schema(value: dict) -> dict: + return _build_theme_schema(tuple(WIDGET_TYPES.items()))(value) FINAL_VALIDATE_SCHEMA = final_validation diff --git a/esphome/components/lvgl/defines.py b/esphome/components/lvgl/defines.py index 15a24f1ad2..d9be881a7f 100644 --- a/esphome/components/lvgl/defines.py +++ b/esphome/components/lvgl/defines.py @@ -335,7 +335,7 @@ TYPE_NONE = "none" DIRECTIONS = LvConstant("LV_DIR_", "LEFT", "RIGHT", "BOTTOM", "TOP") -LV_FONTS = list(f"montserrat_{s}" for s in range(8, 50, 2)) + [ +LV_FONTS = [f"montserrat_{s}" for s in range(8, 50, 2)] + [ "dejavu_16_persian_hebrew", "simsun_16_cjk", "unscii_8", diff --git a/esphome/components/lvgl/lv_validation.py b/esphome/components/lvgl/lv_validation.py index a1b75182eb..27cbfff694 100644 --- a/esphome/components/lvgl/lv_validation.py +++ b/esphome/components/lvgl/lv_validation.py @@ -239,7 +239,7 @@ def color_retmapper(value): else: r, g, b, _ = from_rgbw(cval) return literal(f"lv_color_make({r}, {g}, {b})") - assert False + raise AssertionError(f"Unhandled lv_color value: {value!r}") def option_string(value): diff --git a/esphome/components/lvgl/lvgl_esphome.h b/esphome/components/lvgl/lvgl_esphome.h index 218f9a60ab..3f7f1dce14 100644 --- a/esphome/components/lvgl/lvgl_esphome.h +++ b/esphome/components/lvgl/lvgl_esphome.h @@ -74,11 +74,11 @@ inline void lv_style_set_text_font(lv_style_t *style, const font::Font *font) { lv_style_set_text_font(style, font->get_lv_font()); } #endif -#if defined(USE_LVGL_IMAGE) && defined(USE_IMAGE) -#if LV_USE_IMAGE + +#ifdef USE_IMAGE +#ifdef USE_LVGL_IMAGE // Shortcut / overload, so that the source of an image widget can easily be updated from within a lambda. inline void lv_image_set_src(lv_obj_t *obj, image::Image *image) { ::lv_image_set_src(obj, image->get_lv_image_dsc()); } -#endif // LV_USE_IMAGE inline void lv_obj_set_style_bitmap_mask_src(lv_obj_t *obj, image::Image *image, lv_style_selector_t selector) { ::lv_obj_set_style_bitmap_mask_src(obj, image->get_lv_image_dsc(), selector); @@ -93,7 +93,8 @@ inline void lv_style_set_bg_image_src(lv_style_t *style, image::Image *image) { inline void lv_style_set_bitmap_mask_src(lv_style_t *style, image::Image *image) { ::lv_style_set_bitmap_mask_src(style, image->get_lv_image_dsc()); } -#endif // USE_LVGL_IMAGE +#endif + #ifdef USE_LVGL_ANIMIMG inline void lv_animimg_set_src(lv_obj_t *img, std::vector images) { auto *dsc = static_cast *>(lv_obj_get_user_data(img)); @@ -109,6 +110,7 @@ inline void lv_animimg_set_src(lv_obj_t *img, std::vector images lv_animimg_set_src(img, (const void **) dsc->data(), dsc->size()); } #endif // USE_LVGL_ANIMIMG +#endif // USE_IMAGE #ifdef USE_LVGL_METER int16_t lv_get_needle_angle_for_value(lv_obj_t *obj, int32_t value); diff --git a/esphome/components/lvgl/schemas.py b/esphome/components/lvgl/schemas.py index 553e0f7398..bdaa91f15c 100644 --- a/esphome/components/lvgl/schemas.py +++ b/esphome/components/lvgl/schemas.py @@ -1,4 +1,5 @@ from collections.abc import Callable +from typing import Any from esphome import config_validation as cv from esphome.automation import Trigger, validate_automation @@ -21,6 +22,7 @@ from esphome.const import ( ) from esphome.core import TimePeriod from esphome.core.config import StartupTrigger +from esphome.schema_extractors import EnableSchemaExtraction from . import defines as df, lv_validation as lvalid from .defines import ( @@ -377,18 +379,63 @@ TRIGGER_EVENT_MAP = { } -def part_schema(parts): +def part_dict(parts: tuple[str, ...] | list[str]) -> dict[Any, Any]: + """ + Return the raw mapping used by part_schema, so callers can merge it into a + larger dict and avoid chained .extend() calls (each .extend() recompiles the + whole mapping, turning the build into O(N^2)). + + Invariant: the source schemas spread here (STATE_SCHEMA, FLAG_SCHEMA, the + nested STATE_SCHEMA values) must use the default extra=PREVENT_EXTRA and + required=False and must not register any add_extra/prepend_extra + validators. Reaching into .schema and rebuilding via cv.Schema(...) keeps + only the mapping; non-default extra/required and any _extra_schemas would + be silently dropped. + """ + return { + **STATE_SCHEMA.schema, + **FLAG_SCHEMA.schema, + **{cv.Optional(part): STATE_SCHEMA for part in parts}, + } + + +def part_schema(parts: tuple[str, ...] | list[str]) -> cv.Schema: """ Generate a schema for the various parts (e.g. main:, indicator:) of a widget type :param parts: The parts to include :return: The schema """ - return STATE_SCHEMA.extend(FLAG_SCHEMA).extend( - {cv.Optional(part): STATE_SCHEMA for part in parts} - ) + return cv.Schema(part_dict(parts)) -def automation_schema(typ: LvType): +def _lazy_validate_automation(extra_schema: dict) -> Callable[[Any], Any]: + """Return a validator that defers building the validate_automation schema. + + validate_automation() runs AUTOMATION_SCHEMA.extend(extra_schema), which + voluptuous compiles eagerly. automation_schema() builds ~60 of these per + widget type, and the vast majority of slots are never invoked by a given + user config. Deferring the build to first use removes that work from + schema-construction time. + + When EnableSchemaExtraction is set (build_language_schema.py), fall back + to eager construction so the @schema_extractor("automation") decoration + inside validate_automation is registered. + """ + if EnableSchemaExtraction: + return validate_automation(extra_schema) + + cached: Callable[[Any], Any] | None = None + + def validator(value: Any) -> Any: + nonlocal cached + if cached is None: + cached = validate_automation(extra_schema) + return cached(value) + + return validator + + +def automation_schema(typ: LvType) -> dict[Any, Any]: events = df.LV_EVENT_TRIGGERS + df.SWIPE_TRIGGERS if typ.has_on_value: events = events + (CONF_ON_VALUE, CONF_ON_UPDATE) @@ -403,7 +450,7 @@ def automation_schema(typ: LvType): return { **{ - cv.Optional(event): validate_automation( + cv.Optional(event): _lazy_validate_automation( { cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id( Trigger.template(*get_trigger_args(event)) @@ -412,7 +459,7 @@ def automation_schema(typ: LvType): ) for event in events }, - cv.Optional(CONF_ON_BOOT): validate_automation( + cv.Optional(CONF_ON_BOOT): _lazy_validate_automation( {cv.GenerateID(CONF_TRIGGER_ID): cv.declare_id(StartupTrigger)} ), } @@ -461,23 +508,62 @@ def base_update_schema(widget_type: WidgetType | LvType, parts): return schema -def obj_schema(widget_type: WidgetType): +# Memoize obj_dict() the same way _OBJ_SCHEMA_CACHE memoizes obj_schema(). +# automation_schema(w.w_type) builds fresh Trigger.template(...) objects on +# every call, so without this cache _theme_schema pays that cost per widget +# per validation. Callers must treat the returned dict as immutable. The +# _theme_schema caller spreads it into a fresh dict, which is safe; the +# obj_schema caller passes it directly to cv.Schema(...) -- voluptuous stores +# the mapping by reference but never mutates it (.extend() copies first), so +# the alias is also safe today. Adding in-place mutation of obj_schema(w).schema +# would corrupt this cache. +_OBJ_DICT_CACHE: dict[int, tuple[WidgetType, dict[Any, Any]]] = {} + + +def obj_dict(widget_type: WidgetType) -> dict[Any, Any]: + """ + Return the raw mapping used by obj_schema, so callers can merge it into a + larger dict and avoid chained .extend() calls. + + Inherits the same source-schema invariant documented on part_dict: any + schema spread into this mapping must use the default extra=PREVENT_EXTRA + and required=False and must carry no add_extra/prepend_extra validators. + + The returned mapping is cached and must be treated as immutable by callers. + """ + cached = _OBJ_DICT_CACHE.get(id(widget_type)) + if cached is not None and cached[0] is widget_type: + return cached[1] + built = { + **part_dict(widget_type.parts), + **ALIGN_TO_SCHEMA, + **automation_schema(widget_type.w_type), + cv.Optional(CONF_STATE): SET_STATE_SCHEMA, + cv.Optional(CONF_GROUP): cv.use_id(lv_group_t), + } + _OBJ_DICT_CACHE[id(widget_type)] = (widget_type, built) + return built + + +# Widget types are module-level singletons populated at import time, so we +# can cache compiled obj_schemas by widget_type identity for the lifetime of +# the process. The strong reference in the value keeps the key (an id() +# target) from being recycled. +_OBJ_SCHEMA_CACHE: dict[int, tuple[WidgetType, cv.Schema]] = {} + + +def obj_schema(widget_type: WidgetType) -> cv.Schema: """ Create a schema for a widget type itself i.e. no allowance for children :param widget_type: :return: """ - return ( - part_schema(widget_type.parts) - .extend(ALIGN_TO_SCHEMA) - .extend(automation_schema(widget_type.w_type)) - .extend( - { - cv.Optional(CONF_STATE): SET_STATE_SCHEMA, - cv.Optional(CONF_GROUP): cv.use_id(lv_group_t), - } - ) - ) + cached = _OBJ_SCHEMA_CACHE.get(id(widget_type)) + if cached is not None and cached[0] is widget_type: + return cached[1] + schema = cv.Schema(obj_dict(widget_type)) + _OBJ_SCHEMA_CACHE[id(widget_type)] = (widget_type, schema) + return schema ALIGN_TO_SCHEMA = { @@ -534,7 +620,16 @@ def strip_defaults(schema: cv.Schema): return cv.Schema({cv.Optional(k): v for k, v in schema.schema.items()}) -def container_schema(widget_type: WidgetType, extras=None): +# Keyed by (id(widget_type), id(extras)); strong refs in the value keep both +# alive so id() can't be recycled. +_CONTAINER_SCHEMA_CACHE: dict[ + tuple[int, int], tuple[Any, Any, Callable[[Any], Any]] +] = {} + + +def container_schema( + widget_type: WidgetType, extras: Any = None +) -> Callable[[Any], Any]: """ Create a schema for a container widget of a given type. All obj properties are available, plus the extras passed in, plus any defined for the specific widget being specified. @@ -542,19 +637,31 @@ def container_schema(widget_type: WidgetType, extras=None): :param extras: Additional options to be made available, e.g. layout properties for children :return: The schema for this type of widget. """ - schema = obj_schema(widget_type).extend( - {cv.GenerateID(): cv.declare_id(widget_type.w_type)} - ) - if extras: - schema = schema.extend(extras) - # Delayed evaluation for recursion + cache_key = (id(widget_type), id(extras)) + cached = _CONTAINER_SCHEMA_CACHE.get(cache_key) + if cached is not None: + cached_widget_type, cached_extras, cached_validator = cached + if cached_widget_type is widget_type and cached_extras is extras: + return cached_validator - schema = schema.extend(widget_type.schema) + cached_schema: cv.Schema | None = None - def validator(value): + def get_schema() -> cv.Schema: + nonlocal cached_schema + if cached_schema is None: + schema = obj_schema(widget_type).extend( + {cv.GenerateID(): cv.declare_id(widget_type.w_type)} + ) + if extras: + schema = schema.extend(extras) + cached_schema = schema.extend(widget_type.schema) + return cached_schema + + def validator(value: Any) -> Any: value = value or {} - return append_layout_schema(schema, value)(value) + return append_layout_schema(get_schema(), value)(value) + _CONTAINER_SCHEMA_CACHE[cache_key] = (widget_type, extras, validator) return validator diff --git a/esphome/components/lvgl/styles.py b/esphome/components/lvgl/styles.py index c1441526f9..5911505555 100644 --- a/esphome/components/lvgl/styles.py +++ b/esphome/components/lvgl/styles.py @@ -9,6 +9,7 @@ from .defines import ( CONF_THEME, LValidator, add_lv_use, + get_styles_used, get_theme_widget_map, literal, ) @@ -25,6 +26,7 @@ def has_style_props(config) -> bool: async def style_set(svar, style): for prop, validator in ALL_STYLES.items(): if (value := style.get(prop)) is not None: + get_styles_used().add(prop) if isinstance(validator, LValidator): value = await validator.process(value) if isinstance(value, list): diff --git a/esphome/components/lvgl/widgets/__init__.py b/esphome/components/lvgl/widgets/__init__.py index ab1c61ff88..400f7c709b 100644 --- a/esphome/components/lvgl/widgets/__init__.py +++ b/esphome/components/lvgl/widgets/__init__.py @@ -1,4 +1,6 @@ +from collections.abc import Callable import sys +from typing import Any from esphome import codegen as cg, config_validation as cv from esphome.automation import register_action @@ -15,6 +17,7 @@ from esphome.const import ( from esphome.core import ID, EsphomeError, TimePeriod from esphome.coroutine import FakeAwaitable from esphome.cpp_generator import MockObj +from esphome.schema_extractors import EnableSchemaExtraction from esphome.types import Expression from ..defines import ( @@ -73,6 +76,34 @@ from ..types import ( EVENT_LAMB = "event_lamb__" +def _build_update_schema(widget_type: "WidgetType") -> Schema: + # Local import: ..schemas imports WidgetType from this module. + from ..schemas import base_update_schema + + return base_update_schema(widget_type, widget_type.parts).extend( + widget_type.modify_schema + ) + + +def _update_action_schema( + widget_type: "WidgetType", +) -> Schema | Callable[[Any], Any]: + # Eager when extracting so build_language_schema.py sees the mapping; + # lazy otherwise to skip ~200 ms of import-time voluptuous work. + if EnableSchemaExtraction: + return _build_update_schema(widget_type) + + cached: Schema | None = None + + def validator(value: Any) -> Any: + nonlocal cached + if cached is None: + cached = _build_update_schema(widget_type) + return cached(value) + + return validator + + class WidgetType: """ Describes a type of Widget, e.g. "bar" or "line" @@ -113,18 +144,17 @@ class WidgetType: # Local import to avoid circular import from ..automation import update_to_code - from ..schemas import WIDGET_TYPES, base_update_schema + from ..schemas import WIDGET_TYPES if not is_mock: if self.name in WIDGET_TYPES: raise EsphomeError(f"Duplicate definition of widget type '{self.name}'") WIDGET_TYPES[self.name] = self - # Register the update action automatically, adding widget-specific properties register_action( f"lvgl.{self.name}.update", ObjUpdateAction, - base_update_schema(self, self.parts).extend(self.modify_schema), + _update_action_schema(self), synchronous=True, )(update_to_code) diff --git a/esphome/components/lvgl/widgets/tabview.py b/esphome/components/lvgl/widgets/tabview.py index 5e9e0494dd..ee252ecf0b 100644 --- a/esphome/components/lvgl/widgets/tabview.py +++ b/esphome/components/lvgl/widgets/tabview.py @@ -97,7 +97,7 @@ class TabviewType(WidgetType): tab_bar = Widget(bar_obj, obj_spec) await set_obj_properties(tab_bar, tab_style) if tab_items_style: - for index, tab_conf in enumerate(config[CONF_TABS]): + for index, _tab_conf in enumerate(config[CONF_TABS]): await set_obj_properties( Widget(lv_obj.get_child(bar_obj, index), button_spec), tab_items_style, diff --git a/esphome/components/micro_wake_word/__init__.py b/esphome/components/micro_wake_word/__init__.py index 38926fce99..cba6bcfa50 100644 --- a/esphome/components/micro_wake_word/__init__.py +++ b/esphome/components/micro_wake_word/__init__.py @@ -7,7 +7,7 @@ from urllib.parse import urljoin from esphome import automation, external_files, git from esphome.automation import register_action, register_condition import esphome.codegen as cg -from esphome.components import esp32, microphone, ota +from esphome.components import esp32, microphone, ota, psram import esphome.config_validation as cv from esphome.const import ( CONF_FILE, @@ -20,6 +20,7 @@ from esphome.const import ( CONF_RAW_DATA_ID, CONF_REF, CONF_REFRESH, + CONF_TASK_STACK_IN_PSRAM, CONF_TYPE, CONF_URL, CONF_USERNAME, @@ -358,6 +359,7 @@ CONFIG_SCHEMA = cv.All( ), cv.Optional(CONF_VAD): _maybe_empty_vad_schema, cv.Optional(CONF_STOP_AFTER_DETECTION, default=True): cv.boolean, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_MODEL): cv.invalid( f"The {CONF_MODEL} parameter has moved to be a list element under the {CONF_MODELS} parameter." ), @@ -374,14 +376,14 @@ CONFIG_SCHEMA = cv.All( def _load_model_data(manifest_path: Path): - with open(manifest_path, encoding="utf-8") as f: + with manifest_path.open(encoding="utf-8") as f: manifest = json.load(f) _validate_manifest_version(manifest) model_path = manifest_path.parent / manifest[CONF_MODEL] - with open(model_path, "rb") as f: + with model_path.open("rb") as f: model = f.read() if manifest.get(KEY_VERSION) == 1: @@ -451,6 +453,10 @@ async def to_code(config): cg.add_define("USE_MICRO_WAKE_WORD") ota.request_ota_state_listeners() + if config.get(CONF_TASK_STACK_IN_PSRAM): + cg.add(var.set_task_stack_in_psram(True)) + psram.request_external_task_stack() + esp32.add_idf_component(name="espressif/esp-tflite-micro", ref="1.3.3~1") # Pin esp-nn for stable future builds (esp-tflite-micro depends on esp-nn) esp32.add_idf_component(name="espressif/esp-nn", ref="1.1.2") diff --git a/esphome/components/micro_wake_word/micro_wake_word.cpp b/esphome/components/micro_wake_word/micro_wake_word.cpp index 6877e9e5df..237d72229d 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.cpp +++ b/esphome/components/micro_wake_word/micro_wake_word.cpp @@ -33,7 +33,8 @@ static const uint32_t INFERENCE_TASK_STACK_SIZE = 3072; static const UBaseType_t INFERENCE_TASK_PRIORITY = 3; enum EventGroupBits : uint32_t { - COMMAND_STOP = (1 << 0), // Signals the inference task should stop + COMMAND_STOP = (1 << 0), // Signals the inference task should stop + COMMAND_RESET_RING_BUFFER = (1 << 1), // Signals the inference task to discard buffered audio TASK_STARTING = (1 << 3), TASK_RUNNING = (1 << 4), @@ -114,13 +115,13 @@ void MicroWakeWord::setup() { } std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); if (this->ring_buffer_.use_count() > 1) { - size_t bytes_free = temp_ring_buffer->free(); - - if (bytes_free < data.size()) { - xEventGroupSetBits(this->event_group_, EventGroupBits::WARNING_FULL_RING_BUFFER); - temp_ring_buffer->reset(); + // Producer-only write: never touches consumer state. If the buffer is full, ask the inference task + // to drain it - reset() is a consumer operation and must run on the inference task's thread. + // Disable partial writes so audio chunks are either fully accepted or rejected and handled below. + if (temp_ring_buffer->write_without_replacement(data.data(), data.size(), 0, false) == 0) { + xEventGroupSetBits(this->event_group_, + EventGroupBits::WARNING_FULL_RING_BUFFER | EventGroupBits::COMMAND_RESET_RING_BUFFER); } - temp_ring_buffer->write((void *) data.data(), data.size()); } }); @@ -146,56 +147,65 @@ void MicroWakeWord::inference_task(void *params) { { // Ensures any C++ objects fall out of scope to deallocate before deleting the task - const size_t new_bytes_to_process = - this_mww->microphone_source_->get_audio_stream_info().ms_to_bytes(this_mww->features_step_size_); - std::unique_ptr audio_buffer; + const auto &stream_info = this_mww->microphone_source_->get_audio_stream_info(); + const size_t bytes_per_frame = stream_info.frames_to_bytes(1); + const size_t max_fill_bytes = stream_info.ms_to_bytes(this_mww->features_step_size_); + std::unique_ptr audio_source; int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE]; if (!(xEventGroupGetBits(this_mww->event_group_) & ERROR_BITS)) { - // Allocate audio transfer buffer - audio_buffer = audio::AudioSourceTransferBuffer::create(new_bytes_to_process); - - if (audio_buffer == nullptr) { + // Round ring buffer size down to a frame multiple so the wrap boundary never splits an int16 sample. + const size_t ring_buffer_size = + (stream_info.ms_to_bytes(RING_BUFFER_DURATION_MS) / bytes_per_frame) * bytes_per_frame; + std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create(ring_buffer_size); + if (temp_ring_buffer == nullptr) { xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_MEMORY); + } else { + audio_source = audio::RingBufferAudioSource::create(temp_ring_buffer, max_fill_bytes, + static_cast(bytes_per_frame)); + if (audio_source == nullptr) { + xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_MEMORY); + } else { + this_mww->ring_buffer_ = temp_ring_buffer; + } } } - if (!(xEventGroupGetBits(this_mww->event_group_) & ERROR_BITS)) { - // Allocate ring buffer - std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create( - this_mww->microphone_source_->get_audio_stream_info().ms_to_bytes(RING_BUFFER_DURATION_MS)); - if (temp_ring_buffer.use_count() == 0) { - xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_MEMORY); - } - audio_buffer->set_source(temp_ring_buffer); - this_mww->ring_buffer_ = temp_ring_buffer; - } - if (!(xEventGroupGetBits(this_mww->event_group_) & ERROR_BITS)) { this_mww->microphone_source_->start(); xEventGroupSetBits(this_mww->event_group_, EventGroupBits::TASK_RUNNING); - while (!(xEventGroupGetBits(this_mww->event_group_) & COMMAND_STOP)) { - audio_buffer->transfer_data_from_source(pdMS_TO_TICKS(DATA_TIMEOUT_MS)); - - if (audio_buffer->available() < new_bytes_to_process) { - // Insufficient data to generate new spectrogram features, read more next iteration - continue; + while (!(xEventGroupGetBits(this_mww->event_group_) & (COMMAND_STOP | ERROR_BITS))) { + if (xEventGroupGetBits(this_mww->event_group_) & EventGroupBits::COMMAND_RESET_RING_BUFFER) { + // Producer asked us to drain; run the consumer-side reset from this thread. + audio_source->clear_buffered_data(); + xEventGroupClearBits(this_mww->event_group_, EventGroupBits::COMMAND_RESET_RING_BUFFER); } - // Generate new spectrogram features - uint32_t processed_samples = this_mww->generate_features_( - (int16_t *) audio_buffer->get_buffer_start(), audio_buffer->available() / sizeof(int16_t), features_buffer); - audio_buffer->decrease_buffer_length(processed_samples * sizeof(int16_t)); + audio_source->fill(pdMS_TO_TICKS(DATA_TIMEOUT_MS), false); - // Run inference using the new spectorgram features - if (!this_mww->update_model_probabilities_(features_buffer)) { - xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_INFERENCE); - break; + // The frontend buffers samples internally and only emits a feature once it has a full window, so we can + // hand it whatever the source exposes. The frontend consumes at least one sample per call, so available() + // strictly decreases and this loop always terminates. + while (audio_source->available() >= sizeof(int16_t)) { + const size_t samples_available = audio_source->available() / sizeof(int16_t); + const int16_t *audio_data = reinterpret_cast(audio_source->data()); + + size_t processed_samples = 0; + const bool feature_generated = + this_mww->generate_features_(audio_data, samples_available, features_buffer, &processed_samples); + audio_source->consume(processed_samples * sizeof(int16_t)); + + if (feature_generated) { + if (!this_mww->update_model_probabilities_(features_buffer)) { + xEventGroupSetBits(this_mww->event_group_, EventGroupBits::ERROR_INFERENCE); + break; + } + + // Process each model's probabilities and possibly send a Detection Event to the queue + this_mww->process_probabilities_(); + } } - - // Process each model's probabilities and possibly send a Detection Event to the queue - this_mww->process_probabilities_(); } } } @@ -207,10 +217,7 @@ void MicroWakeWord::inference_task(void *params) { FrontendFreeStateContents(&this_mww->frontend_state_); xEventGroupSetBits(this_mww->event_group_, EventGroupBits::TASK_STOPPED); - while (true) { - // Continuously delay until the main loop deletes the task - delay(10); - } + vTaskSuspend(nullptr); // Suspend this task indefinitely until the loop method deletes it } std::vector MicroWakeWord::get_wake_words() { @@ -233,14 +240,14 @@ void MicroWakeWord::add_vad_model(const uint8_t *model_start, uint8_t probabilit #endif void MicroWakeWord::suspend_task_() { - if (this->inference_task_handle_ != nullptr) { - vTaskSuspend(this->inference_task_handle_); + if (this->inference_task_.is_created()) { + vTaskSuspend(this->inference_task_.get_handle()); } } void MicroWakeWord::resume_task_() { - if (this->inference_task_handle_ != nullptr) { - vTaskResume(this->inference_task_handle_); + if (this->inference_task_.is_created()) { + vTaskResume(this->inference_task_.get_handle()); } } @@ -282,8 +289,7 @@ void MicroWakeWord::loop() { if ((event_group_bits & EventGroupBits::TASK_STOPPED)) { ESP_LOGD(TAG, "Inference task is finished, freeing task resources"); - vTaskDelete(this->inference_task_handle_); - this->inference_task_handle_ = nullptr; + this->inference_task_.deallocate(); xEventGroupClearBits(this->event_group_, ALL_BITS); xQueueReset(this->detection_queue_); this->set_state_(State::STOPPED); @@ -301,7 +307,7 @@ void MicroWakeWord::loop() { switch (this->state_) { case State::STARTING: - if ((this->inference_task_handle_ == nullptr) && !this->status_has_error()) { + if (!this->inference_task_.is_created() && !this->status_has_error()) { // Setup preprocesor feature generator. If done in the task, it would lock the task to its initial core, as it // uses floating point operations. if (!FrontendPopulateState(&this->frontend_config_, &this->frontend_state_, @@ -310,10 +316,8 @@ void MicroWakeWord::loop() { return; } - xTaskCreate(MicroWakeWord::inference_task, "mww", INFERENCE_TASK_STACK_SIZE, (void *) this, - INFERENCE_TASK_PRIORITY, &this->inference_task_handle_); - - if (this->inference_task_handle_ == nullptr) { + if (!this->inference_task_.create(MicroWakeWord::inference_task, "mww", INFERENCE_TASK_STACK_SIZE, + (void *) this, INFERENCE_TASK_PRIORITY, this->task_stack_in_psram_)) { FrontendFreeStateContents(&this->frontend_state_); // Deallocate frontend state this->status_momentary_error("task_start", 1000); } @@ -386,11 +390,15 @@ void MicroWakeWord::set_state_(State state) { } } -size_t MicroWakeWord::generate_features_(int16_t *audio_buffer, size_t samples_available, - int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE]) { - size_t processed_samples = 0; +bool MicroWakeWord::generate_features_(const int16_t *audio_buffer, size_t samples_available, + int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE], size_t *processed_samples) { + *processed_samples = 0; struct FrontendOutput frontend_output = - FrontendProcessSamples(&this->frontend_state_, audio_buffer, samples_available, &processed_samples); + FrontendProcessSamples(&this->frontend_state_, audio_buffer, samples_available, processed_samples); + + if (frontend_output.size == 0) { + return false; + } for (size_t i = 0; i < frontend_output.size; ++i) { // These scaling values are set to match the TFLite audio frontend int8 output. @@ -415,7 +423,7 @@ size_t MicroWakeWord::generate_features_(int16_t *audio_buffer, size_t samples_a features_buffer[i] = static_cast(clamp(value, INT8_MIN, INT8_MAX)); } - return processed_samples; + return true; } void MicroWakeWord::process_probabilities_() { diff --git a/esphome/components/micro_wake_word/micro_wake_word.h b/esphome/components/micro_wake_word/micro_wake_word.h index 5c0c056ac0..e4c590a423 100644 --- a/esphome/components/micro_wake_word/micro_wake_word.h +++ b/esphome/components/micro_wake_word/micro_wake_word.h @@ -11,6 +11,7 @@ #include "esphome/core/automation.h" #include "esphome/core/component.h" #include "esphome/core/defines.h" +#include "esphome/core/static_task.h" #ifdef USE_OTA_STATE_LISTENER #include "esphome/components/ota/ota_backend.h" @@ -59,6 +60,8 @@ class MicroWakeWord : public Component void set_stop_after_detection(bool stop_after_detection) { this->stop_after_detection_ = stop_after_detection; } + void set_task_stack_in_psram(bool task_stack_in_psram) { this->task_stack_in_psram_ = task_stack_in_psram; } + Trigger *get_wake_word_detected_trigger() { return &this->wake_word_detected_trigger_; } void add_wake_word_model(WakeWordModel *model); @@ -93,6 +96,8 @@ class MicroWakeWord : public Component bool stop_after_detection_; + bool task_stack_in_psram_{false}; + uint8_t features_step_size_; // Audio frontend handles generating spectrogram features @@ -105,8 +110,9 @@ class MicroWakeWord : public Component // Used to send messages about the models' states to the main loop QueueHandle_t detection_queue_; + StaticTask inference_task_; + static void inference_task(void *params); - TaskHandle_t inference_task_handle_{nullptr}; /// @brief Suspends the inference task void suspend_task_(); @@ -115,13 +121,16 @@ class MicroWakeWord : public Component void set_state_(State state); - /// @brief Generates spectrogram features from an input buffer of audio samples - /// @param audio_buffer (int16_t *) Buffer containing input audio samples - /// @param samples_available (size_t) Number of samples avaiable in the input buffer - /// @param features_buffer (int8_t *) Buffer to store generated features - /// @return (size_t) Number of samples processed from the input buffer - size_t generate_features_(int16_t *audio_buffer, size_t samples_available, - int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE]); + /// @brief Generates a spectrogram feature from an input buffer of audio samples. The frontend buffers samples + /// internally, so callers may stream arbitrary-sized chunks; a feature is only emitted once enough samples have + /// accumulated to fill a full analysis window. + /// @param audio_buffer (const int16_t *) Buffer containing input audio samples + /// @param samples_available (size_t) Number of samples available in the input buffer + /// @param features_buffer (int8_t *) Buffer to store the generated feature, valid only when the return value is true + /// @param processed_samples (size_t *) Set to the number of samples consumed from the input buffer + /// @return True if a new feature was generated; false if more samples are required + bool generate_features_(const int16_t *audio_buffer, size_t samples_available, + int8_t features_buffer[PREPROCESSOR_FEATURE_SIZE], size_t *processed_samples); /// @brief Processes any new probabilities for each model. If any wake word is detected, it will send a DetectionEvent /// to the detection_queue_. diff --git a/esphome/components/mitsubishi_cn105/climate.py b/esphome/components/mitsubishi_cn105/climate.py index cc44494d89..522b9218fc 100644 --- a/esphome/components/mitsubishi_cn105/climate.py +++ b/esphome/components/mitsubishi_cn105/climate.py @@ -1,8 +1,14 @@ from esphome import automation import esphome.codegen as cg from esphome.components import climate, uart +from esphome.components.climate import validate_climate_swing_mode import esphome.config_validation as cv -from esphome.const import CONF_ID, CONF_TEMPERATURE, CONF_UPDATE_INTERVAL +from esphome.const import ( + CONF_ID, + CONF_SUPPORTED_SWING_MODES, + CONF_TEMPERATURE, + CONF_UPDATE_INTERVAL, +) from esphome.core import ID from esphome.cpp_generator import MockObj from esphome.types import ConfigType, TemplateArgsType @@ -43,6 +49,9 @@ CONFIG_SCHEMA = ( cv.Optional( CONF_CURRENT_TEMPERATURE_MIN_INTERVAL, default="60s" ): cv.update_interval, + cv.Optional( + CONF_SUPPORTED_SWING_MODES, default="OFF" + ): validate_climate_swing_mode, } ) ) @@ -63,6 +72,7 @@ async def to_code(config: ConfigType) -> None: var = await climate.new_climate(config) await cg.register_component(var, config) await uart.register_uart_device(var, config) + cg.add(var.set_supported_swing_mode(config[CONF_SUPPORTED_SWING_MODES])) cg.add( var.set_current_temperature_min_interval( config[CONF_CURRENT_TEMPERATURE_MIN_INTERVAL] diff --git a/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h b/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h index dbeb43068e..742d8e18a9 100644 --- a/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h +++ b/esphome/components/mitsubishi_cn105/mitsubishi_cn105.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include "esphome/components/uart/uart.h" #include "esphome/core/finite_set_mask.h" diff --git a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp index 67a561397a..afffe7ea5e 100644 --- a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp +++ b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.cpp @@ -84,6 +84,8 @@ climate::ClimateTraits MitsubishiCN105Climate::traits() { traits.add_supported_fan_mode(p.second); } + traits.set_supported_swing_modes(this->supported_swing_modes_); + traits.set_visual_min_temperature(16.0f); traits.set_visual_max_temperature(31.0f); traits.set_visual_temperature_step(1.0f); @@ -114,6 +116,37 @@ void MitsubishiCN105Climate::control(const climate::ClimateCall &call) { this->hp_.set_fan_mode(*fan_mode); } + if (const auto swing_mode = call.get_swing_mode()) { + auto vane = this->last_non_swing_vane_mode_; + auto wide = this->last_non_swing_wide_vane_mode_; + + switch (*swing_mode) { + case climate::CLIMATE_SWING_BOTH: + vane = MitsubishiCN105::VaneMode::SWING; + wide = MitsubishiCN105::WideVaneMode::SWING; + break; + + case climate::CLIMATE_SWING_VERTICAL: + vane = MitsubishiCN105::VaneMode::SWING; + break; + + case climate::CLIMATE_SWING_HORIZONTAL: + wide = MitsubishiCN105::WideVaneMode::SWING; + break; + + case climate::CLIMATE_SWING_OFF: + default: + break; + } + + if (this->supported_swing_modes_.count(climate::CLIMATE_SWING_VERTICAL)) { + this->hp_.set_vane_mode(vane); + } + if (this->supported_swing_modes_.count(climate::CLIMATE_SWING_HORIZONTAL)) { + this->hp_.set_wide_vane_mode(wide); + } + } + if (this->hp_.is_status_initialized()) { this->apply_values_(); } @@ -143,7 +176,64 @@ void MitsubishiCN105Climate::apply_values_() { ESP_LOGD(TAG, "Unable to map fan mode"); } + if (!this->supported_swing_modes_.empty()) { + bool vertical_swinging = false; + bool horizontal_swinging = false; + + if (this->supported_swing_modes_.count(climate::CLIMATE_SWING_VERTICAL)) { + if (status.vane_mode == MitsubishiCN105::VaneMode::SWING) { + vertical_swinging = true; + } else if (status.vane_mode != MitsubishiCN105::VaneMode::UNKNOWN) { + this->last_non_swing_vane_mode_ = status.vane_mode; + } + } + + if (this->supported_swing_modes_.count(climate::CLIMATE_SWING_HORIZONTAL)) { + if (status.wide_vane_mode == MitsubishiCN105::WideVaneMode::SWING) { + horizontal_swinging = true; + } else if (status.wide_vane_mode != MitsubishiCN105::WideVaneMode::UNKNOWN) { + this->last_non_swing_wide_vane_mode_ = status.wide_vane_mode; + } + } + + if (vertical_swinging && horizontal_swinging) { + this->swing_mode = climate::CLIMATE_SWING_BOTH; + } else if (vertical_swinging) { + this->swing_mode = climate::CLIMATE_SWING_VERTICAL; + } else if (horizontal_swinging) { + this->swing_mode = climate::CLIMATE_SWING_HORIZONTAL; + } else { + this->swing_mode = climate::CLIMATE_SWING_OFF; + } + } + this->publish_state(); } +void MitsubishiCN105Climate::set_supported_swing_mode(climate::ClimateSwingMode mode) { + this->supported_swing_modes_.clear(); + switch (mode) { + case climate::CLIMATE_SWING_VERTICAL: + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_OFF); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_VERTICAL); + break; + + case climate::CLIMATE_SWING_HORIZONTAL: + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_OFF); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_HORIZONTAL); + break; + + case climate::CLIMATE_SWING_BOTH: + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_OFF); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_VERTICAL); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_HORIZONTAL); + this->supported_swing_modes_.insert(climate::CLIMATE_SWING_BOTH); + break; + + case climate::CLIMATE_SWING_OFF: + default: + break; + } +} + } // namespace esphome::mitsubishi_cn105 diff --git a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h index e09158bfcf..c83a5519c1 100644 --- a/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h +++ b/esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h @@ -25,10 +25,15 @@ class MitsubishiCN105Climate : public climate::Climate, public Component, public void set_remote_temperature(float temperature) { this->hp_.set_remote_temperature(temperature); } void clear_remote_temperature() { this->hp_.clear_remote_temperature(); } + void set_supported_swing_mode(climate::ClimateSwingMode mode); + protected: void apply_values_(); MitsubishiCN105 hp_; + climate::ClimateSwingModeMask supported_swing_modes_{}; + MitsubishiCN105::VaneMode last_non_swing_vane_mode_{MitsubishiCN105::VaneMode::AUTO}; + MitsubishiCN105::WideVaneMode last_non_swing_wide_vane_mode_{MitsubishiCN105::WideVaneMode::CENTER}; }; template diff --git a/esphome/components/mixer/speaker/__init__.py b/esphome/components/mixer/speaker/__init__.py index 59a80d9297..47164a9997 100644 --- a/esphome/components/mixer/speaker/__init__.py +++ b/esphome/components/mixer/speaker/__init__.py @@ -1,6 +1,6 @@ from esphome import automation import esphome.codegen as cg -from esphome.components import audio, esp32, speaker +from esphome.components import audio, psram, speaker import esphome.config_validation as cv from esphome.const import ( CONF_BITS_PER_SAMPLE, @@ -44,20 +44,10 @@ SOURCE_SPEAKER_SCHEMA = speaker.SPEAKER_SCHEMA.extend( cv.positive_time_period_milliseconds, cv.one_of(CONF_NEVER, lower=True), ), - cv.Optional(CONF_BITS_PER_SAMPLE, default=16): cv.int_range(16, 16), } ) -def _set_stream_limits(config): - audio.set_stream_limits( - min_bits_per_sample=16, - max_bits_per_sample=16, - )(config) - - return config - - def _validate_source_speaker(config): fconf = fv.full_config.get() @@ -67,15 +57,25 @@ def _validate_source_speaker(config): output_speaker_id = fconf.get_config_for_path(path) config[CONF_OUTPUT_SPEAKER] = output_speaker_id + inherit_property_from(CONF_BITS_PER_SAMPLE, CONF_OUTPUT_SPEAKER)(config) inherit_property_from(CONF_NUM_CHANNELS, CONF_OUTPUT_SPEAKER)(config) inherit_property_from(CONF_SAMPLE_RATE, CONF_OUTPUT_SPEAKER)(config) + audio.final_validate_audio_schema( + "mixer", + audio_device=CONF_OUTPUT_SPEAKER, + sample_rate=config.get(CONF_SAMPLE_RATE), + )(config) + + return config + + +def _validate_output_speaker(config): audio.final_validate_audio_schema( "mixer", audio_device=CONF_OUTPUT_SPEAKER, bits_per_sample=config.get(CONF_BITS_PER_SAMPLE), channels=config.get(CONF_NUM_CHANNELS), - sample_rate=config.get(CONF_SAMPLE_RATE), )(config) return config @@ -89,24 +89,26 @@ CONFIG_SCHEMA = cv.All( cv.Required(CONF_SOURCE_SPEAKERS): cv.All( cv.ensure_list(SOURCE_SPEAKER_SCHEMA), cv.Length(min=2, max=8), - [_set_stream_limits], ), + cv.Optional(CONF_BITS_PER_SAMPLE): cv.one_of(8, 16, 24, 32, int=True), cv.Optional(CONF_NUM_CHANNELS): cv.int_range(min=1, max=2), cv.Optional(CONF_QUEUE_MODE, default=False): cv.boolean, - cv.Optional(CONF_TASK_STACK_IN_PSRAM, default=False): cv.boolean, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, } ), cv.only_on([PLATFORM_ESP32]), ) FINAL_VALIDATE_SCHEMA = cv.All( + inherit_property_from(CONF_BITS_PER_SAMPLE, CONF_OUTPUT_SPEAKER), + inherit_property_from(CONF_NUM_CHANNELS, CONF_OUTPUT_SPEAKER), cv.Schema( { cv.Optional(CONF_SOURCE_SPEAKERS): [_validate_source_speaker], }, extra=cv.ALLOW_EXTRA, ), - inherit_property_from(CONF_NUM_CHANNELS, CONF_OUTPUT_SPEAKER), + _validate_output_speaker, ) @@ -116,16 +118,14 @@ async def to_code(config): spkr = await cg.get_variable(config[CONF_OUTPUT_SPEAKER]) + cg.add(var.set_output_bits_per_sample(config[CONF_BITS_PER_SAMPLE])) cg.add(var.set_output_channels(config[CONF_NUM_CHANNELS])) cg.add(var.set_output_speaker(spkr)) cg.add(var.set_queue_mode(config[CONF_QUEUE_MODE])) - if task_stack_in_psram := config.get(CONF_TASK_STACK_IN_PSRAM): - cg.add(var.set_task_stack_in_psram(task_stack_in_psram)) - if task_stack_in_psram and config[CONF_TASK_STACK_IN_PSRAM]: - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + if config.get(CONF_TASK_STACK_IN_PSRAM): + cg.add(var.set_task_stack_in_psram(True)) + psram.request_external_task_stack() # Initialize FixedVector with exact count of source speakers cg.add(var.init_source_speakers(len(config[CONF_SOURCE_SPEAKERS]))) diff --git a/esphome/components/mixer/speaker/mixer_speaker.cpp b/esphome/components/mixer/speaker/mixer_speaker.cpp index 1a995a6edf..6128dc3767 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.cpp +++ b/esphome/components/mixer/speaker/mixer_speaker.cpp @@ -7,8 +7,10 @@ #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#include // esp-audio-libs +#include // esp-audio-libs + #include -#include #include namespace esphome::mixer_speaker { @@ -22,19 +24,8 @@ static const uint32_t MIXER_AUTO_STOP_DEBOUNCE_MS = 200; static const size_t TASK_STACK_SIZE = 4096; -static const int16_t MAX_AUDIO_SAMPLE_VALUE = INT16_MAX; -static const int16_t MIN_AUDIO_SAMPLE_VALUE = INT16_MIN; - static const char *const TAG = "speaker_mixer"; -// Gives the Q15 fixed point scaling factor to reduce by 0 dB, 1dB, ..., 50 dB -// dB to PCM scaling factor formula: floating_point_scale_factor = 2^(-db/6.014) -// float to Q15 fixed point formula: q15_scale_factor = floating_point_scale_factor * 2^(15) -static const std::array DECIBEL_REDUCTION_TABLE = { - 32767, 29201, 26022, 23189, 20665, 18415, 16410, 14624, 13032, 11613, 10349, 9222, 8218, 7324, 6527, 5816, 5183, - 4619, 4116, 3668, 3269, 2913, 2596, 2313, 2061, 1837, 1637, 1459, 1300, 1158, 1032, 920, 820, 731, - 651, 580, 517, 461, 411, 366, 326, 291, 259, 231, 206, 183, 163, 146, 130, 116, 103}; - // Event bits for SourceSpeaker command processing enum SourceSpeakerEventBits : uint32_t { SOURCE_SPEAKER_COMMAND_START = (1 << 0), @@ -315,97 +306,17 @@ size_t SourceSpeaker::process_data_from_source(std::shared_ptraudio_stream_info_.bytes_to_samples(bytes_read); if (samples_to_duck > 0) { - int16_t *current_buffer = reinterpret_cast(audio_source->mutable_data()); - - duck_samples(current_buffer, samples_to_duck, &this->current_ducking_db_reduction_, - &this->ducking_transition_samples_remaining_, this->samples_per_ducking_step_, - this->db_change_per_ducking_step_); + esp_audio_libs::ducking::apply(audio_source->mutable_data(), + static_cast(this->audio_stream_info_.get_bits_per_sample() / 8), + samples_to_duck, this->ducking_state_); } return bytes_read; } void SourceSpeaker::apply_ducking(uint8_t decibel_reduction, uint32_t duration) { - if (this->target_ducking_db_reduction_ != decibel_reduction) { - // Start transition from the previous target (which becomes the new current level) - this->current_ducking_db_reduction_ = this->target_ducking_db_reduction_; - - this->target_ducking_db_reduction_ = decibel_reduction; - - // Calculate the number of intermediate dB steps for the transition timing. - // Subtract 1 because the first step is taken immediately after this calculation. - uint8_t total_ducking_steps = 0; - if (this->target_ducking_db_reduction_ > this->current_ducking_db_reduction_) { - // The dB reduction level is increasing (which results in quieter audio) - total_ducking_steps = this->target_ducking_db_reduction_ - this->current_ducking_db_reduction_ - 1; - this->db_change_per_ducking_step_ = 1; - } else { - // The dB reduction level is decreasing (which results in louder audio) - total_ducking_steps = this->current_ducking_db_reduction_ - this->target_ducking_db_reduction_ - 1; - this->db_change_per_ducking_step_ = -1; - } - if ((duration > 0) && (total_ducking_steps > 0)) { - this->ducking_transition_samples_remaining_ = this->audio_stream_info_.ms_to_samples(duration); - - this->samples_per_ducking_step_ = this->ducking_transition_samples_remaining_ / total_ducking_steps; - this->ducking_transition_samples_remaining_ = - this->samples_per_ducking_step_ * total_ducking_steps; // adjust for integer division rounding - - this->current_ducking_db_reduction_ += this->db_change_per_ducking_step_; - } else { - this->ducking_transition_samples_remaining_ = 0; - this->current_ducking_db_reduction_ = this->target_ducking_db_reduction_; - } - } -} - -void SourceSpeaker::duck_samples(int16_t *input_buffer, uint32_t input_samples_to_duck, - int8_t *current_ducking_db_reduction, uint32_t *ducking_transition_samples_remaining, - uint32_t samples_per_ducking_step, int8_t db_change_per_ducking_step) { - if (*ducking_transition_samples_remaining > 0) { - // Ducking level is still transitioning - - // Takes the ceiling of input_samples_to_duck/samples_per_ducking_step - uint32_t ducking_steps_in_batch = - input_samples_to_duck / samples_per_ducking_step + (input_samples_to_duck % samples_per_ducking_step != 0); - - for (uint32_t i = 0; i < ducking_steps_in_batch; ++i) { - uint32_t samples_left_in_step = *ducking_transition_samples_remaining % samples_per_ducking_step; - - if (samples_left_in_step == 0) { - samples_left_in_step = samples_per_ducking_step; - } - - uint32_t samples_to_duck = std::min(input_samples_to_duck, samples_left_in_step); - samples_to_duck = std::min(samples_to_duck, *ducking_transition_samples_remaining); - - // Ensure we only point to valid index in the Q15 scaling factor table - uint8_t safe_db_reduction_index = - clamp(*current_ducking_db_reduction, 0, DECIBEL_REDUCTION_TABLE.size() - 1); - int16_t q15_scale_factor = DECIBEL_REDUCTION_TABLE[safe_db_reduction_index]; - - audio::scale_audio_samples(input_buffer, input_buffer, q15_scale_factor, samples_to_duck); - - if (samples_left_in_step - samples_to_duck == 0) { - // After scaling the current samples, we are ready to transition to the next step - *current_ducking_db_reduction += db_change_per_ducking_step; - } - - input_buffer += samples_to_duck; - *ducking_transition_samples_remaining -= samples_to_duck; - input_samples_to_duck -= samples_to_duck; - } - } - - if ((*current_ducking_db_reduction > 0) && (input_samples_to_duck > 0)) { - // Audio is ducked, but its not in the middle of a transition step - - uint8_t safe_db_reduction_index = - clamp(*current_ducking_db_reduction, 0, DECIBEL_REDUCTION_TABLE.size() - 1); - int16_t q15_scale_factor = DECIBEL_REDUCTION_TABLE[safe_db_reduction_index]; - - audio::scale_audio_samples(input_buffer, input_buffer, q15_scale_factor, input_samples_to_duck); - } + const uint32_t transition_samples = duration > 0 ? this->audio_stream_info_.ms_to_samples(duration) : 0; + esp_audio_libs::ducking::set_target(this->ducking_state_, decibel_reduction, transition_samples); } void SourceSpeaker::enter_stopping_state_() { @@ -417,8 +328,9 @@ void SourceSpeaker::enter_stopping_state_() { void MixerSpeaker::dump_config() { ESP_LOGCONFIG(TAG, "Speaker Mixer:\n" - " Number of output channels: %u", - this->output_channels_); + " Number of output channels: %" PRIu8 "\n" + " Output bits per sample: %" PRIu8, + this->output_channels_, this->output_bits_per_sample_); } void MixerSpeaker::setup() { @@ -512,13 +424,8 @@ void MixerSpeaker::loop() { esp_err_t MixerSpeaker::start(audio::AudioStreamInfo &stream_info) { if (!this->audio_stream_info_.has_value()) { - if (stream_info.get_bits_per_sample() != 16) { - // Audio streams that don't have 16 bits per sample are not supported - return ESP_ERR_NOT_SUPPORTED; - } - - this->audio_stream_info_ = audio::AudioStreamInfo(stream_info.get_bits_per_sample(), this->output_channels_, - stream_info.get_sample_rate()); + this->audio_stream_info_ = + audio::AudioStreamInfo(this->output_bits_per_sample_, this->output_channels_, stream_info.get_sample_rate()); this->output_speaker_->set_audio_stream_info(this->audio_stream_info_.value()); } else { if (!this->queue_mode_ && (stream_info.get_sample_rate() != this->audio_stream_info_.value().get_sample_rate())) { @@ -542,57 +449,6 @@ esp_err_t MixerSpeaker::start(audio::AudioStreamInfo &stream_info) { return ESP_OK; } -void MixerSpeaker::copy_frames(const int16_t *input_buffer, audio::AudioStreamInfo input_stream_info, - int16_t *output_buffer, audio::AudioStreamInfo output_stream_info, - uint32_t frames_to_transfer) { - uint8_t input_channels = input_stream_info.get_channels(); - uint8_t output_channels = output_stream_info.get_channels(); - const uint8_t max_input_channel_index = input_channels - 1; - - if (input_channels == output_channels) { - size_t bytes_to_copy = input_stream_info.frames_to_bytes(frames_to_transfer); - memcpy(output_buffer, input_buffer, bytes_to_copy); - - return; - } - - for (uint32_t frame_index = 0; frame_index < frames_to_transfer; ++frame_index) { - for (uint8_t output_channel_index = 0; output_channel_index < output_channels; ++output_channel_index) { - uint8_t input_channel_index = std::min(output_channel_index, max_input_channel_index); - output_buffer[output_channels * frame_index + output_channel_index] = - input_buffer[input_channels * frame_index + input_channel_index]; - } - } -} - -void MixerSpeaker::mix_audio_samples(const int16_t *primary_buffer, audio::AudioStreamInfo primary_stream_info, - const int16_t *secondary_buffer, audio::AudioStreamInfo secondary_stream_info, - int16_t *output_buffer, audio::AudioStreamInfo output_stream_info, - uint32_t frames_to_mix) { - const uint8_t primary_channels = primary_stream_info.get_channels(); - const uint8_t secondary_channels = secondary_stream_info.get_channels(); - const uint8_t output_channels = output_stream_info.get_channels(); - - const uint8_t max_primary_channel_index = primary_channels - 1; - const uint8_t max_secondary_channel_index = secondary_channels - 1; - - for (uint32_t frames_index = 0; frames_index < frames_to_mix; ++frames_index) { - for (uint8_t output_channel_index = 0; output_channel_index < output_channels; ++output_channel_index) { - const uint32_t secondary_channel_index = std::min(output_channel_index, max_secondary_channel_index); - const int32_t secondary_sample = secondary_buffer[frames_index * secondary_channels + secondary_channel_index]; - - const uint32_t primary_channel_index = std::min(output_channel_index, max_primary_channel_index); - const int32_t primary_sample = - static_cast(primary_buffer[frames_index * primary_channels + primary_channel_index]); - - const int32_t added_sample = secondary_sample + primary_sample; - - output_buffer[frames_index * output_channels + output_channel_index] = - static_cast(clamp(added_sample, MIN_AUDIO_SAMPLE_VALUE, MAX_AUDIO_SAMPLE_VALUE)); - } - } -} - // NOLINTBEGIN(bugprone-unchecked-optional-access) -- audio_stream_info_ always set before this task is created void MixerSpeaker::audio_mixer_task(void *params) { MixerSpeaker *this_mixer = static_cast(params); @@ -662,6 +518,10 @@ void MixerSpeaker::audio_mixer_task(void *params) { uint32_t frames_to_mix = output_frames_free; + const audio::AudioStreamInfo &output_info = this_mixer->audio_stream_info_.value(); + const uint8_t output_bps = output_info.get_bits_per_sample() / 8; + const uint8_t output_channels = output_info.get_channels(); + if ((audio_sources_with_data.size() == 1) || this_mixer->queue_mode_) { // Only one speaker has audio data, just copy samples over @@ -669,14 +529,15 @@ void MixerSpeaker::audio_mixer_task(void *params) { if (active_stream_info.get_sample_rate() == this_mixer->output_speaker_->get_audio_stream_info().get_sample_rate()) { - // Speaker's sample rate matches the output speaker's, copy directly + // Speaker's sample rate matches the output speaker's, convert directly into the output buffer const uint32_t frames_available_in_buffer = active_stream_info.bytes_to_frames(audio_sources_with_data[0]->available()); frames_to_mix = std::min(frames_to_mix, frames_available_in_buffer); - copy_frames(reinterpret_cast(audio_sources_with_data[0]->data()), active_stream_info, - reinterpret_cast(output_transfer_buffer->get_buffer_end()), - this_mixer->audio_stream_info_.value(), frames_to_mix); + esp_audio_libs::pcm_convert::copy_frames( + audio_sources_with_data[0]->data(), output_transfer_buffer->get_buffer_end(), + static_cast(active_stream_info.get_bits_per_sample() / 8), active_stream_info.get_channels(), + output_bps, output_channels, frames_to_mix); // Set playback delay for newly contributing source if (!speakers_with_data[0]->has_contributed_.load(std::memory_order_acquire)) { @@ -690,8 +551,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { audio_sources_with_data[0]->consume(active_stream_info.frames_to_bytes(frames_to_mix)); // Update output transfer buffer length and pipeline frame count - output_transfer_buffer->increase_buffer_length( - this_mixer->audio_stream_info_.value().frames_to_bytes(frames_to_mix)); + output_transfer_buffer->increase_buffer_length(output_info.frames_to_bytes(frames_to_mix)); this_mixer->frames_in_pipeline_.fetch_add(frames_to_mix, std::memory_order_release); } else { // Speaker's stream info doesn't match the output speaker's, so it's a new source speaker @@ -703,7 +563,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { } else { // Speaker has finished writing the current audio, update the stream information and restart the speaker this_mixer->audio_stream_info_ = - audio::AudioStreamInfo(active_stream_info.get_bits_per_sample(), this_mixer->output_channels_, + audio::AudioStreamInfo(this_mixer->output_bits_per_sample_, this_mixer->output_channels_, active_stream_info.get_sample_rate()); this_mixer->output_speaker_->set_audio_stream_info(this_mixer->audio_stream_info_.value()); this_mixer->output_speaker_->start(); @@ -719,21 +579,22 @@ void MixerSpeaker::audio_mixer_task(void *params) { speakers_with_data[i]->get_audio_stream_info().bytes_to_frames(audio_sources_with_data[i]->available()); frames_to_mix = std::min(frames_to_mix, frames_available_in_buffer); } - const int16_t *primary_buffer = reinterpret_cast(audio_sources_with_data[0]->data()); + const uint8_t *primary_buffer = audio_sources_with_data[0]->data(); audio::AudioStreamInfo primary_stream_info = speakers_with_data[0]->get_audio_stream_info(); - // Mix two streams together + // Mix two streams together at a time, accumulating into the output buffer. for (size_t i = 1; i < audio_sources_with_data.size(); ++i) { - mix_audio_samples(primary_buffer, primary_stream_info, - reinterpret_cast(audio_sources_with_data[i]->data()), - speakers_with_data[i]->get_audio_stream_info(), - reinterpret_cast(output_transfer_buffer->get_buffer_end()), - this_mixer->audio_stream_info_.value(), frames_to_mix); + esp_audio_libs::mixer::mix_frames( + primary_buffer, static_cast(primary_stream_info.get_bits_per_sample() / 8), + primary_stream_info.get_channels(), audio_sources_with_data[i]->data(), + static_cast(speakers_with_data[i]->get_audio_stream_info().get_bits_per_sample() / 8), + speakers_with_data[i]->get_audio_stream_info().get_channels(), output_transfer_buffer->get_buffer_end(), + output_bps, output_channels, frames_to_mix); if (i != audio_sources_with_data.size() - 1) { // Need to mix more streams together, point primary buffer and stream info to the already mixed output - primary_buffer = reinterpret_cast(output_transfer_buffer->get_buffer_end()); - primary_stream_info = this_mixer->audio_stream_info_.value(); + primary_buffer = output_transfer_buffer->get_buffer_end(); + primary_stream_info = output_info; } } @@ -754,8 +615,7 @@ void MixerSpeaker::audio_mixer_task(void *params) { } // Update output transfer buffer length and pipeline frame count (once, not per source) - output_transfer_buffer->increase_buffer_length( - this_mixer->audio_stream_info_.value().frames_to_bytes(frames_to_mix)); + output_transfer_buffer->increase_buffer_length(output_info.frames_to_bytes(frames_to_mix)); this_mixer->frames_in_pipeline_.fetch_add(frames_to_mix, std::memory_order_release); } } diff --git a/esphome/components/mixer/speaker/mixer_speaker.h b/esphome/components/mixer/speaker/mixer_speaker.h index f57bead679..f1ae919b50 100644 --- a/esphome/components/mixer/speaker/mixer_speaker.h +++ b/esphome/components/mixer/speaker/mixer_speaker.h @@ -11,6 +11,8 @@ #include "esphome/core/helpers.h" #include "esphome/core/static_task.h" +#include // esp-audio-libs + #include #include @@ -22,7 +24,8 @@ namespace esphome::mixer_speaker { * - Source speaker commands are signaled via event group bits and processed in its loop function to ensure thread * safety * - Directly handles pausing at the SourceSpeaker level; pause state is not passed through to the output speaker. - * - Audio sent to the SourceSpeaker must have 16 bits per sample. + * - Audio sent to the SourceSpeaker can have 8, 16, 24, or 32 bits per sample. Each source is converted to the output + * speaker's bit depth as it is mixed (or copied) into the output buffer. * - Audio sent to the SourceSpeaker can have any number of channels. They are duplicated or ignored as needed to match * the number of channels required for the output speaker. * - In queue mode, the audio sent to the SourceSpeakers can have different sample rates. @@ -93,19 +96,6 @@ class SourceSpeaker : public speaker::Speaker, public Component { void enter_stopping_state_(); void send_command_(uint32_t command_bit, bool wake_loop = false); - /// @brief Ducks audio samples by a specified amount. When changing the ducking amount, it can transition gradually - /// over a specified amount of samples. - /// @param input_buffer buffer with audio samples to be ducked in place - /// @param input_samples_to_duck number of samples to process in ``input_buffer`` - /// @param current_ducking_db_reduction pointer to the current dB reduction - /// @param ducking_transition_samples_remaining pointer to the total number of samples left before the - /// transition is finished - /// @param samples_per_ducking_step total number of samples per ducking step for the transition - /// @param db_change_per_ducking_step the change in dB reduction per step - static void duck_samples(int16_t *input_buffer, uint32_t input_samples_to_duck, int8_t *current_ducking_db_reduction, - uint32_t *ducking_transition_samples_remaining, uint32_t samples_per_ducking_step, - int8_t db_change_per_ducking_step); - MixerSpeaker *parent_; std::shared_ptr audio_source_; @@ -118,11 +108,7 @@ class SourceSpeaker : public speaker::Speaker, public Component { bool pause_state_{false}; - int8_t target_ducking_db_reduction_{0}; - int8_t current_ducking_db_reduction_{0}; - int8_t db_change_per_ducking_step_{1}; - uint32_t ducking_transition_samples_remaining_{0}; - uint32_t samples_per_ducking_step_{0}; + esp_audio_libs::ducking::DuckingState ducking_state_{}; std::atomic pending_playback_frames_{0}; std::atomic playback_delay_frames_{0}; // Frames in output pipeline when this source started contributing @@ -143,12 +129,14 @@ class MixerSpeaker : public Component { /// @brief Starts the mixer task. Called by a source speaker giving the current audio stream information /// @param stream_info The calling source speaker's audio stream information - /// @return ESP_ERR_NOT_SUPPORTED if the incoming stream is incompatible due to unsupported bits per sample - /// ESP_ERR_INVALID_ARG if the incoming stream is incompatible to be mixed with the other input audio stream + /// @return ESP_ERR_INVALID_ARG if the incoming stream is incompatible to be mixed with the other input audio stream /// ESP_OK if the incoming stream is compatible and the mixer task starts esp_err_t start(audio::AudioStreamInfo &stream_info); void set_output_channels(uint8_t output_channels) { this->output_channels_ = output_channels; } + void set_output_bits_per_sample(uint8_t output_bits_per_sample) { + this->output_bits_per_sample_ = output_bits_per_sample; + } void set_output_speaker(speaker::Speaker *speaker) { this->output_speaker_ = speaker; } void set_queue_mode(bool queue_mode) { this->queue_mode_ = queue_mode; } void set_task_stack_in_psram(bool task_stack_in_psram) { this->task_stack_in_psram_ = task_stack_in_psram; } @@ -159,33 +147,6 @@ class MixerSpeaker : public Component { uint32_t get_frames_in_pipeline() const { return this->frames_in_pipeline_.load(std::memory_order_acquire); } protected: - /// @brief Copies audio frames from the input buffer to the output buffer taking into account the number of channels - /// in each stream. If the output stream has more channels, the input samples are duplicated. If the output stream has - /// less channels, the extra channel input samples are dropped. - /// @param input_buffer - /// @param input_stream_info - /// @param output_buffer - /// @param output_stream_info - /// @param frames_to_transfer number of frames (consisting of a sample for each channel) to copy from the input buffer - static void copy_frames(const int16_t *input_buffer, audio::AudioStreamInfo input_stream_info, int16_t *output_buffer, - audio::AudioStreamInfo output_stream_info, uint32_t frames_to_transfer); - - /// @brief Mixes the primary and secondary streams taking into account the number of channels in each stream. Primary - /// and secondary samples are duplicated or dropped as necessary to ensure the output stream has the configured number - /// of channels. Output samples are clamped to the corresponding int16 min or max values if the mixed sample - /// overflows. - /// @param primary_buffer samples buffer for the primary stream - /// @param primary_stream_info stream info for the primary stream - /// @param secondary_buffer samples buffer for secondary stream - /// @param secondary_stream_info stream info for the secondary stream - /// @param output_buffer buffer for the mixed samples - /// @param output_stream_info stream info for the output buffer - /// @param frames_to_mix number of frames in the primary and secondary buffers to mix together - static void mix_audio_samples(const int16_t *primary_buffer, audio::AudioStreamInfo primary_stream_info, - const int16_t *secondary_buffer, audio::AudioStreamInfo secondary_stream_info, - int16_t *output_buffer, audio::AudioStreamInfo output_stream_info, - uint32_t frames_to_mix); - static void audio_mixer_task(void *params); EventGroupHandle_t event_group_{nullptr}; @@ -193,6 +154,7 @@ class MixerSpeaker : public Component { FixedVector source_speakers_; speaker::Speaker *output_speaker_{nullptr}; + uint8_t output_bits_per_sample_; uint8_t output_channels_; bool queue_mode_; bool task_stack_in_psram_{false}; diff --git a/esphome/components/msa3xx/binary_sensor.py b/esphome/components/msa3xx/binary_sensor.py index 793d5190af..732a0ed291 100644 --- a/esphome/components/msa3xx/binary_sensor.py +++ b/esphome/components/msa3xx/binary_sensor.py @@ -26,7 +26,7 @@ CONFIG_SCHEMA = MSA_SENSOR_SCHEMA.extend( ), key=CONF_NAME, ) - for event, icon in zip(EVENT_SENSORS, ICONS) + for event, icon in zip(EVENT_SENSORS, ICONS, strict=True) } ) diff --git a/esphome/components/network/__init__.py b/esphome/components/network/__init__.py index 811e7c875a..2818b8c93e 100644 --- a/esphome/components/network/__init__.py +++ b/esphome/components/network/__init__.py @@ -5,8 +5,9 @@ import esphome.codegen as cg from esphome.components.esp32 import add_idf_sdkconfig_option from esphome.components.psram import is_guaranteed as psram_is_guaranteed import esphome.config_validation as cv -from esphome.const import CONF_ENABLE_IPV6, CONF_MIN_IPV6_ADDR_COUNT +from esphome.const import CONF_ENABLE_IPV6, CONF_ID, CONF_MIN_IPV6_ADDR_COUNT from esphome.core import CORE, CoroPriority, coroutine_with_priority +from esphome.types import ConfigType CODEOWNERS = ["@esphome/core"] AUTO_LOAD = ["mdns"] @@ -19,6 +20,7 @@ KEY_HIGH_PERFORMANCE_NETWORKING = "high_performance_networking" CONF_ENABLE_HIGH_PERFORMANCE = "enable_high_performance" network_ns = cg.esphome_ns.namespace("network") +NetworkComponent = network_ns.class_("NetworkComponent", cg.Component) IPAddress = network_ns.class_("IPAddress") @@ -107,6 +109,7 @@ def has_high_performance_networking() -> bool: CONFIG_SCHEMA = cv.Schema( { + cv.GenerateID(): cv.declare_id(NetworkComponent), cv.SplitDefault( CONF_ENABLE_IPV6, bk72xx=False, @@ -224,3 +227,15 @@ async def to_code(config): cg.add_build_flag("-DPIO_FRAMEWORK_ARDUINO_LWIP2_IPV6_LOW_MEMORY") if CORE.is_rp2040: cg.add_build_flag("-DPIO_FRAMEWORK_ARDUINO_ENABLE_IPV6") + # Pvariable creation lives in a separate coroutine at NETWORK_SERVICES so it + # emits after wifi/ethernet at COMMUNICATION. This keeps compile-time config + # (above) separate from C++ object lifecycle and allows wiring in interface + # pointers via get_variable(). + if CORE.is_esp32: + CORE.add_job(network_component_to_code, config) + + +@coroutine_with_priority(CoroPriority.NETWORK_SERVICES) +async def network_component_to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) diff --git a/esphome/components/network/network_component.cpp b/esphome/components/network/network_component.cpp new file mode 100644 index 0000000000..40cf64906c --- /dev/null +++ b/esphome/components/network/network_component.cpp @@ -0,0 +1,33 @@ +#include "network_component.h" + +#include "esphome/core/defines.h" +#if defined(USE_NETWORK) && defined(USE_ESP32) +#include "esphome/core/log.h" +#include "esp_err.h" +#include "esp_netif.h" +#include "esp_event.h" +namespace esphome::network { + +static const char *const TAG = "network"; + +void NetworkComponent::setup() { + // Initialize ESP-IDF network interfaces and ensure the default event loop exists + esp_err_t err; + err = esp_netif_init(); + if (err != ESP_OK) { + ESP_LOGE(TAG, "esp_netif_init failed: (%d) %s", err, esp_err_to_name(err)); + this->mark_failed(); + return; + } + err = esp_event_loop_create_default(); + // ESP_ERR_INVALID_STATE is returned if the default loop already exists, + // which is fine since we just want to make sure it exists + if (err != ESP_OK && err != ESP_ERR_INVALID_STATE) { + ESP_LOGE(TAG, "esp_event_loop_create_default failed: (%d) %s", err, esp_err_to_name(err)); + this->mark_failed(); + return; + } +} + +} // namespace esphome::network +#endif diff --git a/esphome/components/network/network_component.h b/esphome/components/network/network_component.h new file mode 100644 index 0000000000..dde15940e4 --- /dev/null +++ b/esphome/components/network/network_component.h @@ -0,0 +1,14 @@ +#pragma once +#include "esphome/core/defines.h" +#if defined(USE_NETWORK) && defined(USE_ESP32) +#include "esphome/core/component.h" + +namespace esphome::network { +class NetworkComponent : public Component { + public: + void setup() override; + // AFTER_BLUETOOTH: BLE controller must initialize before esp_netif_init per IDF guidance. + float get_setup_priority() const override { return setup_priority::AFTER_BLUETOOTH; } +}; +} // namespace esphome::network +#endif diff --git a/esphome/components/nrf52/ota.py b/esphome/components/nrf52/ota.py index eb1caa5595..5d608acbac 100644 --- a/esphome/components/nrf52/ota.py +++ b/esphome/components/nrf52/ota.py @@ -139,7 +139,7 @@ async def _smpmgr_upload_connected( already_uploaded = True if not already_uploaded: - with open(firmware, "rb") as file: + with firmware.open("rb") as file: image = file.read() upload_size = len(image) progress = ProgressBar("Uploading") diff --git a/esphome/components/opentherm/generate.py b/esphome/components/opentherm/generate.py index 0b39895798..1c0de329e5 100644 --- a/esphome/components/opentherm/generate.py +++ b/esphome/components/opentherm/generate.py @@ -16,7 +16,7 @@ def define_has_component(component_type: str, keys: list[str]) -> None: cg.add_define( f"OPENTHERM_{component_type.upper()}_LIST(F, sep)", cg.RawExpression( - " sep ".join(map(lambda key: f"F({key}_{component_type.lower()})", keys)) + " sep ".join(f"F({key}_{component_type.lower()})" for key in keys) ), ) for key in keys: @@ -30,12 +30,8 @@ def define_has_settings(keys: list[str], schemas: dict[str, SettingSchema]) -> N "OPENTHERM_SETTING_LIST(F, sep)", cg.RawExpression( " sep ".join( - map( - lambda key: ( - f"F({schemas[key].backing_type}, {key}_setting, {schemas[key].default_value})" - ), - keys, - ) + f"F({schemas[key].backing_type}, {key}_setting, {schemas[key].default_value})" + for key in keys ) ), ) diff --git a/esphome/components/opentherm/output/__init__.py b/esphome/components/opentherm/output/__init__.py index 87307eb051..68977b9e34 100644 --- a/esphome/components/opentherm/output/__init__.py +++ b/esphome/components/opentherm/output/__init__.py @@ -21,7 +21,7 @@ async def new_openthermoutput( var = cg.new_Pvariable(config[CONF_ID]) await cg.register_component(var, config) await output.register_output(var, config) - cg.add(getattr(var, "set_id")(cg.RawExpression(f'"{key}_{config[CONF_ID]}"'))) + cg.add(var.set_id(cg.RawExpression(f'"{key}_{config[CONF_ID]}"'))) input.generate_setters(var, config) return var diff --git a/esphome/components/openthread/openthread_esp.cpp b/esphome/components/openthread/openthread_esp.cpp index 27712bd86a..787f2f5de8 100644 --- a/esphome/components/openthread/openthread_esp.cpp +++ b/esphome/components/openthread/openthread_esp.cpp @@ -35,9 +35,8 @@ void OpenThreadComponent::setup() { esp_vfs_eventfd_config_t eventfd_config = { .max_fds = 3, }; + // Network interface setup handled by network component ESP_ERROR_CHECK(nvs_flash_init()); - ESP_ERROR_CHECK(esp_event_loop_create_default()); - ESP_ERROR_CHECK(esp_netif_init()); ESP_ERROR_CHECK(esp_vfs_eventfd_register(&eventfd_config)); xTaskCreate( diff --git a/esphome/components/packages/__init__.py b/esphome/components/packages/__init__.py index 06a64208b6..c1c5bd2ae3 100644 --- a/esphome/components/packages/__init__.py +++ b/esphome/components/packages/__init__.py @@ -112,7 +112,7 @@ def expand_file_to_files(config: dict): def validate_yaml_filename(value): value = cv.string(value) - if not (value.endswith(".yaml") or value.endswith(".yml")): + if not value.endswith((".yaml", ".yml")): raise cv.Invalid("Only YAML (.yaml / .yml) files are supported.") return value @@ -215,7 +215,7 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: If loading fails after cloning, attempts a revert and retry in case a prior cached checkout is stale. """ - repo_dir, revert = git.clone_or_update( + repo_root, revert = git.clone_or_update( url=config[CONF_URL], ref=config.get(CONF_REF), refresh=config[CONF_REFRESH], @@ -225,6 +225,10 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: ) files: list[dict[str, Any]] = [] + # ``repo_root`` is the directory containing ``.git`` and must be passed + # to git for symlink-stub resolution. ``repo_dir`` may be narrowed to a + # subdirectory via the user's CONF_PATH and is used for file lookups. + repo_dir = repo_root if base_path := config.get(CONF_PATH): repo_dir = repo_dir / base_path @@ -236,13 +240,37 @@ def _process_remote_package(config: dict[str, Any]) -> dict[str, Any]: def _load_package_yaml(yaml_file: Path, filename: str) -> dict: """Load a YAML file from a remote package, validating min_version.""" - try: - new_yaml = yaml_util.load_yaml(yaml_file) - except EsphomeError as e: + + def _load(path: Path) -> dict | str | None: + try: + return yaml_util.load_yaml(path) + except EsphomeError as e: + raise cv.Invalid( + f"{filename} is not a valid YAML file." + f" Please check the file contents.\n{e}" + ) from e + + new_yaml = _load(yaml_file) + if not isinstance(new_yaml, dict): + # On Windows, git defaults to core.symlinks=false unless the user + # has Developer Mode enabled or is running elevated. Files stored + # in the repo as symlinks (tree mode 120000) are then checked out + # as plain text files containing the symlink target path, so + # parsing them as YAML yields a bare scalar instead of a mapping. + # Best-effort: follow the symlink target ourselves and re-load. + target = git.resolve_symlink_stub(repo_root, yaml_file) + if target is not None: + new_yaml = _load(target) + if not isinstance(new_yaml, dict): raise cv.Invalid( - f"{filename} is not a valid YAML file." - f" Please check the file contents.\n{e}" - ) from e + f"{filename} does not contain a YAML mapping at the top level " + f"(got {type(new_yaml).__name__}). " + f"If this file is a git symlink in the source repository, it " + f"may not have been materialized correctly on your platform " + f"(this is a known issue with git on Windows without Developer " + f"Mode enabled). Try pointing your package at the real file " + f"path instead." + ) esphome_config = new_yaml.get(CONF_ESPHOME) or {} min_version = esphome_config.get(CONF_MIN_VERSION) if min_version is not None and cv.Version.parse(min_version) > cv.Version.parse( diff --git a/esphome/components/psram/__init__.py b/esphome/components/psram/__init__.py index 86c17ce9ca..d36d900997 100644 --- a/esphome/components/psram/__init__.py +++ b/esphome/components/psram/__init__.py @@ -1,5 +1,6 @@ import logging import textwrap +from typing import Any import esphome.codegen as cg from esphome.components.const import CONF_IGNORE_NOT_FOUND @@ -94,6 +95,27 @@ def is_guaranteed() -> bool: return CORE.data.get(KEY_PSRAM_GUARANTEED, False) +def request_external_task_stack() -> None: + """Allow FreeRTOS task stacks to be allocated in external RAM (PSRAM). + + Components that expose a ``task_stack_in_psram`` option should call this from their + ``to_code`` when the option is enabled. The sdkconfig option only permits external + stacks; it does not move any stack into PSRAM on its own, so it stays opt-in per task. + """ + add_idf_sdkconfig_option("CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True) + + +def validate_task_stack_in_psram(value: Any) -> bool: + """Validate a ``task_stack_in_psram`` boolean, requiring the psram component only when enabled. + + Validating the boolean first means an explicit ``false`` does not pull in the psram + requirement, so the option can still be set to false on devices without PSRAM. + """ + if value := cv.boolean(value): + return cv.requires_component(DOMAIN)(value) + return value + + def validate_psram_mode(config): esp32_config = fv.full_config.get()[PLATFORM_ESP32] if config[CONF_SPEED] == "120MHZ": diff --git a/esphome/components/resampler/speaker/__init__.py b/esphome/components/resampler/speaker/__init__.py index 3134cf7646..8a13110631 100644 --- a/esphome/components/resampler/speaker/__init__.py +++ b/esphome/components/resampler/speaker/__init__.py @@ -1,5 +1,5 @@ import esphome.codegen as cg -from esphome.components import audio, esp32, speaker +from esphome.components import audio, psram, speaker import esphome.config_validation as cv from esphome.const import ( CONF_BITS_PER_SAMPLE, @@ -63,7 +63,7 @@ CONFIG_SCHEMA = cv.All( cv.Optional( CONF_BUFFER_DURATION, default="100ms" ): cv.positive_time_period_milliseconds, - cv.Optional(CONF_TASK_STACK_IN_PSRAM, default=False): cv.boolean, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_FILTERS, default=16): cv.int_range(min=2, max=1024), cv.Optional(CONF_TAPS, default=16): _validate_taps, } @@ -88,9 +88,7 @@ async def to_code(config): if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() cg.add(var.set_target_bits_per_sample(config[CONF_BITS_PER_SAMPLE])) cg.add(var.set_target_sample_rate(config[CONF_SAMPLE_RATE])) diff --git a/esphome/components/router/__init__.py b/esphome/components/router/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esphome/components/router/speaker/__init__.py b/esphome/components/router/speaker/__init__.py new file mode 100644 index 0000000000..2b2dc56433 --- /dev/null +++ b/esphome/components/router/speaker/__init__.py @@ -0,0 +1,123 @@ +from esphome import automation, core +import esphome.codegen as cg +from esphome.components import audio, speaker +import esphome.config_validation as cv +from esphome.const import ( + CONF_BITS_PER_SAMPLE, + CONF_ID, + CONF_NUM_CHANNELS, + CONF_OUTPUT_SPEAKER, + CONF_SAMPLE_RATE, +) +from esphome.core import ID +from esphome.cpp_generator import MockObj +from esphome.types import ConfigType, TemplateArgsType + +CODEOWNERS = ["@kahrendt"] + +CONF_OUTPUT_SPEAKERS = "output_speakers" +CONF_TARGET_SPEAKER = "target_speaker" + +router_ns = cg.esphome_ns.namespace("router") +Router = router_ns.class_("Router", cg.Component, speaker.Speaker) +SwitchOutputAction = router_ns.class_("SwitchOutputAction", automation.Action) + +SpeakerPtr = speaker.Speaker.operator("ptr") + + +def _set_stream_limits(config: ConfigType) -> ConfigType: + # Lock the router's stream limits to the user-declared format. Limits are set + # at CONFIG_SCHEMA time so they're visible to other components' FINAL_VALIDATE + # (which has no guaranteed ordering vs. ours). + audio.set_stream_limits( + min_bits_per_sample=config[CONF_BITS_PER_SAMPLE], + max_bits_per_sample=config[CONF_BITS_PER_SAMPLE], + min_channels=config[CONF_NUM_CHANNELS], + max_channels=config[CONF_NUM_CHANNELS], + min_sample_rate=config[CONF_SAMPLE_RATE], + max_sample_rate=config[CONF_SAMPLE_RATE], + )(config) + return config + + +CONFIG_SCHEMA = cv.All( + cv.Schema( + { + cv.GenerateID(): cv.declare_id(Router), + cv.Required(CONF_OUTPUT_SPEAKERS): cv.All( + cv.ensure_list(cv.use_id(speaker.Speaker)), + cv.Length(min=2, max=8), + ), + # All outputs must agree on a single format so the producer can keep + # streaming through a switch without reconfiguring. These are required + # rather than inherited because downstream components (e.g. mixer) + # read them from the router's declaration during FINAL_VALIDATE, + # which can't depend on our FINAL_VALIDATE running first. + cv.Required(CONF_BITS_PER_SAMPLE): cv.int_range(8, 32), + cv.Required(CONF_NUM_CHANNELS): cv.int_range(1, 2), + cv.Required(CONF_SAMPLE_RATE): cv.int_range(8000, 96000), + } + ).extend(cv.COMPONENT_SCHEMA), + cv.only_on_esp32, + _set_stream_limits, +) + + +def _final_validate(config: ConfigType) -> ConfigType: + # Validate every configured output speaker can accept the router's format. + # Switching to an output that can't reproduce the format the producer is + # already sending would otherwise fail silently at runtime. + for spk_id in config[CONF_OUTPUT_SPEAKERS]: + proxy = {**config, CONF_OUTPUT_SPEAKER: spk_id} + audio.final_validate_audio_schema( + "router", + audio_device=CONF_OUTPUT_SPEAKER, + bits_per_sample=config[CONF_BITS_PER_SAMPLE], + channels=config[CONF_NUM_CHANNELS], + sample_rate=config[CONF_SAMPLE_RATE], + )(proxy) + return config + + +FINAL_VALIDATE_SCHEMA = _final_validate + + +async def to_code(config: ConfigType) -> None: + var = cg.new_Pvariable(config[CONF_ID]) + await cg.register_component(var, config) + + # The first configured output is the default active output on boot. + speakers = config[CONF_OUTPUT_SPEAKERS] + cg.add(var.set_output_count(len(speakers))) + for spk_id in speakers: + spk = await cg.get_variable(spk_id) + cg.add(var.add_output(spk)) + + +@automation.register_action( + "router.speaker.switch_output", + SwitchOutputAction, + cv.Schema( + { + cv.GenerateID(CONF_ID): cv.use_id(Router), + cv.Required(CONF_TARGET_SPEAKER): cv.templatable( + cv.use_id(speaker.Speaker) + ), + } + ), + synchronous=True, +) +async def switch_output_to_code( + config: ConfigType, + action_id: ID, + template_arg: cg.TemplateArguments, + args: TemplateArgsType, +) -> MockObj: + parent = await cg.get_variable(config[CONF_ID]) + var = cg.new_Pvariable(action_id, template_arg, parent) + target = config[CONF_TARGET_SPEAKER] + if not isinstance(target, core.Lambda): + target = await cg.get_variable(target) + template_ = await cg.templatable(target, args, SpeakerPtr) + cg.add(var.set_target(template_)) + return var diff --git a/esphome/components/router/speaker/router_speaker.cpp b/esphome/components/router/speaker/router_speaker.cpp new file mode 100644 index 0000000000..f4bf7420ab --- /dev/null +++ b/esphome/components/router/speaker/router_speaker.cpp @@ -0,0 +1,236 @@ +#include "router_speaker.h" + +#ifdef USE_ESP32 + +#include "esphome/core/log.h" + +#include "esp_timer.h" + +#include + +namespace esphome::router { + +static const char *const TAG = "router.speaker"; + +static inline uint32_t atomic_subtract_clamped(std::atomic &var, uint32_t amount) { + uint32_t current = var.load(std::memory_order_acquire); + uint32_t subtracted = 0; + if (current > 0) { + uint32_t new_value; + do { + subtracted = std::min(amount, current); + new_value = current - subtracted; + } while (!var.compare_exchange_weak(current, new_value, std::memory_order_release, std::memory_order_acquire)); + } + return subtracted; +} + +void Router::setup() { + // Register a callback on every configured output. Each lambda captures its own + // index and only forwards when that output is the active one. This is required + // because CallbackManager has no remove() API. + for (size_t i = 0; i < this->outputs_.size(); i++) { + this->outputs_[i]->add_audio_output_callback([this, i](uint32_t frames, int64_t timestamp_us) { + // Always suppress the draining previous output during a switch, even if it's + // also the reselected active output (switching back to the bus holder). + // loop() fires one synthetic credit for its in-flight frames instead. + if (this->pending_start_prev_idx_.load(std::memory_order_relaxed) == static_cast(i)) { + return; + } + if (this->active_output_idx_.load(std::memory_order_relaxed) != static_cast(i)) { + return; + } + atomic_subtract_clamped(this->frames_in_pipeline_, frames); + this->audio_output_callback_.call(frames, timestamp_us); + }); + } +} + +void Router::loop() { + speaker::Speaker *active = this->get_active_output(); + + // Mid-switch: the new output's start() is deferred until the previous output + // fully releases shared hardware (e.g. a single i2s_audio bus driving two + // speakers). Starting earlier produces "Parent bus is busy" retries. The + // synthetic-credit callback is also deferred until prev is fully stopped, so + // that once its task has drained no natural callbacks can race ours. + const int8_t pending_prev_idx = this->pending_start_prev_idx_.load(std::memory_order_relaxed); + if (pending_prev_idx >= 0) { + speaker::Speaker *prev = this->outputs_[pending_prev_idx]; + if (prev->is_stopped()) { + this->pending_start_prev_idx_.store(-1, std::memory_order_relaxed); + + // Credit any frames left in prev's ring buffer / DMA so producer frame + // accounting (SpeakerSourceMediaPlayer pending_frames, sendspin/AEC + // clocks) clears cleanly. The leftover audio is intentionally dropped and + // the producer is told it played "now", giving a clean discontinuity that + // keeps frame accounting consistent across the switch. + const uint32_t in_flight = this->frames_in_pipeline_.exchange(0, std::memory_order_acq_rel); + if (in_flight > 0) { + this->audio_output_callback_.call(in_flight, esp_timer_get_time()); + } + + this->apply_cached_state_to_active_(); + this->state_ = speaker::STATE_STARTING; + active->start(); + } + return; + } + + // Mirror the active output's running/stopped state into our own state_ so that + // is_running() / is_stopped() stay accurate from the producer's perspective. + // Also catch the active output self-stopping (e.g. i2s_audio silence timeout): + // without this, our state_ would stay RUNNING forever and the next play() would + // skip start(). The output retains its own volume/mute across a restart (and we + // forward those live regardless), but stream info arrives via the non-virtual + // set_audio_stream_info() and never reaches the output on its own; if the format + // changed while stopped, only start()'s apply_cached_state_to_active_() pushes it + // down before the output's play()-side auto-start locks in the stale format. + if (active->is_stopped()) { + this->state_ = speaker::STATE_STOPPED; + } else if (this->state_ == speaker::STATE_STARTING && active->is_running()) { + this->state_ = speaker::STATE_RUNNING; + } +} + +void Router::dump_config() { + ESP_LOGCONFIG(TAG, + "Router Speaker:\n" + " Outputs: %u", + static_cast(this->outputs_.size())); +} + +size_t Router::play(const uint8_t *data, size_t length, TickType_t ticks_to_wait) { + speaker::Speaker *active = this->get_active_output(); + + // Drop frames during a mid-switch until the old output releases shared hardware; + // forwarding now would trigger the new output's play()-side auto-start while + // the bus is still busy. + if (this->pending_start_prev_idx_.load(std::memory_order_relaxed) >= 0) { + vTaskDelay(ticks_to_wait); + return 0; + } + + // Producers (e.g. mixer) set stream info on us and then drive play() from a + // task without ever calling our start(). i2s_audio's play() auto-starts the + // underlying driver, so we must push our cached stream info to the active + // output before that auto-start, or it locks to its default (16k mono). + if (this->state_ == speaker::STATE_STOPPED) { + this->start(); + vTaskDelay(ticks_to_wait); + ticks_to_wait = 0; + } + + size_t written = active->play(data, length, ticks_to_wait); + if (written > 0) { + const uint32_t frames = this->audio_stream_info_.bytes_to_frames(written); + this->frames_in_pipeline_.fetch_add(frames, std::memory_order_release); + } + return written; +} + +void Router::start() { + this->frames_in_pipeline_.store(0, std::memory_order_release); + this->apply_cached_state_to_active_(); + this->state_ = speaker::STATE_STARTING; + this->get_active_output()->start(); +} + +void Router::stop() { + // Cancel any pending mid-switch start; the producer wants us stopped. + this->pending_start_prev_idx_.store(-1, std::memory_order_relaxed); + this->state_ = speaker::STATE_STOPPING; + this->get_active_output()->stop(); +} + +void Router::finish() { + this->pending_start_prev_idx_.store(-1, std::memory_order_relaxed); + this->state_ = speaker::STATE_STOPPING; + this->get_active_output()->finish(); +} + +bool Router::has_buffered_data() const { return this->get_active_output()->has_buffered_data(); } + +void Router::set_pause_state(bool pause_state) { + this->cached_pause_ = pause_state; + this->get_active_output()->set_pause_state(pause_state); +} + +void Router::set_volume(float volume) { + this->volume_ = volume; + this->get_active_output()->set_volume(volume); +} + +void Router::set_mute_state(bool mute_state) { + this->mute_state_ = mute_state; + this->get_active_output()->set_mute_state(mute_state); +} + +bool Router::switch_to_output(speaker::Speaker *target) { + if (target == nullptr) { + return false; + } + + int8_t new_idx = -1; + for (size_t i = 0; i < this->outputs_.size(); i++) { + if (this->outputs_[i] == target) { + new_idx = static_cast(i); + break; + } + } + if (new_idx < 0) { + ESP_LOGW(TAG, "Switch target is not a configured output"); + return false; + } + if (new_idx == this->active_output_idx_.load(std::memory_order_relaxed)) { + return true; + } + + // A switch is already in flight: pending_start_prev_idx_ is still releasing the + // shared bus and the current active output's start() is still deferred (it never + // started). Just redirect which output we start once the bus frees. Leave the bus + // holder (pending_start_prev_idx_), the in-flight frame counter (loop() still owes one + // synthetic credit for the bus holder's in-flight frames), and state_ alone, and + // don't stop the current active output, which never started. + if (this->pending_start_prev_idx_.load(std::memory_order_relaxed) >= 0) { + this->active_output_idx_.store(new_idx, std::memory_order_relaxed); + return true; + } + + const bool was_active = (this->state_ == speaker::STATE_STARTING || this->state_ == speaker::STATE_RUNNING); + const int8_t old_idx = this->active_output_idx_.load(std::memory_order_relaxed); + + if (was_active) { + this->outputs_[old_idx]->stop(); + } + + this->active_output_idx_.store(new_idx, std::memory_order_relaxed); + + if (was_active) { + // Defer start and the synthetic-credit callback until the old output's + // task is fully stopped; loop() handles both. Firing the synthetic credit + // here would race the old task's still-in-flight natural callbacks, + // dispatching audio_output_callback_ concurrently from two threads, which + // some consumers (e.g. sendspin's progress sync) aren't reentrant-safe for. + // STATE_STOPPING keeps producers from observing a transient stopped state + // and lets our play() short-circuit so the new output's play() doesn't + // auto-start it while the shared bus is still being released. + this->state_ = speaker::STATE_STOPPING; + this->pending_start_prev_idx_.store(old_idx, std::memory_order_relaxed); + } else { + this->frames_in_pipeline_.store(0, std::memory_order_release); + } + return true; +} + +void Router::apply_cached_state_to_active_() { + speaker::Speaker *active = this->get_active_output(); + active->set_audio_stream_info(this->audio_stream_info_); + active->set_volume(this->volume_); + active->set_mute_state(this->mute_state_); + active->set_pause_state(this->cached_pause_); +} + +} // namespace esphome::router + +#endif // USE_ESP32 diff --git a/esphome/components/router/speaker/router_speaker.h b/esphome/components/router/speaker/router_speaker.h new file mode 100644 index 0000000000..13b58a1c72 --- /dev/null +++ b/esphome/components/router/speaker/router_speaker.h @@ -0,0 +1,92 @@ +#pragma once + +#ifdef USE_ESP32 + +#include "esphome/components/speaker/speaker.h" +#include "esphome/core/automation.h" +#include "esphome/core/component.h" +#include "esphome/core/helpers.h" + +#include + +#include + +namespace esphome::router { + +class Router : public Component, public speaker::Speaker { + public: + float get_setup_priority() const override { return setup_priority::DATA; } + + void setup() override; + void loop() override; + void dump_config() override; + + size_t play(const uint8_t *data, size_t length) override { return this->play(data, length, 0); } + size_t play(const uint8_t *data, size_t length, TickType_t ticks_to_wait) override; + + void start() override; + void stop() override; + void finish() override; + + bool has_buffered_data() const override; + + void set_pause_state(bool pause_state) override; + bool get_pause_state() const override { return this->cached_pause_; } + + void set_volume(float volume) override; + float get_volume() override { return this->volume_; } + + void set_mute_state(bool mute_state) override; + bool get_mute_state() override { return this->mute_state_; } + + // Allocate the output list to its final size. Must be called before add_output(). + void set_output_count(size_t count) { this->outputs_.init(count); } + void add_output(speaker::Speaker *spk) { this->outputs_.push_back(spk); } + + /// Switch the active output to the given speaker. Must be one of the configured outputs. + /// Returns false if `target` is not in the output list. + bool switch_to_output(speaker::Speaker *target); + + // Always valid: active_output_idx_ stays within [0, outputs_.size()) and at least + // two outputs are required (validated in Python), so this never returns null. + speaker::Speaker *get_active_output() const { + return this->outputs_[this->active_output_idx_.load(std::memory_order_relaxed)]; + } + + protected: + // Frames written to the active output but not yet played: incremented in play() and decremented + // (clamped at zero) by the active output's audio_output_callback. Mirrors mixer_speaker's + // frames_in_pipeline_. + std::atomic frames_in_pipeline_{0}; + + bool cached_pause_{false}; + + void apply_cached_state_to_active_(); + + // Index of the previously-active output we're waiting on to fully stop before + // starting the new one. -1 means no pending start. Set by switch_to_output() + // when switching mid-playback; cleared by loop() once the old output reports + // is_stopped(). Required because shared-bus drivers (e.g. two i2s_audio + // speakers on one i2s_bus) reject start() until the previous user releases. + std::atomic pending_start_prev_idx_{-1}; + + private: + FixedVector outputs_; + // Index into outputs_, always within [0, outputs_.size()). Defaults to the first + // configured output; updated by switch_to_output(). + std::atomic active_output_idx_{0}; +}; + +template class SwitchOutputAction : public Action { + public: + explicit SwitchOutputAction(Router *parent) : parent_(parent) {} + TEMPLATABLE_VALUE(speaker::Speaker *, target) + void play(const Ts &...x) override { this->parent_->switch_to_output(this->target_.value(x...)); } + + protected: + Router *parent_; +}; + +} // namespace esphome::router + +#endif // USE_ESP32 diff --git a/esphome/components/rp2040/__init__.py b/esphome/components/rp2040/__init__.py index 862d532645..830c961476 100644 --- a/esphome/components/rp2040/__init__.py +++ b/esphome/components/rp2040/__init__.py @@ -1,8 +1,10 @@ +from collections.abc import Callable import logging from pathlib import Path import re from string import ascii_letters, digits import subprocess +from typing import Any import esphome.codegen as cg import esphome.config_validation as cv @@ -12,6 +14,7 @@ from esphome.const import ( CONF_FRAMEWORK, CONF_PLATFORM_VERSION, CONF_SOURCE, + CONF_VARIANT, CONF_VERSION, CONF_WATCHDOG_TIMEOUT, KEY_CORE, @@ -21,12 +24,30 @@ from esphome.const import ( PLATFORM_RP2040, ThreadModel, ) -from esphome.core import CORE, CoroPriority, EsphomeError, coroutine_with_priority +from esphome.core import ( + CORE, + CoroPriority, + EsphomeCore, + EsphomeError, + coroutine_with_priority, +) from esphome.core.config import BOARD_MAX_LENGTH from esphome.helpers import copy_file_if_changed, read_file, write_file_if_changed +from esphome.types import ConfigType from . import boards -from .const import KEY_BOARD, KEY_LWIP_OPTS, KEY_PIO_FILES, KEY_RP2040, rp2040_ns +from .const import ( + KEY_BOARD, + KEY_LWIP_OPTS, + KEY_PIO_FILES, + KEY_RP2040, + KEY_VARIANT, + MCU_TO_VARIANT, + STANDARD_BOARDS, + VARIANT_FRIENDLY, + VARIANTS, + rp2040_ns, +) # force import gpio to register pin schema from .gpio import rp2040_pin_to_code # noqa @@ -68,7 +89,7 @@ def board_id_has_wifi(board_id: str) -> bool: return board_info.get("wifi", False) -def set_core_data(config): +def set_core_data(config: ConfigType) -> ConfigType: CORE.data[KEY_RP2040] = {} CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] = PLATFORM_RP2040 CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] = "arduino" @@ -76,12 +97,46 @@ def set_core_data(config): config[CONF_FRAMEWORK][CONF_VERSION] ) CORE.data[KEY_RP2040][KEY_BOARD] = config[CONF_BOARD] + CORE.data[KEY_RP2040][KEY_VARIANT] = config[CONF_VARIANT] CORE.data[KEY_RP2040][KEY_PIO_FILES] = {} return config +def get_rp2040_variant(core_obj: EsphomeCore | None = None) -> str: + return (core_obj or CORE).data[KEY_RP2040][KEY_VARIANT] + + +def only_on_variant( + *, + supported: str | list[str] | None = None, + unsupported: str | list[str] | None = None, + msg_prefix: str = "This feature", +) -> Callable[[Any], Any]: + """Config validator for features only available on some RP2040 variants.""" + if supported is not None and not isinstance(supported, list): + supported = [supported] + if unsupported is not None and not isinstance(unsupported, list): + unsupported = [unsupported] + + def validator_(obj: Any) -> Any: + if not CORE.is_rp2040: + raise cv.Invalid(f"{msg_prefix} is only available on RP2040") + variant = get_rp2040_variant() + if supported is not None and variant not in supported: + raise cv.Invalid( + f"{msg_prefix} is only available on {', '.join(supported)}" + ) + if unsupported is not None and variant in unsupported: + raise cv.Invalid( + f"{msg_prefix} is not available on {', '.join(unsupported)}" + ) + return obj + + return validator_ + + def get_download_types(storage_json): """Binary-download entries for a built RP2040 firmware. @@ -192,12 +247,52 @@ ARDUINO_FRAMEWORK_SCHEMA = cv.All( _arduino_check_versions, ) + +def _detect_variant(value: ConfigType) -> ConfigType: + value = value.copy() + board: str | None = value.get(CONF_BOARD) + variant: str | None = value.get(CONF_VARIANT) + + if board is None: + # `cv.has_at_least_one_key` guarantees variant is set here. + board = STANDARD_BOARDS[variant] + value[CONF_BOARD] = board + + board_info = boards.BOARDS.get(board) + if board_info is None: + if variant is None: + raise cv.Invalid( + "This board is unknown; please specify the chip variant using " + f"the '{CONF_VARIANT}' option.", + path=[CONF_BOARD], + ) + _LOGGER.warning( + "This board is unknown; the specified variant '%s' will be used " + "but this may not work as expected.", + variant, + ) + else: + board_variant = MCU_TO_VARIANT[board_info["mcu"]] + if variant is None: + variant = board_variant + elif variant != board_variant: + raise cv.Invalid( + f"Option '{CONF_VARIANT}' ({variant}) does not match the " + f"selected board '{board}' ({board_variant}).", + path=[CONF_VARIANT], + ) + + value[CONF_VARIANT] = variant + return value + + CONFIG_SCHEMA = cv.All( cv.Schema( { - cv.Required(CONF_BOARD): cv.All( + cv.Optional(CONF_BOARD): cv.All( cv.string_strict, cv.ByteLength(max=BOARD_MAX_LENGTH) ), + cv.Optional(CONF_VARIANT): cv.one_of(*VARIANTS, upper=True), cv.Optional(CONF_FRAMEWORK, default={}): ARDUINO_FRAMEWORK_SCHEMA, cv.Optional(CONF_WATCHDOG_TIMEOUT, default="8388ms"): cv.All( cv.positive_time_period_milliseconds, @@ -206,6 +301,8 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_ENABLE_FULL_PRINTF, default=False): cv.boolean, } ), + cv.has_at_least_one_key(CONF_BOARD, CONF_VARIANT), + _detect_variant, set_core_data, ) @@ -223,7 +320,9 @@ async def to_code(config): cg.add_define("USE_NATIVE_64BIT_TIME") cg.set_cpp_standard("gnu++20") cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) - cg.add_define("ESPHOME_VARIANT", "RP2040") + variant = config[CONF_VARIANT] + cg.add_build_flag(f"-DUSE_RP2040_VARIANT_{variant}") + cg.add_define("ESPHOME_VARIANT", VARIANT_FRIENDLY[variant]) cg.add_define(ThreadModel.SINGLE) cg.add_platformio_option("extra_scripts", ["post:post_build.py"]) diff --git a/esphome/components/rp2040/const.py b/esphome/components/rp2040/const.py index e381d0482d..959753d95b 100644 --- a/esphome/components/rp2040/const.py +++ b/esphome/components/rp2040/const.py @@ -4,5 +4,31 @@ KEY_BOARD = "board" KEY_LWIP_OPTS = "lwip_opts" KEY_RP2040 = "rp2040" KEY_PIO_FILES = "pio_files" +KEY_VARIANT = "variant" + +VARIANT_RP2040 = "RP2040" +VARIANT_RP2350 = "RP2350" +VARIANTS = [ + VARIANT_RP2040, + VARIANT_RP2350, +] + +VARIANT_FRIENDLY = { + VARIANT_RP2040: "RP2040", + VARIANT_RP2350: "RP2350", +} + +# Map BOARDS[board]["mcu"] (lowercase) to canonical variant constant +MCU_TO_VARIANT = { + "rp2040": VARIANT_RP2040, + "rp2350": VARIANT_RP2350, +} + +# Default board chosen when only `variant` is specified — the Raspberry Pi +# Foundation reference boards (Pico W / Pico 2 W). +STANDARD_BOARDS = { + VARIANT_RP2040: "rpipicow", + VARIANT_RP2350: "rpipico2w", +} rp2040_ns = cg.esphome_ns.namespace("rp2040") diff --git a/esphome/components/rp2040/generate_boards.py b/esphome/components/rp2040/generate_boards.py index 8af261396c..b1a0b17ca3 100644 --- a/esphome/components/rp2040/generate_boards.py +++ b/esphome/components/rp2040/generate_boards.py @@ -67,7 +67,7 @@ def load_boards(arduino_pico_path: Path) -> tuple[dict, dict]: for json_file in sorted(json_dir.glob("*.json")): board_name = json_file.stem - with open(json_file, encoding="utf-8") as f: + with json_file.open(encoding="utf-8") as f: data = json.load(f) build = data.get("build", {}) @@ -136,7 +136,7 @@ def _get_variant(json_file: Path) -> str | None: """Get variant name from a board JSON file.""" if not json_file.exists(): return None - with open(json_file, encoding="utf-8") as f: + with json_file.open(encoding="utf-8") as f: data = json.load(f) return data.get("build", {}).get("variant") diff --git a/esphome/components/sen5x/sensor.py b/esphome/components/sen5x/sensor.py index ce35cf5bf1..480654ee1b 100644 --- a/esphome/components/sen5x/sensor.py +++ b/esphome/components/sen5x/sensor.py @@ -25,7 +25,6 @@ from esphome.const import ( CONF_TEMPERATURE_COMPENSATION, CONF_TIME_CONSTANT, CONF_VOC, - DEVICE_CLASS_AQI, DEVICE_CLASS_HUMIDITY, DEVICE_CLASS_PM1, DEVICE_CLASS_PM10, @@ -77,7 +76,6 @@ def _gas_sensor( return sensor.sensor_schema( icon=ICON_RADIATOR, accuracy_decimals=0, - device_class=DEVICE_CLASS_AQI, state_class=STATE_CLASS_MEASUREMENT, ).extend( { diff --git a/esphome/components/sen6x/sensor.py b/esphome/components/sen6x/sensor.py index 071478e719..19c0cb500e 100644 --- a/esphome/components/sen6x/sensor.py +++ b/esphome/components/sen6x/sensor.py @@ -14,7 +14,6 @@ from esphome.const import ( CONF_TEMPERATURE, CONF_TYPE, CONF_VOC, - DEVICE_CLASS_AQI, DEVICE_CLASS_CARBON_DIOXIDE, DEVICE_CLASS_HUMIDITY, DEVICE_CLASS_PM1, @@ -93,13 +92,11 @@ CONFIG_SCHEMA = ( cv.Optional(CONF_VOC): sensor.sensor_schema( icon=ICON_RADIATOR, accuracy_decimals=0, - device_class=DEVICE_CLASS_AQI, state_class=STATE_CLASS_MEASUREMENT, ), cv.Optional(CONF_NOX): sensor.sensor_schema( icon=ICON_RADIATOR, accuracy_decimals=0, - device_class=DEVICE_CLASS_AQI, state_class=STATE_CLASS_MEASUREMENT, ), cv.Optional(CONF_CO2): sensor.sensor_schema( diff --git a/esphome/components/sendspin/__init__.py b/esphome/components/sendspin/__init__.py index 35280020ba..e8c643f9b9 100644 --- a/esphome/components/sendspin/__init__.py +++ b/esphome/components/sendspin/__init__.py @@ -121,13 +121,6 @@ def register_player_config(config: ConfigType) -> None: data.player_config = config -def _validate_task_stack_in_psram(value): - value = cv.boolean(value) - if value: - return cv.requires_component(psram.DOMAIN)(value) - return value - - def _request_high_performance_networking(config: ConfigType) -> ConfigType: """Request high performance networking for Sendspin streaming. @@ -152,7 +145,7 @@ CONFIG_SCHEMA = cv.All( cv.Schema( { cv.GenerateID(): cv.declare_id(SendspinHub), - cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, } ), cv.only_on_esp32, @@ -201,12 +194,10 @@ async def to_code(config: ConfigType) -> None: if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() # sendspin-cpp library - esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.5.0") + esp32.add_idf_component(name="sendspin/sendspin-cpp", ref="0.6.1") cg.add_define("USE_SENDSPIN", True) # for MDNS @@ -261,9 +252,7 @@ async def to_code(config: ConfigType) -> None: psram_stack = player_cfg.get(CONF_TASK_STACK_IN_PSRAM, False) if psram_stack: - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() # Library defaults: priority 18 (one above httpd_priority 17 so the decoder is not # starved by the HTTP server during the initial encoded-audio burst at stream start), diff --git a/esphome/components/sendspin/media_source/__init__.py b/esphome/components/sendspin/media_source/__init__.py index f689ab01cb..6af244d41f 100644 --- a/esphome/components/sendspin/media_source/__init__.py +++ b/esphome/components/sendspin/media_source/__init__.py @@ -1,6 +1,6 @@ from esphome import automation import esphome.codegen as cg -from esphome.components import media_source +from esphome.components import media_source, psram import esphome.config_validation as cv from esphome.const import ( CONF_BUFFER_SIZE, @@ -19,7 +19,6 @@ from .. import ( CONF_SENDSPIN_ID, MEMORY_LOCATIONS, SendspinHub, - _validate_task_stack_in_psram, register_player_config, request_controller_support, sendspin_ns, @@ -71,7 +70,7 @@ CONFIG_SCHEMA = cv.All( ).extend( { cv.GenerateID(CONF_SENDSPIN_ID): cv.use_id(SendspinHub), - cv.Optional(CONF_TASK_STACK_IN_PSRAM): _validate_task_stack_in_psram, + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_BUFFER_SIZE, default=1000000): cv.int_range(min=25000), cv.Optional(CONF_INITIAL_STATIC_DELAY, default="0ms"): cv.All( cv.positive_time_period_milliseconds, diff --git a/esphome/components/sensor/__init__.py b/esphome/components/sensor/__init__.py index 6bbab76363..5a2ebf03c0 100644 --- a/esphome/components/sensor/__init__.py +++ b/esphome/components/sensor/__init__.py @@ -1192,7 +1192,7 @@ def _std(x): def _correlation_coeff(x, y): m_x, m_y = _mean(x), _mean(y) - s_xy = sum((x_ - m_x) * (y_ - m_y) for x_, y_ in zip(x, y)) + s_xy = sum((x_ - m_x) * (y_ - m_y) for x_, y_ in zip(x, y, strict=True)) s_sq_x = sum((x_ - m_x) ** 2 for x_ in x) s_sq_y = sum((y_ - m_y) ** 2 for y_ in y) return s_xy / math.sqrt(s_sq_x * s_sq_y) @@ -1228,7 +1228,7 @@ def _mat_copy(m): def _mat_transpose(m): - return _mat_copy(zip(*m)) + return _mat_copy(zip(*m, strict=True)) def _mat_identity(n): @@ -1237,7 +1237,10 @@ def _mat_identity(n): def _mat_dot(a, b): b_t = _mat_transpose(b) - return [[sum(x * y for x, y in zip(row_a, col_b)) for col_b in b_t] for row_a in a] + return [ + [sum(x * y for x, y in zip(row_a, col_b, strict=True)) for col_b in b_t] + for row_a in a + ] def _mat_inverse(m): diff --git a/esphome/components/sgp4x/sensor.py b/esphome/components/sgp4x/sensor.py index 1e58a0f26a..d407f20a4e 100644 --- a/esphome/components/sgp4x/sensor.py +++ b/esphome/components/sgp4x/sensor.py @@ -15,7 +15,6 @@ from esphome.const import ( CONF_STORE_BASELINE, CONF_TEMPERATURE_SOURCE, CONF_VOC, - DEVICE_CLASS_AQI, ICON_RADIATOR, STATE_CLASS_MEASUREMENT, ) @@ -72,13 +71,11 @@ CONFIG_SCHEMA = cv.All( cv.Optional(CONF_VOC): sensor.sensor_schema( icon=ICON_RADIATOR, accuracy_decimals=0, - device_class=DEVICE_CLASS_AQI, state_class=STATE_CLASS_MEASUREMENT, ).extend(VOC_SENSOR), cv.Optional(CONF_NOX): sensor.sensor_schema( icon=ICON_RADIATOR, accuracy_decimals=0, - device_class=DEVICE_CLASS_AQI, state_class=STATE_CLASS_MEASUREMENT, ).extend(NOX_SENSOR), cv.Optional(CONF_STORE_BASELINE, default=True): cv.boolean, diff --git a/esphome/components/speaker/media_player/__init__.py b/esphome/components/speaker/media_player/__init__.py index 094043c292..90eb19d73d 100644 --- a/esphome/components/speaker/media_player/__init__.py +++ b/esphome/components/speaker/media_player/__init__.py @@ -7,7 +7,6 @@ import esphome.codegen as cg from esphome.components import ( audio, audio_file, - esp32, media_player, network, ota, @@ -155,9 +154,7 @@ CONFIG_SCHEMA = cv.All( # Remove before 2026.10.0 cv.Optional(CONF_CODEC_SUPPORT_ENABLED): cv.Any(cv.boolean, cv.string), cv.Optional(CONF_FILES): audio_file.audio_files_schema(), - cv.Optional(CONF_TASK_STACK_IN_PSRAM): cv.All( - cv.boolean, cv.requires_component(psram.DOMAIN) - ), + cv.Optional(CONF_TASK_STACK_IN_PSRAM): psram.validate_task_stack_in_psram, cv.Optional(CONF_VOLUME_INCREMENT, default=0.05): cv.percentage, cv.Optional(CONF_VOLUME_INITIAL, default=0.5): cv.percentage, cv.Optional(CONF_VOLUME_MAX, default=1.0): cv.percentage, @@ -198,9 +195,7 @@ async def to_code(config): if config.get(CONF_TASK_STACK_IN_PSRAM): cg.add(var.set_task_stack_in_psram(True)) - esp32.add_idf_sdkconfig_option( - "CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY", True - ) + psram.request_external_task_stack() cg.add(var.set_volume_increment(config[CONF_VOLUME_INCREMENT])) cg.add(var.set_volume_initial(config[CONF_VOLUME_INITIAL])) diff --git a/esphome/components/sx126x/sx126x.cpp b/esphome/components/sx126x/sx126x.cpp index 6e6857fadb..83afeac50a 100644 --- a/esphome/components/sx126x/sx126x.cpp +++ b/esphome/components/sx126x/sx126x.cpp @@ -30,8 +30,8 @@ static constexpr uint8_t OCP_140MA = 0x38; // 140 mA max current static constexpr float LOW_DATA_RATE_OPTIMIZE_THRESHOLD = 16.38f; // 16.38 ms uint8_t SX126x::read_fifo_(uint8_t offset, std::vector &packet) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->transfer_byte(RADIO_READ_BUFFER); this->transfer_byte(offset); uint8_t status = this->transfer_byte(0x00); @@ -43,8 +43,8 @@ uint8_t SX126x::read_fifo_(uint8_t offset, std::vector &packet) { } void SX126x::write_fifo_(uint8_t offset, const std::vector &packet) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->transfer_byte(RADIO_WRITE_BUFFER); this->transfer_byte(offset); for (const uint8_t &byte : packet) { @@ -55,8 +55,8 @@ void SX126x::write_fifo_(uint8_t offset, const std::vector &packet) { } uint8_t SX126x::read_opcode_(uint8_t opcode, uint8_t *data, uint8_t size) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->transfer_byte(opcode); uint8_t status = this->transfer_byte(0x00); for (int32_t i = 0; i < size; i++) { @@ -67,8 +67,8 @@ uint8_t SX126x::read_opcode_(uint8_t opcode, uint8_t *data, uint8_t size) { } void SX126x::write_opcode_(uint8_t opcode, uint8_t *data, uint8_t size) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->transfer_byte(opcode); for (int32_t i = 0; i < size; i++) { this->transfer_byte(data[i]); @@ -78,8 +78,8 @@ void SX126x::write_opcode_(uint8_t opcode, uint8_t *data, uint8_t size) { } void SX126x::read_register_(uint16_t reg, uint8_t *data, uint8_t size) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->write_byte(RADIO_READ_REGISTER); this->write_byte((reg >> 8) & 0xFF); this->write_byte((reg >> 0) & 0xFF); @@ -91,8 +91,8 @@ void SX126x::read_register_(uint16_t reg, uint8_t *data, uint8_t size) { } void SX126x::write_register_(uint16_t reg, uint8_t *data, uint8_t size) { - this->wait_busy_(); this->enable(); + this->wait_busy_(); this->write_byte(RADIO_WRITE_REGISTER); this->write_byte((reg >> 8) & 0xFF); this->write_byte((reg >> 0) & 0xFF); diff --git a/esphome/components/time/__init__.py b/esphome/components/time/__init__.py index 29bb01b499..b3bf2d44d7 100644 --- a/esphome/components/time/__init__.py +++ b/esphome/components/time/__init__.py @@ -30,13 +30,21 @@ from esphome.const import ( CONF_SECONDS, CONF_TIMEZONE, CONF_TRIGGER_ID, + PLATFORM_BK72XX, + PLATFORM_ESP32, + PLATFORM_ESP8266, + PLATFORM_HOST, + PLATFORM_LN882X, + PLATFORM_RP2040, + PLATFORM_RTL87XX, ) -from esphome.core import CORE, CoroPriority, coroutine_with_priority +from esphome.core import CORE, CoroPriority, EsphomeError, coroutine_with_priority _LOGGER = logging.getLogger(__name__) CODEOWNERS = ["@esphome/core"] IS_PLATFORM_COMPONENT = True +DOMAIN = "time" time_ns = cg.esphome_ns.namespace("time") RealTimeClock = time_ns.class_("RealTimeClock", cg.PollingComponent) @@ -88,24 +96,38 @@ def _extract_tz_string(tzfile: bytes) -> str: return tzfile.split(b"\n")[-2].decode() except (IndexError, UnicodeDecodeError): _LOGGER.error("Could not determine TZ string. Please report this issue.") - _LOGGER.error("tzfile contents: %s", tzfile, exc_info=True) + _LOGGER.exception("tzfile contents: %s", tzfile) raise -def detect_tz() -> str: +def detect_tz() -> str | None: + if CORE.target_platform not in { + PLATFORM_ESP8266, + PLATFORM_ESP32, + PLATFORM_RP2040, + PLATFORM_BK72XX, + PLATFORM_RTL87XX, + PLATFORM_LN882X, + PLATFORM_HOST, + }: + return None + # Avoids duplicate logger messages when multiple time components are configured + if cached := CORE.data.setdefault(DOMAIN, {}).get(CONF_TIMEZONE): + return cached iana_key = tzlocal.get_localzone_name() if iana_key is None: - raise cv.Invalid( + raise EsphomeError( "Could not automatically determine timezone, please set timezone manually." ) - _LOGGER.info("Detected timezone '%s'", iana_key) tzfile = _load_tzdata(iana_key) if tzfile is None: - raise cv.Invalid( + raise EsphomeError( "Could not automatically determine timezone, please set timezone manually." ) ret = _extract_tz_string(tzfile) + _LOGGER.info("Detected timezone '%s'", iana_key) _LOGGER.debug(" -> TZ string %s", ret) + CORE.data.setdefault(DOMAIN, {})[CONF_TIMEZONE] = ret return ret @@ -182,7 +204,7 @@ def cron_expression_validator(name, min_value, max_value, special_mapping=None): raise cv.Invalid( f"{name} {v} is out of range (min={min_value} max={max_value})." ) - return list(sorted(value)) + return sorted(value) value = cv.string(value) values = set() for part in value.split(","): @@ -312,16 +334,7 @@ def validate_tz(value: str) -> str: TIME_SCHEMA = cv.Schema( { - cv.SplitDefault( - CONF_TIMEZONE, - esp8266=detect_tz, - esp32=detect_tz, - rp2040=detect_tz, - bk72xx=detect_tz, - rtl87xx=detect_tz, - ln882x=detect_tz, - host=detect_tz, - ): cv.All( + cv.Optional(CONF_TIMEZONE): cv.All( cv.only_with_framework(["arduino", "esp-idf", "host"]), validate_tz, ), @@ -384,7 +397,11 @@ def _emit_parsed_timezone_fields(parsed): async def setup_time_core_(time_var, config): - if timezone := config.get(CONF_TIMEZONE): + timezone = config.get(CONF_TIMEZONE) + # an empty timezone is treated as disabling timezones completely as before + if timezone is None: + timezone = detect_tz() + if timezone: cg.add_define("USE_TIME_TIMEZONE") if CORE.is_host: @@ -392,17 +409,20 @@ async def setup_time_core_(time_var, config): cg.add(time_var.set_timezone(timezone)) else: # Embedded: pre-parse at codegen time, emit struct directly - parsed = parse_posix_tz_python(timezone) - _emit_parsed_timezone_fields(parsed) + try: + parsed = parse_posix_tz_python(timezone) + _emit_parsed_timezone_fields(parsed) + except ValueError as e: + raise EsphomeError(f"Invalid timezone: {timezone}") from e for conf in config.get(CONF_ON_TIME, []): trigger = cg.new_Pvariable(conf[CONF_TRIGGER_ID], time_var) - seconds = conf.get(CONF_SECONDS, list(range(0, 61))) + seconds = conf.get(CONF_SECONDS, list(range(61))) cg.add(trigger.add_seconds(seconds)) - minutes = conf.get(CONF_MINUTES, list(range(0, 60))) + minutes = conf.get(CONF_MINUTES, list(range(60))) cg.add(trigger.add_minutes(minutes)) - hours = conf.get(CONF_HOURS, list(range(0, 24))) + hours = conf.get(CONF_HOURS, list(range(24))) cg.add(trigger.add_hours(hours)) days_of_month = conf.get(CONF_DAYS_OF_MONTH, list(range(1, 32))) cg.add(trigger.add_days_of_month(days_of_month)) diff --git a/esphome/components/tuya/tuya.cpp b/esphome/components/tuya/tuya.cpp index fd14844908..3058d82cc4 100644 --- a/esphome/components/tuya/tuya.cpp +++ b/esphome/components/tuya/tuya.cpp @@ -206,15 +206,17 @@ void Tuya::handle_command_(uint8_t command, uint8_t version, const uint8_t *buff if (this->status_pin_reported_ != -1) { this->init_state_ = TuyaInitState::INIT_DATAPOINT; this->send_empty_command_(TuyaCommandType::DATAPOINT_QUERY); - bool is_pin_equals = - this->status_pin_ != nullptr && this->status_pin_->get_pin() == this->status_pin_reported_; - // Configure status pin toggling (if reported and configured) or WIFI_STATE periodic send - if (!is_pin_equals) { - ESP_LOGW(TAG, "Supplied status_pin does not equals the reported pin %i. Using supplied pin anyway.", + if (this->status_pin_ != nullptr) { + if (this->status_pin_->get_pin() != this->status_pin_reported_) { + ESP_LOGW(TAG, "Supplied status_pin does not equal the reported pin %i. Using supplied pin anyway.", + this->status_pin_reported_); + } + ESP_LOGV(TAG, "Configured status pin %i", this->status_pin_->get_pin()); + this->set_interval("wifi", 1000, [this] { this->set_status_pin_(); }); + } else { + ESP_LOGW(TAG, "MCU reported status_pin %i but no status_pin was configured; running in limited mode.", this->status_pin_reported_); } - ESP_LOGV(TAG, "Configured status pin %i", this->status_pin_->get_pin()); - this->set_interval("wifi", 1000, [this] { this->set_status_pin_(); }); } else { this->init_state_ = TuyaInitState::INIT_WIFI; ESP_LOGV(TAG, "Configured WIFI_STATE periodic send"); diff --git a/esphome/components/uart/__init__.py b/esphome/components/uart/__init__.py index 7075228743..4ea32e26a3 100644 --- a/esphome/components/uart/__init__.py +++ b/esphome/components/uart/__init__.py @@ -513,10 +513,11 @@ async def uart_write_to_code(config, action_id, template_arg, args): @coroutine_with_priority(CoroPriority.FINAL) async def final_step(): """Final code generation step to configure optional UART features.""" - if CORE.is_esp32 and CORE.has_networking: - # Wake-on-RX is essentially free on ESP32 (just an ISR function pointer - # registration) — enable by default to reduce RX buffer overflow risk - # by waking the main loop immediately when data arrives. + if (CORE.is_esp32 or CORE.is_esp8266) and CORE.has_networking: + # Wake-on-RX is essentially free (just an ISR function pointer + # registration on ESP32, an inline flag set on ESP8266 software + # serial) — enable by default to reduce RX buffer overflow risk by + # waking the main loop immediately when data arrives. cg.add_define("USE_UART_WAKE_LOOP_ON_RX") diff --git a/esphome/components/uart/uart_component_esp8266.cpp b/esphome/components/uart/uart_component_esp8266.cpp index 0ea7930760..fc1509f737 100644 --- a/esphome/components/uart/uart_component_esp8266.cpp +++ b/esphome/components/uart/uart_component_esp8266.cpp @@ -4,6 +4,9 @@ #include "esphome/core/defines.h" #include "esphome/core/helpers.h" #include "esphome/core/log.h" +#ifdef USE_UART_WAKE_LOOP_ON_RX +#include "esphome/core/wake.h" +#endif #ifdef USE_LOGGER #include "esphome/components/logger/logger.h" @@ -149,7 +152,11 @@ void ESP8266UartComponent::dump_config() { if (this->hw_serial_ != nullptr) { ESP_LOGCONFIG(TAG, " Using hardware serial interface."); } else { - ESP_LOGCONFIG(TAG, " Using software serial"); + ESP_LOGCONFIG(TAG, " Using software serial" +#ifdef USE_UART_WAKE_LOOP_ON_RX + "\n Wake on data RX: ENABLED" +#endif + ); } this->check_logger_conflict(); } @@ -266,6 +273,12 @@ void IRAM_ATTR ESP8266SoftwareSerial::gpio_intr(ESP8266SoftwareSerial *arg) { arg->rx_in_pos_ = (arg->rx_in_pos_ + 1) % arg->rx_buffer_size_; // Clear RX pin so that the interrupt doesn't re-trigger right away again. arg->rx_pin_.clear_interrupt(); +#ifdef USE_UART_WAKE_LOOP_ON_RX + // Wake the main loop so the consuming component drains the byte promptly + // instead of waiting for the next loop_interval_ tick. Important for timing + // sensitive setups that poll read() in a tight loop (e.g. fingerprint_grow). + wake_loop_isrsafe(); +#endif } void IRAM_ATTR HOT ESP8266SoftwareSerial::write_byte(uint8_t data) { if (this->gpio_tx_pin_ == nullptr) { diff --git a/esphome/components/voice_assistant/__init__.py b/esphome/components/voice_assistant/__init__.py index 958d1cbf91..f41adfd8de 100644 --- a/esphome/components/voice_assistant/__init__.py +++ b/esphome/components/voice_assistant/__init__.py @@ -15,7 +15,7 @@ from esphome.const import ( CONF_SPEAKER, ) -AUTO_LOAD = ["ring_buffer", "socket"] +AUTO_LOAD = ["audio", "ring_buffer", "socket"] DEPENDENCIES = ["api", "microphone"] CODEOWNERS = ["@jesserockz", "@kahrendt"] diff --git a/esphome/components/voice_assistant/voice_assistant.cpp b/esphome/components/voice_assistant/voice_assistant.cpp index 286e6645d2..af1b98da02 100644 --- a/esphome/components/voice_assistant/voice_assistant.cpp +++ b/esphome/components/voice_assistant/voice_assistant.cpp @@ -30,7 +30,7 @@ VoiceAssistant::VoiceAssistant() { global_voice_assistant = this; } void VoiceAssistant::setup() { this->mic_source_->add_data_callback([this](const std::vector &data) { - std::shared_ptr temp_ring_buffer = this->ring_buffer_; + std::shared_ptr temp_ring_buffer = this->ring_buffer_.lock(); if (temp_ring_buffer != nullptr) { temp_ring_buffer->write((void *) data.data(), data.size()); } @@ -39,7 +39,7 @@ void VoiceAssistant::setup() { // Second microphone channel if (this->mic_source2_ != nullptr) { this->mic_source2_->add_data_callback([this](const std::vector &data) { - std::shared_ptr temp_ring_buffer = this->ring_buffer2_; + std::shared_ptr temp_ring_buffer = this->ring_buffer2_.lock(); if (temp_ring_buffer != nullptr) { temp_ring_buffer->write((void *) data.data(), data.size()); } @@ -125,62 +125,47 @@ bool VoiceAssistant::allocate_buffers_() { } #endif - if (this->ring_buffer_ == nullptr) { - this->ring_buffer_ = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE); - if (this->ring_buffer_ == nullptr) { + if (this->audio_source_ == nullptr) { + std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE); + if (temp_ring_buffer == nullptr) { ESP_LOGE(TAG, "Could not allocate ring buffer"); return false; } - } - - if (this->send_buffer_ == nullptr) { - RAMAllocator send_allocator; - this->send_buffer_ = send_allocator.allocate(SEND_BUFFER_SIZE); - if (send_buffer_ == nullptr) { - ESP_LOGW(TAG, "Could not allocate send buffer"); + // Zero-copy source that reads directly from the ring buffer; frame-aligned to never split an int16 sample. + this->audio_source_ = audio::RingBufferAudioSource::create(temp_ring_buffer, SEND_BUFFER_SIZE, sizeof(int16_t)); + if (this->audio_source_ == nullptr) { + ESP_LOGE(TAG, "Could not allocate audio source"); return false; } + this->ring_buffer_ = temp_ring_buffer; } // Second microphone channel - if (this->mic_source2_ != nullptr) { - if (this->ring_buffer2_ == nullptr) { - this->ring_buffer2_ = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE); - if (this->ring_buffer2_ == nullptr) { - ESP_LOGE(TAG, "Could not allocate second ring buffer"); - return false; - } + if ((this->mic_source2_ != nullptr) && (this->audio_source2_ == nullptr)) { + std::shared_ptr temp_ring_buffer = ring_buffer::RingBuffer::create(RING_BUFFER_SIZE); + if (temp_ring_buffer == nullptr) { + ESP_LOGE(TAG, "Could not allocate second ring buffer"); + return false; } - - if (this->send_buffer2_ == nullptr) { - RAMAllocator send_allocator; - this->send_buffer2_ = send_allocator.allocate(SEND_BUFFER_SIZE); - if (this->send_buffer2_ == nullptr) { - ESP_LOGW(TAG, "Could not allocate second send buffer"); - return false; - } + this->audio_source2_ = audio::RingBufferAudioSource::create(temp_ring_buffer, SEND_BUFFER_SIZE, sizeof(int16_t)); + if (this->audio_source2_ == nullptr) { + ESP_LOGE(TAG, "Could not allocate second audio source"); + return false; } + this->ring_buffer2_ = temp_ring_buffer; } return true; } void VoiceAssistant::clear_buffers_() { - if (this->send_buffer_ != nullptr) { - memset(this->send_buffer_, 0, SEND_BUFFER_SIZE); - } - - if (this->ring_buffer_ != nullptr) { - this->ring_buffer_->reset(); + if (this->audio_source_ != nullptr) { + this->audio_source_->clear_buffered_data(); } // Second microphone channel - if (this->send_buffer2_ != nullptr) { - memset(this->send_buffer2_, 0, SEND_BUFFER_SIZE); - } - - if (this->ring_buffer2_ != nullptr) { - this->ring_buffer2_->reset(); + if (this->audio_source2_ != nullptr) { + this->audio_source2_->clear_buffered_data(); } #ifdef USE_SPEAKER @@ -195,22 +180,11 @@ void VoiceAssistant::clear_buffers_() { } void VoiceAssistant::deallocate_buffers_() { - if (this->send_buffer_ != nullptr) { - RAMAllocator send_deallocator; - send_deallocator.deallocate(this->send_buffer_, SEND_BUFFER_SIZE); - this->send_buffer_ = nullptr; - } - - this->ring_buffer_.reset(); + // Destroying each source releases its ring buffer; the matching weak_ptr then expires automatically. + this->audio_source_.reset(); // Second microphone channel - if (this->send_buffer2_ != nullptr) { - RAMAllocator send_deallocator; - send_deallocator.deallocate(this->send_buffer2_, SEND_BUFFER_SIZE); - this->send_buffer2_ = nullptr; - } - - this->ring_buffer2_.reset(); + this->audio_source2_.reset(); #ifdef USE_SPEAKER if ((this->speaker_ != nullptr) && (this->speaker_buffer_ != nullptr)) { @@ -316,52 +290,57 @@ void VoiceAssistant::loop() { break; // State changed when udp server port received } case State::STREAMING_MICROPHONE: { + // pre_shift is ignored by RingBufferAudioSource (no intermediate transfer buffer to compact). if (this->audio_mode_ == AUDIO_MODE_API) { // API audio // Both microphone channels are sent, if configured - bool is_available = this->ring_buffer_->available() >= SEND_BUFFER_SIZE; - bool is_available2 = false; - if (this->mic_source2_) { - is_available2 = this->ring_buffer2_->available() >= SEND_BUFFER_SIZE; + size_t available = this->audio_source_->fill(0, false); + size_t available2 = 0; + if (this->audio_source2_ != nullptr) { + available2 = this->audio_source2_->fill(0, false); } - while (is_available || is_available2) { + while (available > 0 || available2 > 0) { api::VoiceAssistantAudio msg; - if (is_available) { - size_t read_bytes = this->ring_buffer_->read((void *) this->send_buffer_, SEND_BUFFER_SIZE, 0); - msg.data = this->send_buffer_; - msg.data_len = read_bytes; + if (available > 0) { + // Zero-copy: send_message() copies the data out before we consume it + msg.data = this->audio_source_->data(); + msg.data_len = available; } // Second microphone channel - if (is_available2) { - size_t read_bytes = this->ring_buffer2_->read((void *) this->send_buffer2_, SEND_BUFFER_SIZE, 0); - msg.data2 = this->send_buffer2_; - msg.data2_len = read_bytes; + if (available2 > 0) { + msg.data2 = this->audio_source2_->data(); + msg.data2_len = available2; } this->api_client_->send_message(msg); - is_available = this->ring_buffer_->available() >= SEND_BUFFER_SIZE; - if (this->mic_source2_) { - is_available2 = this->ring_buffer2_->available() >= SEND_BUFFER_SIZE; - } else { - is_available2 = false; + + if (available > 0) { + this->audio_source_->consume(available); + } + available = this->audio_source_->fill(0, false); + if (available2 > 0) { + this->audio_source2_->consume(available2); + } + if (this->audio_source2_ != nullptr) { + available2 = this->audio_source2_->fill(0, false); } } } else { // UDP (will eventually be deprecated) // Only the primary microphone channel is used - while (this->ring_buffer_->available() >= SEND_BUFFER_SIZE) { - size_t read_bytes = this->ring_buffer_->read((void *) this->send_buffer_, SEND_BUFFER_SIZE, 0); + while (this->audio_source_->fill(0, false) > 0) { if (!this->udp_socket_running_) { if (!this->start_udp_socket_()) { this->set_state_(State::STOP_MICROPHONE, State::IDLE); break; } } - this->socket_->sendto(this->send_buffer_, read_bytes, 0, (struct sockaddr *) &this->dest_addr_, - sizeof(this->dest_addr_)); + this->socket_->sendto(this->audio_source_->data(), this->audio_source_->available(), 0, + (struct sockaddr *) &this->dest_addr_, sizeof(this->dest_addr_)); + this->audio_source_->consume(this->audio_source_->available()); } } // audio mode break; diff --git a/esphome/components/voice_assistant/voice_assistant.h b/esphome/components/voice_assistant/voice_assistant.h index c4fa7eb615..f3ea669e15 100644 --- a/esphome/components/voice_assistant/voice_assistant.h +++ b/esphome/components/voice_assistant/voice_assistant.h @@ -9,6 +9,7 @@ #include "esphome/core/helpers.h" #include "esphome/components/api/api_connection.h" +#include "esphome/components/audio/audio_transfer_buffer.h" #include "esphome/components/ring_buffer/ring_buffer.h" #include "esphome/components/api/api_pb2.h" #include "esphome/components/microphone/microphone_source.h" @@ -306,8 +307,13 @@ class VoiceAssistant : public Component { std::string wake_word_; - std::shared_ptr ring_buffer_; - std::shared_ptr ring_buffer2_; + // Zero-copy sources that read directly from each microphone channel's ring buffer internal storage. + // Each source owns its ring buffer; the matching ``ring_buffer_``/``ring_buffer2_`` weak_ptr is used by + // the microphone callback (a different thread) to write into it. + std::unique_ptr audio_source_; + std::unique_ptr audio_source2_; + std::weak_ptr ring_buffer_; + std::weak_ptr ring_buffer2_; bool use_wake_word_; uint8_t noise_suppression_level_; @@ -315,9 +321,6 @@ class VoiceAssistant : public Component { float volume_multiplier_; uint32_t conversation_timeout_; - uint8_t *send_buffer_{nullptr}; - uint8_t *send_buffer2_{nullptr}; - bool continuous_{false}; bool silence_detection_; diff --git a/esphome/components/waveshare_epaper/display.py b/esphome/components/waveshare_epaper/display.py index 5db7a1fc3d..7ecc3b4a87 100644 --- a/esphome/components/waveshare_epaper/display.py +++ b/esphome/components/waveshare_epaper/display.py @@ -236,7 +236,7 @@ async def to_code(config): rhs = model.new() var = cg.Pvariable(config[CONF_ID], rhs, model) else: - raise NotImplementedError() + raise NotImplementedError await display.register_display(var, config) await spi.register_spi_device(var, config, write_only=True) diff --git a/esphome/components/web_server/__init__.py b/esphome/components/web_server/__init__.py index 84910b6f90..99a9b7518c 100644 --- a/esphome/components/web_server/__init__.py +++ b/esphome/components/web_server/__init__.py @@ -326,12 +326,12 @@ async def to_code(config): if CONF_CSS_INCLUDE in config: cg.add_define("USE_WEBSERVER_CSS_INCLUDE") path = CORE.relative_config_path(config[CONF_CSS_INCLUDE]) - with open(file=path, encoding="utf-8") as css_file: + with path.open(encoding="utf-8") as css_file: add_resource_as_progmem("CSS_INCLUDE", css_file.read()) if CONF_JS_INCLUDE in config: cg.add_define("USE_WEBSERVER_JS_INCLUDE") path = CORE.relative_config_path(config[CONF_JS_INCLUDE]) - with open(file=path, encoding="utf-8") as js_file: + with path.open(encoding="utf-8") as js_file: add_resource_as_progmem("JS_INCLUDE", js_file.read()) cg.add(var.set_include_internal(config[CONF_INCLUDE_INTERNAL])) if CONF_LOCAL in config and config[CONF_LOCAL]: diff --git a/esphome/components/wifi/__init__.py b/esphome/components/wifi/__init__.py index bad57fc481..f9cb391442 100644 --- a/esphome/components/wifi/__init__.py +++ b/esphome/components/wifi/__init__.py @@ -54,10 +54,18 @@ from esphome.const import ( CONF_TTLS_PHASE_2, CONF_USE_ADDRESS, CONF_USERNAME, + CONF_WIFI, + PLACEHOLDER_WIFI_SSID, Platform, PlatformFramework, ) -from esphome.core import CORE, CoroPriority, HexInt, coroutine_with_priority +from esphome.core import ( + CORE, + CoroPriority, + EsphomeError, + HexInt, + coroutine_with_priority, +) import esphome.final_validate as fv from esphome.types import ConfigType @@ -903,3 +911,45 @@ FILTER_SOURCE_FILES = filter_source_files_from_platform( "wifi_component_pico_w.cpp": {PlatformFramework.RP2040_ARDUINO}, } ) + + +def _placeholder_wifi_credentials(config: ConfigType) -> list[str]: + """Return human-readable locations where the dashboard's placeholder wifi + values still appear. Empty list means no placeholders were found. + """ + placeholders: list[str] = [] + wifi_conf = config.get(CONF_WIFI) + if not wifi_conf: + return placeholders + + for idx, network in enumerate(wifi_conf.get(CONF_NETWORKS, [])): + ssid = network.get(CONF_SSID) + if isinstance(ssid, str) and ssid == PLACEHOLDER_WIFI_SSID: + placeholders.append(f"wifi.networks[{idx}].ssid") + + ap_conf = wifi_conf.get(CONF_AP) + if ap_conf: + ap_ssid = ap_conf.get(CONF_SSID) + if isinstance(ap_ssid, str) and ap_ssid == PLACEHOLDER_WIFI_SSID: + placeholders.append("wifi.ap.ssid") + + return placeholders + + +def check_placeholder_credentials(config: ConfigType) -> None: + """Raise EsphomeError if any wifi credential is the dashboard placeholder. + + Call only at compile time. NEVER from CONFIG_SCHEMA, FINAL_VALIDATE_SCHEMA, + or any path reached by `esphome config`; device-builder relies on + validation passing with the placeholders still in place. + """ + locations = _placeholder_wifi_credentials(config) + if not locations: + return + formatted = ", ".join(locations) + raise EsphomeError( + f"wifi configuration still contains the dashboard placeholder value " + f"'{PLACEHOLDER_WIFI_SSID}' at: {formatted}. " + f"Open secrets.yaml and replace 'wifi_ssid' (and 'wifi_password') " + f"with your real wifi credentials before flashing." + ) diff --git a/esphome/components/wifi/wifi_component.cpp b/esphome/components/wifi/wifi_component.cpp index edfb93bba2..fdbd70bc61 100644 --- a/esphome/components/wifi/wifi_component.cpp +++ b/esphome/components/wifi/wifi_component.cpp @@ -634,9 +634,6 @@ void WiFiComponent::setup() { if (this->enable_on_boot_) { this->start(); } else { -#ifdef USE_ESP32 - esp_netif_init(); -#endif this->state_ = WIFI_COMPONENT_STATE_DISABLED; } } @@ -2193,7 +2190,15 @@ bool WiFiComponent::request_high_performance() { } // Give the semaphore (non-blocking). This increments the count. - return xSemaphoreGive(this->high_performance_semaphore_) == pdTRUE; + bool success = xSemaphoreGive(this->high_performance_semaphore_) == pdTRUE; + + // Wake the main loop so the switch to high-performance mode is applied on the + // next tick instead of waiting up to loop_interval. + if (success) { + App.wake_loop_threadsafe(); + } + + return success; } bool WiFiComponent::release_high_performance() { diff --git a/esphome/components/wifi/wifi_component_esp_idf.cpp b/esphome/components/wifi/wifi_component_esp_idf.cpp index 4f39a3a4b1..11b39b5000 100644 --- a/esphome/components/wifi/wifi_component_esp_idf.cpp +++ b/esphome/components/wifi/wifi_component_esp_idf.cpp @@ -145,23 +145,15 @@ void WiFiComponent::wifi_pre_setup_() { get_mac_address_raw(mac); set_mac_address(mac); } - esp_err_t err = esp_netif_init(); - if (err != ERR_OK) { - ESP_LOGE(TAG, "esp_netif_init failed: %s", esp_err_to_name(err)); - return; - } + // Network interface setup handled by network component s_wifi_event_group = xEventGroupCreate(); if (s_wifi_event_group == nullptr) { ESP_LOGE(TAG, "xEventGroupCreate failed"); return; } - err = esp_event_loop_create_default(); - if (err != ERR_OK) { - ESP_LOGE(TAG, "esp_event_loop_create_default failed: %s", esp_err_to_name(err)); - return; - } esp_event_handler_instance_t instance_wifi_id, instance_ip_id; - err = esp_event_handler_instance_register(WIFI_EVENT, ESP_EVENT_ANY_ID, &event_handler, nullptr, &instance_wifi_id); + esp_err_t err = + esp_event_handler_instance_register(WIFI_EVENT, ESP_EVENT_ANY_ID, &event_handler, nullptr, &instance_wifi_id); if (err != ERR_OK) { ESP_LOGE(TAG, "esp_event_handler_instance_register failed: %s", esp_err_to_name(err)); return; diff --git a/esphome/components/zigbee/__init__.py b/esphome/components/zigbee/__init__.py index 69e3fe9c5a..c75b0773d2 100644 --- a/esphome/components/zigbee/__init__.py +++ b/esphome/components/zigbee/__init__.py @@ -50,6 +50,8 @@ _LOGGER = logging.getLogger(__name__) CODEOWNERS = ["@luar123", "@tomaszduda23"] +CONFLICTS_WITH = ["openthread"] + BASE_SCHEMA = cv.Schema( { cv.Optional(CONF_REPORT): cv.All( diff --git a/esphome/components/zigbee/zigbee_esp32.py b/esphome/components/zigbee/zigbee_esp32.py index e446377a06..a0fadbce8b 100644 --- a/esphome/components/zigbee/zigbee_esp32.py +++ b/esphome/components/zigbee/zigbee_esp32.py @@ -117,15 +117,11 @@ def final_validate_esp32(config: ConfigType) -> ConfigType: if not CORE.is_esp32: return config if CONF_WIFI in fv.full_config.get(): - if config[CONF_ROUTER] and CONF_AP in fv.full_config.get()[CONF_WIFI]: - raise cv.Invalid( - "Only Zigbee End Device can be used together with a Wifi Access Point." - ) if CONF_AP in fv.full_config.get()[CONF_WIFI]: - _LOGGER.warning( - "Wifi Access Point might be unstable while Zigbee is active, use only as fallback." + raise cv.Invalid( + "A Wifi Access Point can not be used together with Zigbee." ) - elif config[CONF_ROUTER]: + if config[CONF_ROUTER]: _LOGGER.warning( "The Zigbee Router might miss packets while Wifi is active and could destabilize " "your network. Use only if Wifi is off most of the time." @@ -133,9 +129,8 @@ def final_validate_esp32(config: ConfigType) -> ConfigType: if CONF_PARTITIONS in fv.full_config.get() and not isinstance( fv.full_config.get()[CONF_PARTITIONS], list ): - with open( - CORE.relative_config_path(fv.full_config.get()[CONF_PARTITIONS]), - encoding="utf8", + with CORE.relative_config_path(fv.full_config.get()[CONF_PARTITIONS]).open( + encoding="utf8" ) as f: partitions_tab = f.read() for partition, types in [ diff --git a/esphome/config_validation.py b/esphome/config_validation.py index c993c1dcc5..ca1fd8f5d4 100644 --- a/esphome/config_validation.py +++ b/esphome/config_validation.py @@ -1862,7 +1862,7 @@ def extract_keys(schema): elif isinstance(skey, vol.Marker) and isinstance(skey.schema, str): keys.append(skey.schema) else: - raise ValueError() + raise ValueError keys.sort() return keys diff --git a/esphome/const.py b/esphome/const.py index 4557380c73..07f6bad771 100644 --- a/esphome/const.py +++ b/esphome/const.py @@ -199,6 +199,7 @@ CONF_BROKER = "broker" CONF_BSSID = "bssid" CONF_BUFFER_DURATION = "buffer_duration" CONF_BUFFER_SIZE = "buffer_size" +CONF_BUILD_FLAGS = "build_flags" CONF_BUILD_PATH = "build_path" CONF_BUS_VOLTAGE = "bus_voltage" CONF_BUSY_PIN = "busy_pin" @@ -1416,3 +1417,12 @@ ENTITY_CATEGORY_DIAGNOSTIC = "diagnostic" # The corresponding constant exists in c++ # when update_interval is set to never, it becomes SCHEDULER_DONT_RUN milliseconds SCHEDULER_DONT_RUN = 4294967295 + +# Sentinel values written by the esphome-device-builder dashboard into +# secrets.yaml on first boot so that !secret wifi_ssid / !secret wifi_password +# references resolve cleanly through validation before the user has finished +# the onboarding wizard. Compilation refuses if these reach the binary so that +# a user who dismisses onboarding can't accidentally flash a device that will +# never associate with their wifi. +PLACEHOLDER_WIFI_SSID = "REPLACE_WITH_YOUR_WIFI_NETWORK" +PLACEHOLDER_WIFI_PASSWORD = "REPLACE_WITH_YOUR_WIFI_PASSWORD" # noqa: S105 diff --git a/esphome/core/__init__.py b/esphome/core/__init__.py index e13d5668af..182be38b18 100644 --- a/esphome/core/__init__.py +++ b/esphome/core/__init__.py @@ -5,7 +5,7 @@ import math import os from pathlib import Path import re -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from esphome.const import ( CONF_COMMENT, @@ -569,6 +569,12 @@ class EsphomeCore: self.build_path: Path | None = None # The validated configuration, this is None until the config has been validated self.config: ConfigType | None = None + # YAML frontmatter loaded from user YAML files. Frontmatter is a leading + # YAML document separated by `---` from the actual configuration. It is + # ignored by config validation and code generation, but kept here so it + # can be inspected by callers (tooling, future features). Keyed by the + # resolved Path of the source file. + self.frontmatter: dict[Path, Any] = {} # The pending tasks in the task queue (mostly for C++ generation) # This is a priority queue (with heapq) # Each item is a tuple of form: (-priority, unique number, task) @@ -634,6 +640,7 @@ class EsphomeCore: self.config_path = None self.build_path = None self.config = None + self.frontmatter = {} self.event_loop = _FakeEventLoop() self.task_counter = 0 self.variables = {} @@ -1074,7 +1081,7 @@ class EnumValue: @enum_value.setter def enum_value(self, value): - setattr(self, "_enum_value", value) + self._enum_value = value CORE = EsphomeCore() diff --git a/esphome/core/config.py b/esphome/core/config.py index 5a98b94781..6125c4ecc9 100644 --- a/esphome/core/config.py +++ b/esphome/core/config.py @@ -13,6 +13,7 @@ from esphome.const import ( CONF_AREA, CONF_AREA_ID, CONF_AREAS, + CONF_BUILD_FLAGS, CONF_BUILD_PATH, CONF_COMMENT, CONF_COMPILE_PROCESS_LIMIT, @@ -288,6 +289,7 @@ CONFIG_SCHEMA = cv.All( cv.string_strict: cv.Any([cv.string], cv.string), } ), + cv.Optional(CONF_BUILD_FLAGS, default=[]): cv.ensure_list(cv.string_strict), cv.Optional(CONF_ENVIRONMENT_VARIABLES, default={}): cv.Schema( { cv.string_strict: cv.string, @@ -510,6 +512,12 @@ async def _add_platformio_options(pio_options): cg.add_platformio_option(key, val) +@coroutine_with_priority(CoroPriority.FINAL) +async def _add_build_flags(flags: list[str]) -> None: + for flag in flags: + cg.add_build_flag(flag) + + @coroutine_with_priority(CoroPriority.FINAL) async def _add_environment_variables(env_vars: dict[str, str]) -> None: # Set environment variables for the build process @@ -705,6 +713,9 @@ async def to_code(config: ConfigType) -> None: if config[CONF_PLATFORMIO_OPTIONS]: CORE.add_job(_add_platformio_options, config[CONF_PLATFORMIO_OPTIONS]) + if config[CONF_BUILD_FLAGS]: + CORE.add_job(_add_build_flags, config[CONF_BUILD_FLAGS]) + if config[CONF_ENVIRONMENT_VARIABLES]: CORE.add_job(_add_environment_variables, config[CONF_ENVIRONMENT_VARIABLES]) diff --git a/esphome/core/defines.h b/esphome/core/defines.h index 6c125c2ed0..837a80f030 100644 --- a/esphome/core/defines.h +++ b/esphome/core/defines.h @@ -71,6 +71,7 @@ #define USE_GRAPH #define USE_GRAPHICAL_DISPLAY_MENU #define USE_HOMEASSISTANT_TIME +#define USE_HOMEASSISTANT_TIMEZONE #define USE_HTTP_REQUEST_OTA_WATCHDOG_TIMEOUT 8000 // NOLINT #define USE_I2S_AUDIO_SPDIF_MODE #define USE_IMAGE @@ -402,6 +403,7 @@ #define USE_LOGGER_USB_CDC #define USE_SOCKET_IMPL_LWIP_TCP #define USE_RP2040_BLE +#define USE_RP2040_VARIANT_RP2040 #define USE_SPI #ifndef USE_ETHERNET #define USE_ETHERNET diff --git a/esphome/core/hal.h b/esphome/core/hal.h index 4babda807d..b44a422836 100644 --- a/esphome/core/hal.h +++ b/esphome/core/hal.h @@ -1,6 +1,7 @@ #pragma once -#include #include +#include +#include #include "gpio.h" #include "esphome/core/defines.h" #include "esphome/core/time_64.h" @@ -42,6 +43,9 @@ void __attribute__((noreturn)) arch_restart(); inline uint8_t progmem_read_byte(const uint8_t *addr) { return *addr; } inline const char *progmem_read_ptr(const char *const *addr) { return *addr; } inline uint16_t progmem_read_uint16(const uint16_t *addr) { return *addr; } +// Bulk copy out of PROGMEM. PROGMEM is a no-op everywhere except ESP8266, so a +// plain `std::memcpy` is correct and the fast path here. +inline void progmem_memcpy(void *dst, const void *src, size_t len) { std::memcpy(dst, src, len); } #endif } // namespace esphome diff --git a/esphome/cpp_generator.py b/esphome/cpp_generator.py index c622207dac..151018baa4 100644 --- a/esphome/cpp_generator.py +++ b/esphome/cpp_generator.py @@ -893,7 +893,7 @@ class MockObj(Expression): def __getattr__(self, attr: str) -> "MockObj": # prevent python dunder methods being replaced by mock objects if attr.startswith("__"): - raise AttributeError() + raise AttributeError next_op = "." if attr.startswith("P") and self.op not in ["::", ""]: attr = attr[1:] @@ -1077,43 +1077,45 @@ class MockObj(Expression): op = BinOpExpression(other, "|", self) return MockObj(op) - def __iadd__(self, other: SafeExpType) -> "MockObj": + # MockObj operator overloads build a new C++ expression rather than mutating self, + # so the PYI034 "augmented assignment returns self" assumption does not apply. + def __iadd__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "+=", other) return MockObj(op) - def __isub__(self, other: SafeExpType) -> "MockObj": + def __isub__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "-=", other) return MockObj(op) - def __imul__(self, other: SafeExpType) -> "MockObj": + def __imul__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "*=", other) return MockObj(op) - def __itruediv__(self, other: SafeExpType) -> "MockObj": + def __itruediv__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "/=", other) return MockObj(op) - def __imod__(self, other: SafeExpType) -> "MockObj": + def __imod__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "%=", other) return MockObj(op) - def __ilshift__(self, other: SafeExpType) -> "MockObj": + def __ilshift__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "<<=", other) return MockObj(op) - def __irshift__(self, other: SafeExpType) -> "MockObj": + def __irshift__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, ">>=", other) return MockObj(op) - def __iand__(self, other: SafeExpType) -> "MockObj": + def __iand__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "&=", other) return MockObj(op) - def __ixor__(self, other: SafeExpType) -> "MockObj": + def __ixor__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "^=", other) return MockObj(op) - def __ior__(self, other: SafeExpType) -> "MockObj": + def __ior__(self, other: SafeExpType) -> "MockObj": # noqa: PYI034 op = BinOpExpression(self, "|=", other) return MockObj(op) diff --git a/esphome/dashboard/dashboard.py b/esphome/dashboard/dashboard.py index 81c10763e7..7fc21f8a44 100644 --- a/esphome/dashboard/dashboard.py +++ b/esphome/dashboard/dashboard.py @@ -6,6 +6,7 @@ from concurrent.futures import ThreadPoolExecutor import contextlib import logging import os +from pathlib import Path import socket import threading from time import monotonic @@ -149,4 +150,4 @@ async def async_start(args) -> None: await dashboard.async_run() finally: if sock: - os.remove(sock) + Path(sock).unlink() diff --git a/esphome/dashboard/status/mdns.py b/esphome/dashboard/status/mdns.py index 881340ab24..9da9bb8f01 100644 --- a/esphome/dashboard/status/mdns.py +++ b/esphome/dashboard/status/mdns.py @@ -115,7 +115,7 @@ class MDNSStatus: results = await asyncio.gather( *(self.aiozc.async_resolve_host(name) for name in poll_names) ) - for name, address_list in zip(poll_names, results): + for name, address_list in zip(poll_names, results, strict=True): result = bool(address_list) host_mdns_state[name] = result for entry in poll_names[name]: diff --git a/esphome/dashboard/status/ping.py b/esphome/dashboard/status/ping.py index b4f106d21a..eb69fbb9b3 100644 --- a/esphome/dashboard/status/ping.py +++ b/esphome/dashboard/status/ping.py @@ -83,7 +83,7 @@ class PingStatus: return_exceptions=True, ) - for entry, result in zip(ping_group, dns_results): + for entry, result in zip(ping_group, dns_results, strict=True): if isinstance(result, Exception): # Only update state if its unknown or from ping # so we don't mark it as offline if we have a state @@ -106,7 +106,7 @@ class PingStatus: return_exceptions=True, ) - for entry_addresses, result in zip(entry_addresses, results): + for entry_address, result in zip(entry_addresses, results, strict=True): if isinstance(result, Exception): ping_result = False elif isinstance(result, BaseException): @@ -114,7 +114,7 @@ class PingStatus: else: host: Host = result ping_result = host.is_alive - entry: DashboardEntry = entry_addresses[0] + entry: DashboardEntry = entry_address[0] # If we can reach it via ping, we always set it # online, however if we can't reach it via ping # we only set it to offline if the state is unknown diff --git a/esphome/dashboard/web_server.py b/esphome/dashboard/web_server.py index 916e937a53..97d6639c1f 100644 --- a/esphome/dashboard/web_server.py +++ b/esphome/dashboard/web_server.py @@ -1030,7 +1030,7 @@ class DownloadListRequestHandler(BaseHandler): try: module = importlib.import_module(f"esphome.components.{platform}") - get_download_types = getattr(module, "get_download_types") + get_download_types = module.get_download_types except AttributeError as exc: raise ValueError(f"Unknown platform {platform}") from exc downloads = get_download_types(storage_json) @@ -1040,7 +1040,7 @@ class DownloadListRequestHandler(BaseHandler): class DownloadBinaryRequestHandler(BaseHandler): def _load_file(self, path: str, compressed: bool) -> bytes: """Load a file from disk and compress it if requested.""" - with open(path, "rb") as f: + with Path(path).open("rb") as f: data = f.read() if compressed: return gzip.compress(data, 9) @@ -1146,7 +1146,7 @@ class MainRequestHandler(BaseHandler): begin = bool(self.get_argument("begin", False)) if settings.using_password: # Simply accessing the xsrf_token sets the cookie for us - self.xsrf_token # pylint: disable=pointless-statement + self.xsrf_token # pylint: disable=pointless-statement # noqa: B018 else: self.clear_cookie("_xsrf") @@ -1292,7 +1292,7 @@ class EditRequestHandler(BaseHandler): def _read_file(self, filename: str, configuration: str) -> bytes | None: """Read a file and return the content as bytes.""" try: - with open(file=filename, encoding="utf-8") as f: + with Path(filename).open(encoding="utf-8") as f: return f.read() except FileNotFoundError: if configuration in const.SECRETS_FILES: @@ -1493,7 +1493,7 @@ def get_base_frontend_path() -> Path: static_path += "/" # This path can be relative, so resolve against the root or else templates don't work - path = Path(os.getcwd()) / static_path / "esphome_dashboard" + path = Path.cwd() / static_path / "esphome_dashboard" return path.resolve() @@ -1519,7 +1519,10 @@ def get_static_file_url(name: str) -> str: return f"{base}?hash={hash_}" -def make_app(debug=get_bool_env(ENV_DEV)) -> tornado.web.Application: +def make_app(debug: bool | None = None) -> tornado.web.Application: + if debug is None: + debug = get_bool_env(ENV_DEV) + def log_function(handler: tornado.web.RequestHandler) -> None: if handler.get_status() < 400: log_method = access_log.info diff --git a/esphome/espidf/component.py b/esphome/espidf/component.py index b9202fb6bf..050002d9e2 100644 --- a/esphome/espidf/component.py +++ b/esphome/espidf/component.py @@ -55,7 +55,7 @@ ESPHOME_DATA_EXTRA_CMAKE_KEY = "EXTRA_CMAKE" class Source: def download(self, dir_suffix: str, force: bool = False) -> Path: - raise NotImplementedError() + raise NotImplementedError class URLSource(Source): @@ -93,7 +93,7 @@ class URLSource(Source): class GitSource(Source): - def __init__(self, url: str, ref: str): + def __init__(self, url: str, ref: str | None): self.url = url self.ref = ref @@ -109,7 +109,7 @@ class GitSource(Source): return path def __str__(self): - return f"{self.url}#{self.ref}" + return f"{self.url}#{self.ref}" if self.ref else self.url class InvalidIDFComponent(Exception): @@ -154,41 +154,6 @@ class IDFComponent: self.path = self.source.download(self.get_sanitized_name(), force=force) -def _sanitize_version(version: str) -> str: - """ - Sanitize a version string by removing common requirement prefixes or a leading v. - - Args: - version: Version string to clean. - - Returns: - Cleaned version string without common requirement symbols. - """ - version = version.strip() - - prefixes = ( - "^", - "~=", - "~", - ">=", - "<=", - "==", - "!=", - ">", - "<", - "=", - "v", - "V", - ) - - for p in prefixes: - if version.startswith(p): - version = version[len(p) :] - break - - return version.strip() - - def _get_package_from_pio_registry( username: str | None, pkgname: str, requirements: str ) -> tuple[str, str, str | None, str | None]: @@ -352,24 +317,26 @@ def _collect_filtered_files(src_dir: PathType, src_filters: list[str]) -> list[s if pattern.endswith("/"): pattern = pattern.rstrip("/") + "/**" - full_pattern = os.path.join(glob.escape(str(src_dir)), pattern) + # glob.escape has no pathlib equivalent and the matcher works on raw + # path strings, so PTH118/PTH207 don't apply here. + full_pattern = os.path.join(glob.escape(str(src_dir)), pattern) # noqa: PTH118 matched = [] - for item in glob.glob(full_pattern, recursive=True): - if not os.path.isdir(item): + for item in glob.glob(full_pattern, recursive=True): # noqa: PTH207 + if not Path(item).is_dir(): matched.append(item) else: # PlatformIO quirk: a directory matched with "*" should include all its # nested files and subdirectories, not just the directory itself. for root, _, files in os.walk(item): - matched.extend([os.path.join(root, f) for f in files]) + matched.extend([str(Path(root) / f) for f in files]) if sign == "+": selected.update(matched) elif sign == "-": selected.difference_update(matched) - return [r for r in selected if os.path.isfile(r)] + return [r for r in selected if Path(r).is_file()] def _convert_library_to_component(library: Library) -> IDFComponent: @@ -387,7 +354,6 @@ def _convert_library_to_component(library: Library) -> IDFComponent: IDFComponent: The resolved component with name, version, and URL Raises: - ValueError: If a repository URL is missing a reference (#) RuntimeError: If no artifact can be found for the library """ name = None @@ -396,20 +362,25 @@ def _convert_library_to_component(library: Library) -> IDFComponent: # Repository is provided directly if library.repository: - # Parse repository URL to extract name and version + # Parse repository URL: path becomes the component name, fragment + # (if any) becomes the git ref stored on GitSource. A missing + # fragment is fine -- clone_or_update leaves the depth-1 clone on + # the remote's default branch, matching PIO's lib_deps behavior + # and external_components handling. split_result = urlsplit(library.repository) - if not split_result.fragment.strip(): - raise ValueError(f"Missing ref in URL {library.repository}") # Sanitize name name = str(split_result.path).strip("/") name = name.removesuffix(".git") - # Sanitize version - version = _sanitize_version(split_result.fragment) + # IDF Component Manager only accepts "*", a 40-char commit hash, or + # semver here. The actual git ref is preserved in GitSource.ref; + # override_path makes this field cosmetic at build time. + version = "*" repository = urlunsplit(split_result._replace(fragment="")) - source = GitSource(str(repository), split_result.fragment) + ref = split_result.fragment.strip() or None + source = GitSource(str(repository), ref) # Version is provided - resolve using PlatformIO registry elif library.version: @@ -517,7 +488,7 @@ def generate_cmakelists_txt(component: IDFComponent) -> str: # Only keep sources build_src_files = [os.path.relpath(p, component.path) for p in build_src_files] build_src_files = [ - f for f in build_src_files if os.path.splitext(f)[1] in SRC_FILE_EXTENSIONS + f for f in build_src_files if Path(f).suffix in SRC_FILE_EXTENSIONS ] # Handle build flags @@ -619,9 +590,6 @@ def generate_idf_component_yml(component: IDFComponent) -> str: if description: data["description"] = description - # Do not use the version from library.json/library.properties; it may be incorrect. - data["version"] = component.version - repository = component.data.get("repository", {}).get("url", None) if repository: data["repository"] = repository @@ -631,20 +599,11 @@ def generate_idf_component_yml(component: IDFComponent) -> str: if "dependencies" not in data: data["dependencies"] = {} - # Add this dependency to dependencies - dep = {} - dep["version"] = dependency.version - - # Should use dependency.path as override path - try: - dep["override_path"] = str(dependency.path) - except RuntimeError as e: - # No local path: only a GitSource can substitute its URL. - if not isinstance(dependency.source, GitSource): - raise e - dep["git"] = dependency.source.url - - data["dependencies"][dependency.get_sanitized_name()] = dep + # Every dependency goes through _generate_idf_component → + # component.download() before this runs, so .path is always set. + data["dependencies"][dependency.get_sanitized_name()] = { + "override_path": str(dependency.path), + } return yaml_util.dump(data) @@ -653,11 +612,17 @@ def _check_library_data(data: dict): """ Check if a library data is compatible with the ESP-IDF framework. + A platform mismatch (e.g. an AVR-only library on ESP32) raises + ``InvalidIDFComponent`` so the caller skips the library. A framework + mismatch only logs a warning — PIO manifests often understate the + frameworks they actually compile under, and IDF (unlike PIO's + ``lib_compat_mode``) has no opt-out, so we include the library anyway. + Args: - component: IDFComponent object being processed + data: PIO library manifest dict being processed. Raises: - ValueError: If library has unsupported platforms or frameworks + InvalidIDFComponent: If the library does not support the ESP32 platform. """ platforms = data.get("platforms", "*") if isinstance(platforms, str): @@ -675,12 +640,21 @@ def _check_library_data(data: dict): frameworks = [a.strip() for a in frameworks.split(",")] frameworks = _ensure_list(frameworks) - # Check if library supports ESP-IDF framework + # Check if library declares the active framework. PIO library manifests + # often list only "arduino" even when the library actually compiles fine + # under ESP-IDF, and IDF (unlike PIO with `lib_compat_mode`) has no way to + # opt out of the check. Warn instead of failing so the user isn't forced to + # fork the library to fix the manifest. framework = "arduino" if CORE.using_arduino else "espidf" valid_framework = "*" in frameworks or framework in frameworks if not valid_framework: - raise InvalidIDFComponent(f"Unsupported library frameworks: {frameworks}") + _LOGGER.warning( + "Library %s declares frameworks %s that do not include '%s'; including anyway", + data.get("name", ""), + frameworks, + framework, + ) def _process_dependencies(component: IDFComponent): @@ -699,6 +673,26 @@ def _process_dependencies(component: IDFComponent): if not dependencies: return + # PIO's library.json accepts both the list-of-dicts form and the + # shorthand dict form ``{"owner/Name": "version_spec"}``. Normalize + # the dict form so the loop below sees a uniform list. Iterating a + # dict gives string keys, which would silently fail the + # ``"name" in dependency`` substring check and skip every entry. + if isinstance(dependencies, dict): + normalized = [] + for raw_name, spec in dependencies.items(): + if "/" in raw_name: + owner, pkgname = raw_name.split("/", 1) + else: + owner, pkgname = None, raw_name + entry = {"name": pkgname, "owner": owner} + if isinstance(spec, dict): + entry.update(spec) + else: + entry["version"] = spec + normalized.append(entry) + dependencies = normalized + _LOGGER.info("Processing %s@%s component dependencies...", name, version) for dependency in dependencies: # Validate dependency structure @@ -748,7 +742,7 @@ def _parse_library_json(library_json_path: PathType): Returns: dict: Parsed JSON content as a Python dictionary. """ - with open(library_json_path, encoding="utf8") as fp: + with Path(library_json_path).open(encoding="utf8") as fp: return json.load(fp) @@ -762,7 +756,7 @@ def _parse_library_properties(library_properties_path: PathType): Returns: dict[str, str]: Mapping of parsed property keys to values. """ - with open(library_properties_path, encoding="utf8") as fp: + with Path(library_properties_path).open(encoding="utf8") as fp: data = {} for line in fp.read().splitlines(): line = line.strip() diff --git a/esphome/espidf/extra_script.py b/esphome/espidf/extra_script.py index 2f22f23c10..bead63ca21 100644 --- a/esphome/espidf/extra_script.py +++ b/esphome/espidf/extra_script.py @@ -108,7 +108,7 @@ def run_extra_script( """ env = _FakeSConsEnv(board_mcu=idf_target, pio_env=f"esphome_{idf_target}") code = compile(script_path.read_text(), str(script_path), "exec") - old_cwd = os.getcwd() + old_cwd = Path.cwd() try: os.chdir(library_dir) exec( # noqa: S102 pylint: disable=exec-used diff --git a/esphome/espidf/framework.py b/esphome/espidf/framework.py index 7ff373aba8..331c2f84b0 100644 --- a/esphome/espidf/framework.py +++ b/esphome/espidf/framework.py @@ -7,6 +7,8 @@ import json import logging import os from pathlib import Path +import platform +import re import shutil import subprocess import sys @@ -17,7 +19,7 @@ import requests from esphome.config_validation import Version from esphome.core import CORE -from esphome.helpers import ProgressBar, get_str_env, rmtree +from esphome.helpers import ProgressBar, get_str_env, rmtree, write_file_if_changed PathType = str | os.PathLike @@ -26,17 +28,19 @@ _LOGGER = logging.getLogger(__name__) _SCRIPTS_DIR = Path(__file__).parent -def _str_to_lst_of_str(a: str) -> list[str]: +def _str_to_lst_of_str(a: str | list[str]) -> list[str]: """ Convert a string to a list of string Args: - a: A string containing semicolon-separated values + a: A string containing semicolon-separated values, or an already-split list Returns: list of strings """ - return list(f.strip() for f in a.split(";") if f.strip()) + if isinstance(a, list): + return a + return [f.strip() for f in a.split(";") if f.strip()] ESPHOME_STAMP_FILE = ".esphome.stamp.json" @@ -67,10 +71,11 @@ ESPHOME_IDF_DEFAULT_FEATURES = _str_to_lst_of_str( ) ESPHOME_IDF_FRAMEWORK_MIRRORS = _str_to_lst_of_str( - os.environ.get( - "ESPHOME_IDF_FRAMEWORK_MIRRORS", - "https://github.com/espressif/esp-idf/releases/download/v{VERSION}/esp-idf-v{VERSION}.zip;https://github.com/espressif/esp-idf/releases/download/v{MAJOR}.{MINOR}/esp-idf-v{MAJOR}.{MINOR}.zip", - ) + os.environ.get("ESPHOME_IDF_FRAMEWORK_MIRRORS") + or [ + "https://github.com/esphome-libs/esp-idf/releases/download/v{VERSION}/esp-idf-v{VERSION}.tar.xz", + "https://github.com/esphome-libs/esp-idf/releases/download/v{MAJOR}.{MINOR}/esp-idf-v{MAJOR}.{MINOR}.tar.xz", + ] ) ESP_IDF_CONSTRAINTS_MIRRORS = _str_to_lst_of_str( @@ -133,7 +138,7 @@ def rmdir(directory: PathType, msg: str | None = None): Raises: RuntimeError: If directory removal fails """ - if os.path.isdir(directory): + if Path(directory).is_dir(): try: if msg: _LOGGER.debug(msg) @@ -187,7 +192,7 @@ def _check_stamp(file: PathType, data: dict[str, str]) -> bool: return False try: - with open(file, encoding="utf-8") as f: + with Path(file).open(encoding="utf-8") as f: return json.load(f) == data except (json.JSONDecodeError, OSError): return False @@ -201,7 +206,7 @@ def _write_stamp(file: PathType, data: dict[str, str]): file: Path to the stamp file to write data: Dictionary containing data to write """ - with open(file, "w", encoding="utf8") as fp: + with Path(file).open("w", encoding="utf8") as fp: json.dump(data, fp) @@ -466,8 +471,12 @@ def _tar_extract_all( import stat import tarfile + # Tar extraction safety: os.path.realpath / commonpath / normpath have no + # pathlib equivalents and Path.resolve() would follow symlinks unsafely. + # Use os.path for the security-sensitive parts; the simple checks move to + # Path. extract_dir = os.fspath(extract_dir) - abs_dest = os.path.abspath(extract_dir) + abs_dest = os.path.abspath(extract_dir) # noqa: PTH100 with tarfile.open(fileobj=data, mode="r") as tar_ref: all_members = tar_ref.getmembers() @@ -486,8 +495,8 @@ def _tar_extract_all( name = name.lstrip("/" + os.sep) # 2. Reject absolute paths (incl. Windows drive) - if os.path.isabs(name) or ( - os.name == "nt" and ":" in name.split(os.sep)[0] + if Path(name).is_absolute() or ( + os.name == "nt" and ":" in name.split(os.sep)[0] # noqa: PTH206 ): continue @@ -501,7 +510,7 @@ def _tar_extract_all( name = norm[len(strip_prefix) :] # 4. Compute final path - target_path = os.path.realpath(os.path.join(abs_dest, name)) + target_path = os.path.realpath(os.path.join(abs_dest, name)) # noqa: PTH118 if os.path.commonpath([abs_dest, target_path]) != abs_dest: continue @@ -510,18 +519,20 @@ def _tar_extract_all( linkname = member.linkname # Reject absolute link targets - if os.path.isabs(linkname): + if Path(linkname).is_absolute(): continue # Strip leading slashes linkname = os.path.normpath(linkname) if member.issym(): - link_target = os.path.join( - abs_dest, os.path.dirname(name), linkname + link_target = os.path.join( # noqa: PTH118 + abs_dest, + os.path.dirname(name), # noqa: PTH120 + linkname, ) else: - link_target = os.path.join(abs_dest, linkname) + link_target = os.path.join(abs_dest, linkname) # noqa: PTH118 link_target = os.path.realpath(link_target) if os.path.commonpath([abs_dest, link_target]) != abs_dest: @@ -546,11 +557,11 @@ def _tar_extract_all( if not (mode & stat.S_IXUSR): mode &= ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) mode |= stat.S_IRUSR | stat.S_IWUSR - elif member.isdir() or member.issym(): - # Ignore mode for directories & symlinks - mode = None - else: - # Block special files + elif not (member.isdir() or member.issym()): + # Block special files. Directories and symlinks keep + # their masked-original mode — passing None here would + # crash tarfile.extract on Python <3.12 (its chmod + # path calls os.chmod unconditionally). continue member.mode = mode @@ -593,7 +604,9 @@ def _zip_extract_all( """ import zipfile - extract_dir = os.path.abspath(extract_dir) + # See note in archive_extract_all_tar: os.path is used intentionally for + # the security-sensitive abspath/commonpath checks below. + extract_dir = os.path.abspath(extract_dir) # noqa: PTH100 with zipfile.ZipFile(data, "r") as zip_ref: all_members = zip_ref.infolist() @@ -613,8 +626,8 @@ def _zip_extract_all( name = member.filename.lstrip("/\\") # 2. Reject absolute paths / Windows drives - if os.path.isabs(name) or ( - os.name == "nt" and ":" in name.split(os.sep)[0] + if Path(name).is_absolute() or ( + os.name == "nt" and ":" in name.split(os.sep)[0] # noqa: PTH206 ): continue @@ -628,7 +641,7 @@ def _zip_extract_all( name = norm[len(strip_prefix) :] # 4. Compute safe target path - target_path = os.path.abspath(os.path.join(extract_dir, name)) + target_path = os.path.abspath(os.path.join(extract_dir, name)) # noqa: PTH100, PTH118 if os.path.commonpath([extract_dir, target_path]) != extract_dir: raise ValueError(f"Unsafe path detected: {member.filename}") @@ -675,7 +688,7 @@ def archive_extract_all( with ExitStack() as stack: archive_ref: io.BufferedIOBase if isinstance(archive, (str, os.PathLike)): - archive_ref = stack.enter_context(open(archive, "rb")) + archive_ref = stack.enter_context(Path(archive).open("rb")) elif isinstance(archive, (io.BufferedReader, io.BufferedRandom)): archive_ref = archive elif isinstance(archive, io.RawIOBase): @@ -722,7 +735,7 @@ def download_from_mirrors( # 1. Open target file for writing if path given with ExitStack() as stack: if isinstance(target, (str, os.PathLike)): - f = stack.enter_context(open(target, "wb")) + f = stack.enter_context(Path(target).open("wb")) elif isinstance(target, (io.RawIOBase, io.IOBase)): f = target else: @@ -780,12 +793,180 @@ def download_from_mirrors( return None +_GITHUB_SHORTHAND_RE = re.compile( + r"^github://([a-zA-Z0-9\-]+)/([a-zA-Z0-9\-\._]+?)(?:@([a-zA-Z0-9\-_.\./]+))?$" +) +_GITHUB_HTTPS_RE = re.compile( + r"^(https://github\.com/[a-zA-Z0-9\-]+/[a-zA-Z0-9\-\._]+?\.git)(?:@([a-zA-Z0-9\-_.\./]+))?$" +) + + +def _parse_git_source(source_url: str) -> tuple[str, str | None] | None: + """Return ``(url, ref)`` for ``github://owner/repo[@ref]`` or + ``https://github.com/owner/repo.git[@ref]``, else ``None``.""" + if m := _GITHUB_SHORTHAND_RE.match(source_url): + owner, repo, ref = m.group(1), m.group(2), m.group(3) + # Tolerate a trailing ".git" on the shorthand repo so the + # github://owner/repo.git form doesn't silently become repo.git.git. + repo = repo.removesuffix(".git") + return f"https://github.com/{owner}/{repo}.git", ref + if m := _GITHUB_HTTPS_RE.match(source_url): + return m.group(1), m.group(2) + return None + + +def _clone_idf_with_submodules( + framework_path: Path, git_url: str, ref: str | None +) -> None: + """Shallow-clone ESP-IDF with submodules into ``framework_path``. + + GitHub's archive zip strips submodules, so vendored components + (mbedtls, openthread, esptool, ...) come down empty and CMake fails. + + Uses clone + ``fetch FETCH_HEAD`` + ``reset --hard`` instead of + ``--branch``: ``--branch`` only accepts branch or tag names, but a + user can also point at a commit SHA. The fetch-then-reset pattern + handles branches, tags, and SHAs uniformly (mirrors the approach in + ``esphome.git.clone_or_update``). + """ + from esphome.git import run_git_command + + _LOGGER.info("Cloning ESP-IDF from %s%s", git_url, f"@{ref}" if ref else "") + run_git_command(["git", "clone", "--depth=1", "--", git_url, str(framework_path)]) + if ref: + run_git_command( + ["git", "fetch", "--depth=1", "--", "origin", ref], + git_dir=framework_path, + ) + run_git_command( + ["git", "reset", "--hard", "FETCH_HEAD"], + git_dir=framework_path, + ) + run_git_command( + [ + "git", + "submodule", + "update", + "--init", + "--recursive", + "--depth=1", + ], + git_dir=framework_path, + ) + + # Sanity-check the resulting tree. run_git_command only raises when + # stderr is non-empty, so a clone that silently produces no working + # tree would otherwise be marked extracted and stuck until + # ``esphome clean``. + if not (framework_path / "tools" / "idf_tools.py").is_file(): + raise RuntimeError( + f"Clone of {git_url} produced no usable ESP-IDF tree at {framework_path}" + ) + + +def _write_idf_version_txt(framework_path: Path, version: str) -> None: + """Write /version.txt if missing. + + IDF's build.cmake picks the version it embeds in the firmware (and + stamps onto the bootloader) in this order: ``${IDF_PATH}/version.txt`` + if present, else ``git describe`` against IDF_PATH, else the + ``IDF_VERSION_MAJOR/MINOR/PATCH`` triplet from ``tools/cmake/version.cmake``. + On a clean esphome-libs tarball ``.git`` is fully stripped, so + git_describe returns ``HEAD-HASH-NOTFOUND`` (falsy) and the triplet + wins -- correct by luck. But a *partial* ``.git`` (e.g. a custom + framework.source pointed at a real git URL where build artifacts + mark the tree dirty) makes git_describe return ``-dirty``, + which is what then gets baked into the bootloader. Dropping + version.txt forces the right answer regardless. + """ + version_txt = framework_path / "version.txt" + if version_txt.exists(): + return + try: + version_txt.write_text(f"v{version}\n", encoding="utf-8") + except OSError as e: + _LOGGER.warning( + "Could not write %s (%s); bootloader version string may be incorrect.", + version_txt, + e, + ) + + +# Backport of espressif/esp-idf#18272: every ESPHome-supported IDF release +# through v6.0 ships a tools.json whose ninja 1.12.1 entry has no +# ``linux-arm64`` source. ``idf_tools.py`` then either fails to find a +# matching binary or grabs the x86_64 one, which can't execute on +# aarch64. cmake is already populated across the same release range; we +# only need to inject ninja. Values lifted verbatim from the IDF v6.0.1 +# tools.json where the fix landed natively. +_NINJA_ARM64_BACKPORT: dict[str, dict[str, str | int]] = { + "1.12.1": { + "rename_dist": "ninja-linux-arm64-v1.12.1.zip", + "sha256": "5c25c6570b0155e95fce5918cb95f1ad9870df5768653afe128db822301a05a1", + "size": 121787, + "url": "https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux-aarch64.zip", + }, +} + + +def _patch_tools_json_for_linux_arm64(framework_path: Path) -> None: + """Inject ninja linux-arm64 entries into the framework's tools.json on aarch64. + + Idempotent: a tools.json that already has the entry, or a host that + isn't aarch64, is a no-op. Applied unconditionally on every install + check so a build dir extracted before the backport got fixed up + without forcing a clean. + """ + if platform.machine() != "aarch64": + return + + tools_json = framework_path / "tools" / "tools.json" + if not tools_json.is_file(): + return + + try: + with tools_json.open(encoding="utf-8") as f: + data = json.load(f) + except (json.JSONDecodeError, OSError) as e: + _LOGGER.warning( + "Could not parse %s for linux-arm64 backport (%s); " + "skipping. A clean reinstall of the framework directory " + "may be needed.", + tools_json, + e, + ) + return + + changed = False + for tool in data.get("tools", []): + if tool.get("name") != "ninja": + continue + for ver in tool.get("versions", []): + entry = _NINJA_ARM64_BACKPORT.get(ver.get("name")) + if entry is None or ver.get("linux-arm64"): + continue + ver["linux-arm64"] = entry + changed = True + + if changed: + # write_file_if_changed stages a tempfile in the destination dir + # and atomically replaces — safe against mid-write interruption + # and concurrent invocations. + write_file_if_changed(tools_json, json.dumps(data, indent=2) + "\n") + _LOGGER.info( + "Patched %s to add ninja linux-arm64 download " + "(espressif/esp-idf#18272 backport).", + tools_json, + ) + + def _check_esphome_idf_framework_install( version: str, targets: list[str], tools: list[str], force: bool = False, env: dict[str, str] | None = None, + source_url: str | None = None, ) -> tuple[Path, bool]: """ Check and install ESP-IDF framework. @@ -796,6 +977,11 @@ def _check_esphome_idf_framework_install( tools: list of tools to install force: If True, force reinstallation env: Optional dictionary of environment variables to set + source_url: Optional override URL for the framework tarball. Supports + the same ``{VERSION}`` / ``{MAJOR}`` / ``{MINOR}`` / ``{PATCH}`` / + ``{EXTRA}`` substitutions as ESPHOME_IDF_FRAMEWORK_MIRRORS. When + set, it replaces the default mirror list — no implicit fallback, + so a misspelled URL fails loudly. Returns: tuple of (framework_path, install_flag) @@ -817,6 +1003,10 @@ def _check_esphome_idf_framework_install( env_stamp_file = framework_path / ESPHOME_STAMP_FILE idf_tools_path = framework_path / "tools" / "idf_tools.py" _LOGGER.info("Checking ESP-IDF %s framework ...", version) + # Logged every invocation (not just on install) so the user can verify the + # override. A changed URL needs ``esphome clean`` to force a re-download. + if source_url: + _LOGGER.info("Using framework source override: %s", source_url) # 2. Download and extract the framework if not already extracted. # The marker is written last after extraction succeeds, so its presence @@ -829,28 +1019,44 @@ def _check_esphome_idf_framework_install( if install: rmdir(framework_path, msg=f"Clean up ESP-IDF {version} framework") - # Download in temporary file - with tempfile.NamedTemporaryFile() as tmp: - _LOGGER.info("Downloading ESP-IDF %s framework ...", version) + git_source = _parse_git_source(source_url) if source_url else None + if git_source is not None: + git_url, ref = git_source + _clone_idf_with_submodules(framework_path, git_url, ref) + else: + # Download in temporary file + with tempfile.NamedTemporaryFile() as tmp: + _LOGGER.info("Downloading ESP-IDF %s framework ...", version) - # Create substitutions for the URLs - substitutions = {"VERSION": version} - try: - ver = Version.parse(version) - substitutions["MAJOR"] = str(ver.major) - substitutions["MINOR"] = str(ver.minor) - substitutions["PATCH"] = str(ver.patch) - substitutions["EXTRA"] = ver.extra - except ValueError: - pass + # Create substitutions for the URLs + substitutions = {"VERSION": version} + try: + ver = Version.parse(version) + substitutions["MAJOR"] = str(ver.major) + substitutions["MINOR"] = str(ver.minor) + substitutions["PATCH"] = str(ver.patch) + substitutions["EXTRA"] = ver.extra + except ValueError: + pass - download_from_mirrors( - ESPHOME_IDF_FRAMEWORK_MIRRORS, substitutions, tmp.file - ) + mirrors = [source_url] if source_url else ESPHOME_IDF_FRAMEWORK_MIRRORS + download_from_mirrors(mirrors, substitutions, tmp.file) - _LOGGER.info("Extracting ESP-IDF %s framework ...", version) - archive_extract_all(tmp.file, framework_path, progress_header="Extracting") - extracted_marker.touch() + _LOGGER.info("Extracting ESP-IDF %s framework ...", version) + archive_extract_all( + tmp.file, framework_path, progress_header="Extracting" + ) + extracted_marker.touch() + + # Idempotent post-extract patch: written every invocation so a build + # dir extracted before this fix gets the file too, without forcing a + # clean. Skips when version.txt already exists. + _write_idf_version_txt(framework_path, version) + + # Apply the ninja linux-arm64 backport on every invocation, not just on + # fresh extracts — idempotent and cheap, and lets a build dir carrying + # a pre-patch tools.json get fixed up without forcing a clean. + _patch_tools_json_for_linux_arm64(framework_path) # 3. Check if the framework tools are the same and correctly installed if not install: @@ -1008,6 +1214,7 @@ def check_esp_idf_install( tools: list[str] | None = None, features: list[str] | None = None, force: bool = False, + source_url: str | None = None, ) -> tuple[Path, Path]: """ Check and install ESP-IDF framework and Python environment. @@ -1018,6 +1225,10 @@ def check_esp_idf_install( tools: list of tools to install features: Features to install force: If True, force reinstallation + source_url: Optional override URL for the framework tarball. When + set, it replaces the default mirror list (no fallback). Forwarded + to ``_check_esphome_idf_framework_install``; supports the same URL + substitutions. Returns: tuple of (framework_path, python_env_path) @@ -1040,7 +1251,7 @@ def check_esp_idf_install( # 1) Framework framework_path, installed = _check_esphome_idf_framework_install( - version, targets, tools, force=force, env=env + version, targets, tools, force=force, env=env, source_url=source_url ) features = features or ESPHOME_IDF_DEFAULT_FEATURES diff --git a/esphome/espidf/get_idf_tool_paths.py b/esphome/espidf/get_idf_tool_paths.py index 2e8859631d..7d99e629b1 100644 --- a/esphome/espidf/get_idf_tool_paths.py +++ b/esphome/espidf/get_idf_tool_paths.py @@ -10,6 +10,7 @@ not installed. import json import os +from pathlib import Path import sys from types import SimpleNamespace @@ -25,7 +26,7 @@ from idf_tools import ( g.idf_path = sys.argv[1] g.idf_tools_path = os.environ.get("IDF_TOOLS_PATH") -g.tools_json = os.path.join(g.idf_path, TOOLS_FILE) +g.tools_json = str(Path(g.idf_path) / TOOLS_FILE) tools_info = filter_tools_info(IDFEnv.get_idf_env(), load_tools_info()) args = SimpleNamespace(prefer_system=False) diff --git a/esphome/espidf/runner.py b/esphome/espidf/runner.py index 65df37c7b2..7c568db7be 100644 --- a/esphome/espidf/runner.py +++ b/esphome/espidf/runner.py @@ -66,6 +66,12 @@ FILTER_IDF_LINES: list[str] = [ # Drop the blank line rich emits after the note so the build log # doesn't end with an orphan gap before ESPHome's own status lines. r"\s*$", + # ESP-IDF shells out to ``git rev-parse`` to embed a commit hash; + # esphome-libs strips ``.git`` from the tarball so those probes fail + # noisily without affecting the build. + r"-- git rev-parse returned ", + r"fatal: not a git repository", + r"Stopping at filesystem boundary", ] @@ -85,6 +91,7 @@ def main() -> int: # ---- end sys.path fix-up ----------------------------------------------- import os + from pathlib import Path import re import runpy @@ -158,6 +165,12 @@ def main() -> int: self._line_buffer = "" def __getattr__(self, name: str): + # Hide ``buffer`` so consumers that use either + # ``getattr(stream, 'buffer', None)`` or + # ``hasattr(stream, 'buffer')`` see this as a text-only stream + # and skip writing raw bytes (which would bypass the filter). + if name == "buffer": + raise AttributeError(name) return getattr(self._stream, name) def isatty(self) -> bool: @@ -217,7 +230,7 @@ def main() -> int: # runpy.run_path does not do this automatically, but idf.py relies # on it to import its sibling modules (python_version_checker, # idf_py_actions, ...). - script_dir = os.path.dirname(os.path.abspath(script_path)) + script_dir = str(Path(script_path).resolve().parent) if script_dir not in sys.path: sys.path.insert(0, script_dir) diff --git a/esphome/espidf/size_summary.py b/esphome/espidf/size_summary.py index 9477e664b3..3ba0bf3b4d 100644 --- a/esphome/espidf/size_summary.py +++ b/esphome/espidf/size_summary.py @@ -94,9 +94,10 @@ def print_summary(size_json: Path, partitions_csv: Path | None) -> None: _LOGGER.debug("Skipping size summary: %s", e) return - dram = data.get("memory_types", {}).get("DRAM") or {} - ram_used = dram.get("used") - ram_total = dram.get("size") + memory_types = data.get("memory_types", {}) + ram_region = memory_types.get("DRAM") or memory_types.get("DIRAM") or {} + ram_used = ram_region.get("used") + ram_total = ram_region.get("size") if ram_total and ram_used is not None: print(f"RAM: {_format_bar(ram_used, ram_total)}") diff --git a/esphome/espidf/toolchain.py b/esphome/espidf/toolchain.py index e0bc5bb393..752f582e74 100644 --- a/esphome/espidf/toolchain.py +++ b/esphome/espidf/toolchain.py @@ -10,6 +10,7 @@ import shutil import subprocess from esphome.components.esp32.const import KEY_ESP32, KEY_FLASH_SIZE, KEY_IDF_VERSION +from esphome.const import CONF_FRAMEWORK, CONF_SOURCE from esphome.core import CORE, EsphomeError from esphome.espidf.framework import check_esp_idf_install, get_framework_env from esphome.espidf.size_summary import print_summary @@ -37,13 +38,27 @@ def _get_core_framework_version(): return str(CORE.data[KEY_ESP32][KEY_IDF_VERSION]) +def _get_framework_source_override() -> str | None: + """Return the user-supplied esp32.framework.source override, if any. + + The override lets a user point the IDF tarball download at a custom URL + (mirror, fork, local server). Substitutions like ``{VERSION}`` / + ``{MAJOR}`` etc. work the same as in the default mirror list. + """ + if CORE.config is None: + return None + return CORE.config.get(KEY_ESP32, {}).get(CONF_FRAMEWORK, {}).get(CONF_SOURCE) + + def _get_esphome_esp_idf_paths( version: str | None = None, ) -> tuple[os.PathLike, os.PathLike]: version = version or _get_core_framework_version() paths = _cache().paths if version not in paths: - paths[version] = check_esp_idf_install(version) + paths[version] = check_esp_idf_install( + version, source_url=_get_framework_source_override() + ) return paths[version] @@ -226,20 +241,21 @@ def has_outdated_files(): dependency_lock_path = CORE.relative_build_path("dependencies.lock") build_ninja_path = CORE.relative_build_path("build/build.ninja") - if not os.path.isdir(build_config_path) or not os.listdir(build_config_path): + if not build_config_path.is_dir() or not any(build_config_path.iterdir()): return True - if not os.path.isfile(cmakecache_txt_path): + if not cmakecache_txt_path.is_file(): return True - if not os.path.isfile(build_ninja_path): + if not build_ninja_path.is_file(): return True - if os.path.isfile(dependency_lock_path) and os.path.getmtime( - dependency_lock_path - ) > os.path.getmtime(build_ninja_path): + if ( + dependency_lock_path.is_file() + and dependency_lock_path.stat().st_mtime > build_ninja_path.stat().st_mtime + ): return True - cmakecache_txt_mtime = os.path.getmtime(cmakecache_txt_path) + cmakecache_txt_mtime = cmakecache_txt_path.stat().st_mtime return any( - os.path.getmtime(f) > cmakecache_txt_mtime + f.stat().st_mtime > cmakecache_txt_mtime for f in [sdkconfig_internal_path, idf_component_yml_path] if f.exists() ) @@ -437,7 +453,7 @@ def create_factory_bin() -> bool: return False try: - with open(flasher_args_path, encoding="utf-8") as f: + with flasher_args_path.open(encoding="utf-8") as f: flash_data = json.load(f) except (json.JSONDecodeError, OSError) as e: _LOGGER.error("Failed to read flasher_args.json: %s", e) diff --git a/esphome/espota2.py b/esphome/espota2.py index 701a125bcd..266702c142 100644 --- a/esphome/espota2.py +++ b/esphome/espota2.py @@ -517,7 +517,7 @@ def run_ota_impl_( continue _LOGGER.info("Connected to %s", sa[0]) - with open(filename, "rb") as file_handle: + with Path(filename).open("rb") as file_handle: try: perform_ota(sock, password, file_handle, filename, ota_type) except OTAError as err: diff --git a/esphome/git.py b/esphome/git.py index 0106f24845..094a6dae19 100644 --- a/esphome/git.py +++ b/esphome/git.py @@ -6,6 +6,7 @@ import logging from pathlib import Path import re import subprocess +import sys import urllib.parse import esphome.config_validation as cv @@ -72,8 +73,9 @@ def run_git_command(cmd: list[str], git_dir: Path | None = None) -> str: ) except FileNotFoundError as err: raise GitNotInstalledError( - "git is not installed but required for external_components.\n" - "Please see https://git-scm.com/book/en/v2/Getting-Started-Installing-Git for installing git" + "git is not installed. See " + "https://git-scm.com/book/en/v2/Getting-Started-Installing-Git " + "for installation instructions." ) from err if ret.returncode != 0 and ret.stderr: @@ -93,6 +95,92 @@ def _compute_destination_path(key: str, domain: str) -> Path: return base_dir / h.hexdigest()[:8] +def resolve_symlink_stub(repo_dir: Path, file_path: Path) -> Path | None: + """Return the symlink target if ``file_path`` is a Windows-checked-out symlink stub. + + On Windows, when ``core.symlinks=false`` (the default unless the user has + SeCreateSymbolicLinkPrivilege — i.e. Developer Mode or running elevated), + git materializes files with tree mode ``120000`` as plain text files + whose content is the literal symlink target path. Opening such a file + yields the target path string instead of the target's content. + + If ``file_path`` is one of those stubs, return the resolved target Path + inside ``repo_dir``. Otherwise return ``None`` and the caller should use + ``file_path`` as-is. + + Designed to be called *only* when normal access has already produced an + unexpected result (e.g. YAML parsed as a top-level scalar), so the + per-file ``git ls-files`` subprocess cost is paid only on the failure + path. Returns ``None`` on any error or check failure — it's purely a + best-effort recovery, never raises. + """ + # On non-Windows, git creates real symlinks; ordinary file access already + # transparently follows them. + if sys.platform != "win32": + return None + if file_path.is_symlink(): + return None + if not file_path.is_file(): + return None + + try: + rel = file_path.relative_to(repo_dir) + except ValueError: + return None + + try: + # ``git ls-files -s `` prints " \t" + # for that single entry, or empty if untracked. + out = run_git_command( + ["git", "ls-files", "-s", "--", rel.as_posix()], + git_dir=repo_dir, + ) + except GitException: + return None + + parts = out.split() + if not parts or parts[0] != "120000": + return None + + # Stubs are short ASCII relative paths. Decode defensively, and only + # strip the trailing newline git's checkout may append — preserving any + # whitespace that could be part of a valid target name. + try: + raw = file_path.read_bytes() + except OSError: + return None + try: + target_str = raw.decode("utf-8").rstrip("\r\n") + except UnicodeDecodeError: + return None + + # ``Path()`` and ``Path.resolve()`` can raise on malformed inputs (e.g. + # embedded NUL bytes from a hostile symlink blob, paths too long for the + # OS, or temporary I/O errors). Catch broadly — this helper is purely a + # best-effort recovery and must never raise. + try: + target_path = (file_path.parent / target_str).resolve() + repo_root_resolved = repo_dir.resolve() + except (OSError, ValueError, RuntimeError): + return None + + # ``Path.resolve()`` follows ``..``; re-verify containment afterwards. + try: + target_path.relative_to(repo_root_resolved) + except ValueError: + _LOGGER.warning( + "Refusing to follow symlink %s -> %s (escapes repository)", + file_path, + target_str, + ) + return None + + if not target_path.is_file(): + return None + + return target_path + + def clone_or_update( *, url: str, diff --git a/esphome/helpers.py b/esphome/helpers.py index d7ddb5c416..733474c9c9 100644 --- a/esphome/helpers.py +++ b/esphome/helpers.py @@ -385,7 +385,7 @@ def rmtree(path: Path | str) -> None: def _onerror(func, path, exc_info): if os.access(path, os.W_OK): raise exc_info[1].with_traceback(exc_info[2]) - os.chmod(path, stat.S_IWUSR | stat.S_IRUSR) + Path(path).chmod(stat.S_IWUSR | stat.S_IRUSR) func(path) # ``onerror`` is deprecated in 3.12 in favour of ``onexc`` (different @@ -512,7 +512,7 @@ def copy_file_if_changed(src: Path, dst: Path) -> bool: # -> delete file (it would be overwritten anyway), and try again # if that fails, use normal error handler with suppress(OSError): - os.unlink(dst) + Path(dst).unlink() shutil.copyfile(src, dst) return True diff --git a/esphome/idf_component.yml b/esphome/idf_component.yml index 35c55cbb4d..5af25fc351 100644 --- a/esphome/idf_component.yml +++ b/esphome/idf_component.yml @@ -2,7 +2,7 @@ dependencies: bblanchon/arduinojson: version: "7.4.2" esphome/esp-audio-libs: - version: 3.0.0 + version: 3.1.0 esphome/esp-micro-speech-features: version: 1.2.3 esphome/micro-decoder: @@ -36,7 +36,7 @@ dependencies: rules: - if: "target in [esp32h2, esp32p4]" espressif/esp_hosted: - version: 2.12.7 + version: 2.12.8 rules: - if: "target in [esp32h2, esp32p4]" zorxx/multipart-parser: @@ -100,6 +100,6 @@ dependencies: esp32async/asynctcp: version: 3.4.91 sendspin/sendspin-cpp: - version: 0.5.0 + version: 0.6.1 lvgl/lvgl: version: 9.5.0 diff --git a/esphome/loader.py b/esphome/loader.py index d50554f8c9..c57c09274e 100644 --- a/esphome/loader.py +++ b/esphome/loader.py @@ -239,12 +239,12 @@ def _lookup_module(domain: str, exception: bool) -> ComponentManifest | None: "Unable to import component %s: %s", domain, str(e), exc_info=False ) else: - _LOGGER.error("Unable to import component %s:", domain, exc_info=True) + _LOGGER.exception("Unable to import component %s:", domain) return None except Exception: # pylint: disable=broad-except if exception: raise - _LOGGER.error("Unable to load component %s:", domain, exc_info=True) + _LOGGER.exception("Unable to load component %s:", domain) return None manif = ComponentManifest(module) diff --git a/esphome/log.py b/esphome/log.py index bfd1875b55..b120c930d0 100644 --- a/esphome/log.py +++ b/esphome/log.py @@ -28,10 +28,12 @@ class AnsiFore(Enum): class AnsiStyle(Enum): + # BOLD/BRIGHT and THIN/DIM are intentional ANSI synonyms; Enum treats the + # second name in each pair as an alias of the first. BRIGHT = "\033[1m" - BOLD = "\033[1m" + BOLD = "\033[1m" # noqa: PIE796 DIM = "\033[2m" - THIN = "\033[2m" + THIN = "\033[2m" # noqa: PIE796 NORMAL = "\033[22m" RESET_ALL = "\033[0m" diff --git a/esphome/mqtt.py b/esphome/mqtt.py index ccacbaea54..d6bde0cbfd 100644 --- a/esphome/mqtt.py +++ b/esphome/mqtt.py @@ -2,7 +2,7 @@ import contextlib from datetime import datetime import json import logging -import os +from pathlib import Path import ssl import tempfile import time @@ -120,8 +120,8 @@ def prepare( key_file.close() context.load_cert_chain(cert_file.name, key_file.name) finally: - os.unlink(cert_file.name) - os.unlink(key_file.name) + Path(cert_file.name).unlink() + Path(key_file.name).unlink() client.tls_set_context(context) try: @@ -159,7 +159,7 @@ def get_esphome_device_ip( username: str | None = None, password: str | None = None, client_id: str | None = None, - timeout: int | float = 25, + timeout: float = 25, ) -> list[str]: if CONF_MQTT not in config: raise EsphomeError( diff --git a/esphome/pins.py b/esphome/pins.py index bdaa0e28ab..d6393508ab 100644 --- a/esphome/pins.py +++ b/esphome/pins.py @@ -272,9 +272,10 @@ def check_strapping_pin(conf, strapping_pin_list: set[int], logger: Logger): num = conf[CONF_NUMBER] if num in strapping_pin_list and not conf.get(CONF_IGNORE_STRAPPING_WARNING): logger.warning( - f"GPIO{num} is a strapping PIN and should only be used for I/O with care.\n" + "GPIO%s is a strapping PIN and should only be used for I/O with care.\n" "Attaching external pullup/down resistors to strapping pins can cause unexpected failures.\n" "See https://esphome.io/guides/faq/#why-am-i-getting-a-warning-about-strapping-pins", + num, ) # mitigate undisciplined use of strapping: if num not in strapping_pin_list and conf.get(CONF_IGNORE_STRAPPING_WARNING): @@ -313,9 +314,7 @@ def gpio_base_schema( :return: A schema for the pin """ mode_default = len(modes) == 1 - mode_dict = dict( - map(lambda m: (cv.Optional(m, default=mode_default), cv.boolean), modes) - ) + mode_dict = {cv.Optional(m, default=mode_default): cv.boolean for m in modes} def _number_validator(value): if isinstance(value, str) and value.upper().startswith("GPIOX"): diff --git a/esphome/platformio/toolchain.py b/esphome/platformio/toolchain.py index 073e134ac4..c81420e6ca 100644 --- a/esphome/platformio/toolchain.py +++ b/esphome/platformio/toolchain.py @@ -96,7 +96,7 @@ def _run_idedata(config): try: return json.loads(match.group()) except ValueError: - _LOGGER.error("Could not parse idedata", exc_info=True) + _LOGGER.exception("Could not parse idedata") _LOGGER.error("Stdout: %s", stdout) raise diff --git a/esphome/storage_json.py b/esphome/storage_json.py index 7d26b22f96..7f8885ba5f 100644 --- a/esphome/storage_json.py +++ b/esphome/storage_json.py @@ -14,6 +14,7 @@ from esphome.const import ( KEY_CORE, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM, + Toolchain, ) from esphome.core import CORE from esphome.helpers import write_file_if_changed @@ -98,6 +99,7 @@ class StorageJSON: no_mdns: bool, framework: str | None = None, core_platform: str | None = None, + toolchain: str | None = None, ) -> None: # Version of the storage JSON schema assert storage_version is None or isinstance(storage_version, int) @@ -134,6 +136,8 @@ class StorageJSON: self.framework = framework # The core platform of this firmware. Like "esp32", "rp2040", "host" etc. self.core_platform = core_platform + # The toolchain used for the build ("platformio" / "esp-idf") + self.toolchain = toolchain def as_dict(self): return { @@ -153,6 +157,7 @@ class StorageJSON: "no_mdns": self.no_mdns, "framework": self.framework, "core_platform": self.core_platform, + "toolchain": self.toolchain, } def to_json(self): @@ -189,6 +194,7 @@ class StorageJSON: ), framework=esph.target_framework, core_platform=esph.target_platform, + toolchain=esph.toolchain.value if esph.toolchain is not None else None, ) @staticmethod @@ -236,6 +242,7 @@ class StorageJSON: no_mdns = storage.get("no_mdns", False) framework = storage.get("framework") core_platform = storage.get("core_platform") + toolchain = storage.get("toolchain") return StorageJSON( storage_version, name, @@ -253,6 +260,7 @@ class StorageJSON: no_mdns, framework, core_platform, + toolchain, ) @staticmethod @@ -273,10 +281,33 @@ class StorageJSON: """ CORE.name = self.name CORE.build_path = self.build_path + # Restore toolchain so upload/logs picks the right firmware_bin path. + # An unknown value (corrupt sidecar, or written by a newer ESPHome) + # just leaves CORE.toolchain None — the fallback then picks PlatformIO. + if self.toolchain and CORE.toolchain is None: + try: + CORE.toolchain = Toolchain(self.toolchain) + except ValueError: + _LOGGER.debug( + "Ignoring unknown toolchain %r from %s", + self.toolchain, + storage_path(), + ) + target_platform = self.core_platform or self.target_platform.lower() CORE.data[KEY_CORE] = { - KEY_TARGET_PLATFORM: self.core_platform or self.target_platform.lower(), + KEY_TARGET_PLATFORM: target_platform, KEY_TARGET_FRAMEWORK: self.framework, } + # The compile pipeline populates CORE.data[KEY_ESP32] when esp32's + # validator runs; on the cache fast path that validator is skipped, + # so populate the variant upload_using_esptool reads via + # esp32.get_esp32_variant(). target_platform on disk is the variant + # (e.g. "ESP32S3"); core_platform is the family (e.g. "esp32"). + if target_platform == const.PLATFORM_ESP32: + from esphome.components.esp32.const import KEY_ESP32 + from esphome.const import KEY_VARIANT + + CORE.data[KEY_ESP32] = {KEY_VARIANT: self.target_platform} def __eq__(self, o) -> bool: return isinstance(o, StorageJSON) and self.as_dict() == o.as_dict() diff --git a/esphome/upload_targets.py b/esphome/upload_targets.py index 302ecf7301..d9d9713fc1 100644 --- a/esphome/upload_targets.py +++ b/esphome/upload_targets.py @@ -57,7 +57,7 @@ def get_port_type(port: str) -> PortType: """ if port == "BOOTSEL": return PortType.BOOTSEL - if port.startswith("/") or port.startswith("COM"): + if port.startswith(("/", "COM")): return PortType.SERIAL if port == "MQTT": return PortType.MQTT diff --git a/esphome/web_server_ota.py b/esphome/web_server_ota.py index 7c31c1b123..8d0fdeecff 100644 --- a/esphome/web_server_ota.py +++ b/esphome/web_server_ota.py @@ -126,7 +126,7 @@ def _try_upload( _LOGGER.info("Connecting to %s port %s...", ip, port) try: - with open(filename, "rb") as fh: + with filename.open("rb") as fh: streamer = _MultipartStreamer(fh, file_size, filename.name) try: response = requests.post( diff --git a/esphome/writer.py b/esphome/writer.py index 72c2c355dc..ab014c5daa 100644 --- a/esphome/writer.py +++ b/esphome/writer.py @@ -87,6 +87,21 @@ def replace_file_content(text, pattern, repl): def storage_should_clean(old: StorageJSON | None, new: StorageJSON) -> bool: + """Return True when the build tree must be wiped before reuse. + + Predicate is True when *old* is missing (first build), + ``src_version`` differs, ``build_path`` differs, or a previously + loaded integration was removed in *new*. Adding integrations or + changing unrelated fields (friendly name, esphome version, etc.) + does not trigger a clean. + + Used by esphome-device-builder (esphome/device-builder) to gate + its remote-build artifact materialiser so a local → remote → local + cycle preserves PlatformIO's local object cache instead of wiping + it on every cycle. The signature, semantics, and ``None`` handling + for *old* are part of the public contract; keep them stable so the + offloader's wipe decision tracks core's. + """ if old is None: return True @@ -343,7 +358,7 @@ def copy_src_tree(): platform = "esphome.components." + CORE.target_platform try: module = importlib.import_module(platform) - copy_files = getattr(module, "copy_files") + copy_files = module.copy_files copy_files() except AttributeError: pass diff --git a/esphome/yaml_util.py b/esphome/yaml_util.py index b56d024418..28f72ab831 100644 --- a/esphome/yaml_util.py +++ b/esphome/yaml_util.py @@ -763,15 +763,40 @@ def parse_yaml(file_name: Path, file_handle: TextIOWrapper, yaml_loader=None) -> def _load_yaml_internal_with_type( - loader_type: type[ESPHomeLoader] | type[ESPHomePurePythonLoader], + loader_type: type[ESPHomeLoader | ESPHomePurePythonLoader], fname: Path, content: TextIOWrapper, yaml_loader: Callable[[Path], dict[str, Any]], ) -> Any: - """Load a YAML file.""" + """Load a YAML file. + + Supports an optional leading YAML frontmatter document: when the file + contains two YAML documents separated by ``---``, the first document is + treated as metadata and stored in :attr:`CORE.frontmatter` keyed by the + resolved file path, while the second document is returned as the actual + configuration. Frontmatter is ignored by config validation and code + generation. + """ loader = loader_type(content, fname, yaml_loader) try: - return loader.get_single_data() or OrderedDict() + documents: list[Any] = [] + while loader.check_data(): + documents.append(loader.get_data()) + if len(documents) > 2: + raise EsphomeError( + f"YAML file '{fname}' contains {len(documents)} documents but " + f"at most two are supported (an optional frontmatter document " + f"followed by the configuration)." + ) + if len(documents) == 2: + frontmatter = documents[0] + config = documents[1] + if frontmatter is not None: + CORE.frontmatter[Path(fname).resolve()] = frontmatter + return config if config is not None else OrderedDict() + if len(documents) == 1: + return documents[0] or OrderedDict() + return OrderedDict() except yaml.YAMLError as exc: raise EsphomeError(exc) from exc finally: diff --git a/esphome/zeroconf.py b/esphome/zeroconf.py index 5d922ea911..a4f4f46097 100644 --- a/esphome/zeroconf.py +++ b/esphome/zeroconf.py @@ -249,7 +249,7 @@ async def async_resolve_hosts( ), return_exceptions=True, ) - for host, result in zip(pending, results): + for host, result in zip(pending, results, strict=True): if isinstance(result, BaseException): _LOGGER.debug("Failed to resolve %s: %s", host, result) diff --git a/pyproject.toml b/pyproject.toml index d16bf2b625..6572078746 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,16 +111,34 @@ exclude = ['generated'] [tool.ruff.lint] select = [ + "B", # flake8-bugbear + "C4", # flake8-comprehensions "E", # pycodestyle + "EXE", # flake8-executable "F", # pyflakes/autoflake + "FA", # flake8-future-annotations "FLY", # flynt: convert string formatting to f-strings "FURB", # refurb + "G", # flake8-logging-format "I", # isort + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "NPY", # numpy-specific rules "PERF", # performance + "PIE", # flake8-pie "PL", # pylint + "PTH", # flake8-use-pathlib + "PYI", # flake8-pyi + "Q", # flake8-quotes + "RSE", # flake8-raise "SIM", # flake8-simplify + "SLOT", # flake8-slots "RET", # flake8-ret + "T10", # flake8-debugger "UP", # pyupgrade + "W", # pycodestyle warnings + "YTT", # flake8-2020 ] ignore = [ diff --git a/requirements.txt b/requirements.txt index d9873081e4..608ca089a1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,19 +12,19 @@ platformio==6.1.19 esptool==5.2.0 click==8.3.3 esphome-dashboard==20260425.0 -aioesphomeapi==45.0.0 -zeroconf==0.148.0 +aioesphomeapi==45.2.2 +zeroconf==0.149.16 puremagic==1.30 ruamel.yaml==0.19.1 # dashboard_import ruamel.yaml.clib==0.2.15 # dashboard_import esphome-glyphsets==0.2.0 pillow==12.2.0 -resvg-py==0.3.1 +resvg-py==0.3.2 freetype-py==2.5.1 jinja2==3.1.6 bleak==2.1.1 smpclient==6.0.0 -requests==2.34.1 +requests==2.34.2 # zstd compression for store_yaml component (stdlib in 3.14+) backports.zstd==1.5.0; python_version < "3.14" diff --git a/requirements_test.txt b/requirements_test.txt index 9050132e70..102a9cae6e 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -1,6 +1,6 @@ pylint==4.0.5 flake8==7.3.0 # also change in .pre-commit-config.yaml when updating -ruff==0.15.13 # also change in .pre-commit-config.yaml when updating +ruff==0.15.14 # also change in .pre-commit-config.yaml when updating pyupgrade==3.21.2 # also change in .pre-commit-config.yaml when updating pre-commit @@ -16,7 +16,7 @@ hypothesis==6.92.1 # CodSpeed benchmarks under tests/benchmarks/python/ # (skipped via pytest.importorskip when missing -- only required for the # benchmarks job in .github/workflows/ci.yml) -pytest-codspeed==5.0.1 +pytest-codspeed==5.0.3 # Used by the import-time regression check (.github/workflows/ci.yml → import-time job) importtime-waterfall==1.0.0 diff --git a/script/api_protobuf/api_protobuf.py b/script/api_protobuf/api_protobuf.py index bf672d0567..451cd9ac1f 100755 --- a/script/api_protobuf/api_protobuf.py +++ b/script/api_protobuf/api_protobuf.py @@ -84,12 +84,7 @@ def indent_list(text: str, padding: str = " ") -> list[str]: """Indent each line of the given text with the specified padding.""" lines = [] for line in text.splitlines(): - if ( - line == "" - or line.startswith("#ifdef") - or line.startswith("#if ") - or line.startswith("#endif") - ): + if line == "" or line.startswith(("#ifdef", "#if ", "#endif")): p = "" else: p = padding @@ -1283,11 +1278,11 @@ class PackedBufferTypeInfo(TypeInfo): """Dump shows buffer info but not decoded values.""" return ( f'out.append(2, \' \').append_p(ESPHOME_PSTR("{self.name}")).append(": ");\n' - + 'out.append_p(ESPHOME_PSTR("packed buffer ["));\n' - + f"append_uint(out, this->{self.field_name}_count_);\n" - + 'out.append_p(ESPHOME_PSTR(" values, "));\n' - + f"append_uint(out, this->{self.field_name}_length_);\n" - + 'out.append_p(ESPHOME_PSTR(" bytes]\\n"));' + 'out.append_p(ESPHOME_PSTR("packed buffer ["));\n' + f"append_uint(out, this->{self.field_name}_count_);\n" + 'out.append_p(ESPHOME_PSTR(" values, "));\n' + f"append_uint(out, this->{self.field_name}_length_);\n" + 'out.append_p(ESPHOME_PSTR(" bytes]\\n"));' ) def dump(self, name: str) -> str: @@ -3163,7 +3158,7 @@ def main() -> None: defines_content += "\n" defines_content += "\nnamespace esphome::api {} // namespace esphome::api\n" - with open(root / "api_pb2_defines.h", "w", encoding="utf-8") as f: + with (root / "api_pb2_defines.h").open("w", encoding="utf-8") as f: f.write(defines_content) content = FILE_HEADER @@ -3448,13 +3443,13 @@ static void dump_bytes_field(DumpBuffer &out, const char *field_name, const uint #endif // HAS_PROTO_MESSAGE_DUMP """ - with open(root / "api_pb2.h", "w", encoding="utf-8") as f: + with (root / "api_pb2.h").open("w", encoding="utf-8") as f: f.write(content) - with open(root / "api_pb2.cpp", "w", encoding="utf-8") as f: + with (root / "api_pb2.cpp").open("w", encoding="utf-8") as f: f.write(cpp) - with open(root / "api_pb2_dump.cpp", "w", encoding="utf-8") as f: + with (root / "api_pb2_dump.cpp").open("w", encoding="utf-8") as f: f.write(dump_cpp) hpp = FILE_HEADER @@ -3551,7 +3546,7 @@ static const char *const TAG = "api.service"; if id_ is not None and not mt.options.deprecated: id_to_msg_name[id_] = mt.name - for id_, (_, _, case_label) in cases: + for id_, (_, _, _case_label) in cases: msg_name = id_to_msg_name.get(id_, "") if msg_name in message_auth_map: needs_auth = message_auth_map[msg_name] @@ -3614,7 +3609,7 @@ static const char *const TAG = "api.service"; # Dispatch switch out += " switch (msg_type) {\n" - for i, (case, ifdef, case_label) in cases: + for _i, (case, ifdef, case_label) in cases: if ifdef is not None: out += _make_ifdef_line(ifdef) + "\n" @@ -3641,10 +3636,10 @@ static const char *const TAG = "api.service"; } // namespace esphome::api """ - with open(root / "api_pb2_service.h", "w", encoding="utf-8") as f: + with (root / "api_pb2_service.h").open("w", encoding="utf-8") as f: f.write(hpp) - with open(root / "api_pb2_service.cpp", "w", encoding="utf-8") as f: + with (root / "api_pb2_service.cpp").open("w", encoding="utf-8") as f: f.write(cpp) prot_file.unlink() diff --git a/script/build_helpers.py b/script/build_helpers.py index fa722aa099..52f7ee317e 100644 --- a/script/build_helpers.py +++ b/script/build_helpers.py @@ -195,7 +195,7 @@ def load_component_yaml_configs(components: list[str], tests_dir: Path) -> dict: yaml_path = tests_dir / component / BENCHMARK_YAML_FILENAME if not yaml_path.is_file(): continue - with open(yaml_path) as f: + with yaml_path.open() as f: component_config = yaml.safe_load(f) if component_config and isinstance(component_config, dict): for key, value in component_config.items(): diff --git a/script/build_language_schema.py b/script/build_language_schema.py index 921ee9d3d7..9dff70af3c 100755 --- a/script/build_language_schema.py +++ b/script/build_language_schema.py @@ -972,7 +972,7 @@ def convert(schema, config_var, path): } elif schema_type == "use_id": if inspect.ismodule(data): - m_attr_obj = getattr(data, "CONFIG_SCHEMA") + m_attr_obj = data.CONFIG_SCHEMA use_schema = known_schemas.get(repr(m_attr_obj)) if use_schema: [output_module, output_name] = use_schema[0][1].split(".") diff --git a/script/bump-version.py b/script/bump-version.py index ed927cb991..e09fc87c60 100755 --- a/script/bump-version.py +++ b/script/bump-version.py @@ -2,6 +2,7 @@ import argparse from dataclasses import dataclass +from pathlib import Path import re import sys @@ -39,12 +40,12 @@ class Version: def sub(path, pattern, repl, expected_count=1): - with open(path, encoding="utf-8") as fh: + with Path(path).open(encoding="utf-8") as fh: content = fh.read() content, count = re.subn(pattern, repl, content, flags=re.MULTILINE) if expected_count is not None: assert count == expected_count, f"Pattern {pattern} replacement failed!" - with open(path, "w", encoding="utf-8") as fh: + with Path(path).open("w", encoding="utf-8") as fh: fh.write(content) diff --git a/script/ci-custom.py b/script/ci-custom.py index 56ca0d0355..1ac13e18f7 100755 --- a/script/ci-custom.py +++ b/script/ci-custom.py @@ -14,7 +14,7 @@ import time import colorama from helpers import filter_changed, git_ls_files, print_error_for_file, styled -sys.path.append(os.path.dirname(__file__)) +sys.path.append(str(Path(__file__).parent)) def find_all(a_str, sub): @@ -341,9 +341,9 @@ def lint_const_ordered(fname, content): matching = [ (i + 1, line) for i, line in enumerate(lines) if line.startswith(start) ] - ordered = list(sorted(matching, key=lambda x: x[1].replace("_", " "))) - ordered = [(mi, ol) for (mi, _), (_, ol) in zip(matching, ordered)] - for (mi, mline), (_, ol) in zip(matching, ordered): + ordered = sorted(matching, key=lambda x: x[1].replace("_", " ")) + ordered = [(mi, ol) for (mi, _), (_, ol) in zip(matching, ordered, strict=True)] + for (mi, mline), (_, ol) in zip(matching, ordered, strict=True): if mline == ol: continue target = next(i for i, line in ordered if line == mline) @@ -562,7 +562,7 @@ def lint_constants_usage(): # Maximum allowed CONF_ constants in esphome/const.py. # This file is frozen — new constants go in esphome/components/const/__init__.py. # Decrease this number when constants are moved out of const.py. -CONST_PY_MAX_CONF = 1012 +CONST_PY_MAX_CONF = 1013 @lint_content_check(include=["esphome/const.py"]) diff --git a/script/ci_add_metadata_to_json.py b/script/ci_add_metadata_to_json.py index 687b5131c0..e884e9a64c 100755 --- a/script/ci_add_metadata_to_json.py +++ b/script/ci_add_metadata_to_json.py @@ -44,7 +44,7 @@ def main() -> int: return 1 try: - with open(json_path, encoding="utf-8") as f: + with Path(json_path).open(encoding="utf-8") as f: data = json.load(f) except (json.JSONDecodeError, OSError) as e: print(f"Error loading JSON: {e}", file=sys.stderr) @@ -74,7 +74,7 @@ def main() -> int: # Write back try: - with open(json_path, "w", encoding="utf-8") as f: + with Path(json_path).open("w", encoding="utf-8") as f: json.dump(data, f, indent=2) print(f"Added metadata to {args.json_file}", file=sys.stderr) except OSError as e: diff --git a/script/ci_helpers.py b/script/ci_helpers.py old mode 100755 new mode 100644 index 48b0e4bbfe..a51a857ada --- a/script/ci_helpers.py +++ b/script/ci_helpers.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from pathlib import Path def write_github_output(outputs: dict[str, str | int]) -> None: @@ -16,7 +17,7 @@ def write_github_output(outputs: dict[str, str | int]) -> None: """ github_output = os.environ.get("GITHUB_OUTPUT") if github_output: - with open(github_output, "a", encoding="utf-8") as f: + with Path(github_output).open("a", encoding="utf-8") as f: f.writelines(f"{key}={value}\n" for key, value in outputs.items()) else: for key, value in outputs.items(): diff --git a/script/ci_memory_impact_comment.py b/script/ci_memory_impact_comment.py index 01316da27f..0908b99595 100755 --- a/script/ci_memory_impact_comment.py +++ b/script/ci_memory_impact_comment.py @@ -91,7 +91,7 @@ def load_analysis_json(json_path: str) -> dict | None: return None try: - with open(json_file, encoding="utf-8") as f: + with Path(json_file).open(encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError) as e: print(f"Failed to load analysis JSON: {e}", file=sys.stderr) diff --git a/script/ci_memory_impact_extract.py b/script/ci_memory_impact_extract.py index 2aa7394b11..feacc2b1af 100755 --- a/script/ci_memory_impact_extract.py +++ b/script/ci_memory_impact_extract.py @@ -127,7 +127,7 @@ def run_detailed_analysis(build_dir: str) -> dict | None: if not idedata_path.exists(): continue try: - with open(idedata_path, encoding="utf-8") as f: + with idedata_path.open(encoding="utf-8") as f: raw_data = json.load(f) idedata = IDEData(raw_data) print(f"Loaded idedata from: {idedata_path}", file=sys.stderr) @@ -264,7 +264,7 @@ def main() -> int: output_path = Path(args.output_json) output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, "w", encoding="utf-8") as f: + with output_path.open("w", encoding="utf-8") as f: json.dump(output_data, f, indent=2) print(f"Saved analysis to {args.output_json}", file=sys.stderr) diff --git a/script/clang-format b/script/clang-format index 028d752c55..df45798a30 100755 --- a/script/clang-format +++ b/script/clang-format @@ -2,6 +2,7 @@ import argparse import os +from pathlib import Path import queue import re import subprocess @@ -70,7 +71,7 @@ def main(): ) args = parser.parse_args() - cwd = os.getcwd() + cwd = Path.cwd() files = [ os.path.relpath(path, cwd) for path in git_ls_files(["*.cpp", "*.h", "*.tcc"]) ] diff --git a/script/clang-tidy b/script/clang-tidy index 1c413ffa23..56c0a9db71 100755 --- a/script/clang-tidy +++ b/script/clang-tidy @@ -2,6 +2,7 @@ import argparse import os +from pathlib import Path import queue import re import shutil @@ -32,7 +33,7 @@ def clang_options(idedata): cmd = [] # extract target architecture from triplet in g++ filename - triplet = os.path.basename(idedata["cxx_path"])[:-4] + triplet = Path(idedata["cxx_path"]).name[:-4] if triplet.startswith("xtensa-"): # clang doesn't support Xtensa (yet?), so compile in 32-bit mode and pretend we're the Xtensa compiler cmd.append("-m32") @@ -153,8 +154,8 @@ def run_tidy(executable, args, options, tmpdir, path_queue, lock, failed_files): if sys.stdout.isatty(): invocation.append("--use-color") - invocation.append(f"--header-filter={os.path.abspath(basepath)}/.*") - invocation.append(os.path.abspath(path)) + invocation.append(f"--header-filter={Path(basepath).resolve()}/.*") + invocation.append(str(Path(path).resolve())) invocation.append("--") invocation.extend(options) @@ -229,7 +230,7 @@ def main(): ) args = parser.parse_args() - cwd = os.getcwd() + cwd = Path.cwd() files = [os.path.relpath(path, cwd) for path in git_ls_files(["*.cpp"])] # Exclude benchmark files — they require google benchmark headers not # available in the ESP32 toolchain and use different naming conventions. diff --git a/script/clang_tidy_hash.py b/script/clang_tidy_hash.py index d0d8438437..f478535567 100755 --- a/script/clang_tidy_hash.py +++ b/script/clang_tidy_hash.py @@ -16,7 +16,7 @@ sys.path.insert(0, str(script_dir)) def read_file_lines(path: Path) -> list[str]: """Read lines from a file.""" - with open(path) as f: + with path.open() as f: return f.readlines() @@ -65,7 +65,7 @@ def get_clang_tidy_version_from_requirements(repo_root: Path | None = None) -> s def read_file_bytes(path: Path) -> bytes: """Read bytes from a file.""" - with open(path, "rb") as f: + with path.open("rb") as f: return f.read() @@ -120,7 +120,7 @@ def read_stored_hash(repo_root: Path | None = None) -> str | None: def write_file_content(path: Path, content: str) -> None: """Write content to a file.""" - with open(path, "w") as f: + with path.open("w") as f: f.write(content) diff --git a/script/determine-jobs.py b/script/determine-jobs.py index 3259fb5836..d91936952e 100755 --- a/script/determine-jobs.py +++ b/script/determine-jobs.py @@ -5,6 +5,7 @@ This script is a centralized way to determine which CI jobs need to run based on what files have changed. It outputs JSON with the following structure: { + "core_ci": true/false, "integration_tests": true/false, "integration_test_buckets": [{"name": "1/3", "tests": ["tests/integration/test_foo.py", ...]}, ...], "clang_tidy": true/false, @@ -22,6 +23,11 @@ what files have changed. It outputs JSON with the following structure: } The CI workflow uses this information to: +- Gate the unconditional jobs (ci-custom, pytest, pre-commit-ci-lite) via core_ci; + false when a pull_request only touches CI-irrelevant meta paths (other workflow + files, .github/actions/build-image/*, .yamllint, .github/dependabot.yml, docker/**) + so workflow-only PRs satisfy the required CI Status check without running the + unconditional jobs. Always true on non-pull_request events and under --force-all. - Skip or run integration tests - Skip or run clang-tidy (and whether to do a full scan) - Skip or run clang-format @@ -300,7 +306,7 @@ def _is_clang_tidy_full_scan() -> bool: """ try: result = subprocess.run( - [os.path.join(root_path, "script", "clang_tidy_hash.py"), "--check"], + [str(Path(root_path) / "script" / "clang_tidy_hash.py"), "--check"], capture_output=True, check=False, ) @@ -477,9 +483,7 @@ def should_run_device_builder(branch: str | None = None) -> bool: True if the device-builder downstream tests should run, False otherwise. """ target_branch = get_target_branch() - if target_branch and ( - target_branch.startswith("release") or target_branch.startswith("beta") - ): + if target_branch and (target_branch.startswith(("release", "beta"))): return False for file in changed_files(branch): @@ -712,6 +716,69 @@ def should_run_benchmarks(branch: str | None = None) -> bool: return any(get_component_from_path(f) in benchmarked_components for f in files) +# Files / path patterns whose changes alone don't warrant running the +# unconditional CI jobs (`ci-custom`, `pytest`, `pre-commit-ci-lite`). +# Single source of truth for what we treat as "CI-irrelevant" on +# pull_request events; ci.yml used to encode this in its own +# `pull_request.paths` filter, but that hid the required `CI Status` +# check on PRs that only touched these files (dependabot Action bumps, +# dependabot.yml edits, docker/ changes, etc.) and forced admin +# force-merges. +# +# ci.yml itself is deliberately *not* ignored — editing the CI workflow +# must still run CI. Workflows that have their own dedicated triggers +# (codeql.yml, ci-docker.yml, ...) are matched via the +# `.github/workflows/*.yml` prefix below and exclude ci.yml explicitly. +CI_IRRELEVANT_EXACT_FILES = frozenset( + { + ".yamllint", + ".github/dependabot.yml", + } +) + + +def _is_ci_irrelevant_path(path: str) -> bool: + """Whether a single changed path is irrelevant to the unconditional CI jobs.""" + if path in CI_IRRELEVANT_EXACT_FILES: + return True + # docker/** — all descendants + if path.startswith("docker/"): + return True + # .github/workflows/*.yml — top-level workflow files other than ci.yml + # (ci.yml itself must still trigger full CI when edited). + if path.startswith(".github/workflows/") and path.endswith(".yml"): + if path == ".github/workflows/ci.yml": + return False + if "/" not in path[len(".github/workflows/") :]: + return True + # .github/actions/build-image/* — direct children only, matches the + # single-star glob the workflow used to encode. + if path.startswith(".github/actions/build-image/"): + rest = path[len(".github/actions/build-image/") :] + if rest and "/" not in rest: + return True + return False + + +def should_run_core_ci(branch: str | None = None) -> bool: + """Determine if the unconditional CI jobs (ci-custom/pytest/pre-commit-ci-lite) should run. + + Returns False only when every changed file is in the CI-irrelevant set + above (see ``_is_ci_irrelevant_path``). Empty diffs return True so we + never accidentally skip CI when the diff probe fails. + + Args: + branch: Branch to compare against. If None, uses default. + + Returns: + True if the unconditional CI jobs should run, False otherwise. + """ + files = changed_files(branch) + if not files: + return True + return any(not _is_ci_irrelevant_path(f) for f in files) + + def _any_changed_file_endswith(branch: str | None, extensions: tuple[str, ...]) -> bool: """Check if a changed file ends with any of the specified extensions.""" return any(file.endswith(extensions) for file in changed_files(branch)) @@ -886,9 +953,7 @@ def detect_memory_impact_config( # all components at once would produce nonsensical memory impact results. # Memory impact analysis is most useful for focused PRs targeting dev. target_branch = get_target_branch() - if target_branch and ( - target_branch.startswith("release") or target_branch.startswith("beta") - ): + if target_branch and (target_branch.startswith(("release", "beta"))): print( f"Memory impact: Skipping analysis for target branch {target_branch} " f"(would try to build all components at once, giving nonsensical results)", @@ -978,7 +1043,7 @@ def detect_memory_impact_config( # Find common platforms supported by ALL components # This ensures we can build all components together in a merged config common_platforms = set(MEMORY_IMPACT_PLATFORM_PREFERENCE) - for component, platforms in component_platforms_map.items(): + for platforms in component_platforms_map.values(): common_platforms &= platforms # Select the most preferred platform from the common set @@ -1075,6 +1140,16 @@ def main() -> None: args = parser.parse_args() # Determine what should run + # core_ci gates the unconditional jobs in ci.yml (ci-custom, pytest, + # pre-commit-ci-lite). Non-pull_request events (push to dev/beta/release + # and merge_group) always run them so behavior like venv-cache saves on + # push to dev is preserved. + event_name = os.environ.get("GITHUB_EVENT_NAME", "") + run_core_ci = ( + True + if args.force_all or event_name != "pull_request" + else should_run_core_ci(args.branch) + ) if args.force_all: integration_run_all, integration_test_files = True, [] run_clang_tidy = True @@ -1232,7 +1307,7 @@ def main() -> None: # (no isolation, all components are groupable) target_branch = get_target_branch() is_release_branch = target_branch and ( - target_branch.startswith("release") or target_branch.startswith("beta") + target_branch.startswith(("release", "beta")) ) if is_release_branch: @@ -1255,6 +1330,7 @@ def main() -> None: component_test_batches = [] output: dict[str, Any] = { + "core_ci": run_core_ci, "integration_tests": run_integration, "integration_test_buckets": integration_test_buckets, "clang_tidy": run_clang_tidy, diff --git a/script/extract_automations.py b/script/extract_automations.py index 4e650ce25f..3cdfb5d32c 100755 --- a/script/extract_automations.py +++ b/script/extract_automations.py @@ -12,9 +12,9 @@ if __name__ == "__main__": components = get_components_with_dependencies(files, True) dump = { - "actions": sorted(list(ACTION_REGISTRY.keys())), - "conditions": sorted(list(CONDITION_REGISTRY.keys())), - "pin_providers": sorted(list(PIN_SCHEMA_REGISTRY.keys())), + "actions": sorted(ACTION_REGISTRY.keys()), + "conditions": sorted(CONDITION_REGISTRY.keys()), + "pin_providers": sorted(PIN_SCHEMA_REGISTRY.keys()), } print(json.dumps(dump, indent=2)) diff --git a/script/helpers.py b/script/helpers.py index cf82a89f93..9839e766e2 100644 --- a/script/helpers.py +++ b/script/helpers.py @@ -17,10 +17,10 @@ from typing import Any import colorama -root_path = os.path.abspath(os.path.normpath(os.path.join(__file__, "..", ".."))) -basepath = os.path.join(root_path, "esphome") -temp_folder = os.path.join(root_path, ".temp") -temp_header_file = os.path.join(temp_folder, "all-include.cpp") +root_path = str(Path(__file__).resolve().parent.parent) +basepath = str(Path(root_path) / "esphome") +temp_folder = str(Path(root_path) / ".temp") +temp_header_file = str(Path(temp_folder) / "all-include.cpp") # C++ file extensions used for clang-tidy and clang-format checks CPP_FILE_EXTENSIONS = (".cpp", ".h", ".hpp", ".cc", ".cxx", ".c", ".tcc") @@ -103,9 +103,7 @@ def get_component_from_path(file_path: str) -> str | None: Returns: Component name if path is in components or tests directory, None otherwise """ - if file_path.startswith(ESPHOME_COMPONENTS_PATH) or file_path.startswith( - ESPHOME_TESTS_COMPONENTS_PATH - ): + if file_path.startswith((ESPHOME_COMPONENTS_PATH, ESPHOME_TESTS_COMPONENTS_PATH)): parts = file_path.split("/") if len(parts) >= 3 and parts[2]: # Verify that parts[2] is actually a component directory, not a file @@ -160,7 +158,7 @@ def is_validate_only_file(test_file: Path) -> bool: ``esphome config`` only and skipped during compile. """ name = test_file.name - return name.startswith("validate.") or name.startswith("validate-") + return name.startswith(("validate.", "validate-")) @dataclass(frozen=True) @@ -339,8 +337,8 @@ def _get_github_event_data() -> dict | None: Parsed event data dictionary, or None if not available """ github_event_path = os.environ.get("GITHUB_EVENT_PATH") - if github_event_path and os.path.exists(github_event_path): - with open(github_event_path) as f: + if github_event_path and Path(github_event_path).exists(): + with Path(github_event_path).open() as f: return json.load(f) return None @@ -464,7 +462,8 @@ def _get_changed_files_from_command(command: list[str]) -> list[str]: raise Exception(f"Command failed: {' '.join(command)}\nstderr: {proc.stderr}") changed_files = splitlines_no_ends(proc.stdout) - changed_files = [os.path.relpath(f, os.getcwd()) for f in changed_files if f] + cwd = Path.cwd() + changed_files = [os.path.relpath(f, cwd) for f in changed_files if f] # noqa: PTH109 changed_files.sort() return changed_files @@ -499,7 +498,7 @@ def get_changed_components() -> list[str] | None: return None # Use list-components.py to get changed components - script_path = os.path.join(root_path, "script", "list-components.py") + script_path = str(Path(root_path) / "script" / "list-components.py") cmd = [script_path, "--changed"] try: @@ -619,7 +618,7 @@ def filter_changed(files: list[str]) -> list[str]: def filter_grep(files: list[str], value: list[str]) -> list[str]: matched = [] for file in files: - with open(file, encoding="utf-8") as handle: + with Path(file).open(encoding="utf-8") as handle: contents = handle.read() if any(v in contents for v in value): matched.append(file) diff --git a/script/lint-python b/script/lint-python index 18281c711e..e4b3314d2a 100755 --- a/script/lint-python +++ b/script/lint-python @@ -2,6 +2,7 @@ import argparse import os +from pathlib import Path import re import sys @@ -66,11 +67,12 @@ def main(): args = parser.parse_args() files = [] + cwd = Path.cwd() for path in git_ls_files(): filetypes = (".py",) - ext = os.path.splitext(path)[1] + ext = Path(path).suffix if ext in filetypes and path.startswith("esphome"): - path = os.path.relpath(path, os.getcwd()) + path = os.path.relpath(path, cwd) files.append(path) # Match against re file_name_re = re.compile("|".join(args.files)) diff --git a/script/split_components_for_ci.py b/script/split_components_for_ci.py index 0d10246bb4..7f06f50f48 100755 --- a/script/split_components_for_ci.py +++ b/script/split_components_for_ci.py @@ -295,7 +295,7 @@ def main() -> int: # Sort groups by signature for readability groupable_groups = [] isolated_groups = [] - for (platform, signature), group_comps in sorted(signature_groups.items()): + for (_platform, signature), group_comps in sorted(signature_groups.items()): if signature.startswith(ISOLATED_SIGNATURE_PREFIX): isolated_groups.append((signature, group_comps)) else: diff --git a/script/sync-device_class.py b/script/sync-device_class.py index 121c89b8f9..660142195a 100755 --- a/script/sync-device_class.py +++ b/script/sync-device_class.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +from pathlib import Path import re # pylint: disable=import-error @@ -34,10 +35,10 @@ DOMAINS = { def sub(path, pattern, repl): - with open(path, encoding="utf-8") as handle: + with Path(path).open(encoding="utf-8") as handle: content = handle.read() content = re.sub(pattern, repl, content, flags=re.MULTILINE) - with open(path, "w", encoding="utf-8") as handle: + with Path(path).open("w", encoding="utf-8") as handle: handle.write(content) diff --git a/script/test_build_components.py b/script/test_build_components.py index 43b71004eb..767b55c94b 100755 --- a/script/test_build_components.py +++ b/script/test_build_components.py @@ -297,7 +297,7 @@ def write_github_summary( test_results: List of all test results """ summary_content = format_github_summary(test_results, toolchain) - with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as f: + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a", encoding="utf-8") as f: f.write(summary_content) @@ -890,7 +890,7 @@ def run_grouped_component_tests( print("=" * 80 + "\n") # Execute grouped tests - for (platform, signature), components in grouped_components.items(): + for (platform, _signature), components in grouped_components.items(): # Only group if we have multiple components with same signature if len(components) <= 1: continue @@ -1055,7 +1055,7 @@ def test_components( # Create empty test files for each platform (or filtered platform) reference_tests: list[Path] = [] - for platform_name, base_file in platform_bases.items(): + for platform_name in platform_bases: if platform_filter and not platform_name.startswith(platform_filter): continue # Create an empty test file named to match the platform diff --git a/tests/component_tests/display/test_display_metadata.py b/tests/component_tests/display/test_display_metadata.py index e569754494..ef3f12cb73 100644 --- a/tests/component_tests/display/test_display_metadata.py +++ b/tests/component_tests/display/test_display_metadata.py @@ -2,6 +2,8 @@ from unittest.mock import patch +import pytest + from esphome.components.display import ( DisplayMetaData, add_metadata, @@ -74,8 +76,5 @@ def test_add_metadata_overwrites_existing(): def test_metadata_is_frozen(): """Test that DisplayMetaData instances are immutable (frozen dataclass).""" meta = DisplayMetaData(320, 240, True, False) - try: + with pytest.raises(AttributeError): meta.width = 640 - assert False, "Expected FrozenInstanceError" - except AttributeError: - pass diff --git a/tests/component_tests/light/test_effect_validation.py b/tests/component_tests/light/test_effect_validation.py index 579e92c62a..aab9072cc8 100644 --- a/tests/component_tests/light/test_effect_validation.py +++ b/tests/component_tests/light/test_effect_validation.py @@ -9,13 +9,17 @@ import pytest from esphome import config_validation as cv from esphome.components.light import ( + EffectCycleRef, EffectRef, _final_validate, _get_data, available_effects_str, find_effect_index, ) -from esphome.components.light.automation import _record_effect_ref +from esphome.components.light.automation import ( + _record_effect_cycle_ref, + _record_effect_ref, +) from esphome.config import Config, path_context from esphome.const import CONF_EFFECT, CONF_EFFECTS, CONF_ID, CONF_NAME from esphome.core import ID, Lambda @@ -215,6 +219,111 @@ def test_final_validate_drains_refs() -> None: fv.full_config.reset(token) +# --- _final_validate: EffectCycleRef --- + + +def _setup_cycle_final_validate( + cycle_refs: list[EffectCycleRef], + light_configs: list[ConfigType], + declare_ids: list[tuple[ID, list[str | int]]], +) -> Token: + """Set up CORE.data and fv.full_config for EffectCycleRef final_validate tests.""" + data = _get_data() + data.effect_cycle_refs = cycle_refs + + full_conf = Config() + full_conf["light"] = light_configs + for id_, path in declare_ids: + full_conf.declare_ids.append((id_, path)) + + return fv.full_config.set(full_conf) + + +def test_final_validate_cycle_accepts_light_with_effects() -> None: + """Cycle ref against a light with effects should not raise.""" + light_id = ID("led1", is_declaration=True) + token = _setup_cycle_final_validate( + cycle_refs=[ + EffectCycleRef(light_id=light_id, component_path=["esphome"]), + ], + light_configs=[{CONF_ID: light_id, CONF_EFFECTS: _make_effects("Fast Pulse")}], + declare_ids=[(light_id, ["light", 0, CONF_ID])], + ) + try: + _final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_final_validate_cycle_rejects_light_without_effects_key() -> None: + """Cycle ref against a light with no CONF_EFFECTS key should raise.""" + light_id = ID("led1", is_declaration=True) + token = _setup_cycle_final_validate( + cycle_refs=[ + EffectCycleRef(light_id=light_id, component_path=["esphome"]), + ], + light_configs=[{CONF_ID: light_id}], + declare_ids=[(light_id, ["light", 0, CONF_ID])], + ) + try: + with pytest.raises(cv.FinalExternalInvalid, match="no effects configured"): + _final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_final_validate_cycle_rejects_light_with_empty_effects() -> None: + """Cycle ref against a light with empty effects list should raise.""" + light_id = ID("led1", is_declaration=True) + token = _setup_cycle_final_validate( + cycle_refs=[ + EffectCycleRef(light_id=light_id, component_path=["esphome"]), + ], + light_configs=[{CONF_ID: light_id, CONF_EFFECTS: []}], + declare_ids=[(light_id, ["light", 0, CONF_ID])], + ) + try: + with pytest.raises(cv.FinalExternalInvalid, match="no effects configured"): + _final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_final_validate_cycle_unknown_light_id_skipped() -> None: + """Cycle refs to unknown light IDs should be silently skipped.""" + data = _get_data() + data.effect_cycle_refs = [ + EffectCycleRef( + light_id=ID("nonexistent", is_declaration=True), + component_path=["esphome"], + ) + ] + + full_conf = Config() + token = fv.full_config.set(full_conf) + try: + _final_validate({}) + finally: + fv.full_config.reset(token) + + +def test_final_validate_drains_cycle_refs() -> None: + """Cycle refs should be drained after validation to avoid redundant runs.""" + light_id = ID("led1", is_declaration=True) + token = _setup_cycle_final_validate( + cycle_refs=[ + EffectCycleRef(light_id=light_id, component_path=["esphome"]), + ], + light_configs=[{CONF_ID: light_id, CONF_EFFECTS: _make_effects("Fast Pulse")}], + declare_ids=[(light_id, ["light", 0, CONF_ID])], + ) + try: + _final_validate({}) + assert _get_data().effect_cycle_refs == [] + finally: + fv.full_config.reset(token) + + # --- _record_effect_ref --- @@ -278,3 +387,19 @@ def test_record_effect_ref_skips_no_effect_key() -> None: config: ConfigType = {CONF_ID: ID("led1", is_declaration=True)} _record_effect_ref(config) assert _get_data().effect_refs == [] + + +# --- _record_effect_cycle_ref --- + + +@pytest.mark.usefixtures("_path_ctx") +def test_record_effect_cycle_ref() -> None: + """Cycle-action config should be recorded with light_id and path.""" + light_id = ID("led1", is_declaration=True) + config: ConfigType = {CONF_ID: light_id} + result = _record_effect_cycle_ref(config) + assert result is config + data = _get_data() + assert len(data.effect_cycle_refs) == 1 + assert data.effect_cycle_refs[0].light_id is light_id + assert data.effect_cycle_refs[0].component_path == ["esphome"] diff --git a/tests/component_tests/lvgl/test_automation_schema_lazy.py b/tests/component_tests/lvgl/test_automation_schema_lazy.py new file mode 100644 index 0000000000..46430824f6 --- /dev/null +++ b/tests/component_tests/lvgl/test_automation_schema_lazy.py @@ -0,0 +1,71 @@ +"""Tests for lvgl automation_schema lazy validate_automation build.""" + +from __future__ import annotations + +from unittest.mock import patch + +import esphome.components.lvgl # noqa: F401 +from esphome.components.lvgl import schemas as lvgl_schemas +from esphome.components.lvgl.schemas import ( + WIDGET_TYPES, + _lazy_validate_automation, + automation_schema, +) +from esphome.components.lvgl.widgets import WidgetType +from esphome.config_validation import GenerateID, declare_id +from esphome.const import CONF_TRIGGER_ID +from esphome.core.config import StartupTrigger + + +def _widget_type(name: str = "obj") -> WidgetType: + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def _trigger_extra_schema() -> dict: + return {GenerateID(CONF_TRIGGER_ID): declare_id(StartupTrigger)} + + +def test_lazy_validator_defers_build_until_first_call() -> None: + with patch( + "esphome.components.lvgl.schemas.validate_automation", + wraps=lvgl_schemas.validate_automation, + ) as va_mock: + validator = _lazy_validate_automation(_trigger_extra_schema()) + assert va_mock.call_count == 0 + validator({"then": []}) + assert va_mock.call_count == 1 + validator({"then": []}) + assert va_mock.call_count == 1 + + +def test_eager_build_when_schema_extraction_enabled() -> None: + with ( + patch("esphome.components.lvgl.schemas.EnableSchemaExtraction", True), + patch( + "esphome.components.lvgl.schemas.validate_automation", + wraps=lvgl_schemas.validate_automation, + ) as va_mock, + ): + _lazy_validate_automation(_trigger_extra_schema()) + assert va_mock.call_count == 1 + + +def test_lazy_and_eager_produce_equivalent_validation() -> None: + extra = _trigger_extra_schema() + with patch("esphome.components.lvgl.schemas.EnableSchemaExtraction", True): + eager = _lazy_validate_automation(extra) + lazy = _lazy_validate_automation(_trigger_extra_schema()) + sample = {"then": []} + assert lazy(sample) == eager(sample) + + +def test_automation_schema_uses_lazy_validators() -> None: + wt = _widget_type("obj") + with patch( + "esphome.components.lvgl.schemas.validate_automation", + wraps=lvgl_schemas.validate_automation, + ) as va_mock: + automation_schema(wt.w_type) + assert va_mock.call_count == 0 diff --git a/tests/component_tests/lvgl/test_container_schema_cache.py b/tests/component_tests/lvgl/test_container_schema_cache.py new file mode 100644 index 0000000000..39e623d720 --- /dev/null +++ b/tests/component_tests/lvgl/test_container_schema_cache.py @@ -0,0 +1,87 @@ +"""Tests for container_schema() memoization and lazy build.""" + +from __future__ import annotations + +from collections.abc import Generator +from unittest.mock import patch + +import pytest + +from esphome import config_validation as cv +import esphome.components.lvgl # noqa: F401 +from esphome.components.lvgl import schemas as lvgl_schemas +from esphome.components.lvgl.schemas import WIDGET_TYPES, container_schema + + +@pytest.fixture(autouse=True) +def _clear_container_schema_cache() -> Generator[None]: + cache = getattr(lvgl_schemas, "_CONTAINER_SCHEMA_CACHE", None) + if cache is not None: + cache.clear() + yield + if cache is not None: + cache.clear() + + +def _widget_type(name: str = "obj"): + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def test_same_args_return_same_validator() -> None: + wt = _widget_type("obj") + assert container_schema(wt) is container_schema(wt) + + +def test_extras_none_vs_truthy_get_different_validators() -> None: + wt = _widget_type("obj") + no_extras = container_schema(wt) + extras = {cv.Optional("custom_extra"): cv.string} + assert no_extras is not container_schema(wt, extras) + + +def test_different_widget_types_get_different_validators() -> None: + assert container_schema(_widget_type("obj")) is not container_schema( + _widget_type("label") + ) + + +def test_schema_build_is_deferred_until_first_validation() -> None: + wt = _widget_type("obj") + with patch.object( + lvgl_schemas, "obj_schema", wraps=lvgl_schemas.obj_schema + ) as obj_schema_mock: + validator = container_schema(wt) + assert obj_schema_mock.call_count == 0 + validator({}) + assert obj_schema_mock.call_count == 1 + validator({}) + assert obj_schema_mock.call_count == 1 + + +def test_cached_validator_produces_equivalent_output() -> None: + wt = _widget_type("obj") + cached = container_schema(wt) + cached_result = cached({}) + lvgl_schemas._CONTAINER_SCHEMA_CACHE.clear() + reference = container_schema(wt) + assert cached is not reference + assert cached_result == reference({}) + + +def test_id_recycling_is_caught_by_identity_guard() -> None: + wt = _widget_type("obj") + real_extras = {cv.Optional("a"): cv.int_} + validator_a = container_schema(wt, real_extras) + + cache_key = (id(wt), id(real_extras)) + cached_entry = lvgl_schemas._CONTAINER_SCHEMA_CACHE[cache_key] + sentinel = {cv.Optional("a"): cv.int_} + lvgl_schemas._CONTAINER_SCHEMA_CACHE[cache_key] = ( + cached_entry[0], + sentinel, + cached_entry[2], + ) + + assert container_schema(wt, real_extras) is not validator_a diff --git a/tests/component_tests/lvgl/test_obj_schema_cache.py b/tests/component_tests/lvgl/test_obj_schema_cache.py new file mode 100644 index 0000000000..860ee211dd --- /dev/null +++ b/tests/component_tests/lvgl/test_obj_schema_cache.py @@ -0,0 +1,67 @@ +"""Tests for obj_schema() memoization.""" + +from __future__ import annotations + +from collections.abc import Generator + +import pytest + +import esphome.components.lvgl # noqa: F401 +from esphome.components.lvgl import schemas as lvgl_schemas +from esphome.components.lvgl.schemas import WIDGET_TYPES, obj_schema + + +@pytest.fixture(autouse=True) +def _clear_obj_schema_cache() -> Generator[None]: + cache = getattr(lvgl_schemas, "_OBJ_SCHEMA_CACHE", None) + if cache is not None: + cache.clear() + yield + if cache is not None: + cache.clear() + + +def _widget_type(name: str = "obj"): + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def test_same_widget_type_returns_same_schema() -> None: + wt = _widget_type("obj") + assert obj_schema(wt) is obj_schema(wt) + + +def test_different_widget_types_return_different_schemas() -> None: + assert obj_schema(_widget_type("obj")) is not obj_schema(_widget_type("label")) + + +def test_cache_is_populated_after_first_call() -> None: + wt = _widget_type("obj") + assert id(wt) not in lvgl_schemas._OBJ_SCHEMA_CACHE + obj_schema(wt) + assert id(wt) in lvgl_schemas._OBJ_SCHEMA_CACHE + + +def test_cached_schema_produces_equivalent_output() -> None: + wt = _widget_type("obj") + cached_result = obj_schema(wt)({}) + lvgl_schemas._OBJ_SCHEMA_CACHE.clear() + fresh_result = obj_schema(wt)({}) + assert cached_result == fresh_result + + +def test_id_recycling_is_caught_by_identity_guard() -> None: + wt = _widget_type("obj") + real_schema = obj_schema(wt) + + cached_widget_type, _ = lvgl_schemas._OBJ_SCHEMA_CACHE[id(wt)] + sentinel_schema = object() + lvgl_schemas._OBJ_SCHEMA_CACHE[id(wt)] = (cached_widget_type, sentinel_schema) + assert obj_schema(wt) is sentinel_schema + + other = _widget_type("label") + lvgl_schemas._OBJ_SCHEMA_CACHE[id(wt)] = (other, sentinel_schema) + rebuilt = obj_schema(wt) + assert rebuilt is not sentinel_schema + assert rebuilt is not real_schema diff --git a/tests/component_tests/lvgl/test_schema_dict_helpers.py b/tests/component_tests/lvgl/test_schema_dict_helpers.py new file mode 100644 index 0000000000..16714f54d7 --- /dev/null +++ b/tests/component_tests/lvgl/test_schema_dict_helpers.py @@ -0,0 +1,236 @@ +"""Tests for part_dict / obj_dict / part_schema / obj_schema mapping contracts. + +These guard the dict-merge refactor: the dict helpers must keep returning the +same logical mapping as the chained-extend version produced, and the +corresponding Schema(...) wrappers must accept and reject the same configs. +""" + +from __future__ import annotations + +from collections.abc import Generator + +import pytest +import voluptuous as vol + +from esphome import config_validation as cv +import esphome.components.lvgl +from esphome.components.lvgl import ( + _theme_schema, + defines as df, + schemas as lvgl_schemas, +) +from esphome.components.lvgl.schemas import ( + ALIGN_TO_SCHEMA, + FLAG_SCHEMA, + FULL_STYLE_SCHEMA, + STATE_SCHEMA, + STYLE_SCHEMA, + WIDGET_TYPES, + automation_schema, + obj_dict, + obj_schema, + part_dict, + part_schema, +) +from esphome.components.lvgl.types import LvType +from esphome.components.lvgl.widgets import WidgetType + + +@pytest.fixture(autouse=True) +def _clear_obj_dict_cache() -> Generator[None]: + cache = getattr(lvgl_schemas, "_OBJ_DICT_CACHE", None) + if cache is not None: + cache.clear() + # The lazily-built theme schema is cached on _build_theme_schema; clear it + # too so each test starts from a clean slate. + build_theme = getattr(esphome.components.lvgl, "_build_theme_schema", None) + if build_theme is not None and hasattr(build_theme, "cache_clear"): + build_theme.cache_clear() + yield + if cache is not None: + cache.clear() + if build_theme is not None and hasattr(build_theme, "cache_clear"): + build_theme.cache_clear() + + +def _marker_names(mapping) -> set[str]: + """Return the underlying string names of every voluptuous Marker key.""" + names: set[str] = set() + for key in mapping: + if isinstance(key, vol.Marker): + schema = key.schema + if isinstance(schema, str): + names.add(schema) + return names + + +def _widget_type(name: str = "obj"): + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def test_part_dict_includes_state_flag_and_part_keys() -> None: + parts = ("indicator", "knob") + keys = _marker_names(part_dict(parts)) + + assert {"indicator", "knob"} <= keys + assert _marker_names(STATE_SCHEMA.schema) <= keys + assert _marker_names(FLAG_SCHEMA.schema) <= keys + + +def test_obj_dict_extends_part_dict_with_align_automation_state_group() -> None: + wt = _widget_type("obj") + part_keys = _marker_names(part_dict(wt.parts)) + obj_keys = _marker_names(obj_dict(wt)) + + assert part_keys <= obj_keys + assert _marker_names(ALIGN_TO_SCHEMA) <= obj_keys + assert _marker_names(automation_schema(wt.w_type)) <= obj_keys + assert {"state", "group"} <= obj_keys + + +def test_obj_dict_is_memoized_by_widget_type() -> None: + wt = _widget_type("obj") + first = obj_dict(wt) + second = obj_dict(wt) + assert first is second + # Different widget type, different dict. + assert obj_dict(_widget_type("label")) is not first + + +def test_part_schema_round_trips_known_state_and_part_settings() -> None: + schema = part_schema(("indicator",)) + out = schema( + { + "bg_color": 0x112233, + "checked": {"bg_color": 0x445566}, + "indicator": {"bg_color": 0x778899}, + } + ) + assert out["bg_color"] == 0x112233 + assert out["checked"]["bg_color"] == 0x445566 + assert out["indicator"]["bg_color"] == 0x778899 + + +def test_part_schema_rejects_unknown_part() -> None: + schema = part_schema(("indicator",)) + with pytest.raises(vol.Invalid): + schema({"definitely_not_a_part": {}}) + + +@pytest.mark.parametrize("name", sorted(WIDGET_TYPES)) +def test_obj_schema_accepts_empty_config_for_every_widget_type(name: str) -> None: + obj_schema(_widget_type(name))({}) + + +def test_obj_schema_accepts_align_to_and_state_group() -> None: + schema = obj_schema(_widget_type("obj")) + out = schema( + { + df.CONF_ALIGN_TO: { + "id": "some_other_widget", + df.CONF_ALIGN: "TOP_LEFT", + }, + "state": {"checked": True}, + } + ) + assert out[df.CONF_ALIGN_TO][df.CONF_ALIGN] == "LV_ALIGN_TOP_LEFT" + assert out["state"]["checked"] is True + + +def test_obj_schema_rejects_unknown_top_level_key() -> None: + with pytest.raises(vol.Invalid): + obj_schema(_widget_type("obj"))({"definitely_not_a_real_key": 1}) + + +def test_part_schema_returns_cv_schema_for_extend_callers() -> None: + schema = part_schema(("indicator",)) + extended = schema.extend({cv.Optional("extra_key"): cv.string}) + out = extended({"extra_key": "value", "bg_color": 0xAABBCC}) + assert out["extra_key"] == "value" + assert out["bg_color"] == 0xAABBCC + + +def test_obj_schema_returns_cv_schema_for_extend_callers() -> None: + schema = obj_schema(_widget_type("obj")) + extended = schema.extend({cv.Optional("extra_key"): cv.string}) + extended({"extra_key": "value"}) + + +@pytest.mark.parametrize( + "schema", + [STATE_SCHEMA, FLAG_SCHEMA, STYLE_SCHEMA, FULL_STYLE_SCHEMA], +) +def test_spread_sources_carry_no_extra_schemas(schema: cv.Schema) -> None: + # part_dict / obj_dict reach into .schema and rebuild via cv.Schema(...), + # which silently drops _extra_schemas and any non-default extra/required. + # Lock the invariant so a future add_extra() on these sources fails CI + # instead of quietly removing validation from part/obj/theme schemas. + assert not schema._extra_schemas + assert schema.extra is vol.PREVENT_EXTRA + assert schema.required is False + + +def test_theme_schema_merges_obj_dict_and_full_style_props() -> None: + # _theme_schema is the riskiest merge: obj_dict(w) and FULL_STYLE_SCHEMA.schema + # share many STYLE_SCHEMA marker instances. Exercise the merged schema + # end-to-end with one key from each side (a STATE_SCHEMA part from obj_dict + # and a FULL_STYLE-only property) to lock the behaviour against future + # regressions in either source. + out = _theme_schema( + { + df.CONF_DARK_MODE: True, + "obj": { + "bg_color": 0x112233, + "checked": {"bg_color": 0x445566}, + df.CONF_PAD_ROW: 4, + df.CONF_GRID_CELL_X_ALIGN: "CENTER", + }, + } + ) + assert out[df.CONF_DARK_MODE] is True + obj_out = out["obj"] + assert obj_out["bg_color"] == 0x112233 + assert obj_out["checked"]["bg_color"] == 0x445566 + assert obj_out[df.CONF_PAD_ROW] == 4 + assert obj_out[df.CONF_GRID_CELL_X_ALIGN] == "LV_GRID_ALIGN_CENTER" + + +def test_theme_schema_self_heals_when_a_widget_type_is_registered_later() -> None: + # _build_theme_schema is functools.cached on a snapshot of WIDGET_TYPES. + # any_widget_schema explicitly supports external components registering + # widgets lazily, and the device builder revalidates in-process, so a + # widget registered after first use must invalidate the cached snapshot. + _theme_schema({df.CONF_DARK_MODE: True}) # populate the cache + + name = "test_self_heal_widget" + assert name not in WIDGET_TYPES + # is_mock=True skips registration side-effects; insert into WIDGET_TYPES + # manually so the next theme call sees the new entry. + WIDGET_TYPES[name] = WidgetType(name, LvType("test_fake_t"), (), is_mock=True) + try: + out = _theme_schema({df.CONF_DARK_MODE: False, name: {"bg_color": 0x010203}}) + assert out[name]["bg_color"] == 0x010203 + finally: + WIDGET_TYPES.pop(name, None) + + +@pytest.mark.parametrize( + "schema", + [STATE_SCHEMA, FLAG_SCHEMA, STYLE_SCHEMA, FULL_STYLE_SCHEMA], +) +def test_spread_sources_have_no_top_level_marker_defaults(schema: cv.Schema) -> None: + # _theme_schema merges obj_dict(w) with FULL_STYLE_SCHEMA.schema; on a key + # collision, dict-spread keeps the first source's marker (and its default) + # but the last source's value, whereas .extend() would take both from the + # later source. The two are equivalent today because the overlapping + # markers are the same instances (both derive from STYLE_SCHEMA) and none + # carry a top-level default. Lock that so a future divergent default would + # fail CI rather than silently drift the merged validation. + offenders = [ + marker.schema + for marker in schema.schema + if isinstance(marker, vol.Optional) and marker.default is not vol.UNDEFINED + ] + assert not offenders, f"top-level Optional with default: {offenders}" diff --git a/tests/component_tests/lvgl/test_update_action_lazy.py b/tests/component_tests/lvgl/test_update_action_lazy.py new file mode 100644 index 0000000000..7fcdc149cf --- /dev/null +++ b/tests/component_tests/lvgl/test_update_action_lazy.py @@ -0,0 +1,53 @@ +"""Tests for lvgl..update lazy schema build.""" + +from __future__ import annotations + +from unittest.mock import patch + +from esphome.automation import ACTION_REGISTRY +import esphome.components.lvgl # noqa: F401 +from esphome.components.lvgl.schemas import WIDGET_TYPES +from esphome.components.lvgl.widgets import _update_action_schema +from esphome.config_validation import Schema + + +def _widget_type(name: str = "obj"): + wt = WIDGET_TYPES.get(name) + assert wt is not None, f"widget type {name!r} not registered" + return wt + + +def test_registry_entry_uses_lazy_validator() -> None: + entry = ACTION_REGISTRY["lvgl.label.update"] + assert callable(entry.raw_schema) + assert not isinstance(entry.raw_schema, Schema) + + +def test_lazy_validator_defers_build_until_first_call() -> None: + wt = _widget_type("label") + with patch( + "esphome.components.lvgl.widgets._build_update_schema", + wraps=lambda w: Schema({}), + ) as build_mock: + validator = _update_action_schema(wt) + assert build_mock.call_count == 0 + validator({}) + assert build_mock.call_count == 1 + validator({}) + assert build_mock.call_count == 1 + + +def test_eager_build_when_schema_extraction_enabled() -> None: + wt = _widget_type("label") + with patch("esphome.components.lvgl.widgets.EnableSchemaExtraction", True): + result = _update_action_schema(wt) + assert isinstance(result, Schema) + + +def test_lazy_and_eager_produce_equivalent_validation() -> None: + wt = _widget_type("label") + with patch("esphome.components.lvgl.widgets.EnableSchemaExtraction", True): + eager = _update_action_schema(wt) + lazy = _update_action_schema(wt) + sample = {"id": "label_id"} + assert lazy(sample) == eager(sample) diff --git a/tests/component_tests/packages/test_packages.py b/tests/component_tests/packages/test_packages.py index 8c809c5e91..66f946a5bd 100644 --- a/tests/component_tests/packages/test_packages.py +++ b/tests/component_tests/packages/test_packages.py @@ -510,15 +510,9 @@ def test_package_merge_by_missing_id() -> None: ], } - error_raised = False - try: + with pytest.raises(cv.Invalid) as exc_info: packages_pass(config) - assert False, "Expected validation error for missing ID" - except cv.Invalid as err: - error_raised = True - assert err.path == [CONF_SENSOR, 2] - - assert error_raised + assert exc_info.value.path == [CONF_SENSOR, 2] def test_package_list_remove_by_id() -> None: diff --git a/tests/component_tests/time/__init__.py b/tests/component_tests/time/__init__.py new file mode 100644 index 0000000000..dc24f4e532 --- /dev/null +++ b/tests/component_tests/time/__init__.py @@ -0,0 +1 @@ +"""Tests for the time component.""" diff --git a/tests/component_tests/time/test_init.py b/tests/component_tests/time/test_init.py new file mode 100644 index 0000000000..44469cfe28 --- /dev/null +++ b/tests/component_tests/time/test_init.py @@ -0,0 +1,369 @@ +"""Tests for time component – ha-timezone branch changes. + +Covers: +- detect_tz() platform guard (returns None for unsupported platforms) +- detect_tz() result caching (avoids duplicate log messages) +- detect_tz() error paths (tzlocal None, tzdata missing) +- validate_tz() accepts/rejects POSIX timezone strings and IANA keys +- TIME_SCHEMA: timezone is now truly optional (was SplitDefault) +- homeassistant/time: USE_HOMEASSISTANT_TIMEZONE define emitted iff + CONF_TIMEZONE is absent from the config +""" + +from __future__ import annotations + +from unittest import mock + +import pytest + +from esphome.components.time import DOMAIN, TIME_SCHEMA, detect_tz, validate_tz +import esphome.config_validation as cv +from esphome.const import ( + CONF_ID, + CONF_TIMEZONE, + KEY_CORE, + KEY_TARGET_FRAMEWORK, + KEY_TARGET_PLATFORM, + Platform, + PlatformFramework, +) +from esphome.core import CORE, EsphomeError +from tests.component_tests.types import SetCoreConfigCallable + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# A minimal TZif v2/v3 file that encodes "EST5EDT" as the footer line. +# The binary content is not validated at this level – what matters is that +# _extract_tz_string() picks up the last-but-one newline-terminated line. +_FAKE_TZFILE = b"\x00" * 44 + b"TZif2\x00" * 1 + b"\n" + b"EST5EDT,M3.2.0,M11.1.0\n" + + +def _set_platform(platform: Platform) -> None: + """Set CORE.data so that CORE.target_platform returns *platform*.""" + CORE.data[KEY_CORE] = { + KEY_TARGET_PLATFORM: platform, + KEY_TARGET_FRAMEWORK: "arduino", + } + + +# --------------------------------------------------------------------------- +# detect_tz – platform guard +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "platform_framework", + [ + PlatformFramework.NRF52_ZEPHYR, + ], +) +def test_detect_tz_returns_none_for_unsupported_platform( + platform_framework: PlatformFramework, + set_core_config: SetCoreConfigCallable, +) -> None: + """detect_tz() must return None for platforms that do not support TZ auto-detection.""" + set_core_config(platform_framework) + result = detect_tz() + assert result is None + + +@pytest.mark.parametrize( + "platform_framework", + [ + PlatformFramework.ESP32_IDF, + PlatformFramework.ESP32_ARDUINO, + PlatformFramework.ESP8266_ARDUINO, + PlatformFramework.RP2040_ARDUINO, + PlatformFramework.BK72XX_ARDUINO, + PlatformFramework.RTL87XX_ARDUINO, + PlatformFramework.LN882X_ARDUINO, + PlatformFramework.HOST_NATIVE, + ], +) +def test_detect_tz_calls_tzlocal_for_supported_platform( + platform_framework: PlatformFramework, + set_core_config: SetCoreConfigCallable, +) -> None: + """detect_tz() must call tzlocal for every supported platform.""" + set_core_config(platform_framework) + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value="America/New_York", + ), + mock.patch( + "esphome.components.time._load_tzdata", + return_value=_FAKE_TZFILE, + ), + ): + result = detect_tz() + assert result is not None + assert isinstance(result, str) + assert len(result) > 0 + + +# --------------------------------------------------------------------------- +# detect_tz – caching +# --------------------------------------------------------------------------- + + +def test_detect_tz_caches_result( + set_core_config: SetCoreConfigCallable, + caplog: pytest.LogCaptureFixture, +) -> None: + """detect_tz() must cache the TZ string after the first call so that + subsequent invocations (e.g. when multiple time platforms are configured) + skip tzlocal and avoid duplicate INFO messages.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value="America/New_York", + ) as mock_tz, + mock.patch( + "esphome.components.time._load_tzdata", + return_value=_FAKE_TZFILE, + ) as mock_load, + ): + first = detect_tz() + second = detect_tz() + + assert first == second + # tzlocal and _load_tzdata must be called exactly once despite two detect_tz() calls + mock_tz.assert_called_once() + mock_load.assert_called_once() + + +def test_detect_tz_cache_stored_in_core_data( + set_core_config: SetCoreConfigCallable, +) -> None: + """The cached TZ string should be stored under CORE.data[DOMAIN][CONF_TIMEZONE].""" + set_core_config(PlatformFramework.ESP32_IDF) + + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value="Europe/London", + ), + mock.patch( + "esphome.components.time._load_tzdata", + return_value=_FAKE_TZFILE, + ), + ): + result = detect_tz() + + assert CORE.data.get(DOMAIN, {}).get(CONF_TIMEZONE) == result + + +def test_detect_tz_returns_pre_seeded_cache( + set_core_config: SetCoreConfigCallable, +) -> None: + """If CORE.data already has a cached TZ string, detect_tz() must return it + without calling tzlocal at all.""" + set_core_config(PlatformFramework.ESP32_IDF) + CORE.data[DOMAIN] = {CONF_TIMEZONE: "CET-1CEST,M3.5.0,M10.5.0/3"} + + with mock.patch("esphome.components.time.tzlocal.get_localzone_name") as mock_tz: + result = detect_tz() + + assert result == "CET-1CEST,M3.5.0,M10.5.0/3" + mock_tz.assert_not_called() + + +# --------------------------------------------------------------------------- +# detect_tz – error paths +# --------------------------------------------------------------------------- + + +def test_detect_tz_raises_when_tzlocal_returns_none( + set_core_config: SetCoreConfigCallable, +) -> None: + """detect_tz() must raise EsphomeError when the local timezone cannot be determined.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value=None, + ), + pytest.raises(EsphomeError, match="Could not automatically determine timezone"), + ): + detect_tz() + + +def test_detect_tz_raises_when_tzdata_not_found( + set_core_config: SetCoreConfigCallable, +) -> None: + """detect_tz() must raise EsphomeError when tzdata has no entry for the IANA key.""" + set_core_config(PlatformFramework.ESP32_IDF) + + with ( + mock.patch( + "esphome.components.time.tzlocal.get_localzone_name", + return_value="Antarctica/Troll", + ), + mock.patch( + "esphome.components.time._load_tzdata", + return_value=None, + ), + pytest.raises(EsphomeError, match="Could not automatically determine timezone"), + ): + detect_tz() + + +# --------------------------------------------------------------------------- +# validate_tz +# --------------------------------------------------------------------------- + + +def test_validate_tz_accepts_valid_posix_string() -> None: + """validate_tz() must accept a syntactically valid POSIX TZ string.""" + result = validate_tz("UTC0") + assert result == "UTC0" + + +def test_validate_tz_accepts_posix_string_with_dst() -> None: + """validate_tz() must accept a full POSIX TZ string with DST rules.""" + tz = "EST5EDT,M3.2.0,M11.1.0" + result = validate_tz(tz) + assert result == tz + + +def test_validate_tz_accepts_iana_key_and_converts() -> None: + """validate_tz() must accept an IANA timezone key and return the POSIX string.""" + with mock.patch( + "esphome.components.time._load_tzdata", + return_value=_FAKE_TZFILE, + ): + result = validate_tz("America/New_York") + + # Should have been converted from IANA to POSIX via _extract_tz_string + assert result == "EST5EDT,M3.2.0,M11.1.0" + + +def test_validate_tz_rejects_invalid_posix_string() -> None: + """validate_tz() must raise cv.Invalid for a malformed POSIX TZ string.""" + with pytest.raises(cv.Invalid, match="Invalid POSIX timezone string"): + validate_tz("NOTAVALIDTZ!!!") + + +def test_validate_tz_accepts_empty_string() -> None: + """An empty string is accepted by validate_tz() and signals 'disable timezone'.""" + result = validate_tz("") + assert result == "" + + +# --------------------------------------------------------------------------- +# TIME_SCHEMA – timezone is now cv.Optional (no SplitDefault) +# --------------------------------------------------------------------------- + + +def test_time_schema_timezone_is_optional( + set_core_config: SetCoreConfigCallable, +) -> None: + """TIME_SCHEMA must accept a config with no timezone key on a supported platform.""" + set_core_config(PlatformFramework.ESP32_IDF) + # Should not raise + config = TIME_SCHEMA({}) + assert CONF_TIMEZONE not in config + + +def test_time_schema_explicit_timezone_accepted( + set_core_config: SetCoreConfigCallable, +) -> None: + """TIME_SCHEMA must accept an explicit valid POSIX timezone on Arduino/IDF.""" + set_core_config(PlatformFramework.ESP32_IDF) + config = TIME_SCHEMA({CONF_TIMEZONE: "UTC0"}) + assert config[CONF_TIMEZONE] == "UTC0" + + +def test_time_schema_explicit_empty_timezone_accepted( + set_core_config: SetCoreConfigCallable, +) -> None: + """An empty timezone string (timezone-disable sentinel) must pass TIME_SCHEMA.""" + set_core_config(PlatformFramework.ESP32_IDF) + config = TIME_SCHEMA({CONF_TIMEZONE: ""}) + assert config[CONF_TIMEZONE] == "" + + +def test_time_schema_timezone_rejected_on_zephyr( + set_core_config: SetCoreConfigCallable, +) -> None: + """TIME_SCHEMA must reject a timezone value on Zephyr with the framework error. + + The platform check (cv.only_with_framework) must run BEFORE validate_tz so + that users receive an actionable "unsupported framework" message rather than a + confusing TZ-parsing error. + """ + set_core_config(PlatformFramework.NRF52_ZEPHYR) + with pytest.raises(cv.Invalid, match="only available with framework"): + TIME_SCHEMA({CONF_TIMEZONE: "UTC0"}) + + +def test_time_schema_invalid_tz_on_zephyr_gives_framework_error( + set_core_config: SetCoreConfigCallable, +) -> None: + """Even a syntactically invalid TZ string must produce the framework error on Zephyr. + + This specifically tests that cv.only_with_framework is evaluated before + validate_tz: if the order were reversed, an invalid POSIX string would + generate a misleading TZ-parsing error instead. + """ + set_core_config(PlatformFramework.NRF52_ZEPHYR) + with pytest.raises(cv.Invalid, match="only available with framework"): + TIME_SCHEMA({CONF_TIMEZONE: "NOTAVALIDTZ!!!"}) + + +# --------------------------------------------------------------------------- +# homeassistant/time: USE_HOMEASSISTANT_TIMEZONE define +# --------------------------------------------------------------------------- + + +@pytest.fixture +def mock_ha_cg(): + """Mock codegen functions used by homeassistant/time to_code.""" + with ( + mock.patch( + "esphome.components.homeassistant.time.cg.new_Pvariable", + return_value=mock.MagicMock(), + ), + mock.patch( + "esphome.components.homeassistant.time.cg.add_define", + ) as mock_add_define, + mock.patch( + "esphome.components.homeassistant.time.cg.register_component", + new_callable=mock.AsyncMock, + ), + mock.patch( + "esphome.components.homeassistant.time.time_.register_time", + new_callable=mock.AsyncMock, + ), + ): + yield mock_add_define + + +@pytest.mark.asyncio +async def test_ha_time_defines_ha_timezone_when_no_explicit_tz(mock_ha_cg) -> None: + """When CONF_TIMEZONE is absent from the config, to_code() must call + cg.add_define('USE_HOMEASSISTANT_TIMEZONE').""" + from esphome.components.homeassistant.time import to_code + + await to_code({CONF_ID: mock.MagicMock()}) + + mock_ha_cg.assert_any_call("USE_HOMEASSISTANT_TIMEZONE") + + +@pytest.mark.asyncio +async def test_ha_time_no_ha_timezone_define_when_explicit_tz(mock_ha_cg) -> None: + """When CONF_TIMEZONE is present in the config, to_code() must NOT call + cg.add_define('USE_HOMEASSISTANT_TIMEZONE').""" + from esphome.components.homeassistant.time import to_code + + await to_code({CONF_ID: mock.MagicMock(), CONF_TIMEZONE: "UTC0"}) + + define_calls = [call.args[0] for call in mock_ha_cg.call_args_list] + assert "USE_HOMEASSISTANT_TIME" in define_calls + assert "USE_HOMEASSISTANT_TIMEZONE" not in define_calls diff --git a/tests/components/api/common-base.yaml b/tests/components/api/common-base.yaml index 504c52a57b..ca86445777 100644 --- a/tests/components/api/common-base.yaml +++ b/tests/components/api/common-base.yaml @@ -120,12 +120,12 @@ api: lambda: 'return condition;' then: - logger.log: - format: "Condition true, value: %d" - args: ['value'] + format: "Condition true, value: %ld" + args: ['(long) value'] else: - logger.log: - format: "Condition false, value: %d" - args: ['value'] + format: "Condition false, value: %ld" + args: ['(long) value'] - logger.log: "After if/else" # Test nested IfAction (multiple ContinuationAction instances) - action: test_nested_if @@ -171,8 +171,8 @@ api: count: !lambda 'return count;' then: - logger.log: - format: "Repeat iteration: %d" - args: ['iteration'] + format: "Repeat iteration: %lu" + args: ['(unsigned long) iteration'] - logger.log: "After repeat" # Test combined continuations (if + while + repeat) - action: test_combined_continuations @@ -193,8 +193,8 @@ api: lambda: 'return id(api_continuation_test_counter) > 0;' then: - logger.log: - format: "Combined: repeat=%d, while=%d" - args: ['iteration', 'id(api_continuation_test_counter)'] + format: "Combined: repeat=%lu, while=%d" + args: ['(unsigned long) iteration', 'id(api_continuation_test_counter)'] - lambda: 'id(api_continuation_test_counter)--;' else: - logger.log: "Skipped loops" @@ -208,8 +208,8 @@ api: - api.respond: success: true - logger.log: - format: "Status response sent (call_id=%d)" - args: [call_id] + format: "Status response sent (call_id=%lu)" + args: ['(unsigned long) call_id'] - action: test_respond_status_error variables: @@ -229,8 +229,8 @@ api: value: float then: - logger.log: - format: "Optional response (call_id=%d, return_response=%d)" - args: [call_id, return_response] + format: "Optional response (call_id=%lu, return_response=%lu)" + args: ['(unsigned long) call_id', '(unsigned long) return_response'] - api.respond: data: !lambda |- root["sensor"] = sensor_name; @@ -264,8 +264,8 @@ api: input: string then: - logger.log: - format: "Only response (call_id=%d)" - args: [call_id] + format: "Only response (call_id=%lu)" + args: ['(unsigned long) call_id'] - api.respond: data: !lambda |- root["input"] = input; diff --git a/tests/components/audio_file/validate.esp32-idf.yaml b/tests/components/audio_file/validate.esp32-idf.yaml new file mode 100644 index 0000000000..085f853c8e --- /dev/null +++ b/tests/components/audio_file/validate.esp32-idf.yaml @@ -0,0 +1,11 @@ +audio_file: + - id: test_audio + file: + type: local + path: $component_dir/test.wav + +media_source: + - platform: audio_file + id: audio_file_source + # task_stack_in_psram: false must validate without a psram: component + task_stack_in_psram: false diff --git a/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml b/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml index 6c27bd35d0..df5b0123b5 100644 --- a/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml +++ b/tests/components/bluetooth_proxy/test.esp32-c6-idf.yaml @@ -1,6 +1,8 @@ <<: !include common.yaml esp32_ble_tracker: + +esp32_ble: max_connections: 9 bluetooth_proxy: diff --git a/tests/components/esp32_hosted/common.yaml b/tests/components/esp32_hosted/common.yaml index ab029e5064..332fe5b070 100644 --- a/tests/components/esp32_hosted/common.yaml +++ b/tests/components/esp32_hosted/common.yaml @@ -3,6 +3,7 @@ esp32_hosted: slot: 1 active_high: true reset_pin: GPIO15 + use_psram: true cmd_pin: GPIO13 clk_pin: GPIO12 d0_pin: GPIO11 diff --git a/tests/components/esphome/common.yaml b/tests/components/esphome/common.yaml index db75b08b38..93f82824e6 100644 --- a/tests/components/esphome/common.yaml +++ b/tests/components/esphome/common.yaml @@ -2,6 +2,8 @@ esphome: debug_scheduler: true platformio_options: board_build.flash_mode: dio + build_flags: + - "-DESPHOME_TEST_BUILD_FLAG" environment_variables: TEST_ENV_VAR: "test_value" BUILD_NUMBER: "12345" diff --git a/tests/components/speaker/spdif_mode.esp32-idf.yaml b/tests/components/i2s_audio/common-spdif_mode.yaml similarity index 52% rename from tests/components/speaker/spdif_mode.esp32-idf.yaml rename to tests/components/i2s_audio/common-spdif_mode.yaml index 4d6859feae..374a4bce1e 100644 --- a/tests/components/speaker/spdif_mode.esp32-idf.yaml +++ b/tests/components/i2s_audio/common-spdif_mode.yaml @@ -1,13 +1,3 @@ -substitutions: - i2s_bclk_pin: GPIO27 - i2s_lrclk_pin: GPIO26 - i2s_mclk_pin: GPIO25 - i2s_dout_pin: GPIO12 - spdif_data_pin: GPIO4 - -packages: - i2c: !include ../../test_build_components/common/i2c/esp32-idf.yaml - i2s_audio: - id: i2s_output @@ -20,6 +10,5 @@ speaker: use_apll: true timeout: 2s sample_rate: 48000 - bits_per_sample: 16bit channel: stereo i2s_mode: primary diff --git a/tests/components/i2s_audio/test-spdif_speaker.esp32-idf.yaml b/tests/components/i2s_audio/test-spdif_speaker.esp32-idf.yaml new file mode 100644 index 0000000000..a69d808d1d --- /dev/null +++ b/tests/components/i2s_audio/test-spdif_speaker.esp32-idf.yaml @@ -0,0 +1,8 @@ +substitutions: + i2s_bclk_pin: GPIO27 + i2s_lrclk_pin: GPIO26 + i2s_mclk_pin: GPIO25 + i2s_dout_pin: GPIO12 + spdif_data_pin: GPIO4 + +<<: !include common-spdif_mode.yaml diff --git a/tests/components/light/common.yaml b/tests/components/light/common.yaml index 044a8144fa..cd9b27768e 100644 --- a/tests/components/light/common.yaml +++ b/tests/components/light/common.yaml @@ -103,6 +103,16 @@ esphome: - light.turn_on: id: test_monochromatic_light effect: !lambda 'return iteration > 1 ? "Strobe" : "none";' + # Cycle through configured effects (skip "None") + - light.effect.next: test_monochromatic_light + - light.effect.previous: test_monochromatic_light + # Cycle through effects including "None" + - light.effect.next: + id: test_monochromatic_light + include_none: true + - light.effect.previous: + id: test_monochromatic_light + include_none: true - light.dim_relative: id: test_monochromatic_light relative_brightness: 5% diff --git a/tests/components/micro_wake_word/common.yaml b/tests/components/micro_wake_word/common.yaml index c051c8dd57..cd060c176e 100644 --- a/tests/components/micro_wake_word/common.yaml +++ b/tests/components/micro_wake_word/common.yaml @@ -1,3 +1,6 @@ +psram: + mode: quad + i2s_audio: i2s_lrclk_pin: GPIO18 i2s_bclk_pin: GPIO19 @@ -12,6 +15,7 @@ microphone: micro_wake_word: microphone: echo_microphone + task_stack_in_psram: true on_wake_word_detected: - logger.log: "Wake word detected" - micro_wake_word.stop: diff --git a/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp b/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp new file mode 100644 index 0000000000..36e0fc90b4 --- /dev/null +++ b/tests/components/mitsubishi_cn105/climate/mitsubishi_cn105_climate_tests.cpp @@ -0,0 +1,165 @@ +#include "../common.h" + +namespace esphome::mitsubishi_cn105::testing { + +TEST(MitsubishiCN105ClimateTests, SupportedSwingModeOffLeavesTraitsEmpty) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_OFF); + + EXPECT_FALSE(sut.traits().get_supports_swing_modes()); +} + +TEST(MitsubishiCN105ClimateTests, SupportedSwingModeVerticalExposesOffAndVertical) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_VERTICAL); + + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_OFF)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_VERTICAL)); + EXPECT_FALSE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_HORIZONTAL)); + EXPECT_FALSE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_BOTH)); +} + +TEST(MitsubishiCN105ClimateTests, SupportedSwingModeHorizontalExposesOffAndHorizontal) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_HORIZONTAL); + + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_OFF)); + EXPECT_FALSE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_VERTICAL)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_HORIZONTAL)); + EXPECT_FALSE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_BOTH)); +} + +TEST(MitsubishiCN105ClimateTests, SupportedSwingModeBothExposesAllExpectedModes) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_OFF)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_VERTICAL)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_HORIZONTAL)); + EXPECT_TRUE(sut.traits().supports_swing_mode(climate::CLIMATE_SWING_BOTH)); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesMapsVerticalSwingWhenSupported) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_VERTICAL); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::SWING; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::CENTER; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_VERTICAL); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesMapsHorizontalSwingWhenSupported) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_HORIZONTAL); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::AUTO; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::SWING; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_HORIZONTAL); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesMapsBothSwingWhenSupported) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::SWING; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::SWING; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_BOTH); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesMapsSwingOffWhenNoSwingActive) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::POSITION_3; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::CENTER; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_OFF); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesRemembersLastNonSwingPositions) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::POSITION_4; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::RIGHT; + + sut.apply_values_(); + + EXPECT_EQ(sut.last_non_swing_vane_mode_, MitsubishiCN105::VaneMode::POSITION_4); + EXPECT_EQ(sut.last_non_swing_wide_vane_mode_, MitsubishiCN105::WideVaneMode::RIGHT); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::SWING; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::SWING; + + sut.apply_values_(); + + EXPECT_EQ(sut.last_non_swing_vane_mode_, MitsubishiCN105::VaneMode::POSITION_4); + EXPECT_EQ(sut.last_non_swing_wide_vane_mode_, MitsubishiCN105::WideVaneMode::RIGHT); + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_BOTH); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesDoesNotOverwriteRememberedPositionWithUnknownValues) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_BOTH); + + sut.last_non_swing_vane_mode_ = MitsubishiCN105::VaneMode::POSITION_2; + sut.last_non_swing_wide_vane_mode_ = MitsubishiCN105::WideVaneMode::LEFT; + + sut.status().vane_mode = MitsubishiCN105::VaneMode::UNKNOWN; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::UNKNOWN; + + sut.apply_values_(); + + EXPECT_EQ(sut.last_non_swing_vane_mode_, MitsubishiCN105::VaneMode::POSITION_2); + EXPECT_EQ(sut.last_non_swing_wide_vane_mode_, MitsubishiCN105::WideVaneMode::LEFT); + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_OFF); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesIgnoresUnsupportedVerticalSwingState) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_HORIZONTAL); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::SWING; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::CENTER; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_OFF); +} + +TEST(MitsubishiCN105ClimateTests, ApplyValuesIgnoresUnsupportedHorizontalSwingState) { + TestableMitsubishiCN105Climate sut; + + sut.set_supported_swing_mode(climate::CLIMATE_SWING_VERTICAL); + + sut.status().vane_mode = MitsubishiCN105::VaneMode::AUTO; + sut.status().wide_vane_mode = MitsubishiCN105::WideVaneMode::SWING; + + sut.apply_values_(); + + EXPECT_EQ(sut.swing_mode, climate::CLIMATE_SWING_OFF); +} + +} // namespace esphome::mitsubishi_cn105::testing diff --git a/tests/components/mitsubishi_cn105/common.h b/tests/components/mitsubishi_cn105/common.h index 59b6203732..798f7283f6 100644 --- a/tests/components/mitsubishi_cn105/common.h +++ b/tests/components/mitsubishi_cn105/common.h @@ -8,6 +8,7 @@ #include #include "esphome/components/uart/uart_component.h" #include "esphome/components/mitsubishi_cn105/mitsubishi_cn105.h" +#include "esphome/components/mitsubishi_cn105/mitsubishi_cn105_climate.h" namespace esphome::mitsubishi_cn105::testing { @@ -44,6 +45,7 @@ class TestableMitsubishiCN105 : public MitsubishiCN105 { using MitsubishiCN105::State; using MitsubishiCN105::UpdateFlag; using MitsubishiCN105::state_; + using MitsubishiCN105::status_; using MitsubishiCN105::operation_start_ms_; using MitsubishiCN105::use_temperature_encoding_b_; using MitsubishiCN105::set_wide_vane_high_bit_; @@ -58,4 +60,13 @@ class TestableMitsubishiCN105 : public MitsubishiCN105 { void set_current_time(uint32_t ms) { test_loop_time_ms = ms; } }; +class TestableMitsubishiCN105Climate : public MitsubishiCN105Climate { + public: + using MitsubishiCN105Climate::apply_values_; + using MitsubishiCN105Climate::last_non_swing_vane_mode_; + using MitsubishiCN105Climate::last_non_swing_wide_vane_mode_; + + MitsubishiCN105::Status &status() { return static_cast(this->hp_).status_; } +}; + } // namespace esphome::mitsubishi_cn105::testing diff --git a/tests/components/mitsubishi_cn105/common.yaml b/tests/components/mitsubishi_cn105/common.yaml index 4b64f51261..5b9c3aaaf6 100644 --- a/tests/components/mitsubishi_cn105/common.yaml +++ b/tests/components/mitsubishi_cn105/common.yaml @@ -3,6 +3,9 @@ climate: id: ac name: "AC Test" uart_id: uart_bus + update_interval: 30s + current_temperature_min_interval: 120s + supported_swing_modes: BOTH esphome: on_boot: diff --git a/tests/components/mixer/common.yaml b/tests/components/mixer/common.yaml index e171b9499c..ef613b82bc 100644 --- a/tests/components/mixer/common.yaml +++ b/tests/components/mixer/common.yaml @@ -16,8 +16,12 @@ speaker: id: speaker_id dac_type: external i2s_dout_pin: ${dout_pin} + bits_per_sample: 32bit + channel: stereo - platform: mixer output_speaker: speaker_id + bits_per_sample: 32 + num_channels: 2 source_speakers: - id: source_speaker_1_id - id: source_speaker_2_id diff --git a/tests/components/router/common.yaml b/tests/components/router/common.yaml new file mode 100644 index 0000000000..f1239de3cb --- /dev/null +++ b/tests/components/router/common.yaml @@ -0,0 +1,40 @@ +esphome: + on_boot: + then: + - router.speaker.switch_output: + id: router_id + target_speaker: speaker_b_id + # id omitted: auto-resolved since there's a single router instance + - router.speaker.switch_output: + target_speaker: !lambda return id(speaker_a_id); + +i2s_audio: + i2s_lrclk_pin: ${a_lrclk_pin} + i2s_bclk_pin: ${a_bclk_pin} + +speaker: + - platform: i2s_audio + id: speaker_a_id + dac_type: external + i2s_dout_pin: ${a_dout_pin} + sample_rate: 48000 + bits_per_sample: 16bit + channel: stereo + - platform: i2s_audio + id: speaker_b_id + dac_type: external + i2s_dout_pin: ${b_dout_pin} + spdif_mode: true + use_apll: true + sample_rate: 48000 + bits_per_sample: 16bit + channel: stereo + i2s_mode: primary + - platform: router + id: router_id + output_speakers: + - speaker_a_id + - speaker_b_id + sample_rate: 48000 + bits_per_sample: 16 + num_channels: 2 diff --git a/tests/components/router/test.esp32-idf.yaml b/tests/components/router/test.esp32-idf.yaml new file mode 100644 index 0000000000..241a9a8903 --- /dev/null +++ b/tests/components/router/test.esp32-idf.yaml @@ -0,0 +1,7 @@ +substitutions: + a_lrclk_pin: GPIO4 + a_bclk_pin: GPIO5 + a_dout_pin: GPIO14 + b_dout_pin: GPIO19 + +<<: !include common.yaml diff --git a/tests/components/rp2040/test.rp2040-ard.yaml b/tests/components/rp2040/test.rp2040-ard.yaml index 1eb315a3b4..09531f914e 100644 --- a/tests/components/rp2040/test.rp2040-ard.yaml +++ b/tests/components/rp2040/test.rp2040-ard.yaml @@ -1,4 +1,5 @@ rp2040: + variant: rp2040 enable_full_printf: false logger: diff --git a/tests/components/rp2040/test.rp2040-pico2-ard.yaml b/tests/components/rp2040/test.rp2040-pico2-ard.yaml new file mode 100644 index 0000000000..c9d795840d --- /dev/null +++ b/tests/components/rp2040/test.rp2040-pico2-ard.yaml @@ -0,0 +1,6 @@ +rp2040: + variant: rp2350 + enable_full_printf: false + +logger: + level: VERBOSE diff --git a/tests/dashboard/test_web_server.py b/tests/dashboard/test_web_server.py index 626aea0216..0ee841e68c 100644 --- a/tests/dashboard/test_web_server.py +++ b/tests/dashboard/test_web_server.py @@ -1503,13 +1503,18 @@ async def test_websocket_refresh_command( ) -> None: """Test WebSocket refresh command triggers dashboard update.""" with patch("esphome.dashboard.web_server.DASHBOARD_SUBSCRIBER") as mock_subscriber: - mock_subscriber.request_refresh = Mock() + # Signal an asyncio.Event when request_refresh is invoked so the + # test can deterministically wait for the server-side handler to run + # instead of relying on a fixed sleep (flaky on Windows CI under load). + called = asyncio.Event() + mock_subscriber.request_refresh = Mock(side_effect=called.set) # Send refresh command await websocket_client.write_message(json.dumps({"event": "refresh"})) - # Give it a moment to process - await asyncio.sleep(0.01) + # Wait for the server to process the message and invoke request_refresh + async with asyncio.timeout(5): + await called.wait() # Verify request_refresh was called mock_subscriber.request_refresh.assert_called_once() diff --git a/tests/dashboard/test_web_server_paths.py b/tests/dashboard/test_web_server_paths.py index b596ebb581..efeafbf3b5 100644 --- a/tests/dashboard/test_web_server_paths.py +++ b/tests/dashboard/test_web_server_paths.py @@ -34,9 +34,7 @@ def test_get_base_frontend_path_dev_mode() -> None: # The function uses Path.resolve() which resolves symlinks # The actual function adds "/" to the path, so we simulate that test_path_with_slash = test_path if test_path.endswith("/") else test_path + "/" - expected = ( - Path(os.getcwd()) / test_path_with_slash / "esphome_dashboard" - ).resolve() + expected = (Path.cwd() / test_path_with_slash / "esphome_dashboard").resolve() assert result == expected @@ -62,9 +60,7 @@ def test_get_base_frontend_path_dev_mode_relative_path() -> None: # The function uses Path.resolve() which resolves symlinks # The actual function adds "/" to the path, so we simulate that test_path_with_slash = test_path if test_path.endswith("/") else test_path + "/" - expected = ( - Path(os.getcwd()) / test_path_with_slash / "esphome_dashboard" - ).resolve() + expected = (Path.cwd() / test_path_with_slash / "esphome_dashboard").resolve() assert result == expected assert result.is_absolute() @@ -157,7 +153,7 @@ def test_load_file_path(tmp_path: Path) -> None: test_file = tmp_path / "test.txt" test_file.write_bytes(b"test content") - with open(test_file, "rb") as f: + with test_file.open("rb") as f: content = f.read() assert content == b"test content" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index fb025ce427..a9c9e0686f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -79,7 +79,7 @@ def shared_platformio_cache() -> Generator[Path]: lock_file = Path.home() / ".esphome-integration-tests-init.lock" # Always acquire the lock to ensure cache is ready before proceeding - with open(lock_file, "w") as lock_fd: + with lock_file.open("w") as lock_fd: fcntl.flock(lock_fd.fileno(), fcntl.LOCK_EX) # Check if the native platform is installed (the actual indicator of a populated cache) @@ -407,8 +407,10 @@ async def wait_and_connect_api_client( # Wait for connection with timeout try: await asyncio.wait_for(connected_future, timeout=timeout) - except TimeoutError: - raise TimeoutError(f"Failed to connect to API after {timeout} seconds") + except TimeoutError as err: + raise TimeoutError( + f"Failed to connect to API after {timeout} seconds" + ) from err if return_disconnect_event: yield client, disconnect_event diff --git a/tests/integration/test_gpio_expander_cache.py b/tests/integration/test_gpio_expander_cache.py index e5f0f2818f..1d36ca3446 100644 --- a/tests/integration/test_gpio_expander_cache.py +++ b/tests/integration/test_gpio_expander_cache.py @@ -43,7 +43,7 @@ async def test_gpio_expander_cache( # ensure logs are in the expected order log_order = [ (digital_read_hw_pattern, 0), - [(digital_read_cache_pattern, i) for i in range(0, 8)], + [(digital_read_cache_pattern, i) for i in range(8)], (digital_read_hw_pattern, 8), [(digital_read_cache_pattern, i) for i in range(8, 16)], (digital_read_hw_pattern, 16), @@ -68,7 +68,7 @@ async def test_gpio_expander_cache( # uint16_t component tests (single bank of 16 pins) (uint16_read_hw_pattern, 0), # First pin triggers hw read [ - (uint16_read_cache_pattern, i) for i in range(0, 16) + (uint16_read_cache_pattern, i) for i in range(16) ], # All 16 pins return via cache # After cache reset (uint16_read_hw_pattern, 5), # First read after reset triggers hw diff --git a/tests/script/test_check_import_time.py b/tests/script/test_check_import_time.py index 223c58002c..528ca0701c 100644 --- a/tests/script/test_check_import_time.py +++ b/tests/script/test_check_import_time.py @@ -4,7 +4,6 @@ from __future__ import annotations import importlib.util import json -import os from pathlib import Path import sys from unittest.mock import patch @@ -13,12 +12,10 @@ import pytest # Load the script-under-test as `check_import_time` (it's a hyphenated path # inside `script/` that mirrors the existing `determine_jobs` pattern). -script_dir = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "..", "script") -) +script_dir = str((Path(__file__).parent / ".." / ".." / "script").resolve()) sys.path.insert(0, script_dir) spec = importlib.util.spec_from_file_location( - "check_import_time", os.path.join(script_dir, "check_import_time.py") + "check_import_time", str(Path(script_dir) / "check_import_time.py") ) check_import_time = importlib.util.module_from_spec(spec) spec.loader.exec_module(check_import_time) diff --git a/tests/script/test_determine_jobs.py b/tests/script/test_determine_jobs.py index 3fd5eada94..ac3c6424bf 100644 --- a/tests/script/test_determine_jobs.py +++ b/tests/script/test_determine_jobs.py @@ -3,7 +3,6 @@ from collections.abc import Generator import importlib.util import json -import os from pathlib import Path import sys from unittest.mock import Mock, call, patch @@ -11,9 +10,7 @@ from unittest.mock import Mock, call, patch import pytest # Add the script directory to Python path so we can import the module -script_dir = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "..", "script") -) +script_dir = str((Path(__file__).parent / ".." / ".." / "script").resolve()) sys.path.insert(0, script_dir) # Import helpers module for patching @@ -22,7 +19,7 @@ import helpers # noqa: E402 import script.helpers # noqa: E402 spec = importlib.util.spec_from_file_location( - "determine_jobs", os.path.join(script_dir, "determine-jobs.py") + "determine_jobs", str(Path(script_dir) / "determine-jobs.py") ) determine_jobs = importlib.util.module_from_spec(spec) spec.loader.exec_module(determine_jobs) @@ -775,6 +772,88 @@ def test_should_run_import_time_with_branch() -> None: mock_changed.assert_called_once_with("release") +@pytest.mark.parametrize( + ("path", "expected_result"), + [ + # Exact-file matches in the CI-irrelevant set. + (".yamllint", True), + (".github/dependabot.yml", True), + # Other top-level workflow files are irrelevant; ci.yml itself is not. + (".github/workflows/codeql.yml", True), + (".github/workflows/release.yml", True), + (".github/workflows/ci.yml", False), + # Nested files under workflows/ are not matched by the single-star glob. + (".github/workflows/matchers/gcc.json", False), + # build-image action: direct children only (single-star glob). + (".github/actions/build-image/action.yml", True), + (".github/actions/build-image/nested/file.yml", False), + # Other actions are CI-relevant. + (".github/actions/restore-python/action.yml", False), + # docker/** covers everything under docker/. + ("docker/Dockerfile", True), + ("docker/scripts/run.sh", True), + # Regular source files are CI-relevant. + ("esphome/__main__.py", False), + ("esphome/components/wifi/wifi_component.cpp", False), + ("README.md", False), + ("tests/script/test_determine_jobs.py", False), + ], +) +def test_is_ci_irrelevant_path(path: str, expected_result: bool) -> None: + """Test _is_ci_irrelevant_path mirrors the historic ci.yml path filter.""" + assert determine_jobs._is_ci_irrelevant_path(path) == expected_result + + +@pytest.mark.parametrize( + ("changed_files", "expected_result"), + [ + # Empty diffs default to True — don't accidentally skip CI on a + # broken probe. + ([], True), + # Any CI-relevant file flips the result to True. + (["esphome/__main__.py"], True), + (["esphome/components/wifi/wifi_component.cpp"], True), + (["README.md"], True), + # All-irrelevant diffs return False. + ([".github/workflows/codeql.yml"], False), + ( + [".github/workflows/codeql.yml", ".github/workflows/release.yml"], + False, + ), + ([".yamllint"], False), + ([".github/dependabot.yml"], False), + (["docker/Dockerfile"], False), + ( + [ + ".github/workflows/codeql.yml", + ".github/dependabot.yml", + "docker/Dockerfile", + ], + False, + ), + # Mixed diffs always trigger CI. + ( + [".github/workflows/codeql.yml", "esphome/__main__.py"], + True, + ), + # ci.yml itself is treated as CI-relevant. + ([".github/workflows/ci.yml"], True), + ], +) +def test_should_run_core_ci(changed_files: list[str], expected_result: bool) -> None: + """Test should_run_core_ci function.""" + with patch.object(determine_jobs, "changed_files", return_value=changed_files): + assert determine_jobs.should_run_core_ci() == expected_result + + +def test_should_run_core_ci_with_branch() -> None: + """Test should_run_core_ci passes the branch through to changed_files.""" + with patch.object(determine_jobs, "changed_files") as mock_changed: + mock_changed.return_value = [] + determine_jobs.should_run_core_ci("release") + mock_changed.assert_called_once_with("release") + + @pytest.mark.parametrize( ("changed_files", "expected_result"), [ @@ -1518,6 +1597,7 @@ def test_clang_tidy_mode_full_scan( mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, + mock_determine_cpp_unit_tests: Mock, mock_changed_files: Mock, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, @@ -1529,6 +1609,9 @@ def test_clang_tidy_mode_full_scan( mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False + # Without this mock, main() runs the real determine_cpp_unit_tests + # which loads the full component graph (~5s import of every component). + mock_determine_cpp_unit_tests.return_value = (False, []) # Mock changed_files to return no component files mock_changed_files.return_value = [] @@ -1584,6 +1667,7 @@ def test_clang_tidy_mode_targeted_scan( mock_should_run_clang_tidy: Mock, mock_should_run_clang_format: Mock, mock_should_run_python_linters: Mock, + mock_determine_cpp_unit_tests: Mock, mock_changed_files: Mock, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, @@ -1595,6 +1679,9 @@ def test_clang_tidy_mode_targeted_scan( mock_should_run_clang_tidy.return_value = True mock_should_run_clang_format.return_value = False mock_should_run_python_linters.return_value = False + # Without this mock, main() runs the real determine_cpp_unit_tests + # which loads the full component graph (~5s import of every component). + mock_determine_cpp_unit_tests.return_value = (False, []) # Create component names components = [f"comp{i}" for i in range(component_count)] @@ -2651,6 +2738,15 @@ def test_main_force_all_overrides_detection( return_value={"should_run": "false"}, ), patch.object(determine_jobs, "should_run_benchmarks", return_value=False), + # create_intelligent_batches scans every tests/components//*.yaml + # under --force-all (~2500 YAML loads, ~10s in CI). This test only + # asserts that main() routes to it and returns non-empty -- the + # batching logic itself has its own dedicated tests. + patch.object( + determine_jobs, + "create_intelligent_batches", + return_value=([["fake_batch"]], None), + ), ): determine_jobs.main() diff --git a/tests/script/test_helpers.py b/tests/script/test_helpers.py index 10f258aa83..82ff5e1411 100644 --- a/tests/script/test_helpers.py +++ b/tests/script/test_helpers.py @@ -12,9 +12,7 @@ import pytest from pytest import MonkeyPatch # Add the script directory to Python path so we can import helpers -sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "script")) -) +sys.path.insert(0, str((Path(__file__).parent / ".." / ".." / "script").resolve())) import helpers # noqa: E402 diff --git a/tests/script/test_test_helpers.py b/tests/script/test_test_helpers.py index 3149712563..a8100252da 100644 --- a/tests/script/test_test_helpers.py +++ b/tests/script/test_test_helpers.py @@ -1,6 +1,5 @@ """Unit tests for script/build_helpers.py manifest override and build helpers.""" -import os from pathlib import Path import sys import textwrap @@ -9,9 +8,7 @@ from unittest.mock import MagicMock, patch import pytest # Add the script directory to Python path so we can import build_helpers -sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "script")) -) +sys.path.insert(0, str((Path(__file__).parent / ".." / ".." / "script").resolve())) import build_helpers # noqa: E402 diff --git a/tests/test_build_components/build_components_base.esp32-c6-idf.yaml b/tests/test_build_components/build_components_base.esp32-c6-idf.yaml index 9dbc465ca2..4105481dc5 100644 --- a/tests/test_build_components/build_components_base.esp32-c6-idf.yaml +++ b/tests/test_build_components/build_components_base.esp32-c6-idf.yaml @@ -4,6 +4,7 @@ esphome: esp32: board: esp32-c6-devkitc-1 + flash_size: 8MB framework: type: esp-idf diff --git a/tests/test_build_components/build_components_base.esp32-s3-idf.yaml b/tests/test_build_components/build_components_base.esp32-s3-idf.yaml index ee209000e9..f3122f977e 100644 --- a/tests/test_build_components/build_components_base.esp32-s3-idf.yaml +++ b/tests/test_build_components/build_components_base.esp32-s3-idf.yaml @@ -7,6 +7,9 @@ esp32: variant: ESP32S3 framework: type: esp-idf + # Use custom partition table with larger app partition (3MB) + # Default IDF partitions only allow 1.75MB which is too small for grouped tests + partitions: ../partitions_testing.csv logger: level: VERY_VERBOSE diff --git a/tests/unit_tests/components/test_rp2040.py b/tests/unit_tests/components/test_rp2040.py index 25a9ade567..8e726933ed 100644 --- a/tests/unit_tests/components/test_rp2040.py +++ b/tests/unit_tests/components/test_rp2040.py @@ -1,6 +1,11 @@ -"""Tests for RP2040 component public helpers.""" +"""Tests for RP2040 component public helpers and variant detection.""" -from esphome.components.rp2040 import board_id_has_wifi +import pytest + +from esphome.components.rp2040 import _detect_variant, board_id_has_wifi +from esphome.components.rp2040.const import VARIANT_RP2040, VARIANT_RP2350 +import esphome.config_validation as cv +from esphome.const import CONF_BOARD, CONF_VARIANT def test_board_id_has_wifi_for_known_wifi_board() -> None: @@ -27,3 +32,61 @@ def test_board_id_has_wifi_for_unknown_board_returns_true() -> None: "no CYW43" guard at compile time. """ assert board_id_has_wifi("not-a-real-board-id") is True + + +def test_detect_variant_derives_variant_from_board() -> None: + """Board alone resolves to the matching variant.""" + result = _detect_variant({CONF_BOARD: "rpipicow"}) + assert result[CONF_BOARD] == "rpipicow" + assert result[CONF_VARIANT] == VARIANT_RP2040 + + +def test_detect_variant_derives_variant_from_rp2350_board() -> None: + """An RP2350 board resolves to ``RP2350``.""" + result = _detect_variant({CONF_BOARD: "rpipico2"}) + assert result[CONF_BOARD] == "rpipico2" + assert result[CONF_VARIANT] == VARIANT_RP2350 + + +def test_detect_variant_only_picks_default_board_rp2040() -> None: + """Variant alone picks Pico W as the canonical RP2040 board.""" + result = _detect_variant({CONF_VARIANT: VARIANT_RP2040}) + assert result[CONF_BOARD] == "rpipicow" + assert result[CONF_VARIANT] == VARIANT_RP2040 + + +def test_detect_variant_only_picks_default_board_rp2350() -> None: + """Variant alone picks Pico 2 W as the canonical RP2350 board.""" + result = _detect_variant({CONF_VARIANT: VARIANT_RP2350}) + assert result[CONF_BOARD] == "rpipico2w" + assert result[CONF_VARIANT] == VARIANT_RP2350 + + +def test_detect_variant_matching_explicit_variant_passes() -> None: + """Specifying both a board and the matching variant is allowed.""" + result = _detect_variant({CONF_BOARD: "rpipico2", CONF_VARIANT: VARIANT_RP2350}) + assert result[CONF_BOARD] == "rpipico2" + assert result[CONF_VARIANT] == VARIANT_RP2350 + + +def test_detect_variant_mismatched_variant_raises() -> None: + """Board/variant mismatch must be rejected and name the offending board.""" + with pytest.raises( + cv.Invalid, match=r"does not match the selected board 'rpipicow'" + ): + _detect_variant({CONF_BOARD: "rpipicow", CONF_VARIANT: VARIANT_RP2350}) + + +def test_detect_variant_unknown_board_without_variant_raises() -> None: + """Unknown board with no variant tells the user how to recover.""" + with pytest.raises(cv.Invalid, match="please specify the chip variant"): + _detect_variant({CONF_BOARD: "not-a-real-board"}) + + +def test_detect_variant_unknown_board_with_variant_passes() -> None: + """Unknown board + explicit variant is accepted (with a warning).""" + result = _detect_variant( + {CONF_BOARD: "not-a-real-board", CONF_VARIANT: VARIANT_RP2040} + ) + assert result[CONF_BOARD] == "not-a-real-board" + assert result[CONF_VARIANT] == VARIANT_RP2040 diff --git a/tests/unit_tests/components/test_time.py b/tests/unit_tests/components/test_time.py index 6325bfbe75..5ae9d787d6 100644 --- a/tests/unit_tests/components/test_time.py +++ b/tests/unit_tests/components/test_time.py @@ -70,11 +70,11 @@ def test_numeric_offset_slash() -> None: def test_star() -> None: - assert _parse_cron_part("*", 0, 59, {}) == set(range(0, 60)) + assert _parse_cron_part("*", 0, 59, {}) == set(range(60)) def test_question() -> None: - assert _parse_cron_part("?", 0, 59, {}) == set(range(0, 60)) + assert _parse_cron_part("?", 0, 59, {}) == set(range(60)) def test_range() -> None: diff --git a/tests/unit_tests/components/test_wifi.py b/tests/unit_tests/components/test_wifi.py index 71a14d7817..9598c1bdd8 100644 --- a/tests/unit_tests/components/test_wifi.py +++ b/tests/unit_tests/components/test_wifi.py @@ -3,8 +3,20 @@ import pytest from esphome.components.esp32 import const -from esphome.components.wifi import has_native_wifi, variant_has_wifi -from esphome.const import Platform +from esphome.components.wifi import ( + check_placeholder_credentials, + has_native_wifi, + variant_has_wifi, +) +from esphome.const import ( + CONF_AP, + CONF_NETWORKS, + CONF_SSID, + CONF_WIFI, + PLACEHOLDER_WIFI_SSID, + Platform, +) +from esphome.core import EsphomeError, Lambda @pytest.mark.parametrize( @@ -123,3 +135,65 @@ def test_has_native_wifi_esp32_without_variant_assumes_wifi() -> None: def test_has_native_wifi_rp2040_without_board_assumes_wifi() -> None: """RP2040 without a board id falls open to True (custom-board default).""" assert has_native_wifi(platform=Platform.RP2040) is True + + +def _wifi_config( + *, + networks: list[dict] | None = None, + ap: dict | None = None, +) -> dict: + """Build a minimal config dict matching the post-validation shape.""" + wifi: dict = {} + if networks is not None: + wifi[CONF_NETWORKS] = networks + if ap is not None: + wifi[CONF_AP] = ap + return {CONF_WIFI: wifi} + + +def test_check_placeholder_credentials_passes_with_real_ssid() -> None: + """A real SSID compiles without complaint.""" + config = _wifi_config(networks=[{CONF_SSID: "home_network"}]) + assert check_placeholder_credentials(config) is None + + +def test_check_placeholder_credentials_refuses_placeholder_ssid() -> None: + """The placeholder SSID is rejected with an actionable message.""" + config = _wifi_config(networks=[{CONF_SSID: PLACEHOLDER_WIFI_SSID}]) + with pytest.raises(EsphomeError) as exc_info: + check_placeholder_credentials(config) + message = str(exc_info.value) + assert "wifi.networks[0].ssid" in message + assert "secrets.yaml" in message + + +def test_check_placeholder_credentials_refuses_placeholder_in_second_network() -> None: + """Index reporting picks the placeholder out of a mixed network list.""" + config = _wifi_config( + networks=[ + {CONF_SSID: "home_network"}, + {CONF_SSID: PLACEHOLDER_WIFI_SSID}, + ], + ) + with pytest.raises(EsphomeError) as exc_info: + check_placeholder_credentials(config) + assert "wifi.networks[1].ssid" in str(exc_info.value) + + +def test_check_placeholder_credentials_refuses_placeholder_ap_ssid() -> None: + """An AP using the placeholder broadcast name is also refused.""" + config = _wifi_config(ap={CONF_SSID: PLACEHOLDER_WIFI_SSID}) + with pytest.raises(EsphomeError) as exc_info: + check_placeholder_credentials(config) + assert "wifi.ap.ssid" in str(exc_info.value) + + +def test_check_placeholder_credentials_no_wifi_passes() -> None: + """Ethernet-only / wifi-less configs skip the check entirely.""" + assert check_placeholder_credentials({}) is None + + +def test_check_placeholder_credentials_skips_template_ssid() -> None: + """A templated (Lambda) SSID is not a string and is skipped.""" + config = _wifi_config(networks=[{CONF_SSID: Lambda('return "x";')}]) + assert check_placeholder_credentials(config) is None diff --git a/tests/unit_tests/core/test_config.py b/tests/unit_tests/core/test_config.py index 4ce862315d..b5b35b5172 100644 --- a/tests/unit_tests/core/test_config.py +++ b/tests/unit_tests/core/test_config.py @@ -486,7 +486,7 @@ def test_preload_core_config_basic(setup_core: Path) -> None: assert CONF_BUILD_PATH in config[CONF_ESPHOME] # Verify default build path is "build/" build_path = config[CONF_ESPHOME][CONF_BUILD_PATH] - assert build_path.endswith(os.path.join("build", "test_device")) + assert build_path.endswith(str(Path("build") / "test_device")) def test_preload_core_config_with_build_path(setup_core: Path) -> None: @@ -523,7 +523,7 @@ def test_preload_core_config_env_build_path(setup_core: Path) -> None: assert "test_device" in config[CONF_ESPHOME][CONF_BUILD_PATH] # Verify it uses the env var path with device name appended build_path = config[CONF_ESPHOME][CONF_BUILD_PATH] - expected_path = os.path.join("/env/build", "test_device") + expected_path = str(Path("/env/build") / "test_device") assert build_path == expected_path or build_path == expected_path.replace( "/", os.sep ) @@ -739,7 +739,7 @@ async def test_add_includes_with_single_file( """Test add_includes copies a single header file to build directory.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create include file include_file = tmp_path / "my_header.h" @@ -769,7 +769,7 @@ async def test_add_includes_with_directory_unix( """Test add_includes copies all files from a directory on Unix.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create include directory with files include_dir = tmp_path / "includes" @@ -814,7 +814,7 @@ async def test_add_includes_with_directory_windows( """Test add_includes copies all files from a directory on Windows.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create include directory with files include_dir = tmp_path / "includes" @@ -856,7 +856,7 @@ async def test_add_includes_with_multiple_sources( """Test add_includes with multiple files and directories.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create various include sources single_file = tmp_path / "single.h" @@ -884,7 +884,7 @@ async def test_add_includes_empty_directory( """Test add_includes with an empty directory doesn't fail.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create empty directory empty_dir = tmp_path / "empty" @@ -906,7 +906,7 @@ async def test_add_includes_preserves_directory_structure_unix( """Test that add_includes preserves relative directory structure on Unix.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create nested directory structure lib_dir = tmp_path / "lib" @@ -940,7 +940,7 @@ async def test_add_includes_preserves_directory_structure_windows( """Test that add_includes preserves relative directory structure on Windows.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create nested directory structure lib_dir = tmp_path / "lib" @@ -973,7 +973,7 @@ async def test_add_includes_overwrites_existing_files( """Test that add_includes overwrites existing files in build directory.""" CORE.config_path = tmp_path / "config.yaml" CORE.build_path = tmp_path / "build" - os.makedirs(CORE.build_path, exist_ok=True) + CORE.build_path.mkdir(parents=True, exist_ok=True) # Create include file include_file = tmp_path / "header.h" diff --git a/tests/unit_tests/test_compiled_config.py b/tests/unit_tests/test_compiled_config.py index 34e811b97b..e12107152b 100644 --- a/tests/unit_tests/test_compiled_config.py +++ b/tests/unit_tests/test_compiled_config.py @@ -22,6 +22,7 @@ from esphome.const import ( KEY_CORE, KEY_TARGET_FRAMEWORK, KEY_TARGET_PLATFORM, + KEY_VARIANT, ) from esphome.core import CORE @@ -47,7 +48,12 @@ wifi: """ -def _write_storage(storage_path: Path) -> None: +def _write_storage( + storage_path: Path, + *, + esp_platform: str = "ESP32", + core_platform: str | None = "esp32", +) -> None: """Write a vanilla StorageJSON sidecar for the cache tests.""" storage_path.parent.mkdir(parents=True, exist_ok=True) data = { @@ -59,14 +65,14 @@ def _write_storage(storage_path: Path) -> None: "src_version": 1, "address": "192.168.1.42", "web_port": None, - "esp_platform": "ESP32", + "esp_platform": esp_platform, "build_path": "/build/lite_test", "firmware_bin_path": "/build/lite_test/firmware.bin", "loaded_integrations": ["api", "logger", "ota", "wifi"], "loaded_platforms": [], "no_mdns": False, "framework": "arduino", - "core_platform": "esp32", + "core_platform": core_platform, } storage_path.write_text(json.dumps(data)) @@ -123,6 +129,50 @@ def test_load_compiled_config_happy_path(fresh_cache_files: Path) -> None: assert CORE.build_path == Path("/build/lite_test") assert CORE.data[KEY_CORE][KEY_TARGET_PLATFORM] == "esp32" assert CORE.data[KEY_CORE][KEY_TARGET_FRAMEWORK] == "arduino" + # upload_using_esptool reads get_esp32_variant() off CORE.data[KEY_ESP32]. + from esphome.components.esp32.const import KEY_ESP32 + + assert CORE.data[KEY_ESP32][KEY_VARIANT] == "ESP32" + + +def test_load_compiled_config_populates_esp32_variant(tmp_path: Path) -> None: + """ESP32 variants survive the cache fast path so esptool gets the right --chip.""" + from esphome.components.esp32.const import KEY_ESP32 + + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + storage_dir = tmp_path / ".esphome" / "storage" + _write_storage(storage_dir / "lite_test.yaml.json", esp_platform="ESP32S3") + cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml") + _set_cache_mtime(cache, yaml_path, offset=5) + + assert load_compiled_config(yaml_path) is not None + assert CORE.data[KEY_ESP32][KEY_VARIANT] == "ESP32S3" + + +def test_load_compiled_config_skips_esp32_block_for_other_platforms( + tmp_path: Path, +) -> None: + """Non-esp32 targets shouldn't fabricate an esp32 data block.""" + from esphome.components.esp32.const import KEY_ESP32 + + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + storage_dir = tmp_path / ".esphome" / "storage" + _write_storage( + storage_dir / "lite_test.yaml.json", + esp_platform="ESP8266", + core_platform="esp8266", + ) + cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml") + _set_cache_mtime(cache, yaml_path, offset=5) + + assert load_compiled_config(yaml_path) is not None + assert KEY_ESP32 not in CORE.data @pytest.mark.parametrize( @@ -203,6 +253,106 @@ def test_run_esphome_upload_and_logs_fall_back_when_no_cache( mock_read.assert_called_once() +def test_run_esphome_upload_does_not_refresh_cache_without_sidecar( + tmp_path: Path, +) -> None: + """Without a StorageJSON sidecar (no compile has run), the fallback + skips the cache write -- load_compiled_config requires the sidecar, + so writing the rendered (secret-resolved) YAML would be inert and + leak secrets to disk for nothing.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + with ( + patch( + "esphome.__main__.read_config", + return_value={"esphome": {"name": "lite_test"}}, + ), + patch("esphome.compiled_config.save_compiled_config") as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"upload": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "upload", str(yaml_path)]) + + mock_save.assert_not_called() + + +@pytest.mark.parametrize("command", ["upload", "logs"]) +def test_run_esphome_upload_and_logs_refresh_cache_on_fallback( + tmp_path: Path, command: str +) -> None: + """A stale-cache fallback rewrites the cache so the next call hits + the fast path. Without this, every upload/logs after a YAML edit + pays for read_config() until the next compile rewrites the cache.""" + yaml_path = tmp_path / "lite_test.yaml" + yaml_path.write_text("esphome:\n name: lite_test\n") + CORE.config_path = yaml_path + + storage_dir = tmp_path / ".esphome" / "storage" + _write_storage(storage_dir / "lite_test.yaml.json") + cache = _write_cache(storage_dir / "lite_test.yaml.validated.yaml") + _set_cache_mtime(cache, yaml_path, offset=-60) # stale + + fresh_config = {"esphome": {"name": "lite_test"}, "logger": {}} + + with ( + patch("esphome.__main__.read_config", return_value=fresh_config), + patch( + "esphome.compiled_config.save_compiled_config", wraps=save_compiled_config + ) as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {command: lambda args, config: 0}, + ), + ): + assert run_esphome(["esphome", command, str(yaml_path)]) == 0 + + mock_save.assert_called_once_with(fresh_config) + # mtime is now newer than the source YAML, so a follow-up call hits + # the fast path instead of repeating read_config. + assert cache.stat().st_mtime >= yaml_path.stat().st_mtime + + +def test_run_esphome_upload_with_substitution_does_not_refresh_cache( + fresh_cache_files: Path, +) -> None: + """`-s` substitutions skip the cache on both read and write -- saving + here would clobber the cache with a substitution-specific config.""" + with ( + patch("esphome.__main__.read_config", return_value={"esphome": {}}), + patch("esphome.compiled_config.save_compiled_config") as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"upload": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "-s", "var", "val", "upload", str(fresh_cache_files)]) + + mock_save.assert_not_called() + + +def test_run_esphome_compile_does_not_refresh_cache_via_fallback( + fresh_cache_files: Path, +) -> None: + """Compile writes the cache through update_storage_json, not via the + upload/logs fallback path -- the fallback save would skip the + storage_should_clean check.""" + with ( + patch("esphome.__main__.read_config", return_value={"esphome": {}}), + patch("esphome.compiled_config.save_compiled_config") as mock_save, + patch.dict( + "esphome.__main__.POST_CONFIG_ACTIONS", + {"compile": lambda args, config: 0}, + ), + ): + run_esphome(["esphome", "compile", str(fresh_cache_files)]) + + mock_save.assert_not_called() + + def test_run_esphome_upload_with_substitution_skips_cache( fresh_cache_files: Path, ) -> None: diff --git a/tests/unit_tests/test_config_normalization.py b/tests/unit_tests/test_config_normalization.py index d70f3c24e0..4ec17b3c7c 100644 --- a/tests/unit_tests/test_config_normalization.py +++ b/tests/unit_tests/test_config_normalization.py @@ -67,7 +67,7 @@ def test_iter_component_configs_with_multi_conf(mock_get_component: Mock) -> Non configs = list(config.iter_component_configs(test_config)) assert len(configs) == 2 - for domain, component, conf in configs: + for domain, _component, conf in configs: assert domain == "switch" assert "name" in conf diff --git a/tests/unit_tests/test_espidf_component.py b/tests/unit_tests/test_espidf_component.py index 8977b05d23..4f0a71053d 100644 --- a/tests/unit_tests/test_espidf_component.py +++ b/tests/unit_tests/test_espidf_component.py @@ -203,7 +203,7 @@ def test_generate_idf_component_yml_basic(tmp_component): tmp_component.data = {"description": "test", "repository": {"url": "http://aaa"}} result = generate_idf_component_yml(tmp_component) - assert result == "description: test\nversion: 1.0.0\nrepository: http://aaa\n" + assert result == "description: test\nrepository: http://aaa\n" def test_generate_idf_component_yml_with_dependencies(tmp_component, tmp_path): @@ -217,18 +217,16 @@ def test_generate_idf_component_yml_with_dependencies(tmp_component, tmp_path): assert ( result - == f"""version: 1.0.0 -dependencies: + == f"""dependencies: dep: - version: '1.0' override_path: {dep.path} """ ) -def test_generate_idf_component_yml_missing_path_reraises(tmp_component): - # A dep without a path and without a recognised source should re-raise - # the underlying RuntimeError instead of silently producing a bad manifest. +def test_generate_idf_component_yml_missing_path_raises(tmp_component): + # A dep without a path is a contract violation — every dep is expected + # to have been downloaded before YAML generation. Raise loudly. dep = IDFComponent("foo/bar", "1.0", source=None) tmp_component.dependencies = [dep] @@ -263,9 +261,14 @@ def test_check_library_data_invalid_platform(esp32_idf_core): _check_library_data({"platforms": ["other"], "frameworks": "*"}) -def test_check_library_data_invalid_framework(esp32_idf_core): - with pytest.raises(InvalidIDFComponent): - _check_library_data({"platforms": "*", "frameworks": ["other"]}) +def test_check_library_data_invalid_framework( + esp32_idf_core: None, caplog: pytest.LogCaptureFixture +) -> None: + # Framework mismatch is a warning, not a hard skip: the library is still + # included so that PIO manifests that only list "arduino" (but actually + # compile under IDF) can be used without forking them. + _check_library_data({"name": "lib", "platforms": "*", "frameworks": ["other"]}) + assert "do not include 'espidf'" in caplog.text def test_extra_script_captures_libpath_libs_and_defines(tmp_path): @@ -290,7 +293,7 @@ def test_extra_script_captures_libpath_libs_and_defines(tmp_path): result = run_extra_script(script, library_dir=tmp_path, idf_target="esp32") - assert result.libpath == [os.path.join("src", "esp32")] + assert result.libpath == [str(Path("src") / "esp32")] assert result.libs == ["algobsec"] assert ("BAR", "1") in result.cppdefines assert "FOO" in result.cppdefines @@ -422,15 +425,37 @@ def test_convert_library_with_repository(): result = _convert_library_to_component(lib) assert result.name == "foo/bar" - assert result.version == "1.2.3" + assert result.version == "*" assert isinstance(result.source, GitSource) + assert result.source.ref == "v1.2.3" -def test_convert_library_missing_ref(): +def test_convert_library_with_branch_ref(): + lib = Library("name", None, "https://github.com/foo/bar.git#some-branch") + + result = _convert_library_to_component(lib) + + assert result.name == "foo/bar" + assert result.version == "*" + assert isinstance(result.source, GitSource) + assert result.source.ref == "some-branch" + + +def test_convert_library_missing_ref_uses_default_branch(): + """A bare URL with no #ref clones the remote's default branch. + + Matches PIO's lib_deps behavior and external_components handling -- + git.clone_or_update with ref=None leaves the depth-1 clone on + whatever branch the remote HEAD points at. + """ lib = Library("name", None, "https://github.com/foo/bar.git") - with pytest.raises(ValueError): - _convert_library_to_component(lib) + result = _convert_library_to_component(lib) + + assert result.name == "foo/bar" + assert result.version == "*" + assert isinstance(result.source, GitSource) + assert result.source.ref is None def test_convert_library_registry(monkeypatch): @@ -485,3 +510,113 @@ def test_process_dependencies_skips_invalid(tmp_component): _process_dependencies(tmp_component) assert tmp_component.dependencies == [] + + +def test_process_dependencies_dict_form(tmp_component, monkeypatch): + """PIO library.json shorthand ``{"owner/Name": "version"}`` is honored. + + Iterating a dict gives string keys, which would silently fail the + ``"name" in dependency`` substring check. Normalize to list-of-dicts + first so the dict form (used by e.g. tesla-ble for its nanopb dep) + is treated the same as the verbose list form. + """ + captured: list[Library] = [] + + def fake_generate(library): + captured.append(library) + return IDFComponent( + library.name, library.version, source=URLSource("http://dummy.com") + ) + + tmp_component.data = { + "dependencies": { + "nanopb/Nanopb": "^0.4.91", + "BareName": "1.2.3", + } + } + monkeypatch.setattr( + esphome.espidf.component, "_generate_idf_component", fake_generate + ) + monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None) + + _process_dependencies(tmp_component) + + assert len(tmp_component.dependencies) == 2 + names = sorted(lib.name for lib in captured) + versions = sorted(lib.version for lib in captured) + assert names == ["BareName", "nanopb/Nanopb"] + assert versions == ["1.2.3", "^0.4.91"] + + +def test_process_dependencies_dict_form_with_url_value(tmp_component, monkeypatch): + """A dict-value that's a URL gets routed to ``repository`` like the list form.""" + captured: list[Library] = [] + + def fake_generate(library): + captured.append(library) + return IDFComponent(library.name, "*", source=URLSource("http://dummy.com")) + + tmp_component.data = { + "dependencies": { + "foo/Bar": "https://github.com/foo/bar.git#main", + } + } + monkeypatch.setattr( + esphome.espidf.component, "_generate_idf_component", fake_generate + ) + monkeypatch.setattr(esphome.espidf.component, "_check_library_data", lambda x: None) + + _process_dependencies(tmp_component) + + assert len(captured) == 1 + assert captured[0].name == "foo/Bar" + assert captured[0].version is None + assert captured[0].repository == "https://github.com/foo/bar.git#main" + + +def test_process_dependencies_dict_form_with_nested_spec(tmp_component, monkeypatch): + """A dict-value that's itself a dict is merged into the entry. + + PIO's library.json allows ``{"owner/Name": {"version": "...", ...}}`` + for entries that need fields beyond just a version (platforms, + frameworks, etc.). The extra fields flow into _check_library_data + via the entry merge. + """ + captured: list[Library] = [] + checked: list[dict] = [] + + def fake_generate(library): + captured.append(library) + return IDFComponent( + library.name, library.version, source=URLSource("http://dummy.com") + ) + + tmp_component.data = { + "dependencies": { + "nanopb/Nanopb": {"version": "^0.4.91", "platforms": "espidf"}, + } + } + monkeypatch.setattr( + esphome.espidf.component, "_generate_idf_component", fake_generate + ) + monkeypatch.setattr( + esphome.espidf.component, + "_check_library_data", + checked.append, + ) + + _process_dependencies(tmp_component) + + assert len(captured) == 1 + assert captured[0].name == "nanopb/Nanopb" + assert captured[0].version == "^0.4.91" + # Extra spec fields reach _check_library_data so platform/framework + # gating still applies. + assert checked == [ + { + "name": "Nanopb", + "owner": "nanopb", + "version": "^0.4.91", + "platforms": "espidf", + } + ] diff --git a/tests/unit_tests/test_espidf_framework.py b/tests/unit_tests/test_espidf_framework.py new file mode 100644 index 0000000000..9f4e4fcca8 --- /dev/null +++ b/tests/unit_tests/test_espidf_framework.py @@ -0,0 +1,156 @@ +"""Tests for esphome.espidf.framework helpers.""" + +# pylint: disable=protected-access + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from esphome.espidf.framework import _clone_idf_with_submodules, _parse_git_source + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + # github:// shorthand + ( + "github://espressif/esp-idf", + ("https://github.com/espressif/esp-idf.git", None), + ), + ( + "github://espressif/esp-idf@master", + ("https://github.com/espressif/esp-idf.git", "master"), + ), + ( + "github://espressif/esp-idf@release/v6.0", + ("https://github.com/espressif/esp-idf.git", "release/v6.0"), + ), + # explicit https://github.com/...git URL + ( + "https://github.com/espressif/esp-idf.git", + ("https://github.com/espressif/esp-idf.git", None), + ), + ( + "https://github.com/espressif/esp-idf.git@master", + ("https://github.com/espressif/esp-idf.git", "master"), + ), + ( + "https://github.com/espressif/esp-idf.git@v6.0.1", + ("https://github.com/espressif/esp-idf.git", "v6.0.1"), + ), + # Tolerate a trailing ".git" on the shorthand so the user doesn't + # silently end up with a doubled "...esp-idf.git.git" URL. + ( + "github://espressif/esp-idf.git", + ("https://github.com/espressif/esp-idf.git", None), + ), + ( + "github://espressif/esp-idf.git@master", + ("https://github.com/espressif/esp-idf.git", "master"), + ), + ], +) +def test_parse_git_source_recognized( + source: str, expected: tuple[str, str | None] +) -> None: + assert _parse_git_source(source) == expected + + +@pytest.mark.parametrize( + "source", + [ + # archive URLs fall through to the existing download path + "https://github.com/espressif/esp-idf/archive/refs/heads/master.zip", + "https://dl.espressif.com/dl/esp-idf/v6.0.1/esp-idf-v6.0.1.zip", + "https://github.com/esphome-libs/esp-idf/releases/download/v5.5.4/esp-idf-v5.5.4.tar.xz", + # SSH and other git protocols are intentionally rejected — match + # external_components, which only recognizes github:// + structured + # dicts for these. + "git@github.com:espressif/esp-idf.git", + "ssh://git@github.com/espressif/esp-idf.git", + "git://github.com/espressif/esp-idf.git", + # non-GitHub .git URLs are intentionally rejected for the same reason + "https://gitlab.com/foo/bar.git", + "https://github.example.com/foo/bar.git", + ], +) +def test_parse_git_source_rejected(source: str) -> None: + assert _parse_git_source(source) is None + + +def _make_idf_tree(framework_path: Path) -> None: + """Create the minimum tree _clone_idf_with_submodules sanity-checks for.""" + (framework_path / "tools").mkdir(parents=True) + (framework_path / "tools" / "idf_tools.py").write_text("# stub\n") + + +def test_clone_idf_with_submodules_without_ref(tmp_path: Path) -> None: + framework_path = tmp_path / "idf" + framework_path.mkdir() + _make_idf_tree(framework_path) + + with patch("esphome.git.run_git_command", return_value="") as run_git_command_mock: + _clone_idf_with_submodules( + framework_path, "https://github.com/espressif/esp-idf.git", None + ) + + # No ref -> just clone + submodule update, no fetch/reset. + calls = [c.args[0] for c in run_git_command_mock.call_args_list] + assert calls[0] == [ + "git", + "clone", + "--depth=1", + "--", + "https://github.com/espressif/esp-idf.git", + str(framework_path), + ] + assert calls[-1][:5] == ["git", "submodule", "update", "--init", "--recursive"] + assert not any(c[1] == "fetch" for c in calls) + assert not any(c[1] == "reset" for c in calls) + + +def test_clone_idf_with_submodules_with_ref(tmp_path: Path) -> None: + framework_path = tmp_path / "idf" + framework_path.mkdir() + _make_idf_tree(framework_path) + + with patch("esphome.git.run_git_command", return_value="") as run_git_command_mock: + _clone_idf_with_submodules( + framework_path, + "https://github.com/espressif/esp-idf.git", + "master", + ) + + calls = [c.args[0] for c in run_git_command_mock.call_args_list] + # clone, fetch ref, reset hard, submodule update + assert calls[0][:2] == ["git", "clone"] + assert calls[1] == [ + "git", + "fetch", + "--depth=1", + "--", + "origin", + "master", + ] + assert calls[2] == ["git", "reset", "--hard", "FETCH_HEAD"] + assert calls[3][:5] == ["git", "submodule", "update", "--init", "--recursive"] + + +def test_clone_idf_with_submodules_raises_when_tree_missing( + tmp_path: Path, +) -> None: + framework_path = tmp_path / "idf" + framework_path.mkdir() + # Deliberately do NOT call _make_idf_tree — simulate a clone that + # returned 0 but produced no tools/idf_tools.py. + + with ( + patch("esphome.git.run_git_command", return_value=""), + pytest.raises(RuntimeError, match="no usable ESP-IDF tree"), + ): + _clone_idf_with_submodules( + framework_path, + "https://github.com/espressif/esp-idf.git", + None, + ) diff --git a/tests/unit_tests/test_espidf_toolchain.py b/tests/unit_tests/test_espidf_toolchain.py new file mode 100644 index 0000000000..adc8bfce63 --- /dev/null +++ b/tests/unit_tests/test_espidf_toolchain.py @@ -0,0 +1,58 @@ +"""Tests for esphome.espidf.toolchain helpers.""" + +# pylint: disable=protected-access + +from unittest.mock import patch + +from esphome.const import CONF_FRAMEWORK, CONF_SOURCE +from esphome.core import CORE +from esphome.espidf import toolchain + + +def test_get_framework_source_override_no_config(): + """When CORE.config hasn't been set, no override is returned.""" + CORE.config = None + assert toolchain._get_framework_source_override() is None + + +def test_get_framework_source_override_no_esp32_section(): + """A config without an esp32 section yields no override.""" + CORE.config = {} + assert toolchain._get_framework_source_override() is None + + +def test_get_framework_source_override_no_framework_source(): + """An esp32 section without framework.source yields no override.""" + CORE.config = {"esp32": {CONF_FRAMEWORK: {}}} + assert toolchain._get_framework_source_override() is None + + +def test_get_framework_source_override_returns_value(): + """A user-supplied framework source is returned verbatim.""" + url = "https://example.com/esp-idf-v{VERSION}.tar.xz" + CORE.config = {"esp32": {CONF_FRAMEWORK: {CONF_SOURCE: url}}} + assert toolchain._get_framework_source_override() == url + + +def test_get_esphome_esp_idf_paths_forwards_source_override(): + """_get_esphome_esp_idf_paths threads the override into check_esp_idf_install.""" + url = "https://my-mirror/esp-idf-v{VERSION}.tar.xz" + CORE.config = {"esp32": {CONF_FRAMEWORK: {CONF_SOURCE: url}}} + # Hit a fresh cache key so check_esp_idf_install is actually called. + toolchain._cache().paths.clear() + with patch.object( + toolchain, "check_esp_idf_install", return_value=("/fw", "/penv") + ) as mock_install: + toolchain._get_esphome_esp_idf_paths("5.5.4") + mock_install.assert_called_once_with("5.5.4", source_url=url) + + +def test_get_esphome_esp_idf_paths_no_override(): + """When no source override is configured, source_url=None is passed.""" + CORE.config = {} + toolchain._cache().paths.clear() + with patch.object( + toolchain, "check_esp_idf_install", return_value=("/fw", "/penv") + ) as mock_install: + toolchain._get_esphome_esp_idf_paths("5.5.4") + mock_install.assert_called_once_with("5.5.4", source_url=None) diff --git a/tests/unit_tests/test_git.py b/tests/unit_tests/test_git.py index eab6bfc2cb..690c47c183 100644 --- a/tests/unit_tests/test_git.py +++ b/tests/unit_tests/test_git.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta import os from pathlib import Path from typing import Any -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest @@ -1001,3 +1001,304 @@ def test_refresh_picks_up_new_remote_commits( "--hard", "old_sha", ] + + +def test_resolve_symlink_stub_returns_none_on_non_windows( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """On non-Windows, resolve_symlink_stub returns None without calling git.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + stub = repo_dir / "file.yaml" + stub.write_text("static/file.yaml") + + with patch("esphome.git.sys.platform", "linux"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_target_for_mode_120000( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A mode-120000 file is recognised as a stub; its target Path is returned.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "static").mkdir() + + target = repo_dir / "static" / "real.yaml" + target.write_text("esphome:\n name: real\n") + + stub = repo_dir / "real.yaml" + stub.write_text("static/real.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\treal.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + # Stub file itself was not modified — only inspected. + assert stub.read_text() == "static/real.yaml" + + +def test_resolve_symlink_stub_resolves_relative_parent_paths( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Symlink targets with ``..`` segments resolve correctly within the repo.""" + repo_dir = tmp_path / "repo" + (repo_dir / "subdir").mkdir(parents=True) + (repo_dir / "static").mkdir() + + target = repo_dir / "static" / "shared.yaml" + target.write_text("shared content") + + stub = repo_dir / "subdir" / "shared.yaml" + stub.write_text("../static/shared.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tsubdir/shared.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + + +def test_resolve_symlink_stub_refuses_escape_outside_repo( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A symlink pointing outside the repository is not followed.""" + outside = tmp_path / "outside.yaml" + outside.write_text("sensitive") + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "escape.yaml" + stub.write_text("../outside.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tescape.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_for_real_symlink( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A real symlink already opens transparently, so the helper short-circuits. + + Skipped on Windows where symlink creation requires + SeCreateSymbolicLinkPrivilege. + """ + if os.name == "nt": + pytest.skip("Requires symlink-creation privilege on Windows") + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + target = repo_dir / "real.yaml" + target.write_text("real content") + + real_link = repo_dir / "link.yaml" + real_link.symlink_to("real.yaml") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, real_link) + + assert result is None + # No git call needed for real symlinks. + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_for_regular_file( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A regular file (mode 100644) whose content looks path-shaped is not + followed.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + regular = repo_dir / "looks_like_path.txt" + regular.write_text("static/something.yaml") + + mock_run_git_command.return_value = "100644 abc123 0\tlooks_like_path.txt" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, regular) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_git_fails( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """If ``git ls-files`` fails (e.g. not a repo), the helper returns None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "real.yaml" + stub.write_text("static/real.yaml") + + mock_run_git_command.side_effect = GitCommandError("ls-files exploded") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_for_non_utf8_content( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file whose bytes are not valid UTF-8 must not raise — return None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "binary.bin" + stub.write_bytes(b"\xff\xfe\x00\xff") + + mock_run_git_command.return_value = "120000 abc123 0\tbinary.bin" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_preserves_whitespace_in_target( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Only trailing CR/LF is stripped — internal whitespace is preserved.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + target_dir = repo_dir / "dir with spaces" + target_dir.mkdir() + target = target_dir / "real.yaml" + target.write_text("hello") + + stub = repo_dir / "link.yaml" + # Trailing newline (as git's checkout may append) is stripped, but + # whitespace inside the target path itself must survive. + stub.write_bytes(b"dir with spaces/real.yaml\n") + + mock_run_git_command.return_value = "120000 abc123 0\tlink.yaml" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result == target.resolve() + + +def test_resolve_symlink_stub_returns_none_for_directory_target( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A symlink pointing at a directory has no file content to load.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "dir_target").mkdir() + + stub = repo_dir / "link_to_dir" + stub.write_text("dir_target") + + mock_run_git_command.return_value = "120000 abc123 0\tlink_to_dir" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_resolve_raises( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Path.resolve() raising (e.g. on a malformed target) must not propagate.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "broken.yaml" + stub.write_text("ignored") + + mock_run_git_command.return_value = "120000 abc123 0\tbroken.yaml" + + with ( + patch("esphome.git.sys.platform", "win32"), + patch.object(Path, "resolve", side_effect=OSError("bad path")), + ): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_file_missing( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file path that doesn't exist is rejected before git is consulted.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + missing = repo_dir / "ghost.yaml" # not created + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, missing) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_when_path_outside_repo( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """A file path that isn't under repo_dir is rejected (ValueError from relative_to).""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + outside = tmp_path / "stray.yaml" + outside.write_text("something") + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, outside) + + assert result is None + mock_run_git_command.assert_not_called() + + +def test_resolve_symlink_stub_returns_none_when_untracked( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """Empty `git ls-files` output (untracked file) makes the helper return None.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "untracked.yaml" + stub.write_text("static/foo.yaml") + + mock_run_git_command.return_value = "" + + with patch("esphome.git.sys.platform", "win32"): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None + + +def test_resolve_symlink_stub_returns_none_when_read_bytes_raises( + tmp_path: Path, mock_run_git_command: Mock +) -> None: + """An OSError from read_bytes() (e.g. file vanished mid-call) must not propagate.""" + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + + stub = repo_dir / "racy.yaml" + stub.write_text("static/racy.yaml") + + mock_run_git_command.return_value = "120000 abc123 0\tracy.yaml" + + with ( + patch("esphome.git.sys.platform", "win32"), + patch.object(Path, "read_bytes", side_effect=OSError("vanished")), + ): + result = git.resolve_symlink_stub(repo_dir, stub) + + assert result is None diff --git a/tests/unit_tests/test_helpers.py b/tests/unit_tests/test_helpers.py index bb00a15bee..efc2d8e42a 100644 --- a/tests/unit_tests/test_helpers.py +++ b/tests/unit_tests/test_helpers.py @@ -7,7 +7,7 @@ import stat from unittest.mock import MagicMock, patch from aioesphomeapi.host_resolver import AddrInfo, IPv4Sockaddr, IPv6Sockaddr -from hypothesis import given +from hypothesis import given, settings from hypothesis.strategies import ip_addresses import pytest @@ -151,6 +151,7 @@ def test_is_ip_address__invalid(host): assert actual is False +@settings(deadline=None) @given(value=ip_addresses(v=4).map(str)) def test_is_ip_address__valid(value): actual = helpers.is_ip_address(value) diff --git a/tests/unit_tests/test_main.py b/tests/unit_tests/test_main.py index 6ec0069b3a..f6b6d0b05f 100644 --- a/tests/unit_tests/test_main.py +++ b/tests/unit_tests/test_main.py @@ -11,7 +11,7 @@ from pathlib import Path import re import sys import time -from typing import Any +from typing import Any, Self from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -5110,11 +5110,11 @@ class MockSerial: self.timeout = 0.1 self._is_open = False - def __enter__(self) -> MockSerial: + def __enter__(self) -> Self: self._is_open = True return self - def __exit__(self, *args: Any) -> None: + def __exit__(self, *args: object) -> None: self._is_open = False @property diff --git a/tests/unit_tests/test_size_summary.py b/tests/unit_tests/test_size_summary.py new file mode 100644 index 0000000000..933be88476 --- /dev/null +++ b/tests/unit_tests/test_size_summary.py @@ -0,0 +1,128 @@ +"""Tests for esphome.espidf.size_summary.print_summary.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from esphome.espidf.size_summary import print_summary + + +def _write_size_json(tmp_path: Path, data: dict) -> Path: + """Drop a fake esp_idf_size.json under ``tmp_path`` and return the path.""" + out = tmp_path / "esp_idf_size.json" + out.write_text(json.dumps(data)) + return out + + +def _esp32_size_data() -> dict: + """Synthetic esp_idf_size.json for the original ESP32 (split IRAM/DRAM).""" + return { + "image_size": 827455, + "memory_types": { + "DRAM": { + "size": 180736, + "used": 47332, + "sections": { + ".dram0.bss": {"abbrev_name": ".bss", "size": 30616}, + ".dram0.data": {"abbrev_name": ".data", "size": 16716}, + }, + }, + "IRAM": { + "size": 131072, + "used": 80351, + "sections": { + ".iram0.text": {"abbrev_name": ".text", "size": 79323}, + ".iram0.vectors": {"abbrev_name": ".vectors", "size": 1028}, + }, + }, + }, + } + + +def _s3_size_data() -> dict: + """Synthetic esp_idf_size.json for ESP32-S3 (unified DIRAM).""" + return { + "image_size": 724215, + "memory_types": { + "DIRAM": { + "size": 341760, + "used": 104999, + "sections": { + ".iram0.text": {"abbrev_name": ".text", "size": 58051}, + ".dram0.bss": {"abbrev_name": ".bss", "size": 27088}, + ".dram0.data": {"abbrev_name": ".data", "size": 19708}, + ".noinit": {"abbrev_name": ".noinit", "size": 152}, + }, + }, + "IRAM": { + "size": 16384, + "used": 16384, + "sections": { + ".iram0.text": {"abbrev_name": ".text", "size": 15356}, + ".iram0.vectors": {"abbrev_name": ".vectors", "size": 1028}, + }, + }, + }, + } + + +def test_print_summary_esp32_uses_dram( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Original ESP32: DRAM has no ``.text``, so RAM = DRAM.used / DRAM.size unchanged.""" + size_json = _write_size_json(tmp_path, _esp32_size_data()) + print_summary(size_json, partitions_csv=None) + out = capsys.readouterr().out + assert "RAM:" in out + assert "used 47332 bytes from 180736 bytes" in out + + +def test_print_summary_s3_falls_back_to_diram( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """ESP32-S3 with no DRAM key falls back to DIRAM and reports raw region usage.""" + size_json = _write_size_json(tmp_path, _s3_size_data()) + print_summary(size_json, partitions_csv=None) + out = capsys.readouterr().out + assert "used 104999 bytes from 341760 bytes" in out + + +def test_print_summary_skips_when_diram_total_collapses( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A zero-size region drops the RAM line rather than divide by zero.""" + size_json = _write_size_json( + tmp_path, + { + "memory_types": { + "DIRAM": { + "size": 0, + "used": 0, + "sections": {}, + }, + }, + }, + ) + print_summary(size_json, partitions_csv=None) + out = capsys.readouterr().out + assert "RAM:" not in out + + +def test_print_summary_handles_missing_json( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Missing size json is non-fatal and prints nothing.""" + print_summary(tmp_path / "does_not_exist.json", partitions_csv=None) + assert capsys.readouterr().out == "" + + +def test_print_summary_handles_no_memory_types( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A size json without ``memory_types`` still doesn't crash.""" + size_json = _write_size_json(tmp_path, {"image_size": 0}) + print_summary(size_json, partitions_csv=None) + assert capsys.readouterr().out == "" diff --git a/tests/unit_tests/test_storage_json.py b/tests/unit_tests/test_storage_json.py index a3a38960e7..ea37492cf4 100644 --- a/tests/unit_tests/test_storage_json.py +++ b/tests/unit_tests/test_storage_json.py @@ -9,7 +9,7 @@ from unittest.mock import MagicMock, Mock, patch import pytest from esphome import storage_json -from esphome.const import CONF_DISABLED, CONF_MDNS +from esphome.const import CONF_DISABLED, CONF_MDNS, Toolchain from esphome.core import CORE @@ -308,6 +308,7 @@ def test_storage_json_from_esphome_core(setup_core: Path) -> None: mock_core.loaded_platforms = {"sensor"} mock_core.config = {CONF_MDNS: {CONF_DISABLED: True}} mock_core.target_framework = "esp-idf" + mock_core.toolchain = Toolchain.ESP_IDF with patch("esphome.components.esp32.get_esp32_variant") as mock_variant: mock_variant.return_value = "ESP32-C3" @@ -327,6 +328,7 @@ def test_storage_json_from_esphome_core(setup_core: Path) -> None: assert result.no_mdns is True assert result.framework == "esp-idf" assert result.core_platform == "esp32" + assert result.toolchain == "esp-idf" def test_storage_json_from_esphome_core_mdns_enabled(setup_core: Path) -> None: @@ -345,10 +347,12 @@ def test_storage_json_from_esphome_core_mdns_enabled(setup_core: Path) -> None: mock_core.loaded_platforms = set() mock_core.config = {} # No MDNS config means enabled mock_core.target_framework = "arduino" + mock_core.toolchain = None result = storage_json.StorageJSON.from_esphome_core(mock_core, old=None) assert result.no_mdns is False + assert result.toolchain is None def test_storage_json_load_valid_file(tmp_path: Path) -> None: @@ -470,6 +474,73 @@ def test_storage_json_equality() -> None: assert storage1 != "not a storage object" +def _make_storage_with_toolchain( + toolchain: str | None, +) -> storage_json.StorageJSON: + return storage_json.StorageJSON( + storage_version=1, + name="dev", + friendly_name=None, + comment=None, + esphome_version="2024.1.0", + src_version=1, + address="dev.local", + web_port=None, + target_platform="ESP32", + build_path=Path("/build"), + firmware_bin_path=Path("/build/firmware.bin"), + loaded_integrations=set(), + loaded_platforms=set(), + no_mdns=False, + framework="esp-idf", + core_platform="esp32", + toolchain=toolchain, + ) + + +def test_storage_json_toolchain_round_trip(setup_core: Path) -> None: + """Sidecar toolchain survives save -> load -> apply_to_core.""" + storage = _make_storage_with_toolchain("esp-idf") + path = setup_core / "storage.json" + path.write_text(storage.to_json()) + + # Serialization key is stable -- device-builder relies on it. + assert json.loads(path.read_text())["toolchain"] == "esp-idf" + + loaded = storage_json.StorageJSON.load(path) + assert loaded is not None + assert loaded.toolchain == "esp-idf" + + CORE.toolchain = None + with patch("esphome.components.esp32.get_esp32_variant"): + loaded.apply_to_core() + assert CORE.toolchain == Toolchain.ESP_IDF + + +def test_storage_json_apply_to_core_preserves_cli_toolchain( + setup_core: Path, +) -> None: + """A CLI-set CORE.toolchain wins over the sidecar value.""" + loaded = _make_storage_with_toolchain("esp-idf") + + CORE.toolchain = Toolchain.PLATFORMIO + with patch("esphome.components.esp32.get_esp32_variant"): + loaded.apply_to_core() + assert CORE.toolchain == Toolchain.PLATFORMIO + + +def test_storage_json_apply_to_core_ignores_unknown_toolchain( + setup_core: Path, +) -> None: + """Unknown enum values (corrupt sidecar / newer ESPHome) fall through to None.""" + loaded = _make_storage_with_toolchain("gcc") + + CORE.toolchain = None + with patch("esphome.components.esp32.get_esp32_variant"): + loaded.apply_to_core() + assert CORE.toolchain is None + + def test_esphome_storage_json_as_dict() -> None: """Test EsphomeStorageJSON.as_dict returns correct dictionary.""" storage = storage_json.EsphomeStorageJSON( diff --git a/tests/unit_tests/test_substitutions.py b/tests/unit_tests/test_substitutions.py index 4783112578..b5816f742e 100644 --- a/tests/unit_tests/test_substitutions.py +++ b/tests/unit_tests/test_substitutions.py @@ -1,4 +1,3 @@ -import glob import logging from pathlib import Path from typing import Any @@ -106,7 +105,7 @@ REMOTES = { # Collect all input YAML files for test_substitutions_fixtures parametrized tests: HERE = Path(__file__).parent BASE_DIR = HERE / "fixtures" / "substitutions" -SOURCES = sorted(glob.glob(str(BASE_DIR / "*.input.yaml"))) +SOURCES = sorted(str(p) for p in BASE_DIR.glob("*.input.yaml")) assert SOURCES, f"test_substitutions_fixtures: No input YAML files found in {BASE_DIR}" @@ -838,3 +837,86 @@ def test_include_vars_applied_to_lambda_value(tmp_path: Path) -> None: assert isinstance(result["value"], Lambda) assert result["value"].value == 'return "bar";' + + +@patch("esphome.git.resolve_symlink_stub") +@patch("esphome.git.clone_or_update") +def test_remote_package_symlink_stub_is_followed( + mock_clone_or_update: MagicMock, + mock_resolve_symlink_stub: MagicMock, + tmp_path: Path, +) -> None: + """When a package YAML is a scalar (symlink stub) and resolve_symlink_stub + returns a target, the loader follows the target and uses its content.""" + CORE.config_path = tmp_path / "test.yaml" + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + (repo_dir / "static").mkdir() + + # Stub file: content is the target path string (simulating Windows behavior). + stub = repo_dir / "file1.yaml" + stub.write_text("static/file1.yaml") + + # Real target with valid YAML mapping. + target = repo_dir / "static" / "file1.yaml" + target.write_text("substitutions:\n hello: world\n") + + mock_clone_or_update.return_value = (repo_dir, None) + mock_resolve_symlink_stub.return_value = target + + config: dict[str, Any] = { + "packages": { + "test_package": { + "url": "https://github.com/esphome/repo1", + "ref": "main", + "files": ["file1.yaml"], + } + } + } + + # Must succeed (does not raise the helpful cv.Invalid) because the stub + # was followed and a valid mapping was loaded from the target. + do_packages_pass(config) + assert mock_resolve_symlink_stub.called + + +@patch("esphome.git.clone_or_update") +def test_remote_package_scalar_yaml_raises_helpful_error( + mock_clone_or_update: MagicMock, tmp_path: Path +) -> None: + """A remote package YAML that is a top-level scalar (e.g. an unmaterialized + git symlink on Windows) raises a clear cv.Invalid, not AttributeError. + + Regression test for the case where a repo containing a YAML symlink, + checked out on Windows without symlink privilege, lands as a short text + file containing the symlink target path. PyYAML parses that as a bare + string scalar; the package loader must reject it with a human-readable + error instead of dying inside ``.get()``. + """ + CORE.config_path = tmp_path / "test.yaml" + + repo_dir = tmp_path / "repo" + repo_dir.mkdir() + # Simulate the broken-symlink state: a YAML file whose entire content is + # the symlink target string. PyYAML parses this as a top-level scalar. + (repo_dir / "file1.yaml").write_text("static/file1.yaml") + + mock_clone_or_update.return_value = (repo_dir, None) + + config: dict[str, Any] = { + "packages": { + "test_package": { + "url": "https://github.com/esphome/repo1", + "ref": "main", + "files": ["file1.yaml"], + } + } + } + + with pytest.raises(cv.Invalid) as exc_info: + do_packages_pass(config) + + msg = str(exc_info.value) + assert "mapping at the top level" in msg + assert "file1.yaml" in msg diff --git a/tests/unit_tests/test_writer.py b/tests/unit_tests/test_writer.py index 91b4bd8e87..fc49f03067 100644 --- a/tests/unit_tests/test_writer.py +++ b/tests/unit_tests/test_writer.py @@ -1358,7 +1358,7 @@ def test_clean_build_handles_readonly_files( # Create a read-only file (simulating git pack files on Windows) readonly_file = git_dir / "pack-abc123.pack" readonly_file.write_text("pack data") - os.chmod(readonly_file, stat.S_IRUSR) # Read-only + readonly_file.chmod(stat.S_IRUSR) # Read-only # Setup mocks mock_core.relative_pioenvs_path.return_value = pioenvs_dir @@ -1393,7 +1393,7 @@ def test_clean_all_handles_readonly_files( subdir.mkdir() readonly_file = subdir / "readonly.txt" readonly_file.write_text("content") - os.chmod(readonly_file, stat.S_IRUSR) # Read-only + readonly_file.chmod(stat.S_IRUSR) # Read-only # Verify file is read-only assert not os.access(readonly_file, os.W_OK) @@ -1422,7 +1422,7 @@ def test_clean_build_reraises_for_other_errors( test_file.write_text("content") # Make subdir read-only so files inside can't be deleted - os.chmod(subdir, stat.S_IRUSR | stat.S_IXUSR) + subdir.chmod(stat.S_IRUSR | stat.S_IXUSR) # Setup mocks mock_core.relative_pioenvs_path.return_value = pioenvs_dir @@ -1440,7 +1440,7 @@ def test_clean_build_reraises_for_other_errors( clean_build() finally: # Cleanup - restore write permission so tmp_path cleanup works - os.chmod(subdir, stat.S_IRWXU) + subdir.chmod(stat.S_IRWXU) # Tests for get_build_info() diff --git a/tests/unit_tests/test_yaml_util.py b/tests/unit_tests/test_yaml_util.py index e97a188be4..d6fb5b81f2 100644 --- a/tests/unit_tests/test_yaml_util.py +++ b/tests/unit_tests/test_yaml_util.py @@ -34,6 +34,14 @@ def clear_secrets_cache() -> None: yaml_util._SECRET_CACHE.clear() +@pytest.fixture(autouse=True) +def clear_core_frontmatter() -> None: + """Reset CORE.frontmatter between tests.""" + core.CORE.frontmatter = {} + yield + core.CORE.frontmatter = {} + + def test_include_with_vars(fixture_path: Path) -> None: yaml_file = fixture_path / "yaml_util" / "includetest.yaml" @@ -899,7 +907,7 @@ def test_format_path_current_obj_without_location_falls_back_to_key(): """An ESPHomeDataBase current_obj with no esp_range falls back to the key's location.""" class _NoRange(ESPHomeDataBase, str): - pass + __slots__ = () obj = _NoRange.__new__(_NoRange, "value") str.__init__(obj) @@ -1182,3 +1190,153 @@ def test_track_yaml_loads_records_resolved_paths(tmp_path: Path) -> None: with track_yaml_loads() as loaded: yaml_util.load_yaml(link) assert target.resolve() in loaded + + +# --------------------------------------------------------------------------- +# YAML frontmatter +# --------------------------------------------------------------------------- + + +def test_frontmatter_parsed_and_stored_on_core(tmp_path: Path) -> None: + """A leading `---`-separated YAML document is stored as frontmatter and + stripped from the returned config.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text( + "author: Jesse\nlabels: [office, climate]\n---\nesphome:\n name: my_node\n" + ) + + config = yaml_util.load_yaml(yaml_file) + + # Config does not contain frontmatter keys + assert "author" not in config + assert "labels" not in config + assert config["esphome"]["name"] == "my_node" + + # Frontmatter is stored on CORE keyed by resolved path + frontmatter = core.CORE.frontmatter[yaml_file.resolve()] + assert frontmatter["author"] == "Jesse" + assert frontmatter["labels"] == ["office", "climate"] + + +def test_frontmatter_absent_when_single_document(tmp_path: Path) -> None: + """A YAML file with a single document does not populate CORE.frontmatter.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text("esphome:\n name: my_node\n") + + yaml_util.load_yaml(yaml_file) + assert yaml_file.resolve() not in core.CORE.frontmatter + + +def test_frontmatter_absent_when_leading_doc_separator(tmp_path: Path) -> None: + """A leading `---` with no content above it is just a document start marker, + not frontmatter, and must not populate CORE.frontmatter.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text("---\nesphome:\n name: my_node\n") + + config = yaml_util.load_yaml(yaml_file) + assert config["esphome"]["name"] == "my_node" + assert yaml_file.resolve() not in core.CORE.frontmatter + + +def test_frontmatter_supports_arbitrary_keys(tmp_path: Path) -> None: + """Frontmatter keys are not validated — any structure is accepted.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text( + "any_key: any_value\n" + "nested:\n" + " count: 42\n" + " items:\n" + " - a\n" + " - b\n" + "---\n" + "esphome:\n" + " name: t\n" + ) + + yaml_util.load_yaml(yaml_file) + frontmatter = core.CORE.frontmatter[yaml_file.resolve()] + assert frontmatter["any_key"] == "any_value" + assert frontmatter["nested"]["count"] == 42 + assert frontmatter["nested"]["items"] == ["a", "b"] + + +def test_frontmatter_supports_deeply_nested_paths(tmp_path: Path) -> None: + """Frontmatter preserves deeply nested dict/list structures intact.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text( + "device:\n" + " metadata:\n" + " location:\n" + " building: HQ\n" + " floor: 3\n" + " room:\n" + " number: 302\n" + " occupants:\n" + " - name: Jesse\n" + " role:\n" + " title: maintainer\n" + " since: 2021\n" + " - name: Alice\n" + " role:\n" + " title: contributor\n" + " since: 2024\n" + "---\n" + "esphome:\n" + " name: t\n" + ) + + yaml_util.load_yaml(yaml_file) + fm = core.CORE.frontmatter[yaml_file.resolve()] + room = fm["device"]["metadata"]["location"]["room"] + assert room["number"] == 302 + assert room["occupants"][0]["name"] == "Jesse" + assert room["occupants"][0]["role"]["title"] == "maintainer" + assert room["occupants"][0]["role"]["since"] == 2021 + assert room["occupants"][1]["role"]["title"] == "contributor" + + +def test_frontmatter_more_than_two_documents_raises(tmp_path: Path) -> None: + """Three or more YAML documents is unsupported and must raise.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text("a: 1\n---\nb: 2\n---\nc: 3\n") + + with pytest.raises(EsphomeError, match="at most two are supported"): + yaml_util.load_yaml(yaml_file) + + +def test_frontmatter_empty_frontmatter_doc_not_stored(tmp_path: Path) -> None: + """An empty (null) frontmatter document is treated as no frontmatter.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text("---\n---\nesphome:\n name: t\n") + + config = yaml_util.load_yaml(yaml_file) + assert config["esphome"]["name"] == "t" + assert yaml_file.resolve() not in core.CORE.frontmatter + + +def test_frontmatter_empty_config_doc(tmp_path: Path) -> None: + """An empty config document after a frontmatter document yields an empty config.""" + yaml_file = tmp_path / "main.yaml" + yaml_file.write_text("only: frontmatter\n---\n") + + config = yaml_util.load_yaml(yaml_file) + assert config == {} + assert core.CORE.frontmatter[yaml_file.resolve()]["only"] == "frontmatter" + + +def test_frontmatter_included_file_stored(tmp_path: Path) -> None: + """Frontmatter on an !include'd file is also captured on CORE, keyed by + that file's resolved path.""" + inc = tmp_path / "child.yaml" + inc.write_text("child_meta: hello\n---\nchild_key: value\n") + main = tmp_path / "main.yaml" + main.write_text("esphome:\n name: t\nchild: !include child.yaml\n") + + config = yaml_util.load_yaml(main) + # !include is deferred; force resolution so the child file actually loads + force_load_include_files(config) + assert config["child"].load()["child_key"] == "value" + # Main file has no frontmatter + assert main.resolve() not in core.CORE.frontmatter + # Included file's frontmatter is captured + assert core.CORE.frontmatter[inc.resolve()]["child_meta"] == "hello"