Bypass the cache when the resolved config enables secure options the snapshot cannot see

Co-authored-by: n-IA-hane <n-IA-hane@users.noreply.github.com>

.github/actions/cache-arduino8266/action.yml

@@ -22,7 +22,7 @@ runs:
         # cached path cannot diverge.
         echo "ESPHOME_ARDUINO8266_PREFIX=$HOME/.esphome-arduino8266" >> "$GITHUB_ENV"
         . venv/bin/activate
-        key=$(python -c 'from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION as f; from esphome.arduino8266.framework import TOOLCHAIN_VERSION as t; print(f"{f}-{t}")')
+        key=$(python -c 'from esphome.components.esp8266 import RECOMMENDED_ARDUINO_FRAMEWORK_VERSION as f; from esphome.arduino8266.framework import FRAMEWORK_RELEASES, TOOLCHAIN_VERSION as t; print(f"{FRAMEWORK_RELEASES[f].tag}-{t}")')
         [ -n "$key" ] || exit 1
         echo "key=$key" >> "$GITHUB_OUTPUT"
     - name: Cache the native toolchain (write on dev)

.github/workflows/ci-docker.yml

@@ -19,6 +19,7 @@ on:
       - "esphome/core/**"
       - "esphome/writer.py"
       - "esphome/build_gen/**"
+      - "esphome/build_helpers/**"
       - "esphome/espidf/**"
       - "esphome/platformio/**"
       - "esphome/components/bk72xx/**"
@@ -206,17 +207,6 @@ jobs:
           - ln882x-arduino
           - nrf52
           - host
-        # Strict by default so a new matrix id cannot silently join in the
-        # degrade-quietly mode the knob exists to catch; the knob is inert
-        # where no pch code runs (esp32-*-platformio, nrf52).
-        # Opt-outs: libretiny GCC rejects its own pch until a toolchain bump.
-        include:
-          - id: bk72xx-arduino
-            pch_strict: "0"
-          - id: rtl87xx-arduino
-            pch_strict: "0"
-          - id: ln882x-arduino
-            pch_strict: "0"
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Download image artifact
@@ -228,7 +218,6 @@ jobs:
       - name: Compile ${{ matrix.id }}
         run: |
           docker run --rm \
-            -e ESPHOME_PCH_STRICT="${{ matrix.pch_strict || '1' }}" \
             -v "${{ github.workspace }}/docker/test_configs:/config" \
             "ghcr.io/esphome/esphome-amd64:${{ needs.check-docker.outputs.tag }}" \
             compile "${{ matrix.id }}.yaml"

.github/workflows/ci.yml

@@ -1226,8 +1226,20 @@ jobs:
           if [ -n "$compile_csv" ]; then
             # Run compilation with grouping and isolation
             python3 script/test_build_components.py -e compile -c "$compile_csv" -f --isolate "$directly_changed_csv"
+            if [[ "${{ matrix.batch.check_idf_py }}" == "true" ]]; then
+              # The real idf.py must find nothing to configure or build in a
+              # tree built above; catches drift on ESP-IDF bumps.
+              echo "Checking the native ESP-IDF build matches idf.py"
+              python3 script/check_idf_py_equivalence.py --require-cached-bootloader
+            fi
           else
             echo "All components in this batch are validate-only -- skipping compile stage."
+            if [[ "${{ matrix.batch.check_idf_py }}" == "true" ]]; then
+              # determine-jobs and this step disagree on what compiles; fail
+              # rather than let the check run nowhere.
+              echo "::error::This batch was picked for the idf.py check but compiled nothing"
+              exit 1
+            fi
           fi
 
       - name: Print ccache statistics
@@ -1307,10 +1319,8 @@ jobs:
           echo ""
 
           # ESP8266 Arduino built directly (no PlatformIO); compile validates
-          # config first, so a separate config pass is redundant. Strict pch:
-          # exercises the native ninja pch (and its probe edge) against real
-          # component configs; the docker matrix smoke-tests both toolchains.
-          ESPHOME_PCH_STRICT=1 python3 script/test_build_components.py -e compile -t esp8266-ard -c "$TEST_COMPONENTS" -f --toolchain arduino --fail-on-no-tests
+          # config first, so a separate config pass is redundant.
+          python3 script/test_build_components.py -e compile -t esp8266-ard -c "$TEST_COMPONENTS" -f --toolchain arduino --fail-on-no-tests
 
   device-builder:
     name: Test downstream esphome/device-builder

.github/workflows/codeql.yml

@@ -56,7 +56,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
+        uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
         with:
           languages: ${{ matrix.language }}
           build-mode: ${{ matrix.build-mode }}
@@ -84,6 +84,6 @@ jobs:
           exit 1
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1
+        uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2
         with:
           category: "/language:${{matrix.language}}"

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ ci:
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.16.8
+    rev: v0.16.9
     hooks:
       # Run the linter.
       - id: ruff
@@ -18,7 +18,7 @@ repos:
       # Run the formatter.
       - id: ruff-format
   - repo: https://github.com/PyCQA/flake8
-    rev: 7.3.0
+    rev: 7.4.1
     hooks:
       - id: flake8
         additional_dependencies:

docker/Dockerfile

@@ -22,7 +22,7 @@ RUN \
     -r /requirements.txt
 
 # Install the ESPHome Device Builder dashboard.
-RUN uv pip install --no-cache-dir esphome-device-builder==1.17.0
+RUN uv pip install --no-cache-dir esphome-device-builder==1.17.2
 
 RUN \
     platformio settings set enable_telemetry No \

docker/docker_entrypoint.sh

@@ -21,10 +21,14 @@ export PLATFORMIO_PLATFORMS_DIR="${pio_cache_base}/platforms"
 export PLATFORMIO_PACKAGES_DIR="${pio_cache_base}/packages"
 export PLATFORMIO_CACHE_DIR="${pio_cache_base}/cache"
 
-# Keep the native toolchain installs on the persistent cache root, not the
-# container's ephemeral user cache dir (re-downloaded on every restart).
+# Keep the native toolchain installs and compiler caches on the persistent
+# cache root, not the container's user cache dir: it is lost on every
+# restart, and not writable when the container runs as a non-root user.
 export ESPHOME_ESP_IDF_PREFIX="$(dirname "${pio_cache_base}")/idf"
 export ESPHOME_SDK_NRF_PREFIX="$(dirname "${pio_cache_base}")/sdk-nrf"
+export ESPHOME_ARDUINO8266_PREFIX="$(dirname "${pio_cache_base}")/arduino8266"
+export ESPHOME_HOST_PREFIX="$(dirname "${pio_cache_base}")/host"
+export ESPHOME_PLATFORMIO_CCACHE_DIR="$(dirname "${pio_cache_base}")/platformio-ccache"
 
 # If /build is mounted, use that as the build path
 # otherwise use path in /config (so that builds aren't lost on container restart)

docker/ha-addon-rootfs/etc/s6-overlay/s6-rc.d/esphome/run

@@ -15,10 +15,14 @@ export PLATFORMIO_PLATFORMS_DIR="${pio_cache_base}/platforms"
 export PLATFORMIO_PACKAGES_DIR="${pio_cache_base}/packages"
 export PLATFORMIO_CACHE_DIR="${pio_cache_base}/cache"
 
-# Keep the native toolchain installs on the persistent /data volume, not the
-# container's ephemeral user cache dir (wiped on every add-on update/restart).
+# Keep the native toolchain installs and compiler caches on the persistent
+# /data volume, not the container's ephemeral user cache dir (wiped on every
+# add-on update/restart).
 export ESPHOME_ESP_IDF_PREFIX=/data/cache/idf
 export ESPHOME_SDK_NRF_PREFIX=/data/cache/sdk-nrf
+export ESPHOME_ARDUINO8266_PREFIX=/data/cache/arduino8266
+export ESPHOME_HOST_PREFIX=/data/cache/host
+export ESPHOME_PLATFORMIO_CCACHE_DIR=/data/cache/platformio-ccache
 
 if bashio::config.true 'leave_front_door_open'; then
     export DISABLE_HA_AUTHENTICATION=true

docker/test_configs/esp8266-arduino.yaml

@@ -3,5 +3,7 @@ esphome:
 
 esp8266:
   board: d1_mini
+  # The PlatformIO path stays covered whatever the default is
+  toolchain: platformio
 
 logger:

esphome/__main__.py

@@ -838,7 +838,7 @@ def compile_program(args: ArgsProtocol, config: ConfigType) -> int:
 
     # Keep this here, NOT in codegen: config-hash and --only-generate must keep
     # working on machines that cannot run the toolchain.
-    if CORE.is_esp8266:
+    if CORE.is_esp8266 and CORE.using_toolchain_platformio:
         from esphome.components.esp8266 import check_rosetta
 
         check_rosetta()
@@ -924,14 +924,11 @@ def _check_and_emit_build_info() -> None:
 
 def _get_configured_xtal_freq() -> int | None:
     """Read the configured crystal frequency from the sdkconfig file."""
+    from esphome.espidf import parse_sdkconfig
+
     sdkconfig_path = CORE.relative_build_path(f"sdkconfig.{CORE.name}")
-    if not sdkconfig_path.is_file():
-        return None
-    with suppress(OSError, ValueError):
-        content = sdkconfig_path.read_text()
-        for line in content.splitlines():
-            if line.startswith("CONFIG_XTAL_FREQ="):
-                return int(line.split("=", 1)[1])
+    with suppress(OSError, ValueError, KeyError):
+        return int(parse_sdkconfig(sdkconfig_path)["CONFIG_XTAL_FREQ"])
     return None
 
 
@@ -2367,7 +2364,8 @@ def parse_args(argv):
         metavar="{" + ",".join(t.value for t in Toolchain) + "}",
         help=(
             "Select toolchain for compiling. Overrides '<platform>.toolchain' in YAML. "
-            f"Default: {Toolchain.PLATFORMIO.value}."
+            "Default: the platform's native toolchain where it has one, else "
+            f"{Toolchain.PLATFORMIO.value}."
         ),
     )
 

esphome/analyze_memory/toolchain.py

@@ -37,7 +37,7 @@ def find_elf_path(build_path: Path) -> Path | None:
     """
     name = build_path.name
     for candidate in (
-        # Native ESP-IDF: idf.py writes build/<name>.elf, which ESPHome copies
+        # Native ESP-IDF: the build writes build/<name>.elf, which ESPHome copies
         # to build/firmware.elf (see espidf.toolchain.create_elf_copy)
         build_path / "build" / "firmware.elf",
         # PlatformIO

esphome/arduino8266/framework.py

@@ -3,12 +3,12 @@
 Artifacts land in a machine-global cache (shared across projects, like the
 ESP-IDF install in ``esphome.espidf.framework``):
 
-    <cache>/arduino8266/frameworks/<version>/   framework-arduinoespressif8266
-    <cache>/arduino8266/toolchains/<version>/   toolchain-xtensa (gcc 10.3)
+    <cache>/arduino8266/frameworks/<tag>/   the Arduino core
+    <cache>/arduino8266/toolchains/<tag>/   xtensa-lx106-elf gcc 10.3
 
-Packages come from the PlatformIO registry (identical bits to the PlatformIO
-backend); ``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs. ninja comes
-from PATH or the ninja PyPI wheel.
+Both come from esphome-libs releases pinned below;
+``ESPHOME_ARDUINO8266_*_MIRRORS`` overrides the URLs, with ``{VERSION}``
+standing for the release tag. ninja comes from PATH or the ninja PyPI wheel.
 """
 
 from __future__ import annotations
@@ -22,13 +22,68 @@ from esphome.build_helpers.ninja import find_ninja
 from esphome.build_helpers.tools_cache import ARDUINO8266_TOOLS_CACHE, tools_cache_path
 from esphome.core import EsphomeError, Version
 from esphome.framework_helpers import str_to_lst_of_str
-from esphome.platformio.registry import install_package, prefetch_packages
+from esphome.platformio.registry import (
+    Download,
+    Resolver,
+    get_systype,
+    install_package,
+    prefetch_packages,
+)
 
-FRAMEWORK_PACKAGE = "framework-arduinoespressif8266"
-TOOLCHAIN_PACKAGE = "toolchain-xtensa"
+FRAMEWORK_PACKAGE = "arduino-esp8266"
+_FRAMEWORK_RELEASES = "https://github.com/esphome-libs/arduino-esp8266/releases/"
+
+
+class FrameworkRelease(NamedTuple):
+    tag: str
+    sha256: str
+    size: int
+
+    def download(self) -> Download:
+        archive = f"{FRAMEWORK_PACKAGE}-{self.tag}.tar.gz"
+        url = f"{_FRAMEWORK_RELEASES}download/{self.tag}/{archive}"
+        return Download(url, self.sha256, self.size)
+
+
+# Arduino core version -> its build in esphome-libs/arduino-esp8266
+FRAMEWORK_RELEASES: dict[Version, FrameworkRelease] = {
+    Version(3, 1, 2): FrameworkRelease(
+        "3.1.2-esphome.1",
+        "e80751e3123676b967143e39c61f2d8693946db4c7806f2a83dcaaf797ecd582",
+        37189311,
+    ),
+}
+
+TOOLCHAIN_PACKAGE = "toolchain-xtensa-lx106-elf"
 # gcc 10.3, the toolchain Arduino core 3.x builds with; the build
 # generator's compile flags are tuned to it.
-TOOLCHAIN_VERSION = "2.100300.220621"
+TOOLCHAIN_VERSION = "10.3.0-esphome.2"
+_TOOLCHAIN_RELEASES = (
+    "https://github.com/esphome-libs/xtensa-lx106-elf-toolchain/releases/"
+)
+# Registry system tag -> (sha256, size) of that host's archive
+TOOLCHAIN_BUILDS: dict[str, tuple[str, int]] = {
+    "darwin_arm64": (
+        "849cede44d4d5c6ea0f14099783239f559f46327bea314281814f2652b486201",
+        60830321,
+    ),
+    "darwin_x86_64": (
+        "ca69904daabf0c5983b372423e5e62f49182a793e992c052e94666852470c897",
+        64149487,
+    ),
+    "linux_aarch64": (
+        "60a49a4f082bf246544bd409a9517dbbcab19bb30ac9decbee544b896aaccbd6",
+        67573397,
+    ),
+    "linux_x86_64": (
+        "1fba33ca1494ec79f2776e0e37eca93282d30f8bb9992f5f4f9a655d6fff1db4",
+        68431336,
+    ),
+    "windows_amd64": (
+        "af9066b0e5bf036f04f2bd9d08b89b81a7f183c57dac0abcaff71dd861cf5f3b",
+        67664137,
+    ),
+}
 
 ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS = str_to_lst_of_str(
     os.environ.get("ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS", "")
@@ -44,36 +99,40 @@ def get_arduino8266_tools_path() -> Path:
     return tools_cache_path(*ARDUINO8266_TOOLS_CACHE)
 
 
-# 3.1.1 rather than 3.1.0: the registry has no packages for 3.0.0, 3.0.1 or 3.1.0
-MIN_FRAMEWORK_VERSION = Version(3, 1, 1)
-
-
-def framework_package_version(ver: Version) -> str:
-    """Map an Arduino core version to its registry package version (3.1.2 ->
-    3.30102.0; the leading 3 is the package major).
-
-    Exact registry names for 3.x cores; callers floor at MIN_FRAMEWORK_VERSION.
-    """
-    if ver.major > 3:
+def framework_release(version: Version) -> FrameworkRelease:
+    if (release := FRAMEWORK_RELEASES.get(version)) is None:
         raise EsphomeError(
-            f"Arduino core {ver} is not supported yet; "
-            "the newest known core series is 3.x"
+            f"'toolchain: arduino' has no build of Arduino core {version}; "
+            f"available: {', '.join(str(v) for v in FRAMEWORK_RELEASES)}. "
+            "Use one of those or 'toolchain: platformio'"
         )
-    if ver.major < 3:
-        raise EsphomeError(
-            f"Arduino core {ver} is not supported; ESPHome requires core 3.x"
-        )
-    return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
+    return release
 
 
-def get_framework_path(package_version: str) -> Path:
-    return get_arduino8266_tools_path() / "frameworks" / package_version
+def get_framework_path(tag: str) -> Path:
+    return get_arduino8266_tools_path() / "frameworks" / tag
 
 
 def get_toolchain_path() -> Path:
     return get_arduino8266_tools_path() / "toolchains" / TOOLCHAIN_VERSION
 
 
+def toolchain_download() -> Download:
+    """The toolchain archive for the current host."""
+    systype = get_systype()
+    if (build := TOOLCHAIN_BUILDS.get(systype)) is None:
+        raise EsphomeError(
+            f"There is no ESP8266 toolchain for this system ({systype}); "
+            f"supported systems are {', '.join(sorted(TOOLCHAIN_BUILDS))}. "
+            "Either set 'toolchain: platformio' under 'esp8266:', or point "
+            "ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS at a toolchain archive"
+        )
+    sha256, size = build
+    archive = f"{TOOLCHAIN_PACKAGE}-{TOOLCHAIN_VERSION}-{systype}.tar.gz"
+    url = f"{_TOOLCHAIN_RELEASES}download/{TOOLCHAIN_VERSION}/{archive}"
+    return Download(url, sha256, size)
+
+
 class InstalledPaths(NamedTuple):
     """Locations of the installed framework, toolchain, and ninja binary."""
 
@@ -84,24 +143,17 @@ class InstalledPaths(NamedTuple):
 
 def check_and_install(framework_version: Version) -> InstalledPaths:
     """Ensure framework, toolchain, and ninja are installed; return their paths."""
-    if framework_version < MIN_FRAMEWORK_VERSION:
-        # Config validation enforces this too; keep the module honest when
-        # called directly.
-        raise EsphomeError(
-            f"The native toolchain requires the Arduino core "
-            f">= {MIN_FRAMEWORK_VERSION}, got {framework_version}"
-        )
+    release = framework_release(framework_version)
     # Probe the cheap local dependency before ~110 MB of downloads
     ninja_path = find_ninja()
-    package_version = framework_package_version(framework_version)
-    framework_path = get_framework_path(package_version)
+    framework_path = get_framework_path(release.tag)
     downloads_dir = get_arduino8266_tools_path() / "downloads"
     toolchain_path = get_toolchain_path()
     # One spec per package: the prefetch and the installs must agree
     specs = (
         (
             FRAMEWORK_PACKAGE,
-            package_version,
+            release.tag,
             framework_path,
             ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS,
             ("cores/esp8266", "tools/sdk", "libraries"),
@@ -115,10 +167,25 @@ def check_and_install(framework_version: Version) -> InstalledPaths:
             ("bin", "xtensa-lx106-elf"),
         ),
     )
+    # Resolved only when a download is needed, so an installed toolchain
+    # keeps working on a host without a build; a mirror override wins
+    resolvers: dict[str, Resolver] = {}
+    if not ESPHOME_ARDUINO8266_FRAMEWORK_MIRRORS:
+        resolvers[FRAMEWORK_PACKAGE] = release.download
+    if not ESPHOME_ARDUINO8266_TOOLCHAIN_MIRRORS:
+        resolvers[TOOLCHAIN_PACKAGE] = toolchain_download
     # Fetch both archives at once; the installs below verify and extract
-    prefetch_packages([spec[:4] for spec in specs], downloads_dir)
+    prefetch_packages([spec[:4] for spec in specs], downloads_dir, resolvers)
     for name, version, dest, mirrors, expect in specs:
-        install_package(name, version, dest, mirrors, downloads_dir, expect=expect)
+        install_package(
+            name,
+            version,
+            dest,
+            mirrors,
+            downloads_dir,
+            expect=expect,
+            resolve=resolvers.get(name),
+        )
     return InstalledPaths(
         framework=framework_path, toolchain=toolchain_path, ninja=ninja_path
     )

esphome/build_gen/arduino8266.py

@@ -41,7 +41,7 @@ from esphome.build_helpers.ninja_gen import (
     pch_rule_lines,
     tool_lines,
 )
-from esphome.build_helpers.pch import PCH_CORE_HEADER
+from esphome.build_helpers.pch import PCH_DEFAULT_HEADERS
 from esphome.components.esp8266 import build_surgery
 from esphome.components.esp8266.boards import (
     BOARDS,
@@ -1053,12 +1053,9 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
         + ["stdc++-exc" if config.exceptions else "stdc++", "m", "c", "gcc"]
     )
 
+    cxx = (toolchain_tool(paths.toolchain, "g++"),)
     lines = [
-        *tool_lines(
-            (toolchain_tool(paths.toolchain, "gcc"),),
-            (toolchain_tool(paths.toolchain, "g++"),),
-            ccache,
-        ),
+        *tool_lines((toolchain_tool(paths.toolchain, "gcc"),), cxx, ccache),
         *compile_rule_lines(),
         *ar_rule_lines(toolchain_tool(paths.toolchain, "ar")),
         *pch_rule_lines(),
@@ -1130,8 +1127,7 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
                 )
             src_includes.append(header)
         elif is_joined_include(tok):
-            # Left in src_other it would precede the pch include and
-            # silently defeat the .gch
+            # Left in src_other it would precede the pch include
             src_includes.append(tok[len("-include") :])
         else:
             src_other.append(_shell_token(tok))
@@ -1143,11 +1139,11 @@ def write_project(paths: InstalledPaths, ccache: str | None) -> bool:
         lines,
         build_dir,
         src_dir,
-        (*src_includes, PCH_CORE_HEADER),
+        (*src_includes, *PCH_DEFAULT_HEADERS),
         flag_sets.cxxflags,
         src_other,
-        ccache,
         (str(paths.framework), str(paths.toolchain)),
+        cxx,
     )
     src_objs = compile_edges(
         lines,

esphome/build_gen/build_tool.py

@@ -6,7 +6,6 @@ started esphome and must not depend on the package being importable.
 Subcommands:
     ar <ar-binary> <archive> <rspfile>   remove stale archive, then ``ar rcs``
     copy <src> <dst>                     copy a file
-    touch <path>                         create/update a stamp file
 
 The ar rspfile carries one object path per line (the generating rule must
 use ``$in_newline``, never ``$in``).
@@ -84,18 +83,9 @@ def _run_copy(src: str, dst: str) -> int:
     return 0
 
 
-def _run_touch(path: str) -> int:
-    try:
-        Path(path).touch()
-    except OSError as err:
-        print(f"touch: {path} failed: {err}", file=sys.stderr)
-        return 1
-    return 0
-
-
 # mode -> (handler, expected operand count); surplus argv means a
 # mis-specified ninja rule and must error, not silently drop operands
-_MODES = {"ar": (_run_ar, 3), "copy": (_run_copy, 2), "touch": (_run_touch, 1)}
+_MODES = {"ar": (_run_ar, 3), "copy": (_run_copy, 2)}
 
 
 def main() -> int:

esphome/build_gen/espidf.py

@@ -3,15 +3,10 @@
 import json
 import logging
 from pathlib import Path
+import re
+import textwrap
 
 from esphome.build_helpers import pch
-from esphome.build_helpers.pch import (
-    PCH_DEFAULT_HEADERS,
-    PCH_HEADER_NAME,
-    mark_pch_emitted,
-    pch_enabled,
-    pch_header_text,
-)
 from esphome.components.esp32 import (
     get_esp32_variant,
     get_excluded_builtin_components,
@@ -86,6 +81,81 @@ def _cmake_quote(value: str) -> str:
     return f'"{escaped}"'
 
 
+# CONFIG_APP_BUILD_BOOTLOADER is hidden and force-selected, so it can only be
+# cleared at the CMake level (the same state IDF's RAM-app build type uses).
+# The macro is IDF's __build_process_project_includes plus two added lines;
+# esphome.espidf.bootloader falls back to in-tree if an IDF update changes it.
+IDF_BOOTLOADER_OVERRIDE = """\
+# ESPHome cached bootloader switch; see esphome/espidf/bootloader.py.
+if(ESPHOME_USE_CACHED_BOOTLOADER)
+    macro(__build_process_project_includes)
+        idf_build_get_property(sdkconfig_cmake SDKCONFIG_CMAKE)
+        include(${sdkconfig_cmake})
+        set(CONFIG_APP_BUILD_BOOTLOADER "")
+        # bt's CMakeLists reads the lowercase idf_target that the (now
+        # skipped) bootloader project_include leaks; keep it defined, or
+        # its empty TARGET_SRC_NAME sends file(GLOB_RECURSE) across /.
+        idf_build_get_property(idf_target IDF_TARGET)
+        idf_build_get_property(build_properties __BUILD_PROPERTIES)
+        foreach(build_property ${build_properties})
+            idf_build_get_property(val ${build_property})
+            set(${build_property} "${val}")
+        endforeach()
+        idf_build_get_property(build_component_targets __BUILD_COMPONENT_TARGETS)
+        foreach(component_target ${build_component_targets})
+            __component_get_property(dir ${component_target} COMPONENT_DIR)
+            __component_get_property(_name ${component_target} COMPONENT_NAME)
+            set(COMPONENT_NAME ${_name})
+            set(COMPONENT_DIR ${dir})
+            set(COMPONENT_PATH ${dir})
+            if(EXISTS ${COMPONENT_DIR}/project_include.cmake)
+                include(${COMPONENT_DIR}/project_include.cmake)
+            endif()
+        endforeach()
+    endmacro()
+endif()
+"""
+
+# The lines the override adds to IDF's macro; the staleness tripwire in
+# esphome.espidf.bootloader strips them before comparing with the live macro.
+BOOTLOADER_OVERRIDE_ADDED_LINES = (
+    'set(CONFIG_APP_BUILD_BOOTLOADER "")',
+    "idf_build_get_property(idf_target IDF_TARGET)",
+)
+
+_MACRO = re.compile(
+    r"macro\(__build_process_project_includes\)(.*?)endmacro\(\)", re.DOTALL
+)
+
+
+def _normalized_macro(text: str) -> list[str] | None:
+    """The macro body as comment-free, whitespace-collapsed lines."""
+    if (match := _MACRO.search(text)) is None:
+        return None
+    return [
+        re.sub(r"\s+", " ", line)
+        for raw in match.group(1).splitlines()
+        if (line := raw.split("#", 1)[0].strip())
+    ]
+
+
+_EXPECTED_MACRO = [
+    line
+    for line in _normalized_macro(IDF_BOOTLOADER_OVERRIDE)
+    if line not in BOOTLOADER_OVERRIDE_ADDED_LINES
+]
+
+
+def idf_macro_matches() -> bool:
+    """Whether IDF's macro still matches the copy the override replays."""
+    from esphome.espidf import toolchain
+
+    # pylint: disable-next=protected-access
+    build_cmake = toolchain._get_idf_path() / "tools" / "cmake" / "build.cmake"
+    live = _normalized_macro(build_cmake.read_text(encoding="utf-8"))
+    return live == _EXPECTED_MACRO
+
+
 def get_project_cmakelists(
     minimal: bool = False, builtin_components: list[str] | None = None
 ) -> str:
@@ -208,6 +278,7 @@ set(EXTRA_COMPONENT_DIRS ${{CMAKE_SOURCE_DIR}}/src)
 
 include($ENV{{IDF_PATH}}/tools/cmake/project.cmake)
 
+{IDF_BOOTLOADER_OVERRIDE}
 {cpp_standard_options}
 
 {cxx_compile_options}
@@ -220,6 +291,13 @@ include($ENV{{IDF_PATH}}/tools/cmake/project.cmake)
 
 project({CORE.name})
 
+# Record the bootloader customization channels the cache cannot replay;
+# esphome.espidf.bootloader bypasses the cache when any is non-empty.
+idf_build_get_property(esphome_bl_dirs BOOTLOADER_EXTRA_COMPONENT_DIRS)
+idf_build_get_property(esphome_bl_args EXTRA_CMAKE_ARGS)
+file(WRITE ${{CMAKE_BINARY_DIR}}/esphome_bootloader_hooks.txt
+     "${{esphome_bl_dirs}}${{BOOTLOADER_IGNORE_EXTRA_COMPONENT}}${{esphome_bl_args}}")
+
 # Emit per-memory-type JSON size data for ESPHome to read post-build.
 # json2 stays small; raw dumps every symbol (~2s on a large map) and
 # this command runs inside the link edge, blocking everything downstream.
@@ -290,72 +368,70 @@ idf_component_register(
 target_link_options(${{COMPONENT_LIB}} PUBLIC
     {link_opts_str}
 )
-{_pch_cmake()}"""
+{_pch_cmake_block()}"""
 
 
-def _pch_cmake() -> str:
-    """The src component's precompiled-header block (C++ TUs only).
+# Where CMake puts the .gch of the src component; ccache reads the checksum
+# next to it in place of the .gch
+_PCH_SUM_PATH = "build/esp-idf/src/CMakeFiles/__idf_src.dir/cmake_pch.hxx.gch.sum"
+# Where the Windows gate records its choice
+_PCH_CHOICE_VAR = "ESPHOME_PCH"
 
-    The -include stays relative (resolved from the compiler cwd, the build
-    dir); an absolute path would poison ccache keys.
-    """
-    if not pch_enabled():
+
+def _pch_cmake_block() -> str:
+    """The CMake block that precompiles the core headers for the C++ sources
+    of the src component; empty when disabled."""
+    if not pch.pch_enabled():
         return ""
-    # Strict inverts: a per-process consumer rejection reds the build.
-    # Baked at generation: a knob flip takes effect when the CMakeLists is
-    # rewritten (every esphome compile); a hand-run idf.py keeps the old one
-    escalation = pch.pch_consumer_escalation()
+    headers = "\n".join(
+        f'    "$<$<COMPILE_LANGUAGE:CXX>:${{CMAKE_CURRENT_SOURCE_DIR}}/{header}>"'
+        for header in pch.PCH_DEFAULT_HEADERS
+    )
+    block = f"""target_precompile_headers(${{COMPONENT_LIB}} PRIVATE
+{headers}
+)"""
+    if not pch.pch_needs_gcc_check():
+        return f"\n# ESPHome precompiled header\n{block}\n"
+    # Before the first configure only CMake knows the compiler version
     return f"""
-# ESPHome precompiled header (see esphome/build_helpers/pch.py).
-# OBJECT_DEPENDS is on the header, not the .gch: pch-baked headers drop
-# out of TU depfiles, and prepare_pch() touches the header on rebuild.
-target_compile_options(${{COMPONENT_LIB}} PRIVATE
-    "$<$<COMPILE_LANGUAGE:CXX>:-Winvalid-pch>"
-    "$<$<COMPILE_LANGUAGE:CXX>:{escalation}>"
-    "$<$<COMPILE_LANGUAGE:CXX>:-include>"
-    "$<$<COMPILE_LANGUAGE:CXX>:{PCH_HEADER_NAME}>"
-)
-set_source_files_properties(${{app_sources}} PROPERTIES
-    OBJECT_DEPENDS "${{CMAKE_BINARY_DIR}}/{PCH_HEADER_NAME}")
+# ESPHome precompiled header, unless GCC bug 14940 keeps it from loading
+if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND ({pch.PCH_WINDOWS_CMAKE_OLD_GCC}))
+  message(STATUS "ESPHome: GCC ${{CMAKE_CXX_COMPILER_VERSION}} cannot load a precompiled header on Windows; compiling without it")
+  set({_PCH_CHOICE_VAR} OFF CACHE BOOL "ESPHome precompiled header in use" FORCE)
+else()
+  set({_PCH_CHOICE_VAR} ON CACHE BOOL "ESPHome precompiled header in use" FORCE)
+{textwrap.indent(block, "  ")}
+endif()
 """
 
 
-def discard_pch() -> None:
-    """Drop the pch sidecars in the IDF build dir."""
-    pch.discard_pch(CORE.relative_build_path("build"))
+def _read_if_exists(path: Path) -> str:
+    return path.read_text(encoding="utf-8") if path.is_file() else ""
 
 
-def prepare_pch() -> None:
-    """Build the .gch right before ninja, after every reconfigure, so the
-    compile_commands.json flags and the sdkconfig are the settled ones."""
-    if not pch_enabled():
-        # Self-cleaning escape hatch: drop any previously built .gch
-        pch.discard_pch(CORE.relative_build_path("build"))
-        pch.pch_disabled_degraded()
+def write_pch_checksum() -> None:
+    """Write the checksum ccache uses in place of the .gch: the core headers,
+    the framework version, the sdkconfig and the managed component versions."""
+    from esphome.espidf.toolchain import get_cmake_cache_value
+
+    if not pch.pch_enabled():
         return
-    sdkconfig_path = CORE.relative_build_path(f"sdkconfig.{CORE.name}")
-    try:
-        sdkconfig = sdkconfig_path.read_text(encoding="utf-8")
-    except OSError as err:
-        # Fail closed: the sdkconfig is the .sum's only config identity for
-        # sdkconfig.h-only options; a stand-in marker would collide
-        _LOGGER.warning(
-            "Could not read %s; compiling without the pch: %s", sdkconfig_path, err
-        )
-        pch.discard_pch(CORE.relative_build_path("build"))
-        pch.pch_degraded(f"sdkconfig unreadable: {err}")
+    # The gate's choice, cached by configure
+    if pch.pch_needs_gcc_check() and get_cmake_cache_value(_PCH_CHOICE_VAR) != "ON":
         return
-    pch.prepare_pch(
-        CORE.relative_build_path("build"),
-        PCH_DEFAULT_HEADERS,
+    pch.log_pch_in_use()
+    checksum = pch.pch_checksum(
+        CORE.relative_src_path(),
+        pch.PCH_DEFAULT_HEADERS,
         (
             str(idf_version()),
-            CORE.cpp_standard or "",
-            sdkconfig,
-            *get_project_compile_flags(),
-            *get_project_cxx_compile_flags(),
+            _read_if_exists(CORE.relative_build_path(f"sdkconfig.{CORE.name}")),
+            _read_if_exists(CORE.relative_build_path("dependencies.lock")),
         ),
     )
+    path = CORE.relative_build_path(_PCH_SUM_PATH)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    write_file_if_changed(path, checksum + "\n")
 
 
 def write_project(
@@ -377,14 +453,6 @@ def write_project(
         get_component_cmakelists(),
     )
 
-    if pch_enabled():
-        write_file_if_changed(
-            CORE.relative_build_path("build", PCH_HEADER_NAME),
-            pch_header_text(PCH_DEFAULT_HEADERS),
-        )
-        # Consumers carry the -include; gate the ccache relaxation on it
-        mark_pch_emitted()
-
     # Snapshot the exclusion set so has_outdated_files() can trigger a
     # discovery reconfigure when it changes. Excluded components never
     # register in project_description.json, so re-including one (e.g. a

esphome/build_gen/host.py

@@ -15,6 +15,7 @@ from collections.abc import Iterable
 import logging
 import os
 from pathlib import Path
+import subprocess
 import sys
 from typing import TYPE_CHECKING
 
@@ -32,7 +33,7 @@ from esphome.build_helpers.ninja_gen import (
     pch_rule_lines,
     tool_lines,
 )
-from esphome.build_helpers.pch import PCH_CORE_HEADER
+from esphome.build_helpers.pch import PCH_DEFAULT_HEADERS, pch_enabled
 from esphome.core import CORE, EsphomeError
 from esphome.framework_helpers import get_project_cxx_compile_flags
 from esphome.helpers import mkdir_p, write_file_if_changed
@@ -192,6 +193,15 @@ def _file_macro_maps(build_dir: Path) -> list[str]:
     ]
 
 
+def _compiler_version(cxx: tuple[str, ...]) -> str:
+    """What the compiler says it is: its path can stay the same across an
+    update (the macOS shims in /usr/bin)."""
+    result = subprocess.run(
+        [*cxx, "--version"], capture_output=True, text=True, check=False
+    )
+    return result.stdout
+
+
 def write_project(compilers: HostCompilers, ccache: str | None) -> bool:
     """Write the ninja build for the current configuration.
 
@@ -255,18 +265,15 @@ def write_project(compilers: HostCompilers, ccache: str | None) -> bool:
 
     archives, direct_objs = library_edges(lines, libraries)
 
-    from esphome.components.host import HOST_PCH_PREFIX
-
-    # Host has no framework force-includes; the per-TU cost is the STL
-    # closure behind the core headers, so those are what is precompiled
     src_cxx_override = pch_edges(
         lines,
         build_dir,
         src_dir,
-        (HOST_PCH_PREFIX, PCH_CORE_HEADER),
-        cxxflags,
+        PCH_DEFAULT_HEADERS,
+        # The arguments of a CXX override come before the flags
+        [*compilers.cxx[1:], *cxxflags],
         (),
-        ccache,
+        (compilers.cxx[0], _compiler_version(compilers.cxx)) if pch_enabled() else (),
         compilers.cxx,
     )
     src_objs = compile_edges(

esphome/build_gen/platformio.py

@@ -1,6 +1,8 @@
+from esphome.build_helpers.pch import pch_script_enabled
 from esphome.const import __version__
 from esphome.core import CORE
 from esphome.helpers import mkdir_p, read_file, write_file_if_changed
+from esphome.platformio.toolchain import copy_pch_script
 from esphome.writer import find_begin_end
 
 INI_AUTO_GENERATE_BEGIN = "; ========== AUTO GENERATED CODE BEGIN ==========="
@@ -62,6 +64,8 @@ def get_ini_content():
 
     # Add extra script for C++ flags
     CORE.add_platformio_option("extra_scripts", [f"pre:{CXX_FLAGS_FILE_NAME}"])
+    if pch_script_enabled():
+        CORE.add_platformio_option("extra_scripts", ["post:pch.py"])
 
     # Add CMake args. A user-supplied value (str or list) is deliberately
     # replaced; this option was always overwritten at FINAL priority.
@@ -106,6 +110,8 @@ def write_project():
 
     # Write extra script for C++ specific flags
     write_cxx_flags_script()
+    if pch_script_enabled():
+        copy_pch_script()
 
 
 CXX_FLAGS_FILE_NAME = "cxx_flags.py"

esphome/build_helpers/ccache.py

@@ -21,13 +21,12 @@ def _ccache_runs(ccache: str) -> bool:
     )
 
 
-def parse_enable_env(name: str, strict: bool = False) -> bool | None:
+def parse_enable_env(name: str) -> bool | None:
     """Strictly parse an on/off environment knob; None when unset or invalid.
 
     ``bool(str)`` truthiness would flip ``no``/``off`` to enabled, so only
     1/true/yes/on and 0/false/no/off count; anything else warns and reads
-    as unset so the caller's default policy applies — or raises when
-    ``strict`` (a typo must not silently disable a CI gate).
+    as unset so the caller's default policy applies.
     """
     raw = os.environ.get(name)
     if raw is None:
@@ -40,10 +39,6 @@ def parse_enable_env(name: str, strict: bool = False) -> bool | None:
         return True
     if lowered in FALSY_ENV_STRINGS:
         return False
-    if strict:
-        from esphome.core import EsphomeError
-
-        raise EsphomeError(f"Unrecognized {name}={raw!r}; use 1 or 0")
     _LOGGER.warning("Ignoring unrecognized %s=%r; use 1 or 0", name, raw)
     return None
 
@@ -108,14 +103,13 @@ def ccache_defaults_env(cache_dir: Path) -> dict[str, str]:
 
 
 def effective_ccache_basedir() -> str:
-    """The prefix ccache rewrites out of hashed paths: a user CCACHE_BASEDIR
-    wins, else the resolved build path (matching ccache_defaults_env)."""
+    """The prefix ccache strips from hashed paths: a usable user
+    CCACHE_BASEDIR, else the resolved build path."""
     from esphome.core import CORE
 
     raw = os.environ.get("CCACHE_BASEDIR")
     if raw is not None and Path(raw).is_absolute() and len(Path(raw).parts) > 1:
         return raw
-    # Unset or degenerate ("", "/", relative): fall back to the build path
     return str(Path(CORE.build_path).resolve())
 
 
@@ -124,7 +118,8 @@ def ccache_env(ccache: str | None, tools_cache: tuple[str, str]) -> dict[str, st
 
     ``ccache`` is the pre-resolved binary (resolve_ccache_path), or None when
     disabled; ``tools_cache`` is the backend's tools cache spec, which holds
-    its ccache dir.
+    its ccache dir. The pch settings include ``time_macros``: a cached
+    object can keep an older ``__DATE__`` or ``__TIME__``.
     """
     if ccache is None:
         return {}

esphome/build_helpers/idedata.py

@@ -50,11 +50,11 @@ def warn_if_idedata_missing(get_idedata: Callable[[], dict | None]) -> None:
         _LOGGER.debug("Idedata failure detail", exc_info=True)
 
 
-# C++ translation-unit suffixes.
-CXX_SOURCE_SUFFIXES = (".cpp", ".cc", ".cxx")
+# C++ translation-unit suffixes used to identify ESPHome source files.
+_CXX_SUFFIXES = (".cpp", ".cc")
 # Suffixes of input/output files that appear bare on the command line (and so
 # must not be mistaken for compiler flags).
-_INPUT_FILE_SUFFIXES = (*CXX_SOURCE_SUFFIXES, ".c", ".o", ".S", ".s")
+_INPUT_FILE_SUFFIXES = (*_CXX_SUFFIXES, ".c", ".o", ".S", ".s")
 # Path marker identifying an ESPHome source translation unit.
 _ESPHOME_SRC_MARKER = "/src/esphome/"
 
@@ -63,11 +63,11 @@ def _is_esphome_src(file: str) -> bool:
     """Whether ``file`` is an ESPHome C++ translation unit; normalized to
     ``/`` first since Windows compile DBs use backslashes."""
     return _ESPHOME_SRC_MARKER in file.replace("\\", "/") and file.endswith(
-        CXX_SOURCE_SUFFIXES
+        _CXX_SUFFIXES
     )
 
 
-def split_command(command: str) -> list[str]:
+def _split_command(command: str) -> list[str]:
     r"""Tokenize a compile_commands.json / response-file command string.
 
     On Windows, tokenize per Windows ``argv`` rules via ``CommandLineToArgvW``.
@@ -103,7 +103,7 @@ def split_command(command: str) -> list[str]:
         ctypes.windll.kernel32.LocalFree(argv)
 
 
-def expand_response_files(tokens: list[str], directory: Path) -> list[str]:
+def _expand_response_files(tokens: list[str], directory: Path) -> list[str]:
     """Inline any ``@response-file`` arguments (paths relative to ``directory``).
 
     GCC response files embed flags that must be expanded so GCC-only flags
@@ -118,8 +118,8 @@ def expand_response_files(tokens: list[str], directory: Path) -> list[str]:
                 rf = directory / rf
             try:
                 out.extend(
-                    expand_response_files(
-                        split_command(rf.read_text(encoding="utf-8")), directory
+                    _expand_response_files(
+                        _split_command(rf.read_text(encoding="utf-8")), directory
                     )
                 )
                 continue
@@ -138,7 +138,7 @@ def _pick_entry(entries: list[dict]) -> dict:
         if _is_esphome_src(entry["file"]):
             return entry
     for entry in entries:
-        if entry["file"].endswith(CXX_SOURCE_SUFFIXES):
+        if entry["file"].endswith(_CXX_SUFFIXES):
             return entry
     raise ValueError("no C++ translation unit found in compile_commands.json")
 
@@ -148,7 +148,7 @@ def _pick_entry(entries: list[dict]) -> dict:
 _LAUNCHER_STEMS = frozenset({"ccache", "sccache", "distcc", "icecc", "buildcache"})
 
 
-def is_launcher(token: str) -> bool:
+def _is_launcher(token: str) -> bool:
     return Path(token).stem.lower() in _LAUNCHER_STEMS
 
 
@@ -166,7 +166,7 @@ def parse_entry(
 ) -> tuple[str, list[str], list[str], list[str]]:
     """Parse one compile_commands entry -> (cxx_path, defines, includes, cxx_flags)."""
     directory = Path(entry["directory"])
-    tokens = expand_response_files(split_command(entry["command"]), directory)
+    tokens = _expand_response_files(_split_command(entry["command"]), directory)
 
     def _include(raw: str) -> str:
         # Resolve against the entry's ``directory`` so cached idedata works
@@ -182,7 +182,7 @@ def parse_entry(
     if not tokens:
         # An empty command, or one that was only the launcher; fail by name
         raise ValueError(f"empty compile command for {entry.get('file')}")
-    if is_launcher(tokens[0]) and len(tokens) > 1 and not tokens[1].startswith("-"):
+    if _is_launcher(tokens[0]) and len(tokens) > 1 and not tokens[1].startswith("-"):
         # Stale DB built with a launcher this run no longer configures; the
         # real compiler is the next token
         _LOGGER.warning("Stripping unconfigured launcher %s", tokens[0])
@@ -195,23 +195,11 @@ def parse_entry(
     defines: list[str] = []
     includes: list[str] = []
     cxx_flags: list[str] = []
-    unresolved_force_includes: list[str] = []
 
     it = iter(tokens[1:])
     for tok in it:
         if tok in ("-c", "-o"):
             next(it, None)  # drop the flag and its argument (input/output)
-        elif tok == "-include" or is_joined_include(tok):
-            # Re-anchor only names next to the compile (the pch); a name
-            # meant for the -I chain must stay untouched
-            raw = next(it, "") if tok == "-include" else tok[len("-include") :]
-            if not raw:
-                _LOGGER.warning("Dropping -include with no argument")
-            elif Path(resolved := _include(raw)).is_file():
-                cxx_flags.extend(("-include", resolved))
-            else:
-                unresolved_force_includes.append(raw)
-                cxx_flags.extend(("-include", raw))
         elif tok.startswith("-D"):
             # ``.strip()`` handles tokens like ``-D CONFIGURED=1`` (a single
             # quoted arg with a space after -D) that some flags arrive as.
@@ -230,14 +218,6 @@ def parse_entry(
             pass  # input/output files
         else:
             cxx_flags.append(tok)
-    for raw in unresolved_force_includes:
-        # A deleted build artifact would otherwise surface only downstream
-        if not any((Path(inc) / raw).is_file() for inc in includes):
-            _LOGGER.warning(
-                "-include %s found neither next to the compile nor on the "
-                "include path; cached idedata may not resolve it",
-                raw,
-            )
     return cxx_path, defines, includes, cxx_flags
 
 
@@ -302,7 +282,7 @@ def _cache_usable(cached: object) -> bool:
     if not isinstance(cached, dict) or "cc_path" not in cached:
         return False
     cxx_path = cached.get("cxx_path")
-    if not isinstance(cxx_path, str) or is_launcher(cxx_path):
+    if not isinstance(cxx_path, str) or _is_launcher(cxx_path):
         return False
     includes = cached.get("includes")
     return isinstance(includes, dict) and isinstance(includes.get("build"), list)
@@ -350,7 +330,7 @@ def load_or_build_idedata(
 def reject_launcher_compiler(cxx_path: str) -> None:
     """Reject a compile DB naming a launcher (ccache) as the compiler; it
     must never be probed, cached, or consumed."""
-    if is_launcher(cxx_path):
+    if _is_launcher(cxx_path):
         raise EsphomeError(
             f"compile_commands.json names the launcher {cxx_path} as the "
             "compiler; the compile database is unusable"

esphome/build_helpers/ninja_gen.py

@@ -13,25 +13,21 @@ from pathlib import Path
 import sys
 from typing import TYPE_CHECKING
 
-from esphome.build_helpers.ccache import effective_ccache_basedir
+from esphome.build_helpers.idedata import is_joined_include
 from esphome.build_helpers.ninja import (
     escape as _e,
     quote_path as _q,
     shell_token as _shell_token,
 )
 from esphome.build_helpers.pch import (
-    PCH_HEADER_NAME,
-    mark_pch_emitted,
-    pch_checksum,
-    pch_consumer_escalation,
-    pch_degraded,
-    pch_disabled_degraded,
-    pch_enabled,
-    pch_header_text,
-    pch_probe_args,
-    pch_strict,
+    PCH_GCH_NAME,
+    PCH_SUM_NAME,
+    log_pch_in_use,
+    pch_consumer_flags,
+    pch_identity,
+    pch_usable,
+    write_pch_headers,
 )
-from esphome.core import CORE
 from esphome.framework_helpers import strip_win_long_path_prefix
 from esphome.helpers import write_file_if_changed
 from esphome.platformio.library import SOURCE_KIND_FOR_SUFFIX
@@ -112,8 +108,7 @@ def compile_rule_lines() -> list[str]:
 def pch_rule_lines() -> list[str]:
     """The precompiled header rule, for a generator that emits one."""
     return [
-        # No $ccache: the .gch is compiled once per build dir and ccache
-        # cannot cache it usefully (its bytes embed build-dir paths)
+        # No $ccache: the .gch embeds build dir paths
         "rule pch",
         "  command = $cxx -MMD -MF $out.d -x c++-header $cxxflags $flags -c $in -o $out",
         "  depfile = $out.d",
@@ -129,97 +124,43 @@ def pch_edges(
     headers: Sequence[str],
     cxxflags: Sequence[str],
     src_flags: Sequence[str],
-    ccache: str | None,
     identity: Sequence[str],
+    cxx: Sequence[Path | str],
 ) -> tuple[str, str] | None:
     """Emit the precompiled header for the C++ src edges.
 
     ``headers`` are folded into one prefix header, ``src_flags`` are the
-    flags every src edge carries, and ``identity`` names what the compile
-    depends on beyond the headers and flags (versioned install paths).
-    Returns the ``cxx_override`` for ``compile_edges``, or None when the
-    build goes without a precompiled header.
+    flags every src edge carries, ``identity`` names what else the compile
+    depends on, the compiler included, and ``cxx`` is what the host rule
+    asks. Returns the ``cxx_override`` for ``compile_edges``, or None
+    without a pch.
     """
-    if not pch_enabled():
-        pch_disabled_degraded()
+    if not pch_usable(cxx):
         return None
-    if any(tok.startswith("-include") for tok in cxxflags):
-        # $cxxflags expands first, so a user -include there means GCC would
-        # never load the .gch
+    if any(
+        tok == "-include" or tok.startswith("--include") or is_joined_include(tok)
+        for tok in cxxflags
+    ):
+        # $cxxflags expands first and GCC only loads a .gch for the first
+        # -include
         _LOGGER.warning(
-            "A -include in build_flags prevents the precompiled header from "
+            "A -include in the compiler flags prevents the precompiled header from "
             "loading; compiling without it"
         )
-        pch_degraded("a user -include precedes the pch")
         return None
-    pch_header = build_dir / PCH_HEADER_NAME
-    pch_text = pch_header_text(headers)
-    checksum = None
-    try:
-        if ccache:
-            # The .sum exists only for CCACHE_PCH_EXTSUM; ninja's depfile
-            # handles staleness. Strip resolved and raw build paths
-            # (symlinks) so identical configs share cache entries
-            flags_id = (
-                " ".join(cxxflags)
-                .replace(effective_ccache_basedir(), "")
-                .replace(str(CORE.build_path), "")
-            )
-            # The header text covers include order
-            checksum = pch_checksum(src_dir, headers, (pch_text, *identity, flags_id))
-    except (OSError, UnicodeError) as err:
-        # Identity unknown: a stale cache entry must never be served
-        _LOGGER.warning(
-            "Could not establish the pch identity; compiling without it: %s", err
-        )
-        pch_degraded(f"identity unknown: {err}")
-        return None
-    _LOGGER.info(
-        "Compiling with a precompiled header (set ESPHOME_PCH_ENABLE=0 to disable)"
+    log_pch_in_use()
+    source = write_pch_headers(build_dir, headers)
+    write_file_if_changed(
+        build_dir / PCH_SUM_NAME,
+        pch_identity([*cxxflags, *src_flags], src_dir, tuple(headers), identity) + "\n",
     )
-    write_file_if_changed(pch_header, pch_text)
-    sum_path = build_dir / f"{PCH_HEADER_NAME}.gch.sum"
-    if checksum is not None:
-        # Generate-time stamp: a hand-run ninja can rebuild the .gch
-        # while this .sum lags
-        write_file_if_changed(sum_path, checksum + "\n")
-    else:
-        # A stale .sum from an earlier ccache run must not survive
-        sum_path.unlink(missing_ok=True)
-    gch = _e(f"{PCH_HEADER_NAME}.gch")
-    lines.append(f"build {gch}: pch {_e(pch_header)}")
+    gch = _e(PCH_GCH_NAME)
+    # The checksum file changes with anything the .gch depends on
+    lines.append(f"build {gch}: pch {_e(source)} | {_e(PCH_SUM_NAME)}")
     if src_flags:
         lines.append(f"  flags = {' '.join(src_flags)}")
-    # Relative -include: absolute would break cross-device ccache.
-    # -Wno-error keeps a rejected .gch a warning under user -Werror;
-    # strict inverts it so any consumer rejection reds the build
-    # (rejection is per-process, so the probe alone cannot prove
-    # the consumers)
-    cxx_parts = [
-        *src_flags,
-        f"-Winvalid-pch {pch_consumer_escalation()} -include {PCH_HEADER_NAME}",
-    ]
-    lines.append(f"srccxxflags = {' '.join(cxx_parts)}")
-    pch_dep = gch
-    if pch_strict():
-        # Consumers wait on the probe stamp, so an unloadable .gch
-        # reds the build here instead of warning ~100 times
-        probe = " ".join(pch_probe_args(PCH_HEADER_NAME, source=os.devnull))
-        lines.append("rule pchprobe")
-        # $out only expands in rule text, hence the inline stamp
-        lines.append(
-            f"  command = $cxx $cxxflags $flags {probe}"
-            " && $python $buildtool touch $out"
-        )
-        lines.append("  description = PCHPROBE $out")
-        # Runs when the .gch is (re)built; strict consumer -Werror
-        # covers a cached .gch this process cannot load
-        lines.append(f"build esphome_pch.probe: pchprobe {gch}")
-        if src_flags:
-            lines.append(f"  flags = {' '.join(src_flags)}")
-        pch_dep = f"{gch} esphome_pch.probe"
-    mark_pch_emitted()
-    return ("$srccxxflags", pch_dep)
+    lines.append(f"srccxxflags = {' '.join([*src_flags, *pch_consumer_flags()])}")
+    return ("$srccxxflags", gch)
 
 
 def ar_rule_lines(ar: Path | str) -> list[str]:

esphome/build_helpers/pch.py

@@ -1,100 +1,62 @@
-"""Shared precompiled-header policy for the build backends.
-
-The prefix either mirrors the TUs' own force-includes (ESP8266) or is a
-curated core-header set (ESP-IDF). ``esphome: includes:`` sources receive
-it too; Arduino.h visibility there is intended (esphome#8693).
-"""
+"""Shared precompiled header policy for the build backends."""
 
 from __future__ import annotations
 
-from collections.abc import Iterable
-from contextlib import suppress
-from dataclasses import dataclass
+from collections.abc import Iterable, Sequence
 import hashlib
-import json
 import logging
 import os
 from pathlib import Path
 import posixpath
 import re
-import stat
 import subprocess
+import sys
 
 from esphome.build_helpers.ccache import effective_ccache_basedir, parse_enable_env
-from esphome.build_helpers.idedata import (
-    CXX_SOURCE_SUFFIXES,
-    expand_response_files,
-    is_launcher,
-    split_command,
-)
-
-_DOMAIN = "pch"
-
-
-@dataclass
-class _PCHData:
-    emitted: bool = False
-
-
-def _pch_data() -> _PCHData:
-    from esphome.core import CORE
-
-    if _DOMAIN not in CORE.data:
-        CORE.data[_DOMAIN] = _PCHData()
-    return CORE.data[_DOMAIN]
-
-
-def mark_pch_emitted() -> None:
-    """Record that this build's consumers reference the pch."""
-    _pch_data().emitted = True
-
+from esphome.const import PLATFORM_NRF52
+from esphome.helpers import write_file_if_changed
 
 _LOGGER = logging.getLogger(__name__)
 
-# The header and its .gch/.sum sidecars live in the build directory.
+# The header and its sidecars live in the build directory
 PCH_HEADER_NAME = "esphome_pch.h"
+PCH_GCH_NAME = f"{PCH_HEADER_NAME}.gch"
+# ccache hashes this instead of the .gch; also the freshness stamp
+PCH_SUM_NAME = f"{PCH_GCH_NAME}.sum"
+# The include list the .gch is compiled from
+PCH_SOURCE_NAME = "esphome_pch_src.h"
 
-# Every artifact the pch machinery can leave behind, for cleanup.
-PCH_ARTIFACT_NAMES = (
-    PCH_HEADER_NAME,
-    f"{PCH_HEADER_NAME}.gch",
-    f"{PCH_HEADER_NAME}.gch.sum",
-    f"{PCH_HEADER_NAME}.gch.failed",
+# GCC can skip a .gch without a diagnostic and read the header of the same
+# name, so that header is an error. Other tools get the include list.
+PCH_GUARD_TEXT = f"""\
+#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTELLISENSE__)
+#error "The precompiled header was not loaded"
+#else
+#include "{PCH_SOURCE_NAME}"
+#endif
+"""
+
+# The cc1plus wrapper the PlatformIO script writes on arm64 macOS
+PCH_CC1_DIR = "pch_cc1"
+
+# What the PlatformIO script leaves in the project root, for cleanup
+PCH_ARTIFACT_NAMES = (PCH_HEADER_NAME, PCH_GCH_NAME, PCH_SUM_NAME, PCH_SOURCE_NAME)
+PCH_ARTIFACT_DIRS = (PCH_CC1_DIR,)
+
+# The core headers every backend precompiles
+PCH_DEFAULT_HEADERS = ("esphome/core/pch_prefix.h",)
+
+# PlatformIO platforms that do not take the pch script
+PCH_SCRIPT_EXCLUDED_PLATFORMS = frozenset(
+    {
+        PLATFORM_NRF52,
+    }
 )
 
-# The core defines header every backend anchors its prefix on.
-PCH_CORE_HEADER = "esphome/core/defines.h"
-
-# Prefix-header contents for backends that inject a curated set (rather
-# than mirroring the TUs' own force-includes), defines.h first so USE_*
-# macros exist for the rest. Deliberately hard-coded: frequency-derived
-# sets measured no better and kept selecting headers that cannot compile
-# standalone (X-macro, platform-variant). Every entry must be safe to
-# include first in an empty TU. Caveat: application.h/automation.h become
-# ambiently visible, so a TU missing those #includes still builds on such
-# backends; ESPHOME_PCH_ENABLE=0 restores the strict view.
-PCH_DEFAULT_HEADERS = (
-    PCH_CORE_HEADER,
-    "esphome/core/component.h",
-    "esphome/core/helpers.h",
-    "esphome/core/log.h",
-    "esphome/core/application.h",
-    "esphome/core/automation.h",
-)
-
-# ccache cannot hash through a .gch; CCACHE_PCH_EXTSUM makes it hash the
-# .sum sidecar instead of the .gch bytes, which are not reproducible.
-# Keep in sync with the literals in platformio/pch.py.script.
-_CCACHE_PCH_ENV = {
-    "CCACHE_SLOPPINESS": "pch_defines,time_macros",
-    "CCACHE_PCH_EXTSUM": "true",
-}
-
-# Both include forms: an angle include resolving under src/ must enter the
-# digest too; ones that do not resolve simply end the walk
-# Compiler failures that clear on their own must not latch the .failed marker
-_TRANSIENT_ERRORS = ("No space left", "Cannot allocate", "Resource temporarily")
+# What ccache needs to cache compiles that load a .gch
+_CCACHE_PCH_SLOPPINESS = ("pch_defines", "time_macros")
 
+# Both include forms: an angle include resolving under src/ enters the digest
 _INCLUDE_RE = re.compile(rb'^\s*#\s*include\s+["<]([^">]+)[">]', re.MULTILINE)
 
 
@@ -103,92 +65,104 @@ def pch_enabled() -> bool:
     return parse_enable_env("ESPHOME_PCH_ENABLE") is not False
 
 
-def pch_strict() -> bool:
-    """CI knob: ``ESPHOME_PCH_STRICT=1`` turns pch degrade paths fatal.
-
-    A set-but-unrecognized value raises: a typo must not silently turn
-    the gate into a no-op that proves nothing.
-    """
-    return parse_enable_env("ESPHOME_PCH_STRICT", strict=True) is True
+def pch_forced() -> bool:
+    """``ESPHOME_PCH_ENABLE=1``: wanted even where the host rule says no."""
+    return parse_enable_env("ESPHOME_PCH_ENABLE") is True
 
 
-def pch_degraded(reason: str) -> None:
-    """Every degrade path funnels through here; strict mode raises."""
-    if pch_strict():
-        from esphome.core import EsphomeError
-
-        raise EsphomeError(f"ESPHOME_PCH_STRICT: {reason}")
+# GCC bug 14940: before these releases the Windows loader maps a .gch only
+# at its saved address. First fixed release per major, 16 on always fixed;
+# PCH_WINDOWS_CMAKE_OLD_GCC and the pch_usable message spell the same table
+PCH_WINDOWS_GCC_FIXED = {14: (14, 4), 15: (15, 3)}
+PCH_WINDOWS_GCC_FIXED_DEFAULT = (16, 0)
+# The same rule for CMake, which alone knows the version before configure
+PCH_WINDOWS_CMAKE_OLD_GCC = (
+    "CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14.4 OR "
+    "(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 15 AND "
+    "CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15.3)"
+)
 
 
-def pch_disabled_degraded() -> None:
-    """Strict CI must not read "no pch at all" as success."""
-    pch_degraded("pch disabled by ESPHOME_PCH_ENABLE")
+def gcc_relocates_pch_on_windows(version: Sequence[int]) -> bool:
+    """Whether a GCC of this version loads a .gch on Windows."""
+    if not version:
+        return False
+    fixed = PCH_WINDOWS_GCC_FIXED.get(version[0], PCH_WINDOWS_GCC_FIXED_DEFAULT)
+    return tuple(version[:2]) >= fixed
 
 
-def pch_probe_tail(source: str = "-") -> list[str]:
-    """The syntax-only compile shared by the probe and its baseline."""
-    return ["-fsyntax-only", "-x", "c++", source]
+# GCC ends the first --version line with its version; clang names itself
+_VERSION_RE = re.compile(r"\d+(?:\.\d+)+")
 
 
-def pch_probe_args(header: str, source: str = "-") -> list[str]:
-    """Flags that load-check a built .gch via a syntax-only compile.
-
-    Rejection must be a nonzero exit (never just a wording match), so the
-    invalid-pch class is always escalated. ``source`` defaults to stdin
-    (host independent); the ninja probe edge passes a real file.
-    """
-    return [
-        "-Winvalid-pch",
-        "-Werror=invalid-pch",
-        "-include",
-        header,
-        *pch_probe_tail(source),
-    ]
+def gcc_version(cxx: Sequence[Path | str]) -> tuple[int, ...] | None:
+    """The GCC version from ``--version``: () when it cannot be read, None
+    for a compiler that is not GCC."""
+    try:
+        result = subprocess.run(
+            [*cxx, "--version"], capture_output=True, text=True, check=False
+        )
+    except OSError as err:
+        _LOGGER.debug("Cannot run %s: %s", cxx[0], err)
+        return ()
+    banner = result.stdout.partition("\n")[0]
+    if "clang" in banner.lower():
+        return None
+    found = _VERSION_RE.findall(banner)
+    return tuple(int(part) for part in found[-1].split(".")) if found else ()
 
 
-def pch_consumer_escalation() -> str:
-    """Consumer-side invalid-pch flag: strict reds the build on rejection
-    (per-process, so the probe alone cannot prove the consumers)."""
-    return "-Werror=invalid-pch" if pch_strict() else "-Wno-error=invalid-pch"
+def pch_needs_gcc_check() -> bool:
+    """Windows host with the knob unset: the compiler version decides."""
+    return sys.platform == "win32" and parse_enable_env("ESPHOME_PCH_ENABLE") is None
+
+
+def pch_usable(cxx: Sequence[Path | str]) -> bool:
+    """The knob plus the host rule; ``ESPHOME_PCH_ENABLE=1`` skips the rule."""
+    if not pch_enabled():
+        return False
+    if not pch_needs_gcc_check():
+        return True
+    version = gcc_version(cxx)
+    if version is None or gcc_relocates_pch_on_windows(version):
+        return True
+    _LOGGER.info(
+        "GCC %s cannot load a precompiled header on Windows (GCC bug 14940, "
+        "fixed in 14.4, 15.3 and 16); compiling without it "
+        "(set ESPHOME_PCH_ENABLE=1 to force)",
+        ".".join(map(str, version)) or "of unknown version",
+    )
+    return False
+
+
+def pch_consumer_flags() -> list[str]:
+    """Flags a C++ src compile loads the pch with. The -include stays
+    relative: an absolute path would enter the ccache key."""
+    return ["-Winvalid-pch", "-Werror=invalid-pch", "-include", PCH_HEADER_NAME]
 
 
 def ccache_pch_env() -> dict[str, str]:
-    """Settings ccache needs to cache compiles that consume the .gch;
-    empty unless this build actually emitted one. User-set values win.
-    Native backends export these process-wide; only time_macros affects
-    non-pch TUs."""
-    if not (pch_enabled() and _pch_data().emitted):
+    """What ccache needs to cache compiles that load a .gch, added to what
+    the user already set."""
+    if not pch_enabled():
         return {}
-    extsum = os.environ.get("CCACHE_PCH_EXTSUM")
-    if extsum is not None and extsum.strip().lower() not in ("1", "true", "yes", "on"):
-        # ccache then hashes the non-reproducible .gch bytes: permanent misses
-        _LOGGER.warning("CCACHE_PCH_EXTSUM=%s disables pch caching", extsum)
-    env = {k: v for k, v in _CCACHE_PCH_ENV.items() if k not in os.environ}
-    user_sloppiness = os.environ.get("CCACHE_SLOPPINESS")
-    if user_sloppiness is not None and (
-        missing := [
-            t
-            for t in ("pch_defines", "time_macros")
-            if t not in {tok.strip() for tok in user_sloppiness.split(",")}
-        ]
-    ):
-        # Without these ccache declines every pch-consuming compile
-        env["CCACHE_SLOPPINESS"] = ",".join((user_sloppiness, *missing))
-        _LOGGER.warning(
-            "Adding %s to CCACHE_SLOPPINESS so ccache can cache compiles "
-            "that use the precompiled header",
-            ",".join(missing),
-        )
+    sloppiness = [
+        item.strip()
+        for item in os.environ.get("CCACHE_SLOPPINESS", "").split(",")
+        if item.strip()
+    ]
+    sloppiness += [item for item in _CCACHE_PCH_SLOPPINESS if item not in sloppiness]
+    env = {"CCACHE_SLOPPINESS": ",".join(sloppiness)}
+    if "CCACHE_PCH_EXTSUM" not in os.environ:
+        env["CCACHE_PCH_EXTSUM"] = "true"
     return env
 
 
-def pch_extra_scripts() -> list[str]:
-    """The extra_scripts entries a PlatformIO platform registers for the
-    pch; empty when disabled (the script itself has no enable check)."""
-    if not pch_enabled():
-        pch_disabled_degraded()
-        return []
-    return ["post:pch.py"]
+def pch_script_enabled() -> bool:
+    """Whether this PlatformIO build takes the pch script."""
+    from esphome.core import CORE
+
+    return pch_enabled() and CORE.target_platform not in PCH_SCRIPT_EXCLUDED_PLATFORMS
 
 
 def pch_header_text(include_headers: Iterable[str]) -> str:
@@ -196,21 +170,21 @@ def pch_header_text(include_headers: Iterable[str]) -> str:
     return "".join(f'#include "{name}"\n' for name in include_headers)
 
 
-def _resolves(path: Path) -> bool:
-    """False when missing; other stat failures propagate (identity unknown,
-    unlike is_file(), which would silently drop the header)."""
-    try:
-        return stat.S_ISREG(path.stat().st_mode)
-    except (FileNotFoundError, NotADirectoryError):
-        return False
+def write_pch_headers(build_dir: Path, include_headers: Iterable[str]) -> Path:
+    """Write the guard header and the include list; return the latter,
+    which is what the .gch compiles from."""
+    write_file_if_changed(build_dir / PCH_HEADER_NAME, PCH_GUARD_TEXT)
+    source = build_dir / PCH_SOURCE_NAME
+    write_file_if_changed(source, pch_header_text(include_headers))
+    return source
 
 
 def _include_closure(src_dir: Path, roots: Iterable[str]) -> dict[str, bytes]:
     """Include closure of ``roots``: src-relative name -> contents.
 
-    Resolution mirrors the compiler (includer's dir, then src root); names
-    outside ``src_dir`` end the walk and are versioned by the caller. No
-    #ifdef evaluation: over-approximating is the safe direction.
+    Resolution mirrors the compiler (includer's dir, then src root). No
+    #ifdef evaluation: including too much is the safe direction. Headers
+    outside ``src_dir`` are covered by the version strings of the caller.
     """
     seen: dict[str, bytes] = {}
     stack: list[tuple[str, str]] = [(name, "") for name in roots]
@@ -218,332 +192,57 @@ def _include_closure(src_dir: Path, roots: Iterable[str]) -> dict[str, bytes]:
         name, from_dir = stack.pop()
         for candidate in (f"{from_dir}/{name}" if from_dir else name, name):
             rel = posixpath.normpath(candidate)
-            if not rel.startswith("..") and _resolves(src_dir / rel):
+            if not rel.startswith("..") and (src_dir / rel).is_file():
                 break
         else:
             continue
         if rel in seen:
             continue
-        try:
-            data = (src_dir / rel).read_bytes()
-        except OSError as err:
-            # A marker would truncate the transitive walk; fail closed
-            _LOGGER.warning("Could not read %s for the pch checksum: %s", rel, err)
-            raise
-        seen[rel] = data
+        data = seen[rel] = (src_dir / rel).read_bytes()
         parent = posixpath.dirname(rel)
-        stack.extend(
-            # surrogateescape: a non-UTF-8 name just fails to resolve
-            (inc.decode(errors="surrogateescape"), parent)
-            for inc in _INCLUDE_RE.findall(data)
-        )
+        stack.extend((inc.decode(), parent) for inc in _INCLUDE_RE.findall(data))
     return seen
 
 
 def pch_checksum(
     src_dir: Path, include_headers: Iterable[str], extra: Iterable[str]
 ) -> str:
-    """Digest standing in for the .gch in ccache's hash: the include closure
-    of the prefix header plus caller-supplied identity strings (versioned
-    install paths, flags). Raises OSError when a header's identity cannot
-    be established at all; callers must then compile without a pch."""
+    """Digest of the prefix header's include closure plus ``extra``."""
     digest = hashlib.sha256()
     closure = _include_closure(src_dir, include_headers)
     for name in sorted(closure):
-        digest.update(name.encode(errors="surrogateescape"))
+        digest.update(name.encode())
         digest.update(closure[name])
         digest.update(b"\0")
     for item in extra:
-        digest.update(item.encode(errors="surrogateescape"))
+        digest.update(item.encode())
         digest.update(b"\0")
     return digest.hexdigest()
 
 
-# Tokens dropped when retargeting a TU's flags at the prefix header
-# (the pch compile must not touch depfiles)
-_PCH_STRIP_FLAGS_WITH_ARG = frozenset({"-o", "-c", "-MT", "-MF", "-MQ"})
-_PCH_STRIP_FLAGS = frozenset({"-MD", "-MMD", "-MP", "-MM", "-M"})
-
-
-def pch_compile_command(
-    build_dir: Path, header: Path, gch: Path
-) -> tuple[list[str], Path] | None:
-    """The exact src C++ flags from compile_commands.json retargeted at the
-    header, with the directory they resolve against (relative -I paths must
-    be expanded and executed from the same root); None (logged) when no
-    configured C++ TU is available yet."""
+def pch_identity(
+    tokens: Iterable[str],
+    src_dir: Path,
+    include_headers: tuple[str, ...],
+    extra: Iterable[str],
+) -> str:
+    """The .sum digest: include closure, header text, ``extra`` and the
+    compile flags with the build path stripped, as ccache does."""
     from esphome.core import CORE
 
-    try:
-        entries = json.loads(
-            (build_dir / "compile_commands.json").read_text(encoding="utf-8")
-        )
-    except (OSError, json.JSONDecodeError) as err:
-        # Configure already succeeded, so an unusable DB is a real anomaly
-        _LOGGER.warning("No usable compile database, skipping pch: %s", err)
-        return None
-    if not isinstance(entries, list):
-        _LOGGER.warning("Malformed compile database, skipping pch")
-        return None
-    # CMake may spell paths through a symlink differently than CORE does
-    # (macOS /tmp vs /private/tmp), so compare resolved paths
-    src_root = Path(CORE.relative_src_path()).resolve()
-    entry = next(
-        (
-            e
-            for e in entries
-            if isinstance(e, dict)
-            and isinstance(e.get("file"), str)
-            and e["file"].endswith(CXX_SOURCE_SUFFIXES)
-            and Path(e["file"]).resolve().is_relative_to(src_root)
-        ),
-        None,
-    )
-    if entry is None:
-        _LOGGER.warning("No src C++ entry in the compile database, skipping pch")
-        return None
-    directory = entry.get("directory")
-    cmd_dir = Path(directory) if isinstance(directory, str) and directory else build_dir
-    command = entry.get("command")
-    tokens = expand_response_files(
-        split_command(command if isinstance(command, str) else ""), cmd_dir
-    )
-    # A DB recorded with ccache enabled prefixes the compiler with the
-    # launcher; the .gch must be compiled directly
-    if tokens and is_launcher(tokens[0]):
-        tokens = tokens[1:]
-    if not tokens:
-        # "arguments"-style or empty entries must skip, not spawn "-x ..."
-        _LOGGER.warning("Compile database entry has no usable command, skipping pch")
-        return None
-    args: list[str] = []
-    arg_it = iter(tokens)
-    for tok in arg_it:
-        if tok in _PCH_STRIP_FLAGS_WITH_ARG:
-            next(arg_it, None)
-            continue
-        if tok in _PCH_STRIP_FLAGS:
-            continue
-        if tok == "-include":
-            # Drop only the injected prefix; user force-includes must reach
-            # the .gch compile or GCC rejects it over the macro mismatch
-            inc = next(arg_it, "")
-            if not inc.endswith(PCH_HEADER_NAME):
-                args.extend(("-include", inc))
-            continue
-        args.append(tok)
-    return [*args, "-x", "c++-header", "-c", str(header), "-o", str(gch)], cmd_dir
-
-
-def _log_pch_in_use() -> None:
-    # The only place a user can discover the knob; emitted only once a
-    # .gch is actually fresh or being built
-    _LOGGER.info(
-        "Compiling with a precompiled header (set ESPHOME_PCH_ENABLE=0 to disable)"
-    )
-
-
-def _read_stamp(path: Path) -> str:
-    """A corrupt sidecar must read as stale, not kill the pch forever."""
-    try:
-        return path.read_text(encoding="utf-8").strip()
-    except (OSError, UnicodeDecodeError):
-        return ""
-
-
-def discard_pch(build_dir: Path) -> None:
-    """Remove the pch sidecars so a stale .gch is never consumed.
-
-    Bumps the header only when a .gch was actually removed: TUs compiled
-    against it have incomplete depfiles, while a repeat failure with no
-    .gch must not force a full rebuild every build. A .gch that survives
-    an unlink failure would be consumed silently (wrong output, not a
-    slow build), so that raises.
-    """
-    header = build_dir / PCH_HEADER_NAME
-    gch = Path(f"{header}.gch")
-    had_gch = gch.is_file()
-    errors = []
-    for sidecar in (gch, Path(f"{gch}.sum")):
-        try:
-            sidecar.unlink(missing_ok=True)
-        except OSError as err:
-            if sidecar.is_file():
-                from esphome.core import EsphomeError
-
-                raise EsphomeError(
-                    f"Could not discard the stale precompiled header: {err}"
-                ) from err
-            errors.append(err)
-    for err in errors:
-        _LOGGER.warning("Could not discard the pch sidecars: %s", err)
-    if had_gch and header.is_file():
-        with suppress(OSError):
-            os.utime(header)
-
-
-def prepare_pch(
-    build_dir: Path, include_headers: tuple[str, ...], extra: Iterable[str]
-) -> None:
-    """Compile ``build_dir``'s .gch from compile_commands.json flags and
-    write its ccache .sum.
-
-    The .sum doubles as the freshness stamp and folds in the compile
-    command, so a flag-only change rebuilds the .gch; ``extra`` carries
-    backend identity (framework version, sdkconfig, ...). A failed
-    compile falls back to the plain header include.
-    """
-    from esphome.core import CORE
-
-    header = build_dir / PCH_HEADER_NAME
-    gch = Path(f"{header}.gch")
-    sum_path = Path(f"{gch}.sum")
-    cmd_and_dir = pch_compile_command(build_dir, header, gch)
-    if cmd_and_dir is None:
-        # Freshness cannot be validated; a leftover .gch must not be consumed
-        discard_pch(build_dir)
-        pch_degraded("no usable compile command")
-        return
-    cmd, cmd_dir = cmd_and_dir
-    # Strip like ccache's rewriting (user CCACHE_BASEDIR wins); the raw
-    # build path covers unresolved (symlinked) spellings
-    cmd_id = (
-        " ".join(cmd)
-        .replace(effective_ccache_basedir(), "")
+    flags = (
+        " ".join(tokens)
         .replace(str(CORE.build_path), "")
+        .replace(effective_ccache_basedir(), "")
+    )
+    # The closure is sorted, so header order only enters via the text
+    return pch_checksum(
+        src_dir, include_headers, (pch_header_text(include_headers), *extra, flags)
     )
-    try:
-        checksum = pch_checksum(
-            CORE.relative_src_path(),
-            include_headers,
-            (
-                # The closure is sorted, so root order only enters via the text
-                pch_header_text(include_headers),
-                *extra,
-                cmd_id,
-            ),
-        )
-    except (OSError, UnicodeError) as err:
-        # Identity unknown: a stale cache entry must never be served
-        _LOGGER.warning(
-            "Could not establish the pch identity; compiling without it: %s", err
-        )
-        discard_pch(build_dir)
-        pch_degraded(f"identity unknown: {err}")
-        return
-    failed_marker = Path(f"{gch}.failed")
 
-    def _run(
-        run_cmd: list[str], what: str, stdin: str | None = None
-    ) -> subprocess.CompletedProcess | None:
-        """Spawn one pch tool step; environmental failures discard and
-        degrade (None): spawn/IO/timeout errors and signal kills never
-        latch the marker."""
-        try:
-            proc = subprocess.run(
-                run_cmd,
-                cwd=cmd_dir,
-                # C locale keeps diagnostics matchable by _TRANSIENT_ERRORS
-                env={**os.environ, "LC_ALL": "C"},
-                input=stdin,
-                capture_output=True,
-                text=True,
-                check=False,
-                timeout=300,
-            )
-        except (OSError, subprocess.SubprocessError) as err:
-            _LOGGER.warning("Precompiled header %s did not run: %s", what, err)
-            discard_pch(build_dir)
-            pch_degraded(f"{what} did not run: {err}")
-            return None
-        if proc.returncode < 0:
-            # Killed by a signal (OOM, ^C): environmental, do not latch
-            _LOGGER.warning(
-                "Precompiled header %s was killed (signal %d); retrying next build",
-                what,
-                -proc.returncode,
-            )
-            discard_pch(build_dir)
-            pch_degraded(f"{what} killed by signal {-proc.returncode}")
-            return None
-        return proc
 
-    def _fail(error: str, reason: str, latch: bool) -> None:
-        """Discard and degrade; deterministic failures latch when asked."""
-        _LOGGER.warning(
-            "Precompiled header failed; compiling without it: %s", error[:400]
-        )
-        # Latching paths keep the full compiler output recoverable
-        _LOGGER.debug("Full pch output: %s", error)
-        discard_pch(build_dir)
-        if latch and not any(m in error for m in _TRANSIENT_ERRORS):
-            # Skip retries until a header/flag/backend-identity/command change
-            failed_marker.write_text(checksum + "\n", encoding="utf-8")
-            os.utime(header)
-        pch_degraded(f"{reason}: {error[:200]}")
+_DISABLE_HINT = " (set ESPHOME_PCH_ENABLE=0 to disable)"
 
-    def _probe(latch: bool = True) -> None:
-        """Load-check the built .gch: some toolchains build one they then
-        refuse to load (per-process ASLR). Dep flags are already stripped
-        from cmd, so no -MF is needed; cmd ends with the fixed
-        "-x c++-header -c -o" tail. A cached-header rejection may not
-        reproduce (per-process), so that caller passes latch=False."""
-        if cmd[-6:-4] != ["-x", "c++-header"]:
-            # The slice below depends on pch_compile_command's fixed tail
-            _LOGGER.warning("Unexpected pch command shape: %s", cmd[-6:])
-            discard_pch(build_dir)
-            pch_degraded("unexpected pch command shape")
-            return
-        base = cmd[:-6]
-        probe = _run([*base, *pch_probe_args(str(header))], "probe", stdin="")
-        if probe is None:
-            return
-        if probe.returncode != 0:
-            # Disambiguate: only blame the pch when the same compile passes
-            # without it; a failing baseline is its own (latchable) problem
-            baseline = _run([*base, *pch_probe_tail()], "probe baseline", stdin="")
-            if baseline is None:
-                return
-            if baseline.returncode == 0:
-                error = probe.stderr.strip() or f"exit code {probe.returncode}"
-                _fail(error, "toolchain cannot load the pch", latch=latch)
-            else:
-                error = baseline.stderr.strip() or f"exit code {baseline.returncode}"
-                _fail(error, "probe cannot run at all", latch=latch)
 
-    if gch.is_file() and _read_stamp(sum_path) == checksum:
-        _log_pch_in_use()
-        if pch_strict():
-            # Rejection is per-process, so a cached .gch must re-prove
-            # loadability for the strict gate (CI-only cost); no latch,
-            # since the rejection may not reproduce either
-            _probe(latch=False)
-        return
-    if _read_stamp(failed_marker) == checksum:
-        _LOGGER.info(
-            "Precompiled header disabled after an earlier failure; delete %s to retry",
-            failed_marker,
-        )
-        pch_degraded("earlier failure latched")
-        return
-    _log_pch_in_use()
-    result = _run(cmd, "compile")
-    if result is None:
-        return
-    error = None
-    if result.returncode != 0:
-        error = result.stderr.strip() or f"exit code {result.returncode}"
-    elif not gch.is_file():
-        error = "compiler produced no .gch"
-    if error is not None:
-        _fail(error, "compile failed", latch=True)
-        return
-    _probe()
-    if not gch.is_file():
-        # The probe discarded a rejected or unrunnable .gch
-        return
-    failed_marker.unlink(missing_ok=True)
-    sum_path.write_text(checksum + "\n", encoding="utf-8")
-    # Consumers depend on the header (depfiles cannot see through a .gch);
-    # bump it so users of the previous .gch recompile
-    os.utime(header)
+def log_pch_in_use() -> None:
+    _LOGGER.info("Compiling with a precompiled header%s", _DISABLE_HINT)

esphome/build_helpers/tool_runner.py

@@ -0,0 +1,180 @@
+"""Run a native build tool (cmake, ninja) and relay its output.
+
+Output is read from a pipe so it can be filtered here: a child that inherits
+our stdout writes straight to the file descriptor, past any Python wrapper.
+"""
+
+from __future__ import annotations
+
+import codecs
+from contextlib import suppress
+import logging
+import os
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import sys
+from typing import Any, TextIO
+
+from esphome.util import ANSI_ESCAPE, RedirectText, shlex_quote
+
+_LOGGER = logging.getLogger(__name__)
+
+# Windows code page identifier for UTF-8, as used by ``chcp 65001``.
+UTF8_CODEPAGE = 65001
+
+# Same pattern idf.py uses to spot ninja status lines (``is_progression``).
+_PROGRESS = re.compile(r"^\[\d+/\d+\]|.*\(\d+ \%\)$")
+_READ_SIZE = 65536
+
+
+def _get_kernel32() -> Any | None:
+    """Return the Windows kernel32 module, or None on any other platform."""
+    if sys.platform != "win32":
+        return None
+    import ctypes
+
+    return ctypes.windll.kernel32
+
+
+class Utf8Console:
+    """Keep an attached Windows console on UTF-8 while a build tool runs.
+
+    esp_idf_size draws its table with Unicode box characters, and CMake
+    re-decodes a child's output with the console code page, which garbles the
+    table on any page but UTF-8. A console already on UTF-8 is left alone so
+    an overlapping build never records UTF-8 as the page to go back to.
+    """
+
+    def __init__(self, kernel32: Any | None) -> None:
+        self._kernel32 = kernel32
+        self._codepages: tuple[int, int] | None = None
+
+    def __enter__(self) -> None:
+        kernel32 = self._kernel32
+        if kernel32 is None:
+            return
+        old_in = kernel32.GetConsoleCP()
+        old_out = kernel32.GetConsoleOutputCP()
+        # Both calls return 0 when no console is attached.
+        if not old_in or not old_out:
+            return
+        if old_in == UTF8_CODEPAGE and old_out == UTF8_CODEPAGE:
+            return
+        # Record first so a switch that fails part way is still undone.
+        self._codepages = (old_in, old_out)
+        kernel32.SetConsoleCP(UTF8_CODEPAGE)
+        kernel32.SetConsoleOutputCP(UTF8_CODEPAGE)
+
+    def __exit__(self, *exc_info: object) -> None:
+        if self._codepages is None:
+            return
+        old_in, old_out = self._codepages
+        self._codepages = None
+        self._kernel32.SetConsoleCP(old_in)
+        self._kernel32.SetConsoleOutputCP(old_out)
+
+
+def _fit_terminal(text: str) -> str:
+    """Elide the middle of ``text`` to fit the terminal, as idf.py does.
+
+    A width of 0 (a pipe, the dashboard) leaves the text whole.
+    """
+    width = shutil.get_terminal_size((0, 0)).columns
+    if not width:
+        return text
+    if width <= 3:
+        return "." * width
+    if len(text) >= width:
+        keep = (width - 3) // 2
+        return f"{text[:keep]}...{text[len(text) - keep :]}"
+    return text
+
+
+class ToolOutput(RedirectText):
+    """RedirectText that can collapse ninja status lines into one line.
+
+    With ``progress`` each ``[n/m]`` line overwrites the previous one, the
+    way idf.py shows a build.
+    """
+
+    def __init__(
+        self, out: TextIO, filter_lines: list[str] | None, progress: bool
+    ) -> None:
+        super().__init__(out, filter_lines=filter_lines)
+        self._progress = progress
+        self._on_progress_line = False
+
+    def _splits_lines(self) -> bool:
+        return self._progress or super()._splits_lines()
+
+    def _emit_line(self, line: str) -> None:
+        if self._progress and _PROGRESS.match(line):
+            if not self._is_filtered(line):
+                text = _fit_terminal(line.strip("\r\n"))
+                self._write_color_replace(f"\r{text}\x1b[K")
+                self._on_progress_line = True
+            return
+        self._end_progress_line()
+        super()._emit_line(line)
+
+    def _end_progress_line(self) -> None:
+        if self._on_progress_line:
+            self._on_progress_line = False
+            self._write_color_replace(os.linesep)
+
+    def drain(self) -> None:
+        super().drain()
+        # Called from cleanup, so a broken stream must not hide the exit code.
+        with suppress(OSError, ValueError):
+            self._end_progress_line()
+            self._out.flush()
+
+
+def run_build_tool(
+    cmd: list[str],
+    *,
+    cwd: Path,
+    env: dict[str, str],
+    filter_lines: list[str] | None = None,
+    progress: bool = False,
+    log_path: Path | None = None,
+) -> int:
+    """Run ``cmd`` and relay stdout and stderr, merged, to our stdout.
+
+    ``log_path`` also gets the full, unfiltered output without color codes, as
+    idf.py wrote its logs (its hint patterns expect plain text). Returns the
+    exit code.
+    """
+    _LOGGER.debug("Running: %s", " ".join(shlex_quote(arg) for arg in cmd))
+    _LOGGER.debug("  in directory: %s", cwd)
+    output = ToolOutput(sys.stdout, filter_lines, progress)
+    decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
+    if log_path is not None:
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+    with (
+        Path(log_path or os.devnull).open("w", encoding="utf-8", newline="") as log,
+        Utf8Console(_get_kernel32()),
+        subprocess.Popen(
+            cmd,
+            cwd=cwd,
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            close_fds=False,
+        ) as proc,
+    ):
+        try:
+            # read1 returns as soon as anything is available, so output
+            # streams while the tool runs.
+            while chunk := proc.stdout.read1(_READ_SIZE):
+                text = decoder.decode(chunk)
+                log.write(ANSI_ESCAPE.sub("", text))
+                output.write(text)
+            if tail := decoder.decode(b"", final=True):
+                log.write(ANSI_ESCAPE.sub("", tail))
+                output.write(tail)
+        finally:
+            output.drain()
+    return proc.returncode

esphome/build_helpers/tools_cache.py

@@ -16,8 +16,8 @@ def tools_cache_path(env_var: str, subdir: str) -> Path:
     from esphome.helpers import get_str_env
 
     if prefix := get_str_env(env_var, "").strip():
-        # resolve(): symlinked prefixes otherwise trip idf.py's
-        # venv-mismatch warning on every build
+        # resolve(): a symlinked prefix would otherwise record a second
+        # spelling of the same paths in the build tree
         return Path(prefix).expanduser().resolve()
     # appauthor=False keeps the Windows path short (no vendor segment);
     # deep IDF trees run into MAX_PATH otherwise
@@ -35,9 +35,12 @@ SDK_NRF_TOOLS_CACHE = ("ESPHOME_SDK_NRF_PREFIX", "sdk-nrf")
 ARDUINO8266_TOOLS_CACHE = ("ESPHOME_ARDUINO8266_PREFIX", "arduino8266")
 # The host backend installs nothing; the entry only holds its ccache dir
 HOST_TOOLS_CACHE = ("ESPHOME_HOST_PREFIX", "host")
+# PlatformIO installs into its own dirs; the entry is its ccache dir itself
+PLATFORMIO_CCACHE = ("ESPHOME_PLATFORMIO_CCACHE_DIR", "platformio-ccache")
 TOOLS_CACHE_SPECS = (
     IDF_TOOLS_CACHE,
     SDK_NRF_TOOLS_CACHE,
     ARDUINO8266_TOOLS_CACHE,
     HOST_TOOLS_CACHE,
+    PLATFORMIO_CCACHE,
 )

esphome/components/as5600/sensor/__init__.py

@@ -87,18 +87,8 @@ async def to_code(config: ConfigType) -> None:
     if out_of_range_mode_config := config.get(CONF_OUT_OF_RANGE_MODE):
         cg.add(var.set_out_of_range_mode(out_of_range_mode_config))
 
-    if raw_position_config := config.get(CONF_RAW_POSITION):
-        sens = await sensor.new_sensor(raw_position_config)
-        cg.add(var.set_raw_position_sensor(sens))
-
-    if gain_config := config.get(CONF_GAIN):
-        sens = await sensor.new_sensor(gain_config)
-        cg.add(var.set_gain_sensor(sens))
-
-    if magnitude_config := config.get(CONF_MAGNITUDE):
-        sens = await sensor.new_sensor(magnitude_config)
-        cg.add(var.set_magnitude_sensor(sens))
-
-    if status_config := config.get(CONF_STATUS):
-        sens = await sensor.new_sensor(status_config)
-        cg.add(var.set_status_sensor(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_RAW_POSITION, var.set_raw_position_sensor)
+    await sensors(CONF_GAIN, var.set_gain_sensor)
+    await sensors(CONF_MAGNITUDE, var.set_magnitude_sensor)
+    await sensors(CONF_STATUS, var.set_status_sensor)

esphome/components/ble_nus/__init__.py

@@ -62,6 +62,7 @@ CONFIG_SCHEMA = cv.All(
 async def to_code(config: ConfigType) -> None:
     var = cg.new_Pvariable(config[CONF_ID])
     zephyr_add_prj_conf("BT_NUS", True)
+    zephyr_add_prj_conf("RING_BUFFER", True)
     request_ble_l2cap_mtu(247)  # DLE sweet spot: one NUS frame per LL PDU
     expose_log = config[CONF_TYPE] == CONF_LOGS
     cg.add(var.set_expose_log(expose_log))

esphome/components/ccs811/sensor.py

@@ -65,17 +65,12 @@ async def to_code(config: ConfigType) -> None:
     await cg.register_component(var, config)
     await i2c.register_i2c_device(var, config)
 
-    if eco2_config := config.get(CONF_ECO2):
-        sens = await sensor.new_sensor(eco2_config)
-        cg.add(var.set_co2(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_ECO2, var.set_co2)
+    await sensors(CONF_TVOC, var.set_tvoc)
 
-    if tvoc_config := config.get(CONF_TVOC):
-        sens = await sensor.new_sensor(tvoc_config)
-        cg.add(var.set_tvoc(sens))
-
-    if version_config := config.get(CONF_VERSION):
-        sens = await text_sensor.new_text_sensor(version_config)
-        cg.add(var.set_version(sens))
+    text_sensors = text_sensor.sub_text_sensors(config)
+    await text_sensors(CONF_VERSION, var.set_version)
 
     if (baseline := config.get(CONF_BASELINE)) is not None:
         cg.add(var.set_baseline(baseline))

esphome/components/debug/__init__.py

@@ -60,6 +60,7 @@ async def to_code(config: ConfigType) -> None:
         zephyr_add_prj_conf("LOG_BLOCK_IN_THREAD", True)
         zephyr_add_prj_conf("LOG_BUFFER_SIZE", 4096)
         zephyr_add_prj_conf("SEGGER_RTT_MODE_BLOCK_IF_FIFO_FULL", True)
+        zephyr_add_prj_conf("PRINTK", True)
     var = cg.new_Pvariable(config[CONF_ID])
     await cg.register_component(var, config)
     cg.add_define("USE_DEBUG")

esphome/components/dfrobot_sen0395/__init__.py

@@ -187,19 +187,18 @@ async def dfrobot_sen0395_settings_to_code(
             template_ = await cg.templatable(segments[7], args, cg.float_)
             cg.add(var.set_det_max4(template_))
     if CONF_OUTPUT_LATENCY in config:
-        template_ = await cg.templatable(
-            config[CONF_OUTPUT_LATENCY][CONF_DELAY_AFTER_DETECT], args, float
-        )
-        if isinstance(template_, cv.TimePeriod):
-            template_ = template_.total_milliseconds / 1000
-        cg.add(var.set_delay_after_detect(template_))
-
-        template_ = await cg.templatable(
-            config[CONF_OUTPUT_LATENCY][CONF_DELAY_AFTER_DISAPPEAR], args, float
-        )
-        if isinstance(template_, cv.TimePeriod):
-            template_ = template_.total_milliseconds / 1000
-        cg.add(var.set_delay_after_disappear(template_))
+        latency = config[CONF_OUTPUT_LATENCY]
+        for key, setter in (
+            (CONF_DELAY_AFTER_DETECT, var.set_delay_after_detect),
+            (CONF_DELAY_AFTER_DISAPPEAR, var.set_delay_after_disappear),
+        ):
+            template_ = await cg.templatable(
+                latency[key],
+                args,
+                float,
+                to_exp=lambda period: period.total_milliseconds / 1000,
+            )
+            cg.add(setter(template_))
     if CONF_SENSITIVITY in config:
         template_ = await cg.templatable(config[CONF_SENSITIVITY], args, cg.int8)
         cg.add(var.set_sensitivity(template_))

esphome/components/epaper_spi/colorconv.h

@@ -16,6 +16,28 @@ namespace esphome::epaper_spi {
 /** Delta for when to regard as gray */
 static constexpr uint8_t COLORCONV_GRAY_THRESHOLD = 50;
 
+/** Rec.601 luma (0.299/0.587/0.114 weights, scaled by 256) for optimum perceptual brightness */
+constexpr uint8_t rec601_luma(Color color) {
+  return (uint8_t) ((77u * color.r + 150u * color.g + 29u * color.b + 128u) >> 8);
+}
+
+/** Map RGB color to a single monochrome bit
+ *
+ * @param color RGB color to convert from
+ * @return      1 = white, 0 = black
+ */
+constexpr uint8_t color_to_mono(Color color) { return rec601_luma(color) >= 128 ? 1 : 0; }
+
+/** Map RGB color to one of 4 discrete gray levels (2 bits per pixel)
+ *
+ * @param color RGB color to convert from
+ * @return      Gray level: 0 = black, 3 = white
+ */
+constexpr uint8_t color_to_gray4(Color color) {
+  const uint8_t level = (uint8_t) ((rec601_luma(color) + 32u) >> 6);  // quantize 0..255 to 0..3, rounded
+  return level > 3 ? 3 : level;
+}
+
 /** Map RGB color to discrete BWYR hex 4 color key
  *
  * @tparam NATIVE_COLOR  Type of native hardware color values
@@ -25,7 +47,6 @@ static constexpr uint8_t COLORCONV_GRAY_THRESHOLD = 50;
  * @param hw_yellow Native value for yellow
  * @param hw_red    Native value for red
  * @return          Converted native hardware color value
- * @internal Constexpr. Does not depend on side effects ("pure").
  */
 template<typename NATIVE_COLOR>
 constexpr NATIVE_COLOR color_to_bwyr(Color color, NATIVE_COLOR hw_black, NATIVE_COLOR hw_white, NATIVE_COLOR hw_yellow,
@@ -38,11 +59,7 @@ constexpr NATIVE_COLOR color_to_bwyr(Color color, NATIVE_COLOR hw_black, NATIVE_
 
   if ((max_rgb - min_rgb) < COLORCONV_GRAY_THRESHOLD) {
     // It's a shade of gray. Map to BLACK or WHITE.
-    // We split the luminance at the halfway point (382 = (255*3)/2)
-    if ((static_cast<int>(color.r) + color.g + color.b) > 382) {
-      return hw_white;
-    }
-    return hw_black;
+    return color_to_mono(color) ? hw_white : hw_black;
   }
 
   // --- Step 2: Check for Primary/Secondary Colors ---
@@ -96,7 +113,6 @@ constexpr NATIVE_COLOR color_to_bwr(Color color, NATIVE_COLOR hw_black, NATIVE_C
  * @param hw_green  Native value for green
  * @param hw_blue   Native value for blue
  * @return          Converted native hardware color value
- * @internal Constexpr. Does not depend on side effects ("pure").
  */
 template<typename NATIVE_COLOR>
 constexpr NATIVE_COLOR color_to_bwyrgb(Color color, NATIVE_COLOR hw_black, NATIVE_COLOR hw_white,
@@ -105,10 +121,7 @@ constexpr NATIVE_COLOR color_to_bwyrgb(Color color, NATIVE_COLOR hw_black, NATIV
   const auto [min_rgb, max_rgb] = std::minmax({color.r, color.g, color.b});
 
   if ((max_rgb - min_rgb) < COLORCONV_GRAY_THRESHOLD) {
-    if ((static_cast<int>(color.r) + color.g + color.b) > 382) {
-      return hw_white;
-    }
-    return hw_black;
+    return color_to_mono(color) ? hw_white : hw_black;
   }
 
   const bool r_on = (color.r > 128);
@@ -158,7 +171,6 @@ constexpr NATIVE_COLOR color_to_bwyrgb(Color color, NATIVE_COLOR hw_black, NATIV
  * @param hw_blue   Native value for blue
  * @param hw_orange Native value for orange
  * @return          Converted native hardware color value
- * @internal Constexpr. Does not depend on side effects ("pure").
  */
 template<typename NATIVE_COLOR>
 constexpr NATIVE_COLOR color_to_bwyrgbo(Color color, NATIVE_COLOR hw_black, NATIVE_COLOR hw_white,
@@ -167,10 +179,7 @@ constexpr NATIVE_COLOR color_to_bwyrgbo(Color color, NATIVE_COLOR hw_black, NATI
   const auto [min_rgb, max_rgb] = std::minmax({color.r, color.g, color.b});
 
   if ((max_rgb - min_rgb) < COLORCONV_GRAY_THRESHOLD) {
-    if ((static_cast<int>(color.r) + color.g + color.b) > 382) {
-      return hw_white;
-    }
-    return hw_black;
+    return color_to_mono(color) ? hw_white : hw_black;
   }
 
   const bool r_on = (color.r > 128);

esphome/components/epaper_spi/display.py

@@ -1,7 +1,9 @@
+from collections.abc import Callable
 import importlib
 import pkgutil
+from typing import Any
 
-from esphome import core, pins
+from esphome import automation, core, pins
 import esphome.codegen as cg
 from esphome.components import display, spi
 from esphome.components.display import CONF_SHOW_TEST_CARD, validate_rotation
@@ -54,6 +56,12 @@ EPaperBase = epaper_spi_ns.class_(
 )
 Transform = epaper_spi_ns.enum("Transform")
 
+automation.register_apply_action(
+    "epaper_spi.full_update_next",
+    automation.maybe_simple_id({cv.Required(CONF_ID): cv.use_id(EPaperBase)}),
+    automation.ApplyCall("request_full_update()"),
+)
+
 # Import all models dynamically from the models package
 for module_info in pkgutil.iter_modules(models.__path__):
     importlib.import_module(f".models.{module_info.name}", package=__package__)
@@ -70,6 +78,23 @@ DIMENSION_SCHEMA = cv.Schema(
 TRANSFORM_OPTIONS = {CONF_MIRROR_X, CONF_MIRROR_Y, CONF_SWAP_XY}
 
 
+def _full_update_every_validator(
+    model: models.EpaperModel,
+) -> Callable[[Any], int]:
+    if model.get_default("partial_update"):
+        return cv.int_range(1, 255)
+
+    def validate(value: Any) -> int:
+        value = cv.int_range(1, 255)(value)
+        if value != 1:
+            raise cv.Invalid(
+                f"{model.name} does not support partial update; full_update_every must be 1"
+            )
+        return value
+
+    return validate
+
+
 def model_schema(config):
     model = MODELS[config[CONF_MODEL]]
     class_name = epaper_spi_ns.class_(model.class_name, EPaperBase)
@@ -96,7 +121,9 @@ def model_schema(config):
                     cv.Required(CONF_MIRROR_Y): cv.boolean,
                 }
             ),
-            cv.Optional(CONF_FULL_UPDATE_EVERY, default=1): cv.int_range(1, 255),
+            cv.Optional(
+                CONF_FULL_UPDATE_EVERY, default=1
+            ): _full_update_every_validator(model),
             model.option(CONF_BUSY_PIN): pins.gpio_input_pin_schema,
             model.option(CONF_CS_PIN): pins.gpio_output_pin_schema,
             model.option(CONF_DC_PIN, fallback=None): pins.gpio_output_pin_schema,
@@ -132,8 +159,15 @@ def customise_schema(config):
         extra=cv.ALLOW_EXTRA,
     )(config)
     model = MODELS[config[CONF_MODEL]]
+    model.check_requirements()
     config = model_schema(config)(config)
+    config = model.validate_config(config)
     width, height = model.get_dimensions(config)
+    if width % (width_multiple := model.get_default("width_multiple", 1)):
+        raise cv.Invalid(
+            f"{model.name} requires a width that is a multiple of {width_multiple}",
+            path=[CONF_DIMENSIONS],
+        )
     display.add_metadata(
         config[CONF_ID],
         width,

esphome/components/epaper_spi/epaper_spi.cpp

@@ -196,6 +196,15 @@ void EPaperBase::process_state_() {
       break;
     case EPaperState::UPDATE:
       this->do_update_();  // Calls ESPHome (current page) lambda
+      if (this->full_update_requested_) {
+        // Refresh the whole panel even if nothing was drawn
+        this->full_update_requested_ = false;
+        this->update_count_ = 0;
+        this->x_low_ = 0;
+        this->y_low_ = 0;
+        this->x_high_ = this->width_;
+        this->y_high_ = this->height_;
+      }
       if (this->x_high_ < this->x_low_ || this->y_high_ < this->y_low_) {
         this->set_state_(EPaperState::IDLE);
         return;
@@ -327,9 +336,9 @@ void HOT EPaperBase::draw_pixel_at(int x, int y, Color color) {
     return;
   const size_t byte_position = y * this->row_width_ + x / 8;
   const uint8_t bit_position = x % 8;
-  const uint8_t pixel_bit = 0x80 >> bit_position;
+  const uint8_t pixel_bit = 0x80u >> bit_position;
   const auto original = this->buffer_[byte_position];
-  if ((color_to_bit(color) == 0)) {
+  if (color_to_mono(color) == 0) {
     this->buffer_[byte_position] = original & ~pixel_bit;
   } else {
     this->buffer_[byte_position] = original | pixel_bit;

esphome/components/epaper_spi/epaper_spi.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "colorconv.h"
 #include "esphome/components/display/display.h"
 #include "esphome/components/spi/spi.h"
 #include "esphome/components/split_buffer/split_buffer.h"
@@ -80,15 +81,7 @@ class EPaperBase : public Display,
 
   DisplayType get_display_type() override { return this->display_type_; };
 
-  // Default implementations for monochrome displays
-  static uint8_t color_to_bit(Color color) {
-    // It's always a shade of gray. Map to BLACK or WHITE.
-    // We split the luminance at a suitable point
-    if ((color.r + color.g + color.b) >= 382) {
-      return 1;
-    }
-    return 0;
-  }
+  // Default implementation for monochrome displays
   void fill(Color color) override {
     // If clipping is active, fall back to base implementation
     if (this->get_clipping().is_set()) {
@@ -96,7 +89,7 @@ class EPaperBase : public Display,
       return;
     }
 
-    auto pixel_color = color_to_bit(color) ? 0xFF : 0x00;
+    auto pixel_color = color_to_mono(color) ? 0xFF : 0x00;
 
     // We store 8 pixels per byte
     this->buffer_.fill(pixel_color);
@@ -114,6 +107,8 @@ class EPaperBase : public Display,
   int get_width() override { return this->effective_transform_ & SWAP_XY ? this->height_ : this->width_; }
   int get_height() override { return this->effective_transform_ & SWAP_XY ? this->width_ : this->height_; }
   void draw_pixel_at(int x, int y, Color color) override;
+  // Make the next update a full one. Applied when that update starts, so one in progress is not affected.
+  void request_full_update() { this->full_update_requested_ = true; }
 
  protected:
   int get_height_internal() override { return this->height_; };
@@ -185,6 +180,7 @@ class EPaperBase : public Display,
   uint8_t transform_{};
   uint8_t effective_transform_{};
   uint8_t update_count_{};
+  bool full_update_requested_{};
   // these values represent the bounds of the updated buffer. Note that x_high and y_high
   // point to the pixel past the last one updated, i.e. may range up to width/height.
   uint16_t x_low_{}, y_low_{}, x_high_{}, y_high_{};

esphome/components/epaper_spi/epaper_spi_mono.h

@@ -9,8 +9,8 @@ namespace esphome::epaper_spi {
 class EPaperMono : public EPaperBase {
  public:
   EPaperMono(const char *name, uint16_t width, uint16_t height, const uint8_t *init_sequence,
-             size_t init_sequence_length)
-      : EPaperBase(name, width, height, init_sequence, init_sequence_length, DISPLAY_TYPE_BINARY) {
+             size_t init_sequence_length, DisplayType display_type = DISPLAY_TYPE_BINARY)
+      : EPaperBase(name, width, height, init_sequence, init_sequence_length, display_type) {
     this->buffer_length_ = (width + 7) / 8 * height;  // 8 pixels per byte, rounded up
   }
 

esphome/components/epaper_spi/epaper_spi_ssd1677.cpp

@@ -0,0 +1,101 @@
+#include "epaper_spi_ssd1677.h"
+
+#include <algorithm>
+
+#include "esphome/core/helpers.h"
+#include "esphome/core/log.h"
+
+namespace esphome::epaper_spi {
+static constexpr const char *const TAG = "epaper_spi.ssd1677";
+
+void EPaperSSD1677::setup() {
+  EPaperMono::setup();
+  if (!this->is_failed())
+    this->init_comparison_frame_();
+}
+
+void EPaperSSD1677::init_comparison_frame_() {
+  if (!this->is_using_partial_update_())
+    return;
+  if (!this->sent_.init(this->plane_row_length_() * this->height_)) {
+    ESP_LOGW(TAG, "No memory for the comparison frame; partial updates will degrade unchanged areas");
+  }
+}
+
+void EPaperSSD1677::plane_row(size_t y, uint8_t *out) {
+  const size_t row_length = this->plane_row_length_();
+  const size_t data_idx = y * row_length;
+  for (size_t i = 0; i != row_length; i++)
+    out[i] = this->buffer_[data_idx + i];
+}
+
+// Nothing a partial update needs is kept in controller RAM any more, so skip the reset for those.
+bool EPaperSSD1677::reset() {
+  if (this->update_count_ != 0 && this->sent_.is_valid())
+    return true;
+  return EPaperMono::reset();
+}
+
+// The window always covers the whole panel, so each plane is the frame's bytes in order. Where
+// those bytes are already stored as the plane needs them (the comparison frame, and a 1-bit
+// buffer) they are written straight from the buffer, as many at a time as the time slice allows;
+// otherwise they are built a row at a time by plane_row().
+bool HOT EPaperSSD1677::transfer_data() {
+  if (!this->sent_.is_valid())
+    return EPaperMono::transfer_data();
+
+  const auto start_time = millis();
+  if (this->current_data_index_ == 0) {
+    if (this->plane_ == 0) {
+      this->x_low_ = 0;
+      this->x_high_ = this->width_;
+      this->y_low_ = 0;
+      this->y_high_ = this->height_;
+    }
+    this->set_window();
+    this->command(this->plane_ == 0 ? 0x26 : 0x24);
+  }
+  // A full update ignores 0x26, and the copy may not hold a real frame yet (first update after
+  // boot): send the new frame to both planes.
+  const bool send_copy = this->plane_ == 0 && this->update_count_ != 0;
+  const bool direct = send_copy || this->buffer_is_plane();
+  const auto &source = send_copy ? this->sent_ : this->buffer_;
+  const size_t row_length = this->plane_row_length_();
+  const size_t plane_length = row_length * this->height_;
+  // Roughly what the bus moves in one time slice, so a slice is not overrun by much
+  const size_t max_chunk = std::max<size_t>(this->data_rate_ / 8000 * MAX_TRANSFER_TIME, MAX_TRANSFER_SIZE);
+  SmallBufferWithHeapFallback<128> row_alloc(direct ? 0 : row_length);
+  this->start_data_();
+  while (this->current_data_index_ != plane_length) {
+    size_t length;
+    const uint8_t *data;
+    if (direct) {
+      data = source.get_span(this->current_data_index_, length);
+      length = std::min(length, max_chunk);
+    } else {
+      // Always at the start of a row here, since this path sends whole rows only
+      this->plane_row(this->current_data_index_ / row_length, row_alloc.get());
+      data = row_alloc.get();
+      length = row_length;
+    }
+    this->write_array(data, length);
+    if (this->plane_ == 1)
+      this->sent_.write(this->current_data_index_, data, length);
+    this->current_data_index_ += length;
+    if (this->current_data_index_ != plane_length && millis() - start_time > MAX_TRANSFER_TIME) {
+      // Let the main loop run and come back next loop
+      this->disable();
+      return false;
+    }
+  }
+  this->disable();
+  this->current_data_index_ = 0;
+  if (this->plane_ == 0) {
+    this->plane_ = 1;
+    return false;
+  }
+  this->plane_ = 0;
+  return true;
+}
+
+}  // namespace esphome::epaper_spi

esphome/components/epaper_spi/epaper_spi_ssd1677.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include "epaper_spi_mono.h"
+
+namespace esphome::epaper_spi {
+
+/**
+ * Monochrome SSD1677 with partial refreshes that leave unchanged pixels alone.
+ *
+ * A partial refresh drives each pixel from the pair (RAM 0x26 = the image on the panel,
+ * RAM 0x24 = the new image) across the whole panel; the RAM window only scopes a write.
+ * EPaperMono writes 0x26 once and afterwards only the changed window of 0x24, which relies on
+ * the controller's RAM being unchanged from one update to the next. On this controller it is not:
+ * the hardware reset at the start of each update loses it, and even without resets, keeping 0x26
+ * in step one window at a time left unchanged areas alternating between older frames. Either way
+ * unchanged pixels get driven on every partial and wash out.
+ *
+ * So before every refresh this class writes both planes over the whole panel: 0x26 from a copy of
+ * the frame last sent, 0x24 from the buffer. GxEPD2 likewise rewrites both planes after each
+ * partial on this controller. The copy is taken as the data goes out, not from the buffer, which
+ * may already hold the next frame by the time the refresh completes.
+ */
+class EPaperSSD1677 : public EPaperMono {
+ public:
+  EPaperSSD1677(const char *name, uint16_t width, uint16_t height, const uint8_t *init_sequence,
+                size_t init_sequence_length, DisplayType display_type = DISPLAY_TYPE_BINARY)
+      : EPaperMono(name, width, height, init_sequence, init_sequence_length, display_type) {}
+
+  void setup() override;
+
+ protected:
+  // Allocates the comparison frame when partial updates are enabled. Separate from setup() so it
+  // can run without a bus.
+  void init_comparison_frame_();
+  // Bytes in one row of a RAM plane: 8 pixels per byte, whatever the buffer holds.
+  size_t plane_row_length_() const { return (this->width_ + 7) / 8; }
+  // Whether the buffer already holds the frame as a RAM plane does, so it can be sent as it is.
+  // A subclass with a deeper buffer returns false and overrides plane_row().
+  virtual bool buffer_is_plane() const { return true; }
+  // Row y of the frame as a RAM plane holds it, 1 bit per pixel with 1 = white.
+  virtual void plane_row(size_t y, uint8_t *out);
+  bool reset() override;
+  bool transfer_data() override;
+
+  split_buffer::SplitBuffer sent_{};  // the frame last sent to 0x24, i.e. what the panel shows
+  uint8_t plane_{0};                  // 0 while sending 0x26, 1 while sending 0x24
+};
+
+}  // namespace esphome::epaper_spi

esphome/components/epaper_spi/epaper_spi_ssd1677_gray4.cpp

@@ -0,0 +1,129 @@
+#include "epaper_spi_ssd1677_gray4.h"
+
+#include "esphome/core/log.h"
+
+namespace esphome::epaper_spi {
+
+static constexpr const char *const TAG = "epaper_spi.ssd1677_gray4";
+
+// Combine two source bytes (4 pixels each, 2 bits per pixel, most significant pixel first) into
+// one plane byte covering the same 8 pixels (1 bit per pixel), choosing high or low bit
+static uint8_t plane_byte(uint8_t first, uint8_t second, bool high_bit) {
+  uint8_t out = 0;
+  for (const uint8_t src : {first, second}) {
+    for (uint8_t shift = 6;; shift -= 2) {
+      const uint8_t level = (src >> shift) & 0x03;
+      out = (uint8_t) ((out << 1) | (high_bit ? (level >> 1) : (level & 1)));
+      if (shift == 0)
+        break;
+    }
+  }
+  return out;
+}
+
+void EPaperSSD1677Gray4::fill(Color color) {
+  if (this->get_clipping().is_set()) {
+    // Falls back to the generic per-pixel implementation for the clipped rectangle.
+    EPaperBase::fill(color);
+    return;
+  }
+  const uint8_t level = color_to_gray4(color);
+  this->buffer_.fill((uint8_t) (level | (level << 2) | (level << 4) | (level << 6)));
+  this->x_low_ = 0;
+  this->y_low_ = 0;
+  this->x_high_ = this->width_;
+  this->y_high_ = this->height_;
+}
+
+void HOT EPaperSSD1677Gray4::draw_pixel_at(int x, int y, Color color) {
+  if (!this->rotate_coordinates_(x, y))
+    return;
+  const uint8_t level = color_to_gray4(color);
+  const size_t byte_position = (size_t) y * this->row_width_ + x / 4;
+  const uint8_t shift = (uint8_t) (6 - 2 * (x % 4));  // most significant pixel first
+  const uint8_t original = this->buffer_[byte_position];
+  this->buffer_[byte_position] = (uint8_t) ((original & ~(0x03 << shift)) | (level << shift));
+}
+
+// A partial update reduces each pixel to its high bit: levels 2 and 3 are light, 0 and 1 dark.
+void EPaperSSD1677Gray4::plane_row(size_t y, uint8_t *out) {
+  const size_t src_row = y * this->row_width_;
+  for (size_t i = 0; i != this->plane_row_length_(); i++)
+    out[i] = plane_byte(this->buffer_[src_row + 2 * i], this->buffer_[src_row + 2 * i + 1], true);
+}
+
+// the high bit of every pixel's level goes to the new (bw) plane
+// (0x24), the low bit to the old (red) plane (0x26)
+bool HOT EPaperSSD1677Gray4::transfer_data() {
+  if (this->is_partial_push_())
+    return EPaperSSD1677::transfer_data();
+
+  auto start_time = millis();
+  const bool first_pass = this->send_red_;
+  if (this->current_data_index_ == 0) {
+    if (first_pass) {
+      // With partial updates enabled the window follows the changed area, but the four-level
+      // refresh drives every pixel from both planes, and the reset before it does not keep RAM.
+      this->x_low_ = 0;
+      this->x_high_ = this->width_;
+      this->y_low_ = 0;
+      this->y_high_ = this->height_;
+      this->set_window();
+    }
+    this->command(first_pass ? 0x24 : 0x26);
+    this->current_data_index_ = this->y_low_;
+  }
+  const size_t plane_row_length = (this->x_high_ - this->x_low_) / 8;
+  // Stack-backed for every panel width in practice; only a custom `dimensions:` far wider than any
+  // supported panel would fall back to the heap.
+  SmallBufferWithHeapFallback<128> bytes_to_send_alloc(plane_row_length);
+  uint8_t *bytes_to_send = bytes_to_send_alloc.get();
+  ESP_LOGV(TAG, "Writing %u bytes at line %zu at %ums", plane_row_length, this->current_data_index_,
+           (unsigned) millis());
+  this->start_data_();
+  while (this->current_data_index_ != this->y_high_) {
+    const size_t src_row = this->current_data_index_ * this->row_width_ + this->x_low_ / 4;
+    for (size_t i = 0; i != plane_row_length; i++) {
+      const uint8_t plane = plane_byte(this->buffer_[src_row + 2 * i], this->buffer_[src_row + 2 * i + 1], first_pass);
+      // The OTP grayscale waveform treats data as inverted relative to monochrome
+      bytes_to_send[i] = (uint8_t) ~plane;
+      // What the next partial update compares against: the high bits, as a black-and-white frame.
+      if (first_pass && this->sent_.is_valid())
+        this->sent_[this->current_data_index_ * plane_row_length + i] = plane;
+    }
+    ++this->current_data_index_;
+    this->write_array(bytes_to_send, plane_row_length);
+    if (millis() - start_time > MAX_TRANSFER_TIME) {
+      // Let the main loop run and come back next loop
+      this->disable();
+      return false;
+    }
+  }
+
+  this->disable();
+  this->current_data_index_ = 0;
+  if (first_pass) {
+    this->send_red_ = false;
+    return false;
+  }
+  this->send_red_ = true;
+  return true;
+}
+
+void EPaperSSD1677Gray4::refresh_screen(bool partial) {
+  if (this->is_partial_push_()) {
+    ESP_LOGV(TAG, "Black-and-white partial refresh");
+    // The border follows the LUT selected in 0x3C. The model's setting (sent with the init
+    // sequence) picks the LUT that is white under the four-level waveform's inverted data; under
+    // the black-and-white waveform that LUT drives black and the border darkens, so use LUT1.
+    this->cmd_data(0x3C, {0x01});
+    EPaperSSD1677::refresh_screen(true);
+    return;
+  }
+  ESP_LOGV(TAG, "Four-level refresh");
+  this->cmd_data(0x1A, {0x67, 0x00});  // force temperature by OTP
+  this->cmd_data(0x22, {0xD7});        // four-level update sequence, panel's OTP waveform
+  this->command(0x20);                 // master activation
+}
+
+}  // namespace esphome::epaper_spi

esphome/components/epaper_spi/epaper_spi_ssd1677_gray4.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include "colorconv.h"
+#include "epaper_spi_ssd1677.h"
+
+namespace esphome::epaper_spi {
+
+/**
+ * Four-level grayscale for SSD1677 panels.
+ *
+ * The SSD1677 has two independent 1-bit RAM planes, normally used for a
+ * black/white and a red plane. This class writes image data to both,
+ * splitting each pixel's 2-bit gray level across them, and triggers the
+ * panel's own OTP grayscale waveform instead of the normal monochrome update
+ * sequence. No custom LUT upload needed for any currently supported panel,
+ * the OTP waveform is used instead.
+ *
+ * The framebuffer therefore packs 2 bits per pixel (4 per byte, most
+ * significant pixel first) instead of EPaperMono's 1 bit.
+ *
+ * The grayscale waveform has no partial form: master activation redraws the whole panel
+ * regardless of the RAM window. So a full update is a four-level refresh, and when partial updates
+ * are enabled (full_update_every > 1, which the model only allows on explicit request) a partial
+ * update is EPaperSSD1677's black-and-white one, each pixel reduced to light or dark. The partial
+ * waveform also drives unchanged pixels towards black or white, so from the first partial update
+ * the whole panel loses its gray levels until the next full update.
+ */
+class EPaperSSD1677Gray4 : public EPaperSSD1677 {
+ public:
+  EPaperSSD1677Gray4(const char *name, uint16_t width, uint16_t height, const uint8_t *init_sequence,
+                     size_t init_sequence_length)
+      : EPaperSSD1677(name, width, height, init_sequence, init_sequence_length, DISPLAY_TYPE_GRAYSCALE) {
+    this->row_width_ = (width + 3) / 4;  // 4 pixels per byte
+    this->buffer_length_ = (size_t) this->row_width_ * height;
+  }
+
+  void fill(Color color) override;
+  void draw_pixel_at(int x, int y, Color color) override;
+
+ protected:
+  // A partial update if partial updates are enabled and this is not a full one.
+  bool is_partial_push_() const { return this->update_count_ != 0 && this->sent_.is_valid(); }
+  bool buffer_is_plane() const override { return false; }
+  void plane_row(size_t y, uint8_t *out) override;
+  void refresh_screen(bool partial) override;
+  bool transfer_data() override;
+};
+
+}  // namespace esphome::epaper_spi

esphome/components/epaper_spi/models/__init__.py

@@ -2,7 +2,9 @@ from typing import Any, Self
 
 import esphome.config_validation as cv
 from esphome.const import CONF_DIMENSIONS, CONF_HEIGHT, CONF_WIDTH
+from esphome.core import CORE
 from esphome.cpp_generator import MockObj
+from esphome.types import ConfigType
 
 
 class EpaperModel:
@@ -48,6 +50,16 @@ class EpaperModel:
         """
         return {}
 
+    def validate_config(self, config: ConfigType) -> ConfigType:
+        """
+        Validate the configuration as a whole, once the schema has been applied.
+        The base implementation accepts it unchanged; specific models override this for
+        rules that span several options.
+        :param config: The validated configuration
+        :return: The configuration, possibly updated
+        """
+        return config
+
     async def to_code(self, var: MockObj, config: dict) -> dict:
         """
         Generate model-specific code for the options added by add_options().
@@ -89,3 +101,26 @@ class EpaperModel:
         defaults = self.defaults.copy()
         defaults.update(kwargs)
         return self.__class__(name, initsequence=tuple(initsequence), **defaults)
+
+    def check_requirements(self) -> None:
+        """
+        Raise a friendly error if any component this model requires is not configured.
+
+        This runs during schema validation (before ID references are resolved) so that a
+        model whose default pins live on a pin expander reports the missing expander clearly
+        instead of a cryptic "Couldn't find ID" from the unresolved pin reference.
+        """
+        if requirements := self.get_default("requires", set()):
+            # ``raw_config`` is populated before any component schema runs during a real
+            # validation, so presence of a required component is simply a top-level key.
+            # When it is absent (e.g. a unit test that invokes the schema directly) there
+            # is no config to check against, so skip.
+            global_config = CORE.raw_config
+            if global_config is None:
+                return
+            missing = {x for x in requirements if x not in global_config}
+            if missing:
+                reqstr = ", ".join(f"'{x}'" for x in sorted(missing))
+                raise cv.Invalid(
+                    f"{self.name} requires component{'s' if len(missing) > 1 else ''} {reqstr} to be configured"
+                )

esphome/components/epaper_spi/models/jd79660.py

@@ -10,7 +10,9 @@ from . import EpaperModel
 
 class JD79660(EpaperModel):
     def __init__(self, name, class_name="EPaperJD79660", fast_update=None, **kwargs):
-        super().__init__(name, class_name, **kwargs)
+        # Only a fast_update sequence lets the driver do anything but a full refresh
+        kwargs.setdefault("partial_update", fast_update is not None)
+        super().__init__(name, class_name=class_name, **kwargs)
         self.fast_update = fast_update
 
     def option(self, name, fallback=cv.UNDEFINED) -> cv.Optional | cv.Required:

esphome/components/epaper_spi/models/ssd1677.py

@@ -1,12 +1,56 @@
-from esphome.const import CONF_DATA_RATE
+from typing import Any
+
+import esphome.config_validation as cv
+from esphome.const import CONF_DATA_RATE, CONF_FULL_UPDATE_EVERY
+from esphome.types import ConfigType
 
 from . import EpaperModel
 
+CONF_BORDER_WAVEFORM = "border_waveform"
+CONF_MONOCHROME_PARTIAL_UPDATES = "monochrome_partial_updates"
+
+# partial_update value for models whose partial updates are black and white only
+MONOCHROME = "monochrome"
+
 
 class SSD1677(EpaperModel):
-    def __init__(self, name, class_name="EPaperMono", data_rate="20MHz", **defaults):
+    def __init__(
+        self,
+        name: str,
+        class_name: str = "EPaperSSD1677",
+        data_rate: str = "20MHz",
+        border_waveform: int = 0x01,
+        **defaults: Any,
+    ) -> None:
         defaults[CONF_DATA_RATE] = data_rate
-        super().__init__(name, class_name, **defaults)
+        defaults[CONF_BORDER_WAVEFORM] = border_waveform
+        defaults.setdefault("partial_update", True)
+        super().__init__(name, class_name=class_name, **defaults)
+
+    def get_config_options(self) -> dict:
+        options = {
+            self.option(CONF_BORDER_WAVEFORM): cv.hex_uint8_t,
+        }
+        if self.get_default("partial_update") == MONOCHROME:
+            options[cv.Optional(CONF_MONOCHROME_PARTIAL_UPDATES, default=False)] = (
+                cv.boolean
+            )
+        return options
+
+    def validate_config(self, config: ConfigType) -> ConfigType:
+        if (
+            self.get_default("partial_update") == MONOCHROME
+            and config[CONF_FULL_UPDATE_EVERY] > 1
+            and not config[CONF_MONOCHROME_PARTIAL_UPDATES]
+        ):
+            raise cv.Invalid(
+                f"{self.name} can only update partially in black and white, and a partial "
+                "update reduces the whole panel to black and white until the next full "
+                f"update. Set '{CONF_MONOCHROME_PARTIAL_UPDATES}: true' to accept this, "
+                "or leave full_update_every at 1",
+                path=[CONF_FULL_UPDATE_EVERY],
+            )
+        return config
 
     # fmt: off
     def get_init_sequence(self, config: dict):
@@ -15,13 +59,14 @@ class SSD1677(EpaperModel):
             (0x18, 0x80),    # Select internal Temp sensor
             (0x0C, 0xAE, 0xC7, 0xC3, 0xC0, 0x80),  # inrush current level 2
             (0x01, (height - 1) % 256, (height - 1) // 256, 0x02),    # Set gate limit (number of rows-1)
-            (0x3C, 0x01),    # Set border waveform
+            (0x3C, config[CONF_BORDER_WAVEFORM]),    # Set border waveform
             (0x11, 3),      # Set transform
         )
 
 
 ssd1677 = SSD1677("ssd1677")
 
+
 wave_4_26 = ssd1677.extend(
     "waveshare-4.26in",
     width=800,
@@ -52,7 +97,8 @@ ssd1677.extend(
     mirror_x=True,
 )
 
-ssd1677.extend(
+# Sticky - monochrome version
+seeed_sticky = ssd1677.extend(
     "seeed-reterminal-sticky",
     width=800,
     height=480,
@@ -63,4 +109,15 @@ ssd1677.extend(
     reset_pin=17,
     busy_pin=18,
     data_rate="10MHz",
+    requires={"psram"},
+)
+
+# Sticky - 4 level grayscale; partial updates only in black and white, on request
+seeed_sticky.extend(
+    "seeed-reterminal-sticky-gray4",
+    class_name="EPaperSSD1677Gray4",
+    border_waveform=0x00,
+    partial_update=MONOCHROME,
+    # each plane byte is built from two whole buffer bytes
+    width_multiple=8,
 )

esphome/components/epaper_spi/models/ssd1683.py

@@ -6,7 +6,8 @@ from . import EpaperModel
 class SSD1683(EpaperModel):
     def __init__(self, name, class_name="EPaperSSD1683", data_rate="20MHz", **defaults):
         defaults[CONF_DATA_RATE] = data_rate
-        super().__init__(name, class_name, **defaults)
+        defaults.setdefault("partial_update", True)
+        super().__init__(name, class_name=class_name, **defaults)
 
     # fmt: off
     def get_init_sequence(self, config: dict):

esphome/components/epaper_spi/models/uc8179.py

@@ -32,7 +32,8 @@ class UC8179(EpaperModel):
         **defaults: Any,
     ) -> None:
         defaults.setdefault(CONF_DATA_RATE, data_rate)
-        super().__init__(name, class_name, **defaults)
+        defaults.setdefault("partial_update", True)
+        super().__init__(name, class_name=class_name, **defaults)
 
     def get_init_sequence(self, config: dict) -> tuple:
         """Generate the initialization sequence for UC8179 mono displays.

esphome/components/epaper_spi/models/waveshare.py

@@ -6,8 +6,12 @@ from . import EpaperModel
 
 
 class WaveshareModel(EpaperModel):
-    def __init__(self, name, lut, lut_partial=None, **defaults):
-        super().__init__(name, "EpaperWaveshare", **defaults)
+    def __init__(
+        self, name, lut, lut_partial=None, class_name="EpaperWaveshare", **defaults
+    ):
+        # A partial LUT is what lets EpaperWaveshare do partial refresh
+        defaults.setdefault("partial_update", lut_partial is not None)
+        super().__init__(name, class_name=class_name, **defaults)
         self.lut = lut
         self.lut_partial = lut_partial
 

esphome/components/esp32/gpio.cpp

@@ -30,7 +30,7 @@ static const gpio_hal_context_t GPIO_HAL = {.dev = GPIO_HAL_GET_HW(GPIO_PORT_0)}
 bool ESP32InternalGPIOPin::isr_service_installed = false;  // NOLINT(cppcoreguidelines-avoid-non-const-global-variables)
 
 static gpio_mode_t flags_to_mode(gpio::Flags flags) {
-  flags = (gpio::Flags)(flags & ~(gpio::FLAG_PULLUP | gpio::FLAG_PULLDOWN));
+  flags = flags & (gpio::FLAG_INPUT | gpio::FLAG_OUTPUT | gpio::FLAG_OPEN_DRAIN);
   if (flags == gpio::FLAG_INPUT)
     return GPIO_MODE_INPUT;
   if (flags == gpio::FLAG_OUTPUT)

esphome/components/esp32_ble/__init__.py

@@ -474,12 +474,17 @@ def final_validation(config: ConfigType) -> None:
 
     # Check if hosted bluetooth is being used
     if "esp32_hosted" in full_config:
+        from esphome.components.esp32_hosted import uses_esp_hosted_3x
+
         add_idf_sdkconfig_option("CONFIG_BT_CLASSIC_ENABLED", False)
         add_idf_sdkconfig_option("CONFIG_BT_BLE_ENABLED", True)
         add_idf_sdkconfig_option("CONFIG_BT_BLUEDROID_ENABLED", True)
         add_idf_sdkconfig_option("CONFIG_BT_CONTROLLER_DISABLED", True)
-        add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID", True)
-        add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_BLUEDROID_HCI_VHCI", True)
+        if uses_esp_hosted_3x():
+            add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_HOST_FEAT_BT", True)
+        else:
+            add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID", True)
+            add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_BLUEDROID_HCI_VHCI", True)
 
     # Check if BLE Client is needed (via esp32_ble_tracker or esp32_ble_client)
     has_ble_client = (

esphome/components/esp32_ble/ble.cpp

@@ -6,16 +6,21 @@
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
 
-#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifndef CONFIG_BT_CONTROLLER_DISABLED
 #include <esp_bt.h>
 #else
 #include "esphome/components/watchdog/watchdog.h"
 #include <cinttypes>
 extern "C" {
 #include <esp_hosted.h>
+#ifndef CONFIG_ESP_HOSTED_HOST_FEAT_BT
 #include <esp_hosted_misc.h>
 #include <esp_hosted_bluedroid.h>
+#endif
 }
+#ifdef CONFIG_ESP_HOSTED_HOST_FEAT_BT
+#include <esp_hosted_bt_host_stack.h>
+#endif
 #endif
 #include <esp_bt_device.h>
 #include <esp_bt_main.h>
@@ -35,7 +40,7 @@ namespace esphome::esp32_ble {
 
 static const char *const TAG = "esp32_ble";
 
-#ifdef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifdef CONFIG_BT_CONTROLLER_DISABLED
 // Bringing up the remote BT controller issues synchronous RPCs to the
 // co-processor with 5 second response timeouts, and the default task watchdog
 // is also 5 seconds. If the co-processor firmware does not answer (for example
@@ -43,8 +48,9 @@ static const char *const TAG = "esp32_ble";
 // device before the RPC could return an error, causing a boot loop. Raise the
 // watchdog for the duration of the bring-up so failures surface as error
 // returns instead. 60 seconds covers the worst case: transport reconnect
-// (up to ~20s), version preflight (1s), controller init/enable (5s each) and
-// the bluedroid host bring-up over the hosted HCI transport.
+// (up to ~20s), version preflight (1s), the controller init retry window
+// (5s of 5s RPCs), controller enable (5s) and the bluedroid host bring-up
+// over the hosted HCI transport.
 static constexpr uint32_t HOSTED_BT_WDT_TIMEOUT_MS = 60000;
 #endif
 
@@ -202,10 +208,10 @@ void ESP32BLE::advertising_init_() {
 
 bool ESP32BLE::ble_setup_() {
   esp_err_t err;
-#ifdef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifdef CONFIG_BT_CONTROLLER_DISABLED
   watchdog::WatchdogManager wdt(HOSTED_BT_WDT_TIMEOUT_MS);
 #endif
-#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifndef CONFIG_BT_CONTROLLER_DISABLED
   if (esp_bt_controller_get_status() != ESP_BT_CONTROLLER_STATUS_ENABLED) {
     // start bt controller
     if (esp_bt_controller_get_status() == ESP_BT_CONTROLLER_STATUS_IDLE) {
@@ -239,8 +245,7 @@ bool ESP32BLE::ble_setup_() {
   }
 
   // Fast preflight (1 second RPC timeout): verifies the co-processor answers
-  // RPCs at all before the 5 second timeout BT controller RPCs below, and
-  // before hosted_hci_bluedroid_open(), which aborts if the transport is down.
+  // RPCs at all before the 5 second timeout BT controller RPCs below.
   esp_hosted_coprocessor_fwver_t fw_ver{};
   if (esp_hosted_get_coprocessor_fwversion(&fw_ver) != ESP_OK) {
     ESP_LOGE(TAG, "Co-processor not responding; BLE disabled. Update its firmware with the esp32_hosted "
@@ -249,6 +254,19 @@ bool ESP32BLE::ble_setup_() {
   }
   ESP_LOGD(TAG, "Co-processor firmware %" PRIu32 ".%" PRIu32 ".%" PRIu32, fw_ver.major1, fw_ver.minor1, fw_ver.patch1);
 
+#ifdef CONFIG_ESP_HOSTED_HOST_FEAT_BT
+  esp_hosted_bt_host_stack_cfg_t bt_cfg{};
+  bt_cfg.stack = ESP_HOSTED_BT_HOST_STACK_BLUEDROID;
+  bt_cfg.bring_up_controller = true;
+  bt_cfg.controller_ready_timeout_ms = EH_BT_CTRL_DEFAULT_READY_TIMEOUT_MS;
+  if (esp_hosted_bt_host_stack_setup(&bt_cfg) != ESP_OK) {
+    ESP_LOGE(TAG,
+             "BT controller bring-up failed; co-processor firmware %" PRIu32 ".%" PRIu32 ".%" PRIu32
+             " may lack BT support. Update it with the esp32_hosted update component; BLE disabled",
+             fw_ver.major1, fw_ver.minor1, fw_ver.patch1);
+    return false;
+  }
+#else
   if (esp_hosted_bt_controller_init() != ESP_OK) {
     ESP_LOGE(TAG,
              "BT controller init failed; co-processor firmware %" PRIu32 ".%" PRIu32 ".%" PRIu32
@@ -273,6 +291,7 @@ bool ESP32BLE::ble_setup_() {
       .register_host_callback = hosted_hci_bluedroid_register_host_callback,
   };
   esp_bluedroid_attach_hci_driver(&operations);
+#endif
 #endif
 
   err = esp_bluedroid_init();
@@ -393,7 +412,7 @@ bool ESP32BLE::ble_setup_() {
 }
 
 bool ESP32BLE::ble_dismantle_() {
-#ifdef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifdef CONFIG_BT_CONTROLLER_DISABLED
   // Same 5 second RPCs as the bring-up path; see HOSTED_BT_WDT_TIMEOUT_MS
   watchdog::WatchdogManager wdt(HOSTED_BT_WDT_TIMEOUT_MS);
 #endif
@@ -416,7 +435,7 @@ bool ESP32BLE::ble_dismantle_() {
     ESP_LOGD(TAG, "Already deinitialized");
   }
 
-#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifndef CONFIG_BT_CONTROLLER_DISABLED
   if (esp_bt_controller_get_status() != ESP_BT_CONTROLLER_STATUS_IDLE) {
     // stop bt controller
     if (esp_bt_controller_get_status() == ESP_BT_CONTROLLER_STATUS_ENABLED) {
@@ -440,6 +459,11 @@ bool ESP32BLE::ble_dismantle_() {
       return false;
     }
   }
+#elif defined(CONFIG_ESP_HOSTED_HOST_FEAT_BT)
+  if (esp_hosted_bt_host_stack_teardown() != ESP_OK) {
+    ESP_LOGE(TAG, "esp_hosted_bt_host_stack_teardown failed");
+    return false;
+  }
 #else
   if (esp_hosted_bt_controller_disable() != ESP_OK) {
     ESP_LOGE(TAG, "esp_hosted_bt_controller_disable failed");

esphome/components/esp32_ble_beacon/esp32_ble_beacon.cpp

@@ -3,7 +3,7 @@
 
 #ifdef USE_ESP32
 
-#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifndef CONFIG_BT_CONTROLLER_DISABLED
 #include <esp_bt.h>
 #endif
 #include <esp_bt_main.h>
@@ -35,7 +35,7 @@ void ESP32BLEBeacon::dump_config() {
     }
   }
   *bpos = '\0';
-#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifndef CONFIG_BT_CONTROLLER_DISABLED
   ESP_LOGCONFIG(TAG,
                 "  UUID: %s, Major: %u, Minor: %u, Min Interval: %ums, Max Interval: %ums, Measured Power: %d"
                 ", TX Power: %ddBm",
@@ -81,7 +81,7 @@ void ESP32BLEBeacon::on_advertise_() {
   ibeacon_adv_data.ibeacon_vendor.measured_power = static_cast<uint8_t>(this->measured_power_);
 
   esp_err_t err;
-#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifndef CONFIG_BT_CONTROLLER_DISABLED
   ESP_LOGD(TAG, "Setting BLE TX power");
   err = esp_ble_tx_power_set(ESP_BLE_PWR_TYPE_ADV, this->tx_power_);
   if (err != ESP_OK) {

esphome/components/esp32_ble_beacon/esp32_ble_beacon.h

@@ -5,7 +5,7 @@
 
 #ifdef USE_ESP32
 
-#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifndef CONFIG_BT_CONTROLLER_DISABLED
 #include <esp_bt.h>
 #endif
 #include <esp_gap_ble_api.h>
@@ -47,7 +47,7 @@ class ESP32BLEBeacon final : public Component {
   void set_min_interval(uint16_t val) { this->min_interval_ = val; }
   void set_max_interval(uint16_t val) { this->max_interval_ = val; }
   void set_measured_power(int8_t val) { this->measured_power_ = val; }
-#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifndef CONFIG_BT_CONTROLLER_DISABLED
   void set_tx_power(esp_power_level_t val) { this->tx_power_ = val; }
 #endif
   void gap_event_handler(esp_gap_ble_cb_event_t event, esp_ble_gap_cb_param_t *param);
@@ -61,7 +61,7 @@ class ESP32BLEBeacon final : public Component {
   uint16_t min_interval_{};
   uint16_t max_interval_{};
   int8_t measured_power_{};
-#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifndef CONFIG_BT_CONTROLLER_DISABLED
   esp_power_level_t tx_power_{};
 #endif
   esp_ble_adv_params_t ble_adv_params_;

esphome/components/esp32_ble_server/ble_server.cpp

@@ -10,7 +10,7 @@
 #include <nvs_flash.h>
 #include <freertos/FreeRTOSConfig.h>
 #include <esp_bt_main.h>
-#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifndef CONFIG_BT_CONTROLLER_DISABLED
 #include <esp_bt.h>
 #endif
 #include <freertos/task.h>

esphome/components/esp32_ble_tracker/esp32_ble_tracker.cpp

@@ -7,7 +7,7 @@
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
 
-#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifndef CONFIG_BT_CONTROLLER_DISABLED
 #include <esp_bt.h>
 #endif
 #include <esp_bt_defs.h>
@@ -635,7 +635,7 @@ void ESP32BLETracker::log_unexpected_state_(const char *operation, ScannerState
 
 #ifdef USE_ESP32_BLE_SOFTWARE_COEXISTENCE
 void ESP32BLETracker::update_coex_preference_(bool force_ble) {
-#ifndef CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#ifndef CONFIG_BT_CONTROLLER_DISABLED
   if (force_ble && !this->coex_prefer_ble_) {
     ESP_LOGD(TAG, "Setting coexistence to Bluetooth to make connection.");
     this->coex_prefer_ble_ = true;
@@ -645,7 +645,7 @@ void ESP32BLETracker::update_coex_preference_(bool force_ble) {
     this->coex_prefer_ble_ = false;
     esp_coex_preference_set(ESP_COEX_PREFER_BALANCE);  // Reset to default
   }
-#endif  // CONFIG_ESP_HOSTED_ENABLE_BT_BLUEDROID
+#endif  // CONFIG_BT_CONTROLLER_DISABLED
 }
 #endif
 

esphome/components/esp32_hosted/__init__.py

@@ -1,5 +1,6 @@
 import os
 from pathlib import Path
+import re
 
 from esphome import pins
 from esphome.components import esp32
@@ -7,15 +8,22 @@ from esphome.components.const import CONF_SLOT, CONF_USE_PSRAM
 import esphome.config_validation as cv
 from esphome.const import (
     CONF_CLK_PIN,
+    CONF_COMPONENTS,
     CONF_CS_PIN,
+    CONF_FRAMEWORK,
     CONF_FREQUENCY,
     CONF_MISO_PIN,
     CONF_MOSI_PIN,
+    CONF_NAME,
+    CONF_REF,
     CONF_RESET_PIN,
     CONF_TYPE,
     CONF_VARIANT,
+    KEY_ESP32,
 )
+from esphome.core import CORE
 from esphome.cpp_generator import add_define
+import esphome.final_validate as fv
 from esphome.types import ConfigType
 
 CODEOWNERS = ["@swoboda1337"]
@@ -23,6 +31,9 @@ DEPENDENCIES = ["esp32"]
 # esp32_ble raises the task watchdog around the remote BT controller bring-up
 AUTO_LOAD = ["watchdog"]
 
+# 3.x is opt-in via a pin under esp32.framework.components (needs ESP-IDF 5.5)
+ESP_HOSTED_COMPONENT = "espressif/esp_hosted"
+
 CONF_ACTIVE_HIGH = "active_high"
 CONF_BUS_WIDTH = "bus_width"
 CONF_CMD_PIN = "cmd_pin"
@@ -145,6 +156,34 @@ CONFIG_SCHEMA = cv.typed_schema(
 )
 
 
+# Version specs that stay within one major (exact, ^, ~, ~=); other ranges are 3.x
+_PINNED_MAJOR = re.compile(r"^(?:==|\^|~=|~)?(\d+)(?:\.(?:\d+|\*))*$")
+
+
+def user_esp_hosted_major() -> int | None:
+    """Major version of a user-pinned esp_hosted, or None without a pin."""
+    try:
+        full_config = fv.full_config.get()
+    except LookupError:
+        full_config = CORE.config
+    esp32_config = full_config.get(KEY_ESP32) or {}
+    for component in esp32_config.get(CONF_FRAMEWORK, {}).get(CONF_COMPONENTS, []):
+        if component.get(CONF_NAME) != ESP_HOSTED_COMPONENT:
+            continue
+        ref = component.get(CONF_REF, "").strip()
+        if not ref:
+            return 2
+        if match := _PINNED_MAJOR.match(ref):
+            return int(match.group(1))
+        return 3
+    return None
+
+
+def uses_esp_hosted_3x() -> bool:
+    """Whether the user pinned esp_hosted 3.x."""
+    return (major := user_esp_hosted_major()) is not None and major >= 3
+
+
 def _final_validate(config: ConfigType) -> None:
     # The esp_hosted releases compatible with older ESP-IDF versions crash at
     # boot with a heap double free in the SDIO RX path (fixed in esp_hosted
@@ -155,12 +194,34 @@ def _final_validate(config: ConfigType) -> None:
             "Remove the framework version from your configuration to use the "
             "recommended version, or pin a version at or above 5.3."
         )
+    if (major := user_esp_hosted_major()) is None or major < 3:
+        return
+    if idf_ver < cv.Version(5, 5, 0):
+        raise cv.Invalid(
+            f"esp_hosted 3.x requires ESP-IDF 5.5 or newer, got {idf_ver}. "
+            f"Remove the {ESP_HOSTED_COMPONENT} pin from esp32.framework."
+            "components to stay on the 2.x line, or use ESP-IDF 5.5 or newer."
+        )
+    if not config[CONF_ACTIVE_HIGH]:
+        raise cv.Invalid(
+            "esp_hosted 3.x always parks the reset line high with a low pulse, so "
+            "'active_high: false' cannot be expressed on it. Remove the "
+            f"{ESP_HOSTED_COMPONENT} pin from esp32.framework.components to stay "
+            "on the 2.x line."
+        )
+    if config[CONF_TYPE] == "sdio" and config[CONF_BUS_WIDTH] == 1:
+        raise cv.Invalid(
+            "esp_hosted 3.x cannot be built with a 1-bit SDIO bus "
+            "(espressif/esp-hosted-mcu#245). Remove the "
+            f"{ESP_HOSTED_COMPONENT} pin from esp32.framework.components to stay "
+            "on the 2.x line, or use a 4-bit bus."
+        )
 
 
 FINAL_VALIDATE_SCHEMA = _final_validate
 
 
-def _configure_sdio(config: ConfigType) -> None:
+def _configure_sdio_2x(config: ConfigType) -> None:
     slot = config[CONF_SLOT]
     esp32.add_idf_sdkconfig_option(
         f"CONFIG_ESP_HOSTED_SDIO_SLOT_{slot}",
@@ -202,7 +263,42 @@ def _configure_sdio(config: ConfigType) -> None:
     )
 
 
-def _configure_spi(config: ConfigType) -> None:
+def _configure_sdio_3x(config: ConfigType) -> None:
+    esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_HOST_TRANSPORT_BUS_SDIO", True)
+    esp32.add_idf_sdkconfig_option(
+        f"CONFIG_ESP_HOSTED_SDIO_SLOT_{config[CONF_SLOT]}",
+        True,
+    )
+    esp32.add_idf_sdkconfig_option(
+        f"CONFIG_ESP_HOSTED_HOST_SDIO_BUS_WIDTH_{config[CONF_BUS_WIDTH]}",
+        True,
+    )
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_SDIO_PIN_CLK", config[CONF_CLK_PIN]
+    )
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_SDIO_PIN_CMD", config[CONF_CMD_PIN]
+    )
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_SDIO_PIN_D0", config[CONF_D0_PIN]
+    )
+    if config[CONF_BUS_WIDTH] == 4:
+        esp32.add_idf_sdkconfig_option(
+            "CONFIG_ESP_HOSTED_HOST_SDIO_PIN_D1", config[CONF_D1_PIN]
+        )
+        esp32.add_idf_sdkconfig_option(
+            "CONFIG_ESP_HOSTED_HOST_SDIO_PIN_D2", config[CONF_D2_PIN]
+        )
+        esp32.add_idf_sdkconfig_option(
+            "CONFIG_ESP_HOSTED_HOST_SDIO_PIN_D3", config[CONF_D3_PIN]
+        )
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_SDIO_CLK_KHZ",
+        int(config[CONF_SDIO_FREQUENCY] // 1000),
+    )
+
+
+def _configure_spi_2x(config: ConfigType) -> None:
     esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_SPI_HOST_INTERFACE", True)
     # SPI mode is set via per-variant choice options
     variant = config[CONF_VARIANT]
@@ -250,11 +346,53 @@ def _configure_spi(config: ConfigType) -> None:
         esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_DR_ACTIVE_LOW", True)
 
 
-async def to_code(config: ConfigType) -> None:
-    add_define("USE_ESP32_HOSTED")
-    transport = config[CONF_TYPE]
-    transport_prefix = "SDIO" if transport == "sdio" else "SPI"
+def _configure_spi_3x(config: ConfigType) -> None:
+    esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_HOST_TRANSPORT_BUS_SPI", True)
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_SPI_MODE", config[CONF_SPI_MODE]
+    )
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_SPI_CLK_MHZ", int(config[CONF_FREQUENCY] // 1e6)
+    )
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_SPI_MOSI_GPIO", config[CONF_MOSI_PIN]
+    )
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_SPI_MISO_GPIO", config[CONF_MISO_PIN]
+    )
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_SPI_CLK_GPIO", config[CONF_CLK_PIN]
+    )
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_SPI_CS_GPIO", config[CONF_CS_PIN]
+    )
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_HANDSHAKE_GPIO", config[CONF_HANDSHAKE_PIN]
+    )
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_DATA_READY_GPIO", config[CONF_DATA_READY_PIN]
+    )
+    # Handshake and data_ready polarity
+    if config[CONF_HANDSHAKE_ACTIVE_HIGH]:
+        esp32.add_idf_sdkconfig_option(
+            "CONFIG_ESP_HOSTED_SPI_HANDSHAKE_ACTIVE_HIGH", True
+        )
+    else:
+        esp32.add_idf_sdkconfig_option(
+            "CONFIG_ESP_HOSTED_SPI_HANDSHAKE_ACTIVE_LOW", True
+        )
+    if config[CONF_DATA_READY_ACTIVE_HIGH]:
+        esp32.add_idf_sdkconfig_option(
+            "CONFIG_ESP_HOSTED_SPI_DATAREADY_ACTIVE_HIGH", True
+        )
+    else:
+        esp32.add_idf_sdkconfig_option(
+            "CONFIG_ESP_HOSTED_SPI_DATAREADY_ACTIVE_LOW", True
+        )
 
+
+def _configure_2x(config: ConfigType) -> None:
+    transport_prefix = "SDIO" if config[CONF_TYPE] == "sdio" else "SPI"
     # Reset polarity
     if config[CONF_ACTIVE_HIGH]:
         esp32.add_idf_sdkconfig_option(
@@ -274,24 +412,11 @@ async def to_code(config: ConfigType) -> None:
         f"CONFIG_SLAVE_IDF_TARGET_{config[CONF_VARIANT]}",  # NOLINT
         True,
     )
-
-    # Transport-specific configuration
-    if transport == "sdio":
-        _configure_sdio(config)
+    if config[CONF_TYPE] == "sdio":
+        _configure_sdio_2x(config)
     else:
-        _configure_spi(config)
-
-    # ESP-NOW-over-hosted shim: only the radio-less ESP32-P4 host needs it (see
-    # the note by _MAX_CUSTOM_MSG_HANDLERS). Enabled for every P4 host, not
-    # gated on the `espnow` component being present: the shim is tiny and the
-    # esp_now_* symbols/CustomRpc calls it defines require these Kconfig options
-    # to link whenever esp_now_hosted.cpp compiles (which is on any P4 host), so
-    # coupling the two keeps the build consistent. When `espnow` is absent the
-    # symbols are simply unused and never register a callback at runtime.
+        _configure_spi_2x(config)
     if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4:
-        add_define("USE_ESP_NOW_HOSTED")
-        # esp_now_hosted.cpp includes esp_now.h, which esp_wifi provides
-        esp32.include_builtin_idf_component("esp_wifi")
         # esp-hosted's CustomRpc ("peer data transfer") path — off by default.
         esp32.add_idf_sdkconfig_option(
             "CONFIG_ESP_HOSTED_ENABLE_PEER_DATA_TRANSFER", True
@@ -299,18 +424,64 @@ async def to_code(config: ConfigType) -> None:
         esp32.add_idf_sdkconfig_option(
             "CONFIG_ESP_HOSTED_MAX_CUSTOM_MSG_HANDLERS", _MAX_CUSTOM_MSG_HANDLERS
         )
-
     # Place the transport mempool in PSRAM. Required on memory-tight host
     # configurations (e.g. P4 with a large LVGL UI) where the internal-RAM
     # mempool allocation fails at boot with `sdio_mempool_create` assert.
     if config[CONF_USE_PSRAM]:
         esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_MEMPOOL_PREFER_SPIRAM", True)
 
+
+def _configure_3x(config: ConfigType) -> None:
+    esp32.add_idf_sdkconfig_option(
+        "CONFIG_ESP_HOSTED_HOST_RESET_GPIO", config[CONF_RESET_PIN]
+    )
+    # Co-processor variant
+    esp32.add_idf_sdkconfig_option(
+        f"CONFIG_ESP_HOSTED_CP_TARGET_{config[CONF_VARIANT]}",
+        True,
+    )
+    if config[CONF_TYPE] == "sdio":
+        _configure_sdio_3x(config)
+    else:
+        _configure_spi_3x(config)
+    if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4:
+        esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_HOST_FEAT_PEER_DATA", True)
+        esp32.add_idf_sdkconfig_option(
+            "CONFIG_ESP_HOSTED_HOST_FEAT_PEER_DATA_MAX_CUSTOM_MSG_HANDLERS",
+            _MAX_CUSTOM_MSG_HANDLERS,
+        )
+    if config[CONF_USE_PSRAM]:
+        esp32.add_idf_sdkconfig_option("CONFIG_EH_HOST_PORT_DMA_PREFER_SPIRAM", True)
+        esp32.add_idf_sdkconfig_option("CONFIG_ESP_HOSTED_DFLT_TASK_FROM_SPIRAM", True)
+
+
+async def to_code(config: ConfigType) -> None:
+    add_define("USE_ESP32_HOSTED")
+    use_3x = uses_esp_hosted_3x()
+
+    if use_3x:
+        _configure_3x(config)
+    else:
+        _configure_2x(config)
+
+    # ESP-NOW-over-hosted shim: only the radio-less ESP32-P4 host needs it (see
+    # the note by _MAX_CUSTOM_MSG_HANDLERS). Enabled for every P4 host, not
+    # gated on the `espnow` component being present: the shim is tiny and the
+    # esp_now_* symbols/CustomRpc calls it defines require the peer-data Kconfig
+    # options (set above) to link whenever esp_now_hosted.cpp compiles (which is
+    # on any P4 host), so coupling the two keeps the build consistent. When
+    # `espnow` is absent the symbols are simply unused and never register a
+    # callback at runtime.
+    if esp32.get_esp32_variant() == esp32.VARIANT_ESP32P4:
+        add_define("USE_ESP_NOW_HOSTED")
+        # esp_now_hosted.cpp includes esp_now.h, which esp_wifi provides
+        esp32.include_builtin_idf_component("esp_wifi")
+
     # Library versions; this component set requires ESP-IDF 5.3 or newer,
     # which is enforced at validation time.
     idf_ver = esp32.idf_version()
     os.environ["ESP_IDF_VERSION"] = f"{idf_ver.major}.{idf_ver.minor}"
-    esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="1.6.3")
+    esp32.add_idf_component(name="espressif/esp_wifi_remote", ref="1.6.5")
     esp32.add_idf_component(name="espressif/wifi_remote_over_eppp", ref="0.3.3")
     esp32.add_idf_component(name="espressif/eppp_link", ref="1.1.5")
     esp32.add_idf_component(name="espressif/esp_hosted", ref="2.12.13")

esphome/components/esp32_hosted/update/esp32_hosted_update.cpp

@@ -26,6 +26,8 @@ static const char *const TAG = "esp32_hosted.update";
 
 // Older coprocessor firmware versions have a 1500-byte limit per RPC call
 constexpr size_t CHUNK_SIZE = 1500;
+// OTA begin blocks while the coprocessor erases its partition
+constexpr uint32_t OTA_WDT_TIMEOUT_MS = 60000;
 
 #ifdef USE_ESP32_HOSTED_HTTP_UPDATE
 // Interval/timeout IDs (uint32_t to avoid string comparison)
@@ -336,7 +338,11 @@ bool Esp32HostedUpdate::stream_firmware_to_coprocessor_() {
   ESP_LOGI(TAG, "Firmware size: %zu bytes", total_size);
 
   // Begin OTA on coprocessor
-  esp_err_t err = esp_hosted_slave_ota_begin();  // NOLINT
+  esp_err_t err;
+  {
+    watchdog::WatchdogManager wdt(OTA_WDT_TIMEOUT_MS);
+    err = esp_hosted_slave_ota_begin();  // NOLINT
+  }
   if (err != ESP_OK) {
     ESP_LOGE(TAG, "Failed to begin OTA: %s", esp_err_to_name(err));
     container->end();
@@ -425,7 +431,11 @@ bool Esp32HostedUpdate::write_embedded_firmware_to_coprocessor_() {
 
   ESP_LOGI(TAG, "Starting OTA update (%zu bytes)", this->firmware_size_);
 
-  esp_err_t err = esp_hosted_slave_ota_begin();  // NOLINT
+  esp_err_t err;
+  {
+    watchdog::WatchdogManager wdt(OTA_WDT_TIMEOUT_MS);
+    err = esp_hosted_slave_ota_begin();  // NOLINT
+  }
   if (err != ESP_OK) {
     ESP_LOGE(TAG, "Failed to begin OTA: %s", esp_err_to_name(err));
     this->status_set_error(LOG_STR("Failed to begin OTA"));
@@ -472,8 +482,6 @@ void Esp32HostedUpdate::perform(bool force) {
   this->update_info_.has_progress = false;
   this->publish_state();
 
-  watchdog::WatchdogManager watchdog(60000);
-
 #ifdef USE_ESP32_HOSTED_HTTP_UPDATE
   if (!this->stream_firmware_to_coprocessor_())
 #else

esphome/components/esp8266/__init__.py

@@ -6,7 +6,6 @@ import subprocess
 from typing import Any
 
 from esphome.build_helpers.native import native_backend
-from esphome.build_helpers.pch import pch_extra_scripts
 import esphome.codegen as cg
 import esphome.config_validation as cv
 from esphome.const import (
@@ -35,7 +34,7 @@ from esphome.core import (
 )
 from esphome.core.config import BOARD_MAX_LENGTH
 from esphome.helpers import IS_MACOS, copy_file_if_changed
-from esphome.platformio.toolchain import copy_ccache_script, copy_pch_script
+from esphome.platformio.toolchain import copy_ccache_script
 from esphome.storage_json import StorageJSON
 from esphome.types import ConfigType
 
@@ -112,22 +111,32 @@ def set_core_data(config: ConfigType) -> ConfigType:
 
 _TOOLCHAINS = (Toolchain.PLATFORMIO, Toolchain.ARDUINO)
 _validate_toolchain = cv.toolchain_enum(_TOOLCHAINS)
-_resolve_toolchain = cv.resolve_toolchain("ESP8266", _TOOLCHAINS, Toolchain.PLATFORMIO)
+_resolve_toolchain = cv.resolve_toolchain("ESP8266", _TOOLCHAINS, Toolchain.ARDUINO)
+
+
+def _warn_platformio_toolchain(config: ConfigType) -> ConfigType:
+    # Remove before 2027.4.0
+    if CORE.using_toolchain_platformio:
+        _LOGGER.warning(
+            "The 'platformio' toolchain for ESP8266 is deprecated and will be "
+            "removed in ESPHome 2027.4.0; the native 'arduino' toolchain is the "
+            "default."
+        )
+    return config
 
 
 def _validate_native_toolchain(config: ConfigType) -> ConfigType:
     """Constraints of the native (non-PlatformIO) Arduino toolchain."""
     if not CORE.using_toolchain_arduino:
         return config
-    from esphome.arduino8266.framework import MIN_FRAMEWORK_VERSION
+    from esphome.arduino8266.framework import framework_release
 
     conf = config[CONF_FRAMEWORK]
     version = cv.Version.parse(conf[CONF_VERSION])
-    if version < MIN_FRAMEWORK_VERSION:
-        raise cv.Invalid(
-            "'toolchain: arduino' requires framework version "
-            f"{MIN_FRAMEWORK_VERSION} or newer"
-        )
+    try:
+        framework_release(version)
+    except EsphomeError as err:
+        raise cv.Invalid(str(err), path=[CONF_FRAMEWORK, CONF_VERSION]) from err
     # platform_version is a PlatformIO concept; drop it, warning when a
     # custom pin is discarded
     if (
@@ -138,6 +147,8 @@ def _validate_native_toolchain(config: ConfigType) -> ConfigType:
             "'platform_version' is ignored by 'toolchain: arduino'; the native "
             "toolchain downloads the framework and compiler directly"
         )
+    # The native path fetches its own build; source may only be the
+    # PlatformIO default the schema filled in
     if conf[CONF_SOURCE] != _format_framework_arduino_version(version):
         raise cv.Invalid(
             "'toolchain: arduino' does not support a custom framework source; "
@@ -175,16 +186,27 @@ def get_download_types(storage_json: StorageJSON) -> list[dict[str, str]]:
     ]
 
 
+def _framework_package_version(ver: cv.Version) -> str:
+    """Map an Arduino core version to its PlatformIO registry package version
+    (3.1.2 -> 3.30102.0; the leading 3 is the package major)."""
+    if ver.major > 3:
+        raise EsphomeError(
+            f"Arduino core {ver} is not supported yet; "
+            "the newest known core series is 3.x"
+        )
+    if ver.major < 3:
+        raise EsphomeError(
+            f"Arduino core {ver} is not supported; ESPHome requires core 3.x"
+        )
+    return f"3.{ver.major}{ver.minor:02d}{ver.patch:02d}.0"
+
+
 def _format_framework_arduino_version(ver: cv.Version) -> str:
     # format the given arduino (https://github.com/esp8266/Arduino/releases) version to
     # a PIO platformio/framework-arduinoespressif8266 value
     # List of package versions: https://api.registry.platformio.org/v3/packages/platformio/tool/framework-arduinoespressif8266
-    # Same encoding the native toolchain uses for its package download, so a
-    # custom-source check against this value cannot drift from what it fetches.
-    from esphome.arduino8266.framework import framework_package_version
-
     try:
-        return f"~{framework_package_version(ver)}"
+        return f"~{_framework_package_version(ver)}"
     except EsphomeError as err:
         # Anchor the 4.x rejection to the framework version line instead of
         # aborting with a bare traceback-level error
@@ -195,7 +217,8 @@ def _format_framework_arduino_version(ver: cv.Version) -> str:
 #  * New framework historically have had some regressions, especially for WiFi.
 #    The new version needs to be thoroughly validated before changing the
 #    recommended version as otherwise a bunch of devices could be bricked
-#  * For all constants below, update platformio.ini (in this repo)
+#  * For all constants below, update platformio.ini (in this repo) and
+#    FRAMEWORK_RELEASES in esphome/arduino8266/framework.py
 
 # The default/recommended arduino framework version
 #  - https://github.com/esp8266/Arduino/releases
@@ -307,6 +330,7 @@ CONFIG_SCHEMA = cv.All(
         }
     ),
     _resolve_toolchain,
+    _warn_platformio_toolchain,
     _validate_native_toolchain,
     set_core_data,
 )
@@ -315,8 +339,8 @@ CONFIG_SCHEMA = cv.All(
 def check_rosetta() -> None:
     """Fail fast when the x86_64 ESP8266 toolchain cannot run on this Mac.
 
-    There is no native arm64 build of the xtensa-lx106 toolchain; on Apple
-    Silicon it runs under Rosetta 2, which macOS updates can remove.
+    PlatformIO's xtensa-lx106 toolchain has no arm64 build; on Apple Silicon
+    it runs under Rosetta 2, which macOS updates can remove.
     """
     if not IS_MACOS or platform.machine() != "arm64":
         return
@@ -331,9 +355,10 @@ def check_rosetta() -> None:
         return  # arch(1) unavailable; let the build proceed
     if result.returncode != 0:
         raise EsphomeError(
-            "ESP8266 builds on Apple Silicon Macs use an Intel (x86_64) "
-            "compiler that requires Rosetta 2, which is not installed on "
-            "this system. Install it with:\n"
+            "ESP8266 builds with PlatformIO on Apple Silicon Macs use an Intel "
+            "(x86_64) compiler that requires Rosetta 2, which is not installed "
+            "on this system. Either set 'toolchain: arduino' under 'esp8266:', "
+            "which needs no Rosetta, or install it with:\n"
             "  softwareupdate --install-rosetta --agree-to-license"
         )
 
@@ -381,7 +406,6 @@ async def to_code(config: ConfigType) -> None:
         ]
         if not enable_scanf_float:
             extra_scripts.append("pre:remove_float_scanf.py")
-        extra_scripts.extend(pch_extra_scripts())
         extra_scripts.append("post:post_build.py")
         cg.add_platformio_option("extra_scripts", extra_scripts)
 
@@ -538,7 +562,6 @@ def copy_files() -> None:
             CORE.relative_build_path(f"{script}.py"),
         )
     copy_ccache_script()
-    copy_pch_script()
 
 
 # ESP logs stack trace decoder, based on https://github.com/me-no-dev/EspExceptionDecoder

esphome/components/esp8266/gpio.cpp

@@ -8,6 +8,8 @@ namespace esphome::esp8266 {
 static const char *const TAG = "esp8266";
 
 static int flags_to_mode(gpio::Flags flags, uint8_t pin) {
+  flags =
+      flags & (gpio::FLAG_INPUT | gpio::FLAG_OUTPUT | gpio::FLAG_OPEN_DRAIN | gpio::FLAG_PULLUP | gpio::FLAG_PULLDOWN);
   if (flags == gpio::FLAG_OUTPUT || flags == (gpio::FLAG_OUTPUT | gpio::FLAG_INPUT)) {
     return OUTPUT;
   }

esphome/components/ethernet/ethernet_component_esp32.cpp

@@ -10,6 +10,9 @@
 #include <lwip/dns.h>
 #include <cinttypes>
 #include "esp_event.h"
+#ifdef USE_PSRAM
+#include <esp_psram.h>
+#endif
 
 // IDF 6.0 moved per-chip PHY/MAC drivers to the Espressif Component Registry;
 // they are no longer included via esp_eth.h and need explicit includes.
@@ -71,6 +74,32 @@ static const char *const TAG = "ethernet";
 // PHY register size for hex logging
 static constexpr size_t PHY_REG_SIZE = 2;
 
+// Dual wifi + ethernet SPI builds: the one place internal RAM is short and lwip's other buffers are
+// already in PSRAM. Not with L2 TAP, whose filter lives in the glue's input path this replaces.
+#if defined(USE_PSRAM) && defined(USE_ETHERNET_SPI) && defined(USE_WIFI) && !defined(CONFIG_ESP_NETIF_L2_TAP)
+#define USE_ETHERNET_RX_PSRAM
+// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) - reported by dump_config()
+static bool rx_psram_installed = false;
+
+// ESP-IDF ethernet drivers malloc() every received frame in internal RAM, where it stays until lwIP
+// hands it to the application. Move it to PSRAM; if that fails the frame is passed on where it is.
+static esp_err_t eth_input_to_psram(esp_eth_handle_t handle, uint8_t *buffer, uint32_t length, void *priv) {
+  auto *copy = static_cast<uint8_t *>(heap_caps_malloc(length, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
+  if (copy != nullptr) {
+    memcpy(copy, buffer, length);
+    free(buffer);  // NOLINT(cppcoreguidelines-no-malloc) - allocated by the driver with malloc()
+    buffer = copy;
+  } else {
+    static bool warned = false;  // once, this runs per frame in the driver's task
+    if (!warned) {
+      warned = true;
+      ESP_LOGW(TAG, "PSRAM allocation failed, frame kept in internal RAM (reported once)");
+    }
+  }
+  return esp_netif_receive(static_cast<esp_netif_t *>(priv), buffer, length, nullptr);
+}
+#endif
+
 void EthernetComponent::log_error_and_mark_failed_(esp_err_t err, const char *message) {
   ESP_LOGE(TAG, "%s: (%d) %s", message, err, esp_err_to_name(err));
   this->mark_failed();
@@ -453,6 +482,16 @@ void EthernetComponent::ethernet_lazy_init_() {
   /* attach Ethernet driver to TCP/IP stack */
   err = esp_netif_attach(this->eth_netif_, esp_eth_new_netif_glue(this->eth_handle_));
   ESPHL_ERROR_CHECK(err, "ETH netif attach error");
+#ifdef USE_ETHERNET_RX_PSRAM
+  // The glue frees every receive buffer with free(), so the replacement buffer must come from the heap
+  if (esp_psram_is_initialized()) {
+    err = esp_eth_update_input_path(this->eth_handle_, eth_input_to_psram, this->eth_netif_);
+    rx_psram_installed = err == ESP_OK;
+    if (!rx_psram_installed) {
+      ESP_LOGW(TAG, "PSRAM RX path not installed: %s", esp_err_to_name(err));
+    }
+  }
+#endif
 
   // Register user defined event handers
   err = esp_event_handler_register(ETH_EVENT, ESP_EVENT_ANY_ID, &EthernetComponent::eth_event_handler, nullptr);
@@ -630,6 +669,13 @@ void EthernetComponent::dump_config() {
                 this->clk_pin_, this->mdc_pin_, this->mdio_pin_, this->phy_addr_);
 #endif
   ESP_LOGCONFIG(TAG, "  Type: %s", eth_type);
+#ifdef USE_ETHERNET_RX_PSRAM
+  // Only known once the driver is up; with enable_on_boot: false that is after this dump
+  if (this->ethernet_initialized_) {
+    ESP_LOGCONFIG(TAG, "  RX frames: %s",
+                  rx_psram_installed ? LOG_STR_LITERAL("PSRAM") : LOG_STR_LITERAL("internal RAM"));
+  }
+#endif
 }
 
 network::IPAddresses EthernetComponent::get_ip_addresses() {

esphome/components/hlw8032/sensor.py

@@ -79,20 +79,11 @@ async def to_code(config: ConfigType) -> None:
     await cg.register_component(var, config)
     await uart.register_uart_device(var, config)
 
-    if voltage_config := config.get(CONF_VOLTAGE):
-        sens = await sensor.new_sensor(voltage_config)
-        cg.add(var.set_voltage_sensor(sens))
-    if current_config := config.get(CONF_CURRENT):
-        sens = await sensor.new_sensor(current_config)
-        cg.add(var.set_current_sensor(sens))
-    if power_config := config.get(CONF_POWER):
-        sens = await sensor.new_sensor(power_config)
-        cg.add(var.set_power_sensor(sens))
-    if apparent_power_config := config.get(CONF_APPARENT_POWER):
-        sens = await sensor.new_sensor(apparent_power_config)
-        cg.add(var.set_apparent_power_sensor(sens))
-    if power_factor_config := config.get(CONF_POWER_FACTOR):
-        sens = await sensor.new_sensor(power_factor_config)
-        cg.add(var.set_power_factor_sensor(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_VOLTAGE, var.set_voltage_sensor)
+    await sensors(CONF_CURRENT, var.set_current_sensor)
+    await sensors(CONF_POWER, var.set_power_sensor)
+    await sensors(CONF_APPARENT_POWER, var.set_apparent_power_sensor)
+    await sensors(CONF_POWER_FACTOR, var.set_power_factor_sensor)
     cg.add(var.set_current_resistor(config[CONF_CURRENT_RESISTOR]))
     cg.add(var.set_voltage_divider(config[CONF_VOLTAGE_DIVIDER]))

esphome/components/hmc5883l/sensor.py

@@ -136,15 +136,8 @@ async def to_code(config: ConfigType) -> None:
     cg.add(var.set_oversampling(config[CONF_OVERSAMPLING]))
     cg.add(var.set_datarate(auto_data_rate(config)))
     cg.add(var.set_range(config[CONF_RANGE]))
-    if CONF_FIELD_STRENGTH_X in config:
-        sens = await sensor.new_sensor(config[CONF_FIELD_STRENGTH_X])
-        cg.add(var.set_x_sensor(sens))
-    if CONF_FIELD_STRENGTH_Y in config:
-        sens = await sensor.new_sensor(config[CONF_FIELD_STRENGTH_Y])
-        cg.add(var.set_y_sensor(sens))
-    if CONF_FIELD_STRENGTH_Z in config:
-        sens = await sensor.new_sensor(config[CONF_FIELD_STRENGTH_Z])
-        cg.add(var.set_z_sensor(sens))
-    if CONF_HEADING in config:
-        sens = await sensor.new_sensor(config[CONF_HEADING])
-        cg.add(var.set_heading_sensor(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_FIELD_STRENGTH_X, var.set_x_sensor)
+    await sensors(CONF_FIELD_STRENGTH_Y, var.set_y_sensor)
+    await sensors(CONF_FIELD_STRENGTH_Z, var.set_z_sensor)
+    await sensors(CONF_HEADING, var.set_heading_sensor)

esphome/components/host/__init__.py

@@ -20,9 +20,6 @@ from .const import KEY_HOST
 # force import gpio to register pin schema
 from .gpio import host_pin_to_code  # noqa: F401
 
-# The curated headers the host build precompiles
-HOST_PCH_PREFIX = "esphome/core/pch_prefix.h"
-
 CODEOWNERS = ["@esphome/core", "@clydebarrow"]
 AUTO_LOAD = ["network", "preferences"]
 IS_TARGET_PLATFORM = True

esphome/components/htu21d/sensor.py

@@ -67,17 +67,10 @@ async def to_code(config: ConfigType) -> None:
     await cg.register_component(var, config)
     await i2c.register_i2c_device(var, config)
 
-    if CONF_TEMPERATURE in config:
-        sens = await sensor.new_sensor(config[CONF_TEMPERATURE])
-        cg.add(var.set_temperature(sens))
-
-    if CONF_HUMIDITY in config:
-        sens = await sensor.new_sensor(config[CONF_HUMIDITY])
-        cg.add(var.set_humidity(sens))
-
-    if CONF_HEATER in config:
-        sens = await sensor.new_sensor(config[CONF_HEATER])
-        cg.add(var.set_heater(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_TEMPERATURE, var.set_temperature)
+    await sensors(CONF_HUMIDITY, var.set_humidity)
+    await sensors(CONF_HEATER, var.set_heater)
 
     cg.add(var.set_sensor_model(config[CONF_MODEL]))
 

esphome/components/inkbird_ibsth1_mini/sensor.py

@@ -71,15 +71,8 @@ async def to_code(config: ConfigType) -> None:
 
     cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex))
 
-    if CONF_TEMPERATURE in config:
-        sens = await sensor.new_sensor(config[CONF_TEMPERATURE])
-        cg.add(var.set_temperature(sens))
-    if CONF_EXTERNAL_TEMPERATURE in config:
-        sens = await sensor.new_sensor(config[CONF_EXTERNAL_TEMPERATURE])
-        cg.add(var.set_external_temperature(sens))
-    if CONF_HUMIDITY in config:
-        sens = await sensor.new_sensor(config[CONF_HUMIDITY])
-        cg.add(var.set_humidity(sens))
-    if CONF_BATTERY_LEVEL in config:
-        sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL])
-        cg.add(var.set_battery_level(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_TEMPERATURE, var.set_temperature)
+    await sensors(CONF_EXTERNAL_TEMPERATURE, var.set_external_temperature)
+    await sensors(CONF_HUMIDITY, var.set_humidity)
+    await sensors(CONF_BATTERY_LEVEL, var.set_battery_level)

esphome/components/libretiny/__init__.py

@@ -2,7 +2,6 @@ import json
 import logging
 from pathlib import Path
 
-from esphome.build_helpers.pch import pch_extra_scripts
 import esphome.codegen as cg
 import esphome.config_validation as cv
 from esphome.const import (
@@ -27,7 +26,7 @@ from esphome.const import (
 from esphome.core import CORE
 from esphome.core.config import BOARD_MAX_LENGTH
 from esphome.helpers import copy_file_if_changed
-from esphome.platformio.toolchain import copy_ccache_script, copy_pch_script
+from esphome.platformio.toolchain import copy_ccache_script
 from esphome.storage_json import StorageJSON
 
 from . import gpio  # noqa: F401
@@ -516,7 +515,7 @@ async def component_to_code(config):
     cg.add_platformio_option("build_src_flags", build_src_flags)
     # Must run before the platform's builder scripts are loaded; see the script.
     cg.add_platformio_option("extra_scripts", ["pre:scons_dont_inherit.py"])
-    cg.add_platformio_option("extra_scripts", ["pre:ccache.py", *pch_extra_scripts()])
+    cg.add_platformio_option("extra_scripts", ["pre:ccache.py"])
     # IRAM_ATTR is a no-op on BK72xx (SDK masks FIQ+IRQ around flash ops).
     # On other families, patch_linker.py routes .sram.text into the right
     # RAM-executable output section and prints a post-link placement summary.
@@ -626,4 +625,3 @@ def copy_files() -> None:
         CORE.relative_build_path("scons_dont_inherit.py"),
     )
     copy_ccache_script()
-    copy_pch_script()

esphome/components/libretiny/gpio_arduino.cpp

@@ -8,6 +8,8 @@ namespace esphome::libretiny {
 static const char *const TAG = "libretiny.gpio";
 
 static int IRAM_ATTR flags_to_mode(gpio::Flags flags) {
+  flags =
+      flags & (gpio::FLAG_INPUT | gpio::FLAG_OUTPUT | gpio::FLAG_OPEN_DRAIN | gpio::FLAG_PULLUP | gpio::FLAG_PULLDOWN);
   if (flags == gpio::FLAG_INPUT) {
     return INPUT;
   } else if (flags == gpio::FLAG_OUTPUT) {

esphome/components/logger/__init__.py

@@ -504,6 +504,7 @@ async def _late_logger_init(config: ConfigType) -> None:
         zephyr_add_prj_conf("RESET_ON_FATAL_ERROR", False)
         zephyr_add_prj_conf("THREAD_LOCAL_STORAGE", True)
         if has_serial_logging:
+            zephyr_add_prj_conf("SERIAL", True)
             if config[CONF_HARDWARE_UART] == UART0:
                 zephyr_add_overlay("""&uart0 { status = "okay";};""")
             if config[CONF_HARDWARE_UART] == UART1:

esphome/components/logger/logger_zephyr.cpp

@@ -54,6 +54,7 @@ void Logger::cdc_loop_() {
 #endif
 
 void Logger::pre_setup() {
+#ifdef CONFIG_SERIAL
   if (this->baud_rate_ > 0) {
     static const struct device *uart_dev = nullptr;
     switch (this->uart_) {
@@ -87,6 +88,7 @@ void Logger::pre_setup() {
 #endif
     }
   }
+#endif
   global_logger = this;
   ESP_LOGI(TAG, "Log initialized");
 #ifdef USE_LOGGER_EARLY_MESSAGE

esphome/components/ltr501/sensor.py

@@ -229,29 +229,13 @@ async def to_code(config: ConfigType) -> None:
     await cg.register_component(var, config)
     await i2c.register_i2c_device(var, config)
 
-    if als_config := config.get(CONF_AMBIENT_LIGHT):
-        sens = await sensor.new_sensor(als_config)
-        cg.add(var.set_ambient_light_sensor(sens))
-
-    if infrared_cnt_config := config.get(CONF_INFRARED_COUNTS):
-        sens = await sensor.new_sensor(infrared_cnt_config)
-        cg.add(var.set_infrared_counts_sensor(sens))
-
-    if full_spect_cnt_config := config.get(CONF_FULL_SPECTRUM_COUNTS):
-        sens = await sensor.new_sensor(full_spect_cnt_config)
-        cg.add(var.set_full_spectrum_counts_sensor(sens))
-
-    if act_gain_config := config.get(CONF_ACTUAL_GAIN):
-        sens = await sensor.new_sensor(act_gain_config)
-        cg.add(var.set_actual_gain_sensor(sens))
-
-    if act_itime_config := config.get(CONF_ACTUAL_INTEGRATION_TIME):
-        sens = await sensor.new_sensor(act_itime_config)
-        cg.add(var.set_actual_integration_time_sensor(sens))
-
-    if prox_cnt_config := config.get(CONF_PS_COUNTS):
-        sens = await sensor.new_sensor(prox_cnt_config)
-        cg.add(var.set_proximity_counts_sensor(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_AMBIENT_LIGHT, var.set_ambient_light_sensor)
+    await sensors(CONF_INFRARED_COUNTS, var.set_infrared_counts_sensor)
+    await sensors(CONF_FULL_SPECTRUM_COUNTS, var.set_full_spectrum_counts_sensor)
+    await sensors(CONF_ACTUAL_GAIN, var.set_actual_gain_sensor)
+    await sensors(CONF_ACTUAL_INTEGRATION_TIME, var.set_actual_integration_time_sensor)
+    await sensors(CONF_PS_COUNTS, var.set_proximity_counts_sensor)
 
     await automation.build_callback_automations(var, config, _CALLBACK_AUTOMATIONS)
 

esphome/components/mipi/__init__.py

@@ -606,11 +606,11 @@ class DriverChip:
         """
         Create the init sequence for the display.
         Use the default sequence from the model, if any, and append any custom sequence provided in the config.
-        Append SLPOUT (if not already in the sequence) and DISPON to the end of the sequence
+        Append SLPOUT (if not suppressed by the model) and DISPON to the end of the sequence
         MADCTL will be set if add_madctl is True
         If add_reset is True, a reset is prepended: a software reset when no reset pin
         is configured (and the model doesn't skip it), followed by a settling delay that
-        both a software and a hardware reset require.
+        both a software and a hardware reset require. The delay length is set via reset_delay, and defaults to 10ms.
         Returns the init sequence
         """
         sequence = list(self.initsequence or ())
@@ -620,12 +620,16 @@ class DriverChip:
         sequence = [x if isinstance(x, tuple) else (x,) for x in sequence]
 
         if add_reset:
+            # Matches the 1-255ms range map_sequence() already allows for a "delay N" entry.
+            reset_delay = self.get_default("reset_delay", 10)
+            if reset_delay < 1 or reset_delay > 255:
+                raise ValueError("reset_delay must be between 1 and 255ms")
             reset: list = []
             # A software reset is only needed when there is no hardware reset pin.
             if CONF_RESET_PIN not in config and not self.skip_command("SWRESET"):
                 reset.append((SWRESET,))
             # Both a software and a hardware reset need a settling delay before further commands.
-            reset.append(delay(10))
+            reset.append(delay(reset_delay))
             sequence = reset + sequence
 
         # Set pixel format if not already in the custom sequence

esphome/components/mipi_rgb/display.py

@@ -285,7 +285,7 @@ async def to_code(config: ConfigType) -> None:
 
     if CONF_SPI_ID in config:
         await spi.register_spi_device(var, config, write_only=True)
-        sequence = model.get_sequence(config)
+        sequence = model.get_sequence(config, add_reset=True)
         cg.add(var.set_init_sequence(sequence))
 
     cg.add(var.set_color_mode(COLOR_ORDERS[config[CONF_COLOR_ORDER]]))

esphome/components/mipi_rgb/mipi_rgb.cpp

@@ -44,8 +44,10 @@ void MipiRgb::setup_enables_() {
 void MipiRgbSpi::setup() {
   this->setup_enables_();
   this->spi_setup();
-  this->write_init_sequence_();
   this->common_setup_();
+  if (this->is_failed())
+    return;
+  this->write_init_sequence_();
 }
 void MipiRgbSpi::write_command_(uint8_t value) {
   this->enable();

esphome/components/mipi_rgb/models/st7701s.py

@@ -7,6 +7,10 @@ SDIR_CMD = 0xC7
 
 
 class ST7701S(RgbDriverChip):
+    def __init__(self, *args, reset_delay=50, **kwargs):
+        kwargs["reset_delay"] = reset_delay
+        super().__init__(*args, **kwargs)
+
     # The ST7701s does not use the standard MADCTL bits for x/y mirroring
     def add_madctl(self, sequence: list, config: dict) -> int:
         transform = self.get_transform(config)
@@ -49,7 +53,6 @@ st7701s = ST7701S(
     pclk_frequency="16MHz",
     pclk_inverted=True,
     initsequence=(
-        (0x01,),  # Software Reset
         (0xFF, 0x77, 0x01, 0x00, 0x00, 0x10),  # Page 0
         (0xC0, 0x3B, 0x00), (0xC1, 0x0D, 0x02), (0xC2, 0x31, 0x05),
         (0xB0, 0x00, 0x11, 0x18, 0x0E, 0x11, 0x06, 0x07, 0x08, 0x07, 0x22, 0x04, 0x12, 0x0F, 0xAA, 0x31, 0x18,),

esphome/components/mopeka_std_check/sensor.py

@@ -128,15 +128,8 @@ async def to_code(config: ConfigType) -> None:
     if CONF_PROPANE_BUTANE_MIX in config:
         cg.add(var.set_propane_butane_mix(config[CONF_PROPANE_BUTANE_MIX]))
 
-    if CONF_TEMPERATURE in config:
-        sens = await sensor.new_sensor(config[CONF_TEMPERATURE])
-        cg.add(var.set_temperature(sens))
-    if CONF_LEVEL in config:
-        sens = await sensor.new_sensor(config[CONF_LEVEL])
-        cg.add(var.set_level(sens))
-    if CONF_DISTANCE in config:
-        sens = await sensor.new_sensor(config[CONF_DISTANCE])
-        cg.add(var.set_distance(sens))
-    if CONF_BATTERY_LEVEL in config:
-        sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL])
-        cg.add(var.set_battery_level(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_TEMPERATURE, var.set_temperature)
+    await sensors(CONF_LEVEL, var.set_level)
+    await sensors(CONF_DISTANCE, var.set_distance)
+    await sensors(CONF_BATTERY_LEVEL, var.set_battery_level)

esphome/components/network/__init__.py

@@ -9,9 +9,11 @@ from esphome.components.zephyr import zephyr_add_prj_conf
 import esphome.config_validation as cv
 from esphome.const import (
     CONF_ENABLE_IPV6,
+    CONF_ETHERNET,
     CONF_ID,
     CONF_MIN_IPV6_ADDR_COUNT,
     CONF_PRIORITY,
+    CONF_WIFI,
 )
 from esphome.core import CORE, CoroPriority, coroutine_with_priority
 import esphome.final_validate as fv
@@ -28,6 +30,16 @@ KEY_HIGH_PERFORMANCE_NETWORKING = "high_performance_networking"
 CONF_ENABLE_HIGH_PERFORMANCE = "enable_high_performance"
 CONF_TCP_SEND_BUFFER = "tcp_send_buffer"
 
+# TCP receive window of the optimized lwip tier (PSRAM not guaranteed)
+TCP_WND_OPTIMIZED = 65534
+# Ethernet drivers keep every queued frame in internal RAM, so ethernet-only builds
+# get a window sized for a LAN round trip and the stock lwip input mailbox. Dual
+# wifi + ethernet builds keep wifi's sizes; the ethernet component moves their
+# received frames to PSRAM instead.
+TCP_WND_ETHERNET = 16384
+TCPIP_RECVMBOX_ETHERNET = 32
+TCPIP_RECVMBOX_OPTIMIZED = 64
+
 # lwIP queues at most this many unsent/unacked bytes per TCP socket; the
 # stock ESP-IDF default (5744 bytes) stalls bursty senders like a Bluetooth
 # proxy streaming GATT notifications. Bounds follow the lwIP guidance for the
@@ -166,6 +178,8 @@ def require_high_performance_networking() -> None:
     Configuration is PSRAM-aware:
     - With PSRAM guaranteed: Aggressive settings (512 RX buffers, 512KB TCP windows)
     - Without PSRAM: Conservative optimized settings (64 buffers, 65KB TCP windows)
+    - Ethernet only: 16KB TCP windows regardless of PSRAM, because ESP-IDF
+      ethernet drivers keep received frames in internal RAM
 
     Example:
         from esphome.components import network
@@ -417,8 +431,11 @@ async def to_code(config: ConfigType) -> None:
     if CORE.is_esp32 and should_enable:
         # Check if PSRAM is guaranteed (set by psram component during final validation)
         psram_guaranteed = psram_is_guaranteed()
+        # ESP-IDF ethernet drivers malloc() received frames into internal RAM, so
+        # lwip is never sized for PSRAM on an ethernet-only build.
+        ethernet_only = CONF_ETHERNET in CORE.config and CONF_WIFI not in CORE.config
 
-        if psram_guaranteed:
+        if psram_guaranteed and not ethernet_only:
             _LOGGER.info(
                 "Applying high-performance lwip settings (PSRAM guaranteed): 512KB TCP windows, 512 mailbox sizes"
             )
@@ -451,15 +468,25 @@ async def to_code(config: ConfigType) -> None:
             add_idf_sdkconfig_option("CONFIG_LWIP_TCP_OVERSIZE_MSS", True)
             add_idf_sdkconfig_option("CONFIG_LWIP_TCP_QUEUE_OOSEQ", True)
         else:
+            # Every queued byte is internal RAM on ethernet and a LAN round trip needs
+            # little window; wifi keeps the larger sizes.
+            if ethernet_only:
+                tcp_window, tcpip_mailbox = TCP_WND_ETHERNET, TCPIP_RECVMBOX_ETHERNET
+            else:
+                tcp_window, tcpip_mailbox = TCP_WND_OPTIMIZED, TCPIP_RECVMBOX_OPTIMIZED
             _LOGGER.info(
-                "Applying optimized lwip settings: 65KB TCP windows, 64 mailbox sizes"
+                "Applying optimized lwip settings: %dKB TCP windows, %d entry input mailbox",
+                tcp_window // 1000,
+                tcpip_mailbox,
             )
             # PSRAM not guaranteed - use more conservative, but still optimized settings
             # Based on https://github.com/espressif/esp-idf/blob/release/v5.4/examples/wifi/iperf/sdkconfig.defaults.esp32
+            # The send buffer stays at 65534 on ethernet too: it only fills under outbound
+            # load, and bursty senders such as a Bluetooth proxy stall on the IDF default.
             add_idf_sdkconfig_option("CONFIG_LWIP_TCP_SND_BUF_DEFAULT", 65534)
-            add_idf_sdkconfig_option("CONFIG_LWIP_TCP_WND_DEFAULT", 65534)
+            add_idf_sdkconfig_option("CONFIG_LWIP_TCP_WND_DEFAULT", tcp_window)
             add_idf_sdkconfig_option("CONFIG_LWIP_TCP_RECVMBOX_SIZE", 64)
-            add_idf_sdkconfig_option("CONFIG_LWIP_TCPIP_RECVMBOX_SIZE", 64)
+            add_idf_sdkconfig_option("CONFIG_LWIP_TCPIP_RECVMBOX_SIZE", tcpip_mailbox)
 
     # After the high performance block so an explicit size wins over the
     # bundle's 65534 (last write wins in the sdkconfig store).

esphome/components/nrf52/__init__.py

@@ -8,6 +8,7 @@ import shutil
 import subprocess
 
 from esphome import pins
+from esphome.build_helpers import pch
 import esphome.codegen as cg
 from esphome.components.zephyr import (
     add_extra_script,
@@ -74,6 +75,7 @@ from .framework import (
     get_build_env,
     get_build_paths,
     setup_platformio_python_env,
+    toolchain_tool,
 )
 
 # force import gpio to register pin schema
@@ -175,6 +177,8 @@ def _detect_bootloader(config: ConfigType) -> ConfigType:
     """Detect the bootloader for the given board."""
     config = config.copy()
     bootloaders: list[str] = []
+    if CONF_BOARD not in config:
+        raise cv.Invalid("'board' is a required option for [nrf52].")
     board = config[CONF_BOARD]
 
     if board in BOARDS_ZEPHYR and KEY_BOOTLOADER in BOARDS_ZEPHYR[board]:
@@ -249,7 +253,7 @@ CONFIG_SCHEMA = cv.All(
             ): cv.Schema(
                 {
                     cv.Optional(CONF_VERSION): cv.string_strict,
-                    cv.Optional(CONF_LIBC_NANO, default=True): cv.boolean,
+                    cv.Optional(CONF_LIBC_NANO): cv.boolean,
                     cv.Optional(
                         CONF_ADVANCED, default={}, visibility=cv.Visibility.YAML_ONLY
                     ): cv.Schema(
@@ -295,7 +299,7 @@ def _final_validate(config):
     conf = config[CONF_FRAMEWORK]
     advanced = conf[CONF_ADVANCED]
 
-    if conf[CONF_LIBC_NANO] and "logger" in CORE.loaded_integrations:
+    if conf.get(CONF_LIBC_NANO, False) and "logger" in CORE.loaded_integrations:
         _LOGGER.warning(
             "Logger is enabled with newlib-nano (libc_nano: true). Some format specifiers "
             "such as %%zu are not supported and will print incorrectly. "
@@ -401,7 +405,10 @@ async def to_code(config: ConfigType) -> None:
         cg.add_define("USE_OTA_ROLLBACK")
     zephyr_add_prj_conf("NEWLIB_LIBC", True)
     zephyr_add_prj_conf("NEWLIB_LIBC_FLOAT_PRINTF", True)
-    zephyr_add_prj_conf("NEWLIB_LIBC_NANO", conf[CONF_LIBC_NANO])
+    zephyr_add_prj_conf(
+        "NEWLIB_LIBC_NANO",
+        conf.get(CONF_LIBC_NANO, "logger" not in CORE.loaded_integrations),
+    )
     # c++ support
     if framework_ver < cv.Version(2, 9, 2):
         zephyr_add_prj_conf("CPLUSPLUS", True)
@@ -412,9 +419,6 @@ async def to_code(config: ConfigType) -> None:
     # watchdog
     zephyr_add_prj_conf("WATCHDOG", True)
     zephyr_add_prj_conf("WDT_DISABLE_AT_BOOT", False)
-    # disable console
-    zephyr_add_prj_conf("UART_CONSOLE", False)
-    zephyr_add_prj_conf("CONSOLE", False, False)
     # use NFC pins as GPIO
     if framework_ver < cv.Version(2, 9, 2):
         zephyr_add_prj_conf("NFCT_PINS_AS_GPIOS", True)
@@ -428,9 +432,17 @@ async def to_code(config: ConfigType) -> None:
         )
     zephyr_add_prj_conf("REBOOT", True)
 
-    # some boards enable USB by default.
+    # some boards enable USB and UART by default.
     # disable it to prevent extra current consumption.
     zephyr_add_prj_conf("USB_DEVICE_STACK", False, False)
+    zephyr_add_prj_conf("SERIAL", False, False)
+
+    # disable stuff to make image smaller by default
+    zephyr_add_prj_conf("NCS_BOOT_BANNER", False, False)
+    zephyr_add_prj_conf("BOOT_BANNER", False, False)
+    zephyr_add_prj_conf("PRINTK", False, False)
+    zephyr_add_prj_conf("CONSOLE", False, False)
+    zephyr_add_prj_conf("UART_CONSOLE", False)
 
 
 @coroutine_with_priority(CoroPriority.DIAGNOSTICS)
@@ -445,6 +457,7 @@ async def _dfu_to_code(dfu_config):
     zephyr_add_prj_conf("USB_DEVICE_STACK", True)
     zephyr_add_prj_conf("USB_CDC_ACM", True)
     zephyr_add_prj_conf("CDC_ACM_DTE_RATE_CALLBACK_SUPPORT", True)
+    zephyr_add_prj_conf("SERIAL", True)
     await cg.register_component(var, dfu_config)
 
 
@@ -770,7 +783,72 @@ def process_stacktrace(config: ConfigType, line: str, backtrace_state: bool) ->
     return False
 
 
-def _generate_cmake_lists() -> bool:
+# GCC only loads a precompiled header ahead of every other forced header, and
+# Zephyr forces two with -imacros. They hold macros only, so the C++ sources
+# of the app get them through the precompiled header.
+_PCH_CMAKE_LINES = [
+    "",
+    "# ESPHome precompiled header",
+    "get_property(esphome_options TARGET zephyr_interface",
+    "    PROPERTY INTERFACE_COMPILE_OPTIONS)",
+    "set(esphome_kept_options)",
+    "set(esphome_pch_headers)",
+    "foreach(option IN LISTS esphome_options)",
+    '  if(option MATCHES "imacros> (.+)$")',
+    '    list(APPEND esphome_pch_headers "${CMAKE_MATCH_1}")',
+    "    list(APPEND esphome_kept_options",
+    '        "$<$<NOT:$<AND:$<COMPILE_LANGUAGE:CXX>,$<STREQUAL:$<TARGET_PROPERTY:NAME>,app>>>:${option}>")',
+    "  else()",
+    '    list(APPEND esphome_kept_options "${option}")',
+    "  endif()",
+    "endforeach()",
+    "if(NOT esphome_pch_headers)",
+    '  message(FATAL_ERROR "ESPHome: the headers Zephyr forces were not found, so "',
+    '      "the precompiled header would not load (set ESPHOME_PCH_ENABLE=0)")',
+    "endif()",
+    "set_property(TARGET zephyr_interface",
+    '    PROPERTY INTERFACE_COMPILE_OPTIONS "${esphome_kept_options}")',
+    *(
+        f'list(APPEND esphome_pch_headers "${{CMAKE_CURRENT_LIST_DIR}}/../src/{header}")'
+        for header in pch.PCH_DEFAULT_HEADERS
+    ),
+    'list(TRANSFORM esphome_pch_headers REPLACE "(.+)" "$<$<COMPILE_LANGUAGE:CXX>:\\\\1>")',
+    "target_precompile_headers(app PRIVATE ${esphome_pch_headers})",
+]
+# Where CMake puts the .gch of the app, below its binary dir
+_PCH_SUM_PATH = "CMakeFiles/app.dir/cmake_pch.hxx.gch.sum"
+
+
+def _write_pch_checksum(build_dir: Path, source_dir: Path) -> None:
+    """Write the checksum ccache reads in place of the .gch. The app binary
+    dir only exists after the first configure; sysbuild nests it."""
+    app_dir = build_dir / "zephyr"
+    if not (app_dir / "CMakeCache.txt").is_file():
+        app_dir = build_dir
+    if not (app_dir / "CMakeCache.txt").is_file():
+        return
+    checksum = pch.pch_checksum(
+        CORE.relative_src_path(),
+        pch.PCH_DEFAULT_HEADERS,
+        (
+            str(CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]),
+            zephyr_data()[KEY_BOARD],
+            # What the Zephyr configuration is generated from
+            *(
+                path.read_text(encoding="utf-8")
+                for path in sorted(source_dir.iterdir())
+                if path.suffix in (".conf", ".overlay")
+            ),
+        ),
+    )
+    write_file_if_changed(app_dir / _PCH_SUM_PATH, checksum + "\n")
+
+
+def _pch_usable() -> bool:
+    return pch.pch_usable((toolchain_tool("g++"),))
+
+
+def _generate_cmake_lists(pch_on: bool) -> bool:
     """Write the project CMakeLists.txt, returning True if it changed."""
     compile_flags = get_project_compile_flags()
     link_flags = get_project_link_flags()
@@ -813,6 +891,9 @@ def _generate_cmake_lists() -> bool:
             ")",
         ]
 
+    if pch_on:
+        lines += _PCH_CMAKE_LINES
+
     if link_flags:
         lines += [
             "",
@@ -869,7 +950,8 @@ def run_compile(args, config: ConfigType) -> bool:
     paths = get_build_paths()
     env = get_build_env()
 
-    cmake_lists_changed = _generate_cmake_lists()
+    pch_on = _pch_usable()
+    cmake_lists_changed = _generate_cmake_lists(pch_on)
 
     board = zephyr_data()[KEY_BOARD]
     build_dir = CORE.relative_pioenvs_path(CORE.name)
@@ -886,6 +968,12 @@ def run_compile(args, config: ConfigType) -> bool:
         _LOGGER.info("Build inputs changed, cleaning %s", build_dir)
         rmtree(build_dir)
 
+    if pch_on:
+        pch.log_pch_in_use()
+        # Zephyr turns ccache on by itself when it is installed
+        env.update(pch.ccache_pch_env())
+        _write_pch_checksum(build_dir, source_dir)
+
     west_cmd = _west_build_command(
         paths["python_executable"], board, build_dir, source_dir
     )

esphome/components/pzemdc/sensor.py

@@ -85,19 +85,8 @@ async def to_code(config: ConfigType) -> None:
     await cg.register_component(var, config)
     await modbus.register_modbus_client_device(var, config)
 
-    if CONF_VOLTAGE in config:
-        conf = config[CONF_VOLTAGE]
-        sens = await sensor.new_sensor(conf)
-        cg.add(var.set_voltage_sensor(sens))
-    if CONF_CURRENT in config:
-        conf = config[CONF_CURRENT]
-        sens = await sensor.new_sensor(conf)
-        cg.add(var.set_current_sensor(sens))
-    if CONF_POWER in config:
-        conf = config[CONF_POWER]
-        sens = await sensor.new_sensor(conf)
-        cg.add(var.set_power_sensor(sens))
-    if CONF_ENERGY in config:
-        conf = config[CONF_ENERGY]
-        sens = await sensor.new_sensor(conf)
-        cg.add(var.set_energy_sensor(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_VOLTAGE, var.set_voltage_sensor)
+    await sensors(CONF_CURRENT, var.set_current_sensor)
+    await sensors(CONF_POWER, var.set_power_sensor)
+    await sensors(CONF_ENERGY, var.set_energy_sensor)

esphome/components/qmc5883l/sensor.py

@@ -150,21 +150,12 @@ async def to_code(config: ConfigType) -> None:
     cg.add(var.set_oversampling(config[CONF_OVERSAMPLING]))
     cg.add(var.set_datarate(config[CONF_DATA_RATE]))
     cg.add(var.set_range(config[CONF_RANGE]))
-    if CONF_FIELD_STRENGTH_X in config:
-        sens = await sensor.new_sensor(config[CONF_FIELD_STRENGTH_X])
-        cg.add(var.set_x_sensor(sens))
-    if CONF_FIELD_STRENGTH_Y in config:
-        sens = await sensor.new_sensor(config[CONF_FIELD_STRENGTH_Y])
-        cg.add(var.set_y_sensor(sens))
-    if CONF_FIELD_STRENGTH_Z in config:
-        sens = await sensor.new_sensor(config[CONF_FIELD_STRENGTH_Z])
-        cg.add(var.set_z_sensor(sens))
-    if CONF_HEADING in config:
-        sens = await sensor.new_sensor(config[CONF_HEADING])
-        cg.add(var.set_heading_sensor(sens))
-    if CONF_TEMPERATURE in config:
-        sens = await sensor.new_sensor(config[CONF_TEMPERATURE])
-        cg.add(var.set_temperature_sensor(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_FIELD_STRENGTH_X, var.set_x_sensor)
+    await sensors(CONF_FIELD_STRENGTH_Y, var.set_y_sensor)
+    await sensors(CONF_FIELD_STRENGTH_Z, var.set_z_sensor)
+    await sensors(CONF_HEADING, var.set_heading_sensor)
+    await sensors(CONF_TEMPERATURE, var.set_temperature_sensor)
     if CONF_DRDY_PIN in config:
         pin = await cg.gpio_pin_expression(config[CONF_DRDY_PIN])
         cg.add(var.set_drdy_pin(pin))

esphome/components/rp2/__init__.py

@@ -6,7 +6,6 @@ from string import ascii_letters, digits
 import subprocess
 from typing import Any
 
-from esphome.build_helpers.pch import pch_extra_scripts
 import esphome.codegen as cg
 import esphome.config_validation as cv
 from esphome.const import (
@@ -34,7 +33,7 @@ from esphome.core import (
 )
 from esphome.core.config import BOARD_MAX_LENGTH
 from esphome.helpers import copy_file_if_changed, read_file, write_file_if_changed
-from esphome.platformio.toolchain import copy_ccache_script, copy_pch_script
+from esphome.platformio.toolchain import copy_ccache_script
 from esphome.storage_json import StorageJSON
 from esphome.types import ConfigType
 
@@ -198,20 +197,21 @@ def _parse_platform_version(value: Any) -> str:
 
 # The default/recommended arduino framework version
 #  - https://github.com/earlephilhower/arduino-pico/releases
-RECOMMENDED_ARDUINO_FRAMEWORK_VERSION = cv.Version(6, 1, 0)
+RECOMMENDED_ARDUINO_FRAMEWORK_VERSION = cv.Version(6, 1, 1)
 
 # The raspberrypi platform version to use for arduino frameworks
 #  - https://github.com/maxgerhardt/platform-raspberrypi/tags
-# develop-branch commit carrying the arduino-pico 6.1.0 update and the board
-# JSON files it adds; replace with a release tag when one is cut
+# develop-branch commit carrying the arduino-pico 6.1.0 update; it predates
+# 6.1.1, so the boards that release adds have no board JSON on it yet; replace
+# with a release tag when one is cut
 RECOMMENDED_ARDUINO_PLATFORM_VERSION = "5d4561a05e3b212660ac6fdd3fbfb328d1988aa1"
 
 
 def _arduino_check_versions(value: ConfigType) -> ConfigType:
     value = value.copy()
     lookups = {
-        "dev": (cv.Version(6, 1, 0), "https://github.com/earlephilhower/arduino-pico"),
-        "latest": (cv.Version(6, 1, 0), None),
+        "dev": (cv.Version(6, 1, 1), "https://github.com/earlephilhower/arduino-pico"),
+        "latest": (cv.Version(6, 1, 1), None),
         "recommended": (RECOMMENDED_ARDUINO_FRAMEWORK_VERSION, None),
     }
 
@@ -341,10 +341,7 @@ async def to_code(config: ConfigType) -> None:
     cg.add_define("ESPHOME_VARIANT", VARIANT_FRIENDLY[variant])
     cg.add_define(ThreadModel.SINGLE)
 
-    cg.add_platformio_option(
-        "extra_scripts",
-        ["pre:ccache.py", *pch_extra_scripts(), "post:post_build.py"],
-    )
+    cg.add_platformio_option("extra_scripts", ["pre:ccache.py", "post:post_build.py"])
 
     conf = config[CONF_FRAMEWORK]
     cg.add_platformio_option("framework", "arduino")
@@ -648,7 +645,6 @@ def copy_files() -> None:
         CORE.relative_build_path("inject_lwip_include.py"),
     )
     copy_ccache_script()
-    copy_pch_script()
     _generate_lwipopts_h()
     if generate_pio_files():
         path = CORE.relative_src_path("esphome.h")

esphome/components/rp2/boards.py

@@ -1442,6 +1442,19 @@ RP2_BOARD_PINS = {
         "SS": 5,
         "TX": 0,
     },
+    "waveshare_rp2350_core": {
+        "LED": 39,
+        "MISO": 16,
+        "MOSI": 19,
+        "RX": 1,
+        "SCK": 18,
+        "SCL": 9,
+        "SCL1": 11,
+        "SDA": 8,
+        "SDA1": 10,
+        "SS": 17,
+        "TX": 0,
+    },
     "waveshare_rp2350_lcd_0_96": {
         "MISO": 16,
         "MOSI": 19,
@@ -1479,6 +1492,18 @@ RP2_BOARD_PINS = {
         "SS": 17,
         "TX": 0,
     },
+    "waveshare_rp2350_poe_eth": {
+        "MISO": 16,
+        "MOSI": 19,
+        "RX": 1,
+        "SCK": 18,
+        "SCL": 5,
+        "SCL1": 27,
+        "SDA": 4,
+        "SDA1": 26,
+        "SS": 17,
+        "TX": 0,
+    },
     "waveshare_rp2350_zero": {
         "MISO": 4,
         "MOSI": 3,
@@ -2286,6 +2311,12 @@ BOARDS = {
         "mcu": "rp2040",
         "max_pin": 29,
     },
+    "waveshare_rp2350_core": {
+        "name": "Waveshare RP2350 Core",
+        "mcu": "rp2350",
+        "max_pin": 47,
+        "die": "B",
+    },
     "waveshare_rp2350_lcd_0_96": {
         "name": "Waveshare RP2350 LCD 0.96",
         "mcu": "rp2350",
@@ -2304,6 +2335,12 @@ BOARDS = {
         "max_pin": 29,
         "die": "A",
     },
+    "waveshare_rp2350_poe_eth": {
+        "name": "Waveshare RP2350-POE-ETH",
+        "mcu": "rp2350",
+        "max_pin": 29,
+        "die": "A",
+    },
     "waveshare_rp2350_zero": {
         "name": "Waveshare RP2350 Zero",
         "mcu": "rp2350",

esphome/components/rp2/gpio.cpp

@@ -9,6 +9,8 @@ namespace rp2 {
 static const char *const TAG = "rp2";
 
 static int flags_to_mode(gpio::Flags flags, uint8_t pin) {
+  flags =
+      flags & (gpio::FLAG_INPUT | gpio::FLAG_OUTPUT | gpio::FLAG_OPEN_DRAIN | gpio::FLAG_PULLUP | gpio::FLAG_PULLDOWN);
   if (flags == gpio::FLAG_INPUT) {  // NOLINT(bugprone-branch-clone)
     return INPUT;
   } else if (flags == gpio::FLAG_OUTPUT) {

esphome/components/rp2040_ble/btstack_memory.cpp

@@ -20,7 +20,7 @@
 namespace esphome::rp2040_ble {
 namespace {
 
-// Pinned against arduino-pico 6.1.0's prebuilt archives: a framework bump (or
+// Pinned against arduino-pico 6.1.1's prebuilt archives: a framework bump (or
 // a changed ENABLE_* macro) shifting the struct layout must fail the build
 // here, not overrun the pool blocks at runtime. Sizes differ per core
 // architecture (measured from each archive's own storage symbols). GCC only:

esphome/components/sm300d2/sensor.py

@@ -99,24 +99,11 @@ async def to_code(config: ConfigType) -> None:
     await cg.register_component(var, config)
     await uart.register_uart_device(var, config)
 
-    if CONF_CO2 in config:
-        sens = await sensor.new_sensor(config[CONF_CO2])
-        cg.add(var.set_co2_sensor(sens))
-    if CONF_FORMALDEHYDE in config:
-        sens = await sensor.new_sensor(config[CONF_FORMALDEHYDE])
-        cg.add(var.set_formaldehyde_sensor(sens))
-    if CONF_TVOC in config:
-        sens = await sensor.new_sensor(config[CONF_TVOC])
-        cg.add(var.set_tvoc_sensor(sens))
-    if CONF_PM_2_5 in config:
-        sens = await sensor.new_sensor(config[CONF_PM_2_5])
-        cg.add(var.set_pm_2_5_sensor(sens))
-    if CONF_PM_10_0 in config:
-        sens = await sensor.new_sensor(config[CONF_PM_10_0])
-        cg.add(var.set_pm_10_0_sensor(sens))
-    if CONF_TEMPERATURE in config:
-        sens = await sensor.new_sensor(config[CONF_TEMPERATURE])
-        cg.add(var.set_temperature_sensor(sens))
-    if CONF_HUMIDITY in config:
-        sens = await sensor.new_sensor(config[CONF_HUMIDITY])
-        cg.add(var.set_humidity_sensor(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_CO2, var.set_co2_sensor)
+    await sensors(CONF_FORMALDEHYDE, var.set_formaldehyde_sensor)
+    await sensors(CONF_TVOC, var.set_tvoc_sensor)
+    await sensors(CONF_PM_2_5, var.set_pm_2_5_sensor)
+    await sensors(CONF_PM_10_0, var.set_pm_10_0_sensor)
+    await sensors(CONF_TEMPERATURE, var.set_temperature_sensor)
+    await sensors(CONF_HUMIDITY, var.set_humidity_sensor)

esphome/components/speaker/media_player/speaker_media_player.cpp

@@ -154,6 +154,7 @@ void SpeakerMediaPlayer::watch_media_commands_() {
           // Ensure the loaded next item doesn't start playing, clear the queue, start the file, and unpause
           this->cancel_timeout("next_ann");
           this->announcement_playlist_.clear();
+          this->announcement_item_failed_ = false;
           if (media_command.file.has_value()) {
             this->announcement_pipeline_->start_file(playlist_item.file.value());
           } else if (media_command.url.has_value()) {
@@ -167,6 +168,7 @@ void SpeakerMediaPlayer::watch_media_commands_() {
           // Ensure the loaded next item doesn't start playing, clear the queue, start the file, and unpause
           this->cancel_timeout("next_media");
           this->media_playlist_.clear();
+          this->media_item_failed_ = false;
           if (this->is_paused_) {
             // If paused, stop the media pipeline and unpause it after confirming its stopped. This avoids playing a
             // short segment of the paused file before starting the new one.
@@ -236,6 +238,7 @@ void SpeakerMediaPlayer::watch_media_commands_() {
             if (this->announcement_pipeline_ != nullptr) {
               this->cancel_timeout("next_ann");
               this->announcement_playlist_.clear();
+              this->announcement_item_failed_ = false;
               this->announcement_pipeline_->stop();
               this->unpause_announcement_remaining_ = 3;
               this->set_interval("unpause_ann", 50, [this]() {
@@ -251,6 +254,7 @@ void SpeakerMediaPlayer::watch_media_commands_() {
             if (this->media_pipeline_ != nullptr) {
               this->cancel_timeout("next_media");
               this->media_playlist_.clear();
+              this->media_item_failed_ = false;
               this->stop_and_unpause_media_();
             }
           }
@@ -351,8 +355,10 @@ void SpeakerMediaPlayer::loop() {
 
   if (this->media_pipeline_state_ == AudioPipelineState::ERROR_READING) {
     ESP_LOGE(TAG, "The media pipeline's file reader encountered an error.");
+    this->media_item_failed_ = true;
   } else if (this->media_pipeline_state_ == AudioPipelineState::ERROR_DECODING) {
     ESP_LOGE(TAG, "The media pipeline's audio decoder encountered an error.");
+    this->media_item_failed_ = true;
   }
 
   AudioPipelineState old_announcement_pipeline_state = this->announcement_pipeline_state_;
@@ -362,8 +368,10 @@ void SpeakerMediaPlayer::loop() {
 
   if (this->announcement_pipeline_state_ == AudioPipelineState::ERROR_READING) {
     ESP_LOGE(TAG, "The announcement pipeline's file reader encountered an error.");
+    this->announcement_item_failed_ = true;
   } else if (this->announcement_pipeline_state_ == AudioPipelineState::ERROR_DECODING) {
     ESP_LOGE(TAG, "The announcement pipeline's audio decoder encountered an error.");
+    this->announcement_item_failed_ = true;
   }
 
   if (this->announcement_pipeline_state_ != AudioPipelineState::STOPPED) {
@@ -371,7 +379,12 @@ void SpeakerMediaPlayer::loop() {
   } else {
     if (!this->announcement_playlist_.empty()) {
       uint32_t timeout_ms = 0;
-      if (old_announcement_pipeline_state == AudioPipelineState::PLAYING) {
+      if (this->announcement_item_failed_) {
+        // Drop the item that failed, even with repeat enabled; otherwise it is restarted as soon as the pipeline
+        // stops, which after an error is usually on the next loop
+        this->announcement_item_failed_ = false;
+        this->announcement_playlist_.pop_front();
+      } else if (old_announcement_pipeline_state == AudioPipelineState::PLAYING) {
         // Finished the current announcement file
         if (!this->announcement_repeat_one_) {
           //  Pop item off the playlist if repeat is disabled
@@ -399,6 +412,8 @@ void SpeakerMediaPlayer::loop() {
         }
       }
     } else {
+      // Nothing left to retry
+      this->announcement_item_failed_ = false;
       if (this->is_paused_) {
 #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF
         if (this->state != media_player::MEDIA_PLAYER_STATE_OFF) {
@@ -412,7 +427,12 @@ void SpeakerMediaPlayer::loop() {
       } else if (this->media_pipeline_state_ == AudioPipelineState::STOPPED) {
         if (!media_playlist_.empty()) {
           uint32_t timeout_ms = 0;
-          if (old_media_pipeline_state == AudioPipelineState::PLAYING) {
+          if (this->media_item_failed_) {
+            // Drop the item that failed, even with repeat enabled; otherwise it is restarted as soon as the pipeline
+            // stops. The flag also covers an error that happened while an announcement was playing.
+            this->media_item_failed_ = false;
+            this->media_playlist_.pop_front();
+          } else if (old_media_pipeline_state == AudioPipelineState::PLAYING) {
             // Finished the current media file
             if (!this->media_repeat_one_) {
               // Pop item off the playlist if repeat is disabled
@@ -439,6 +459,8 @@ void SpeakerMediaPlayer::loop() {
             }
           }
         } else {
+          // Nothing left to retry
+          this->media_item_failed_ = false;
 #ifdef USE_SPEAKER_MEDIA_PLAYER_ON_OFF
           if (this->state != media_player::MEDIA_PLAYER_STATE_OFF) {
             this->state = media_player::MEDIA_PLAYER_STATE_IDLE;

esphome/components/speaker/media_player/speaker_media_player.h

@@ -125,11 +125,15 @@ class SpeakerMediaPlayer final : public Component,
   optional<media_player::MediaPlayerSupportedFormat> media_format_;
   AudioPipelineState media_pipeline_state_{AudioPipelineState::STOPPED};
   bool media_repeat_one_{false};
+  // Set when the media pipeline reports an error, consumed when it stops, so the failed item is dropped
+  bool media_item_failed_{false};
   uint32_t media_playlist_delay_ms_{0};
 
   optional<media_player::MediaPlayerSupportedFormat> announcement_format_;
   AudioPipelineState announcement_pipeline_state_{AudioPipelineState::STOPPED};
   bool announcement_repeat_one_{false};
+  // Set when the announcement pipeline reports an error, consumed when it stops, so the failed item is dropped
+  bool announcement_item_failed_{false};
   uint32_t announcement_playlist_delay_ms_{0};
 
   QueueHandle_t media_control_command_queue_;

esphome/components/speaker_source/speaker_source_media_player.cpp

@@ -196,10 +196,19 @@ size_t SpeakerSourceMediaPlayer::handle_media_output_(uint8_t pipeline, media_so
       vTaskDelay(pdMS_TO_TICKS(timeout_ms));
       return 0;
     }
+    // Reserve frames before the speaker can play them and report their progress,
+    // then release the portion it did not accept.
+    const uint32_t reserved_frames = stream_info.bytes_to_frames(length);
+    ps.pending_frames.fetch_add(reserved_frames, std::memory_order_relaxed);
     size_t bytes_written = ps.speaker->play(data, length, pdMS_TO_TICKS(timeout_ms));
-    if (bytes_written > 0) {
-      // Track frames sent to speaker for this source
-      ps.pending_frames.fetch_add(stream_info.bytes_to_frames(bytes_written), std::memory_order_relaxed);
+    const uint32_t unused_frames = reserved_frames - stream_info.bytes_to_frames(bytes_written);
+    if (unused_frames > 0) {
+      // The main loop may reset pending_frames while play() is blocked.
+      // Clamp the release so that a concurrent reset cannot cause underflow.
+      uint32_t pending = ps.pending_frames.load(std::memory_order_relaxed);
+      while (!ps.pending_frames.compare_exchange_weak(pending, pending - std::min(pending, unused_frames),
+                                                      std::memory_order_relaxed)) {
+      }
     }
     return bytes_written;
   }

esphome/components/split_buffer/split_buffer.cpp

@@ -1,5 +1,8 @@
 #include "split_buffer.h"
 
+#include <algorithm>
+#include <cstring>
+
 #include "esphome/core/helpers.h"
 #include "esphome/core/log.h"
 
@@ -8,15 +11,14 @@ static constexpr const char *const TAG = "split_buffer";
 
 SplitBuffer::~SplitBuffer() { this->free(); }
 
-bool SplitBuffer::init(size_t total_length) {
+bool SplitBuffer::init(size_t total_length, size_t max_buffer_size) {
   this->free();  // Clean up any existing allocation
 
-  if (total_length == 0) {
+  if (total_length == 0 || max_buffer_size == 0) {
     return false;
   }
 
-  this->total_length_ = total_length;
-  size_t current_buffer_size = total_length;
+  size_t current_buffer_size = std::min(total_length, max_buffer_size);
 
   RAMAllocator<uint8_t *> ptr_allocator;
   RAMAllocator<uint8_t> allocator;
@@ -63,6 +65,7 @@ bool SplitBuffer::init(size_t total_length) {
       this->buffers_ = temp_buffers;
       this->buffer_count_ = needed_buffers;
       this->buffer_size_ = current_buffer_size;
+      this->total_length_ = total_length;
       ESP_LOGD(TAG, "Allocated %zu * %zu bytes - %zu bytes", this->buffer_count_, this->buffer_size_,
                this->total_length_);
       return true;
@@ -122,6 +125,34 @@ uint8_t &SplitBuffer::operator[](size_t index) {
   return const_cast<uint8_t &>(static_cast<const SplitBuffer *>(this)->operator[](index));
 }
 
+const uint8_t *SplitBuffer::get_span(size_t index, size_t &length) const {
+  if (index >= this->total_length_) {
+    length = 0;
+    return nullptr;
+  }
+  const size_t offset = index % this->buffer_size_;
+  length = std::min(this->buffer_size_ - offset, this->total_length_ - index);
+  return this->buffers_[index / this->buffer_size_] + offset;
+}
+
+uint8_t *SplitBuffer::get_span(size_t index, size_t &length) {
+  return const_cast<uint8_t *>(static_cast<const SplitBuffer *>(this)->get_span(index, length));
+}
+
+void SplitBuffer::write(size_t index, const uint8_t *data, size_t length) {
+  while (length != 0) {
+    size_t span_length;
+    uint8_t *span = this->get_span(index, span_length);
+    if (span == nullptr)
+      return;
+    span_length = std::min(span_length, length);
+    memcpy(span, data, span_length);
+    index += span_length;
+    data += span_length;
+    length -= span_length;
+  }
+}
+
 /**
  * Fill the entire buffer with a single byte value
  * @param value Fill value

esphome/components/split_buffer/split_buffer.h

@@ -16,8 +16,8 @@ class SplitBuffer {
   SplitBuffer() = default;
   ~SplitBuffer();
 
-  // Initialize the buffer with the desired total length
-  bool init(size_t total_length);
+  // Initialize the buffer with the desired total length; no sub-buffer will be larger than `max_buffer_size`
+  bool init(size_t total_length, size_t max_buffer_size = SIZE_MAX);
 
   // Free all allocated buffers
   void free();
@@ -27,6 +27,13 @@ class SplitBuffer {
   const uint8_t &operator[](size_t index) const;
   void fill(uint8_t value) const;
 
+  // Pointer to the byte at `index`; `length` is set to how many bytes are contiguous from there.
+  // Returns nullptr with `length` 0 if `index` is out of range.
+  const uint8_t *get_span(size_t index, size_t &length) const;
+  uint8_t *get_span(size_t index, size_t &length);
+  // Copy `length` bytes from `data` into the buffer starting at `index`; bytes past the end are dropped.
+  void write(size_t index, const uint8_t *data, size_t length);
+
   // Get the total length
   size_t size() const { return this->total_length_; }
 

esphome/components/thermopro_ble/sensor.py

@@ -82,18 +82,9 @@ async def to_code(config: ConfigType) -> None:
 
     cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex))
 
-    if temperature_config := config.get(CONF_TEMPERATURE):
-        sens = await sensor.new_sensor(temperature_config)
-        cg.add(var.set_temperature(sens))
-    if external_temperature_config := config.get(CONF_EXTERNAL_TEMPERATURE):
-        sens = await sensor.new_sensor(external_temperature_config)
-        cg.add(var.set_external_temperature(sens))
-    if humidity_config := config.get(CONF_HUMIDITY):
-        sens = await sensor.new_sensor(humidity_config)
-        cg.add(var.set_humidity(sens))
-    if battery_level_config := config.get(CONF_BATTERY_LEVEL):
-        sens = await sensor.new_sensor(battery_level_config)
-        cg.add(var.set_battery_level(sens))
-    if signal_strength_config := config.get(CONF_SIGNAL_STRENGTH):
-        sens = await sensor.new_sensor(signal_strength_config)
-        cg.add(var.set_signal_strength(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_TEMPERATURE, var.set_temperature)
+    await sensors(CONF_EXTERNAL_TEMPERATURE, var.set_external_temperature)
+    await sensors(CONF_HUMIDITY, var.set_humidity)
+    await sensors(CONF_BATTERY_LEVEL, var.set_battery_level)
+    await sensors(CONF_SIGNAL_STRENGTH, var.set_signal_strength)

esphome/components/tsl2591/sensor.py

@@ -140,30 +140,12 @@ async def to_code(config: ConfigType) -> None:
     await cg.register_component(var, config)
     await i2c.register_i2c_device(var, config)
 
-    if CONF_FULL_SPECTRUM in config:
-        conf = config[CONF_FULL_SPECTRUM]
-        sens = await sensor.new_sensor(conf)
-        cg.add(var.set_full_spectrum_sensor(sens))
-
-    if CONF_INFRARED in config:
-        conf = config[CONF_INFRARED]
-        sens = await sensor.new_sensor(conf)
-        cg.add(var.set_infrared_sensor(sens))
-
-    if CONF_VISIBLE in config:
-        conf = config[CONF_VISIBLE]
-        sens = await sensor.new_sensor(conf)
-        cg.add(var.set_visible_sensor(sens))
-
-    if CONF_CALCULATED_LUX in config:
-        conf = config[CONF_CALCULATED_LUX]
-        sens = await sensor.new_sensor(conf)
-        cg.add(var.set_calculated_lux_sensor(sens))
-
-    if CONF_ACTUAL_GAIN in config:
-        conf = config[CONF_ACTUAL_GAIN]
-        sens = await sensor.new_sensor(conf)
-        cg.add(var.set_actual_gain_sensor(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_FULL_SPECTRUM, var.set_full_spectrum_sensor)
+    await sensors(CONF_INFRARED, var.set_infrared_sensor)
+    await sensors(CONF_VISIBLE, var.set_visible_sensor)
+    await sensors(CONF_CALCULATED_LUX, var.set_calculated_lux_sensor)
+    await sensors(CONF_ACTUAL_GAIN, var.set_actual_gain_sensor)
 
     cg.add(var.set_name(config[CONF_NAME]))
     cg.add(var.set_power_save_mode(config[CONF_POWER_SAVE_MODE]))

esphome/components/wifi/__init__.py

@@ -771,7 +771,7 @@ async def to_code(config):
                 "Applying high-performance WiFi settings (PSRAM guaranteed): 512 RX buffers, 32 TX buffers"
             )
             # PSRAM is guaranteed - use aggressive settings
-            # Higher maximum values are allowed because CONFIG_LWIP_WND_SCALE is set to true in networking component
+            # Higher maximum values are allowed because CONFIG_LWIP_WND_SCALE may be set to true in networking component
             # Based on https://github.com/espressif/esp-adf/issues/297#issuecomment-783811702
 
             # Large dynamic RX buffers (requires PSRAM)

esphome/components/wireguard/binary_sensor.py

@@ -24,12 +24,8 @@ CONFIG_SCHEMA = {
 
 
 async def to_code(config):
-    parent = await cg.get_variable(config[CONF_WIREGUARD_ID])
+    hub = await cg.get_variable(config[CONF_WIREGUARD_ID])
 
-    if status_config := config.get(CONF_STATUS):
-        sens = await binary_sensor.new_binary_sensor(status_config)
-        cg.add(parent.set_status_sensor(sens))
-
-    if enabled_config := config.get(CONF_ENABLED):
-        sens = await binary_sensor.new_binary_sensor(enabled_config)
-        cg.add(parent.set_enabled_sensor(sens))
+    binary_sensors = binary_sensor.sub_binary_sensors(config)
+    await binary_sensors(CONF_STATUS, hub.set_status_sensor)
+    await binary_sensors(CONF_ENABLED, hub.set_enabled_sensor)

esphome/components/wireguard/sensor.py

@@ -19,8 +19,7 @@ CONFIG_SCHEMA = {
 
 
 async def to_code(config):
-    parent = await cg.get_variable(config[CONF_WIREGUARD_ID])
+    hub = await cg.get_variable(config[CONF_WIREGUARD_ID])
 
-    if latest_handshake_config := config.get(CONF_LATEST_HANDSHAKE):
-        sens = await sensor.new_sensor(latest_handshake_config)
-        cg.add(parent.set_handshake_sensor(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_LATEST_HANDSHAKE, hub.set_handshake_sensor)

esphome/components/wireguard/text_sensor.py

@@ -16,8 +16,7 @@ CONFIG_SCHEMA = {
 
 
 async def to_code(config):
-    parent = await cg.get_variable(config[CONF_WIREGUARD_ID])
+    hub = await cg.get_variable(config[CONF_WIREGUARD_ID])
 
-    if address_config := config.get(CONF_ADDRESS):
-        sens = await text_sensor.new_text_sensor(address_config)
-        cg.add(parent.set_address_sensor(sens))
+    text_sensors = text_sensor.sub_text_sensors(config)
+    await text_sensors(CONF_ADDRESS, hub.set_address_sensor)

esphome/components/xiaomi_lywsd03mmc/sensor.py

@@ -68,12 +68,7 @@ async def to_code(config: ConfigType) -> None:
     cg.add(var.set_address(config[CONF_MAC_ADDRESS].as_hex))
     cg.add(var.set_bindkey(config[CONF_BINDKEY]))
 
-    if CONF_TEMPERATURE in config:
-        sens = await sensor.new_sensor(config[CONF_TEMPERATURE])
-        cg.add(var.set_temperature(sens))
-    if CONF_HUMIDITY in config:
-        sens = await sensor.new_sensor(config[CONF_HUMIDITY])
-        cg.add(var.set_humidity(sens))
-    if CONF_BATTERY_LEVEL in config:
-        sens = await sensor.new_sensor(config[CONF_BATTERY_LEVEL])
-        cg.add(var.set_battery_level(sens))
+    sensors = sensor.sub_sensors(config)
+    await sensors(CONF_TEMPERATURE, var.set_temperature)
+    await sensors(CONF_HUMIDITY, var.set_humidity)
+    await sensors(CONF_BATTERY_LEVEL, var.set_battery_level)

esphome/components/zephyr/__init__.py

@@ -210,6 +210,7 @@ def zephyr_add_cdc_acm(config: ConfigType, id: int) -> None:
         zephyr_add_prj_conf("CONFIG_USB_DEVICE_STACK_NEXT", False)
     zephyr_add_prj_conf("USB_DEVICE_STACK", True)
     zephyr_add_prj_conf("USB_CDC_ACM", True)
+    zephyr_add_prj_conf("SERIAL", True)
     # prevent device to go to susspend, without this communication stop working in python
     # there should be a way to solve it
     zephyr_add_prj_conf("USB_DEVICE_REMOTE_WAKEUP", False)

esphome/components/zephyr_mcumgr/ota/__init__.py

@@ -1,3 +1,4 @@
+from esphome import pins
 import esphome.codegen as cg
 from esphome.components.nrf52.boards import BOOTLOADER_CONFIG
 from esphome.components.nrf52.framework import include_west_project
@@ -19,6 +20,10 @@ import esphome.config_validation as cv
 from esphome.const import (
     CONF_HARDWARE_UART,
     CONF_ID,
+    CONF_INVERTED,
+    CONF_NUMBER,
+    CONF_PIN,
+    CONF_STATUS,
     KEY_CORE,
     KEY_FRAMEWORK_VERSION,
     Framework,
@@ -67,6 +72,11 @@ CONFIG_SCHEMA = cv.All(
                     ): cv.one_of(*UARTS, upper=True),
                 }
             ),
+            cv.Optional(CONF_STATUS): cv.Schema(
+                {
+                    cv.Required(CONF_PIN): pins.gpio_output_pin_schema,
+                }
+            ),
         }
     )
     .extend(BASE_OTA_SCHEMA)
@@ -147,6 +157,7 @@ async def to_code(config: ConfigType) -> None:
         zephyr_add_prj_conf("NCS_SAMPLE_MCUMGR_BT_OTA_DFU_SPEEDUP", True)
         request_ble_l2cap_mtu(498)  # matches NCS_SAMPLE_MCUMGR_BT_OTA_DFU_SPEEDUP
     if CONF_HARDWARE_UART in transport:
+        zephyr_add_prj_conf("SERIAL", True)
         uart = UARTS[transport[CONF_HARDWARE_UART]]
         uart_name = uart[0]
         cdc_id = uart[1]
@@ -164,6 +175,32 @@ async def to_code(config: ConfigType) -> None:
                 }};
                 """
         )
+    if CONF_STATUS in config:
+        pin_conf = config[CONF_STATUS][CONF_PIN]
+        pin_num = pin_conf[CONF_NUMBER]
+        port = pin_num // 32
+        pin_in_port = pin_num % 32
+        active_flag = (
+            "GPIO_ACTIVE_LOW" if pin_conf[CONF_INVERTED] else "GPIO_ACTIVE_HIGH"
+        )
+        zephyr_add_prj_conf("MCUBOOT_INDICATION_LED", True, image="mcuboot")
+        zephyr_add_overlay(
+            f"""
+                #include <dt-bindings/gpio/gpio.h>
+                / {{
+                    mcuboot_leds {{
+                        compatible = "gpio-leds";
+                        mcuboot_led0: mcuboot_led0 {{
+                            gpios = <&gpio{port} {pin_in_port} {active_flag}>;
+                        }};
+                    }};
+                    aliases {{
+                        mcuboot-led0 = &mcuboot_led0;
+                    }};
+                }};
+                """,
+            image="mcuboot",
+        )
     framework_ver = CORE.data[KEY_CORE][KEY_FRAMEWORK_VERSION]
     if framework_ver >= cv.Version(2, 9, 2):
         zephyr_data()[KEY_SYSBUILD] = True

esphome/core/defines.h

@@ -546,7 +546,7 @@
 // rp2/__init__.py codegen also defines USE_RP2040 as a back-compat alias
 // for external custom components that may still test for it.
 #ifdef USE_RP2
-#define USE_ARDUINO_VERSION_CODE VERSION_CODE(6, 1, 0)
+#define USE_ARDUINO_VERSION_CODE VERSION_CODE(6, 1, 1)
 #define USE_RP2_CRASH_HANDLER
 #define USE_HTTP_REQUEST_RESPONSE
 #define USE_I2C

esphome/core/pch_prefix.h

@@ -1,8 +1,5 @@
 #pragma once
-// Curated precompiled-header prefix for backends that force-include it via
-// build_src_flags (the pch script folds it into the .gch). Guarded because
-// build_src_flags also reaches C and assembly src edges.
-#ifdef __cplusplus
+// The core headers the precompiled header holds
+#include "esphome/core/defines.h"
 #include "esphome/core/application.h"
 #include "esphome/core/automation.h"
-#endif

esphome/espidf/__init__.py

@@ -5,7 +5,36 @@ package without the esp32 component package, so nothing here may pull
 in codegen or validation.
 """
 
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+# Written by the generated CMakeLists after project(); holds the bootloader
+# customization channels the cache cannot replay (see espidf.bootloader).
+BOOTLOADER_HOOKS_FILE = "esphome_bootloader_hooks.txt"
+
 
 def variant_to_idf_target(variant: str) -> str:
     """Map an esp32 variant name (e.g. "ESP32S3") to its ESP-IDF target name."""
     return variant.lower().replace("-", "")
+
+
+def parse_sdkconfig(path: Path) -> dict[str, str]:
+    """Map each CONFIG_X=value line of a text sdkconfig to its raw value.
+
+    Comment lines ("# CONFIG_X is not set") are skipped and quoted values
+    are unquoted.
+    """
+    result = {}
+    for line in path.read_text(encoding="utf-8").splitlines():
+        if not line.startswith("CONFIG_"):
+            continue
+        name, _, value = line.partition("=")
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] == '"':
+            value = value[1:-1]
+        result[name] = value
+    return result

esphome/espidf/bootloader.py

@@ -0,0 +1,493 @@
+"""Machine-global bootloader cache for the native ESP-IDF toolchain.
+
+Only a bootloader built twice with identical bytes is cached; secure and
+signing builds keep the stock in-tree path.
+"""
+
+# pylint: disable=protected-access
+# This module is part of the espidf toolchain and shares its private helpers.
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+import contextlib
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+import re
+import shutil
+import tempfile
+import time
+
+from esphome.build_helpers.ccache import parse_enable_env
+from esphome.build_helpers.tool_runner import run_build_tool
+from esphome.core import CORE, EsphomeError
+from esphome.espidf import BOOTLOADER_HOOKS_FILE, framework, parse_sdkconfig, toolchain
+from esphome.framework_helpers import _rename_with_retry
+from esphome.helpers import copy_file_if_changed, file_compare, rmtree, write_file
+
+_LOGGER = logging.getLogger(__name__)
+
+BOOTLOADER_CACHE_ENV = "ESPHOME_BOOTLOADER_CACHE"
+# Bump when the build or storage recipe changes; old entries then miss.
+_CACHE_SCHEMA = "1"
+_SECURE_OPTION = re.compile(r"CONFIG_(SECURE_|FLASH_ENCRYPTION)")
+_DISABLED_VALUES = frozenset(("n", "0", ""))
+# Marks a config name the app's sdkconfig does not define.
+_ABSENT = "\x00absent"
+# CMake reads these from the environment on the initial configure.
+_FLAG_ENV_VARS = ("CFLAGS", "CXXFLAGS", "ASMFLAGS", "LDFLAGS")
+_OUTPUTS = ("bootloader.bin", "bootloader.elf", "bootloader.map")
+
+
+def bootloader_cache_enabled() -> bool:
+    """Whether this build may take its bootloader from the cache."""
+    cache = toolchain._cache()
+    if cache.bootloader_enabled is None:
+        cache.bootloader_enabled = _compute_enabled()
+    return cache.bootloader_enabled
+
+
+def _compute_enabled() -> bool:
+    if parse_enable_env(BOOTLOADER_CACHE_ENV) is False:
+        return False
+    if "IDF_PATH" in os.environ:
+        # An unmanaged IDF checkout can change under the cache.
+        return False
+    try:
+        if toolchain._get_framework_source_override():
+            # A custom IDF source can rewire the bootloader subproject in
+            # ways the key and the macro tripwire cannot see.
+            return False
+        # The snapshot holds what was configured; the live sdkconfig is
+        # rewritten by kconfgen with always-on *_SUPPORTED constants.
+        snapshot = CORE.relative_build_path(f"sdkconfig.{CORE.name}.esphomeinternal")
+        if not snapshot.is_file() or _snapshot_blocks_cache(snapshot):
+            return False
+        if CORE.relative_build_path("bootloader_components").exists():
+            # Project-local bootloader overrides are inputs the key can't see.
+            return False
+        if _recorded_bootloader_hooks():
+            # The last configure saw a component customizing the bootloader.
+            _LOGGER.info("Bootloader cache off: build customizes the bootloader")
+            return False
+        resolved = toolchain._load_sdkconfig_json(toolchain._build_dir())
+        if resolved is not None and _resolved_blocks_cache(resolved):
+            # Hand edits to the real sdkconfig survive snapshot preservation.
+            _LOGGER.info("Bootloader cache off: secure options in the tree")
+            return False
+        if not os.access(tools := framework.get_idf_tools_path(), os.W_OK):
+            # A read-only shared prefix would fail the cache on every build.
+            _LOGGER.info("Bootloader cache off: %s is not writable", tools)
+            return False
+        from esphome.build_gen.espidf import idf_macro_matches
+
+        if not idf_macro_matches():
+            _LOGGER.info("IDF changed its bootloader macro; building in-tree")
+            return False
+        return True
+    except (OSError, EsphomeError) as err:
+        _LOGGER.info("Bootloader cache disabled: %s", err)
+        return False
+    except (KeyError, ValueError) as err:
+        # More likely a regression than the environment; still fail safe.
+        _LOGGER.warning("Bootloader cache disabled: %s", err)
+        return False
+
+
+def _recorded_bootloader_hooks() -> str | None:
+    """The bootloader customization channels the last configure recorded.
+
+    Dumped by the generated CMakeLists after project(), so every source
+    counts; None means no configure has written the record yet.
+    """
+    path = toolchain._build_dir() / BOOTLOADER_HOOKS_FILE
+    try:
+        return path.read_text(encoding="utf-8").strip()
+    except FileNotFoundError:
+        return None
+
+
+def _snapshot_blocks_cache(sdkconfig_path: Path) -> bool:
+    """Whether the configured options rule the cache out up front.
+
+    Secure options block because key file contents are inputs the key cannot
+    see; reproducible build off blocks to skip the doomed probe every compile.
+    """
+    config = parse_sdkconfig(sdkconfig_path)
+    if any(
+        _SECURE_OPTION.match(name) and value not in _DISABLED_VALUES
+        for name, value in config.items()
+    ):
+        return True
+    return config.get("CONFIG_APP_REPRODUCIBLE_BUILD", "") in _DISABLED_VALUES
+
+
+def _resolved_blocks_cache(app_config: dict) -> bool:
+    """Secure options in the resolved config; catches hand-edited sdkconfigs.
+
+    Key files are inputs the cache key cannot hash; *_SUPPORTED capability
+    constants are always on and say nothing about what is enabled.
+    """
+    return any(
+        name.startswith(("SECURE_", "FLASH_ENCRYPTION"))
+        and not name.endswith("_SUPPORTED")
+        and bool(value)
+        for name, value in app_config.items()
+    )
+
+
+def tree_uses_cached_bootloader(build_dir: Path) -> bool:
+    """Whether a configured tree was set up for the cached bootloader."""
+    cmakecache = build_dir / "CMakeCache.txt"
+    if not cmakecache.is_file():
+        return False
+    cache = toolchain._parse_cmakecache(cmakecache)
+    return cache.get(toolchain.USE_CACHED_BOOTLOADER_DEFINE) == "1"
+
+
+def _cache_root() -> Path:
+    return (
+        framework.get_idf_tools_path()
+        / "bootloaders"
+        / toolchain._get_core_framework_version()
+        / toolchain._idf_target()
+    )
+
+
+def _load_config_names() -> list[str]:
+    """The known union of config option names the bootloader consumes.
+
+    Each harvest is an immutable file named by its digest; the union over
+    them is monotonic without any cross-process locking.
+    """
+    union: set[str] = set()
+    for path in _cache_root().glob("config_names/*.json"):
+        try:
+            names = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, ValueError) as err:
+            _LOGGER.info("Ignoring unreadable %s: %s", path, err)
+            continue
+        if isinstance(names, list) and all(isinstance(n, str) for n in names):
+            union.update(names)
+        else:
+            _LOGGER.info("Ignoring corrupt %s", path)
+    return sorted(union)
+
+
+def _merge_config_names(names: Iterable[str]) -> list[str]:
+    """Record this build's harvested names and return the resulting union.
+
+    A set that was already recorded is not rewritten; concurrent publishes
+    can only add files, never lose another's discovered names.
+    """
+    merged = sorted(set(names))
+    path = _cache_root() / "config_names" / f"{_compute_key(merged)}.json"
+    if not path.is_file():
+        write_file(path, json.dumps(merged, separators=(",", ":")))
+    # The union guards against our own file being unreadable right back.
+    return sorted(set(merged) | set(_load_config_names()))
+
+
+def _subproject_cmake_args(
+    cmake: str, python: str, sdkconfig: str, idf_path: str, project_dir: str
+) -> list[str]:
+    """The configure argv, mirroring IDF's bootloader ExternalProject args."""
+    from esphome.components.esp32 import idf_version
+    import esphome.config_validation as cv
+
+    # 5.x self-appends the bootloader dir to EXTRA_COMPONENT_DIRS; 6.x
+    # instead forwards IDF_BUILD_V2 (empty: the subproject never opts in).
+    idf6 = idf_version() >= cv.Version(6, 0, 0)
+    args = [
+        cmake,
+        "-G",
+        "Ninja",
+        f"-DSDKCONFIG={sdkconfig}",
+        f"-DIDF_PATH={idf_path}",
+        f"-DIDF_TARGET={toolchain._idf_target()}",
+        "-DPYTHON_DEPS_CHECKED=1",
+        f"-DPYTHON={python}",
+        f"-DEXTRA_COMPONENT_DIRS={'' if idf6 else f'{idf_path}/components/bootloader'}",
+    ]
+    if idf6:
+        args.append("-DIDF_BUILD_V2=")
+    args += [
+        f"-DPROJECT_SOURCE_DIR={project_dir}",
+        "-DIGNORE_EXTRA_COMPONENT=",
+        f"{idf_path}/components/bootloader/subproject",
+    ]
+    return args
+
+
+def _key_payload(names: list[str], app_config: dict, compiler: str, stamp: str) -> dict:
+    """Everything that can influence the built bootloader."""
+    return {
+        "schema": _CACHE_SCHEMA,
+        "idf": toolchain._get_core_framework_version(),
+        "stamp": stamp,
+        "target": toolchain._idf_target(),
+        "compiler": compiler,
+        # Placeholder paths: any change to the invocation shape misses.
+        "args": _subproject_cmake_args(
+            "cmake", "@PYTHON@", "@SDKCONFIG@", "@IDF@", "@PROJECT@"
+        ),
+        "config": {n: app_config.get(n, _ABSENT) for n in names},
+        "env": {n: os.environ.get(n, _ABSENT) for n in _FLAG_ENV_VARS},
+    }
+
+
+def _compute_key(payload: dict | list) -> str:
+    """The digest naming a cache entry or a recorded config name set."""
+    text = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(text.encode()).hexdigest()[:16]
+
+
+def _build_standalone(build_dir: Path, verbose: bool) -> int:
+    """Configure and build the bootloader subproject into ``build_dir``."""
+    build_dir.mkdir(parents=True, exist_ok=True)
+    args = _subproject_cmake_args(
+        toolchain._get_idf_tool("cmake"),
+        toolchain._get_idf_tool("python"),
+        str(CORE.relative_build_path(f"sdkconfig.{CORE.name}")),
+        str(toolchain._get_idf_path()),
+        os.path.realpath(CORE.build_path),
+    )
+    filter_lines = None if verbose else toolchain.FILTER_IDF_LINES
+    for name, cmd in (("cmake", args), ("ninja", [toolchain._get_idf_tool("ninja")])):
+        log_path = build_dir / "log" / f"{name}_output.log"
+        rc = run_build_tool(
+            cmd,
+            cwd=build_dir,
+            env=toolchain._tool_env(),
+            filter_lines=filter_lines,
+            log_path=log_path,
+        )
+        if rc != 0:
+            _LOGGER.error("Bootloader %s failed with exit code %d", name, rc)
+            toolchain._print_hints(log_path)
+            return rc
+    return 0
+
+
+def _work_dir(prefix: str) -> Path:
+    """A private work dir in the cache; unique across shared tools prefixes."""
+    root = _cache_root()
+    root.mkdir(parents=True, exist_ok=True)
+    return Path(tempfile.mkdtemp(prefix=prefix, dir=root))
+
+
+def _bin_sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _entry_intact(entry: Path) -> bool:
+    """Complete with the published bin bytes; anything else is a miss.
+
+    False only for provable damage; a transient read error raises so a
+    valid shared entry is never mistaken for a corrupt one and deleted.
+    """
+    if not all((entry / name).is_file() for name in _OUTPUTS):
+        return False
+    try:
+        meta = json.loads((entry / "meta.json").read_text(encoding="utf-8"))
+    except (FileNotFoundError, ValueError):
+        return False
+    expected = meta.get("bootloader_bin_sha256") if isinstance(meta, dict) else None
+    return expected == _bin_sha256(entry / "bootloader.bin")
+
+
+def _publish(build_dir: Path, key: str, payload: dict) -> Path:
+    """Move the distilled outputs into the cache; losing a race is fine."""
+    entry = _cache_root() / key
+    if entry.is_dir() and not _entry_intact(entry):
+        # A partial or corrupted entry (manual cleanup) must not win.
+        _LOGGER.debug("Discarding damaged entry %s", entry)
+        _remove_dir(entry)
+    stage = _work_dir(f".stage-{key}-")
+    try:
+        for name in _OUTPUTS:
+            shutil.copy2(build_dir / name, stage / name)
+        # The resolved bootloader config, kept for debugging entries.
+        shutil.copy2(build_dir / "config" / "sdkconfig.json", stage / "sdkconfig.json")
+        meta = {
+            **payload,
+            "bootloader_bin_sha256": _bin_sha256(stage / "bootloader.bin"),
+        }
+        write_file(stage / "meta.json", json.dumps(meta, indent=2, sort_keys=True))
+        stage.chmod(0o755)  # mkdtemp creates 0o700
+        _rename_with_retry(stage, entry)
+    except (OSError, EsphomeError) as err:
+        # Failed to stage, or another process published the same key first.
+        _remove_dir(stage)
+        with contextlib.suppress(OSError):
+            if _entry_intact(entry):
+                _LOGGER.debug("Reusing published %s after: %s", entry, err)
+                return entry
+        raise  # keep the original error even if the intact check breaks
+    return entry
+
+
+def _remove_dir(path: Path) -> None:
+    """Best-effort rmtree with the repo's read-only and retry hardening."""
+    try:
+        rmtree(path)
+    except OSError as err:
+        # Leftover work dirs are cleared by the daily prune; leave a trace.
+        _LOGGER.debug("Could not remove %s: %s", path, err)
+
+
+def _install_into_build(entry: Path) -> Path:
+    """Copy the cached outputs to build/bootloader, where every consumer reads."""
+    dest = toolchain._build_dir() / "bootloader"
+    if (dest / "CMakeCache.txt").is_file():
+        # A leftover in-tree sub-build; its ninja state must not linger.
+        rmtree(dest)
+    if (stamps := dest.parent / "bootloader-prefix").is_dir():
+        # Stale ExternalProject stamps would make a later flip back to
+        # in-tree skip the sub-configure and fail on the missing cache.
+        rmtree(stamps)
+    for name in _OUTPUTS:
+        copy_file_if_changed(entry / name, dest / name)
+    return dest
+
+
+def _bootloader_offset(app_config: dict | None) -> int | None:
+    """The flash offset the bootloader is written to, or None when unknown."""
+    value = (app_config or {}).get("BOOTLOADER_OFFSET_IN_FLASH")
+    return None if value is None else int(value)
+
+
+def _check_bootloader_size(bin_path: Path, offset: int, table_offset: int) -> None:
+    """The check_sizes.py bootloader check, without spawning a process."""
+    size = bin_path.stat().st_size
+    free = table_offset - offset - size
+    if free < 0:
+        raise EsphomeError(
+            f"Bootloader binary size {size:#x} bytes is too large for "
+            f"partition table offset {table_offset:#x} (overflows by "
+            f"{-free:#x} bytes). Increase CONFIG_PARTITION_TABLE_OFFSET."
+        )
+    _LOGGER.info(
+        "Bootloader binary size %#x bytes. %#x bytes (%d%%) free.",
+        size,
+        free,
+        round(100 * free / (table_offset - offset)),
+    )
+
+
+def _prune_stale_dirs() -> None:
+    """Drop day-old work dirs from crashed builds; live ones are younger."""
+    cutoff = time.time() - 86400
+    try:
+        paths = list(_cache_root().glob(".*"))
+    except OSError as err:
+        _LOGGER.debug("Prune skipped: %s", err)
+        return
+    for path in paths:
+        # A racing build may remove its own dir mid-scan; skip just that one.
+        with contextlib.suppress(OSError):
+            if path.is_dir() and path.stat().st_mtime < cutoff:
+                _remove_dir(path)
+
+
+def ensure_cached_bootloader(verbose: bool = False) -> bool:
+    """Put the cached bootloader into build/bootloader, building on a miss.
+
+    False tells the caller to fall back to the in-tree build; a too-large
+    bootloader raises instead, since in-tree would fail the same way.
+    """
+    try:
+        if _recorded_bootloader_hooks() != "":
+            # Non-empty: configure just recorded a bootloader customization.
+            # None: this configure did not write the record at all.
+            _LOGGER.info("Bootloader customized by the build; building in-tree")
+            return False
+        app_config = toolchain._load_sdkconfig_json(toolchain._build_dir())
+        if app_config is not None and _resolved_blocks_cache(app_config):
+            # Configure just resolved a secure option the snapshot cannot see.
+            _LOGGER.info("Secure options in the resolved config; building in-tree")
+            return False
+        offset = _bootloader_offset(app_config)
+        table_offset = (app_config or {}).get("PARTITION_TABLE_OFFSET")
+        compiler = toolchain._resolved_c_compiler()
+        if offset is None or table_offset is None or compiler is None:
+            _LOGGER.info(
+                "Bootloader cache unusable: offset=%s table_offset=%s compiler=%s",
+                offset,
+                table_offset,
+                compiler,
+            )
+            return False
+        dest = _install_cached(app_config, compiler, verbose)
+    except (OSError, ValueError, EsphomeError) as err:
+        _LOGGER.warning("Bootloader cache failed: %s", err)
+        return False
+    if dest is None:
+        return False
+    try:
+        # The size overflow raises out: the in-tree build would fail the same.
+        _check_bootloader_size(dest / "bootloader.bin", offset, int(table_offset))
+    except (OSError, ValueError) as err:
+        _LOGGER.warning("Bootloader cache failed: %s", err)
+        return False
+    return True
+
+
+def _install_cached(app_config: dict, compiler: str, verbose: bool) -> Path | None:
+    """The installed build/bootloader dir, building and publishing on a miss;
+    None means fall back to the in-tree build."""
+    if not (stamp := framework.read_idf_version_txt(toolchain._get_idf_path())):
+        _LOGGER.info("Framework version.txt missing; cannot key the bootloader")
+        return None
+    if names := _load_config_names():
+        entry = _cache_root() / _compute_key(
+            _key_payload(names, app_config, compiler, stamp)
+        )
+        if _entry_intact(entry):
+            return _install_into_build(entry)
+    with contextlib.ExitStack() as cleanup:
+        tmp = _work_dir(".build-")
+        cleanup.callback(_remove_dir, tmp)
+        if _build_standalone(tmp, verbose) != 0:
+            return None
+        if (built_config := toolchain._load_sdkconfig_json(tmp)) is None:
+            # After a successful build this most likely means an IDF change.
+            _LOGGER.warning(
+                "Bootloader build produced no %s", tmp / "config" / "sdkconfig.json"
+            )
+            return None
+        # Only a build byte identical to a second one in a different dir may
+        # be cached; any nondeterminism fails here instead of serving stale.
+        probe = _work_dir(".build-")
+        cleanup.callback(_remove_dir, probe)
+        if _build_standalone(probe, verbose) != 0:
+            return None
+        if not file_compare(tmp / "bootloader.bin", probe / "bootloader.bin"):
+            _LOGGER.info("Bootloader build is not byte reproducible; not caching")
+            return None
+        names = _merge_config_names(built_config)
+        payload = _key_payload(names, app_config, compiler, stamp)
+        key = _compute_key(payload)
+        entry = _publish(tmp, key, payload)
+        _LOGGER.info("Cached bootloader %s for later builds", key)
+    _prune_stale_dirs()
+    return _install_into_build(entry)
+
+
+def inject_bootloader_flash_file(flash_data: dict, build_dir: Path) -> bool:
+    """Add the cached bootloader to a flasher_args ``flash_files`` map.
+
+    Mode gated: IDF omits the entry on purpose for some secure-boot builds.
+    False means a cached-mode tree is missing pieces; the image would not boot.
+    """
+    if not tree_uses_cached_bootloader(build_dir):
+        return True
+    offset = _bootloader_offset(toolchain._load_sdkconfig_json(build_dir))
+    if offset is None or not (build_dir / "bootloader" / "bootloader.bin").is_file():
+        _LOGGER.error("Cached bootloader missing; factory image has no bootloader")
+        return False
+    flash_data.setdefault("flash_files", {})[hex(offset)] = "bootloader/bootloader.bin"
+    return True

esphome/espidf/clang_tidy.py

@@ -11,7 +11,7 @@ running codegen on a config, it generates a minimal ESP-IDF CMake project:
   component, so their public include dirs land on the translation unit;
 * the repo ``sdkconfig.defaults`` enables sdkconfig-gated components (bt, ...).
 
-then runs ``idf.py reconfigure`` (configure only, no compile) and reads the
+then runs the CMake configure (no compile) and reads the
 resulting ``build/compile_commands.json``. The IDF version is the esp32
 component's recommended version.
 
@@ -367,7 +367,7 @@ def _write_tidy_project(
 def _generate_compile_commands(
     work_dir: Path, settings: _Settings, platformio_ini: Path
 ) -> Path:
-    """Generate the tidy project and run ``idf.py reconfigure`` (no build).
+    """Generate the tidy project and run the CMake configure (no build).
 
     Two-phase, like a real ESPHome build: a first configure with no builtin
     requires discovers which components actually register for the target (e.g.
@@ -398,7 +398,7 @@ def _generate_compile_commands(
     # Phase 1: discover the components available for this target.
     _write_tidy_project(work_dir, [], extra_deps, settings)
     if toolchain.run_reconfigure() != 0:
-        raise RuntimeError("idf.py reconfigure (discovery) failed")
+        raise RuntimeError("ESP-IDF CMake configure (discovery) failed")
 
     requires = sorted(
         set(get_available_components() or []) - _NON_REQUIRABLE_COMPONENTS
@@ -407,7 +407,7 @@ def _generate_compile_commands(
     # Phase 2: require every available builtin component.
     _write_tidy_project(work_dir, requires, extra_deps, settings)
     if toolchain.run_reconfigure() != 0:
-        raise RuntimeError("idf.py reconfigure failed")
+        raise RuntimeError("ESP-IDF CMake configure failed")
 
     return work_dir / "build" / "compile_commands.json"
 

esphome/espidf/framework.py

@@ -481,6 +481,17 @@ def _clone_idf_with_submodules(
         )
 
 
+def read_idf_version_txt(framework_path: Path) -> str:
+    """The installed framework's version.txt stamp, or empty."""
+    try:
+        return (framework_path / "version.txt").read_text(encoding="utf-8").strip()
+    except FileNotFoundError:
+        return ""
+    except OSError as err:
+        _LOGGER.info("Cannot read %s: %s", framework_path / "version.txt", err)
+        return ""
+
+
 def _write_idf_version_txt(framework_path: Path, version: str) -> None:
     """Write <framework_path>/version.txt if missing.
 
@@ -1206,54 +1217,45 @@ def _ccache_env() -> dict[str, str]:
 
     Only values the user has not already set in the environment are returned, so
     a custom ``CCACHE_DIR`` / ``CCACHE_MAXSIZE`` / etc. is respected.
-    The pch settings add ``time_macros`` sloppiness process-wide; the visible
-    effect is a cached TU can keep an older ``esp_app_desc`` build timestamp.
-    """
-    if not _ccache_enabled():
-        # The raw knob value (e.g. "disable") is still inherited by idf.py
-        # via os.environ, where a non-false-constant string reads as
-        # truthy; export the canonical off spelling instead
-        return {"IDF_CCACHE_ENABLE": "0"}
-
-    env = ccache_defaults_env(get_idf_tools_path() / "ccache")
-    env.update(ccache_pch_env())
-    # Exactly one canonical spelling ever reaches idf.py, whatever the
-    # accepted input spelling was ("enable", "yes", ...)
-    env["IDF_CCACHE_ENABLE"] = "1"
-    return env
-
-
-def _ccache_enabled() -> bool:
-    """Whether ESP-IDF compiles run under ccache.
-
-    IDF_CCACHE_ENABLE (the backend-native knob) wins over the shared
-    ESPHOME_CCACHE_ENABLE; when unset, enabled iff a runnable binary is
-    on PATH.
     """
+    # IDF_CCACHE_ENABLE (the backend-native knob) wins over the shared
+    # ESPHOME_CCACHE_ENABLE.
     idf_knob = parse_enable_env("IDF_CCACHE_ENABLE")
     if idf_knob is False:
-        return False
+        # Replace the inherited raw value (e.g. "disable") with the canonical
+        # off spelling, so every reader of the env sees the same answer
+        return {"IDF_CCACHE_ENABLE": "0"}
     if idf_knob is True:
         # Forced on ignores the runnability verdict, but the outcome is
         # worth saying out loud. Probed directly (not via the resolver,
         # whose failure message says "compiling without ccache" -- exactly
         # what forced-on does NOT do): only the truly-missing case means
-        # idf.py compiles without ccache; a broken binary is still used,
-        # since idf.py does its own PATH lookup.
+        # the build compiles without ccache; a broken binary is still used,
+        # since IDF's CMake does its own PATH lookup.
         if (ccache := shutil.which("ccache")) is None:
             _LOGGER.warning(
                 "IDF_CCACHE_ENABLE=1 but no ccache binary is on PATH; "
-                "idf.py will compile without ccache"
+                "the build will compile without ccache"
             )
         else:
             # The probe warns with this message iff the binary fails
             tool_version_runs(
                 ccache,
                 "IDF_CCACHE_ENABLE=1 forces on the ccache at %s even though "
-                "it failed to run; idf.py will use it anyway",
+                "it failed to run; the build will use it anyway",
             )
-        return True
-    return resolve_ccache_path() is not None
+    elif resolve_ccache_path() is None:
+        # ESP-IDF silently skips ccache without the binary; export the
+        # canonical off spelling so an unparsable inherited value (or a
+        # probe-rejected ccache CMake would still find) cannot enable it
+        return {"IDF_CCACHE_ENABLE": "0"}
+
+    env = ccache_defaults_env(get_idf_tools_path() / "ccache")
+    env.update(ccache_pch_env())
+    # Exactly one canonical spelling ever reaches the build, whatever the
+    # accepted input spelling was ("enable", "yes", ...)
+    env["IDF_CCACHE_ENABLE"] = "1"
+    return env
 
 
 def get_framework_env(

esphome/espidf/runner.py

@@ -1,379 +0,0 @@
-r"""Subprocess entry point for running ``idf.py`` with stdio wrapping.
-
-Invoked as ``python runner.py <script_path> [script args...]``.
-
-Wraps ``sys.stdout`` and ``sys.stderr`` with a ``_FilteringTTYStream``
-shim so that:
-
-1. ``isatty()`` unconditionally returns True. CMake, Ninja, and idf.py's
-   own progress-bar code all check ``stream.isatty()`` to decide between
-   TTY-format output (``\\r`` cursor moves, ANSI colors, fancy progress
-   bars) and a plain fallback. With the wrapper in place they always
-   emit TTY format, even when our real stdout is a pipe to the parent
-   process (e.g. running under the Home Assistant dashboard add-on).
-   Downstream consumers — local terminals and the HA dashboard log
-   viewer — render the TTY control sequences correctly.
-
-2. ``FILTER_IDF_LINES`` is applied inside the shim's ``write()`` so
-   noisy idf.py output is dropped before it leaves this subprocess.
-   Filtering is skipped when ``-v`` / ``--verbose`` appears in argv so
-   verbose mode still shows everything.
-
-ESP-IDF runs under its own Python virtual environment which does not
-have the ``esphome`` package installed, so the runner is intentionally
-self-contained: no imports from ``esphome`` at all. The line-filtering
-wrapper is inlined below rather than imported from
-``esphome.util.RedirectText`` for that reason.
-"""
-
-import sys
-
-# Regex patterns matched against each line of idf.py / CMake / Ninja
-# output. Lines that match are dropped before reaching the parent
-# process. Patterns are anchored at the start of the line (the shim
-# uses ``re.match``). Disabled when the user passes ``-v`` /
-# ``--verbose`` to ``esphome compile``.
-FILTER_IDF_LINES: list[str] = [
-    # idf.py's "how to flash" block at the end of a successful build.
-    # ESPHome handles flashing itself, so these instructions just clutter
-    # the output.
-    r"Project build complete\.",
-    r" idf\.py ",
-    r" python -m esptool ",
-    r"or$",
-    r"or from the ",
-    # CMake dumps the full list of IDF component paths on one giant line.
-    # It's purely informational and bloats the log.
-    r"-- Component paths:",
-    # CMake lists every linker script it adds (dozens of lines) and the
-    # complete flat list of IDF components on one giant line. Neither
-    # has diagnostic value for end users.
-    r"-- Adding linker script ",
-    r"-- Components:",
-    # IDF component manager notices: emitted on first build (no lock),
-    # once per stubbed dependency, plus the final "Processing N
-    # dependencies" enumeration. Patterns allow a leading run of dots
-    # because the component manager prints progress dots on the same
-    # line, so a NOTICE often arrives prefixed with ".NOTICE:" or
-    # "...........NOTICE:".
-    r"\.*NOTICE: ",
-    # ``idf.py size`` prefaces its table with a centered banner; the
-    # per-region table below already makes the structure obvious.
-    r"\s*Memory Type Usage Summary",
-    # Prefix match for esp-idf-size's trailing "Note:" paragraph (no
-    # upstream flag suppresses it).
-    r"Note: The reported total sizes may be smaller than those in the",
-    # Drop the blank line rich emits after the note so the build log
-    # doesn't end with an orphan gap before ESPHome's own status lines.
-    r"\s*$",
-    # ESP-IDF shells out to ``git rev-parse`` to embed a commit hash;
-    # esphome-libs strips ``.git`` from the tarball so those probes fail
-    # noisily without affecting the build.
-    r"-- git rev-parse returned ",
-    r"fatal: not a git repository",
-    r"Stopping at filesystem boundary",
-]
-
-# Windows code page identifier for UTF-8, as used by ``chcp 65001``.
-UTF8_CODEPAGE = 65001
-
-
-def _get_kernel32():
-    """Return the Windows kernel32 module, or None on any other platform."""
-    if sys.platform != "win32":
-        return None
-    import ctypes
-
-    return ctypes.windll.kernel32
-
-
-class _Utf8Console:
-    """Keep an attached Windows console on UTF-8 for the length of the build.
-
-    The build tree runs in UTF-8 mode, so esp_idf_size draws its table with
-    Unicode box characters. ``idf.py size`` reaches it through ``cmake -P``,
-    and CMake re-decodes the child's output with the console code page before
-    printing it, which turns the table into mojibake on any code page but
-    UTF-8. Every process in the build shares this console, so switching it
-    here covers CMake too. The old code pages go back on exit so the user's
-    terminal is left as it was.
-
-    A console that is already on UTF-8 is left alone. The code page belongs to
-    the console, not to this process, so a build that overlaps another one
-    must not save UTF-8 as the page to go back to.
-    """
-
-    def __init__(self, kernel32) -> None:
-        self._kernel32 = kernel32
-        self._codepages: tuple[int, int] | None = None
-
-    def __enter__(self) -> None:
-        kernel32 = self._kernel32
-        if kernel32 is None:
-            return
-        old_in = kernel32.GetConsoleCP()
-        old_out = kernel32.GetConsoleOutputCP()
-        # Both calls return 0 when no console is attached.
-        if not old_in or not old_out:
-            return
-        if old_in == UTF8_CODEPAGE and old_out == UTF8_CODEPAGE:
-            return
-        # Record the old pages first so a switch that fails part way through
-        # still gets put back on exit.
-        self._codepages = (old_in, old_out)
-        kernel32.SetConsoleCP(UTF8_CODEPAGE)
-        kernel32.SetConsoleOutputCP(UTF8_CODEPAGE)
-
-    def __exit__(self, *exc_info: object) -> None:
-        if self._codepages is None:
-            return
-        old_in, old_out = self._codepages
-        self._codepages = None
-        self._kernel32.SetConsoleCP(old_in)
-        self._kernel32.SetConsoleOutputCP(old_out)
-
-
-def main() -> int:
-    # ---- sys.path fix-up ---------------------------------------------------
-    #
-    # When Python runs this file as ``python runner.py``, it prepends the
-    # script's directory — ``<site-packages>/esphome/espidf/`` — to
-    # ``sys.path[0]``. That directory is part of the esphome package whose
-    # sibling ``types.py`` (in ``esphome/``) collides with stdlib ``types``.
-    # Any subsequent import that transitively touches ``types`` (``runpy``,
-    # ``pathlib``, ``functools``, ``typing``, ...) could resolve the wrong
-    # module. Drop the entry pre-emptively. ``sys`` is a built-in so
-    # importing it at module level earlier did not trigger the shadow.
-    if sys.path and sys.path[0]:
-        sys.path.pop(0)
-    # ---- end sys.path fix-up -----------------------------------------------
-
-    import contextlib
-    import os
-    from pathlib import Path
-    import re
-    import runpy
-
-    # Patch ``os.get_terminal_size`` to return a fallback size instead
-    # of raising ``OSError`` when the underlying fd isn't a real
-    # terminal.
-    #
-    # idf.py's ``fit_text_in_terminal`` (in ``idf_py_actions/tools.py``)
-    # unconditionally calls ``os.get_terminal_size()`` to format ninja
-    # progress lines. When that raises ``[Errno 25] Inappropriate
-    # ioctl for device`` on our pipe-backed stdout, idf.py catches the
-    # exception as ``EnvironmentError`` and silently exits its stdout
-    # reader coroutine — dropping all ninja build output from that
-    # point on. Returning a valid value keeps the coroutine alive so
-    # progress and error lines continue to flow through to the parent
-    # process.
-    #
-    # Honour the ``COLUMNS`` / ``LINES`` env vars if the caller set
-    # them explicitly. Otherwise fall back to ``(0, 0)``, which
-    # ``fit_text_in_terminal`` treats as "unknown width, don't
-    # truncate" (see the ``if not terminal_width: return out`` guard).
-    # Downstream log viewers (local terminals, the HA dashboard) wrap
-    # or scroll long lines themselves, so we'd rather emit the full
-    # file path than have idf.py elide its middle.
-    _orig_get_terminal_size = os.get_terminal_size
-
-    def _get_terminal_size_fallback(fd: int = 1) -> os.terminal_size:
-        try:
-            return _orig_get_terminal_size(fd)
-        except OSError:
-            try:
-                columns = int(os.environ.get("COLUMNS", "0"))
-            except ValueError:
-                columns = 0
-            try:
-                lines = int(os.environ.get("LINES", "0"))
-            except ValueError:
-                lines = 0
-            return os.terminal_size((columns, lines))
-
-    os.get_terminal_size = _get_terminal_size_fallback  # type: ignore[assignment]
-
-    # Strip ANSI escape sequences before comparing a line against the filter
-    # patterns, so colorized lines still match plain-text patterns.
-    ansi_escape = re.compile(r"\033[@-_][0-?]*[ -/]*[@-~]")
-
-    class _FilteringTTYStream:
-        r"""Minimal stdout/stderr wrapper.
-
-        * ``isatty()`` unconditionally returns True, tricking downstream
-          code into emitting TTY-format output.
-        * Input is split with ``str.splitlines(keepends=True)``, which
-          breaks on more than ``\\n`` and ``\\r``; form feed and a few
-          other control characters count too. Any piece whose
-          ANSI-stripped, right-stripped form matches one of
-          ``filter_lines`` is dropped.
-        * Only the final piece can still be waiting for more text, so
-          that one is held until a ``\\n`` or ``\\r`` arrives. A piece
-          that ended on one of the other breaks goes out as it is.
-
-        Mirrors the matching semantics of ``esphome.util.RedirectText``
-        so filter patterns behave identically in both the PlatformIO
-        and IDF runner paths.
-        """
-
-        def __init__(self, stream, filter_lines: list[str] | None) -> None:
-            self._stream = stream
-            if filter_lines:
-                combined = r"|".join(r"(?:" + p + r")" for p in filter_lines)
-                self._filter_pattern: re.Pattern[str] | None = re.compile(combined)
-            else:
-                self._filter_pattern = None
-            self._line_buffer = ""
-
-        def __getattr__(self, name: str):
-            # Hide ``buffer`` so consumers that use either
-            # ``getattr(stream, 'buffer', None)`` or
-            # ``hasattr(stream, 'buffer')`` see this as a text-only stream
-            # and skip writing raw bytes (which would bypass the filter).
-            if name == "buffer":
-                raise AttributeError(name)
-            return getattr(self._stream, name)
-
-        def isatty(self) -> bool:
-            return True
-
-        def flush(self) -> None:
-            self._stream.flush()
-
-        def _emit(self, line: str) -> None:
-            if self._filter_pattern is not None:
-                stripped = ansi_escape.sub("", line).rstrip()
-                if self._filter_pattern.match(stripped) is not None:
-                    return
-            self._stream.write(line)
-
-        def drain(self) -> None:
-            """Write out a held-back line that never got its terminator.
-
-            idf.py and CMake do not always end their last line with a
-            newline, and a build that dies part way through can stop mid
-            line. Without this the user is left staring at a build that
-            ended with no explanation.
-            """
-            if not self._line_buffer:
-                return
-            line, self._line_buffer = self._line_buffer, ""
-            try:
-                # Add the terminator the line never got, so whatever ESPHome
-                # prints next does not run onto the same line.
-                self._emit(line + "\n")
-                self._stream.flush()
-            except (OSError, ValueError) as err:
-                # We are called from cleanup, so raising would replace the
-                # build's real exit code. Saying so must not raise either:
-                # under the dashboard our stdout and stderr are the same
-                # pipe, so whatever broke the write has most likely broken
-                # the report, and ``sys.__stderr__`` is None on some
-                # interpreters. Carry the line along; it is usually the
-                # message saying why the build failed.
-                if (real_stderr := sys.__stderr__) is not None:
-                    with contextlib.suppress(OSError, ValueError):
-                        print(
-                            f"Could not write out remaining output ({err}): {line}",
-                            file=real_stderr,
-                        )
-
-        def write(self, data) -> int:
-            # Text streams normally hand us ``str``; decode in case
-            # somebody writes bytes directly.
-            if not isinstance(data, str):
-                data = data.decode(errors="replace")
-
-            if self._filter_pattern is None:
-                # Nothing to match against, so no need to wait for a full line.
-                self._emit(data)
-            else:
-                lines = (self._line_buffer + data).splitlines(keepends=True)
-                # Every piece but the last ends with something
-                # ``str.splitlines`` treats as a break, so only the last one
-                # can still be waiting for more text. Hold that one, write
-                # out the rest.
-                #
-                # Some of those breaks are not line endings to us, a form
-                # feed for one, so a piece can go out without ending in a
-                # newline. That beats what we did before, which was to stop
-                # at the first such piece and drop every complete line
-                # behind it.
-                if lines and not lines[-1].endswith(("\n", "\r")):
-                    self._line_buffer = lines.pop()
-                else:
-                    self._line_buffer = ""
-                for line in lines:
-                    self._emit(line)
-
-            # We tell idf.py it is talking to a terminal, so it sends progress
-            # bars and cursor moves. Our own stdout is usually a pipe, which is
-            # block buffered, so without this the build looks frozen until
-            # 8 KiB of output piles up.
-            self._stream.flush()
-            return len(data)
-
-    if len(sys.argv) < 2:
-        print(
-            "usage: runner.py <script_path> [args...]",
-            file=sys.stderr,
-        )
-        return 2
-
-    script_path = sys.argv[1]
-
-    # Mirror the platformio runner behaviour: verbose mode disables the
-    # line filter so all output reaches the user.
-    is_verbose = any(arg in ("-v", "--verbose") for arg in sys.argv[2:])
-    filter_lines = None if is_verbose else FILTER_IDF_LINES or None
-
-    class _FilteredStreams:
-        """Route ``sys.stdout`` and ``sys.stderr`` through the filtering shims.
-
-        On exit each shim releases a last line that never got its
-        terminator. The shims made here are drained rather than whatever
-        ``sys.stdout`` holds by then, which the script is free to replace.
-        """
-
-        def __init__(self, filter_lines: list[str] | None) -> None:
-            self._stdout = _FilteringTTYStream(sys.stdout, filter_lines)
-            self._stderr = _FilteringTTYStream(sys.stderr, filter_lines)
-
-        def __enter__(self) -> None:
-            sys.stdout = self._stdout  # type: ignore[assignment]
-            sys.stderr = self._stderr  # type: ignore[assignment]
-
-        def __exit__(self, *exc_info: object) -> None:
-            # Drain stderr from a finally so a surprise from the first one
-            # cannot strand the second.
-            try:
-                self._stdout.drain()
-            finally:
-                self._stderr.drain()
-
-    # Shift argv so the target script sees its own path as argv[0] and
-    # its own arguments starting at argv[1]. runpy.run_path does not
-    # modify sys.argv itself.
-    sys.argv = [script_path] + sys.argv[2:]
-
-    # Emulate Python's default behaviour of prepending the script's
-    # directory to sys.path[0] when running ``python script.py``.
-    # runpy.run_path does not do this automatically, but idf.py relies
-    # on it to import its sibling modules (python_version_checker,
-    # idf_py_actions, ...).
-    script_dir = str(Path(script_path).resolve().parent)
-    if script_dir not in sys.path:
-        sys.path.insert(0, script_dir)
-
-    # If idf.py calls sys.exit(), SystemExit propagates out of run_path
-    # and carries the exit code back to our caller. For normal returns,
-    # fall through and exit with 0. Either way the context managers drain
-    # the streams and put the console back on the way out, and they report
-    # instead of raising so cleanup cannot bury the real exit code.
-    with _FilteredStreams(filter_lines), _Utf8Console(_get_kernel32()):
-        runpy.run_path(script_path, run_name="__main__")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())