Merge remote-tracking branch 'origin/integration-debug-rerun' into integration-ccache-seed

Decode gdb's output with replacement characters and treat an OSError like
a timeout.

.github/workflows/ci.yml

@@ -406,6 +406,11 @@ jobs:
       # Registry libraries (noise-c, libsodium, ArduinoJson, lvgl) the host builds
       # download, shared per xdist worker by tests/integration/conftest.py
       INTEGRATION_LIBRARY_CACHE_PATH: ~/.esphome-integration-tests/pio_components
+      # The host build's ccache (get_build_env() in esphome/host/toolchain.py)
+      INTEGRATION_CCACHE_PATH: ~/.cache/esphome/host/ccache
+      # Only a full matrix run saves the ccache seed: a partial run would
+      # trim it to the few tests it ran
+      INTEGRATION_CCACHE_SAVE: ${{ needs.determine-jobs.outputs.integration-run-all == 'true' && (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) }}
     steps:
       - name: Check out code from GitHub
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -432,6 +437,54 @@ jobs:
         with:
           path: ${{ env.INTEGRATION_LIBRARY_CACHE_PATH }}
           key: integration-libraries-v1-${{ runner.os }}-${{ hashFiles('esphome/components/json/__init__.py', 'esphome/components/noise/__init__.py', 'esphome/components/lvgl/__init__.py', 'esphome/components/improv_base/__init__.py') }}
+      # A full run saves one ccache slice per bucket and restores only its own; a
+      # partial run groups tests differently, so it restores all of them. One step
+      # per slice of INTEGRATION_TESTS_SPLIT_BUCKETS in script/determine-jobs.py.
+      - name: Restore integration ccache (slice 1/5)
+        if: needs.determine-jobs.outputs.integration-run-all != 'true' || strategy.job-index == 0
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.INTEGRATION_CCACHE_PATH }}
+          key: integration-ccache-v1-0-${{ github.run_id }}
+          restore-keys: integration-ccache-v1-0-
+      - name: Restore integration ccache (slice 2/5)
+        if: needs.determine-jobs.outputs.integration-run-all != 'true' || strategy.job-index == 1
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.INTEGRATION_CCACHE_PATH }}
+          key: integration-ccache-v1-1-${{ github.run_id }}
+          restore-keys: integration-ccache-v1-1-
+      - name: Restore integration ccache (slice 3/5)
+        if: needs.determine-jobs.outputs.integration-run-all != 'true' || strategy.job-index == 2
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.INTEGRATION_CCACHE_PATH }}
+          key: integration-ccache-v1-2-${{ github.run_id }}
+          restore-keys: integration-ccache-v1-2-
+      - name: Restore integration ccache (slice 4/5)
+        if: needs.determine-jobs.outputs.integration-run-all != 'true' || strategy.job-index == 3
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.INTEGRATION_CCACHE_PATH }}
+          key: integration-ccache-v1-3-${{ github.run_id }}
+          restore-keys: integration-ccache-v1-3-
+      - name: Restore integration ccache (slice 5/5)
+        if: needs.determine-jobs.outputs.integration-run-all != 'true' || strategy.job-index == 4
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.INTEGRATION_CCACHE_PATH }}
+          key: integration-ccache-v1-4-${{ github.run_id }}
+          restore-keys: integration-ccache-v1-4-
+      - name: Mark the restored ccache as unused
+        # A hit refreshes an entry's mtime, so the trim before the save keeps
+        # exactly what this run used; the zeroed statistics cover this run only
+        run: |
+          dir="${INTEGRATION_CCACHE_PATH/#\~/$HOME}"
+          if [ -d "$dir" ]; then
+            find "$dir" -type f -exec touch -m -d 2000-01-01 {} +
+          fi
+          echo "CCACHE_DIR=$dir" >> "$GITHUB_ENV"
+          CCACHE_DIR="$dir" ccache -z
       - name: Restore Python virtual environment
         id: cache-venv
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@@ -462,6 +515,7 @@ jobs:
       - name: Register matcher
         run: echo "::add-matcher::.github/workflows/matchers/pytest.json"
       - name: Run integration tests
+        id: integration
         env:
           # JSON array of test paths; parsed into a bash array below to avoid
           # shell word-splitting / glob hazards.
@@ -477,6 +531,32 @@ jobs:
           # A hung test fails after 30 min: the 15 min shared build lock wait plus a cold compile
           pytest -vv --no-cov --tb=native --durations=30 -n auto --dist worksteal --timeout=1800 \
             --junitxml=junit-integration.xml "${test_files[@]}"
+      - name: Re-run the failed tests with debug symbols
+        # The first run builds without -g. A crash in the re-run prints a gdb
+        # backtrace of its core; exit 1 keeps the job red when the re-run passes.
+        if: failure() && steps.integration.outcome == 'failure'
+        timeout-minutes: 30
+        env:
+          ESPHOME_INTEGRATION_DEBUG_SYMBOLS: "1"
+          ESPHOME_INTEGRATION_CORE_DIR: /tmp/esphome-cores
+        run: |
+          . venv/bin/activate
+          # The image's package lists usually suffice; refresh them only if not
+          if ! command -v gdb >/dev/null &&
+            ! sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 90 apt-get install -y -qq gdb; then
+            sudo timeout -k 15 300 apt-get update -qq &&
+              sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 300 apt-get install -y -qq gdb ||
+              echo "::warning::gdb install failed; no backtraces"
+          fi
+          {
+            mkdir -p "$ESPHOME_INTEGRATION_CORE_DIR" &&
+              echo "$ESPHOME_INTEGRATION_CORE_DIR/core.%p" | sudo tee /proc/sys/kernel/core_pattern &&
+              ulimit -c unlimited
+          } || echo "::warning::core dumps not enabled; no backtraces"
+          # Under the step limit, so a hung test still reports before Actions kills the step
+          pytest -vv --no-cov --tb=native -n auto --dist worksteal --timeout=600 \
+            --lf --lfnf=none tests/integration || true
+          exit 1
       - name: Upload test artifacts
         # Tests that compare rendered output write the image they actually got here, so a
         # failure can be looked at without reproducing the whole build locally.
@@ -499,10 +579,19 @@ jobs:
           if-no-files-found: error
           # A full cron period of margin for the weekly refresh
           retention-days: 14
+      - name: Trim the integration ccache to this run
+        # Restored entries this run did not hit are stale; dropping them keeps
+        # the seed at one full run's working set instead of growing per save
+        if: success() && env.INTEGRATION_CCACHE_SAVE == 'true'
+        run: ccache --evict-older-than 1d
       - name: Print ccache statistics
-        # esphome stores the host build's ccache under the machine-global
-        # cache dir (see get_build_env() in esphome/host/toolchain.py).
-        run: CCACHE_DIR="$HOME/.cache/esphome/host/ccache" ccache -s
+        run: ccache -s
+      - name: Save integration ccache slice
+        if: success() && env.INTEGRATION_CCACHE_SAVE == 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.INTEGRATION_CCACHE_PATH }}
+          key: integration-ccache-v1-${{ strategy.job-index }}-${{ github.run_id }}
       - name: Save integration library cache
         # Bucket 0 only; the others would race the same immutable key.
         if: success() && (github.ref == 'refs/heads/dev' || contains(github.event.pull_request.labels.*.name, 'ci-cache-write')) && strategy.job-index == 0 && steps.library-cache.outputs.cache-hit != 'true'
@@ -511,6 +600,38 @@ jobs:
           path: ${{ env.INTEGRATION_LIBRARY_CACHE_PATH }}
           key: ${{ steps.library-cache.outputs.cache-primary-key }}
 
+  integration-ccache-prune:
+    name: Prune replaced integration ccache slices
+    runs-on: ubuntu-24.04
+    needs:
+      - determine-jobs
+      - integration-tests
+    # Dev only: a pull request's saves stay in its own scope and expire unused.
+    # A separate job so no test code runs with the cache write scope.
+    if: >-
+      !cancelled() &&
+      github.ref == 'refs/heads/dev' &&
+      needs.determine-jobs.outputs.integration-run-all == 'true'
+    permissions:
+      actions: write
+    steps:
+      - name: Delete the slices this run replaced
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
+        run: |
+          entries=$(gh cache list --ref refs/heads/dev --key integration-ccache-v1- --limit 100 --json id,key)
+          # A slice whose bucket failed did not save; its older entry stays
+          for key in $(jq -r --arg run "-$GITHUB_RUN_ID" '.[].key | select(endswith($run))' <<<"$entries"); do
+            prefix="${key%-"$GITHUB_RUN_ID"}-"
+            jq -r --arg p "$prefix" --argjson run "$GITHUB_RUN_ID" \
+              '.[] | select((.key | startswith($p)) and (.key | ltrimstr($p) | tonumber) < $run) | .id' <<<"$entries" |
+              while read -r id; do
+                # An overlapping dev run may have deleted it already
+                gh cache delete "$id" || echo "::notice::cache $id already gone"
+              done
+          done
+
   import-time:
     name: Check import esphome.__main__ time
     runs-on: ubuntu-24.04

script/determine-jobs.py

@@ -107,6 +107,7 @@ CLANG_TIDY_SPLIT_THRESHOLD = 65
 # bucket, not wall time (calibrated with the conftest compile cap); it
 # sizes the bucket count for small subsets.
 INTEGRATION_TESTS_SPLIT_THRESHOLD = 10
+# ci.yml restores one integration ccache slice per bucket; add a step there too
 INTEGRATION_TESTS_SPLIT_BUCKETS = 5
 INTEGRATION_TESTS_TARGET_BUCKET_WEIGHT = 360.0
 

tests/integration/conftest.py

@@ -16,6 +16,7 @@ import re
 import shutil
 import signal
 import socket
+import subprocess
 import sys
 import tempfile
 import time
@@ -28,6 +29,7 @@ import pytest_asyncio
 import esphome.config
 from esphome.core import CORE
 from esphome.helpers import (
+    get_bool_env,
     get_usable_cpu_count,
     read_file,
     rmtree,
@@ -39,6 +41,7 @@ from esphome.host.toolchain import get_elf_path
 from .const import (
     API_CONNECTION_TIMEOUT,
     DEFAULT_API_PORT,
+    GDB_TIMEOUT,
     LOCALHOST,
     PORT_POLL_INTERVAL,
     PORT_WAIT_TIMEOUT,
@@ -80,6 +83,13 @@ REPO_ROOT = Path(__file__).resolve().parent.parent.parent
 # CI caches parts of this path; keep in sync with ci.yml integration-tests.
 INTEGRATION_TESTS_ROOT = Path.home() / ".esphome-integration-tests"
 
+# Opt-in -g and gdb core backtraces; CI enables both only to re-run failed tests
+DEBUG_SYMBOLS = get_bool_env("ESPHOME_INTEGRATION_DEBUG_SYMBOLS")
+CORE_DIR = os.environ.get("ESPHOME_INTEGRATION_CORE_DIR")
+_CRASH_SIGNALS = frozenset(
+    {signal.SIGABRT, signal.SIGBUS, signal.SIGFPE, signal.SIGILL, signal.SIGSEGV}
+)
+
 
 def _get_build_env() -> dict[str, str]:
     """Environment for an ``esphome compile`` subprocess."""
@@ -227,9 +237,12 @@ async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> s
             "  build_flags:\n"
             '    - "-DDEBUG"  # Enable assert() statements\n'
             '    - "-DESPHOME_DEBUG"  # Enable ESPHOME_DEBUG_ASSERT checks\n'
-            '    - "-DESPHOME_DEBUG_API"  # Enable API protocol asserts\n'
-            '    - "-g"       # Add debug symbols',
+            '    - "-DESPHOME_DEBUG_API"  # Enable API protocol asserts'
+            + ('\n    - "-g"  # Add debug symbols' if DEBUG_SYMBOLS else ""),
         )
+    elif DEBUG_SYMBOLS and "  build_flags:\n" in content:
+        # A fixture with its own flags still gets symbols for a debug re-run
+        content = content.replace("  build_flags:\n", '  build_flags:\n    - "-g"\n', 1)
 
     # Replace external component path placeholder if present
     if "EXTERNAL_COMPONENT_PATH" in content:
@@ -772,16 +785,46 @@ async def _read_stream_lines(
             line_callback(decoded_line.rstrip())
 
 
+def _print_crash_backtrace(binary_path: Path, returncode: int, pid: int) -> None:
+    """Print a gdb backtrace from the core of a binary that died from a crash signal."""
+    if CORE_DIR is None or -returncode not in _CRASH_SIGNALS:
+        return
+    core = Path(CORE_DIR) / f"core.{pid}"
+    if not core.is_file() or (gdb := shutil.which("gdb")) is None:
+        print(f"No core or gdb for the crash of {binary_path} (looked for {core})")
+        return
+    try:
+        result = subprocess.run(
+            [gdb, "-batch", "-iex", "set debuginfod enabled off"]
+            + ["-ex", "thread apply all bt", str(binary_path), str(core)],
+            capture_output=True,
+            text=True,
+            errors="replace",
+            timeout=GDB_TIMEOUT,
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired) as err:
+        # Best effort: never replace the test's own failure
+        print(f"gdb could not read {core}: {err}")
+        return
+    print(f"--- Backtrace of {binary_path} ({core}) ---")
+    print(result.stdout + result.stderr)
+
+
 @asynccontextmanager
 async def run_binary(
     binary_path: Path,
     line_callback: Callable[[str], None] | None = None,
+    symbols_path: Path | None = None,
 ) -> AsyncGenerator[tuple[asyncio.subprocess.Process, list[str]]]:
     """Run a binary under a PTY, capture log output, and clean up on exit.
 
     Yields the running ``Process`` and a live list of captured log lines.
     No port wait -- callers that need that should use
-    ``run_binary_and_wait_for_port``."""
+    ``run_binary_and_wait_for_port``.
+    ``symbols_path`` is the unstripped build when ``binary_path`` is a stripped
+    copy; a crash backtrace reads its symbols from it.
+    """
     # Create a pseudo-terminal to make the binary think it's running interactively
     # This is needed because the ESPHome host logger checks isatty()
     controller_fd, device_fd = pty.openpty()
@@ -810,6 +853,9 @@ async def run_binary(
     )
 
     if process.returncode is not None:
+        _print_crash_backtrace(
+            symbols_path or binary_path, process.returncode, process.pid
+        )
         raise RuntimeError(
             f"Process died immediately with return code {process.returncode}. "
             "Ensure the binary is valid and can run successfully."
@@ -851,6 +897,9 @@ async def run_binary(
                     # Last resort: SIGKILL
                     process.kill()
                     await process.wait()
+        _print_crash_backtrace(
+            symbols_path or binary_path, process.returncode, process.pid
+        )
 
 
 @asynccontextmanager

tests/integration/const.py

@@ -19,3 +19,4 @@ KEY_ACTIVATION_DELAY = 0.5  # seconds
 # Process shutdown timeouts
 SIGINT_TIMEOUT = 5.0  # seconds
 SIGTERM_TIMEOUT = 2.0  # seconds
+GDB_TIMEOUT = 60.0  # seconds, reading a crashed binary's core

tests/integration/test_host_ota.py

@@ -141,9 +141,10 @@ async def _build(
     write_yaml_config: ConfigWriter,
     compile_esphome: CompileFunction,
     reserved_tcp_port: tuple[int, socket.socket],
-) -> tuple[int, int, Path]:
+) -> tuple[int, int, Path, Path]:
     """Reserve an OTA port, compile the fixture with it, and release both
-    ports right before the binary is started."""
+    ports right before the binary is started. Returns the stripped binary to
+    run and upload, then the unstripped build for crash backtraces."""
     api_port, api_socket = reserved_tcp_port
     with _reserve_port() as (ota_port, ota_socket):
         config_path = await write_yaml_config(
@@ -152,11 +153,11 @@ async def _build(
         binary_path = await compile_esphome(config_path)
         api_socket.close()
         ota_socket.close()
-    return api_port, ota_port, await _strip_debug(binary_path)
+    return api_port, ota_port, await _strip_debug(binary_path), binary_path
 
 
 async def _strip_debug(binary_path: Path) -> Path:
-    """The fixtures build with -g; every upload sends the image, so drop the
+    """A debug re-run builds with -g; every upload sends the image, so drop the
     debug info (GNU and Apple strip both take -S)."""
     if (strip := shutil.which("strip")) is None:
         _LOGGER.warning(
@@ -207,6 +208,7 @@ class _Device:
     api_port: int
     ota_port: int
     binary_path: Path
+    symbols_path: Path
     proc: asyncio.subprocess.Process | None = None
     waiter: LineWaiter = field(default_factory=LineWaiter)
 
@@ -302,7 +304,9 @@ async def test_host_ota_self_update(
             yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port
         )
     )
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (
         proc,
         _lines,
     ):
@@ -341,7 +345,9 @@ async def test_host_ota_deflate(
         """A stream that inflates past the size the client announced."""
         return _REAL_COMPRESS(data + bytes(8192), *args, **kwargs)
 
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (
         proc,
         _lines,
     ):
@@ -422,7 +428,9 @@ async def test_host_ota_encrypted(
             yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port
         )
     )
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines):
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (proc, lines):
         dev.proc = proc
         await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT)
         await asyncio.get_running_loop().run_in_executor(
@@ -458,7 +466,9 @@ async def test_host_ota_encrypted_safe_mode(
     )
     # The api port never opens in safe mode, so wait for the log line instead
     force_safe_mode(DEVICE_NAME)
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines):
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (proc, lines):
         dev.proc = proc
         await _wait_for_line(lines, "SAFE MODE IS ACTIVE", PORT_WAIT_TIMEOUT)
         await _wait_for_port(LOCALHOST, dev.ota_port, PORT_WAIT_TIMEOUT)
@@ -485,7 +495,9 @@ async def test_host_ota_api_key_offer_with_password(
             yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port
         )
     )
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines):
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (proc, lines):
         dev.proc = proc
         await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT)
         await _wait_for_line(lines, "Encryption: offered")
@@ -535,7 +547,9 @@ async def test_host_ota_provisioned_api_key(
             yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port
         )
     )
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines):
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (proc, lines):
         dev.proc = proc
         await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT)
         await _wait_for_line(lines, "once the api key is provisioned")
@@ -570,7 +584,9 @@ async def test_host_ota_provisioned_api_key_safe_mode(
             yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port
         )
     )
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, _lines):
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (proc, _lines):
         dev.proc = proc
         await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT)
         await _provision_key(dev, api_client_connected)
@@ -579,7 +595,9 @@ async def test_host_ota_provisioned_api_key_safe_mode(
     # exits, so safe mode takes a second start
     force_safe_mode(DEVICE_NAME)
     key = PROVISIONING_PSK.decode()
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines):
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (proc, lines):
         dev.proc = proc
         await _wait_for_line(lines, "SAFE MODE IS ACTIVE", PORT_WAIT_TIMEOUT)
         await _wait_for_port(LOCALHOST, dev.ota_port, PORT_WAIT_TIMEOUT)
@@ -611,7 +629,10 @@ async def test_host_ota_rejects_garbage(
     bogus_path = integration_test_dir / "bogus.bin"
     bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8)
 
-    async with run_binary(dev.binary_path) as (proc, _lines):
+    async with run_binary(dev.binary_path, symbols_path=dev.symbols_path) as (
+        proc,
+        _lines,
+    ):
         dev.proc = proc
         await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT)
         pid_before = proc.pid

tests/script/test_determine_jobs.py

@@ -9,6 +9,7 @@ import sys
 from unittest.mock import Mock, patch
 
 import pytest
+import yaml
 
 # Add the script directory to Python path so we can import the module
 script_dir = str((Path(__file__).parent / ".." / ".." / "script").resolve())
@@ -1308,6 +1309,20 @@ def test_esp_idf_infra_changed(changed_files: list[str], expected: bool) -> None
     assert determine_jobs._esp_idf_infra_changed(changed_files) is expected
 
 
+def test_integration_ccache_restores_every_bucket_slice() -> None:
+    """ci.yml restores one ccache slice per full-run bucket; a bucket count
+    change without a matching restore step would leave a slice unread."""
+    repo_root = Path(__file__).resolve().parents[2]
+    workflow = yaml.safe_load((repo_root / ".github/workflows/ci.yml").read_text())
+    keys = [
+        step["with"]["restore-keys"]
+        for step in workflow["jobs"]["integration-tests"]["steps"]
+        if step.get("name", "").startswith("Restore integration ccache (slice")
+    ]
+    buckets = determine_jobs.INTEGRATION_TESTS_SPLIT_BUCKETS
+    assert keys == [f"integration-ccache-v1-{i}-" for i in range(buckets)]
+
+
 def test_esp_idf_infra_trigger_paths_exist() -> None:
     """A renamed or moved trigger module must fail here, not silently stop
     forcing the esp32 IDF compile."""