Keep a gdb failure from replacing the test's own error

Decode gdb's output with replacement characters and treat an OSError like
a timeout.

.github/workflows/ci.yml

@@ -462,6 +462,7 @@ jobs:
       - name: Register matcher
         run: echo "::add-matcher::.github/workflows/matchers/pytest.json"
       - name: Run integration tests
+        id: integration
         env:
           # JSON array of test paths; parsed into a bash array below to avoid
           # shell word-splitting / glob hazards.
@@ -477,6 +478,32 @@ jobs:
           # A hung test fails after 30 min: the 15 min shared build lock wait plus a cold compile
           pytest -vv --no-cov --tb=native --durations=30 -n auto --dist worksteal --timeout=1800 \
             --junitxml=junit-integration.xml "${test_files[@]}"
+      - name: Re-run the failed tests with debug symbols
+        # The first run builds without -g. A crash in the re-run prints a gdb
+        # backtrace of its core; exit 1 keeps the job red when the re-run passes.
+        if: failure() && steps.integration.outcome == 'failure'
+        timeout-minutes: 30
+        env:
+          ESPHOME_INTEGRATION_DEBUG_SYMBOLS: "1"
+          ESPHOME_INTEGRATION_CORE_DIR: /tmp/esphome-cores
+        run: |
+          . venv/bin/activate
+          # The image's package lists usually suffice; refresh them only if not
+          if ! command -v gdb >/dev/null &&
+            ! sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 90 apt-get install -y -qq gdb; then
+            sudo timeout -k 15 300 apt-get update -qq &&
+              sudo DEBIAN_FRONTEND=noninteractive timeout -k 15 300 apt-get install -y -qq gdb ||
+              echo "::warning::gdb install failed; no backtraces"
+          fi
+          {
+            mkdir -p "$ESPHOME_INTEGRATION_CORE_DIR" &&
+              echo "$ESPHOME_INTEGRATION_CORE_DIR/core.%p" | sudo tee /proc/sys/kernel/core_pattern &&
+              ulimit -c unlimited
+          } || echo "::warning::core dumps not enabled; no backtraces"
+          # Under the step limit, so a hung test still reports before Actions kills the step
+          pytest -vv --no-cov --tb=native -n auto --dist worksteal --timeout=600 \
+            --lf --lfnf=none tests/integration || true
+          exit 1
       - name: Upload test artifacts
         # Tests that compare rendered output write the image they actually got here, so a
         # failure can be looked at without reproducing the whole build locally.

tests/integration/conftest.py

@@ -16,6 +16,7 @@ import re
 import shutil
 import signal
 import socket
+import subprocess
 import sys
 import tempfile
 import time
@@ -28,6 +29,7 @@ import pytest_asyncio
 import esphome.config
 from esphome.core import CORE
 from esphome.helpers import (
+    get_bool_env,
     get_usable_cpu_count,
     read_file,
     rmtree,
@@ -39,6 +41,7 @@ from esphome.host.toolchain import get_elf_path
 from .const import (
     API_CONNECTION_TIMEOUT,
     DEFAULT_API_PORT,
+    GDB_TIMEOUT,
     LOCALHOST,
     PORT_POLL_INTERVAL,
     PORT_WAIT_TIMEOUT,
@@ -80,6 +83,13 @@ REPO_ROOT = Path(__file__).resolve().parent.parent.parent
 # CI caches parts of this path; keep in sync with ci.yml integration-tests.
 INTEGRATION_TESTS_ROOT = Path.home() / ".esphome-integration-tests"
 
+# Opt-in -g and gdb core backtraces; CI enables both only to re-run failed tests
+DEBUG_SYMBOLS = get_bool_env("ESPHOME_INTEGRATION_DEBUG_SYMBOLS")
+CORE_DIR = os.environ.get("ESPHOME_INTEGRATION_CORE_DIR")
+_CRASH_SIGNALS = frozenset(
+    {signal.SIGABRT, signal.SIGBUS, signal.SIGFPE, signal.SIGILL, signal.SIGSEGV}
+)
+
 
 def _get_build_env() -> dict[str, str]:
     """Environment for an ``esphome compile`` subprocess."""
@@ -227,9 +237,12 @@ async def yaml_config(request: pytest.FixtureRequest, unused_tcp_port: int) -> s
             "  build_flags:\n"
             '    - "-DDEBUG"  # Enable assert() statements\n'
             '    - "-DESPHOME_DEBUG"  # Enable ESPHOME_DEBUG_ASSERT checks\n'
-            '    - "-DESPHOME_DEBUG_API"  # Enable API protocol asserts\n'
-            '    - "-g"       # Add debug symbols',
+            '    - "-DESPHOME_DEBUG_API"  # Enable API protocol asserts'
+            + ('\n    - "-g"  # Add debug symbols' if DEBUG_SYMBOLS else ""),
         )
+    elif DEBUG_SYMBOLS and "  build_flags:\n" in content:
+        # A fixture with its own flags still gets symbols for a debug re-run
+        content = content.replace("  build_flags:\n", '  build_flags:\n    - "-g"\n', 1)
 
     # Replace external component path placeholder if present
     if "EXTERNAL_COMPONENT_PATH" in content:
@@ -772,16 +785,46 @@ async def _read_stream_lines(
             line_callback(decoded_line.rstrip())
 
 
+def _print_crash_backtrace(binary_path: Path, returncode: int, pid: int) -> None:
+    """Print a gdb backtrace from the core of a binary that died from a crash signal."""
+    if CORE_DIR is None or -returncode not in _CRASH_SIGNALS:
+        return
+    core = Path(CORE_DIR) / f"core.{pid}"
+    if not core.is_file() or (gdb := shutil.which("gdb")) is None:
+        print(f"No core or gdb for the crash of {binary_path} (looked for {core})")
+        return
+    try:
+        result = subprocess.run(
+            [gdb, "-batch", "-iex", "set debuginfod enabled off"]
+            + ["-ex", "thread apply all bt", str(binary_path), str(core)],
+            capture_output=True,
+            text=True,
+            errors="replace",
+            timeout=GDB_TIMEOUT,
+            check=False,
+        )
+    except (OSError, subprocess.TimeoutExpired) as err:
+        # Best effort: never replace the test's own failure
+        print(f"gdb could not read {core}: {err}")
+        return
+    print(f"--- Backtrace of {binary_path} ({core}) ---")
+    print(result.stdout + result.stderr)
+
+
 @asynccontextmanager
 async def run_binary(
     binary_path: Path,
     line_callback: Callable[[str], None] | None = None,
+    symbols_path: Path | None = None,
 ) -> AsyncGenerator[tuple[asyncio.subprocess.Process, list[str]]]:
     """Run a binary under a PTY, capture log output, and clean up on exit.
 
     Yields the running ``Process`` and a live list of captured log lines.
     No port wait -- callers that need that should use
-    ``run_binary_and_wait_for_port``."""
+    ``run_binary_and_wait_for_port``.
+    ``symbols_path`` is the unstripped build when ``binary_path`` is a stripped
+    copy; a crash backtrace reads its symbols from it.
+    """
     # Create a pseudo-terminal to make the binary think it's running interactively
     # This is needed because the ESPHome host logger checks isatty()
     controller_fd, device_fd = pty.openpty()
@@ -810,6 +853,9 @@ async def run_binary(
     )
 
     if process.returncode is not None:
+        _print_crash_backtrace(
+            symbols_path or binary_path, process.returncode, process.pid
+        )
         raise RuntimeError(
             f"Process died immediately with return code {process.returncode}. "
             "Ensure the binary is valid and can run successfully."
@@ -851,6 +897,9 @@ async def run_binary(
                     # Last resort: SIGKILL
                     process.kill()
                     await process.wait()
+        _print_crash_backtrace(
+            symbols_path or binary_path, process.returncode, process.pid
+        )
 
 
 @asynccontextmanager

tests/integration/const.py

@@ -19,3 +19,4 @@ KEY_ACTIVATION_DELAY = 0.5  # seconds
 # Process shutdown timeouts
 SIGINT_TIMEOUT = 5.0  # seconds
 SIGTERM_TIMEOUT = 2.0  # seconds
+GDB_TIMEOUT = 60.0  # seconds, reading a crashed binary's core

tests/integration/test_host_ota.py

@@ -141,9 +141,10 @@ async def _build(
     write_yaml_config: ConfigWriter,
     compile_esphome: CompileFunction,
     reserved_tcp_port: tuple[int, socket.socket],
-) -> tuple[int, int, Path]:
+) -> tuple[int, int, Path, Path]:
     """Reserve an OTA port, compile the fixture with it, and release both
-    ports right before the binary is started."""
+    ports right before the binary is started. Returns the stripped binary to
+    run and upload, then the unstripped build for crash backtraces."""
     api_port, api_socket = reserved_tcp_port
     with _reserve_port() as (ota_port, ota_socket):
         config_path = await write_yaml_config(
@@ -152,11 +153,11 @@ async def _build(
         binary_path = await compile_esphome(config_path)
         api_socket.close()
         ota_socket.close()
-    return api_port, ota_port, await _strip_debug(binary_path)
+    return api_port, ota_port, await _strip_debug(binary_path), binary_path
 
 
 async def _strip_debug(binary_path: Path) -> Path:
-    """The fixtures build with -g; every upload sends the image, so drop the
+    """A debug re-run builds with -g; every upload sends the image, so drop the
     debug info (GNU and Apple strip both take -S)."""
     if (strip := shutil.which("strip")) is None:
         _LOGGER.warning(
@@ -207,6 +208,7 @@ class _Device:
     api_port: int
     ota_port: int
     binary_path: Path
+    symbols_path: Path
     proc: asyncio.subprocess.Process | None = None
     waiter: LineWaiter = field(default_factory=LineWaiter)
 
@@ -302,7 +304,9 @@ async def test_host_ota_self_update(
             yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port
         )
     )
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (
         proc,
         _lines,
     ):
@@ -341,7 +345,9 @@ async def test_host_ota_deflate(
         """A stream that inflates past the size the client announced."""
         return _REAL_COMPRESS(data + bytes(8192), *args, **kwargs)
 
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (
         proc,
         _lines,
     ):
@@ -422,7 +428,9 @@ async def test_host_ota_encrypted(
             yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port
         )
     )
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines):
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (proc, lines):
         dev.proc = proc
         await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT)
         await asyncio.get_running_loop().run_in_executor(
@@ -458,7 +466,9 @@ async def test_host_ota_encrypted_safe_mode(
     )
     # The api port never opens in safe mode, so wait for the log line instead
     force_safe_mode(DEVICE_NAME)
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines):
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (proc, lines):
         dev.proc = proc
         await _wait_for_line(lines, "SAFE MODE IS ACTIVE", PORT_WAIT_TIMEOUT)
         await _wait_for_port(LOCALHOST, dev.ota_port, PORT_WAIT_TIMEOUT)
@@ -485,7 +495,9 @@ async def test_host_ota_api_key_offer_with_password(
             yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port
         )
     )
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines):
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (proc, lines):
         dev.proc = proc
         await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT)
         await _wait_for_line(lines, "Encryption: offered")
@@ -535,7 +547,9 @@ async def test_host_ota_provisioned_api_key(
             yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port
         )
     )
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines):
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (proc, lines):
         dev.proc = proc
         await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT)
         await _wait_for_line(lines, "once the api key is provisioned")
@@ -570,7 +584,9 @@ async def test_host_ota_provisioned_api_key_safe_mode(
             yaml_config, write_yaml_config, compile_esphome, reserved_tcp_port
         )
     )
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, _lines):
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (proc, _lines):
         dev.proc = proc
         await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT)
         await _provision_key(dev, api_client_connected)
@@ -579,7 +595,9 @@ async def test_host_ota_provisioned_api_key_safe_mode(
     # exits, so safe mode takes a second start
     force_safe_mode(DEVICE_NAME)
     key = PROVISIONING_PSK.decode()
-    async with run_binary(dev.binary_path, line_callback=dev.on_log) as (proc, lines):
+    async with run_binary(
+        dev.binary_path, symbols_path=dev.symbols_path, line_callback=dev.on_log
+    ) as (proc, lines):
         dev.proc = proc
         await _wait_for_line(lines, "SAFE MODE IS ACTIVE", PORT_WAIT_TIMEOUT)
         await _wait_for_port(LOCALHOST, dev.ota_port, PORT_WAIT_TIMEOUT)
@@ -611,7 +629,10 @@ async def test_host_ota_rejects_garbage(
     bogus_path = integration_test_dir / "bogus.bin"
     bogus_path.write_bytes(b"NOT-AN-EXECUTABLE-AT-ALL" * 8)
 
-    async with run_binary(dev.binary_path) as (proc, _lines):
+    async with run_binary(dev.binary_path, symbols_path=dev.symbols_path) as (
+        proc,
+        _lines,
+    ):
         dev.proc = proc
         await _wait_for_port(LOCALHOST, dev.api_port, PORT_WAIT_TIMEOUT)
         pid_before = proc.pid