From 3a7a552f0db443beb0cf550988f3b9a017411e16 Mon Sep 17 00:00:00 2001 From: "J. Nick Koston" Date: Wed, 11 Mar 2026 10:34:00 -1000 Subject: [PATCH] [esp32] Add crash handler to capture and report backtrace across reboots When an ESP32 crashes, the backtrace is printed to UART and lost. Users without a serial cable never see this diagnostic information. This adds a crash handler that: - Intercepts esp_panic_handler() via --wrap linker flag - Captures the faulting PC and backtrace into .noinit memory - Supports both Xtensa (ESP32/S2/S3) and RISC-V (C3/C6/H2/C2) - Logs crash data at boot via ESP_LOGE (serial output) - Re-logs when HA subscribes to logs (visible in HA log viewer) - Adds CLI stacktrace decoding for the new log format --- esphome/components/api/api_connection.h | 6 + esphome/components/esp32/__init__.py | 1 + esphome/components/esp32/core.cpp | 4 + esphome/components/esp32/crash_handler.cpp | 177 +++++++++++++++++++++ esphome/components/esp32/crash_handler.h | 20 +++ esphome/components/logger/logger_esp32.cpp | 2 + esphome/platformio_api.py | 7 + 7 files changed, 217 insertions(+) create mode 100644 esphome/components/esp32/crash_handler.cpp create mode 100644 esphome/components/esp32/crash_handler.h diff --git a/esphome/components/api/api_connection.h b/esphome/components/api/api_connection.h index 3356511684f..6af142fecbc 100644 --- a/esphome/components/api/api_connection.h +++ b/esphome/components/api/api_connection.h @@ -14,6 +14,9 @@ #include "api_server.h" #include "esphome/core/application.h" #include "esphome/core/component.h" +#ifdef USE_ESP32 +#include "esphome/components/esp32/crash_handler.h" +#endif #include "esphome/core/entity_base.h" #include "esphome/core/string_ref.h" @@ -235,6 +238,9 @@ class APIConnection final : public APIServerConnectionBase { this->flags_.log_subscription = msg.level; if (msg.dump_config) App.schedule_dump_config(); +#ifdef USE_ESP32 + esp32::crash_handler_log(); +#endif } #ifdef USE_API_HOMEASSISTANT_SERVICES void on_subscribe_homeassistant_services_request() override { this->flags_.service_call_subscription = true; } diff --git a/esphome/components/esp32/__init__.py b/esphome/components/esp32/__init__.py index 52e70501dcf..630db2271b4 100644 --- a/esphome/components/esp32/__init__.py +++ b/esphome/components/esp32/__init__.py @@ -1442,6 +1442,7 @@ async def to_code(config): cg.add_build_flag("-DUSE_ESP32") cg.add_define("USE_NATIVE_64BIT_TIME") cg.add_build_flag("-Wl,-z,noexecstack") + cg.add_build_flag("-Wl,--wrap=esp_panic_handler") cg.add_define("ESPHOME_BOARD", config[CONF_BOARD]) variant = config[CONF_VARIANT] cg.add_build_flag(f"-DUSE_ESP32_VARIANT_{variant}") diff --git a/esphome/components/esp32/core.cpp b/esphome/components/esp32/core.cpp index 46c000562e1..7b13f4b4a35 100644 --- a/esphome/components/esp32/core.cpp +++ b/esphome/components/esp32/core.cpp @@ -1,5 +1,6 @@ #ifdef USE_ESP32 +#include "crash_handler.h" #include "esphome/core/defines.h" #include "esphome/core/hal.h" #include "esphome/core/helpers.h" @@ -36,6 +37,9 @@ void arch_restart() { } void arch_init() { + // Read crash data from previous boot before anything else + esp32::crash_handler_read_and_clear(); + // Enable the task watchdog only on the loop task (from which we're currently running) esp_task_wdt_add(nullptr); diff --git a/esphome/components/esp32/crash_handler.cpp b/esphome/components/esp32/crash_handler.cpp new file mode 100644 index 00000000000..a1ecca8062f --- /dev/null +++ b/esphome/components/esp32/crash_handler.cpp @@ -0,0 +1,177 @@ +#ifdef USE_ESP32 + +#include "crash_handler.h" +#include "esphome/core/log.h" + +#include +#include +#include +#include + +#if CONFIG_IDF_TARGET_ARCH_XTENSA +#include +#include +#include +#elif CONFIG_IDF_TARGET_ARCH_RISCV +#include +#endif + +static constexpr uint32_t CRASH_MAGIC = 0xDEADBEEF; +static constexpr size_t MAX_BACKTRACE = 8; + +// Check if an address looks like code (flash-mapped or IRAM). +// Must be safe to call from panic context (no flash access needed). +static inline bool IRAM_ATTR is_code_addr(uint32_t addr) { + return (addr >= SOC_IROM_LOW && addr < SOC_IROM_HIGH) || (addr >= SOC_IRAM_LOW && addr < SOC_IRAM_HIGH); +} + +// Raw crash data written by the panic handler wrapper. +// Lives in .noinit so it survives software reset. +// Defined at file scope (outside any namespace) because both the namespace +// functions and the extern "C" panic handler wrapper need to access it. +struct RawCrashData { + uint32_t magic; + uint32_t pc; + uint32_t backtrace[MAX_BACKTRACE]; + uint8_t backtrace_count; +}; +extern RawCrashData s_raw_crash_data; + +namespace esphome::esp32 { + +static const char *const TAG = "esp32.crash"; + +// Validated crash data — populated by crash_handler_read_and_clear() from the +// raw NOINIT data written by the panic handler wrapper. +static struct { + bool valid; + uint32_t pc; + uint32_t backtrace[MAX_BACKTRACE]; + uint8_t backtrace_count; +} s_crash_data; + +void crash_handler_read_and_clear() { + s_crash_data.valid = false; + if (s_raw_crash_data.magic == CRASH_MAGIC) { + s_crash_data.valid = true; + s_crash_data.pc = s_raw_crash_data.pc; + s_crash_data.backtrace_count = s_raw_crash_data.backtrace_count; + if (s_crash_data.backtrace_count > MAX_BACKTRACE) + s_crash_data.backtrace_count = MAX_BACKTRACE; + for (uint8_t i = 0; i < s_crash_data.backtrace_count; i++) { + s_crash_data.backtrace[i] = s_raw_crash_data.backtrace[i]; + } + } + // Clear magic regardless so we don't re-report on next normal reboot + s_raw_crash_data.magic = 0; +} + +bool crash_handler_has_data() { return s_crash_data.valid; } + +// Intentionally uses separate ESP_LOGE calls per line instead of combining into +// one multi-line log message. This ensures each address appears as its own line +// on the serial console, making it possible to see partial output if the device +// crashes again during boot, and allowing the CLI's process_stacktrace to match +// and decode each address individually. +void crash_handler_log() { + if (!s_crash_data.valid) + return; + + ESP_LOGE(TAG, "*** CRASH DETECTED ON PREVIOUS BOOT ***"); + ESP_LOGE(TAG, " PC: 0x%08" PRIX32 " (fault location)", s_crash_data.pc); + for (uint8_t i = 0; i < s_crash_data.backtrace_count; i++) { + ESP_LOGE(TAG, " BT%d: 0x%08" PRIX32 " (backtrace)", i, s_crash_data.backtrace[i]); + } + // Build addr2line hint with all captured addresses for easy copy-paste + char hint[256]; + int pos = snprintf(hint, sizeof(hint), "Use: addr2line -pfiaC -e firmware.elf 0x%08" PRIX32, s_crash_data.pc); + for (uint8_t i = 0; i < s_crash_data.backtrace_count && pos < (int) sizeof(hint) - 12; i++) { + pos += snprintf(hint + pos, sizeof(hint) - pos, " 0x%08" PRIX32, s_crash_data.backtrace[i]); + } + ESP_LOGE(TAG, "%s", hint); +} + +} // namespace esphome::esp32 + +// --- Panic handler wrapper --- +// Intercepts esp_panic_handler() via --wrap linker flag to capture crash data +// into NOINIT memory before the normal panic handler runs. +// +// The raw crash data struct must be separate from the read-side struct to avoid +// BSS initialization conflicts. It lives in .noinit so it survives software reset. + +RawCrashData __attribute__((section(".noinit"))) s_raw_crash_data; + +extern "C" { +extern void __real_esp_panic_handler(panic_info_t *info); + +void IRAM_ATTR __wrap_esp_panic_handler(panic_info_t *info) { + // Save the faulting PC + s_raw_crash_data.pc = (uint32_t) info->addr; + s_raw_crash_data.backtrace_count = 0; + +#if CONFIG_IDF_TARGET_ARCH_XTENSA + // Xtensa: walk the backtrace using the public API + if (info->frame != nullptr) { + auto *xt_frame = (XtExcFrame *) info->frame; + esp_backtrace_frame_t bt_frame = { + .pc = (uint32_t) xt_frame->pc, + .sp = (uint32_t) xt_frame->a1, + .next_pc = (uint32_t) xt_frame->a0, + .exc_frame = xt_frame, + }; + + uint8_t count = 0; + // First frame PC + if (is_code_addr(esp_cpu_process_stack_pc(bt_frame.pc))) { + s_raw_crash_data.backtrace[count++] = esp_cpu_process_stack_pc(bt_frame.pc); + } + // Walk remaining frames + while (count < MAX_BACKTRACE && bt_frame.next_pc != 0) { + if (!esp_backtrace_get_next_frame(&bt_frame)) { + break; + } + uint32_t pc = esp_cpu_process_stack_pc(bt_frame.pc); + if (is_code_addr(pc)) { + s_raw_crash_data.backtrace[count++] = pc; + } + } + s_raw_crash_data.backtrace_count = count; + } + +#elif CONFIG_IDF_TARGET_ARCH_RISCV + // RISC-V: capture MEPC + RA, then scan stack for code addresses + if (info->frame != nullptr) { + auto *rv_frame = (RvExcFrame *) info->frame; + uint8_t count = 0; + + // Save MEPC (fault PC) and RA (return address) + if (is_code_addr(rv_frame->mepc)) { + s_raw_crash_data.backtrace[count++] = rv_frame->mepc; + } + if (is_code_addr(rv_frame->ra) && rv_frame->ra != rv_frame->mepc) { + s_raw_crash_data.backtrace[count++] = rv_frame->ra; + } + + // Scan stack for additional code addresses (like RP2040 approach) + auto *scan_start = (uint32_t *) rv_frame->sp; + for (uint32_t i = 0; i < 64 && count < MAX_BACKTRACE; i++) { + uint32_t val = scan_start[i]; + if (is_code_addr(val) && val != rv_frame->mepc && val != rv_frame->ra) { + s_raw_crash_data.backtrace[count++] = val; + } + } + s_raw_crash_data.backtrace_count = count; + } +#endif + + // Write magic last — ensures all data is written before we mark it valid + s_raw_crash_data.magic = CRASH_MAGIC; + + // Call the real panic handler (prints to UART, does core dump, reboots, etc.) + __real_esp_panic_handler(info); +} + +} // extern "C" + +#endif // USE_ESP32 diff --git a/esphome/components/esp32/crash_handler.h b/esphome/components/esp32/crash_handler.h new file mode 100644 index 00000000000..39a764e8042 --- /dev/null +++ b/esphome/components/esp32/crash_handler.h @@ -0,0 +1,20 @@ +#pragma once + +#ifdef USE_ESP32 + +#include + +namespace esphome::esp32 { + +/// Read crash data from NOINIT memory and clear the magic marker. +void crash_handler_read_and_clear(); + +/// Log crash data if a crash was detected on previous boot. +void crash_handler_log(); + +/// Returns true if crash data was found this boot. +bool crash_handler_has_data(); + +} // namespace esphome::esp32 + +#endif // USE_ESP32 diff --git a/esphome/components/logger/logger_esp32.cpp b/esphome/components/logger/logger_esp32.cpp index d6ad77ff4fa..53768b37969 100644 --- a/esphome/components/logger/logger_esp32.cpp +++ b/esphome/components/logger/logger_esp32.cpp @@ -1,6 +1,7 @@ #ifdef USE_ESP32 #include "logger.h" +#include "esphome/components/esp32/crash_handler.h" #include #include @@ -117,6 +118,7 @@ void Logger::pre_setup() { esp_log_set_vprintf(esp_idf_log_vprintf_); ESP_LOGI(TAG, "Log initialized"); + esp32::crash_handler_log(); } void HOT Logger::write_msg_(const char *msg, uint16_t len) { diff --git a/esphome/platformio_api.py b/esphome/platformio_api.py index 5d4065207f0..cb080b2a953 100644 --- a/esphome/platformio_api.py +++ b/esphome/platformio_api.py @@ -340,6 +340,8 @@ STACKTRACE_ESP32_BACKTRACE_RE = re.compile( r"Backtrace:(?:\s*0x[0-9a-fA-F]{8}:0x[0-9a-fA-F]{8})+" ) STACKTRACE_ESP32_BACKTRACE_PC_RE = re.compile(r"4[0-9a-f]{7}") +# ESP32 crash handler (stored backtrace from previous boot) +STACKTRACE_ESP32_CRASH_BT_RE = re.compile(r"BT\d+:\s*0x([0-9a-fA-F]{8})") STACKTRACE_ESP8266_BACKTRACE_PC_RE = re.compile(r"4[0-9a-f]{7}") @@ -371,6 +373,11 @@ def process_stacktrace(config, line, backtrace_state): ) _decode_pc(config, match.group(1)) + # ESP32 crash handler backtrace (from previous boot) + match = re.search(STACKTRACE_ESP32_CRASH_BT_RE, line) + if match is not None: + _decode_pc(config, match.group(1)) + # ESP32 single-line backtrace match = re.match(STACKTRACE_ESP32_BACKTRACE_RE, line) if match is not None: