Add crash dump collection through CMUX.

Author: MarkusLassila

app/boards/nrf9151dk_nrf9151_ns.overlay

@@ -14,6 +14,7 @@
 &uart0 {
 	status = "okay";
 	hw-flow-control;
+	current-speed = <1000000>;
 
 	dtr_uart0: dtr-uart {
 		compatible = "nordic,dtr-uart";

app/overlay-trace-backend-cmux.conf

@@ -0,0 +1,12 @@
+#
+# Copyright (c) 2025 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
+#
+
+# CMUX trace backend for modem coredump
+CONFIG_NRF_MODEM_LIB_TRACE=y
+CONFIG_NRF_MODEM_LIB_TRACE_BACKEND_CMUX=y
+# Configure the modem trace level to coredump only
+CONFIG_NRF_MODEM_LIB_TRACE_LEVEL_OVERRIDE=y
+CONFIG_NRF_MODEM_LIB_TRACE_LEVEL_COREDUMP_ONLY=y

app/prj.conf

@@ -136,6 +136,8 @@ CONFIG_SM_EXTERNAL_XTAL=n
 #CONFIG_LOG_MODE_IMMEDIATE=y
 #CONFIG_DEBUG_OPTIMIZATIONS=y
 #CONFIG_DTR_UART_LOG_LEVEL_DBG=y
+#CONFIG_THREAD_MONITOR=y
+#CONFIG_NRF_MODEM_LIB_LOG_LEVEL_DBG=y
 
 # For using external GNSS antenna
 #CONFIG_MODEM_ANTENNA=y

app/scripts/sm_start_ppp.sh

@@ -185,7 +185,7 @@ PPP_CMUX=$(ls /dev/gsmtty* | sort -V | head -n 2 | tail -n 1)
 log_dbg "AT CMUX:  $AT_CMUX"
 log_dbg "PPP CMUX: $PPP_CMUX"
 
-sleep 1
+sleep 5
 stty -F $AT_CMUX clocal
 
 echo "Connect and wait for PPP link..."

app/src/sm_cmux.c

@@ -302,6 +302,11 @@ static bool cmux_is_started(void)
 	return (cmux.uart_pipe != NULL);
 }
 
+bool sm_cmux_is_started(void)
+{
+	return cmux_is_started();
+}
+
 void sm_cmux_init(void)
 {
 	const struct modem_cmux_config cmux_config = {
@@ -370,6 +375,18 @@ static struct cmux_dlci *cmux_get_dlci(enum cmux_channel channel)
 		return &cmux.dlcis[!cmux.at_channel];
 	}
 #endif
+#if defined(CONFIG_NRF_MODEM_LIB_TRACE_BACKEND_CMUX)
+	if (channel == CMUX_COREDUMP_CHANNEL) {
+#if defined(CONFIG_SM_GNSS_OUTPUT_NMEA_ON_CMUX_CHANNEL)
+		/* The second to last DLCI. */
+		return &cmux.dlcis[CHANNEL_COUNT - 2];
+#else
+		/* The last DLCI. */
+		return &cmux.dlcis[CHANNEL_COUNT - 1];
+#endif
+	}
+#endif
+
 #if defined(CONFIG_SM_GNSS_OUTPUT_NMEA_ON_CMUX_CHANNEL)
 	if (channel == CMUX_GNSS_CHANNEL) {
 		/* The last DLCI. */

app/src/sm_cmux.h

@@ -21,12 +21,15 @@ enum cmux_channel {
 #if defined(CONFIG_SM_PPP)
 	CMUX_PPP_CHANNEL,
 #endif
+#if defined(CONFIG_NRF_MODEM_LIB_TRACE_BACKEND_CMUX)
+	CMUX_COREDUMP_CHANNEL,
+#endif
 #if defined(CONFIG_SM_GNSS_OUTPUT_NMEA_ON_CMUX_CHANNEL)
 	CMUX_GNSS_CHANNEL,
 #endif
 	CMUX_EXT_CHANNEL_COUNT
 };
 struct modem_pipe *sm_cmux_reserve(enum cmux_channel);
 void sm_cmux_release(enum cmux_channel);
-
+bool sm_cmux_is_started(void);
 #endif

drivers/CMakeLists.txt

@@ -3,3 +3,4 @@
 # SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
 
 add_subdirectory_ifdef(CONFIG_DTR_UART dtr_uart)
+add_subdirectory_ifdef(CONFIG_NRF_MODEM_LIB_TRACE_BACKEND_CMUX trace_backend_cmux)

drivers/Kconfig

@@ -3,6 +3,20 @@
 #
 # SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
 
+if NRF_MODEM_LIB_TRACE
+
+choice NRF_MODEM_LIB_TRACE_BACKEND
+
+config NRF_MODEM_LIB_TRACE_BACKEND_CMUX
+	bool "Stream modem coredump to CMUX"
+	depends on SM_CMUX
+	help
+	  Use CMUX trace backend for modem coredump.
+
+endchoice # NRF_MODEM_LIB_TRACE_BACKEND
+
+endif
+
 menu "Drivers"
 rsource "dtr_uart/Kconfig"
 endmenu

drivers/trace_backend_cmux/CMakeLists.txt

@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2025 Nordic Semiconductor ASA
+#
+# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
+#
+
+zephyr_library()
+zephyr_library_sources(cmux.c)

drivers/trace_backend_cmux/cmux.c

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2025 Nordic Semiconductor ASA
+ *
+ * SPDX-License-Identifier: LicenseRef-Nordic-5-Clause
+ */
+
+#include <zephyr/kernel.h>
+#include <zephyr/logging/log.h>
+#include <zephyr/modem/pipe.h>
+#include <modem/nrf_modem_lib_trace.h>
+#include <modem/trace_backend.h>
+
+#include "sm_cmux.h"
+
+LOG_MODULE_REGISTER(modem_trace_backend, CONFIG_MODEM_TRACE_BACKEND_LOG_LEVEL);
+
+static K_SEM_DEFINE(tx_idle_sem, 0, 1);
+
+static trace_backend_processed_cb trace_processed_callback;
+
+static struct modem_pipe *pipe;
+
+static void modem_pipe_event_handler(struct modem_pipe *pipe,
+				     enum modem_pipe_event event, void *user_data)
+{
+	switch (event) {
+	case MODEM_PIPE_EVENT_OPENED:
+		LOG_DBG("Coredump DLCI opened.");
+		break;
+
+	case MODEM_PIPE_EVENT_CLOSED:
+		LOG_DBG("Coredump DLCI closed.");
+		break;
+
+	case MODEM_PIPE_EVENT_TRANSMIT_IDLE:
+		k_sem_give(&tx_idle_sem);
+		break;
+
+	default:
+		break;
+	}
+}
+
+int trace_backend_init(trace_backend_processed_cb trace_processed_cb)
+{
+	if (trace_processed_cb == NULL) {
+		return -EFAULT;
+	}
+
+	trace_processed_callback = trace_processed_cb;
+
+	return 0;
+}
+
+int trace_backend_deinit(void)
+{
+	if (pipe != NULL) {
+		modem_pipe_release(pipe);
+		sm_cmux_release(CMUX_COREDUMP_CHANNEL, false);
+		pipe = NULL;
+	}
+	return 0;
+}
+
+int trace_backend_write(const void *data, size_t len)
+{
+	size_t sent_len = 0;
+	int ret = 0;
+
+	if (!sm_cmux_is_started()) {
+		/* Drop the trace data */
+		trace_processed_callback(len);
+		return len;
+	}
+
+	if (pipe == NULL) {
+		pipe = sm_cmux_reserve(CMUX_COREDUMP_CHANNEL);
+		modem_pipe_attach(pipe, modem_pipe_event_handler, NULL);
+	}
+
+	const uint8_t *buf = (const uint8_t *)data;
+
+	while (sent_len < len) {
+		k_sem_take(&tx_idle_sem, K_FOREVER);
+		ret = modem_pipe_transmit(pipe, buf, len - sent_len);
+		if (ret <= 0) {
+			break;
+		}
+
+		trace_processed_callback(ret);
+
+		sent_len += ret;
+		buf += ret;
+	}
+
+	if (ret == 0) {
+		/* Retry by trace thread in modem lib */
+		return -EAGAIN;
+	} else if (ret < 0) {
+		LOG_INF("Coredump: Sent %u out of %u bytes. (%d)", sent_len, len, ret);
+		return ret;
+	}
+
+	return sent_len;
+}
+
+struct nrf_modem_lib_trace_backend trace_backend = {
+	.init = trace_backend_init,
+	.deinit = trace_backend_deinit,
+	.write = trace_backend_write,
+};