From 3c1590774010143b5267b85f6de4227576c7bf44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simen=20S=2E=20R=C3=B8stad?= Date: Wed, 22 Jan 2025 14:37:00 +0100 Subject: [PATCH] modules: memfault: Add automatic sending of coredumps over LTE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add automatic sending of coredumps over LTE: - Add new file memfault_lte_coredump.c that uses LTE cereg and PDN to determine if the device is connected to the network. If connected, the layer will trigger sending of a stored coredump. Library is implemented with retry logic and backoff. - Add overlay file to SLM that enables Memfault features - Update Memfault sample to use the new coredump feature. - Add missing features to Memfault sample - Enable assertions by default in SLM, there is no reason why it should be enabled as long as there is space. Signed-off-by: Simen S. Røstad --- .../serial_lte_modem/doc/slm_description.rst | 3 + .../serial_lte_modem/overlay-memfault.conf | 28 + applications/serial_lte_modem/prj.conf | 3 +- applications/serial_lte_modem/sample.yaml | 23 + .../src/lwm2m_carrier/slm_at_carrier.c | 5 +- .../serial_lte_modem/src/slm_at_commands.c | 15 +- doc/nrf/libraries/debug/memfault_ncs.rst | 14 + .../releases/release-notes-changelog.rst | 6 +- modules/memfault-firmware-sdk/CMakeLists.txt | 4 + modules/memfault-firmware-sdk/Kconfig | 37 ++ .../memfault_lte_coredump.c | 502 ++++++++++++++++++ .../memfault/boards/nrf9151dk_nrf9151_ns.conf | 3 + .../memfault/boards/nrf9160dk_nrf9160_ns.conf | 3 + .../memfault/boards/nrf9161dk_nrf9161_ns.conf | 3 + .../memfault/boards/thingy91_nrf9160_ns.conf | 3 + .../memfault/boards/thingy91x_nrf9151_ns.conf | 4 + samples/debug/memfault/prj.conf | 7 +- samples/debug/memfault/src/main.c | 5 + 18 files changed, 656 insertions(+), 12 deletions(-) create mode 100644 applications/serial_lte_modem/overlay-memfault.conf create mode 100644 modules/memfault-firmware-sdk/memfault_lte_coredump.c diff --git a/applications/serial_lte_modem/doc/slm_description.rst b/applications/serial_lte_modem/doc/slm_description.rst index 1f1ff3c352ec..75833af73261 100644 --- a/applications/serial_lte_modem/doc/slm_description.rst +++ b/applications/serial_lte_modem/doc/slm_description.rst @@ -336,6 +336,9 @@ The following configuration files are provided: It can be customized to fit your configuration (UART, baud rate, and so on). By default, it sets the baud rate of the PPP UART to 1 000 000. +* :file:`overlay-memfault.conf` - Configuration file that enables `Memfault`_. + For more information about Memfault features in |NCS|, see :ref:`mod_memfault`. + * :file:`overlay-zephyr-modem.conf`, :file:`overlay-zephyr-modem-external-mcu.conf`, :file:`overlay-zephyr-modem-nrf9160dk-nrf52840.conf`, :file:`overlay-external-mcu.overlay`, and :file:`overlay-zephyr-modem-nrf9160dk-nrf52840.overlay` - These configuration files are used when compiling SLM to turn an nRF91 Series SiP into a Zephyr-compatible standalone modem. See :ref:`slm_as_zephyr_modem` for more information. diff --git a/applications/serial_lte_modem/overlay-memfault.conf b/applications/serial_lte_modem/overlay-memfault.conf new file mode 100644 index 000000000000..71c70ae4a664 --- /dev/null +++ b/applications/serial_lte_modem/overlay-memfault.conf @@ -0,0 +1,28 @@ +# +# Copyright (c) 2025 Nordic Semiconductor +# +# SPDX-License-Identifier: LicenseRef-Nordic-5-Clause +# + +CONFIG_MEMFAULT=y +CONFIG_MEMFAULT_HTTP_PERIODIC_UPLOAD=y +CONFIG_MODEM_KEY_MGMT=y +CONFIG_MEMFAULT_LOGGING_ENABLE=y +CONFIG_MEMFAULT_HTTP_ENABLE=y +CONFIG_MEMFAULT_NCS_LTE_METRICS=y +CONFIG_MEMFAULT_NCS_STACK_METRICS=y +CONFIG_MEMFAULT_NCS_INTERNAL_FLASH_BACKED_COREDUMP=y +CONFIG_MEMFAULT_NCS_DEVICE_ID_IMEI=y +CONFIG_MEMFAULT_LOGGING_RAM_SIZE=4096 +CONFIG_MEMFAULT_HEAP_STATS=y +CONFIG_MEMFAULT_HTTP_DEDICATED_WORKQUEUE_STACK_SIZE=1560 +CONFIG_MEMFAULT_COREDUMP_FULL_THREAD_STACKS=y +CONFIG_MEMFAULT_EVENT_STORAGE_SIZE=2048 +CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y +CONFIG_PDN=y +CONFIG_LTE_LINK_CONTROL=y +CONFIG_LTE_LC_EDRX_MODULE=y +CONFIG_LTE_LC_PSM_MODULE=y + +# Memfault depends on POSIX, disable unneeded POSIX features +CONFIG_POSIX_FILE_SYSTEM=n diff --git a/applications/serial_lte_modem/prj.conf b/applications/serial_lte_modem/prj.conf index 222c1d3acd9e..fc9ac692ad6a 100644 --- a/applications/serial_lte_modem/prj.conf +++ b/applications/serial_lte_modem/prj.conf @@ -5,6 +5,8 @@ # # General config CONFIG_LOG=y +CONFIG_ASSERT=y +CONFIG_ASSERT_VERBOSE=n CONFIG_LOG_DEFAULT_LEVEL=3 CONFIG_STACK_SENTINEL=y CONFIG_PICOLIBC_IO_FLOAT=y @@ -118,7 +120,6 @@ CONFIG_SLM_CUSTOMER_VERSION="" CONFIG_SLM_EXTERNAL_XTAL=n # debug options -#CONFIG_ASSERT=y #CONFIG_LOG_BUFFER_SIZE=16384 #CONFIG_SLM_LOG_LEVEL_DBG=y #CONFIG_LOG_PRINTK=n diff --git a/applications/serial_lte_modem/sample.yaml b/applications/serial_lte_modem/sample.yaml index c7da81025ba6..125179a04cab 100644 --- a/applications/serial_lte_modem/sample.yaml +++ b/applications/serial_lte_modem/sample.yaml @@ -21,6 +21,29 @@ tests: tags: - ci_build - sysbuild + applications.serial_lte_modem.memfault: + sysbuild: true + build_only: true + extra_args: + - EXTRA_CONF_FILE=overlay-memfault.conf + - CONFIG_MEMFAULT_NCS_PROJECT_KEY="dummy-key" + platform_allow: + - nrf9160dk/nrf9160/ns + - nrf9161dk/nrf9161/ns + - nrf9151dk/nrf9151/ns + - nrf9131ek/nrf9131/ns + - thingy91/nrf9160/ns + - thingy91x/nrf9151/ns + integration_platforms: + - nrf9160dk/nrf9160/ns + - nrf9161dk/nrf9161/ns + - nrf9151dk/nrf9151/ns + - nrf9131ek/nrf9131/ns + - thingy91/nrf9160/ns + - thingy91x/nrf9151/ns + tags: + - ci_build + - sysbuild applications.serial_lte_modem.native_tls: sysbuild: true build_only: true diff --git a/applications/serial_lte_modem/src/lwm2m_carrier/slm_at_carrier.c b/applications/serial_lte_modem/src/lwm2m_carrier/slm_at_carrier.c index 78d64f37fde0..5824e11028db 100644 --- a/applications/serial_lte_modem/src/lwm2m_carrier/slm_at_carrier.c +++ b/applications/serial_lte_modem/src/lwm2m_carrier/slm_at_carrier.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -51,7 +52,7 @@ static void print_err(const lwm2m_carrier_event_t *evt) "Connection failure", }; - __ASSERT(PART_OF_ARRAY(strerr[err->type]), + __ASSERT(PART_OF_ARRAY(strerr, strerr[err->type]), "Unhandled liblwm2m_carrier error"); LOG_ERR("LWM2M_CARRIER_EVENT_ERROR: %s, reason %d", strerr[err->type], err->value); @@ -87,7 +88,7 @@ static void print_deferred(const lwm2m_carrier_event_t *evt) "Waiting for SIM MSISDN", }; - __ASSERT(PART_OF_ARRAY(strdef[def->reason]), + __ASSERT(PART_OF_ARRAY(strdef, strdef[def->reason]), "Unhandled deferred reason"); LOG_INF("LWM2M_CARRIER_EVENT_DEFERRED: reason %s, timeout %d seconds", diff --git a/applications/serial_lte_modem/src/slm_at_commands.c b/applications/serial_lte_modem/src/slm_at_commands.c index f5c42d0b08b8..d2155d0cbbb8 100644 --- a/applications/serial_lte_modem/src/slm_at_commands.c +++ b/applications/serial_lte_modem/src/slm_at_commands.c @@ -81,7 +81,7 @@ enum sleep_modes { static struct { struct k_work_delayable work; uint32_t mode; -} sleep; +} sleep_control; #endif bool verify_datamode_control(uint16_t time_limit, uint16_t *time_limit_min); @@ -130,13 +130,13 @@ static int handle_at_slmver(enum at_parser_cmd_type cmd_type, struct at_parser * static void go_sleep_wk(struct k_work *) { - if (sleep.mode == SLEEP_MODE_IDLE) { + if (sleep_control.mode == SLEEP_MODE_IDLE) { if (slm_at_host_power_off() == 0) { slm_enter_idle(); } else { LOG_ERR("failed to power off UART"); } - } else if (sleep.mode == SLEEP_MODE_DEEP) { + } else if (sleep_control.mode == SLEEP_MODE_DEEP) { slm_enter_sleep(); } } @@ -148,12 +148,13 @@ static int handle_at_sleep(enum at_parser_cmd_type cmd_type, struct at_parser *p int ret = -EINVAL; if (cmd_type == AT_PARSER_CMD_TYPE_SET) { - ret = at_parser_num_get(parser, 1, &sleep.mode); + ret = at_parser_num_get(parser, 1, &sleep_control.mode); if (ret) { return -EINVAL; } - if (sleep.mode == SLEEP_MODE_DEEP || sleep.mode == SLEEP_MODE_IDLE) { - k_work_reschedule(&sleep.work, SLM_UART_RESPONSE_DELAY); + if (sleep_control.mode == SLEEP_MODE_DEEP || + sleep_control.mode == SLEEP_MODE_IDLE) { + k_work_reschedule(&sleep_control.work, SLM_UART_RESPONSE_DELAY); } else { ret = -EINVAL; } @@ -376,7 +377,7 @@ int slm_at_init(void) int err; #if POWER_PIN_IS_ENABLED - k_work_init_delayable(&sleep.work, go_sleep_wk); + k_work_init_delayable(&sleep_control.work, go_sleep_wk); #endif err = slm_at_tcp_proxy_init(); diff --git a/doc/nrf/libraries/debug/memfault_ncs.rst b/doc/nrf/libraries/debug/memfault_ncs.rst index 05410fbf0468..3f949d4a9ea4 100644 --- a/doc/nrf/libraries/debug/memfault_ncs.rst +++ b/doc/nrf/libraries/debug/memfault_ncs.rst @@ -58,6 +58,19 @@ To have these configuration files in the include path, add the following to the .. memfault_config_files_end +Automatic sending of coredumps to Memfault +========================================== + +To post a stored coredump from a previous crash to Memfault upon network connection, set the :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED` Kconfig option to ``y``. +The option is only supported for nRF91 Series devices. + +The library has built-in connection awareness and tries to post the coredump to a maximum of the number set in the :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRIES_MAX` Kconfig option, at an interval of the time set in the :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRY_INTERVAL_SECONDS` Kconfig option between each attempt. +If unsuccessful within the number of attempts, the library gives up. +If at any point the network is lost during the retry process, the library waits for the device to reconnect before restarting the retry process. + +This feature is useful when you want to post the coredump as soon as possible after a crash and it is not desirable to wait for the next periodic upload set by :kconfig:option:`CONFIG_MEMFAULT_HTTP_PERIODIC_UPLOAD_INTERVAL_SECS`. +Alternatively, you can manually trigger the coredump upload by calling the :c:func:`memfault_zephyr_port_post_data` function. +You can use the :c:func:`memfault_coredump_has_valid_coredump` function to check whether a coredump is available. Configuration options in Memfault SDK ===================================== @@ -93,6 +106,7 @@ Configuration options in |NCS| The Kconfig options for Memfault that are defined in |NCS| provide some additional features compared to the options that are already implemented in Memfault SDK: +* :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED` * :kconfig:option:`CONFIG_MEMFAULT_NCS_PROJECT_KEY` * :kconfig:option:`CONFIG_MEMFAULT_NCS_PROVISION_CERTIFICATES` * :kconfig:option:`CONFIG_MEMFAULT_NCS_INTERNAL_FLASH_BACKED_COREDUMP` diff --git a/doc/nrf/releases_and_maturity/releases/release-notes-changelog.rst b/doc/nrf/releases_and_maturity/releases/release-notes-changelog.rst index 2c40a21b3ae8..9cc225164c5c 100644 --- a/doc/nrf/releases_and_maturity/releases/release-notes-changelog.rst +++ b/doc/nrf/releases_and_maturity/releases/release-notes-changelog.rst @@ -236,6 +236,8 @@ nRF Machine Learning (Edge Impulse) Serial LTE modem ---------------- +* Added an overlay :file:`overlay-memfault.conf` file to enable Memfault. + For more information about Memfault features in |NCS|, see :ref:`mod_memfault`. * Updated the application to use the :ref:`lib_downloader` library instead of the deprecated :ref:`lib_download_client` library. Thingy:53: Matter weather station @@ -651,7 +653,9 @@ Edge Impulse integration Memfault integration -------------------- -|no_changes_yet_note| +* Added a new feature to automatically post coredumps to Memfault when network connectivity is available. + To enable this feature, set the :kconfig:option:`CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED` Kconfig option to ``y``. + Only supported for nRF91 Series devices. AVSystem integration -------------------- diff --git a/modules/memfault-firmware-sdk/CMakeLists.txt b/modules/memfault-firmware-sdk/CMakeLists.txt index b0d972ef529a..79ed87400217 100644 --- a/modules/memfault-firmware-sdk/CMakeLists.txt +++ b/modules/memfault-firmware-sdk/CMakeLists.txt @@ -31,6 +31,10 @@ zephyr_library_sources_ifdef( CONFIG_MEMFAULT_NCS_BT_METRICS memfault_bt_metrics.c) +zephyr_library_sources_ifdef( + CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED + memfault_lte_coredump.c) + zephyr_library_sources_ifdef( CONFIG_MEMFAULT_NCS_ETB_CAPTURE memfault_etb_trace_capture.c) diff --git a/modules/memfault-firmware-sdk/Kconfig b/modules/memfault-firmware-sdk/Kconfig index 3b8137b8b480..469a67ba4ad3 100644 --- a/modules/memfault-firmware-sdk/Kconfig +++ b/modules/memfault-firmware-sdk/Kconfig @@ -191,6 +191,43 @@ config MEMFAULT_NCS_IMPLEMENT_METRICS_COLLECTION Implement the Memfault 'memfault_metrics_heartbeat_collect_data()' function for the selected metrics. Disable this to override the implementation. +config MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED + bool "Post coredump on network connected" + depends on PDN + depends on LTE_LINK_CONTROL + depends on SOC_SERIES_NRF91X + select SMF + select SMF_ANCESTOR_SUPPORT + select SMF_INITIAL_TRANSITION + help + Post coredump to Memfault when the device is connected to LTE network. + This option is only supported for nRF91 targets. + +config MEMFAULT_NCS_POST_COREDUMP_AFTER_INITIAL_DELAY + bool "Post coredump after initial delay" + default y + help + Delay coredump posting to Memfault to avoid conflicts with the application's + TLS connection setup that occurs typically LTE network is obtained. + The delay duration is defined by CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRY_INTERVAL_SECONDS. +#§ +config MEMFAULT_NCS_POST_COREDUMP_RETRIES_MAX + int "Max number of coredump post retries" + default 3 + default 4 if MEMFAULT_NCS_POST_COREDUMP_AFTER_INITIAL_WAIT + help + Maximum number of retries to post a coredump to Memfault. + +config MEMFAULT_NCS_POST_COREDUMP_RETRY_INTERVAL_SECONDS + int "Retry interval in seconds" + default 30 + help + Interval in seconds between coredump post retries. + +config MEMFAULT_NCS_POST_COREDUMP_THREAD_STACK_SIZE + int "Post coredump thread size" + default 512 + config MEMFAULT_NCS_ETB_CAPTURE bool "Enable ETB trace capture" depends on ETB_TRACE diff --git a/modules/memfault-firmware-sdk/memfault_lte_coredump.c b/modules/memfault-firmware-sdk/memfault_lte_coredump.c new file mode 100644 index 000000000000..b05b0daf720c --- /dev/null +++ b/modules/memfault-firmware-sdk/memfault_lte_coredump.c @@ -0,0 +1,502 @@ +/* + * Copyright (c) 2025 Nordic Semiconductor ASA + * + * SPDX-License-Identifier: LicenseRef-Nordic-5-Clause + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +LOG_MODULE_DECLARE(memfault_ncs, CONFIG_MEMFAULT_NCS_LOG_LEVEL); + +NRF_MODEM_LIB_ON_INIT(memfault_ncs_lte_coredump_init_hook, on_modem_lib_init, NULL); +NRF_MODEM_LIB_ON_SHUTDOWN(memfault_ncs_lte_coredump_shutdown_hook, on_modem_lib_shutdown, NULL); + +#define MSG_QUEUE_ENTRY_COUNT 4 +#define MSG_QUEUE_BYTE_ALIGNMENT 4 + +enum library_state { + STATE_RUNNING, + STATE_WAITING_FOR_NRF_MODEM_LIB_INIT, + STATE_WAITING_FOR_NETWORK_CONNECTION, + STATE_NETWORK_CONNECTED, + STATE_COREDUMP_SEND_ATTEMPT, + STATE_COREDUMP_SEND_BACKOFF, + STATE_FINISHED, + STATE_ERROR +}; + +enum library_event { + EVENT_NRF_MODEM_LIB_INITED, + EVENT_NRF_MODEM_LIB_SHUTDOWN, + EVENT_LTE_REGISTERED, + EVENT_LTE_DEREGISTERED, + EVENT_PDN_ACTIVATED, + EVENT_PDN_DEACTIVATED, + EVENT_NETWORK_CONNECTED, + EVENT_NETWORK_DISCONNECTED, + EVENT_BACKOFF_TIMER_EXPIRED, + EVENT_COREDUMP_SEND_FAIL, + EVENT_COREDUMP_SEND_SUCCESS, + EVENT_ERROR +}; + +K_MSGQ_DEFINE(mflt_lte_coredump_queue, + sizeof(enum library_event), + MSG_QUEUE_ENTRY_COUNT, + MSG_QUEUE_BYTE_ALIGNMENT); + +struct fsm_state_object { + /* This must be first */ + struct smf_ctx ctx; + + enum library_event event; + + uint8_t retry_count; + + struct k_work_delayable backoff_work; + + bool lte_registered; + bool pdn_active; +}; + +/* Forward declarations: Local functions */ +static void pdn_event_handler(uint8_t cid, enum pdn_event event, int reason); +static void lte_handler(const struct lte_lc_evt *const evt); +static void on_modem_lib_init(int ret, void *ctx); +static void on_modem_lib_shutdown(void *ctx); +static void event_send(enum library_event event); +static void schedule_send_backoff(struct fsm_state_object *state_object); + +/* Forward declarations: State Machine Functions */ +static void state_running_entry(void *o); +static void state_running_run(void *o); +static void state_running_exit(void *o); +static void state_waiting_for_nrf_modem_lib_init_entry(void *o); +static void state_waiting_for_nrf_modem_lib_init_run(void *o); +static void state_waiting_for_network_connection_run(void *o); +static void state_network_connected_entry(void *o); +static void state_network_connected_run(void *o); +static void state_coredump_send_attempt_entry(void *o); +static void state_coredump_send_attempt_run(void *o); +static void state_coredump_send_backoff_entry(void *o); +static void state_coredump_send_backoff_run(void *o); +static void state_finished_entry(void *o); +static void state_error_entry(void *o); + +static const struct smf_state states[] = { + [STATE_WAITING_FOR_NRF_MODEM_LIB_INIT] = + SMF_CREATE_STATE(state_waiting_for_nrf_modem_lib_init_entry, + state_waiting_for_nrf_modem_lib_init_run, + NULL, + NULL, + NULL), + [STATE_RUNNING] = + SMF_CREATE_STATE(state_running_entry, + state_running_run, + state_running_exit, + NULL, + &states[STATE_WAITING_FOR_NETWORK_CONNECTION]), + [STATE_WAITING_FOR_NETWORK_CONNECTION] = + SMF_CREATE_STATE(NULL, + state_waiting_for_network_connection_run, + NULL, + &states[STATE_RUNNING], + NULL), + [STATE_NETWORK_CONNECTED] = + SMF_CREATE_STATE(state_network_connected_entry, + state_network_connected_run, + NULL, + &states[STATE_RUNNING], +#if defined(CONFIG_MEMFAULT_NCS_POST_COREDUMP_AFTER_INITIAL_DELAY) + &states[STATE_COREDUMP_SEND_BACKOFF]), +#else + &states[STATE_COREDUMP_SEND_ATTEMPT]), +#endif /* CONFIG_MEMFAULT_NCS_POST_COREDUMP_AFTER_INITIAL_DELAY */ + [STATE_COREDUMP_SEND_ATTEMPT] = + SMF_CREATE_STATE(state_coredump_send_attempt_entry, + state_coredump_send_attempt_run, + NULL, + &states[STATE_NETWORK_CONNECTED], + NULL), + [STATE_COREDUMP_SEND_BACKOFF] = + SMF_CREATE_STATE(state_coredump_send_backoff_entry, + state_coredump_send_backoff_run, + NULL, + &states[STATE_NETWORK_CONNECTED], + NULL), + [STATE_FINISHED] = + SMF_CREATE_STATE(state_finished_entry, + NULL, + NULL, + NULL, + NULL), + [STATE_ERROR] = + SMF_CREATE_STATE(state_error_entry, + NULL, + NULL, + NULL, + NULL) +}; + +static void timer_work_fn(struct k_work *work) +{ + event_send(EVENT_BACKOFF_TIMER_EXPIRED); +} + +static struct fsm_state_object state_object = { 0 }; + +/* nRF Modem Library Handlers */ + +static void on_modem_lib_init(int ret, void *ctx) +{ + event_send(EVENT_NRF_MODEM_LIB_INITED); +} + +static void on_modem_lib_shutdown(void *ctx) +{ + event_send(EVENT_NRF_MODEM_LIB_SHUTDOWN); +} + +/* Callback Handlers for Link Controller and PDN Libraries */ + +static void lte_handler(const struct lte_lc_evt *const evt) +{ + if (evt->type == LTE_LC_EVT_NW_REG_STATUS) { + switch (evt->nw_reg_status) { + case LTE_LC_NW_REG_REGISTERED_HOME: + __fallthrough; + case LTE_LC_NW_REG_REGISTERED_ROAMING: + event_send(EVENT_LTE_REGISTERED); + break; + case LTE_LC_NW_REG_UNKNOWN: + __fallthrough; + case LTE_LC_NW_REG_UICC_FAIL: + __fallthrough; + case LTE_LC_NW_REG_NOT_REGISTERED: + __fallthrough; + case LTE_LC_NW_REG_REGISTRATION_DENIED: + event_send(EVENT_LTE_DEREGISTERED); + default: + /* Don't care */ + break; + } + } +} + +static void pdn_event_handler(uint8_t cid, enum pdn_event event, int reason) +{ + switch (event) { + case PDN_EVENT_ACTIVATED: + event_send(EVENT_PDN_ACTIVATED); + break; + case PDN_EVENT_NETWORK_DETACH: + __fallthrough; + case PDN_EVENT_DEACTIVATED: + event_send(EVENT_PDN_DEACTIVATED); + break; + default: + /* Don't care */ + break; + } +} + +/* SMF State Handlers */ + +static void state_waiting_for_nrf_modem_lib_init_entry(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_waiting_for_nrf_modem_lib_init_entry"); + + state_object->lte_registered = false; + state_object->pdn_active = false; + state_object->retry_count = 0; +} + +static void state_waiting_for_nrf_modem_lib_init_run(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_waiting_for_nrf_modem_lib_init_run"); + + if (state_object->event == EVENT_NRF_MODEM_LIB_INITED) { + smf_set_state(SMF_CTX(state_object), &states[STATE_RUNNING]); + } +} + +static void state_running_entry(void *o) +{ + int err; + + ARG_UNUSED(o); + + LOG_DBG("state_running_entry"); + + /* Subscribe to +CEREG notifications, level 5 */ + err = nrf_modem_at_printf("AT+CEREG=5"); + if (err) { + LOG_ERR("nrf_modem_at_printf, error: %d", err); + event_send(EVENT_ERROR); + return; + } + + lte_lc_register_handler(lte_handler); + + err = pdn_default_ctx_cb_reg(pdn_event_handler); + if (err) { + LOG_ERR("pdn_default_ctx_cb_reg, error: %d", err); + event_send(EVENT_ERROR); + return; + } +} + +static void state_running_run(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_running_run"); + + if (state_object->event == EVENT_NRF_MODEM_LIB_SHUTDOWN) { + smf_set_state(SMF_CTX(state_object), &states[STATE_WAITING_FOR_NRF_MODEM_LIB_INIT]); + } else if (state_object->event == EVENT_ERROR) { + smf_set_state(SMF_CTX(state_object), &states[STATE_ERROR]); + } +} + +static void state_running_exit(void *o) +{ + int err; + + struct fsm_state_object *state_object = o; + + LOG_DBG("state_running_exit"); + + err = lte_lc_deregister_handler(lte_handler); + if (err) { + LOG_ERR("lte_lc_deregister_handler, error: %d", err); + event_send(EVENT_ERROR); + return; + } + + err = pdn_default_ctx_cb_dereg(pdn_event_handler); + if (err) { + LOG_ERR("pdn_default_ctx_cb_dereg, error: %d", err); + event_send(EVENT_ERROR); + return; + } + + (void)k_work_cancel_delayable(&state_object->backoff_work); +} + +static void state_waiting_for_network_connection_run(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_waiting_for_network_connection_run"); + + switch (state_object->event) { + case EVENT_LTE_REGISTERED: + state_object->lte_registered = true; + break; + case EVENT_LTE_DEREGISTERED: + state_object->lte_registered = false; + break; + case EVENT_PDN_ACTIVATED: + state_object->pdn_active = true; + break; + case EVENT_PDN_DEACTIVATED: + state_object->pdn_active = false; + break; + default: + /* Don't care */ + break; + } + + if (state_object->lte_registered && state_object->pdn_active) { + smf_set_state(SMF_CTX(state_object), &states[STATE_NETWORK_CONNECTED]); + } +} + +static void state_network_connected_entry(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_network_connected_entry"); + + state_object->retry_count = 0; +} + +static void state_network_connected_run(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_network_connected_run"); + + if (state_object->event == EVENT_LTE_DEREGISTERED) { + state_object->lte_registered = false; + + smf_set_state(SMF_CTX(state_object), &states[STATE_WAITING_FOR_NETWORK_CONNECTION]); + } else if (state_object->event == EVENT_PDN_DEACTIVATED) { + state_object->pdn_active = false; + + smf_set_state(SMF_CTX(state_object), &states[STATE_WAITING_FOR_NETWORK_CONNECTION]); + } +} + +static void state_coredump_send_attempt_entry(void *o) +{ + int err; + + ARG_UNUSED(o); + + LOG_DBG("state_coredump_send_attempt_entry"); + LOG_DBG("Triggering heartbeat"); + LOG_DBG("Attempting to send coredump"); + + memfault_metrics_heartbeat_debug_trigger(); + + err = memfault_zephyr_port_post_data(); + if (err) { + LOG_DBG("Failed to post data to Memfault"); + event_send(EVENT_COREDUMP_SEND_FAIL); + } else { + LOG_DBG("Succeeded posting data to Memfault"); + event_send(EVENT_COREDUMP_SEND_SUCCESS); + } +} + +static void state_coredump_send_attempt_run(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_coredump_send_attempt_run"); + + if (state_object->event == EVENT_COREDUMP_SEND_FAIL) { + smf_set_state(SMF_CTX(state_object), &states[STATE_COREDUMP_SEND_BACKOFF]); + } else if (state_object->event == EVENT_COREDUMP_SEND_SUCCESS) { + smf_set_state(SMF_CTX(state_object), &states[STATE_FINISHED]); + } +} + +static void state_coredump_send_backoff_entry(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_coredump_send_backoff_entry"); + + schedule_send_backoff(state_object); +} + +static void state_coredump_send_backoff_run(void *o) +{ + struct fsm_state_object *state_object = o; + + LOG_DBG("state_coredump_send_backoff_run"); + + if (state_object->event == EVENT_BACKOFF_TIMER_EXPIRED) { + smf_set_state(SMF_CTX(state_object), &states[STATE_COREDUMP_SEND_ATTEMPT]); + } +} + +static void state_finished_entry(void *o) +{ + ARG_UNUSED(o); + + LOG_DBG("state_finished_entry"); +} + +static void state_error_entry(void *o) +{ + ARG_UNUSED(o); + + LOG_DBG("state_error_entry"); + + __ASSERT_NO_MSG(false); +} + +/* Local Convenience Functions */ + +static void event_send(enum library_event event) +{ + int ret; + enum library_event event_new = event; + + ret = k_msgq_put(&mflt_lte_coredump_queue, &event_new, K_NO_WAIT); + if (ret) { + LOG_ERR("k_msgq_put, error: %d", ret); + __ASSERT_NO_MSG(false); + return; + } +} + +static void schedule_send_backoff(struct fsm_state_object *state_object) +{ + state_object->retry_count++; + + if (state_object->retry_count >= CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRIES_MAX) { + event_send(STATE_FINISHED); + return; + } + + LOG_DBG("Retry number: %d", state_object->retry_count); + LOG_DBG("Scheduling backoff, next attempt in %d seconds", + CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRY_INTERVAL_SECONDS); + + (void)k_work_schedule(&state_object->backoff_work, + K_SECONDS(CONFIG_MEMFAULT_NCS_POST_COREDUMP_RETRY_INTERVAL_SECONDS)); +} + +/* Library thread */ + +static void mflt_lte_coredump_fn(void) +{ + int err; + + k_work_init_delayable(&state_object.backoff_work, timer_work_fn); + + bool has_coredump = memfault_coredump_has_valid_coredump(NULL); + + if (has_coredump) { + LOG_DBG("Coredump found"); + smf_set_initial(SMF_CTX(&state_object), + &states[STATE_WAITING_FOR_NRF_MODEM_LIB_INIT]); + } else { + LOG_DBG("No coredump found"); + smf_set_initial(SMF_CTX(&state_object), + &states[STATE_FINISHED]); + + return; + } + + while (1) { + err = k_msgq_get(&mflt_lte_coredump_queue, &state_object.event, K_FOREVER); + if (err) { + LOG_ERR("k_msgq_get, error: %d", err); + __ASSERT_NO_MSG(false); + return; + } + + err = smf_run_state(SMF_CTX(&state_object)); + if (err) { + LOG_ERR("smf_run_state, error: %d", err); + __ASSERT_NO_MSG(false); + return; + } + } +} + +K_THREAD_DEFINE(mflt_lte_coredump_thread_id, 4096, + mflt_lte_coredump_fn, NULL, NULL, NULL, + K_LOWEST_APPLICATION_THREAD_PRIO, 0, 0); diff --git a/samples/debug/memfault/boards/nrf9151dk_nrf9151_ns.conf b/samples/debug/memfault/boards/nrf9151dk_nrf9151_ns.conf index b7395a7e1d90..1b0e2ceca52f 100644 --- a/samples/debug/memfault/boards/nrf9151dk_nrf9151_ns.conf +++ b/samples/debug/memfault/boards/nrf9151dk_nrf9151_ns.conf @@ -10,6 +10,9 @@ CONFIG_NET_SOCKETS_OFFLOAD=y # Memfault CONFIG_MEMFAULT_NCS_LTE_METRICS=y +CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y +CONFIG_LTE_LC_EDRX_MODULE=y +CONFIG_LTE_LC_PSM_MODULE=y CONFIG_MEMFAULT_NCS_DEVICE_ID_IMEI=y CONFIG_MODEM_INFO=y diff --git a/samples/debug/memfault/boards/nrf9160dk_nrf9160_ns.conf b/samples/debug/memfault/boards/nrf9160dk_nrf9160_ns.conf index 8b9b2695cf42..53c1b66a3b7f 100644 --- a/samples/debug/memfault/boards/nrf9160dk_nrf9160_ns.conf +++ b/samples/debug/memfault/boards/nrf9160dk_nrf9160_ns.conf @@ -10,6 +10,9 @@ CONFIG_NET_SOCKETS_OFFLOAD=y # Memfault CONFIG_MEMFAULT_NCS_LTE_METRICS=y +CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y +CONFIG_LTE_LC_EDRX_MODULE=y +CONFIG_LTE_LC_PSM_MODULE=y CONFIG_MODEM_INFO=y # Certificate management diff --git a/samples/debug/memfault/boards/nrf9161dk_nrf9161_ns.conf b/samples/debug/memfault/boards/nrf9161dk_nrf9161_ns.conf index 8b9b2695cf42..53c1b66a3b7f 100644 --- a/samples/debug/memfault/boards/nrf9161dk_nrf9161_ns.conf +++ b/samples/debug/memfault/boards/nrf9161dk_nrf9161_ns.conf @@ -10,6 +10,9 @@ CONFIG_NET_SOCKETS_OFFLOAD=y # Memfault CONFIG_MEMFAULT_NCS_LTE_METRICS=y +CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y +CONFIG_LTE_LC_EDRX_MODULE=y +CONFIG_LTE_LC_PSM_MODULE=y CONFIG_MODEM_INFO=y # Certificate management diff --git a/samples/debug/memfault/boards/thingy91_nrf9160_ns.conf b/samples/debug/memfault/boards/thingy91_nrf9160_ns.conf index 4c6cad97c8bf..195b4b6e80c0 100644 --- a/samples/debug/memfault/boards/thingy91_nrf9160_ns.conf +++ b/samples/debug/memfault/boards/thingy91_nrf9160_ns.conf @@ -10,6 +10,9 @@ CONFIG_NET_SOCKETS_OFFLOAD=y # Memfault CONFIG_MEMFAULT_NCS_LTE_METRICS=y +CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y +CONFIG_LTE_LC_EDRX_MODULE=y +CONFIG_LTE_LC_PSM_MODULE=y # Certificate management CONFIG_MEMFAULT_ROOT_CERT_STORAGE_NRF9160_MODEM=y diff --git a/samples/debug/memfault/boards/thingy91x_nrf9151_ns.conf b/samples/debug/memfault/boards/thingy91x_nrf9151_ns.conf index b7395a7e1d90..96fc888348b4 100644 --- a/samples/debug/memfault/boards/thingy91x_nrf9151_ns.conf +++ b/samples/debug/memfault/boards/thingy91x_nrf9151_ns.conf @@ -10,6 +10,10 @@ CONFIG_NET_SOCKETS_OFFLOAD=y # Memfault CONFIG_MEMFAULT_NCS_LTE_METRICS=y +CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED=y +CONFIG_LTE_LC_EDRX_MODULE=y +CONFIG_LTE_LC_PSM_MODULE=y + CONFIG_MEMFAULT_NCS_DEVICE_ID_IMEI=y CONFIG_MODEM_INFO=y diff --git a/samples/debug/memfault/prj.conf b/samples/debug/memfault/prj.conf index 33c199a1441b..e8ba4d94ca36 100644 --- a/samples/debug/memfault/prj.conf +++ b/samples/debug/memfault/prj.conf @@ -10,7 +10,7 @@ CONFIG_ASSERT=y # Logging CONFIG_LOG=y -CONFIG_LOG_MODE_IMMEDIATE=y +CONFIG_LOG_BUFFER_SIZE=4096 # For more verbose and detailed log output, set the log level to # CONFIG_MEMFAULT_SAMPLE_LOG_LEVEL_DBG=y instead. @@ -31,12 +31,17 @@ CONFIG_MEMFAULT_NCS_PROJECT_KEY="" CONFIG_MEMFAULT_SHELL=y CONFIG_MEMFAULT_HTTP_PERIODIC_UPLOAD=y CONFIG_MEMFAULT_LOGGING_ENABLE=y +CONFIG_MEMFAULT_LOGGING_RAM_SIZE=4096 CONFIG_MEMFAULT_LOG_LEVEL_DBG=y +CONFIG_MEMFAULT_NCS_STACK_METRICS=y +CONFIG_MEMFAULT_COREDUMP_FULL_THREAD_STACKS=y +CONFIG_MEMFAULT_HEAP_STATS=y # Add to pick up support for sending Memfault data over HTTP CONFIG_MEMFAULT_HTTP_ENABLE=y # Store coredump to flash +CONFIG_MEMFAULT_NCS_LOG_LEVEL_DBG=y CONFIG_MEMFAULT_NCS_INTERNAL_FLASH_BACKED_COREDUMP=y CONFIG_MEMFAULT_COREDUMP_COLLECT_BSS_REGIONS=y diff --git a/samples/debug/memfault/src/main.c b/samples/debug/memfault/src/main.c index 0011da377c8d..6560f9f320fc 100644 --- a/samples/debug/memfault/src/main.c +++ b/samples/debug/memfault/src/main.c @@ -92,6 +92,11 @@ static void on_connect(void) LOG_INF("Time to connect: %d ms", time_to_lte_connection); #endif /* IS_ENABLED(MEMFAULT_NCS_LTE_METRICS) */ + if (IS_ENABLED(CONFIG_MEMFAULT_NCS_POST_COREDUMP_ON_NETWORK_CONNECTED)) { + /* Coredump sending handled internally */ + return; + } + LOG_INF("Sending already captured data to Memfault"); /* Trigger collection of heartbeat data. */