From 3c47432e3de11419088fe0be59e587db1a927929 Mon Sep 17 00:00:00 2001 From: Joshua Hursey Date: Fri, 27 Jan 2017 09:15:23 -0600 Subject: [PATCH] orterun: Add parameter to control when we give up on stack traces * MCA option to control how long we wait for stack traces: - orte_timeout_for_stack_trace INTEGER Default: 30 Setting to <= 0 will cause it to wait forever * Useful when gathering stack traces from large jobs which might take a long time. Signed-off-by: Joshua Hursey --- orte/orted/orted_submit.c | 21 +++++++++++++-------- orte/runtime/orte_globals.c | 3 +++ orte/runtime/orte_globals.h | 4 ++++ orte/runtime/orte_mca_params.c | 9 +++++++++ 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/orte/orted/orted_submit.c b/orte/orted/orted_submit.c index d2ee5d7abee..365203bdeb6 100644 --- a/orte/orted/orted_submit.c +++ b/orte/orted/orted_submit.c @@ -17,6 +17,7 @@ * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. * Copyright (c) 2015-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -3071,8 +3072,10 @@ static void stack_trace_recv(int status, orte_process_name_t* sender, } ++ntraces; if (orte_process_info.num_procs == ntraces) { - /* cancel the timeout */ - OBJ_DESTRUCT(&stack_trace_timer); + if( orte_stack_trace_wait_timeout > 0 ) { + /* cancel the timeout */ + OBJ_DESTRUCT(&stack_trace_timer); + } /* abort the job */ ORTE_ACTIVATE_JOB_STATE(NULL, ORTE_JOB_STATE_ALL_JOBS_COMPLETE); /* set the global abnormal exit flag */ @@ -3173,12 +3176,14 @@ void orte_timeout_wakeup(int sd, short args, void *cbdata) OBJ_RELEASE(sig); /* we will terminate after we get the stack_traces, but set a timeout * just in case we never hear back from everyone */ - OBJ_CONSTRUCT(&stack_trace_timer, orte_timer_t); - opal_event_evtimer_set(orte_event_base, - stack_trace_timer.ev, stack_trace_timeout, NULL); - opal_event_set_priority(stack_trace_timer.ev, ORTE_ERROR_PRI); - stack_trace_timer.tv.tv_sec = 30; - opal_event_evtimer_add(stack_trace_timer.ev, &stack_trace_timer.tv); + if( orte_stack_trace_wait_timeout > 0 ) { + OBJ_CONSTRUCT(&stack_trace_timer, orte_timer_t); + opal_event_evtimer_set(orte_event_base, + stack_trace_timer.ev, stack_trace_timeout, NULL); + opal_event_set_priority(stack_trace_timer.ev, ORTE_ERROR_PRI); + stack_trace_timer.tv.tv_sec = orte_stack_trace_wait_timeout; + opal_event_evtimer_add(stack_trace_timer.ev, &stack_trace_timer.tv); + } return; } giveup: diff --git a/orte/runtime/orte_globals.c b/orte/runtime/orte_globals.c index 1090aa6a3a5..75712e11d52 100644 --- a/orte/runtime/orte_globals.c +++ b/orte/runtime/orte_globals.c @@ -16,6 +16,7 @@ * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -132,6 +133,8 @@ orte_timer_t *orte_mpiexec_timeout = NULL; opal_buffer_t *orte_tree_launch_cmd = NULL; +int orte_stack_trace_wait_timeout = 30; + /* global arrays for data storage */ opal_hash_table_t *orte_job_data = NULL; opal_pointer_array_t *orte_node_pool = NULL; diff --git a/orte/runtime/orte_globals.h b/orte/runtime/orte_globals.h index a794610f613..c60a43303c6 100644 --- a/orte/runtime/orte_globals.h +++ b/orte/runtime/orte_globals.h @@ -14,6 +14,7 @@ * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -576,6 +577,9 @@ ORTE_DECLSPEC extern char *orte_base_user_debugger; */ ORTE_DECLSPEC extern char *orte_daemon_cores; +/* Max time to wait for stack straces to return */ +ORTE_DECLSPEC extern int orte_stack_trace_wait_timeout; + END_C_DECLS #endif /* ORTE_RUNTIME_ORTE_GLOBALS_H */ diff --git a/orte/runtime/orte_mca_params.c b/orte/runtime/orte_mca_params.c index e637ad4fd38..6dd76682d1a 100644 --- a/orte/runtime/orte_mca_params.c +++ b/orte/runtime/orte_mca_params.c @@ -16,6 +16,7 @@ * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. * Copyright (c) 2014 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -752,5 +753,13 @@ int orte_register_params(void) MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &orte_mgmt_transport); + /* Amount of time to wait for a stack trace to return from the daemons */ + orte_stack_trace_wait_timeout = 30; + (void) mca_base_var_register ("orte", "orte", NULL, "timeout_for_stack_trace", + "Seconds to wait for stack traces to return before terminating the job (<= 0 wait forever)", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, + &orte_stack_trace_wait_timeout); + return ORTE_SUCCESS; }