Add a timeout cmd line option and an option to report state info upon…

… timeout to assist with debugging Jenkins tests

If requested, obtain stacktraces for each application process and report it to stderr upon timeout

stack traces: minor improvements

- Also include the hostname and PID of the each process for which
  we're sending the stack traces (vs. just including the ORTE process
  name)
- Send a specific error message if we couldn't find "gstack" in the
  $PATH (e.g., on OS X)
- Send a sepcific error message if gstack fails to run
- Print a message that obtaining the stack traces may take a few
  seconds so that users don't wonder what's happening

Signed-off-by: Jeff Squyres <jsquyres@cisco.com>

help-orterun.txt: minor tweaks

Trivial update: show "--timeout" (instead of "-timeout") in the help
message, just to encourage the use of double-dash options.

Signed-off-by: Jeff Squyres <jsquyres@cisco.com>

trivial: stacktrace -> stack trace

Trivial word smything.

Signed-off-by: Jeff Squyres <jsquyres@cisco.com>

orte/mca/odls/odls_types.h

@@ -9,7 +9,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
- * Copyright (c) 2011      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2011-2016 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011-2012 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
@@ -80,6 +80,9 @@ typedef uint8_t orte_daemon_cmd_flag_t;
 /* add procs for the DVM */
 #define ORTE_DAEMON_DVM_ADD_PROCS           (orte_daemon_cmd_flag_t) 30
 
+/* for debug purposes, get stack traces from all application procs */
+#define ORTE_DAEMON_GET_STACK_TRACES        (orte_daemon_cmd_flag_t) 31
+
 /*
  * Struct written up the pipe from the child to the parent.
  */

orte/mca/rml/rml_types.h

@@ -163,6 +163,9 @@ BEGIN_C_DECLS
 /* error notifications */
 #define ORTE_RML_TAG_NOTIFICATION           59
 
+/* stacktrace for debug */
+#define ORTE_RML_TAG_STACK_TRACE            60
+
 #define ORTE_RML_TAG_MAX                   100
 
 #define ORTE_RML_TAG_NTOH(t) ntohl(t)

orte/mca/schizo/ompi/schizo_ompi.c

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2013 Los Alamos National Security, LLC.
  *                         All rights reserved.
- * Copyright (c) 2009-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011      Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2013-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
@@ -92,6 +92,18 @@ static opal_cmd_line_init_t cmd_line_init[] = {
       &orte_cmd_options.report_uri, OPAL_CMD_LINE_TYPE_STRING,
       "Printout URI on stdout [-], stderr [+], or a file [anything else]" },
 
+    /* testing options */
+    { NULL, '\0', "timeout", "timeout", 1,
+      &orte_cmd_options.timeout, OPAL_CMD_LINE_TYPE_INT,
+      "Timeout the job after the specified number of seconds" },
+    { NULL, '\0', "report-state-on-timeout", "report-state-on-timeout", 0,
+      &orte_cmd_options.report_state_on_timeout, OPAL_CMD_LINE_TYPE_BOOL,
+      "Report all job and process states upon timeout" },
+    { NULL, '\0', "get-stack-traces", "get-stack-traces", 0,
+      &orte_cmd_options.get_stack_traces, OPAL_CMD_LINE_TYPE_BOOL,
+      "Get stack traces of all application procs on timeout" },
+
+
     /* exit status reporting */
     { "orte_report_child_jobs_separately", '\0', "report-child-jobs-separately", "report-child-jobs-separately", 0,
       NULL, OPAL_CMD_LINE_TYPE_BOOL,

orte/orted/orted_comm.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2016 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2007-2012 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2009      Sun Microsystems, Inc. All rights reserved.
@@ -47,6 +47,7 @@
 #include "opal/mca/base/base.h"
 #include "opal/util/output.h"
 #include "opal/util/opal_environ.h"
+#include "opal/util/path.h"
 #include "opal/runtime/opal.h"
 #include "opal/runtime/opal_progress.h"
 #include "opal/dss/dss.h"
@@ -111,6 +112,9 @@ void orte_daemon_recv(int status, orte_process_name_t* sender,
     bool found = false;
     orte_node_t *node;
     orte_grpcomm_signature_t *sig;
+    FILE *fp;
+    char gscmd[256], path[1035], *pathptr;
+    char string[256], *string_ptr = string;
 
     /* unpack the command */
     n = 1;
@@ -1071,6 +1075,82 @@ void orte_daemon_recv(int status, orte_process_name_t* sender,
         }
         break;
 
+    case ORTE_DAEMON_GET_STACK_TRACES:
+        /* prep the response */
+        answer = OBJ_NEW(opal_buffer_t);
+        pathptr = path;
+
+        // Try to find the "gstack" executable.  Failure to find the
+        // executable will be handled below, because the receiver
+        // expects to have the process name, hostname, and PID in the
+        // buffer before finding an error message.
+        char *gstack_exec;
+        gstack_exec = opal_find_absolute_path("gstack");
+
+        /* hit each local process with a gstack command */
+        for (i=0; i < orte_local_children->size; i++) {
+            if (NULL != (proct = (orte_proc_t*)opal_pointer_array_get_item(orte_local_children, i)) &&
+                ORTE_FLAG_TEST(proct, ORTE_PROC_FLAG_ALIVE)) {
+                relay_msg = OBJ_NEW(opal_buffer_t);
+                if (OPAL_SUCCESS != opal_dss.pack(relay_msg, &proct->name, 1, ORTE_NAME) ||
+                    OPAL_SUCCESS != opal_dss.pack(relay_msg, &proct->node->name, 1, OPAL_STRING) ||
+                    OPAL_SUCCESS != opal_dss.pack(relay_msg, &proct->pid, 1, OPAL_PID)) {
+                    OBJ_RELEASE(relay_msg);
+                    break;
+                }
+
+                // If we were able to find the gstack executable,
+                // above, then run the command here.
+                fp = NULL;
+                if (NULL != gstack_exec) {
+                    (void) snprintf(gscmd, sizeof(gscmd), "%s %lu",
+                                    gstack_exec, (unsigned long) proct->pid);
+                    fp = popen(gscmd, "r");
+                }
+
+                // If either we weren't able to find or run the gstack
+                // exectuable, send back a nice error message here.
+                if (NULL == gstack_exec || NULL == fp) {
+                    (void) snprintf(string, sizeof(string),
+                                    "Failed to %s \"%s\" on %s to obtain stack traces",
+                                    (NULL == gstack_exec) ? "find" : "run",
+                                    (NULL == gstack_exec) ? "gstack" : gstack_exec,
+                                    proct->node->name);
+                    if (OPAL_SUCCESS ==
+                        opal_dss.pack(relay_msg, &string_ptr, 1, OPAL_STRING)) {
+                        opal_dss.pack(answer, &relay_msg, 1, OPAL_BUFFER);
+                    }
+                    OBJ_RELEASE(relay_msg);
+                    break;
+                }
+                /* Read the output a line at a time and pack it for transmission */
+                memset(path, 0, sizeof(path));
+                while (fgets(path, sizeof(path)-1, fp) != NULL) {
+                    if (OPAL_SUCCESS != opal_dss.pack(relay_msg, &pathptr, 1, OPAL_STRING)) {
+                        OBJ_RELEASE(relay_msg);
+                        break;
+                    }
+                    memset(path, 0, sizeof(path));
+                }
+                /* close */
+                pclose(fp);
+                /* transfer this load */
+                if (OPAL_SUCCESS != opal_dss.pack(answer, &relay_msg, 1, OPAL_BUFFER)) {
+                    OBJ_RELEASE(relay_msg);
+                    break;
+                }
+                OBJ_RELEASE(relay_msg);
+            }
+        }
+        /* always send our response */
+        if (0 > (ret = orte_rml.send_buffer_nb(ORTE_PROC_MY_HNP, answer,
+                                               ORTE_RML_TAG_STACK_TRACE,
+                                               orte_rml_send_callback, NULL))) {
+            ORTE_ERROR_LOG(ret);
+            OBJ_RELEASE(answer);
+        }
+        break;
+
     default:
         ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
     }
@@ -1139,6 +1219,9 @@ static char *get_orted_comm_cmd_str(int command)
     case ORTE_DAEMON_DVM_ADD_PROCS:
         return strdup("ORTE_DAEMON_DVM_ADD_PROCS");
 
+    case ORTE_DAEMON_GET_STACK_TRACES:
+        return strdup("ORTE_DAEMON_GET_STACK_TRACES");
+
     default:
         return strdup("Unknown Command!");
     }