Improve packing efficiency by raising the initial buffer size and modifying the extension code. Flag if a job map has had its nodes added so we don't have to loop repeatedly to check it.

Signed-off-by: Ralph Castain <rhc@open-mpi.org>

Author: Ralph Castain

opal/dss/dss_internal.h

@@ -11,7 +11,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
@@ -47,12 +47,12 @@ BEGIN_C_DECLS
 /*
  * The default starting chunk size
  */
-#define OPAL_DSS_DEFAULT_INITIAL_SIZE  128
+#define OPAL_DSS_DEFAULT_INITIAL_SIZE  2048
 /*
  * The default threshold size when we switch from doubling the
  * buffer size to addatively increasing it
  */
-#define OPAL_DSS_DEFAULT_THRESHOLD_SIZE 1024
+#define OPAL_DSS_DEFAULT_THRESHOLD_SIZE 4096
 
 /*
  * Internal type corresponding to size_t.  Do not use this in

opal/dss/dss_internal_functions.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2017      Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -35,6 +36,7 @@ char* opal_dss_buffer_extend(opal_buffer_t *buffer, size_t bytes_to_add)
 {
     size_t required, to_alloc;
     size_t pack_offset, unpack_offset;
+    char *tmp;
 
     /* Check to see if we have enough space already */
 
@@ -43,34 +45,19 @@ char* opal_dss_buffer_extend(opal_buffer_t *buffer, size_t bytes_to_add)
     }
 
     required = buffer->bytes_used + bytes_to_add;
-    if(required >= (size_t)opal_dss_threshold_size) {
-        to_alloc = ((required + opal_dss_threshold_size - 1)
-                    / opal_dss_threshold_size) * opal_dss_threshold_size;
+    if (required >= (size_t)opal_dss_threshold_size) {
+        to_alloc = (required + opal_dss_threshold_size - 1) & ~(opal_dss_threshold_size - 1);
     } else {
-        to_alloc = buffer->bytes_allocated;
-        if(0 == to_alloc) {
-            to_alloc = opal_dss_initial_size;
-        }
-        while(to_alloc < required) {
-            to_alloc <<= 1;
-        }
+        to_alloc = buffer->bytes_allocated ? buffer->bytes_allocated : (size_t)opal_dss_initial_size;
     }
 
-    if (NULL != buffer->base_ptr) {
-        pack_offset = ((char*) buffer->pack_ptr) - ((char*) buffer->base_ptr);
-        unpack_offset = ((char*) buffer->unpack_ptr) -
-            ((char*) buffer->base_ptr);
-        buffer->base_ptr = (char*)realloc(buffer->base_ptr, to_alloc);
-    } else {
-        pack_offset = 0;
-        unpack_offset = 0;
-        buffer->bytes_used = 0;
-        buffer->base_ptr = (char*)malloc(to_alloc);
-    }
-
-    if (NULL == buffer->base_ptr) {
+    pack_offset = ((char*) buffer->pack_ptr) - ((char*) buffer->base_ptr);
+    unpack_offset = ((char*) buffer->unpack_ptr) - ((char*) buffer->base_ptr);
+    tmp = (char*)realloc(buffer->base_ptr, to_alloc);
+    if (NULL == tmp) {
         return NULL;
     }
+    buffer->base_ptr = tmp;
     buffer->pack_ptr = ((char*) buffer->base_ptr) + pack_offset;
     buffer->unpack_ptr = ((char*) buffer->base_ptr) + unpack_offset;
     buffer->bytes_allocated = to_alloc;

orte/mca/odls/base/odls_base_default_fns.c

@@ -261,7 +261,6 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer,
     orte_proc_t *pptr, *dmn;
     opal_buffer_t *bptr;
     orte_app_context_t *app;
-    bool found;
     orte_node_t *node;
     bool newmap = false;
 
@@ -409,6 +408,13 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer,
     if (NULL == jdata->map) {
         jdata->map = OBJ_NEW(orte_job_map_t);
         newmap = true;
+    } else if (ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_MAP_INITIALIZED)) {
+        /* zero all the node map flags */
+        for (n=0; n < jdata->map->nodes->size; n++) {
+            if (NULL != (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, n))) {
+                ORTE_FLAG_UNSET(node, ORTE_NODE_FLAG_MAPPED);
+            }
+        }
     }
 
     /* if we have a file map, then we need to load it */
@@ -454,17 +460,7 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer,
         opal_pointer_array_add(dmn->node->procs, pptr);
 
         /* add the node to the map, if not already there */
-        found = false;
-        for (k=0; k < jdata->map->nodes->size; k++) {
-            if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, k))) {
-                continue;
-            }
-            if (node->daemon == dmn) {
-                found = true;
-                break;
-            }
-        }
-        if (!found) {
+        if (!ORTE_FLAG_TEST(dmn->node, ORTE_NODE_FLAG_MAPPED)) {
             OBJ_RETAIN(dmn->node);
             opal_pointer_array_add(jdata->map->nodes, dmn->node);
             if (newmap) {
@@ -497,6 +493,7 @@ int orte_odls_base_default_construct_child_list(opal_buffer_t *buffer,
             app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, pptr->app_idx);
             ORTE_FLAG_SET(app, ORTE_APP_FLAG_USED_ON_NODE);
         }
+        ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_MAP_INITIALIZED);
     }
 
  COMPLETE:

orte/mca/rmaps/base/rmaps_base_map_job.c

@@ -389,6 +389,8 @@ void orte_rmaps_base_map_job(int fd, short args, void *cbdata)
         OBJ_RELEASE(caddy);
         return;
     }
+    /* mark that nodes were assigned to this job */
+    ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_MAP_INITIALIZED);
 
     /* if any node is oversubscribed, then check to see if a binding
      * directive was given - if not, then we want to clear the default

orte/orted/pmix/pmix_server_register_fns.c

@@ -74,6 +74,11 @@ int orte_pmix_server_register_nspace(orte_job_t *jdata)
                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
                         ORTE_JOBID_PRINT(jdata->jobid));
 
+    /* if this job has no local procs, then no need to register them */
+    if (0 == jdata->num_local_procs) {
+        return ORTE_SUCCESS;
+    }
+
     /* setup the info list */
     info = OBJ_NEW(opal_list_t);
     uid = geteuid();

orte/runtime/data_type_support/orte_dt_unpacking_fns.c

@@ -12,7 +12,7 @@
  * Copyright (c) 2011      Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2011-2013 Los Alamos National Security, LLC.
  *                         All rights reserved.
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -168,9 +168,9 @@ int orte_dt_unpack_job(opal_buffer_t *buffer, void *dest,
             return rc;
         }
 
-        /* if the map is NULL, then we din't pack it as there was
+        /* if the map is NULL, then we didn't pack it as there was
          * nothing to pack. Instead, we packed a flag to indicate whether or not
-         * the map is included         */
+         * the map is included */
         n = 1;
         if (ORTE_SUCCESS != (rc = opal_dss_unpack_buffer(buffer,
                                             &j, &n, ORTE_STD_CNTR))) {
@@ -204,6 +204,8 @@ int orte_dt_unpack_job(opal_buffer_t *buffer, void *dest,
             ORTE_ERROR_LOG(rc);
             return rc;
         }
+        /* mark the map as uninitialized as we don't pack the node map */
+        ORTE_FLAG_UNSET(jobs[i], ORTE_JOB_FLAG_MAP_INITIALIZED);
 
         /* unpack the attributes */
         n=1;

orte/util/attr.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014-2016 Intel, Inc. All rights reserved
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
@@ -89,7 +89,7 @@ typedef uint16_t orte_job_flags_t;
 #define ORTE_JOB_FLAG_RESTART            0x0200   //
 #define ORTE_JOB_FLAG_PROCS_MIGRATING    0x0400   // some procs in job are migrating from one node to another
 #define ORTE_JOB_FLAG_OVERSUBSCRIBED     0x0800   // at least one node in the job is oversubscribed
-
+#define ORTE_JOB_FLAG_MAP_INITIALIZED    0x1000   // nodes have been assigned to this job map
 
 /***   JOB ATTRIBUTE KEYS   ***/
 #define ORTE_JOB_START_KEY   ORTE_NODE_MAX_KEY