Merge pull request ESMCI#1381 from NCAR/ejh_perf2

Moving map arrays from stack to heap...

src/clib/pio_rearrange.c

@@ -338,7 +338,11 @@ int create_mpi_datatypes(MPI_Datatype mpitype, int msgcnt,
         if (mcount[i] > 0)
         {
             int len = mcount[i] / blocksize;
-            int displace[len];
+            int *displace;
+
+            if (!(displace = malloc(sizeof(int) * len)))
+                return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
             LOG((3, "blocksize = %d i = %d mcount[%d] = %d len = %d", blocksize, i, i,
                  mcount[i], len));
             if (blocksize == 1)
@@ -376,8 +380,11 @@ int create_mpi_datatypes(MPI_Datatype mpitype, int msgcnt,
             LOG((3, "calling MPI_Type_create_indexed_block len = %d blocksize = %d "
                  "mpitype = %d", len, blocksize, mpitype));
             /* Create an indexed datatype with constant-sized blocks. */
-            if ((mpierr = MPI_Type_create_indexed_block(len, blocksize, displace,
-                                                        mpitype, &mtype[i])))
+            mpierr = MPI_Type_create_indexed_block(len, blocksize, displace,
+                                                   mpitype, &mtype[i]);
+
+            free(displace);
+            if (mpierr)
                 return check_mpi(NULL, mpierr, __FILE__, __LINE__);
 
             if (mtype[i] == PIO_DATATYPE_NULL)

src/clib/pioc.c

@@ -693,8 +693,9 @@ int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, i
                      const PIO_Offset *compmap, int *ioidp, int rearranger,
                      const PIO_Offset *iostart, const PIO_Offset *iocount)
 {
-    PIO_Offset compmap_1_based[maplen];
+    PIO_Offset *compmap_1_based;
     int *rearrangerp = NULL;
+    int ret;
 
     LOG((1, "PIOc_init_decomp iosysid = %d pio_type = %d ndims = %d maplen = %d",
          iosysid, pio_type, ndims, maplen));
@@ -703,6 +704,10 @@ int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, i
     if (rearranger)
         rearrangerp = &rearranger;
 
+    /* Allocate storage for compmap that's one-based. */
+    if (!(compmap_1_based = malloc(sizeof(PIO_Offset) * maplen)))
+        return PIO_ENOMEM;
+
     /* Add 1 to all elements in compmap. */
     for (int e = 0; e < maplen; e++)
     {
@@ -711,8 +716,12 @@ int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, i
     }
 
     /* Call the legacy version of the function. */
-    return PIOc_InitDecomp(iosysid, pio_type, ndims, gdimlen, maplen, compmap_1_based,
-                           ioidp, rearrangerp, iostart, iocount);
+    ret = PIOc_InitDecomp(iosysid, pio_type, ndims, gdimlen, maplen, compmap_1_based,
+                          ioidp, rearrangerp, iostart, iocount);
+
+    free(compmap_1_based);
+
+    return ret;
 }
 
 /**

src/clib/pioc_support.c

@@ -1154,7 +1154,10 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm
     /* Now initialize the iodesc on each task for this decomposition. */
     if (!ret)
     {
-        PIO_Offset compmap[task_maplen[my_rank]];
+        PIO_Offset *compmap;
+
+        if (!(compmap = malloc(sizeof(PIO_Offset) * task_maplen[my_rank])))
+            return PIO_ENOMEM;
 
         /* Copy array into PIO_Offset array. Make it 1 based. */
         for (int e = 0; e < task_maplen[my_rank]; e++)
@@ -1163,6 +1166,8 @@ int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm
         /* Initialize the decomposition. */
         ret = PIOc_InitDecomp(iosysid, pio_type, ndims, global_dimlen, task_maplen[my_rank],
                               compmap, ioidp, NULL, NULL, NULL);
+
+        free(compmap);
     }
 
     /* Free resources. */

tests/cunit/test_perf2.c

@@ -33,8 +33,8 @@
 #define NDIM3 3
 
 /* The length of our sample data along each dimension. */
-#define X_DIM_LEN 256
-#define Y_DIM_LEN 256
+#define X_DIM_LEN 512
+#define Y_DIM_LEN 512
 #define Z_DIM_LEN 64
 
 /* This is the length of the map for each task. */
@@ -113,10 +113,11 @@ int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid)
  * @param flavor array of available iotypes.
  * @param my_rank rank of this task.
  * @param provide_fill 1 if fillvalue should be provided to PIOc_write_darray().
+ * @param rearranger the rearranger in use.
  * @returns 0 for success, error code otherwise.
  */
 int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
-                int provide_fill)
+                int provide_fill, int rearranger)
 {
     char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
     int dimids[NDIM];      /* The dimension IDs. */
@@ -220,7 +221,7 @@ int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank
         endt = (1000000 * endtime.tv_sec) + endtime.tv_usec;
         delta = (endt - startt)/NUM_TIMESTEPS;
         if (!my_rank)
-            printf("%d\t%lld\n", fmt, delta);
+            printf("%d\t%d\t%d\t%lld\n", rearranger, provide_fill, fmt, delta);
 
         /* Reopen the file. */
         if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
@@ -382,7 +383,8 @@ int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
     for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++)
     {
         /* Run a simple darray test. */
-        if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, provide_fill)))
+        if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank,
+                               provide_fill, rearranger)))
             return ret;
     }
 
@@ -424,6 +426,10 @@ int main(int argc, char **argv)
         if ((ret = get_iotypes(&num_flavors, flavor)))
             ERR(ret);
 
+        if (!my_rank)
+            printf("rearr\tfill\tformat\ttime\n");
+
+
         for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
         {
             /* Initialize the PIO IO system. This specifies how