Merge pull request ESMCI#1319 from NCAR/develop

merge latest development to master

cmake/LibMPI.cmake

@@ -16,7 +16,7 @@ endif ()
 #
 
 #==============================================================================
-# - Get the machine platform-specific 
+# - Get the machine platform-specific
 #
 # Syntax:  platform_name (RETURN_VARIABLE)
 #
@@ -25,30 +25,25 @@ function (platform_name RETURN_VARIABLE)
     # Determine platform name from site name...
     site_name (SITENAME)
 
-    # UCAR/NWSC Machines
-    if (SITENAME MATCHES "^yslogin" OR
-        SITENAME MATCHES "^geyser" OR
-        SITENAME MATCHES "^caldera")
-
-        set (${RETURN_VARIABLE} "nwsc" PARENT_SCOPE)
 
-    # New NWSC SGI machine
-    elseif (SITENAME MATCHES "^laramie")
-
+    if (SITENAME MATCHES "^laramie" OR
+            SITENAME MATCHES "^cheyenne" OR
+	    SITENAME MATCHES "^chadmin")
+
 	set (${RETURN_VARIABLE} "nwscla" PARENT_SCOPE)
-        
+
     # ALCF/Argonne Machines
     elseif (SITENAME MATCHES "^mira" OR
             SITENAME MATCHES "^cetus" OR
             SITENAME MATCHES "^vesta" OR
             SITENAME MATCHES "^cooley")
-        
+
         set (${RETURN_VARIABLE} "alcf" PARENT_SCOPE)
-        
+
     # NERSC Machines
     elseif (SITENAME MATCHES "^edison" OR
         SITENAME MATCHES "^cori")
-        
+
         set (${RETURN_VARIABLE} "nersc" PARENT_SCOPE)
 
     # NCSA Machine (Blue Waters)
@@ -61,11 +56,11 @@ function (platform_name RETURN_VARIABLE)
             SITENAME MATCHES "^titan")
 
         set (${RETURN_VARIABLE} "olcf" PARENT_SCOPE)
-        
+
     else ()
 
         set (${RETURN_VARIABLE} "unknown" PARENT_SCOPE)
-    
+
     endif ()
 
 endfunction ()
@@ -85,35 +80,35 @@ function (add_mpi_test TESTNAME)
     set (oneValueArgs NUMPROCS TIMEOUT EXECUTABLE)
     set (multiValueArgs ARGUMENTS)
     cmake_parse_arguments (${TESTNAME} "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-    
+
     # Store parsed arguments for convenience
     set (exec_file ${${TESTNAME}_EXECUTABLE})
     set (exec_args ${${TESTNAME}_ARGUMENTS})
     set (num_procs ${${TESTNAME}_NUMPROCS})
     set (timeout ${${TESTNAME}_TIMEOUT})
-    
+
     # Get the platform name
     platform_name (PLATFORM)
-    
+
     # Default ("unknown" platform) execution
     if (PLATFORM STREQUAL "unknown")
 
         # Run tests directly from the command line
-        set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs} 
-                    ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} 
+        set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs}
+                    ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file}
                     ${MPIEXEC_POSTFLAGS} ${exec_args})
 
     else ()
-                        
+
         # Run tests from the platform-specific executable
-        set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM} 
+        set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM}
                      ${num_procs} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${exec_args})
-                     
+
     endif ()
-    
+
     # Add the test to CTest
     add_test(NAME ${TESTNAME} COMMAND ${EXE_CMD})
-    
+
     # Adjust the test timeout
     set_tests_properties(${TESTNAME} PROPERTIES TIMEOUT ${timeout})
 

cmake/mpiexec.nwscla

@@ -8,5 +8,9 @@
 
 NP=$1
 shift
-
-mpiexec_mpt -n $NP $@
+if [[ "$LMOD_FAMILY_MPI" == "openmpi" ]]
+then
+  mpirun -np $NP $@
+else
+  mpiexec_mpt -n $NP $@
+fi

ctest/CTestEnvironment-nwscla.cmake

@@ -10,7 +10,7 @@
 # set with existing environment variables: NETCDF, PNETCDF, HDF5, etc.
 
 # Define the extra CMake configure options
-set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_ASYNC=TRUE")
+set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE ")
 
 # If MPISERIAL environment variable is set, then enable MPISERIAL
 if (DEFINED ENV{MPISERIAL})

ctest/runcdash-nwsc-gnu.sh → ctest/runcdash-nwscla-gnu.sh

@@ -9,17 +9,17 @@ fi
 
 module reset
 module unload netcdf
-module swap intel gnu/7.1.0
-module swap mpt openmpi/3.0.0
+module swap intel gnu/8.1.0
+module swap mpt openmpi/3.1.0
 module load git/2.10.2
-module load cmake/3.9.1
-module load netcdf/4.4.1.1
-export PNETCDF=/glade/u/home/jedwards/pnetcdf/svn3652/openmpi300/gnu710
+module load cmake/3.12.1
+module load netcdf/4.6.1
+export PNETCDF=/glade/u/home/jedwards/pnetcdf/df0b42f19/gnu/8.1.0/openmpi/3.1.0
 
 export CC=mpicc
 export FC=mpif90
 
-export PIO_DASHBOARD_ROOT=`pwd`/dashboard
+export PIO_DASHBOARD_ROOT=/glade/u/home/jedwards/sandboxes/dashboard
 export PIO_COMPILER_ID=GNU-`$CC --version | head -n 1 | tail -n 1 | cut -d' ' -f3`
 
 if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then

ctest/runcdash-nwsc-pgi.sh → ctest/runcdash-nwscla-pgi.sh

@@ -9,16 +9,16 @@ fi
 
 module reset
 module unload netcdf
-module swap intel pgi/16.5
-module load git/2.3.0
-module load cmake/3.0.2
-module load netcdf-mpi/4.4.1
-module load pnetcdf/1.7.0
+module swap intel pgi/17.9
+module load git/2.10.2
+module load cmake/3.12.1
+module load netcdf-mpi/4.5.0
+module load pnetcdf/1.9.0
 
 export CC=mpicc
 export FC=mpif90
 
-export PIO_DASHBOARD_ROOT=`pwd`/dashboard
+export PIO_DASHBOARD_ROOT=/glade/u/home/jedwards/sandboxes/dashboard
 export PIO_COMPILER_ID=PGI-`$CC --version | head -n 2 | tail -n 1 | cut -d' ' -f2`
 
 if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then

src/clib/pio.h

@@ -267,6 +267,12 @@ typedef struct io_desc_t
      * 1-based mappings to the global array for that task. */
     PIO_Offset *map;
 
+    /** If the map passed in is not monotonically increasing
+     *	then map is sorted and remap is an array of original
+     * indices of map. */
+
+    int *remap;
+
     /** Number of tasks involved in the communication between comp and
      * io tasks. */
     int nrecvs;
@@ -294,6 +300,9 @@ typedef struct io_desc_t
      * everywhere (false) */
     bool needsfill;
 
+    /** If the map is not monotonically increasing we will need to sort it. */
+    bool needssort;
+
     /** The maximum number of bytes of this iodesc before flushing. */
     int maxbytes;
 

src/clib/pio_darray.c

@@ -113,6 +113,7 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
     int fndims;            /* Number of dims in the var in the file. */
     int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function calls. */
     int ierr;              /* Return code. */
+    void *tmparray;
 
     /* Get the file info. */
     if ((ierr = pio_get_file(ncid, &file)))
@@ -264,9 +265,19 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
             return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
         LOG((3, "allocated token for variable buffer"));
     }
+    if (iodesc->needssort)
+    {
+	if (!(tmparray = malloc(arraylen*nvars*iodesc->piotype_size)))
+	    return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+	pio_sorted_copy(array, tmparray, iodesc, nvars, 0);
+    }
+    else
+    {
+	tmparray = array;
+    }
 
     /* Move data from compute to IO tasks. */
-    if ((ierr = rearrange_comp2io(ios, iodesc, array, file->iobuf, nvars)))
+    if ((ierr = rearrange_comp2io(ios, iodesc, tmparray, file->iobuf, nvars)))
         return pio_err(ios, file, ierr, __FILE__, __LINE__);
 
     /* Write the darray based on the iotype. */
@@ -364,6 +375,9 @@ int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
         }
     }
 
+    if(iodesc->needssort && tmparray != NULL)
+	free(tmparray);
+
     /* Flush data to disk for pnetcdf. */
     if (ios->ioproc && file->iotype == PIO_IOTYPE_PNETCDF)
         if ((ierr = flush_output_buffer(file, flushtodisk, 0)))
@@ -403,7 +417,7 @@ pio_inq_var_fill_expected(int ncid, int varid, int pio_type, PIO_Offset type_siz
     unsigned long long uint64_fill_value = NC_FILL_UINT64;
     char *string_fill_value = "";
     int ret;
-    
+
     /* Check inputs. */
     assert(fillvalue);
 
@@ -412,7 +426,7 @@ pio_inq_var_fill_expected(int ncid, int varid, int pio_type, PIO_Offset type_siz
 
     /* Is there a _FillValue attribute? */
     ret = PIOc_inq_att_eh(ncid, varid, "_FillValue", 0, NULL, NULL);
-    
+
     LOG((3, "pio_inq_var_fill_expected ret %d", ret));
 
     /* If there is a fill value, get it. */
@@ -472,7 +486,7 @@ pio_inq_var_fill_expected(int ncid, int varid, int pio_type, PIO_Offset type_siz
             return PIO_EBADTYPE;
         }
     }
-    
+
     return PIO_NOERR;
 }
 
@@ -831,8 +845,8 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
     io_desc_t *iodesc;     /* Pointer to IO description information. */
     void *iobuf = NULL;    /* holds the data as read on the io node. */
     size_t rlen = 0;       /* the length of data in iobuf. */
-    int ierr;           /* Return code. */
-
+    int ierr;              /* Return code. */
+    void *tmparray;        /* unsorted copy of array buf if required */
     /* Get the file info. */
     if ((ierr = pio_get_file(ncid, &file)))
         return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
@@ -872,10 +886,26 @@ int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
         return pio_err(NULL, NULL, PIO_EBADIOTYPE, __FILE__, __LINE__);
     }
 
+    if (iodesc->needssort)
+    {
+	if (!(tmparray = malloc(iodesc->piotype_size*iodesc->maplen)))
+	    return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+	for(int m=0; m<iodesc->maplen;m++)
+	    ((int *) array)[m] = -1;
+    }
+    else
+	tmparray = array;
+
     /* Rearrange the data. */
-    if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, array)))
+    if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, tmparray)))
         return pio_err(ios, file, ierr, __FILE__, __LINE__);
 
+    if (iodesc->needssort)
+    {
+	pio_sorted_copy(tmparray, array, iodesc, 1, 1);
+	free(tmparray);
+    }
+
     /* Free the buffer. */
     if (rlen > 0)
         brel(iobuf);