Skip to content

Commit

Permalink
update pnetcdf output; parallel io test in 3D are currently failling …
Browse files Browse the repository at this point in the history 
…(both hdf5 and pnetcdf), there is a bug in OpenMPI version 2.1 (used in Ubuntu 17.10 at least) that is reported here open-mpi/ompi#3695; let's try another version of openmpi
  • Loading branch information
@pkestene
pkestene committed May 3, 2018
1 parent d91dba8 commit 73f229f
Show file tree
Hide file tree
Showing 3 changed files with 88 additions and 35 deletions.
72 changes: 51 additions & 21 deletions src/utils/io/IO_PNETCDF.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,31 +59,54 @@ class Save_PNETCDF
~Save_PNETCDF() {};

template<DimensionType d_ = d>
void copy_buffer(typename std::enable_if<d_==TWO_D, real_t>::type *data,
int iStop, int jStop, int kStop, int iVar)
void copy_buffer(typename std::enable_if<d_==TWO_D, real_t>::type *&data,
int iStop, int jStop, int kStop, int iVar, KokkosLayout layout)
{
int dI=0;
for (int j= 0; j < jStop; j++) {
if (layout == KOKKOS_LAYOUT_RIGHT) { // transpose array to make data contiguous in memory
int dI=0;
for (int j= 0; j < jStop; j++) {
for(int i = 0; i < iStop; i++) {
data[dI] = Uhost(i,j,iVar);
dI++;
}
}
} else {
//data = Uhost.ptr_on_device() + isize*jsize*nvar;
int dI=0;
for(int i = 0; i < iStop; i++) {
data[dI] = Uhost(i,j,iVar);
dI++;
for (int j= 0; j < jStop; j++) {
data[dI] = Uhost(i,j,iVar);
dI++;
}
}
}

} // copy_buffer

template<DimensionType d_=d>
void copy_buffer(typename std::enable_if<d_==THREE_D, real_t>::type *data,
int iStop, int jStop, int kStop, int iVar)
void copy_buffer(typename std::enable_if<d_==THREE_D, real_t>::type *&data,
int iStop, int jStop, int kStop, int iVar, KokkosLayout layout)
{
int dI=0;
for (int k= 0; k < kStop; k++)
for (int j= 0; j < jStop; j++)
for(int i = 0; i < iStop; i++) {
data[dI] = Uhost(i,j,k,iVar);
dI++;
}

if (layout == KOKKOS_LAYOUT_RIGHT) { // transpose array to make data contiguous in memory

int dI=0;
for (int k= 0; k < kStop; k++)
for (int j= 0; j < jStop; j++)
for(int i = 0; i < iStop; i++) {
data[dI] = Uhost(i,j,k,iVar);
dI++;
}
} else {
//data = Uhost.ptr_on_device() + isize*jsize*ksize*nvar;
int dI=0;
for(int i = 0; i < iStop; i++)
for (int j= 0; j < jStop; j++)
for (int k= 0; k < kStop; k++) {
data[dI] = Uhost(i,j,k,iVar);
dI++;
}
}

} // copy_buffer / 3D

// =======================================================
Expand All @@ -110,7 +133,6 @@ class Save_PNETCDF
*/
void save()
{

using namespace hydroSimu; // for MpiComm (this namespace is tout-pourri)

// sub-domain sizes
Expand Down Expand Up @@ -174,13 +196,12 @@ class Save_PNETCDF
}
// broadcast stringDate size to all other MPI tasks
params.communicator->bcast(&stringDateSize, 1, MpiComm::INT, 0);

// broadcast stringDate to all other MPI task
if (myRank != 0) stringDate.reserve(stringDateSize);
char* cstr = const_cast<char*>(stringDate.c_str());
params.communicator->bcast(cstr, stringDateSize, MpiComm::CHAR, 0);


/*
* get MPI coords corresponding to MPI rank iPiece
*/
Expand All @@ -196,6 +217,7 @@ class Save_PNETCDF
*/
std::string outputDir = configMap.getString("output", "outputDir", "./");
std::string outputPrefix = configMap.getString("output", "outputPrefix", "output");

std::ostringstream outNum;
outNum.width(7);
outNum.fill('0');
Expand All @@ -207,6 +229,14 @@ class Save_PNETCDF
// copy device data to host
Kokkos::deep_copy(Uhost, Udata);

// here we need to check Uhost memory layout
KokkosLayout layout;
if (std::is_same<typename DataArray::array_layout, Kokkos::LayoutLeft>::value)
layout = KOKKOS_LAYOUT_LEFT;
else
layout = KOKKOS_LAYOUT_RIGHT;


// measure time ??
if (pnetcdf_verbose) {
MPI_Barrier(params.communicator->getComm());
Expand Down Expand Up @@ -434,7 +464,7 @@ class Save_PNETCDF
for (int iVar=0; iVar<nbvar; iVar++) {

// copy needed data into data !
copy_buffer(data,iStop,jStop,kStop,iVar);
copy_buffer(data,iStop,jStop,kStop,iVar,layout);

// write on disk
err = ncmpi_put_vara_all(ncFileId, varIds[iVar], starts, counts, data, nItems, mpiDataType);
Expand Down
6 changes: 6 additions & 0 deletions test/io/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ endif(USE_MPI)
if(USE_HDF5)
add_executable(test_io_hdf5
test_io_hdf5.cpp
${BACKWARD_ENABLE}
)

target_include_directories(test_io_hdf5
Expand All @@ -51,6 +52,8 @@ if(USE_HDF5)
target_link_libraries(test_io_hdf5
mpiUtils)
endif(USE_MPI)

add_backward(test_io_hdf5)

endif(USE_HDF5)

Expand All @@ -59,6 +62,7 @@ if(USE_MPI)

add_executable(test_io_pnetcdf
test_io_pnetcdf.cpp
${BACKWARD_ENABLE}
)

target_include_directories(test_io_pnetcdf
Expand All @@ -74,5 +78,7 @@ if(USE_MPI)
target_link_libraries(test_io_pnetcdf
kokkos shared config monitoring io mpiUtils ${HDF5_PNETCDF})

add_backward(test_io_pnetcdf)

endif(USE_PNETCDF)
endif(USE_MPI)
45 changes: 31 additions & 14 deletions test/io/test_io_pnetcdf.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,10 @@ class InitData
real_t x = xmin + dx/2 + (i+nx*i_mpi-ghostWidth)*dx;
real_t y = ymin + dy/2 + (j+ny*j_mpi-ghostWidth)*dy;

data(i,j,ID) = x+y;
data(i,j,ID) = x+y;
data(i,j,IE) = 2*x+y;
data(i,j,IU) = 3*x+y;
data(i,j,IV) = 4*x+y;

}

Expand Down Expand Up @@ -99,7 +102,17 @@ class InitData
real_t y = ymin + dy/2 + (j+ny*j_mpi-ghostWidth)*dy;
real_t z = zmin + dz/2 + (k+nz*k_mpi-ghostWidth)*dz;

data(i,j,k,ID) = x+y+z;
data(i,j,k,ID) = x+y+z;
data(i,j,k,IE) = 2*x+y+z;
data(i,j,k,IU) = 3*x+y+z;
data(i,j,k,IV) = 4*x+y+z;
data(i,j,k,IW) = 5*x+y+z;

// data(i,j,k,ID) = index + 0*isize*jsize*ksize;// x+y+z;
// data(i,j,k,IE) = index + 1*isize*jsize*ksize;//2*x+y+z;
// data(i,j,k,IU) = index + 2*isize*jsize*ksize;//3*x+y+z;
// data(i,j,k,IV) = index + 3*isize*jsize*ksize;//4*x+y+z;
// data(i,j,k,IW) = index + 4*isize*jsize*ksize;//5*x+y+z;
}

HydroParams params;
Expand All @@ -120,8 +133,10 @@ int main(int argc, char* argv[])
hydroSimu::GlobalMpiSession mpiSession(&argc,&argv);

Kokkos::initialize(argc, argv);

{

int mpi_rank = 0;
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
if (mpi_rank==0) {
std::cout << "##########################\n";
std::cout << "KOKKOS CONFIG \n";
std::cout << "##########################\n";
Expand All @@ -141,11 +156,11 @@ int main(int argc, char* argv[])
std::cout << "##########################\n";
}

if (argc != 2) {
fprintf(stderr, "Error: wrong number of argument; input filename must be the only parameter on the command line\n");
Kokkos::finalize();
exit(EXIT_FAILURE);
}
// if (argc != 2) {
// fprintf(stderr, "Error: wrong number of argument; input filename must be the only parameter on the command line\n");
// Kokkos::finalize();
// exit(EXIT_FAILURE);
// }

// read parameter file and initialize parameter
// parse parameters from input file
Expand All @@ -159,16 +174,17 @@ int main(int argc, char* argv[])
std::map<int, std::string> var_names;
var_names[ID] = "rho";
var_names[IP] = "energy";
var_names[IU] = "mx";
var_names[IV] = "my";
var_names[IW] = "mz";
var_names[IU] = "rho_vx";
var_names[IV] = "rho_vy";
var_names[IW] = "rho_vz";

// =================
// ==== 2D test ====
// =================
if (params.nz == 1) {

std::cout << "2D test\n";
if (mpi_rank==0)
std::cout << "2D test\n";

DataArray2d data("data",params.isize,params.jsize,HYDRO_2D_NBVAR);
DataArray2dHost data_host = Kokkos::create_mirror(data);
Expand All @@ -188,7 +204,8 @@ int main(int argc, char* argv[])
// =================
if (params.nz > 1) {

std::cout << "3D test\n";
if (mpi_rank==0)
std::cout << "3D test\n";

DataArray3d data("data",params.isize,params.jsize,params.ksize,HYDRO_3D_NBVAR);
DataArray3dHost data_host = Kokkos::create_mirror(data);
Expand Down

0 comments on commit 73f229f

Please sign in to comment.