Skip to content

Commit

Permalink
Merge pull request #66 from NCAR/ejh_pio_file_async
Browse files Browse the repository at this point in the history 
changing in pio_file.c to support async
  • Loading branch information
@jedwards4b
jedwards4b committed May 5, 2016
2 parents 32228f5 + aebdcdd commit 0c4cbe6
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 22 deletions.
54 changes: 33 additions & 21 deletions src/clib/pio_file.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,10 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
if(ios->comp_rank==0)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
}

if(ios->ioproc){
Expand Down Expand Up @@ -130,8 +131,10 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,

ierr = check_netcdf(file, ierr, __FILE__,__LINE__);

if(ierr==PIO_NOERR){
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm);
if (!ierr) {
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm);
mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm);
*ncidp = file->fh;
pio_add_to_file_list(file);
*ncidp = file->fh;
}
Expand Down Expand Up @@ -199,10 +202,11 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype,

if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send( &msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
mpierr = MPI_Bcast((void *) filename,len, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&(file->iotype), 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
}

Expand Down Expand Up @@ -248,8 +252,10 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype,
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);

if(ierr == PIO_NOERR){
mpierr = MPI_Bcast(&(file->mode), 1, MPI_INT, ios->ioroot, ios->union_comm);
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm);
file->mode = file->mode | PIO_WRITE; // This flag is implied by netcdf create functions but we need to know if its set
mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm);
*ncidp = file->fh;
pio_add_to_file_list(file);
*ncidp = file->fh;
}
Expand Down Expand Up @@ -279,14 +285,20 @@ int PIOc_closefile(int ncid)
if(file == NULL)
return PIO_EBADID;
ios = file->iosystem;
msg = 0;
msg = PIO_MSG_CLOSE_FILE;
if((file->mode & PIO_WRITE)){
PIOc_sync(ncid);
}
if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);

/* If async is in use and this is a comp tasks, then the compmaster
* sends a msg to the pio_msg_handler running on the IO master and
* waiting for a message. Then broadcast the ncid over the intercomm
* to the IO tasks. */
if(ios->async_interface && !ios->ioproc){
if(ios->comp_rank==0) {
mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
}
mpierr = MPI_Bcast(&(file->fh), 1, MPI_INT, ios->compmaster, ios->intercomm);
}

if(ios->ioproc){
Expand Down Expand Up @@ -325,7 +337,6 @@ int PIOc_closefile(int ncid)

int iret = pio_delete_file_from_list(ncid);


return ierr;
}

Expand All @@ -342,19 +353,22 @@ int PIOc_deletefile(const int iosysid, const char filename[])
int mpierr;
int chkerr;
iosystem_desc_t *ios;
size_t len;

ierr = PIO_NOERR;
ios = pio_get_iosystem_from_id(iosysid);

if(ios == NULL)
return PIO_EBADID;

msg = 0;
msg = PIO_MSG_DELETE_FILE;

if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
// mpierr = MPI_Bcast(iosysid,1, MPI_INT, ios->compmaster, ios->intercomm);
len = strlen(filename);
mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
}
// The barriers are needed to assure that no task is trying to operate on the file while it is being deleted.
if(ios->ioproc){
Expand All @@ -371,8 +385,6 @@ int PIOc_deletefile(const int iosysid, const char filename[])
}
// Special case - always broadcast the return from the
MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm);



return ierr;
}
Expand Down Expand Up @@ -405,9 +417,9 @@ int PIOc_sync (int ncid)
msg = PIO_MSG_SYNC;

if(ios->async_interface && ! ios->ioproc){
if(ios->compmaster)
if(ios->comp_rank == 0)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm);
}

if((file->mode & PIO_WRITE)){
Expand Down
12 changes: 11 additions & 1 deletion src/clib/pio_internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,8 @@ void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk);
}
#endif

/** These are the messages that can be sent over the intercomm when
* async is being used. */
enum PIO_MSG{
PIO_MSG_OPEN_FILE,
PIO_MSG_CREATE_FILE,
Expand Down Expand Up @@ -365,7 +367,15 @@ enum PIO_MSG{
PIO_MSG_SET_CHUNK_CACHE,
PIO_MSG_GET_CHUNK_CACHE,
PIO_MSG_SET_VAR_CHUNK_CACHE,
PIO_MSG_GET_VAR_CHUNK_CACHE
PIO_MSG_GET_VAR_CHUNK_CACHE,
PIO_MSG_INITDECOMP_DOF,
PIO_MSG_WRITEDARRAY,
PIO_MSG_READDARRAY,
PIO_MSG_SETERRORHANDLING,
PIO_MSG_FREEDECOMP,
PIO_MSG_CLOSE_FILE,
PIO_MSG_DELETE_FILE,
PIO_MSG_EXIT
};

#endif

0 comments on commit 0c4cbe6

Please sign in to comment.