Skip to content

Commit

Permalink
cleaning up call to netcdf layer in PIOc_nc_async
Browse files Browse the repository at this point in the history
  • Loading branch information
@edhartnett
edhartnett committed May 14, 2016
1 parent 9020e2d commit 0337dbe
Show file tree
Hide file tree
Showing 5 changed files with 101 additions and 109 deletions.
4 changes: 4 additions & 0 deletions src/clib/pio.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,10 @@ typedef struct file_desc_t
int mode;
struct wmulti_buffer buffer;
struct file_desc_t *next;

/** True if this task should participate in IO (only true for one
* task with netcdf serial files. */
int do_io;
} file_desc_t;

/**
Expand Down
61 changes: 44 additions & 17 deletions src/clib/pio_file.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,14 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
file->buffer.frame=NULL;
file->buffer.fillvalue=NULL;

/** Set to true if this task should participate in IO (only true for
* one task with netcdf serial files. */
if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF ||
ios->io_rank == 0)
file->do_io = 1;
else
file->do_io = 0;

/* If async is in use, and this is not an IO task, bcast the parameters. */
if (ios->async_interface)
{
Expand All @@ -75,13 +83,13 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,

len = strlen(filename);
if (!mpierr)
mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
}

/* Handle MPI errors. */
Expand Down Expand Up @@ -193,7 +201,7 @@ int PIOc_openfile(const int iosysid, int *ncidp, int *iotype,
** @param mode : The netcdf mode for the open operation
*/

int PIOc_createfile(const int iosysid, int *ncidp, int *iotype,
int PIOc_createfile(const int iosysid, int *ncidp, int *iotype,
const char filename[], const int mode)
{
int ierr;
Expand Down Expand Up @@ -235,18 +243,37 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype,
msg = PIO_MSG_CREATE_FILE;
file->mode = mode;

/** Set to true if this task should participate in IO (only true for
* one task with netcdf serial files. */
if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF ||
ios->io_rank == 0)
file->do_io = 1;
else
file->do_io = 0;

if(ios->async_interface && ! ios->ioproc){
if(ios->comp_rank==0)
mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
}

/* If async is in use, and this is not an IO task, bcast the parameters. */
if (ios->async_interface)
{
if (!ios->ioproc)
{
if(ios->comp_rank==0)
mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
if (!mpierr)
mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
if (!mpierr)
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
}

/* Handle MPI errors. */
mpierr = MPI_Bcast(&mpierr, 1, MPI_INT, ios->ioroot, ios->my_comm);
check_mpi(file, mpierr, __FILE__, __LINE__);
}

if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
Expand Down Expand Up @@ -288,9 +315,9 @@ int PIOc_createfile(const int iosysid, int *ncidp, int *iotype,
ierr = check_netcdf(file, ierr, __FILE__,__LINE__);

if(ierr == PIO_NOERR){
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm);
mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm);
file->mode = file->mode | PIO_WRITE; // This flag is implied by netcdf create functions but we need to know if its set
mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm);
mpierr = MPI_Bcast(&file->fh, 1, MPI_INT, ios->ioroot, ios->union_comm);
*ncidp = file->fh;
pio_add_to_file_list(file);
*ncidp = file->fh;
Expand Down Expand Up @@ -403,7 +430,7 @@ int PIOc_deletefile(const int iosysid, const char filename[])
if(ios->comp_rank==0)
mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
len = strlen(filename);
mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
}
// The barriers are needed to assure that no task is trying to operate on the file while it is being deleted.
Expand Down
129 changes: 45 additions & 84 deletions src/clib/pio_nc_async.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp,
int *unlimdimidp)
{
int msg = PIO_MSG_INQ; /** Message for async notification. */
iosystem_desc_t *ios; /** Pointer to io system information. */
file_desc_t *file; /** Pointer to file information. */
int ierr = PIO_NOERR; /** Return code from function calls. */
Expand All @@ -56,6 +55,7 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp,
{
if (!ios->ioproc)
{
int msg = PIO_MSG_INQ; /** Message for async notification. */
char ndims_present = ndimsp ? true : false;
char nvars_present = nvarsp ? true : false;
char ngatts_present = ngattsp ? true : false;
Expand Down Expand Up @@ -86,43 +86,29 @@ int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp,
/* If this is an IO task, then call the netCDF function. */
if (ios->ioproc)
{
switch (file->iotype)
{
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp);
break;
case PIO_IOTYPE_NETCDF4C:
#endif
case PIO_IOTYPE_NETCDF:
if (!file->iosystem->io_rank)
{
/* Should not be necessary to do this - nc_inq should
* handle null pointers. This has been reported as a bug
* to netCDF developers. */
int tmp_ndims, tmp_nvars, tmp_ngatts, tmp_unlimdimid;
ierr = nc_inq(ncid, &tmp_ndims, &tmp_nvars, &tmp_ngatts, &tmp_unlimdimid);
if (ndimsp)
*ndimsp = tmp_ndims;
if (nvarsp)
*nvarsp = tmp_nvars;
if (ngattsp)
*ngattsp = tmp_ngatts;
if (unlimdimidp)
*unlimdimidp = tmp_unlimdimid;
}
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp);
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}

if (file->iotype == PIO_IOTYPE_PNETCDF)
ierr = ncmpi_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp);
#endif /* _PNETCDF */
#ifdef _NETCDF
if (file->iotype == PIO_IOTYPE_NETCDF && file->do_io)
{
/* Should not be necessary to do this - nc_inq should
* handle null pointers. This has been reported as a bug
* to netCDF developers. */
int tmp_ndims, tmp_nvars, tmp_ngatts, tmp_unlimdimid;
ierr = nc_inq(ncid, &tmp_ndims, &tmp_nvars, &tmp_ngatts, &tmp_unlimdimid);
if (ndimsp)
*ndimsp = tmp_ndims;
if (nvarsp)
*nvarsp = tmp_nvars;
if (ngattsp)
*ngattsp = tmp_ngatts;
if (unlimdimidp)
*unlimdimidp = tmp_unlimdimid;
} else if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
ierr = nc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp);
#endif /* _NETCDF */
LOG((2, "PIOc_inq netcdf call returned %d", ierr));
}

Expand Down Expand Up @@ -274,25 +260,13 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep)
check_mpi(file, mpierr, __FILE__, __LINE__);
}

/* If this is an IO task, then call the netCDF function. */
/* If this is an IO task, then call the netCDF function. */
if (ios->ioproc)
{
switch (file->iotype)
{
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_inq_type(ncid, xtype, name, (size_t *)sizep);
break;
case PIO_IOTYPE_NETCDF4C:
#endif
case PIO_IOTYPE_NETCDF:
if (!file->iosystem->io_rank)
ierr = nc_inq_type(ncid, xtype, name, (size_t *)sizep);
break;
#endif
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
if (file->iotype == PIO_IOTYPE_PNETCDF)
{
switch (xtype)
{
case NC_UBYTE:
Expand Down Expand Up @@ -320,12 +294,12 @@ int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep)
*sizep = typelen;
if (name)
strcpy(name, "some type");
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}

}
#endif /* _PNETCDF */
#ifdef _NETCDF
if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
ierr = nc_inq_type(ncid, xtype, name, (size_t *)sizep);
#endif /* _NETCDF */
LOG((2, "PIOc_inq_type netcdf call returned %d", ierr));
}

Expand Down Expand Up @@ -445,9 +419,9 @@ int PIOc_inq_format (int ncid, int *formatp)
*/
int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp)
{
iosystem_desc_t *ios;
file_desc_t *file;
int ierr = PIO_NOERR;
iosystem_desc_t *ios; /** Pointer to io system information. */
file_desc_t *file; /** Pointer to file information. */
int ierr = PIO_NOERR; /** Return code from function calls. */
int mpierr = MPI_SUCCESS, mpierr2; /** Return code from MPI function codes. */

LOG((1, "PIOc_inq_dim"));
Expand Down Expand Up @@ -487,30 +461,17 @@ int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp)
check_mpi(file, mpierr, __FILE__, __LINE__);
}

/* Make the call to the netCDF layer. */
if(ios->ioproc){
switch(file->iotype){
#ifdef _NETCDF
#ifdef _NETCDF4
case PIO_IOTYPE_NETCDF4P:
ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);;
break;
case PIO_IOTYPE_NETCDF4C:
#endif
case PIO_IOTYPE_NETCDF:
if (ios->io_rank == 0){
ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);;
}
break;
#endif
/* If this is an IO task, then call the netCDF function. */
if (ios->ioproc)
{
#ifdef _PNETCDF
case PIO_IOTYPE_PNETCDF:
ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);;
break;
#endif
default:
ierr = iotype_error(file->iotype,__FILE__,__LINE__);
}
if (file->iotype == PIO_IOTYPE_PNETCDF)
ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);;
#endif /* _PNETCDF */
#ifdef _NETCDF
if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);;
#endif /* _NETCDF */
}

/* Broadcast and check the return code. */
Expand Down
12 changes: 6 additions & 6 deletions src/clib/pioc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,10 +350,9 @@ int PIOc_InitDecomp_bc(const int iosysid, const int basetype,const int ndims, co
** @param rearr the rearranger to use by default, this may be overriden in the @ref PIO_initdecomp
** @param iosysidp index of the defined system descriptor
*/

int PIOc_Init_Intracomm(const MPI_Comm comp_comm,
const int num_iotasks, const int stride,
const int base,const int rearr, int *iosysidp)
int PIOc_Init_Intracomm(const MPI_Comm comp_comm, const int num_iotasks,
const int stride, const int base, const int rearr,
int *iosysidp)
{
iosystem_desc_t *iosys;
int ierr = PIO_NOERR;
Expand Down Expand Up @@ -437,10 +436,11 @@ int PIOc_Init_Intracomm(const MPI_Comm comp_comm,
&(iosys->iogroup)),__FILE__,__LINE__);

/* Create an MPI communicator for the IO tasks. */
CheckMPIReturn(MPI_Comm_create(iosys->comp_comm, iosys->iogroup, &(iosys->io_comm)),__FILE__,__LINE__);
CheckMPIReturn(MPI_Comm_create(iosys->comp_comm, iosys->iogroup, &(iosys->io_comm))
,__FILE__,__LINE__);

/* For the tasks that are doing IO, get their rank. */
if(iosys->ioproc)
if (iosys->ioproc)
CheckMPIReturn(MPI_Comm_rank(iosys->io_comm, &(iosys->io_rank)),__FILE__,__LINE__);
else
iosys->io_rank = -1;
Expand Down
4 changes: 2 additions & 2 deletions tests/unit/test_intercomm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,8 +361,8 @@ main(int argc, char **argv)
* and when the do, they should go straight to finalize. */
if (comp_task)
{
/* for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) */
for (int fmt = 0; fmt < 1; fmt++)
for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++)
/* for (int fmt = 0; fmt < 1; fmt++) */
{
int ncid, varid, dimid;
PIO_Offset start[NDIM], count[NDIM] = {0};
Expand Down

0 comments on commit 0337dbe

Please sign in to comment.