From 647e71fcea8165a262ebf55665cb4242f8e1dbed Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 13 Jan 2016 11:36:50 -0700 Subject: [PATCH 01/48] allocate requests in chunks --- src/clib/pio_darray.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index 1e7f8e07ae74..c5cf3f08cbb5 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -282,13 +282,17 @@ void compute_buffer_init(iosystem_desc_t ios) // IOBUF, iodesc->llen, iodesc->basetype); int reqn=0; - if(vdesc->request == NULL){ - vdesc->request = malloc(sizeof(int)*iodesc->maxregions); - for(int i=0;imaxregions;i++){ + + if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + + for(int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ vdesc->request[i]=NC_REQ_NULL; } + reqn = vdesc->nreqs; }else{ - while(vdesc->request[reqn] != NC_REQ_NULL){ + while(vdesc->request[reqn] != NC_REQ_NULL ){ reqn++; } } @@ -581,11 +585,14 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ bufptr = (void *)((char *) IOBUF + nv*tsize*llen); int reqn=0; - if(vdesc->request == NULL){ - vdesc->request = malloc(sizeof(int)*maxregions); - for(int i=0;inreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ + vdesc->request = realloc(vdesc->request, + sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); + + for(int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ vdesc->request[i]=NC_REQ_NULL; } + reqn = vdesc->nreqs; }else{ while(vdesc->request[reqn] != NC_REQ_NULL){ reqn++; @@ -1419,9 +1426,14 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) // printf("%s %d %d\n",__FILE__,__LINE__,rcnt); // } if(rcnt>0){ - // if(file->iosystem->io_rank==0){ - // printf("%s %d %d\n",__FILE__,__LINE__,rcnt); - // } + /* + if(file->iosystem->io_rank==0){ + printf("%s %d %d ",__FILE__,__LINE__,rcnt); + for(int i=0; ifh, rcnt, request,status); } for(int i=0; i Date: Wed, 13 Jan 2016 23:08:53 -0700 Subject: [PATCH 02/48] separate serial and parallel in pio_darray --- src/clib/pio.h | 1 + src/clib/pio_darray.c | 570 +++++++++++++++++++++++++++------------- src/clib/pio_spmd.c | 1 + src/clib/pioc.c | 27 ++ src/flib/pio.F90 | 2 +- src/flib/piolib_mod.F90 | 15 +- 6 files changed, 438 insertions(+), 178 deletions(-) diff --git a/src/clib/pio.h b/src/clib/pio.h index 9534c6b02a9e..f4e8fe9251c2 100644 --- a/src/clib/pio.h +++ b/src/clib/pio.h @@ -601,6 +601,7 @@ int PIOc_set_blocksize(const int newblocksize); int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf); int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf); int PIOc_get_var_schar (int ncid, int varid, signed char *buf); + int PIOc_iotype_available(const int iotype); #if defined(__cplusplus) } diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index c5cf3f08cbb5..ee0476628617 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -443,7 +443,6 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ } switch(file->iotype){ -#ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: for(int nv=0; nvio_rank==0){ - for(int iorank=0;iorankio_comm); // handshake - tell the sending task I'm ready - mpierr = MPI_Recv( &buflen, 1, MPI_INT, iorank, 1, ios->io_comm, &status); - if(buflen>0){ - mpierr = MPI_Recv( tstart, fndims, MPI_OFFSET, iorank, ios->num_iotasks+iorank, ios->io_comm, &status); - mpierr = MPI_Recv( tcount, fndims, MPI_OFFSET, iorank,2*ios->num_iotasks+iorank, ios->io_comm, &status); - } - } - - if(buflen>0){ - for(int nv=0; nvrecord >= 0 && ndimsllen, buflen, iodesc->maxiobuflen, nvars, tcount[0]); - - - if(iorank>0){ - bufptr = malloc(buflen*tsize) ; - mpierr = MPI_Recv( bufptr, buflen, basetype, iorank, iorank, ios->io_comm, &status); - }else{ - bufptr = (void *)((char *) IOBUF+ tsize*(nv*llen + region->loffset)); - } - // printf("%s %d %d %X %X\n",__FILE__,__LINE__,vid[nv],IOBUF,bufptr); - if(basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid[nv], tstart, tcount, (const int *) bufptr); - }else if(basetype == MPI_DOUBLE || basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid[nv], tstart, tcount, (const double *) bufptr); - }else if(basetype == MPI_FLOAT || basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid,vid[nv], tstart, tcount, (const float *) bufptr); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) basetype); - } - if(iorank>0){ - free(bufptr); - } - if(ierr != PIO_NOERR){ - for(i=0;iio_rank < num_aiotasks ){ - buflen=1; - for(i=0;iio_rank,tstart[0],tstart[1],tcount[0],tcount[1],buflen,ndims,fndims); - mpierr = MPI_Recv( &ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve - mpierr = MPI_Rsend( &buflen, 1, MPI_INT, 0, 1, ios->io_comm); - if(buflen>0) { - mpierr = MPI_Rsend( tstart, fndims, MPI_OFFSET, 0, ios->num_iotasks+ios->io_rank, ios->io_comm); - mpierr = MPI_Rsend( tcount, fndims, MPI_OFFSET, 0,2*ios->num_iotasks+ios->io_rank, ios->io_comm); - - for(int nv=0; nvloffset)); - // printf("%d %d %d\n",__LINE__,iodesc->llen,region->loffset); - // printf("%d %d %d %d %d %d\n",__LINE__,((int *)bufptr)[0],((int *)bufptr)[1],((int *)bufptr)[2],((int *)bufptr)[3],((int *)bufptr)[4]); - mpierr = MPI_Send( bufptr, buflen, basetype, 0, ios->io_rank, ios->io_comm); - } - } - } - break; - } - break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: @@ -639,6 +553,191 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ return ierr; } +/** @brief Write a set of one or more aggregated arrays to output file + * @ingroup PIO_write_darray + * + * This routine is used if aggregation is enabled, data is already on the + * io-tasks + * @param[in] file: a pointer to the open file descriptor for the file that will be written to + * @param[in] nvars: the number of variables to be written with this decomposition + * @param[in] vid: an array of the variable ids to be written + * @param[in] iodesc_ndims: the number of dimensions explicitly in the iodesc + * @param[in] basetype : the basic type of the minimal data unit + * @param[in] gsize : array of the global dimensions of the field to be written + * @param[in] maxregions : max number of blocks to be written from this iotask + * @param[in] firstregion : pointer to the first element of a linked list of region descriptions. + * @param[in] llen : length of the iobuffer on this task for a single field + * @param[in] maxiobuflen : maximum llen participating + * @param[in] num_aiotasks : actual number of iotasks participating + * @param[in] IOBUF: the buffer to be written from this mpi task + * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF + */ +int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const int vid[], + const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset gsize[], + const int maxregions, io_region *firstregion, const PIO_Offset llen, + const int maxiobuflen, const int num_aiotasks, + void *IOBUF, const int frame[]) + { + iosystem_desc_t *ios; + var_desc_t *vdesc; + int ierr; + int i; + int msg; + int mpierr; + int dsize; + MPI_Status status; + PIO_Offset usage; + int fndims; + PIO_Offset tdsize; + int tsize; + int ncid; + tdsize=0; + ierr = PIO_NOERR; +#ifdef TIMING + GPTLstart("PIO:write_darray_multi_nc_serial"); +#endif + + ios = file->iosystem; + if(ios == NULL){ + fprintf(stderr,"Failed to find iosystem handle \n"); + return PIO_EBADID; + } + vdesc = (file->varlist)+vid[0]; + ncid = file->fh; + + if(vdesc == NULL){ + fprintf(stderr,"Failed to find variable handle %d\n",vid[0]); + return PIO_EBADID; + } + msg = 0; + + if(ios->async_interface && ! ios->ioproc){ + if(ios->comp_rank==0) + mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); + mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); + } + + ierr = PIOc_inq_varndims(file->fh, vid[0], &fndims); + MPI_Type_size(basetype, &tsize); + + if(ios->ioproc){ + io_region *region; + int regioncnt; + int rrcnt; + void *bufptr; + int buflen, j; + size_t tmp_start[fndims*maxregions]; + size_t tmp_count[fndims*maxregions]; + + int ndims = iodesc_ndims; + + ncid = file->fh; + region = firstregion; + + + rrcnt=0; + for(regioncnt=0;regioncntrecord >= 0){ + for(i=fndims-ndims;istart[i-(fndims-ndims)]; + tmp_count[i+regioncnt*fndims] = region->count[i-(fndims-ndims)]; + } + + if(fndims>1 && ndims0){ + tmp_count[regioncnt*fndims] = 1; + tmp_start[regioncnt*fndims] = frame[0]; + }else if(fndims==ndims){ + tmp_start[regioncnt*fndims]+=vdesc->record; + } + // Non-time dependent array + }else{ + for( i=0;istart[i]; + tmp_count[i+regioncnt*fndims] = region->count[i]; + } + } + } + region = region->next; + } + if(ios->io_rank>0){ + mpierr = MPI_Recv( &ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve + MPI_Send( &llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); + if(llen>0){ + MPI_Send( &maxregions, 1, MPI_INT, 0, ios->io_rank+ios->num_iotasks, ios->io_comm); + MPI_Send( tmp_start, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+2*ios->num_iotasks, ios->io_comm); + MPI_Send( tmp_count, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+3*ios->num_iotasks, ios->io_comm); + MPI_Send( IOBUF, nvars*llen, basetype, 0, ios->io_rank+4*ios->num_iotasks, ios->io_comm); + } + }else{ + size_t rlen; + int rregions; + size_t start[fndims], count[fndims]; + size_t loffset; + mpierr = MPI_Type_size(basetype, &dsize); + + for(int rtask=0; rtasknum_iotasks; rtask++){ + if(rtask>0){ + mpierr = MPI_Send( &ierr, 1, MPI_INT, rtask, 0, ios->io_comm); // handshake - tell the sending task I'm ready + MPI_Recv( &rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); + if(rlen>0){ + MPI_Recv( &rregions, 1, MPI_INT, rtask, rtask+ios->num_iotasks, ios->io_comm, &status); + MPI_Recv( tmp_start, rregions*fndims, MPI_OFFSET, rtask, rtask+2*ios->num_iotasks, ios->io_comm, &status); + MPI_Recv( tmp_count, rregions*fndims, MPI_OFFSET, rtask, rtask+3*ios->num_iotasks, ios->io_comm, &status); + MPI_Recv( IOBUF, nvars*rlen, basetype, rtask, rtask+4*ios->num_iotasks, ios->io_comm, &status); + } + }else{ + rlen = llen; + rregions = maxregions; + } + if(rlen>0){ + loffset = 0; + for(regioncnt=0;regioncntmaxregions;regioncnt++){ + } // if(rlen>0) + } // for(int rtask=0; rtasknum_iotasks; rtask++){ + + } + } // if(ios->ioproc) + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +#ifdef TIMING + GPTLstop("PIO:write_darray_multi_nc_serial"); +#endif + + return ierr; + } + + + /** @brief Write one or more arrays with the same IO decomposition to the file * @ingroup PIO_write_darray * @@ -716,12 +815,26 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con else{ vdesc0->iobuf = array; } */ - ierr = pio_write_darray_multi_nc(file, nvars, vid, - iodesc->ndims, iodesc->basetype, iodesc->gsize, - iodesc->maxregions, iodesc->firstregion, iodesc->llen, - iodesc->maxiobuflen, iodesc->num_aiotasks, - vdesc0->iobuf, frame); - + switch(file->iotype){ + case PIO_IOTYPE_NETCDF4P: + case PIO_IOTYPE_PNETCDF: + ierr = pio_write_darray_multi_nc(file, nvars, vid, + iodesc->ndims, iodesc->basetype, iodesc->gsize, + iodesc->maxregions, iodesc->firstregion, iodesc->llen, + iodesc->maxiobuflen, iodesc->num_aiotasks, + vdesc0->iobuf, frame); + break; + case PIO_IOTYPE_NETCDF4C: + case PIO_IOTYPE_NETCDF: + ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, + iodesc->ndims, iodesc->basetype, iodesc->gsize, + iodesc->maxregions, iodesc->firstregion, iodesc->llen, + iodesc->maxiobuflen, iodesc->num_aiotasks, + vdesc0->iobuf, frame); + brel(vdesc0->iobuf); + vdesc0->iobuf = NULL; + + } @@ -745,13 +858,25 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con } } } - - ierr = pio_write_darray_multi_nc(file, nvars, vid, - iodesc->ndims, iodesc->basetype, iodesc->gsize, - iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, - iodesc->holegridsize, iodesc->num_aiotasks, - vdesc0->fillbuf, frame); - + switch(file->iotype){ + case PIO_IOTYPE_NETCDF4P: + case PIO_IOTYPE_PNETCDF: + ierr = pio_write_darray_multi_nc(file, nvars, vid, + iodesc->ndims, iodesc->basetype, iodesc->gsize, + iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, + iodesc->holegridsize, iodesc->num_aiotasks, + vdesc0->fillbuf, frame); + break; + case PIO_IOTYPE_NETCDF4C: + case PIO_IOTYPE_NETCDF: + ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, + iodesc->ndims, iodesc->basetype, iodesc->gsize, + iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, + iodesc->holegridsize, iodesc->num_aiotasks, + vdesc0->fillbuf, frame); + brel(vdesc0->fillbuf); + vdesc0->fillbuf = NULL; + } } flush_output_buffer(file, flushtodisk, 0); @@ -1064,7 +1189,7 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con } #endif -/** @brief Read an array of data from a file to the IO library. +/** @brief Read an array of data from a file to the (parallel) IO library. * @ingroup PIO_read_darray */ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF) @@ -1157,7 +1282,6 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void } switch(file->iotype){ -#ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: if(iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ @@ -1170,79 +1294,6 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void fprintf(stderr,"Type not recognized %d in pioc_read_darray\n",(int) iodesc->basetype); } break; - case PIO_IOTYPE_NETCDF4C: -#endif - case PIO_IOTYPE_NETCDF: - if(ios->io_rank>0){ - tmp_bufsize=1; - for( i=0;iio_rank, ios->io_comm); - if(tmp_bufsize > 0){ - MPI_Send( tmp_start, ndims, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); - // printf("%s %d %d\n",__FILE__,__LINE__,tmp_bufsize); - MPI_Recv( bufptr, tmp_bufsize, iodesc->basetype, 0, ios->io_rank, ios->io_comm, &status); - } - // printf("%s %d %d %d %d %d %d %d\n",__FILE__,__LINE__,regioncnt,tmp_start[1],tmp_start[2],tmp_count[1],tmp_count[2], ndims); - }else if(ios->io_rank==0){ - for( i=ios->num_iotasks-1; i>=0; i--){ - if(i==0){ - for(int k=0;kloffset); - }else{ - MPI_Recv(tmp_count, ndims, MPI_OFFSET, i, i, ios->io_comm, &status); - } - tmp_bufsize=1; - for(int j=0;j0){ - if(i==0){ - for(int k=0;kio_comm, &status); - } - if(iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - if(i>0) - bufptr = malloc(tmp_bufsize *sizeof(double)); - ierr = nc_get_vara_double (file->fh, vid, tmp_start, tmp_count, bufptr); - }else if(iodesc->basetype == MPI_INTEGER){ - if(i>0) - bufptr = malloc(tmp_bufsize *sizeof(int)); - ierr = nc_get_vara_int (file->fh, vid, tmp_start, tmp_count, bufptr); - }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - if(i>0) - bufptr = malloc(tmp_bufsize *sizeof(float)); - ierr = nc_get_vara_float (file->fh, vid, tmp_start, tmp_count, bufptr); - }else{ - fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) iodesc->basetype); - } - - if(ierr != PIO_NOERR){ - printf("%s %d ",__FILE__,__LINE__); - for(int j=0;j0){ - // printf("%s %d %d %d\n",__FILE__,__LINE__,i,tmp_bufsize); - MPI_Rsend(bufptr, tmp_bufsize, iodesc->basetype, i, i, ios->io_comm); - free(bufptr); - } - } - } - } - break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: @@ -1292,6 +1343,168 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void } +/** @brief Read an array of data from a file to the (serial) IO library. + * @ingroup PIO_read_darray + */ +int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vid, void *IOBUF) +{ + int ierr=PIO_NOERR; + iosystem_desc_t *ios; + var_desc_t *vdesc; + int ndims, fndims; + MPI_Status status; + int i; + +#ifdef TIMING + GPTLstart("PIO:read_darray_nc_serial"); +#endif + ios = file->iosystem; + if(ios == NULL) + return PIO_EBADID; + + vdesc = (file->varlist)+vid; + + if(vdesc == NULL) + return PIO_EBADID; + + ndims = iodesc->ndims; + ierr = PIOc_inq_varndims(file->fh, vid, &fndims); + + if(fndims==ndims) + vdesc->record=-1; + + if(ios->ioproc){ + io_region *region; + size_t start[fndims]; + size_t count[fndims]; + size_t tmp_start[fndims*iodesc->maxregions]; + size_t tmp_count[fndims*iodesc->maxregions]; + size_t tmp_bufsize; + int regioncnt; + void *bufptr; + int tsize; + + int rrlen=0; + + // buffer is incremented by byte and loffset is in terms of the iodessc->basetype + // so we need to multiply by the size of the basetype + // We can potentially allow for one iodesc to have multiple datatypes by allowing the + // calling program to change the basetype. + region = iodesc->firstregion; + MPI_Type_size(iodesc->basetype, &tsize); + if(fndims>ndims){ + ndims++; + if(vdesc->record<0) + vdesc->record=0; + } + for(regioncnt=0;regioncntmaxregions;regioncnt++){ + if(region==NULL || iodesc->llen==0){ + for(i=0;irecord >= 0 && fndims>1){ + tmp_start[regioncnt*fndims] = vdesc->record; + for(i=1;istart[i-1]; + tmp_count[i+regioncnt*fndims] = region->count[i-1]; + } + if(tmp_count[1+regioncnt*fndims]>0) + tmp_count[regioncnt*fndims] = 1; + }else{ + // Non-time dependent array + for(i=0;istart[i]; + tmp_count[i+regioncnt*fndims] = region->count[i]; + } + } + } + if(region != NULL) + region = region->next; + } // for(regioncnt=0;...) + + if(ios->io_rank>0){ + MPI_Send( &(iodesc->llen), 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); + if(iodesc->llen > 0){ + MPI_Send( &(iodesc->maxregions), 1, MPI_INT, 0, ios->num_iotasks+ios->io_rank, ios->io_comm); + MPI_Send( tmp_count, iodesc->maxregions*fndims, MPI_OFFSET, 0, 2*ios->num_iotasks+ios->io_rank, ios->io_comm); + MPI_Send( tmp_start, iodesc->maxregions*fndims, MPI_OFFSET, 0, 3*ios->num_iotasks+ios->io_rank, ios->io_comm); + MPI_Recv(IOBUF, iodesc->llen, iodesc->basetype, 0, 4*ios->num_iotasks+ios->io_rank, ios->io_comm, &status); + } + }else if(ios->io_rank==0){ + int maxregions=0; + size_t loffset, regionsize; + size_t this_start[fndims*iodesc->maxregions]; + size_t this_count[fndims*iodesc->maxregions]; + // for( i=ios->num_iotasks-1; i>=0; i--){ + for(int rtask=1;rtask<=ios->num_iotasks;rtask++){ + if(rtasknum_iotasks){ + MPI_Recv(&tmp_bufsize, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status); + if(tmp_bufsize>0){ + MPI_Recv(&maxregions, 1, MPI_INT, rtask, ios->num_iotasks+rtask, ios->io_comm, &status); + MPI_Recv(this_count, maxregions*fndims, MPI_OFFSET, rtask, 2*ios->num_iotasks+rtask, ios->io_comm, &status); + MPI_Recv(this_start, maxregions*fndims, MPI_OFFSET, rtask, 3*ios->num_iotasks+rtask, ios->io_comm, &status); + } + }else{ + maxregions=iodesc->maxregions; + tmp_bufsize=iodesc->llen; + } + loffset = 0; + for(regioncnt=0;regioncntnum_iotasks){ + for(int m=0; mbasetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ + ierr = nc_get_vara_double (file->fh, vid,start, count, bufptr); + }else if(iodesc->basetype == MPI_INTEGER){ + ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); + }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ + ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); + }else{ + fprintf(stderr,"Type not recognized %d in pioc_write_darray_nc_serial\n",(int) iodesc->basetype); + } + if(ierr != PIO_NOERR){ + for(int i=0;inum_iotasks){ + MPI_Send(IOBUF, tmp_bufsize, iodesc->basetype, rtask,4*ios->num_iotasks+rtask, ios->io_comm); + } + } + } + } + + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); +#ifdef TIMING + GPTLstop("PIO:read_darray_nc_serial"); +#endif + + return ierr; +} + + /** @brief Read a field from a file to the IO library. * @ingroup PIO_read_darray * @@ -1337,11 +1550,16 @@ int PIOc_read_darray(const int ncid, const int vid, const int ioid, const PIO_Of } switch(file->iotype){ - case PIO_IOTYPE_PNETCDF: case PIO_IOTYPE_NETCDF: - case PIO_IOTYPE_NETCDF4P: case PIO_IOTYPE_NETCDF4C: + ierr = pio_read_darray_nc_serial(file, iodesc, vid, iobuf); + break; + case PIO_IOTYPE_PNETCDF: + case PIO_IOTYPE_NETCDF4P: ierr = pio_read_darray_nc(file, iodesc, vid, iobuf); + break; + default: + ierr = iotype_error(file->iotype,__FILE__,__LINE__); } if(iodesc->rearranger > 0){ ierr = rearrange_io2comp(*ios, iodesc, iobuf, array); diff --git a/src/clib/pio_spmd.c b/src/clib/pio_spmd.c index 4b5fcc423ccd..9645ed45b0f7 100644 --- a/src/clib/pio_spmd.c +++ b/src/clib/pio_spmd.c @@ -248,6 +248,7 @@ int pio_swapm(void *sndbuf, int sndlths[], int sdispls[], MPI_Datatype stypes CheckMPIReturn(MPI_Comm_size(comm, &nprocs),__FILE__,__LINE__); CheckMPIReturn(MPI_Comm_rank(comm, &mytask),__FILE__,__LINE__); + if(max_requests == 0) { #ifdef DEBUG int totalrecv=0; diff --git a/src/clib/pioc.c b/src/clib/pioc.c index a2d7a6e0aa48..eb9a790ffa2f 100644 --- a/src/clib/pioc.c +++ b/src/clib/pioc.c @@ -547,3 +547,30 @@ int PIOc_iotask_rank(const int iosysid, int *iorank) } +/** + ** @brief return true if this iotype is supported in the build, 0 otherwise + */ + +int PIOc_iotype_available(const int iotype) +{ + + switch(iotype){ +#ifdef _NETCDF +#ifdef _NETCDF4 + case PIO_IOTYPE_NETCDF4P: + case PIO_IOTYPE_NETCDF4C: + return(1); +#endif + case PIO_IOTYPE_NETCDF: + return(1); +#endif +#ifdef _PNETCDF + case PIO_IOTYPE_PNETCDF: + return(1); + break; +#endif + default: + return(0); + } + +} diff --git a/src/flib/pio.F90 b/src/flib/pio.F90 index 0db4c2127f9f..8b1da831efbc 100644 --- a/src/flib/pio.F90 +++ b/src/flib/pio.F90 @@ -16,7 +16,7 @@ module pio pio_seterrorhandling, pio_setframe, pio_init, pio_get_local_array_size, & pio_freedecomp, pio_syncfile, & pio_finalize, pio_set_hint, pio_getnumiotasks, pio_file_is_open, & - PIO_deletefile, PIO_get_numiotasks + PIO_deletefile, PIO_get_numiotasks, PIO_iotype_available use pio_types, only : io_desc_t, file_desc_t, var_desc_t, iosystem_desc_t, & pio_int, pio_real, pio_double, pio_noerr, iotype_netcdf, & diff --git a/src/flib/piolib_mod.F90 b/src/flib/piolib_mod.F90 index 89453d50d88f..79f1573fcb9e 100644 --- a/src/flib/piolib_mod.F90 +++ b/src/flib/piolib_mod.F90 @@ -49,7 +49,8 @@ module piolib_mod PIO_set_hint, & PIO_FILE_IS_OPEN, & PIO_deletefile, & - PIO_get_numiotasks + PIO_get_numiotasks, & + PIO_iotype_available #ifdef MEMCHK !> this is an internal variable for memory leak debugging @@ -1191,6 +1192,18 @@ end function PIOc_get_numiotasks end subroutine getnumiotasks + logical function pio_iotype_available( iotype) result(available) + integer, intent(in) :: iotype + interface + integer(C_INT) function PIOc_iotype_available(iotype) & + bind(C,name="PIOc_iotype_available") + use iso_c_binding + integer(C_INT), intent(in), value :: iotype + end function PIOc_iotype_available + end interface + available= (PIOc_iotype_available(iotype) == 1) + + end function pio_iotype_available !> From 0cf9924548b5ac4b3c785985bef4e535af788610 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 13 Jan 2016 23:16:41 -0700 Subject: [PATCH 03/48] minor bug fix --- src/clib/pio_darray.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index ee0476628617..53dca38bb86c 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -662,8 +662,8 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i tmp_count[i+regioncnt*fndims] = region->count[i]; } } + region = region->next; } - region = region->next; } if(ios->io_rank>0){ mpierr = MPI_Recv( &ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status); // task0 is ready to recieve From dc374a14dd1b2fbc2c5a506f7f77d184ce23ceac Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 15 Jan 2016 14:49:20 -0700 Subject: [PATCH 04/48] latest changes from cesm dev --- src/clib/bget.c | 14 +++++---- src/clib/pio_darray.c | 67 ++++++++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 25 deletions(-) diff --git a/src/clib/bget.c b/src/clib/bget.c index 9c95fe63d88c..752ac219e749 100644 --- a/src/clib/bget.c +++ b/src/clib/bget.c @@ -397,6 +397,7 @@ BGET CONFIGURATION ================== */ +//#define PIO_USE_MALLOC 1 #ifdef PIO_USE_MALLOC #include #endif @@ -610,11 +611,13 @@ void *bget(requested_size) #endif #ifdef PIO_USE_MALLOC - if(requested_size>maxsize){ - maxsize=requested_size; - printf("%s %d %d\n",__FILE__,__LINE__,maxsize); - } - return(malloc(requested_size)); + // if(requested_size>maxsize){ + // maxsize=requested_size; + // printf("%s %d %d\n",__FILE__,__LINE__,maxsize); + // } + buf = malloc(requested_size); + // printf("bget allocate %ld %x\n",requested_size,buf); + return(buf); #endif @@ -875,6 +878,7 @@ void brel(buf) struct bfhead *b, *bn; #ifdef PIO_USE_MALLOC + // printf("bget free %d %x\n",__LINE__,buf); free(buf); return; #endif diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index 53dca38bb86c..b7de478d0b50 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -648,13 +648,6 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i tmp_start[i+regioncnt*fndims] = region->start[i-(fndims-ndims)]; tmp_count[i+regioncnt*fndims] = region->count[i-(fndims-ndims)]; } - - if(fndims>1 && ndims0){ - tmp_count[regioncnt*fndims] = 1; - tmp_start[regioncnt*fndims] = frame[0]; - }else if(fndims==ndims){ - tmp_start[regioncnt*fndims]+=vdesc->record; - } // Non-time dependent array }else{ for( i=0;iio_rank+ios->num_iotasks, ios->io_comm); MPI_Send( tmp_start, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+2*ios->num_iotasks, ios->io_comm); MPI_Send( tmp_count, maxregions*fndims, MPI_OFFSET, 0, ios->io_rank+3*ios->num_iotasks, ios->io_comm); + // printf("%s %d %ld\n",__FILE__,__LINE__,nvars*llen); MPI_Send( IOBUF, nvars*llen, basetype, 0, ios->io_rank+4*ios->num_iotasks, ios->io_comm); } }else{ @@ -689,6 +683,7 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i MPI_Recv( &rregions, 1, MPI_INT, rtask, rtask+ios->num_iotasks, ios->io_comm, &status); MPI_Recv( tmp_start, rregions*fndims, MPI_OFFSET, rtask, rtask+2*ios->num_iotasks, ios->io_comm, &status); MPI_Recv( tmp_count, rregions*fndims, MPI_OFFSET, rtask, rtask+3*ios->num_iotasks, ios->io_comm, &status); + // printf("%s %d %d %ld\n",__FILE__,__LINE__,rtask,nvars*rlen); MPI_Recv( IOBUF, nvars*rlen, basetype, rtask, rtask+4*ios->num_iotasks, ios->io_comm, &status); } }else{ @@ -706,6 +701,18 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i for(int nv=0; nvrecord>=0){ + if(fndims>1 && ndims0){ + count[0] = 1; + start[0] = frame[nv]; + }else if(fndims==ndims){ + start[0]+=vdesc->record; + } + } + + + + if(basetype == MPI_INTEGER){ ierr = nc_put_vara_int (ncid, vid[nv], start, count, (const int *) bufptr); }else if(basetype == MPI_DOUBLE || basetype == MPI_REAL8){ @@ -721,6 +728,12 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i fprintf(stderr,"vid %d dim %d start %ld count %ld \n",vid[nv],i,start[i],count[i]); } } + size_t tsize; + tsize = 1; + for(int i=0;imaxregions;regioncnt++){ } // if(rlen>0) } // for(int rtask=0; rtasknum_iotasks; rtask++){ @@ -786,9 +799,9 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con } if(iodesc->rearranger>0){ if(rlen>0){ - // printf("rlen = %ld\n",rlen); MPI_Type_size(iodesc->basetype, &vsize); - //iobuf = malloc((size_t) vsize* (size_t) rlen); + //printf("rlen*vsize = %ld\n",rlen*vsize); + vdesc0->iobuf = bget((size_t) vsize* (size_t) rlen); if(vdesc0->iobuf==NULL){ printf("%s %d %d %ld\n",__FILE__,__LINE__,nvars,vsize*rlen); @@ -831,8 +844,11 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con iodesc->maxregions, iodesc->firstregion, iodesc->llen, iodesc->maxiobuflen, iodesc->num_aiotasks, vdesc0->iobuf, frame); - brel(vdesc0->iobuf); - vdesc0->iobuf = NULL; + if(vdesc0->iobuf != NULL){ + brel(vdesc0->iobuf); + vdesc0->iobuf = NULL; + } + break; } @@ -845,6 +861,7 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con } vdesc0->fillbuf = bget(iodesc->holegridsize*vsize*nvars); + //printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); if(vsize==4){ for(int nv=0;nvholegridsize;i++){ @@ -869,13 +886,19 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con break; case PIO_IOTYPE_NETCDF4C: case PIO_IOTYPE_NETCDF: - ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, + /* ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, iodesc->ndims, iodesc->basetype, iodesc->gsize, iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, iodesc->holegridsize, iodesc->num_aiotasks, vdesc0->fillbuf, frame); - brel(vdesc0->fillbuf); - vdesc0->fillbuf = NULL; + */ + /* if(vdesc0->fillbuf != NULL){ + printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); + brel(vdesc0->fillbuf); + vdesc0->fillbuf = NULL; + } + */ + break; } } @@ -1393,7 +1416,6 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vi region = iodesc->firstregion; MPI_Type_size(iodesc->basetype, &tsize); if(fndims>ndims){ - ndims++; if(vdesc->record<0) vdesc->record=0; } @@ -1407,7 +1429,7 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vi }else{ if(vdesc->record >= 0 && fndims>1){ tmp_start[regioncnt*fndims] = vdesc->record; - for(i=1;istart[i-1]; tmp_count[i+regioncnt*fndims] = region->count[i-1]; } @@ -1415,11 +1437,15 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vi tmp_count[regioncnt*fndims] = 1; }else{ // Non-time dependent array - for(i=0;istart[i]; tmp_count[i+regioncnt*fndims] = region->count[i]; } } + /* for(i=0;inext; @@ -1481,10 +1507,11 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vi }else{ fprintf(stderr,"Type not recognized %d in pioc_write_darray_nc_serial\n",(int) iodesc->basetype); } + if(ierr != PIO_NOERR){ - for(int i=0;i Date: Mon, 1 Feb 2016 18:32:07 -0500 Subject: [PATCH 05/48] fixed compile problem when timing turned off --- examples/c/example1.c | 5 ++++- examples/c/example2.c | 4 +++- examples/c/examplePio.c | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/c/example1.c b/examples/c/example1.c index 698eb9c37bfc..8402c8261259 100644 --- a/examples/c/example1.c +++ b/examples/c/example1.c @@ -10,6 +10,7 @@ * processors. */ +#include #include #include #include @@ -280,6 +281,9 @@ int check_file(int ntasks, char *filename) { /** Used for command line processing. */ int c; + /** Return value. */ + int ret; + /* Parse command line. */ while ((c = getopt(argc, argv, "v")) != -1) switch (c) @@ -293,7 +297,6 @@ int check_file(int ntasks, char *filename) { #ifdef TIMING /* Initialize the GPTL timing library. */ - int ret; if ((ret = GPTLinitialize ())) return ret; #endif diff --git a/examples/c/example2.c b/examples/c/example2.c index 1f16a7ae1a4d..f4b75d075b74 100644 --- a/examples/c/example2.c +++ b/examples/c/example2.c @@ -21,6 +21,7 @@ * */ +#include #include #include #include @@ -437,6 +438,8 @@ int main(int argc, char* argv[]) /** Needed for command line processing. */ int c; + int ret; + /* Parse command line. */ while ((c = getopt(argc, argv, "v")) != -1) switch (c) @@ -450,7 +453,6 @@ int main(int argc, char* argv[]) #ifdef TIMING /* Initialize the GPTL timing library. */ - int ret; if ((ret = GPTLinitialize ())) return ret; #endif diff --git a/examples/c/examplePio.c b/examples/c/examplePio.c index 2f9c9c0f6624..d0820a7327dc 100644 --- a/examples/c/examplePio.c +++ b/examples/c/examplePio.c @@ -10,6 +10,7 @@ * processors. */ +#include #include #include #include From 260864da7f3071d97b9de9abe323586bba1746ee Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 2 Feb 2016 15:20:54 -0700 Subject: [PATCH 06/48] fix pio_inq_unlimdimid and add a test --- src/flib/pio.F90 | 1 + src/flib/pio_nf.F90 | 3 ++- tests/unit/basic_tests.F90 | 9 ++++++++- tests/unit/driver.F90 | 4 ---- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/flib/pio.F90 b/src/flib/pio.F90 index 8b1da831efbc..14c3237ca028 100644 --- a/src/flib/pio.F90 +++ b/src/flib/pio.F90 @@ -47,6 +47,7 @@ module pio PIO_inq_dimid , & PIO_inq_dimname , & PIO_inq_dimlen , & + PIO_inq_unlimdim, & PIO_def_dim , & PIO_def_var , & PIO_redef , & diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90 index d9b9f0bd589c..d3f863dbd797 100644 --- a/src/flib/pio_nf.F90 +++ b/src/flib/pio_nf.F90 @@ -26,6 +26,7 @@ module pio_nf pio_inq_dimname , & pio_inq_dimlen , & pio_inq_dimid , & + pio_inq_unlimdim , & pio_inquire , & pio_enddef , & pio_redef @@ -509,7 +510,7 @@ integer(C_INT) function PIOc_inq_unlimdim(ncid ,unlimdim) & end function PIOc_inq_unlimdim end interface ierr = PIOc_inq_unlimdim(ncid ,unlimdim) - unlimdim=unlimdim+1 + if(unlimdim>=0) unlimdim=unlimdim+1 end function inq_unlimdim_id !> diff --git a/tests/unit/basic_tests.F90 b/tests/unit/basic_tests.F90 index ec8b4086cd11..de41cef69d44 100644 --- a/tests/unit/basic_tests.F90 +++ b/tests/unit/basic_tests.F90 @@ -144,6 +144,7 @@ Subroutine test_open(test_id, err_msg) integer, dimension(1) :: dims type(io_desc_t) :: iodesc_nCells integer :: pio_dim + integer :: unlimdimid type(var_desc_t) :: pio_var err_msg = "no_error" @@ -260,7 +261,6 @@ Subroutine test_open(test_id, err_msg) data_buffer = -1 call PIO_read_darray(pio_file, pio_var, iodesc_nCells, data_buffer, ret_val) call mpi_barrier(MPI_COMM_WORLD,ret_val) - if (ret_val.ne.PIO_NOERR) then ! Error in PIO_read_darray @@ -276,6 +276,13 @@ Subroutine test_open(test_id, err_msg) call mpi_abort(MPI_COMM_WORLD,0,ret_val) end if + ret_val = PIO_inq_unlimdim(pio_file, unlimdimid) + if(unlimdimid /= -1) then + err_msg = "Error in inq_unlimdim" + call PIO_closefile(pio_file) + print *,__FILE__,__LINE__,iotype, trim(err_msg) + call mpi_abort(MPI_COMM_WORLD,0,ret_val) + end if diff --git a/tests/unit/driver.F90 b/tests/unit/driver.F90 index 853554acd52e..6622bd6bf126 100644 --- a/tests/unit/driver.F90 +++ b/tests/unit/driver.F90 @@ -149,14 +149,12 @@ Program pio_unit_test_driver if (master_task) write(*,"(3x,A,1x)") "testing PIO_createfile..." call test_create(test_id, err_msg) call parse(err_msg, fail_cnt) - print *,__FILE__,__LINE__,fail_cnt ! test_open() if (master_task) write(*,"(3x,A,I3)", advance="no") "testing PIO_openfile...",test_id call test_open(test_id, err_msg) call parse(err_msg, fail_cnt) - print *,__FILE__,__LINE__,fail_cnt call mpi_barrier(mpi_comm_world,ierr) ! netcdf-specific tests @@ -165,12 +163,10 @@ Program pio_unit_test_driver if (master_task) write(*,"(3x,A,1x)", advance="no") "testing PIO_redef..." call test_redef(test_id, err_msg) call parse(err_msg, fail_cnt) - print *,__FILE__,__LINE__,fail_cnt if (master_task) write(*,"(3x,A,1x)", advance="no") "testing PIO_enddef..." call test_enddef(test_id, err_msg) call parse(err_msg, fail_cnt) - print *,__FILE__,__LINE__,fail_cnt end if From 93d430583fefbe2ef3c5f6407a7a4aea74e37301 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 4 Feb 2016 13:25:17 -0700 Subject: [PATCH 07/48] avoid fill pass with netcdf4p --- src/clib/pio_darray.c | 270 +++++++++++++++++++++--------------------- 1 file changed, 135 insertions(+), 135 deletions(-) diff --git a/src/clib/pio_darray.c b/src/clib/pio_darray.c index b7de478d0b50..75a451cba59b 100644 --- a/src/clib/pio_darray.c +++ b/src/clib/pio_darray.c @@ -1,12 +1,12 @@ /** @file - * + * * @brief This file contains the routines that read and write - * distributed arrays in PIO. + * distributed arrays in PIO. * * When arrays are distributed, each processor holds some of the * array. Only by combining the distributed arrays from all processor * can the full array be obtained. - * + * * @author Jim Edwards * @bug No Known bugs */ @@ -16,19 +16,19 @@ #define PIO_WRITE_BUFFERING 1 PIO_Offset PIO_BUFFER_SIZE_LIMIT=10485760; // 10MB default limit -bufsize PIO_CNBUFFER_LIMIT=33554432; -static void *CN_bpool=NULL; +bufsize PIO_CNBUFFER_LIMIT=33554432; +static void *CN_bpool=NULL; static PIO_Offset maxusage=0; /** @brief Set the pio buffer size limit, this is the size of the data buffer on the IO nodes. * * - * The pio_buffer_size_limit will only apply to files opened after the setting is changed. + * The pio_buffer_size_limit will only apply to files opened after the setting is changed. * @param limit the size of the buffer on the IO nodes * @return The previous limit setting. */ PIO_Offset PIOc_set_buffer_size_limit(const PIO_Offset limit) { - PIO_Offset oldsize; + PIO_Offset oldsize; oldsize = PIO_BUFFER_SIZE_LIMIT; if(limit>0) PIO_BUFFER_SIZE_LIMIT=limit; @@ -63,7 +63,7 @@ void compute_buffer_init(iosystem_desc_t ios) #endif } -/** @ingroup PIO_write_darray +/** @ingroup PIO_write_darray * @brief Write a single distributed field to output. This routine is only used if aggregation is off. * @param[in] file: a pointer to the open file descriptor for the file that will be written to * @param[in] iodesc: a pointer to the defined iodescriptor for the buffer @@ -107,7 +107,7 @@ void compute_buffer_init(iosystem_desc_t ios) msg = 0; if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) + if(ios->comp_rank==0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -174,11 +174,11 @@ void compute_buffer_init(iosystem_desc_t ios) case PIO_IOTYPE_NETCDF4P: ierr = nc_var_par_access(ncid, vid, NC_COLLECTIVE); if(iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid,(size_t *) start,(size_t *) count, (const double *) bufptr); + ierr = nc_put_vara_double (ncid, vid,(size_t *) start,(size_t *) count, (const double *) bufptr); } else if(iodesc->basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid, (size_t *) start, (size_t *) count, (const int *) bufptr); + ierr = nc_put_vara_int (ncid, vid, (size_t *) start, (size_t *) count, (const int *) bufptr); }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid, vid, (size_t *) start, (size_t *) count, (const float *) bufptr); + ierr = nc_put_vara_float (ncid, vid, (size_t *) start, (size_t *) count, (const float *) bufptr); }else{ fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) iodesc->basetype); } @@ -192,7 +192,7 @@ void compute_buffer_init(iosystem_desc_t ios) if(ios->io_rank==0){ for(i=0;inum_aiotasks;i++){ - if(i==0){ + if(i==0){ buflen=1; for(j=0;j0){ mpierr = MPI_Recv( tstart, ndims, MPI_OFFSET, i, ios->num_iotasks+i, ios->io_comm, &status); mpierr = MPI_Recv( tcount, ndims, MPI_OFFSET, i,2*ios->num_iotasks+i, ios->io_comm, &status); - tmp_buf = malloc(buflen * dsize); + tmp_buf = malloc(buflen * dsize); mpierr = MPI_Recv( tmp_buf, buflen, iodesc->basetype, i, i, ios->io_comm, &status); } } if(buflen>0){ if(iodesc->basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid, tstart, tcount, (const int *) tmp_buf); + ierr = nc_put_vara_int (ncid, vid, tstart, tcount, (const int *) tmp_buf); }else if(iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid, tstart, tcount, (const double *) tmp_buf); + ierr = nc_put_vara_double (ncid, vid, tstart, tcount, (const double *) tmp_buf); }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid,vid, tstart, tcount, (const float *) tmp_buf); + ierr = nc_put_vara_float (ncid,vid, tstart, tcount, (const float *) tmp_buf); }else{ fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) iodesc->basetype); } @@ -275,18 +275,18 @@ void compute_buffer_init(iosystem_desc_t ios) countlist[rrcnt][i]=count[i]; } rrcnt++; - } + } if(regioncnt==iodesc->maxregions-1){ // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,ios->io_rank,iodesc->llen, tdsize); - // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, + // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, // IOBUF, iodesc->llen, iodesc->basetype); int reqn=0; if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, + vdesc->request = realloc(vdesc->request, sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - + for(int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ vdesc->request[i]=NC_REQ_NULL; } @@ -296,8 +296,8 @@ void compute_buffer_init(iosystem_desc_t ios) reqn++; } } - - ierr = ncmpi_bput_varn(ncid, vid, rrcnt, startlist, countlist, + + ierr = ncmpi_bput_varn(ncid, vid, rrcnt, startlist, countlist, IOBUF, iodesc->llen, iodesc->basetype, vdesc->request+reqn); if(vdesc->request[reqn] == NC_REQ_NULL){ vdesc->request[reqn] = PIO_REQ_NULL; //keeps wait calls in sync @@ -330,24 +330,24 @@ void compute_buffer_init(iosystem_desc_t ios) /** @brief Write a set of one or more aggregated arrays to output file * @ingroup PIO_write_darray - * + * * This routine is used if aggregation is enabled, data is already on the * io-tasks * @param[in] file: a pointer to the open file descriptor for the file that will be written to * @param[in] nvars: the number of variables to be written with this decomposition - * @param[in] vid: an array of the variable ids to be written + * @param[in] vid: an array of the variable ids to be written * @param[in] iodesc_ndims: the number of dimensions explicitly in the iodesc * @param[in] basetype : the basic type of the minimal data unit * @param[in] gsize : array of the global dimensions of the field to be written * @param[in] maxregions : max number of blocks to be written from this iotask * @param[in] firstregion : pointer to the first element of a linked list of region descriptions. * @param[in] llen : length of the iobuffer on this task for a single field - * @param[in] maxiobuflen : maximum llen participating + * @param[in] maxiobuflen : maximum llen participating * @param[in] num_aiotasks : actual number of iotasks participating * @param[in] IOBUF: the buffer to be written from this mpi task - * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF + * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF */ -int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[], +int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[], const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset gsize[], const int maxregions, io_region *firstregion, const PIO_Offset llen, const int maxiobuflen, const int num_aiotasks, @@ -387,7 +387,7 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ msg = 0; if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) + if(ios->comp_rank==0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -411,7 +411,7 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ ncid = file->fh; region = firstregion; - + rrcnt=0; for(regioncnt=0;regioncntstart[0],region->count[0],ndims,fndims,vdesc->record); @@ -455,11 +455,11 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ ierr = nc_var_par_access(ncid, vid[nv], NC_COLLECTIVE); if(basetype == MPI_DOUBLE ||basetype == MPI_REAL8){ - ierr = nc_put_vara_double (ncid, vid[nv],(size_t *) start,(size_t *) count, (const double *) bufptr); + ierr = nc_put_vara_double (ncid, vid[nv],(size_t *) start,(size_t *) count, (const double *) bufptr); } else if(basetype == MPI_INTEGER){ - ierr = nc_put_vara_int (ncid, vid[nv], (size_t *) start, (size_t *) count, (const int *) bufptr); + ierr = nc_put_vara_int (ncid, vid[nv], (size_t *) start, (size_t *) count, (const int *) bufptr); }else if(basetype == MPI_FLOAT || basetype == MPI_REAL4){ - ierr = nc_put_vara_float (ncid, vid[nv], (size_t *) start, (size_t *) count, (const float *) bufptr); + ierr = nc_put_vara_float (ncid, vid[nv], (size_t *) start, (size_t *) count, (const float *) bufptr); }else{ fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",(int) basetype); } @@ -482,12 +482,12 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ countlist[rrcnt][i]=count[i]; } rrcnt++; - } + } if(regioncnt==maxregions-1){ //printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,ios->io_rank,iodesc->llen, tdsize); - // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, + // ierr = ncmpi_put_varn_all(ncid, vid, iodesc->maxregions, startlist, countlist, // IOBUF, iodesc->llen, iodesc->basetype); - + //printf("%s %d %ld \n",__FILE__,__LINE__,IOBUF); for(int nv=0; nvvarlist)+vid[nv]; @@ -500,9 +500,9 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ int reqn=0; if(vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){ - vdesc->request = realloc(vdesc->request, + vdesc->request = realloc(vdesc->request, sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK)); - + for(int i=vdesc->nreqs;inreqs+PIO_REQUEST_ALLOC_CHUNK;i++){ vdesc->request[i]=NC_REQ_NULL; } @@ -512,10 +512,10 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ reqn++; } } - ierr = ncmpi_iput_varn(ncid, vid[nv], rrcnt, startlist, countlist, + ierr = ncmpi_iput_varn(ncid, vid[nv], rrcnt, startlist, countlist, bufptr, llen, basetype, vdesc->request+reqn); /* - ierr = ncmpi_bput_varn(ncid, vid[nv], rrcnt, startlist, countlist, + ierr = ncmpi_bput_varn(ncid, vid[nv], rrcnt, startlist, countlist, bufptr, llen, basetype, &(vdesc->request)); */ if(vdesc->request[reqn] == NC_REQ_NULL){ @@ -555,24 +555,24 @@ int pio_write_darray_multi_nc(file_desc_t *file, const int nvars, const int vid[ /** @brief Write a set of one or more aggregated arrays to output file * @ingroup PIO_write_darray - * + * * This routine is used if aggregation is enabled, data is already on the * io-tasks * @param[in] file: a pointer to the open file descriptor for the file that will be written to * @param[in] nvars: the number of variables to be written with this decomposition - * @param[in] vid: an array of the variable ids to be written + * @param[in] vid: an array of the variable ids to be written * @param[in] iodesc_ndims: the number of dimensions explicitly in the iodesc * @param[in] basetype : the basic type of the minimal data unit * @param[in] gsize : array of the global dimensions of the field to be written * @param[in] maxregions : max number of blocks to be written from this iotask * @param[in] firstregion : pointer to the first element of a linked list of region descriptions. * @param[in] llen : length of the iobuffer on this task for a single field - * @param[in] maxiobuflen : maximum llen participating + * @param[in] maxiobuflen : maximum llen participating * @param[in] num_aiotasks : actual number of iotasks participating * @param[in] IOBUF: the buffer to be written from this mpi task - * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF + * @param[in] frame : the frame or record dimension for each of the nvars variables in IOBUF */ -int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const int vid[], +int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const int vid[], const int iodesc_ndims, MPI_Datatype basetype, const PIO_Offset gsize[], const int maxregions, io_region *firstregion, const PIO_Offset llen, const int maxiobuflen, const int num_aiotasks, @@ -612,7 +612,7 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i msg = 0; if(ios->async_interface && ! ios->ioproc){ - if(ios->comp_rank==0) + if(ios->comp_rank==0) mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, ios->compmaster, ios->intercomm); } @@ -634,7 +634,7 @@ int pio_write_darray_multi_nc_serial(file_desc_t *file, const int nvars, const i ncid = file->fh; region = firstregion; - + rrcnt=0; for(regioncnt=0;regioncntmaxregions;regioncnt++){ + }// for(regioncnt=0;regioncntmaxregions;regioncnt++){ } // if(rlen>0) } // for(int rtask=0; rtasknum_iotasks; rtask++){ @@ -776,16 +776,16 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con fprintf(stderr,"ERROR: Attempt to write to read-only file\n"); return PIO_EPERM; } - + iodesc = pio_get_iodesc_from_id(ioid); if(iodesc == NULL){ // print_trace(NULL); //fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); return PIO_EBADID; } - + vdesc0 = file->varlist+vid[0]; - + pioassert(nvars>0,"nvars <= 0",__FILE__,__LINE__); ios = file->iosystem; @@ -799,7 +799,7 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con } if(iodesc->rearranger>0){ if(rlen>0){ - MPI_Type_size(iodesc->basetype, &vsize); + MPI_Type_size(iodesc->basetype, &vsize); //printf("rlen*vsize = %ld\n",rlen*vsize); vdesc0->iobuf = bget((size_t) vsize* (size_t) rlen); @@ -831,19 +831,19 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con switch(file->iotype){ case PIO_IOTYPE_NETCDF4P: case PIO_IOTYPE_PNETCDF: - ierr = pio_write_darray_multi_nc(file, nvars, vid, + ierr = pio_write_darray_multi_nc(file, nvars, vid, iodesc->ndims, iodesc->basetype, iodesc->gsize, iodesc->maxregions, iodesc->firstregion, iodesc->llen, iodesc->maxiobuflen, iodesc->num_aiotasks, - vdesc0->iobuf, frame); + vdesc0->iobuf, frame); break; case PIO_IOTYPE_NETCDF4C: case PIO_IOTYPE_NETCDF: - ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, + ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, iodesc->ndims, iodesc->basetype, iodesc->gsize, iodesc->maxregions, iodesc->firstregion, iodesc->llen, iodesc->maxiobuflen, iodesc->num_aiotasks, - vdesc0->iobuf, frame); + vdesc0->iobuf, frame); if(vdesc0->iobuf != NULL){ brel(vdesc0->iobuf); vdesc0->iobuf = NULL; @@ -851,15 +851,15 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con break; } - - - + + + if(iodesc->rearranger == PIO_REARR_SUBSET && iodesc->needsfill && iodesc->holegridsize>0){ if(vdesc0->fillbuf != NULL){ piodie("Attempt to overwrite existing buffer",__FILE__,__LINE__); } - + vdesc0->fillbuf = bget(iodesc->holegridsize*vsize*nvars); //printf("%s %d %x\n",__FILE__,__LINE__,vdesc0->fillbuf); if(vsize==4){ @@ -876,17 +876,17 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con } } switch(file->iotype){ - case PIO_IOTYPE_NETCDF4P: case PIO_IOTYPE_PNETCDF: - ierr = pio_write_darray_multi_nc(file, nvars, vid, + ierr = pio_write_darray_multi_nc(file, nvars, vid, iodesc->ndims, iodesc->basetype, iodesc->gsize, iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, iodesc->holegridsize, iodesc->num_aiotasks, vdesc0->fillbuf, frame); break; + case PIO_IOTYPE_NETCDF4P: case PIO_IOTYPE_NETCDF4C: case PIO_IOTYPE_NETCDF: - /* ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, + /* ierr = pio_write_darray_multi_nc_serial(file, nvars, vid, iodesc->ndims, iodesc->basetype, iodesc->gsize, iodesc->maxfillregions, iodesc->fillregion, iodesc->holegridsize, iodesc->holegridsize, iodesc->num_aiotasks, @@ -903,13 +903,13 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con } flush_output_buffer(file, flushtodisk, 0); - + return ierr; } -/** @brief Write a distributed array to the output file. +/** @brief Write a distributed array to the output file. * @ingroup PIO_write_darray * * This routine aggregates output on the compute nodes and only sends @@ -933,7 +933,7 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con * missing data. * * @returns 0 for success, non-zero error code for failure. - */ + */ #ifdef PIO_WRITE_BUFFERING int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) { @@ -965,7 +965,7 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con fprintf(stderr,"ERROR: Attempt to write to read-only file\n"); return PIO_EPERM; } - + iodesc = pio_get_iodesc_from_id(ioid); if(iodesc == NULL){ fprintf(stderr,"iodesc handle not found %d %d\n",ioid,__LINE__); @@ -1035,9 +1035,9 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con wmb->fillvalue=NULL; } - + MPI_Type_size(iodesc->basetype, &tsize); - // At this point wmb should be pointing to a new or existing buffer + // At this point wmb should be pointing to a new or existing buffer // so we can add the data // printf("%s %d %X %d %d %d\n",__FILE__,__LINE__,wmb->data,wmb->validvars,arraylen,tsize); // cn_buffer_report(*ios, true); @@ -1052,7 +1052,7 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con // need to flush first // printf("%s %d %ld %d %ld %ld\n",__FILE__,__LINE__,maxfree, wmb->validvars, (1+wmb->validvars)*arraylen*tsize,totfree); cn_buffer_report(*ios, true); - + flush_buffer(ncid,wmb, needsflush==2); // if needsflush == 2 flush to disk otherwise just flush to io node } if(arraylen > 0){ @@ -1077,32 +1077,32 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con piomemerror(*ios, (1+wmb->validvars)*tsize , __FILE__,__LINE__); } } - + if(iodesc->needsfill){ if(fillvalue != NULL){ - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars,fillvalue, tsize); + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars,fillvalue, tsize); }else{ vtype = (MPI_Datatype) iodesc->basetype; if(vtype == MPI_INTEGER){ int fill = PIO_FILL_INT; - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); }else if(vtype == MPI_FLOAT || vtype == MPI_REAL4){ float fill = PIO_FILL_FLOAT; - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); }else if(vtype == MPI_DOUBLE || vtype == MPI_REAL8){ double fill = PIO_FILL_DOUBLE; - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); }else if(vtype == MPI_CHARACTER){ char fill = PIO_FILL_CHAR; - memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); + memcpy((char *) wmb->fillvalue+tsize*wmb->validvars, &fill, tsize); }else{ fprintf(stderr,"Type not recognized %d in pioc_write_darray\n",vtype); } } } - + wmb->arraylen = arraylen; wmb->vid[wmb->validvars]=vid; bufptr = (void *)((char *) wmb->data + arraylen*tsize*wmb->validvars); @@ -1133,14 +1133,14 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con } return ierr; - + } #else -/** @brief Write a distributed array to the output file +/** @brief Write a distributed array to the output file * @ingroup PIO_write_darray * - * This version of the routine does not buffer, all data is communicated to the io tasks + * This version of the routine does not buffer, all data is communicated to the io tasks * before the routine returns */ int PIOc_write_darray(const int ncid, const int vid, const int ioid, const PIO_Offset arraylen, void *array, void *fillvalue) @@ -1174,14 +1174,14 @@ int PIOc_write_darray_multi(const int ncid, const int vid[], const int ioid, con rlen = iodesc->llen; if(iodesc->rearranger>0){ if(rlen>0){ - MPI_Type_size(iodesc->basetype, &tsize); + MPI_Type_size(iodesc->basetype, &tsize); // iobuf = bget(tsize*rlen); iobuf = malloc((size_t) tsize*rlen); if(iobuf==NULL){ piomemerror(*ios,rlen*(size_t) tsize, __FILE__,__LINE__); } } - // printf(" rlen = %d %ld\n",rlen,iobuf); + // printf(" rlen = %d %ld\n",rlen,iobuf); // } @@ -1230,18 +1230,18 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void ios = file->iosystem; if(ios == NULL) return PIO_EBADID; - + vdesc = (file->varlist)+vid; - + if(vdesc == NULL) return PIO_EBADID; - + ndims = iodesc->ndims; ierr = PIOc_inq_varndims(file->fh, vid, &fndims); - - if(fndims==ndims) + + if(fndims==ndims) vdesc->record=-1; - + if(ios->ioproc){ io_region *region; size_t start[fndims]; @@ -1260,12 +1260,12 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void // buffer is incremented by byte and loffset is in terms of the iodessc->basetype // so we need to multiply by the size of the basetype // We can potentially allow for one iodesc to have multiple datatypes by allowing the - // calling program to change the basetype. + // calling program to change the basetype. region = iodesc->firstregion; MPI_Type_size(iodesc->basetype, &tsize); if(fndims>ndims){ ndims++; - if(vdesc->record<0) + if(vdesc->record<0) vdesc->record=0; } for(regioncnt=0;regioncntmaxregions;regioncnt++){ @@ -1277,21 +1277,21 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void count[i] = 0; } bufptr=NULL; - }else{ + }else{ if(regioncnt==0 || region==NULL) bufptr = IOBUF; else bufptr=(void *)((char *) IOBUF + tsize*region->loffset); - + // printf("%s %d %d %d %d\n",__FILE__,__LINE__,iodesc->llen - region->loffset, iodesc->llen, region->loffset); - + if(vdesc->record >= 0 && fndims>1){ start[0] = vdesc->record; for(i=1;istart[i-1]; count[i] = region->count[i-1]; - // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); - } + // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); + } if(count[1]>0) count[0] = 1; }else{ @@ -1299,20 +1299,20 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void for(i=0;istart[i]; count[i] = region->count[i]; - // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); + // printf("%s %d %d %ld %ld\n",__FILE__,__LINE__,i,start[i],count[i]); } } } - + switch(file->iotype){ #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: if(iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_get_vara_double (file->fh, vid,start,count, bufptr); + ierr = nc_get_vara_double (file->fh, vid,start,count, bufptr); } else if(iodesc->basetype == MPI_INTEGER){ - ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); + ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); + ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); }else{ fprintf(stderr,"Type not recognized %d in pioc_read_darray\n",(int) iodesc->basetype); } @@ -1338,7 +1338,7 @@ int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, const int vid, void rrlen++; } if(regioncnt==iodesc->maxregions-1){ - ierr = ncmpi_get_varn_all(file->fh, vid, rrlen, startlist, + ierr = ncmpi_get_varn_all(file->fh, vid, rrlen, startlist, countlist, IOBUF, iodesc->llen, iodesc->basetype); for(i=0;iiotype,__FILE__,__LINE__); - + } if(region != NULL) region = region->next; } // for(regioncnt=0;...) } - + ierr = check_netcdf(file, ierr, __FILE__,__LINE__); #ifdef TIMING GPTLstop("PIO:read_darray_nc"); @@ -1384,18 +1384,18 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vi ios = file->iosystem; if(ios == NULL) return PIO_EBADID; - + vdesc = (file->varlist)+vid; - + if(vdesc == NULL) return PIO_EBADID; - + ndims = iodesc->ndims; ierr = PIOc_inq_varndims(file->fh, vid, &fndims); - - if(fndims==ndims) + + if(fndims==ndims) vdesc->record=-1; - + if(ios->ioproc){ io_region *region; size_t start[fndims]; @@ -1412,11 +1412,11 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vi // buffer is incremented by byte and loffset is in terms of the iodessc->basetype // so we need to multiply by the size of the basetype // We can potentially allow for one iodesc to have multiple datatypes by allowing the - // calling program to change the basetype. + // calling program to change the basetype. region = iodesc->firstregion; MPI_Type_size(iodesc->basetype, &tsize); if(fndims>ndims){ - if(vdesc->record<0) + if(vdesc->record<0) vdesc->record=0; } for(regioncnt=0;regioncntmaxregions;regioncnt++){ @@ -1426,13 +1426,13 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vi tmp_count[i+regioncnt*fndims] = 0; } bufptr=NULL; - }else{ + }else{ if(vdesc->record >= 0 && fndims>1){ tmp_start[regioncnt*fndims] = vdesc->record; for(i=1;istart[i-1]; tmp_count[i+regioncnt*fndims] = region->count[i-1]; - } + } if(tmp_count[1+regioncnt*fndims]>0) tmp_count[regioncnt*fndims] = 1; }else{ @@ -1449,8 +1449,8 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vi } if(region != NULL) region = region->next; - } // for(regioncnt=0;...) - + } // for(regioncnt=0;...) + if(ios->io_rank>0){ MPI_Send( &(iodesc->llen), 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm); if(iodesc->llen > 0){ @@ -1495,23 +1495,23 @@ int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, const int vi } } loffset+=regionsize; - + #ifdef _NETCDF // Cant use switch here because MPI_DATATYPE may not be simple (openmpi) if(iodesc->basetype == MPI_DOUBLE || iodesc->basetype == MPI_REAL8){ - ierr = nc_get_vara_double (file->fh, vid,start, count, bufptr); + ierr = nc_get_vara_double (file->fh, vid,start, count, bufptr); }else if(iodesc->basetype == MPI_INTEGER){ - ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); + ierr = nc_get_vara_int (file->fh, vid, start, count, bufptr); }else if(iodesc->basetype == MPI_FLOAT || iodesc->basetype == MPI_REAL4){ - ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); + ierr = nc_get_vara_float (file->fh, vid, start, count, bufptr); }else{ fprintf(stderr,"Type not recognized %d in pioc_write_darray_nc_serial\n",(int) iodesc->basetype); - } + } if(ierr != PIO_NOERR){ for(int i=0;irearranger > 0){ if(ios->ioproc && rlen>0){ - MPI_Type_size(iodesc->basetype, &tsize); + MPI_Type_size(iodesc->basetype, &tsize); iobuf = bget(((size_t) tsize)*rlen); if(iobuf==NULL){ piomemerror(*ios,rlen*((size_t) tsize), __FILE__,__LINE__); @@ -1617,7 +1617,7 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) if(!force && file->iosystem->io_comm != MPI_COMM_NULL){ usage += addsize; - MPI_Allreduce(MPI_IN_PLACE, &usage, 1, MPI_OFFSET, MPI_MAX, + MPI_Allreduce(MPI_IN_PLACE, &usage, 1, MPI_OFFSET, MPI_MAX, file->iosystem->io_comm); } @@ -1645,7 +1645,7 @@ int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize) for(int i=0; i<=maxreq; i++){ vdesc = file->varlist+i; #ifdef MPIO_ONESIDED - /*onesided optimization requires that all of the requests in a wait_all call represent + /*onesided optimization requires that all of the requests in a wait_all call represent a contiguous block of data in the file */ if(rcnt>0 && (prev_record != vdesc->record || vdesc->nreqs==0)){ @@ -1760,14 +1760,14 @@ void flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk) wmb->fillvalue=NULL; wmb->frame=NULL; } -} - +} + void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc) { int maxbytesoniotask=INT_MAX; int maxbytesoncomputetask=INT_MAX; int maxbytes; - + // printf("%s %d %d %d\n",__FILE__,__LINE__,iodesc->maxiobuflen, iodesc->ndof); if(ios.ioproc && iodesc->maxiobuflen>0){ @@ -1783,5 +1783,5 @@ void compute_maxaggregate_bytes(const iosystem_desc_t ios, io_desc_t *iodesc) MPI_Allreduce(MPI_IN_PLACE, &maxbytes, 1, MPI_INT, MPI_MIN, ios.union_comm); iodesc->maxbytes=maxbytes; // printf("%s %d %d %d\n",__FILE__,__LINE__,iodesc->maxbytes,iodesc->maxiobuflen); - + } From 70c007b92e5909a1c7e3168e7c0b07e4a01d93da Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 8 Feb 2016 12:10:28 -0500 Subject: [PATCH 08/48] getting test_names ready --- tests/unit/CMakeLists.txt | 9 ++ tests/unit/test_names.c | 294 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 303 insertions(+) create mode 100644 tests/unit/test_names.c diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index ba775ff6e15c..a6a6c69d69f1 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -26,6 +26,8 @@ if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU") PRIVATE -ffree-line-length-none) endif() +add_executable (test_names EXCLUDE_FROM_ALL test_names.c) +target_link_libraries (test_names pioc) add_executable (test_nc4 EXCLUDE_FROM_ALL test_nc4.c) target_link_libraries (test_nc4 pioc) @@ -35,6 +37,7 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG") # PRIVATE -mismatch_all) endif () +add_dependencies (tests test_names) add_dependencies (tests test_nc4) add_dependencies (tests pio_unit_test) @@ -42,6 +45,8 @@ add_dependencies (tests pio_unit_test) set (DEFAULT_TEST_TIMEOUT 240) if (PIO_USE_MPISERIAL) + add_test(NAME test_names + COMMAND test_names) add_test(NAME test_nc4 COMMAND test_nc4) add_test(NAME pio_unit_test @@ -49,6 +54,10 @@ if (PIO_USE_MPISERIAL) set_tests_properties(pio_unit_test PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT}) else () + add_mpi_test(test_names + EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_names + NUMPROCS 4 + TIMEOUT ${DEFAULT_TEST_TIMEOUT}) add_mpi_test(test_nc4 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_nc4 NUMPROCS 4 diff --git a/tests/unit/test_names.c b/tests/unit/test_names.c new file mode 100644 index 000000000000..a09967aef329 --- /dev/null +++ b/tests/unit/test_names.c @@ -0,0 +1,294 @@ +/** + * @file + * Tests for names of vars, atts, and dims. + * + */ +#include +#ifdef TIMING +#include +#endif + +#define NUM_NETCDF_FLAVORS 4 +#define NDIM 3 +#define X_DIM_LEN 400 +#define Y_DIM_LEN 400 +#define NUM_TIMESTEPS 6 +#define VAR_NAME "foo" +#define START_DATA_VAL 42 +#define ERR_AWFUL 1111 +#define VAR_CACHE_SIZE (1024 * 1024) +#define VAR_CACHE_NELEMS 10 +#define VAR_CACHE_PREEMPTION 0.5 + +/** Handle MPI errors. This should only be used with MPI library + * function calls. */ +#define MPIERR(e) do { \ + MPI_Error_string(e, err_buffer, &resultlen); \ + fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \ + MPI_Finalize(); \ + return ERR_AWFUL; \ + } while (0) + +/** Handle non-MPI errors by finalizing the MPI library and exiting + * with an exit code. */ +#define ERR(e) do { \ + fprintf(stderr, "Error %d in %s, line %d\n", e, __FILE__, __LINE__); \ + MPI_Finalize(); \ + return e; \ + } while (0) + +/** Global err buffer for MPI. */ +char err_buffer[MPI_MAX_ERROR_STRING]; +int resultlen; + +/** The dimension names. */ +char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"}; + +/** Length of the dimensions in the sample data. */ +int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; + +/** Length of chunksizes to use in netCDF-4 files. */ +size_t chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2}; + +/** Run Tests for NetCDF-4 Functions. + * + * @param argc argument count + * @param argv array of arguments + */ +int +main(int argc, char **argv) +{ + int verbose = 1; + + /** Zero-based rank of processor. */ + int my_rank; + + /** Number of processors involved in current execution. */ + int ntasks; + + /** Specifies the flavor of netCDF output format. */ + int iotype; + + /** Different output flavors. */ + int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, + PIO_IOTYPE_NETCDF, + PIO_IOTYPE_NETCDF4C, + PIO_IOTYPE_NETCDF4P}; + + /** Names for the output files. */ + char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_nc4_pnetcdf.nc", + "test_nc4_classic.nc", + "test_nc4_serial4.nc", + "test_nc4_parallel4.nc"}; + + /** Number of processors that will do IO. In this test we + * will do IO from all processors. */ + int niotasks; + + /** Stride in the mpi rank between io tasks. Always 1 in this + * test. */ + int ioproc_stride = 1; + + /** Number of the aggregator? Always 0 in this test. */ + int numAggregator = 0; + + /** Zero based rank of first processor to be used for I/O. */ + int ioproc_start = 0; + + /** The dimension IDs. */ + int dimids[NDIM]; + + /** Array index per processing unit. */ + PIO_Offset elements_per_pe; + + /** The ID for the parallel I/O system. */ + int iosysid; + + /** The ncid of the netCDF file. */ + int ncid = 0; + + /** The ID of the netCDF varable. */ + int varid; + + /** Storage of netCDF-4 files (contiguous vs. chunked). */ + int storage; + + /** Chunksizes set in the file. */ + size_t my_chunksize[NDIM]; + + /** The shuffle filter setting in the netCDF-4 test file. */ + int shuffle; + + /** Non-zero if deflate set for the variable in the netCDF-4 test file. */ + int deflate; + + /** The deflate level set for the variable in the netCDF-4 test file. */ + int deflate_level; + + /** Non-zero if fletcher32 filter is used for variable. */ + int fletcher32; + + /** Endianness of variable. */ + int endianness; + + /* Size of the file chunk cache. */ + size_t chunk_cache_size; + + /* Number of elements in file cache. */ + size_t nelems; + + /* File cache preemption. */ + float preemption; + + /* Size of the var chunk cache. */ + size_t var_cache_size; + + /* Number of elements in var cache. */ + size_t var_cache_nelems; + + /* Var cache preemption. */ + float var_cache_preemption; + + /** The I/O description ID. */ + int ioid; + + /** A buffer for sample data. */ + float *buffer; + + /** A buffer for reading data back from the file. */ + int *read_buffer; + + /** The decomposition mapping. */ + PIO_Offset *compdof; + + /** Return code. */ + int ret; + + /** Index for loops. */ + int fmt, d, d1, i; + +#ifdef TIMING + /* Initialize the GPTL timing library. */ + if ((ret = GPTLinitialize ())) + return ret; +#endif + + /* Initialize MPI. */ + if ((ret = MPI_Init(&argc, &argv))) + MPIERR(ret); + + /* Learn my rank and the total number of processors. */ + if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank))) + MPIERR(ret); + if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks))) + MPIERR(ret); + + /* Check that a valid number of processors was specified. */ + if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 || + ntasks == 8 || ntasks == 16)) + fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n"); + if (verbose) + printf("%d: ParallelIO Library example1 running on %d processors.\n", + my_rank, ntasks); + + /* keep things simple - 1 iotask per MPI process */ + niotasks = ntasks; + + /* Initialize the PIO IO system. This specifies how + * many and which processors are involved in I/O. */ + if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride, + ioproc_start, PIO_REARR_SUBSET, &iosysid))) + ERR(ret); + + /* Describe the decomposition. This is a 1-based array, so add 1! */ + elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks; + if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset)))) + return PIO_ENOMEM; + for (i = 0; i < elements_per_pe; i++) { + compdof[i] = my_rank * elements_per_pe + i + 1; + } + + /* Create the PIO decomposition for this test. */ + if (verbose) + printf("rank: %d Creating decomposition...\n", my_rank); + if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe, + compdof, &ioid, NULL, NULL, NULL))) + ERR(ret); + free(compdof); + + /* How many flavors will we be running for? */ + int num_flavors = 0; + int fmtidx = 0; +#ifdef _PNETCDF + num_flavors++; + format[fmtidx++] = PIO_IOTYPE_PNETCDF; +#endif +#ifdef _NETCDF + num_flavors++; + format[fmtidx++] = PIO_IOTYPE_NETCDF; +#endif +#ifdef _NETCDF4 + num_flavors += 2; + format[fmtidx++] = PIO_IOTYPE_NETCDF4C; + format[fmtidx] = PIO_IOTYPE_NETCDF4P; +#endif + + /* Use PIO to create the example file in each of the four + * available ways. */ + for (fmt = 0; fmt < num_flavors; fmt++) + { + /* Create the netCDF output file. */ + if (verbose) + printf("rank: %d Creating sample file %s with format %d...\n", + my_rank, filename[fmt], format[fmt]); + if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt], + PIO_CLOBBER))) + ERR(ret); + + /* Define netCDF dimensions and variable. */ + if (verbose) + printf("rank: %d Defining netCDF metadata...\n", my_rank); + for (d = 0; d < NDIM; d++) { + if (verbose) + printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank, + dim_name[d], dim_len[d]); + if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) + ERR(ret); + } + if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) + ERR(ret); + + if ((ret = PIOc_enddef(ncid))) + ERR(ret); + + /* Close the netCDF file. */ + if (verbose) + printf("rank: %d Closing the sample data file...\n", my_rank); + if ((ret = PIOc_closefile(ncid))) + ERR(ret); + } + + /* Free the PIO decomposition. */ + if (verbose) + printf("rank: %d Freeing PIO decomposition...\n", my_rank); + if ((ret = PIOc_freedecomp(iosysid, ioid))) + ERR(ret); + + /* Finalize the IO system. */ + if (verbose) + printf("rank: %d Freeing PIO resources...\n", my_rank); + if ((ret = PIOc_finalize(iosysid))) + ERR(ret); + + /* Finalize the MPI library. */ + MPI_Finalize(); + +#ifdef TIMING + /* Finalize the GPTL timing library. */ + if ((ret = GPTLfinalize ())) + return ret; +#endif + + + return 0; +} From f2691ba74b513fcb81a73b5abcdbbc77cc977799 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 8 Feb 2016 12:56:01 -0500 Subject: [PATCH 09/48] test and fixes for name null terminator problem in processors not rank=0 --- src/clib/pio_nc.c | 6 +- tests/unit/test_names.c | 130 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 129 insertions(+), 7 deletions(-) diff --git a/src/clib/pio_nc.c b/src/clib/pio_nc.c index e7247b9d5475..354c16eda2d3 100644 --- a/src/clib/pio_nc.c +++ b/src/clib/pio_nc.c @@ -558,7 +558,7 @@ int PIOc_inq_varname (int ncid, int varid, char *name) if(name != NULL){ int slen; if(ios->iomaster) - slen = (int) strlen(name); + slen = (int) strlen(name) + 1; mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); } @@ -1826,7 +1826,7 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) if(name != NULL){ int slen; if(ios->iomaster) - slen = (int) strlen(name); + slen = (int) strlen(name) + 1; mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); } @@ -3267,7 +3267,7 @@ int PIOc_inq_dimname (int ncid, int dimid, char *name) if(name != NULL){ int slen; if(ios->iomaster) - slen = (int) strlen(name); + slen = (int) strlen(name) + 1; mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); } diff --git a/tests/unit/test_names.c b/tests/unit/test_names.c index a09967aef329..ea493ddc0f5d 100644 --- a/tests/unit/test_names.c +++ b/tests/unit/test_names.c @@ -14,6 +14,7 @@ #define Y_DIM_LEN 400 #define NUM_TIMESTEPS 6 #define VAR_NAME "foo" +#define ATT_NAME "bar" #define START_DATA_VAL 42 #define ERR_AWFUL 1111 #define VAR_CACHE_SIZE (1024 * 1024) @@ -50,6 +51,103 @@ int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; /** Length of chunksizes to use in netCDF-4 files. */ size_t chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2}; +/** Check the dimension names. + * + * @param my_rank rank of process + * @param ncid ncid of open netCDF file + * + * @returns 0 for success, error code otherwise. */ +int +check_dim_names(int my_rank, int ncid, int verbose) +{ + char dim_name[NC_MAX_NAME + 1]; + char zero_dim_name[NC_MAX_NAME + 1]; + int ret; + + for (int d = 0; d < NDIM; d++) + { + strcpy(dim_name, "11111111111111111111111111111111"); + if ((ret = PIOc_inq_dimname(ncid, d, dim_name))) + return ret; + if (verbose) + printf("my_rank %d dim %d name %s\n", my_rank, d, dim_name); + + /* Did other ranks get the same name? */ + if (!my_rank) + strcpy(zero_dim_name, dim_name); + /* if (verbose) */ + /* printf("rank %d dim_name %s zero_dim_name %s\n", my_rank, dim_name, zero_dim_name); */ + if ((ret = MPI_Bcast(&zero_dim_name, strlen(dim_name) + 1, MPI_CHAR, 0, + MPI_COMM_WORLD))) + MPIERR(ret); + if (strcmp(dim_name, zero_dim_name)) + return ERR_AWFUL; + } + return 0; +} + +/** Check the variable name. + * + * @param my_rank rank of process + * @param ncid ncid of open netCDF file + * + * @returns 0 for success, error code otherwise. */ +int +check_var_name(int my_rank, int ncid, int verbose) +{ + char var_name[NC_MAX_NAME + 1]; + char zero_var_name[NC_MAX_NAME + 1]; + int ret; + + strcpy(var_name, "11111111111111111111111111111111"); + if ((ret = PIOc_inq_varname(ncid, 0, var_name))) + return ret; + if (verbose) + printf("my_rank %d var name %s\n", my_rank, var_name); + + /* Did other ranks get the same name? */ + if (!my_rank) + strcpy(zero_var_name, var_name); + if ((ret = MPI_Bcast(&zero_var_name, strlen(var_name) + 1, MPI_CHAR, 0, + MPI_COMM_WORLD))) + MPIERR(ret); + if (strcmp(var_name, zero_var_name)) + return ERR_AWFUL; + return 0; +} + +/** Check the attribute name. + * + * @param my_rank rank of process + * @param ncid ncid of open netCDF file + * + * @returns 0 for success, error code otherwise. */ +int +check_att_name(int my_rank, int ncid, int verbose) +{ + char att_name[NC_MAX_NAME + 1]; + char zero_att_name[NC_MAX_NAME + 1]; + int ret; + + strcpy(att_name, "11111111111111111111111111111111"); + if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, att_name))) + return ret; + if (verbose) + printf("my_rank %d att name %s\n", my_rank, att_name); + + /* Did everyone ranks get the same length name? */ +/* if (strlen(att_name) != strlen(ATT_NAME)) + return ERR_AWFUL;*/ + if (!my_rank) + strcpy(zero_att_name, att_name); + if ((ret = MPI_Bcast(&zero_att_name, strlen(att_name) + 1, MPI_CHAR, 0, + MPI_COMM_WORLD))) + MPIERR(ret); + if (strcmp(att_name, zero_att_name)) + return ERR_AWFUL; + return 0; +} + /** Run Tests for NetCDF-4 Functions. * * @param argc argument count @@ -76,10 +174,10 @@ main(int argc, char **argv) PIO_IOTYPE_NETCDF4P}; /** Names for the output files. */ - char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_nc4_pnetcdf.nc", - "test_nc4_classic.nc", - "test_nc4_serial4.nc", - "test_nc4_parallel4.nc"}; + char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_names_pnetcdf.nc", + "test_names_classic.nc", + "test_names_serial4.nc", + "test_names_parallel4.nc"}; /** Number of processors that will do IO. In this test we * will do IO from all processors. */ @@ -255,9 +353,28 @@ main(int argc, char **argv) if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d]))) ERR(ret); } + + /* Check the dimension names. */ + if ((ret = check_dim_names(my_rank, ncid, verbose))) + ERR(ret); + + /* Define a global attribute. */ + int att_val = 42; + if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, NC_INT, 1, &att_val))) + ERR(ret); + + /* Check the attribute name. */ + if ((ret = check_att_name(my_rank, ncid, verbose))) + ERR(ret); + + /* Define a variable. */ if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid))) ERR(ret); + /* Check the variable name. */ + if ((ret = check_var_name(my_rank, ncid, verbose))) + ERR(ret); + if ((ret = PIOc_enddef(ncid))) ERR(ret); @@ -266,6 +383,11 @@ main(int argc, char **argv) printf("rank: %d Closing the sample data file...\n", my_rank); if ((ret = PIOc_closefile(ncid))) ERR(ret); + + /* Put a barrier here to make verbose output look better. */ + if ((ret = MPI_Barrier(MPI_COMM_WORLD))) + MPIERR(ret); + } /* Free the PIO decomposition. */ From 6ff466c6cf3e4cd258452d0c677414af3c23c742 Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Mon, 8 Feb 2016 16:10:26 -0500 Subject: [PATCH 10/48] moved name null-terminator fixes to perl code which generates pio_nc.c --- src/clib/ncparser.pl | 6 +- src/clib/pio_nc.c | 1256 +++++++++++++++++++++--------------------- 2 files changed, 631 insertions(+), 631 deletions(-) diff --git a/src/clib/ncparser.pl b/src/clib/ncparser.pl index a962a3eac072..960e3db688a3 100644 --- a/src/clib/ncparser.pl +++ b/src/clib/ncparser.pl @@ -301,7 +301,7 @@ print F " if(name != NULL){ \n"; print F " int slen;\n"; print F " if(ios->iomaster)\n"; - print F " slen = (int) strlen(name);\n"; + print F " slen = (int) strlen(name) + 1;\n"; print F " mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm);\n"; print F " mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm);\n }\n"; @@ -316,7 +316,7 @@ print F " if(name != NULL){ \n"; print F " int slen;\n"; print F " if(ios->iomaster)\n"; - print F " slen = (int) strlen(name);\n"; + print F " slen = (int) strlen(name) + 1;\n"; print F " mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm);\n"; print F " mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm);\n }\n"; print F " if(lenp != NULL) mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm);\n"; @@ -324,7 +324,7 @@ print F " if(name != NULL){\n"; print F " int slen;\n"; print F " if(ios->iomaster)\n"; - print F " slen = (int) strlen(name);\n"; + print F " slen = (int) strlen(name) + 1;\n"; print F " mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm);\n"; print F " mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm);\n }\n"; }elsif($func =~ /inq_vardimid/){ diff --git a/src/clib/pio_nc.c b/src/clib/pio_nc.c index 354c16eda2d3..f511475a4f6c 100644 --- a/src/clib/pio_nc.c +++ b/src/clib/pio_nc.c @@ -14,22 +14,19 @@ #include /** - * @ingroup PIOc_inq_att - * The PIO-C interface for the NetCDF function nc_inq_att. + * @ingroup PIOc_inq + * The PIO-C interface for the NetCDF function nc_inq. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp) +int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp) { int ierr; int msg; @@ -45,7 +42,7 @@ int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Of if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATT; + msg = PIO_MSG_INQ; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -59,19 +56,19 @@ int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Of #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; + ierr = nc_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; + ierr = nc_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);; + ierr = ncmpi_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);; break; #endif default: @@ -84,27 +81,32 @@ int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Of sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(xtypep != NULL) mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(lenp != NULL) mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); + if(ndimsp != NULL) + mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(nvarsp != NULL) + mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(ngattsp != NULL) + mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(unlimdimidp != NULL) + mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm); + if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_format - * The PIO-C interface for the NetCDF function nc_inq_format. + * @ingroup PIOc_inq_dimname + * The PIO-C interface for the NetCDF function nc_inq_dimname. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param formatp a pointer that will get the file format * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_format (int ncid, int *formatp) +int PIOc_inq_dimname (int ncid, int dimid, char *name) { int ierr; int msg; @@ -120,7 +122,7 @@ int PIOc_inq_format (int ncid, int *formatp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_FORMAT; + msg = PIO_MSG_INQ_DIMNAME; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -134,19 +136,19 @@ int PIOc_inq_format (int ncid, int *formatp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_format(file->fh, formatp);; + ierr = nc_inq_dimname(file->fh, dimid, name);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_format(file->fh, formatp);; + ierr = nc_inq_dimname(file->fh, dimid, name);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_format(file->fh, formatp);; + ierr = ncmpi_inq_dimname(file->fh, dimid, name);; break; #endif default: @@ -159,27 +161,32 @@ int PIOc_inq_format (int ncid, int *formatp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(name != NULL){ + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_varid - * The PIO-C interface for the NetCDF function nc_inq_varid. + * @ingroup PIOc_put_att_short + * The PIO-C interface for the NetCDF function nc_put_att_short. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param varidp a pointer that will get the variable id * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_varid (int ncid, const char *name, int *varidp) +int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op) { int ierr; int msg; @@ -195,7 +202,7 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARID; + msg = PIO_MSG_PUT_ATT_SHORT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -209,19 +216,19 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varid(file->fh, name, varidp);; + ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_varid(file->fh, name, varidp);; + ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varid(file->fh, name, varidp);; + ierr = ncmpi_put_att_short(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -234,27 +241,24 @@ int PIOc_inq_varid (int ncid, const char *name, int *varidp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(varidp,1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_varnatts - * The PIO-C interface for the NetCDF function nc_inq_varnatts. + * @ingroup PIOc_rename_dim + * The PIO-C interface for the NetCDF function nc_rename_dim. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param nattsp a pointer that will get the number of attributes * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) +int PIOc_rename_dim (int ncid, int dimid, const char *name) { int ierr; int msg; @@ -270,7 +274,7 @@ int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARNATTS; + msg = PIO_MSG_RENAME_DIM; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -284,19 +288,19 @@ int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varnatts(file->fh, varid, nattsp);; + ierr = nc_rename_dim(file->fh, dimid, name);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_varnatts(file->fh, varid, nattsp);; + ierr = nc_rename_dim(file->fh, dimid, name);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varnatts(file->fh, varid, nattsp);; + ierr = ncmpi_rename_dim(file->fh, dimid, name);; break; #endif default: @@ -309,27 +313,25 @@ int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(nattsp,1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_def_var - * The PIO-C interface for the NetCDF function nc_def_var. + * @ingroup PIOc_get_att_double + * The PIO-C interface for the NetCDF function nc_get_att_double. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param varidp a pointer that will get the variable id * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp) +int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) { int ierr; int msg; @@ -345,7 +347,7 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const in if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_DEF_VAR; + msg = PIO_MSG_GET_ATT_DOUBLE; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -359,26 +361,19 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const in #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; + ierr = nc_get_att_double(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: - if(ios->io_rank==0){ - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); - if(ierr == PIO_NOERR){ - ierr = nc_def_var_deflate(file->fh, *varidp, 0,1,1); - } - } - break; #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; + ierr = nc_get_att_double(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; + ierr = ncmpi_get_att_double(file->fh, varid, name, ip);; break; #endif default: @@ -387,32 +382,33 @@ int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const in } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_DOUBLE, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_var - * The PIO-C interface for the NetCDF function nc_inq_var. + * @ingroup PIOc_set_fill + * The PIO-C interface for the NetCDF function nc_set_fill. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. - * @param nattsp a pointer that will get the number of attributes * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp, int *nattsp) +int PIOc_set_fill (int ncid, int fillmode, int *old_modep) { int ierr; int msg; @@ -428,7 +424,7 @@ int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VAR; + msg = PIO_MSG_SET_FILL; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -442,19 +438,19 @@ int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + ierr = nc_set_fill(file->fh, fillmode, old_modep);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + ierr = nc_set_fill(file->fh, fillmode, old_modep);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; + ierr = ncmpi_set_fill(file->fh, fillmode, old_modep);; break; #endif default: @@ -467,28 +463,13 @@ int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(xtypep != NULL) mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); - if(ndimsp != NULL){ mpierr = MPI_Bcast(ndimsp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); - file->varlist[varid].ndims = (*ndimsp);} - if(nattsp != NULL) mpierr = MPI_Bcast(nattsp,1, MPI_INT, ios->ioroot, ios->my_comm); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name); - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(dimidsp != NULL) {int ndims; - PIOc_inq_varndims(file->fh, varid, &ndims); - mpierr = MPI_Bcast(dimidsp , ndims, MPI_INT, ios->ioroot, ios->my_comm); - } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_varname - * The PIO-C interface for the NetCDF function nc_inq_varname. + * @ingroup PIOc_def_var + * The PIO-C interface for the NetCDF function nc_def_var. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -498,9 +479,10 @@ int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. + * @param varidp a pointer that will get the variable id * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_varname (int ncid, int varid, char *name) +int PIOc_def_var (int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp) { int ierr; int msg; @@ -516,7 +498,7 @@ int PIOc_inq_varname (int ncid, int varid, char *name) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARNAME; + msg = PIO_MSG_DEF_VAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -530,19 +512,26 @@ int PIOc_inq_varname (int ncid, int varid, char *name) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varname(file->fh, varid, name);; + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; break; case PIO_IOTYPE_NETCDF4C: + if(ios->io_rank==0){ + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp); + if(ierr == PIO_NOERR){ + ierr = nc_def_var_deflate(file->fh, *varidp, 0,1,1); + } + } + break; #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_varname(file->fh, varid, name);; + ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varname(file->fh, varid, name);; + ierr = ncmpi_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);; break; #endif default: @@ -555,13 +544,7 @@ int PIOc_inq_varname (int ncid, int varid, char *name) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name) + 1; - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } + mpierr = MPI_Bcast(varidp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } @@ -640,20 +623,20 @@ int PIOc_put_att_double (int ncid, int varid, const char *name, nc_type xtype, P } /** - * @ingroup PIOc_put_att_int - * The PIO-C interface for the NetCDF function nc_put_att_int. + * @ingroup PIOc_inq_dim + * The PIO-C interface for the NetCDF function nc_inq_dim. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. + * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op) +int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp) { int ierr; int msg; @@ -669,7 +652,7 @@ int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_ if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_INT; + msg = PIO_MSG_INQ_DIM; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -683,19 +666,19 @@ int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_ #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_int(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);; break; #endif default: @@ -708,13 +691,21 @@ int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_ sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(name != NULL){ + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(lenp != NULL) mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_rename_att - * The PIO-C interface for the NetCDF function nc_rename_att. + * @ingroup PIOc_get_att_uchar + * The PIO-C interface for the NetCDF function nc_get_att_uchar. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -726,7 +717,7 @@ int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_ * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) +int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip) { int ierr; int msg; @@ -742,7 +733,7 @@ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_RENAME_ATT; + msg = PIO_MSG_GET_ATT_UCHAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -756,19 +747,19 @@ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_att(file->fh, varid, name, newname);; + ierr = nc_get_att_uchar(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_rename_att(file->fh, varid, name, newname);; + ierr = nc_get_att_uchar(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_att(file->fh, varid, name, newname);; + ierr = ncmpi_get_att_uchar(file->fh, varid, name, ip);; break; #endif default: @@ -777,29 +768,34 @@ int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_CHAR, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_ubyte - * The PIO-C interface for the NetCDF function nc_get_att_ubyte. + * @ingroup PIOc_inq_var_fill + * The PIO-C interface for the NetCDF function nc_inq_var_fill. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip) +int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) { int ierr; int msg; @@ -815,7 +811,7 @@ int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_UBYTE; + msg = PIO_MSG_INQ_VAR_FILL; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -829,19 +825,19 @@ int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_ubyte(file->fh, varid, name, ip);; + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_ubyte(file->fh, varid, name, ip);; + ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_ubyte(file->fh, varid, name, ip);; + ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_value);; break; #endif default: @@ -850,22 +846,18 @@ int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_BYTE, ios->ioroot, ios->my_comm); - } + mpierr = MPI_Bcast(fill_value, 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_natts - * The PIO-C interface for the NetCDF function nc_inq_natts. + * @ingroup PIOc_inq_attid + * The PIO-C interface for the NetCDF function nc_inq_attid. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -874,9 +866,11 @@ int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param idp a pointer that will get the id of the variable or attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_natts (int ncid, int *ngattsp) +int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) { int ierr; int msg; @@ -892,7 +886,7 @@ int PIOc_inq_natts (int ncid, int *ngattsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_NATTS; + msg = PIO_MSG_INQ_ATTID; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -906,19 +900,19 @@ int PIOc_inq_natts (int ncid, int *ngattsp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_natts(file->fh, ngattsp);; + ierr = nc_inq_attid(file->fh, varid, name, idp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_natts(file->fh, ngattsp);; + ierr = nc_inq_attid(file->fh, varid, name, idp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_natts(file->fh, ngattsp);; + ierr = ncmpi_inq_attid(file->fh, varid, name, idp);; break; #endif default: @@ -931,26 +925,27 @@ int PIOc_inq_natts (int ncid, int *ngattsp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(ngattsp,1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_del_att - * The PIO-C interface for the NetCDF function nc_del_att. + * @ingroup PIOc_inq_vartype + * The PIO-C interface for the NetCDF function nc_inq_vartype. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_del_att (int ncid, int varid, const char *name) +int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) { int ierr; int msg; @@ -966,7 +961,7 @@ int PIOc_del_att (int ncid, int varid, const char *name) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_DEL_ATT; + msg = PIO_MSG_INQ_VARTYPE; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -980,19 +975,19 @@ int PIOc_del_att (int ncid, int varid, const char *name) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_del_att(file->fh, varid, name);; + ierr = nc_inq_vartype(file->fh, varid, xtypep);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_del_att(file->fh, varid, name);; + ierr = nc_inq_vartype(file->fh, varid, xtypep);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_del_att(file->fh, varid, name);; + ierr = ncmpi_inq_vartype(file->fh, varid, xtypep);; break; #endif default: @@ -1005,24 +1000,26 @@ int PIOc_del_att (int ncid, int varid, const char *name) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq - * The PIO-C interface for the NetCDF function nc_inq. + * @ingroup PIOc_put_att_schar + * The PIO-C interface for the NetCDF function nc_put_att_schar. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp) +int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op) { int ierr; int msg; @@ -1038,7 +1035,7 @@ int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ; + msg = PIO_MSG_PUT_ATT_SCHAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1052,19 +1049,19 @@ int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);; + ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);; + ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);; + ierr = ncmpi_put_att_schar(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -1077,33 +1074,25 @@ int PIOc_inq (int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ndimsp != NULL) - mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(nvarsp != NULL) - mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(ngattsp != NULL) - mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(unlimdimidp != NULL) - mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm); - if(errstr != NULL) free(errstr); + if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_text - * The PIO-C interface for the NetCDF function nc_get_att_text. + * @ingroup PIOc_inq_vardimid + * The PIO-C interface for the NetCDF function nc_inq_vardimid. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) +int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp) { int ierr; int msg; @@ -1119,7 +1108,7 @@ int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_TEXT; + msg = PIO_MSG_INQ_VARDIMID; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1133,19 +1122,19 @@ int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_text(file->fh, varid, name, ip);; + ierr = nc_inq_vardimid(file->fh, varid, dimidsp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_text(file->fh, varid, name, ip);; + ierr = nc_inq_vardimid(file->fh, varid, dimidsp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_text(file->fh, varid, name, ip);; + ierr = ncmpi_inq_vardimid(file->fh, varid, dimidsp);; break; #endif default: @@ -1154,22 +1143,22 @@ int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm); - } + if(ierr==PIO_NOERR){ + int ndims; + PIOc_inq_varndims(file->fh, varid, &ndims); + mpierr = MPI_Bcast(dimidsp , ndims, MPI_INT, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_short - * The PIO-C interface for the NetCDF function nc_get_att_short. + * @ingroup PIOc_get_att_ushort + * The PIO-C interface for the NetCDF function nc_get_att_ushort. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1181,7 +1170,7 @@ int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) +int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip) { int ierr; int msg; @@ -1197,7 +1186,7 @@ int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_SHORT; + msg = PIO_MSG_GET_ATT_USHORT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1211,19 +1200,19 @@ int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_short(file->fh, varid, name, ip);; + ierr = nc_get_att_ushort(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_short(file->fh, varid, name, ip);; + ierr = nc_get_att_ushort(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_short(file->fh, varid, name, ip);; + ierr = ncmpi_get_att_ushort(file->fh, varid, name, ip);; break; #endif default: @@ -1239,27 +1228,28 @@ int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) if(ierr == PIO_NOERR){ PIO_Offset attlen; PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_SHORT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_SHORT, ios->ioroot, ios->my_comm); } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_long - * The PIO-C interface for the NetCDF function nc_put_att_long. + * @ingroup PIOc_inq_varid + * The PIO-C interface for the NetCDF function nc_inq_varid. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. + * @param varidp a pointer that will get the variable id * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op) +int PIOc_inq_varid (int ncid, const char *name, int *varidp) { int ierr; int msg; @@ -1275,7 +1265,7 @@ int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_LONG; + msg = PIO_MSG_INQ_VARID; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1289,19 +1279,19 @@ int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_varid(file->fh, name, varidp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_varid(file->fh, name, varidp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_long(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_inq_varid(file->fh, name, varidp);; break; #endif default: @@ -1314,24 +1304,27 @@ int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(varidp,1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_redef - * The PIO-C interface for the NetCDF function nc_redef. + * @ingroup PIOc_inq_attlen + * The PIO-C interface for the NetCDF function nc_inq_attlen. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_redef (int ncid) +int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) { int ierr; int msg; @@ -1347,7 +1340,7 @@ int PIOc_redef (int ncid) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_REDEF; + msg = PIO_MSG_INQ_ATTLEN; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1361,19 +1354,19 @@ int PIOc_redef (int ncid) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_redef(file->fh);; + ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_redef(file->fh);; + ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_redef(file->fh);; + ierr = ncmpi_inq_attlen(file->fh, varid, name, lenp);; break; #endif default: @@ -1386,24 +1379,27 @@ int PIOc_redef (int ncid) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_set_fill - * The PIO-C interface for the NetCDF function nc_set_fill. + * @ingroup PIOc_inq_atttype + * The PIO-C interface for the NetCDF function nc_inq_atttype. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_set_fill (int ncid, int fillmode, int *old_modep) +int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep) { int ierr; int msg; @@ -1419,7 +1415,7 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_SET_FILL; + msg = PIO_MSG_INQ_ATTTYPE; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1433,19 +1429,19 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_set_fill(file->fh, fillmode, old_modep);; + ierr = nc_inq_atttype(file->fh, varid, name, xtypep);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_set_fill(file->fh, fillmode, old_modep);; + ierr = nc_inq_atttype(file->fh, varid, name, xtypep);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_set_fill(file->fh, fillmode, old_modep);; + ierr = ncmpi_inq_atttype(file->fh, varid, name, xtypep);; break; #endif default: @@ -1458,24 +1454,26 @@ int PIOc_set_fill (int ncid, int fillmode, int *old_modep) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_enddef - * The PIO-C interface for the NetCDF function nc_enddef. + * @ingroup PIOc_rename_var + * The PIO-C interface for the NetCDF function nc_rename_var. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_enddef (int ncid) +int PIOc_rename_var (int ncid, int varid, const char *name) { int ierr; int msg; @@ -1491,7 +1489,7 @@ int PIOc_enddef (int ncid) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_ENDDEF; + msg = PIO_MSG_RENAME_VAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1505,19 +1503,19 @@ int PIOc_enddef (int ncid) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_enddef(file->fh);; + ierr = nc_rename_var(file->fh, varid, name);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_enddef(file->fh);; + ierr = nc_rename_var(file->fh, varid, name);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_enddef(file->fh);; + ierr = ncmpi_rename_var(file->fh, varid, name);; break; #endif default: @@ -1535,20 +1533,19 @@ int PIOc_enddef (int ncid) } /** - * @ingroup PIOc_rename_var - * The PIO-C interface for the NetCDF function nc_rename_var. + * @ingroup PIOc_inq_natts + * The PIO-C interface for the NetCDF function nc_inq_natts. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_rename_var (int ncid, int varid, const char *name) +int PIOc_inq_natts (int ncid, int *ngattsp) { int ierr; int msg; @@ -1564,7 +1561,7 @@ int PIOc_rename_var (int ncid, int varid, const char *name) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_RENAME_VAR; + msg = PIO_MSG_INQ_NATTS; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1578,19 +1575,19 @@ int PIOc_rename_var (int ncid, int varid, const char *name) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_var(file->fh, varid, name);; + ierr = nc_inq_natts(file->fh, ngattsp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_rename_var(file->fh, varid, name);; + ierr = nc_inq_natts(file->fh, ngattsp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_var(file->fh, varid, name);; + ierr = ncmpi_inq_natts(file->fh, ngattsp);; break; #endif default: @@ -1603,13 +1600,14 @@ int PIOc_rename_var (int ncid, int varid, const char *name) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(ngattsp,1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_short - * The PIO-C interface for the NetCDF function nc_put_att_short. + * @ingroup PIOc_put_att_ulonglong + * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1621,7 +1619,7 @@ int PIOc_rename_var (int ncid, int varid, const char *name) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const short *op) +int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op) { int ierr; int msg; @@ -1637,7 +1635,7 @@ int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_SHORT; + msg = PIO_MSG_PUT_ATT_ULONGLONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1651,19 +1649,19 @@ int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PI #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_short(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_short(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_put_att_ulonglong(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -1681,20 +1679,22 @@ int PIOc_put_att_short (int ncid, int varid, const char *name, nc_type xtype, PI } /** - * @ingroup PIOc_put_att_text - * The PIO-C interface for the NetCDF function nc_put_att_text. + * @ingroup PIOc_inq_var + * The PIO-C interface for the NetCDF function nc_inq_var. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @param nattsp a pointer that will get the number of attributes * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, const char *op) +int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp, int *nattsp) { int ierr; int msg; @@ -1710,7 +1710,7 @@ int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, co if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_TEXT; + msg = PIO_MSG_INQ_VAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1724,19 +1724,19 @@ int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, co #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; + ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; + ierr = nc_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_text(file->fh, varid, name, len, op);; + ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);; break; #endif default: @@ -1749,13 +1749,28 @@ int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, co sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(xtypep != NULL) mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(ndimsp != NULL){ mpierr = MPI_Bcast(ndimsp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + file->varlist[varid].ndims = (*ndimsp);} + if(nattsp != NULL) mpierr = MPI_Bcast(nattsp,1, MPI_INT, ios->ioroot, ios->my_comm); + if(name != NULL){ + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } + if(dimidsp != NULL) {int ndims; + PIOc_inq_varndims(file->fh, varid, &ndims); + mpierr = MPI_Bcast(dimidsp , ndims, MPI_INT, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_attname - * The PIO-C interface for the NetCDF function nc_inq_attname. + * @ingroup PIOc_rename_att + * The PIO-C interface for the NetCDF function nc_rename_att. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1765,10 +1780,9 @@ int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, co * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param attnum the attribute ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) +int PIOc_rename_att (int ncid, int varid, const char *name, const char *newname) { int ierr; int msg; @@ -1784,7 +1798,7 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATTNAME; + msg = PIO_MSG_RENAME_ATT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1798,19 +1812,19 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_attname(file->fh, varid, attnum, name);; + ierr = nc_rename_att(file->fh, varid, name, newname);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_attname(file->fh, varid, attnum, name);; + ierr = nc_rename_att(file->fh, varid, name, newname);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);; + ierr = ncmpi_rename_att(file->fh, varid, name, newname);; break; #endif default: @@ -1823,20 +1837,13 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name) + 1; - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_ulonglong - * The PIO-C interface for the NetCDF function nc_get_att_ulonglong. + * @ingroup PIOc_put_att_ushort + * The PIO-C interface for the NetCDF function nc_put_att_ushort. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -1848,7 +1855,7 @@ int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip) +int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op) { int ierr; int msg; @@ -1864,7 +1871,7 @@ int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_ULONGLONG; + msg = PIO_MSG_PUT_ATT_USHORT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1878,19 +1885,19 @@ int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);; + ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);; + ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_ulonglong(file->fh, varid, name, ip);; + ierr = ncmpi_put_att_ushort(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -1899,34 +1906,29 @@ int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_LONG_LONG, ios->ioroot, ios->my_comm); - } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_ushort - * The PIO-C interface for the NetCDF function nc_get_att_ushort. + * @ingroup PIOc_inq_dimid + * The PIO-C interface for the NetCDF function nc_inq_dimid. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. + * @param idp a pointer that will get the id of the variable or attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short *ip) +int PIOc_inq_dimid (int ncid, const char *name, int *idp) { int ierr; int msg; @@ -1942,7 +1944,7 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_USHORT; + msg = PIO_MSG_INQ_DIMID; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -1956,19 +1958,19 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_ushort(file->fh, varid, name, ip);; + ierr = nc_inq_dimid(file->fh, name, idp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_ushort(file->fh, varid, name, ip);; + ierr = nc_inq_dimid(file->fh, name, idp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_ushort(file->fh, varid, name, ip);; + ierr = ncmpi_inq_dimid(file->fh, name, idp);; break; #endif default: @@ -1977,22 +1979,18 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_SHORT, ios->ioroot, ios->my_comm); - } + mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_ulonglong - * The PIO-C interface for the NetCDF function nc_put_att_ulonglong. + * @ingroup PIOc_put_att_text + * The PIO-C interface for the NetCDF function nc_put_att_text. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2004,7 +2002,7 @@ int PIOc_get_att_ushort (int ncid, int varid, const char *name, unsigned short * * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned long long *op) +int PIOc_put_att_text (int ncid, int varid, const char *name, PIO_Offset len, const char *op) { int ierr; int msg; @@ -2020,7 +2018,7 @@ int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_ULONGLONG; + msg = PIO_MSG_PUT_ATT_TEXT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2034,19 +2032,19 @@ int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_ulonglong(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_put_att_text(file->fh, varid, name, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_ulonglong(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_put_att_text(file->fh, varid, name, len, op);; break; #endif default: @@ -2064,20 +2062,20 @@ int PIOc_put_att_ulonglong (int ncid, int varid, const char *name, nc_type xtype } /** - * @ingroup PIOc_inq_dimlen - * The PIO-C interface for the NetCDF function nc_inq_dimlen. + * @ingroup PIOc_get_att_uint + * The PIO-C interface for the NetCDF function nc_get_att_uint. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param lenp a pointer that will get the number of values + * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp) +int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) { int ierr; int msg; @@ -2093,7 +2091,7 @@ int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_DIMLEN; + msg = PIO_MSG_GET_ATT_UINT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2107,19 +2105,19 @@ int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);; + ierr = nc_get_att_uint(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);; + ierr = nc_get_att_uint(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dimlen(file->fh, dimid, lenp);; + ierr = ncmpi_get_att_uint(file->fh, varid, name, ip);; break; #endif default: @@ -2128,30 +2126,34 @@ int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_var_fill - * The PIO-C interface for the NetCDF function nc_inq_var_fill. + * @ingroup PIOc_inq_format + * The PIO-C interface for the NetCDF function nc_inq_format. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. + * @param formatp a pointer that will get the file format * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) +int PIOc_inq_format (int ncid, int *formatp) { int ierr; int msg; @@ -2167,7 +2169,7 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VAR_FILL; + msg = PIO_MSG_INQ_FORMAT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2181,19 +2183,19 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; + ierr = nc_inq_format(file->fh, formatp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_value);; + ierr = nc_inq_format(file->fh, formatp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_value);; + ierr = ncmpi_inq_format(file->fh, formatp);; break; #endif default: @@ -2206,14 +2208,14 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(fill_value, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_uint - * The PIO-C interface for the NetCDF function nc_get_att_uint. + * @ingroup PIOc_get_att_long + * The PIO-C interface for the NetCDF function nc_get_att_long. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2225,7 +2227,7 @@ int PIOc_inq_var_fill (int ncid, int varid, int *no_fill, void *fill_value) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) +int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) { int ierr; int msg; @@ -2241,7 +2243,7 @@ int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_UINT; + msg = PIO_MSG_GET_ATT_LONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2255,19 +2257,19 @@ int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_uint(file->fh, varid, name, ip);; + ierr = nc_get_att_long(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_uint(file->fh, varid, name, ip);; + ierr = nc_get_att_long(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_uint(file->fh, varid, name, ip);; + ierr = ncmpi_get_att_long(file->fh, varid, name, ip);; break; #endif default: @@ -2283,15 +2285,15 @@ int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) if(ierr == PIO_NOERR){ PIO_Offset attlen; PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG, ios->ioroot, ios->my_comm); } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_longlong - * The PIO-C interface for the NetCDF function nc_get_att_longlong. + * @ingroup PIOc_inq_attname + * The PIO-C interface for the NetCDF function nc_inq_attname. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2301,9 +2303,10 @@ int PIOc_get_att_uint (int ncid, int varid, const char *name, unsigned int *ip) * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. + * @param attnum the attribute ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) +int PIOc_inq_attname (int ncid, int varid, int attnum, char *name) { int ierr; int msg; @@ -2319,7 +2322,7 @@ int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_LONGLONG; + msg = PIO_MSG_INQ_ATTNAME; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2333,19 +2336,19 @@ int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_longlong(file->fh, varid, name, ip);; + ierr = nc_inq_attname(file->fh, varid, attnum, name);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_longlong(file->fh, varid, name, ip);; + ierr = nc_inq_attname(file->fh, varid, attnum, name);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_longlong(file->fh, varid, name, ip);; + ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);; break; #endif default: @@ -2354,22 +2357,24 @@ int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG_LONG, ios->ioroot, ios->my_comm); - } + if(name != NULL){ + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_schar - * The PIO-C interface for the NetCDF function nc_put_att_schar. + * @ingroup PIOc_inq_att + * The PIO-C interface for the NetCDF function nc_inq_att. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2379,9 +2384,11 @@ int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. + * @param xtypep a pointer that will get the type of the attribute. + * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const signed char *op) +int PIOc_inq_att (int ncid, int varid, const char *name, nc_type *xtypep, PIO_Offset *lenp) { int ierr; int msg; @@ -2397,7 +2404,7 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_SCHAR; + msg = PIO_MSG_INQ_ATT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2411,19 +2418,19 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_schar(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_schar(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);; break; #endif default: @@ -2436,13 +2443,15 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(xtypep != NULL) mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(lenp != NULL) mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_float - * The PIO-C interface for the NetCDF function nc_put_att_float. + * @ingroup PIOc_put_att_long + * The PIO-C interface for the NetCDF function nc_put_att_long. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2454,7 +2463,7 @@ int PIOc_put_att_schar (int ncid, int varid, const char *name, nc_type xtype, PI * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const float *op) +int PIOc_put_att_long (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long *op) { int ierr; int msg; @@ -2470,7 +2479,7 @@ int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_FLOAT; + msg = PIO_MSG_PUT_ATT_LONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2484,19 +2493,19 @@ int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PI #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_put_att_long(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_float(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_put_att_long(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -2514,19 +2523,19 @@ int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PI } /** - * @ingroup PIOc_inq_nvars - * The PIO-C interface for the NetCDF function nc_inq_nvars. + * @ingroup PIOc_inq_unlimdim + * The PIO-C interface for the NetCDF function nc_inq_unlimdim. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_nvars (int ncid, int *nvarsp) +int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) { int ierr; int msg; @@ -2542,7 +2551,7 @@ int PIOc_inq_nvars (int ncid, int *nvarsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_NVARS; + msg = PIO_MSG_INQ_UNLIMDIM; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2556,19 +2565,19 @@ int PIOc_inq_nvars (int ncid, int *nvarsp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_nvars(file->fh, nvarsp);; + ierr = nc_inq_unlimdim(file->fh, unlimdimidp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_nvars(file->fh, nvarsp);; + ierr = nc_inq_unlimdim(file->fh, unlimdimidp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_nvars(file->fh, nvarsp);; + ierr = ncmpi_inq_unlimdim(file->fh, unlimdimidp);; break; #endif default: @@ -2581,25 +2590,26 @@ int PIOc_inq_nvars (int ncid, int *nvarsp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(nvarsp,1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(unlimdimidp,1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_rename_dim - * The PIO-C interface for the NetCDF function nc_rename_dim. + * @ingroup PIOc_get_att_float + * The PIO-C interface for the NetCDF function nc_get_att_float. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_rename_dim (int ncid, int dimid, const char *name) +int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) { int ierr; int msg; @@ -2615,7 +2625,7 @@ int PIOc_rename_dim (int ncid, int dimid, const char *name) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_RENAME_DIM; + msg = PIO_MSG_GET_ATT_FLOAT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2629,19 +2639,19 @@ int PIOc_rename_dim (int ncid, int dimid, const char *name) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_rename_dim(file->fh, dimid, name);; + ierr = nc_get_att_float(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_rename_dim(file->fh, dimid, name);; + ierr = nc_get_att_float(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_rename_dim(file->fh, dimid, name);; + ierr = ncmpi_get_att_float(file->fh, varid, name, ip);; break; #endif default: @@ -2650,29 +2660,33 @@ int PIOc_rename_dim (int ncid, int dimid, const char *name) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_FLOAT, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_varndims - * The PIO-C interface for the NetCDF function nc_inq_varndims. + * @ingroup PIOc_inq_ndims + * The PIO-C interface for the NetCDF function nc_inq_ndims. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) +int PIOc_inq_ndims (int ncid, int *ndimsp) { int ierr; int msg; @@ -2688,11 +2702,7 @@ int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARNDIMS; - if(file->varlist[varid].ndims > 0){ - (*ndimsp) = file->varlist[varid].ndims; - return PIO_NOERR; - } + msg = PIO_MSG_INQ_NDIMS; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2706,19 +2716,19 @@ int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_varndims(file->fh, varid, ndimsp);; + ierr = nc_inq_ndims(file->fh, ndimsp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_varndims(file->fh, varid, ndimsp);; + ierr = nc_inq_ndims(file->fh, ndimsp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_varndims(file->fh, varid, ndimsp);; + ierr = ncmpi_inq_ndims(file->fh, ndimsp);; break; #endif default: @@ -2731,15 +2741,14 @@ int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(ndimsp,1, MPI_INT, ios->ioroot, ios->my_comm); - file->varlist[varid].ndims = (*ndimsp); + mpierr = MPI_Bcast(ndimsp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_long - * The PIO-C interface for the NetCDF function nc_get_att_long. + * @ingroup PIOc_put_att_int + * The PIO-C interface for the NetCDF function nc_put_att_int. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -2751,7 +2760,7 @@ int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) +int PIOc_put_att_int (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const int *op) { int ierr; int msg; @@ -2767,7 +2776,7 @@ int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_LONG; + msg = PIO_MSG_PUT_ATT_INT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2781,19 +2790,19 @@ int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_long(file->fh, varid, name, ip);; + ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_long(file->fh, varid, name, ip);; + ierr = nc_put_att_int(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_long(file->fh, varid, name, ip);; + ierr = ncmpi_put_att_int(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -2802,34 +2811,28 @@ int PIOc_get_att_long (int ncid, int varid, const char *name, long *ip) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG, ios->ioroot, ios->my_comm); - } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_dim - * The PIO-C interface for the NetCDF function nc_inq_dim. + * @ingroup PIOc_inq_nvars + * The PIO-C interface for the NetCDF function nc_inq_nvars. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp) +int PIOc_inq_nvars (int ncid, int *nvarsp) { int ierr; int msg; @@ -2845,7 +2848,7 @@ int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_DIM; + msg = PIO_MSG_INQ_NVARS; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2859,19 +2862,19 @@ int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; + ierr = nc_inq_nvars(file->fh, nvarsp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);; + ierr = nc_inq_nvars(file->fh, nvarsp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);; + ierr = ncmpi_inq_nvars(file->fh, nvarsp);; break; #endif default: @@ -2884,33 +2887,25 @@ int PIOc_inq_dim (int ncid, int dimid, char *name, PIO_Offset *lenp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name); - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } - if(lenp != NULL) mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(nvarsp,1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_dimid - * The PIO-C interface for the NetCDF function nc_inq_dimid. + * @ingroup PIOc_enddef + * The PIO-C interface for the NetCDF function nc_enddef. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param idp a pointer that will get the id of the variable or attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_dimid (int ncid, const char *name, int *idp) +int PIOc_enddef (int ncid) { int ierr; int msg; @@ -2926,7 +2921,7 @@ int PIOc_inq_dimid (int ncid, const char *name, int *idp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_DIMID; + msg = PIO_MSG_ENDDEF; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -2940,19 +2935,19 @@ int PIOc_inq_dimid (int ncid, const char *name, int *idp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dimid(file->fh, name, idp);; + ierr = nc_enddef(file->fh);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_dimid(file->fh, name, idp);; + ierr = nc_enddef(file->fh);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dimid(file->fh, name, idp);; + ierr = ncmpi_enddef(file->fh);; break; #endif default: @@ -2965,25 +2960,25 @@ int PIOc_inq_dimid (int ncid, const char *name, int *idp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_unlimdim - * The PIO-C interface for the NetCDF function nc_inq_unlimdim. + * @ingroup PIOc_put_att_uchar + * The PIO-C interface for the NetCDF function nc_put_att_uchar. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) +int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) { int ierr; int msg; @@ -2999,7 +2994,7 @@ int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_UNLIMDIM; + msg = PIO_MSG_PUT_ATT_UCHAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3013,19 +3008,19 @@ int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_unlimdim(file->fh, unlimdimidp);; + ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_unlimdim(file->fh, unlimdimidp);; + ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_unlimdim(file->fh, unlimdimidp);; + ierr = ncmpi_put_att_uchar(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -3038,26 +3033,25 @@ int PIOc_inq_unlimdim (int ncid, int *unlimdimidp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(unlimdimidp,1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_vardimid - * The PIO-C interface for the NetCDF function nc_inq_vardimid. + * @ingroup PIOc_put_att_longlong + * The PIO-C interface for the NetCDF function nc_put_att_longlong. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp) +int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long long *op) { int ierr; int msg; @@ -3073,7 +3067,7 @@ int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARDIMID; + msg = PIO_MSG_PUT_ATT_LONGLONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3087,19 +3081,19 @@ int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_vardimid(file->fh, varid, dimidsp);; + ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_vardimid(file->fh, varid, dimidsp);; + ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_vardimid(file->fh, varid, dimidsp);; + ierr = ncmpi_put_att_longlong(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -3112,31 +3106,26 @@ int PIOc_inq_vardimid (int ncid, int varid, int *dimidsp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr==PIO_NOERR){ - int ndims; - PIOc_inq_varndims(file->fh, varid, &ndims); - mpierr = MPI_Bcast(dimidsp , ndims, MPI_INT, ios->ioroot, ios->my_comm); - } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_attlen - * The PIO-C interface for the NetCDF function nc_inq_attlen. + * @ingroup PIOc_inq_varnatts + * The PIO-C interface for the NetCDF function nc_inq_varnatts. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param lenp a pointer that will get the number of values + * @param nattsp a pointer that will get the number of attributes * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) +int PIOc_inq_varnatts (int ncid, int varid, int *nattsp) { int ierr; int msg; @@ -3152,7 +3141,7 @@ int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATTLEN; + msg = PIO_MSG_INQ_VARNATTS; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3166,19 +3155,19 @@ int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);; + ierr = nc_inq_varnatts(file->fh, varid, nattsp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_attlen(file->fh, varid, name, (size_t *)lenp);; + ierr = nc_inq_varnatts(file->fh, varid, nattsp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_attlen(file->fh, varid, name, lenp);; + ierr = ncmpi_inq_varnatts(file->fh, varid, nattsp);; break; #endif default: @@ -3191,25 +3180,26 @@ int PIOc_inq_attlen (int ncid, int varid, const char *name, PIO_Offset *lenp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(nattsp,1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_dimname - * The PIO-C interface for the NetCDF function nc_inq_dimname. + * @ingroup PIOc_get_att_ubyte + * The PIO-C interface for the NetCDF function nc_get_att_ubyte. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_dimname (int ncid, int dimid, char *name) +int PIOc_get_att_ubyte (int ncid, int varid, const char *name, unsigned char *ip) { int ierr; int msg; @@ -3225,7 +3215,7 @@ int PIOc_inq_dimname (int ncid, int dimid, char *name) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_DIMNAME; + msg = PIO_MSG_GET_ATT_UBYTE; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3239,19 +3229,19 @@ int PIOc_inq_dimname (int ncid, int dimid, char *name) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_dimname(file->fh, dimid, name);; + ierr = nc_get_att_ubyte(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_dimname(file->fh, dimid, name);; + ierr = nc_get_att_ubyte(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_dimname(file->fh, dimid, name);; + ierr = ncmpi_get_att_ubyte(file->fh, varid, name, ip);; break; #endif default: @@ -3260,24 +3250,22 @@ int PIOc_inq_dimname (int ncid, int dimid, char *name) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(name != NULL){ - int slen; - if(ios->iomaster) - slen = (int) strlen(name) + 1; - mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); - mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); - } + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_BYTE, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_ushort - * The PIO-C interface for the NetCDF function nc_put_att_ushort. + * @ingroup PIOc_get_att_text + * The PIO-C interface for the NetCDF function nc_get_att_text. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -3289,7 +3277,7 @@ int PIOc_inq_dimname (int ncid, int dimid, char *name) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned short *op) +int PIOc_get_att_text (int ncid, int varid, const char *name, char *ip) { int ierr; int msg; @@ -3305,7 +3293,7 @@ int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, P if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_USHORT; + msg = PIO_MSG_GET_ATT_TEXT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3319,19 +3307,19 @@ int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, P #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_get_att_text(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_ushort(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_get_att_text(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_ushort(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_get_att_text(file->fh, varid, name, ip);; break; #endif default: @@ -3340,17 +3328,22 @@ int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, P } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_float - * The PIO-C interface for the NetCDF function nc_get_att_float. + * @ingroup PIOc_del_att + * The PIO-C interface for the NetCDF function nc_del_att. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -3362,7 +3355,7 @@ int PIOc_put_att_ushort (int ncid, int varid, const char *name, nc_type xtype, P * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) +int PIOc_del_att (int ncid, int varid, const char *name) { int ierr; int msg; @@ -3378,7 +3371,7 @@ int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_FLOAT; + msg = PIO_MSG_DEL_ATT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3392,19 +3385,19 @@ int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_float(file->fh, varid, name, ip);; + ierr = nc_del_att(file->fh, varid, name);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_float(file->fh, varid, name, ip);; + ierr = nc_del_att(file->fh, varid, name);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_float(file->fh, varid, name, ip);; + ierr = ncmpi_del_att(file->fh, varid, name);; break; #endif default: @@ -3413,34 +3406,29 @@ int PIOc_get_att_float (int ncid, int varid, const char *name, float *ip) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_FLOAT, ios->ioroot, ios->my_comm); - } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_longlong - * The PIO-C interface for the NetCDF function nc_put_att_longlong. + * @ingroup PIOc_inq_dimlen + * The PIO-C interface for the NetCDF function nc_inq_dimlen. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. + * @param lenp a pointer that will get the number of values * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const long long *op) +int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp) { int ierr; int msg; @@ -3456,7 +3444,7 @@ int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_LONGLONG; + msg = PIO_MSG_INQ_DIMLEN; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3470,19 +3458,19 @@ int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_longlong(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_inq_dimlen(file->fh, dimid, (size_t *)lenp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_longlong(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_inq_dimlen(file->fh, dimid, lenp);; break; #endif default: @@ -3495,13 +3483,14 @@ int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_uint - * The PIO-C interface for the NetCDF function nc_put_att_uint. + * @ingroup PIOc_get_att_schar + * The PIO-C interface for the NetCDF function nc_get_att_schar. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -3513,7 +3502,7 @@ int PIOc_put_att_longlong (int ncid, int varid, const char *name, nc_type xtype, * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned int *op) +int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) { int ierr; int msg; @@ -3529,7 +3518,7 @@ int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_UINT; + msg = PIO_MSG_GET_ATT_SCHAR; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3543,19 +3532,19 @@ int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_get_att_schar(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_get_att_schar(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_uint(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_get_att_schar(file->fh, varid, name, ip);; break; #endif default: @@ -3564,17 +3553,22 @@ int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_schar - * The PIO-C interface for the NetCDF function nc_get_att_schar. + * @ingroup PIOc_get_att_ulonglong + * The PIO-C interface for the NetCDF function nc_get_att_ulonglong. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -3586,7 +3580,7 @@ int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) +int PIOc_get_att_ulonglong (int ncid, int varid, const char *name, unsigned long long *ip) { int ierr; int msg; @@ -3602,7 +3596,7 @@ int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_SCHAR; + msg = PIO_MSG_GET_ATT_ULONGLONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3616,19 +3610,19 @@ int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_schar(file->fh, varid, name, ip);; + ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_schar(file->fh, varid, name, ip);; + ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_schar(file->fh, varid, name, ip);; + ierr = ncmpi_get_att_ulonglong(file->fh, varid, name, ip);; break; #endif default: @@ -3644,28 +3638,27 @@ int PIOc_get_att_schar (int ncid, int varid, const char *name, signed char *ip) if(ierr == PIO_NOERR){ PIO_Offset attlen; PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_CHAR, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_LONG_LONG, ios->ioroot, ios->my_comm); } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_attid - * The PIO-C interface for the NetCDF function nc_inq_attid. + * @ingroup PIOc_inq_varndims + * The PIO-C interface for the NetCDF function nc_inq_varndims. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param idp a pointer that will get the id of the variable or attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) +int PIOc_inq_varndims (int ncid, int varid, int *ndimsp) { int ierr; int msg; @@ -3681,7 +3674,11 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATTID; + msg = PIO_MSG_INQ_VARNDIMS; + if(file->varlist[varid].ndims > 0){ + (*ndimsp) = file->varlist[varid].ndims; + return PIO_NOERR; + } if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3695,19 +3692,19 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_attid(file->fh, varid, name, idp);; + ierr = nc_inq_varndims(file->fh, varid, ndimsp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_attid(file->fh, varid, name, idp);; + ierr = nc_inq_varndims(file->fh, varid, ndimsp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_attid(file->fh, varid, name, idp);; + ierr = ncmpi_inq_varndims(file->fh, varid, ndimsp);; break; #endif default: @@ -3720,26 +3717,27 @@ int PIOc_inq_attid (int ncid, int varid, const char *name, int *idp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(ndimsp,1, MPI_INT, ios->ioroot, ios->my_comm); + file->varlist[varid].ndims = (*ndimsp); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_def_dim - * The PIO-C interface for the NetCDF function nc_def_dim. + * @ingroup PIOc_inq_varname + * The PIO-C interface for the NetCDF function nc_inq_varname. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param idp a pointer that will get the id of the variable or attribute. + * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) +int PIOc_inq_varname (int ncid, int varid, char *name) { int ierr; int msg; @@ -3755,7 +3753,7 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_DEF_DIM; + msg = PIO_MSG_INQ_VARNAME; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3769,19 +3767,19 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; + ierr = nc_inq_varname(file->fh, varid, name);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; + ierr = nc_inq_varname(file->fh, varid, name);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_def_dim(file->fh, name, len, idp);; + ierr = ncmpi_inq_varname(file->fh, varid, name);; break; #endif default: @@ -3794,14 +3792,20 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(name != NULL){ + int slen; + if(ios->iomaster) + slen = (int) strlen(name) + 1; + mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(name, slen, MPI_CHAR, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_ndims - * The PIO-C interface for the NetCDF function nc_inq_ndims. + * @ingroup PIOc_def_dim + * The PIO-C interface for the NetCDF function nc_def_dim. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -3810,9 +3814,10 @@ int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). + * @param idp a pointer that will get the id of the variable or attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_ndims (int ncid, int *ndimsp) +int PIOc_def_dim (int ncid, const char *name, PIO_Offset len, int *idp) { int ierr; int msg; @@ -3828,7 +3833,7 @@ int PIOc_inq_ndims (int ncid, int *ndimsp) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_NDIMS; + msg = PIO_MSG_DEF_DIM; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3842,19 +3847,19 @@ int PIOc_inq_ndims (int ncid, int *ndimsp) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_ndims(file->fh, ndimsp);; + ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_ndims(file->fh, ndimsp);; + ierr = nc_def_dim(file->fh, name, (size_t)len, idp);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_ndims(file->fh, ndimsp);; + ierr = ncmpi_def_dim(file->fh, name, len, idp);; break; #endif default: @@ -3867,27 +3872,26 @@ int PIOc_inq_ndims (int ncid, int *ndimsp) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(ndimsp , 1, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_inq_vartype - * The PIO-C interface for the NetCDF function nc_inq_vartype. + * @ingroup PIOc_put_att_uint + * The PIO-C interface for the NetCDF function nc_put_att_uint. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) +int PIOc_put_att_uint (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned int *op) { int ierr; int msg; @@ -3903,7 +3907,7 @@ int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_VARTYPE; + msg = PIO_MSG_PUT_ATT_UINT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3917,19 +3921,19 @@ int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_vartype(file->fh, varid, xtypep);; + ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_vartype(file->fh, varid, xtypep);; + ierr = nc_put_att_uint(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_vartype(file->fh, varid, xtypep);; + ierr = ncmpi_put_att_uint(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -3942,14 +3946,13 @@ int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_int - * The PIO-C interface for the NetCDF function nc_get_att_int. + * @ingroup PIOc_get_att_short + * The PIO-C interface for the NetCDF function nc_get_att_short. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -3961,7 +3964,7 @@ int PIOc_inq_vartype (int ncid, int varid, nc_type *xtypep) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) +int PIOc_get_att_short (int ncid, int varid, const char *name, short *ip) { int ierr; int msg; @@ -3977,7 +3980,7 @@ int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_INT; + msg = PIO_MSG_GET_ATT_SHORT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -3991,19 +3994,19 @@ int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_int(file->fh, varid, name, ip);; + ierr = nc_get_att_short(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_int(file->fh, varid, name, ip);; + ierr = nc_get_att_short(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_int(file->fh, varid, name, ip);; + ierr = ncmpi_get_att_short(file->fh, varid, name, ip);; break; #endif default: @@ -4019,27 +4022,26 @@ int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) if(ierr == PIO_NOERR){ PIO_Offset attlen; PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_INT, ios->ioroot, ios->my_comm); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_SHORT, ios->ioroot, ios->my_comm); } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_double - * The PIO-C interface for the NetCDF function nc_get_att_double. + * @ingroup PIOc_redef + * The PIO-C interface for the NetCDF function nc_redef. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand * please read about this function in the NetCDF documentation at: - * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html + * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html * * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). - * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) +int PIOc_redef (int ncid) { int ierr; int msg; @@ -4055,7 +4057,7 @@ int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_DOUBLE; + msg = PIO_MSG_REDEF; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4069,19 +4071,19 @@ int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_double(file->fh, varid, name, ip);; + ierr = nc_redef(file->fh);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_double(file->fh, varid, name, ip);; + ierr = nc_redef(file->fh);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_double(file->fh, varid, name, ip);; + ierr = ncmpi_redef(file->fh);; break; #endif default: @@ -4090,15 +4092,10 @@ int PIOc_get_att_double (int ncid, int varid, const char *name, double *ip) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_DOUBLE, ios->ioroot, ios->my_comm); - } if(errstr != NULL) free(errstr); return ierr; } @@ -4177,8 +4174,8 @@ int PIOc_put_att_ubyte (int ncid, int varid, const char *name, nc_type xtype, PI } /** - * @ingroup PIOc_inq_atttype - * The PIO-C interface for the NetCDF function nc_inq_atttype. + * @ingroup PIOc_get_att_int + * The PIO-C interface for the NetCDF function nc_get_att_int. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -4188,10 +4185,9 @@ int PIOc_put_att_ubyte (int ncid, int varid, const char *name, nc_type xtype, PI * @param ncid the ncid of the open file, obtained from * PIOc_openfile() or PIOc_createfile(). * @param varid the variable ID. - * @param xtypep a pointer that will get the type of the attribute. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep) +int PIOc_get_att_int (int ncid, int varid, const char *name, int *ip) { int ierr; int msg; @@ -4207,7 +4203,7 @@ int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep) if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_INQ_ATTTYPE; + msg = PIO_MSG_GET_ATT_INT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4221,19 +4217,19 @@ int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep) #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_inq_atttype(file->fh, varid, name, xtypep);; + ierr = nc_get_att_int(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_inq_atttype(file->fh, varid, name, xtypep);; + ierr = nc_get_att_int(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_inq_atttype(file->fh, varid, name, xtypep);; + ierr = ncmpi_get_att_int(file->fh, varid, name, ip);; break; #endif default: @@ -4242,18 +4238,22 @@ int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep) } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - mpierr = MPI_Bcast(xtypep , 1, MPI_INT, ios->ioroot, ios->my_comm); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_INT, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_put_att_uchar - * The PIO-C interface for the NetCDF function nc_put_att_uchar. + * @ingroup PIOc_get_att_longlong + * The PIO-C interface for the NetCDF function nc_get_att_longlong. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -4265,7 +4265,7 @@ int PIOc_inq_atttype (int ncid, int varid, const char *name, nc_type *xtypep) * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const unsigned char *op) +int PIOc_get_att_longlong (int ncid, int varid, const char *name, long long *ip) { int ierr; int msg; @@ -4281,7 +4281,7 @@ int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PI if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_PUT_ATT_UCHAR; + msg = PIO_MSG_GET_ATT_LONGLONG; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4295,19 +4295,19 @@ int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PI #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_get_att_longlong(file->fh, varid, name, ip);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_put_att_uchar(file->fh, varid, name, xtype, (size_t)len, op);; + ierr = nc_get_att_longlong(file->fh, varid, name, ip);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_put_att_uchar(file->fh, varid, name, xtype, len, op);; + ierr = ncmpi_get_att_longlong(file->fh, varid, name, ip);; break; #endif default: @@ -4316,17 +4316,22 @@ int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PI } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); - sprintf(errstr,"in file %s",__FILE__); + errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); + sprintf(errstr,"name %s in file %s",name,__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); + if(ierr == PIO_NOERR){ + PIO_Offset attlen; + PIOc_inq_attlen(file->fh, varid, name, &attlen); + mpierr = MPI_Bcast(ip , (int) attlen, MPI_LONG_LONG, ios->ioroot, ios->my_comm); + } if(errstr != NULL) free(errstr); return ierr; } /** - * @ingroup PIOc_get_att_uchar - * The PIO-C interface for the NetCDF function nc_get_att_uchar. + * @ingroup PIOc_put_att_float + * The PIO-C interface for the NetCDF function nc_put_att_float. * * This routine is called collectively by all tasks in the communicator * ios.union_comm. For more information on the underlying NetCDF commmand @@ -4338,7 +4343,7 @@ int PIOc_put_att_uchar (int ncid, int varid, const char *name, nc_type xtype, PI * @param varid the variable ID. * @return PIO_NOERR for success, error code otherwise. See PIOc_Set_File_Error_Handling */ -int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip) +int PIOc_put_att_float (int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const float *op) { int ierr; int msg; @@ -4354,7 +4359,7 @@ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip if(file == NULL) return PIO_EBADID; ios = file->iosystem; - msg = PIO_MSG_GET_ATT_UCHAR; + msg = PIO_MSG_PUT_ATT_FLOAT; if(ios->async_interface && ! ios->ioproc){ if(ios->compmaster) @@ -4368,19 +4373,19 @@ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip #ifdef _NETCDF #ifdef _NETCDF4 case PIO_IOTYPE_NETCDF4P: - ierr = nc_get_att_uchar(file->fh, varid, name, ip);; + ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);; break; case PIO_IOTYPE_NETCDF4C: #endif case PIO_IOTYPE_NETCDF: if(ios->io_rank==0){ - ierr = nc_get_att_uchar(file->fh, varid, name, ip);; + ierr = nc_put_att_float(file->fh, varid, name, xtype, (size_t)len, op);; } break; #endif #ifdef _PNETCDF case PIO_IOTYPE_PNETCDF: - ierr = ncmpi_get_att_uchar(file->fh, varid, name, ip);; + ierr = ncmpi_put_att_float(file->fh, varid, name, xtype, len, op);; break; #endif default: @@ -4389,15 +4394,10 @@ int PIOc_get_att_uchar (int ncid, int varid, const char *name, unsigned char *ip } if(ierr != PIO_NOERR){ - errstr = (char *) malloc((strlen(name)+strlen(__FILE__) + 40)* sizeof(char)); - sprintf(errstr,"name %s in file %s",name,__FILE__); + errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char)); + sprintf(errstr,"in file %s",__FILE__); } ierr = check_netcdf(file, ierr, errstr,__LINE__); - if(ierr == PIO_NOERR){ - PIO_Offset attlen; - PIOc_inq_attlen(file->fh, varid, name, &attlen); - mpierr = MPI_Bcast(ip , (int) attlen, MPI_UNSIGNED_CHAR, ios->ioroot, ios->my_comm); - } if(errstr != NULL) free(errstr); return ierr; } From ce3e95f5280596a101387e8ff634e50a4836433a Mon Sep 17 00:00:00 2001 From: Ed Hartnett Date: Tue, 9 Feb 2016 12:23:45 -0500 Subject: [PATCH 11/48] fixing def_var_deflate issues --- doc/source/mach_walkthrough.txt | 21 +++- src/clib/pio_nc4.c | 38 +++++-- src/flib/pio.F90 | 2 + src/flib/pio_nf.F90 | 111 +++++++++++++++++++- tests/unit/ncdf_tests.F90 | 181 ++++++++++++++++++++++++++++++++ tests/unit/test_nc4.c | 21 +++- 6 files changed, 360 insertions(+), 14 deletions(-) diff --git a/doc/source/mach_walkthrough.txt b/doc/source/mach_walkthrough.txt index 0c2114cb9d42..7741c44e8e4c 100644 --- a/doc/source/mach_walkthrough.txt +++ b/doc/source/mach_walkthrough.txt @@ -349,7 +349,7 @@ with: 
tar zxf parallel-netcdf-1.6.1.tar.gz

Build with: 

cd parallel-netcdf-1.6.1
-FC=mpifort CC=mpicc ./configure --prefix=/usr/local
+FC=mpifort CC=mpicc CFLAGS=-fPIC ./configure --prefix=/usr/local
 make all check
 sudo make install
@@ -371,16 +371,29 @@ sudo make install

Download the netcdf C library from the NetCDF download -page. (These instructions were tested using version 4.3.3.1). Untar -with: 

tar zxf netcdf-4.3.3.1.tar.gz
+page. (These instructions were tested using version 4.4.0). Untar +with: 
tar zxf netcdf-c-4.4.0.tar.gz

Build with: -

cd netcdf-4.3.3.1
+
cd netcdf-c-4.4.0
 CC=mpicc ./configure --with-zlib=/usr/local --prefix=/usr/local --with-szlib=/usr/local --with-hdf5=/usr/local --enable-pnetcdf --enable-parallel-tests --enable-logging
 make all check
 sudo make install 

 
+
  • Installing NetCDF-4 Fortran Library
+
+
    Download the netcdf Fortran library from the NetCDF download
+page. (These instructions were tested using version 4.4.3). Untar
+with: 
    tar zxf netcdf-fortran-4.4.3.tar.gz
    
+
+
+
    Build with:
+
    cd netcdf-fortran-4.4.3
+F77=mpif77 F90=mpif90  ./configure --with-zlib=/usr/local --prefix=/usr/local --with-szlib=/usr/local --with-hdf5=/usr/local --enable-pnetcdf --enable-parallel-tests --enable-logging
+make all check
+sudo make install 
    
 
 
  • Installing ParallelIO Library
 
diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index 6d126870ecf9..ca96f09c7daf 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -57,7 +57,12 @@ int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
-	    ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level);
+	    /* Versions of netCDF 4.4 and earlier do not return an
+	     * error when attempting to turn on deflation with
+	     * parallel I.O. But this is not allowed by HDF5. So
+	     * return the correct error code. */
+	    ierr = NC_EINVAL;
+	    //ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
 	    if (!ios->io_rank)
@@ -65,12 +70,12 @@ int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
-	    return PIO_ENOTNC4;
+	    ierr = PIO_ENOTNC4;
 	    break;
 #endif
 #ifdef _PNETCDF
 	case PIO_IOTYPE_PNETCDF:
-	    return PIO_ENOTNC4;
+	    ierr = PIO_ENOTNC4;
 	    break;
 #endif
 	default:
@@ -120,6 +125,7 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
     iosystem_desc_t *ios;
     file_desc_t *file;
     char *errstr;
+    int ret;
 
     errstr = NULL;
     ierr = PIO_NOERR;
@@ -127,7 +133,7 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
     if (!(file = pio_get_file_from_id(ncid)))
 	return PIO_EBADID;
     ios = file->iosystem;
-    msg = PIO_MSG_INQ_VAR_CHUNKING;
+    msg = PIO_MSG_INQ_VAR_DEFLATE;
 
     if(ios->async_interface && ! ios->ioproc){
 	if(ios->compmaster) 
@@ -143,9 +149,14 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
 	    ierr = nc_inq_var_deflate(file->fh, varid, shufflep, deflatep, deflate_levelp);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
-	    if(ios->io_rank==0){
+	    if(ios->io_rank == 0)
 		ierr = nc_inq_var_deflate(file->fh, varid, shufflep, deflatep, deflate_levelp);
-	    }
+	    if ((ret = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+		return PIO_EIO;
+	    if ((ret = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+		return PIO_EIO;
+	    if ((ret = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+		return PIO_EIO;
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
@@ -531,6 +542,7 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizes
     iosystem_desc_t *ios;
     file_desc_t *file;
     char *errstr;
+    int ndims;
 
     errstr = NULL;
     ierr = PIO_NOERR;
@@ -554,9 +566,19 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizes
 	    ierr = nc_inq_var_chunking(file->fh, varid, storagep, chunksizesp);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
-	    if(ios->io_rank==0){
-		ierr = nc_inq_var_chunking(file->fh, varid, storagep, chunksizesp);
+	    if (ios->io_rank == 0)
+	    {
+		if ((ierr = nc_inq_var_chunking(file->fh, varid, storagep, chunksizesp)))
+		    return ierr;
+		if ((ierr = nc_inq_varndims(file->fh, varid, &ndims)))
+		    return ierr;
 	    }
+	    if ((ierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+		return PIO_EIO;
+	    if ((ierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+		return PIO_EIO;
+	    if ((ierr = MPI_Bcast(chunksizesp, ndims, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
+		return PIO_EIO;
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
diff --git a/src/flib/pio.F90 b/src/flib/pio.F90
index 14c3237ca028..d71297a33944 100644
--- a/src/flib/pio.F90
+++ b/src/flib/pio.F90
@@ -44,12 +44,14 @@ module pio
        PIO_inq_varndims ,&
        PIO_inq_vardimid ,&
        PIO_inq_varnatts ,&
+       PIO_inq_vardeflate ,&
        PIO_inq_dimid ,      &
        PIO_inq_dimname ,  &
        PIO_inq_dimlen ,    &
        PIO_inq_unlimdim, &
        PIO_def_dim   ,        &
        PIO_def_var   ,        &
+       PIO_def_var_deflate   ,        &
        PIO_redef     ,          &
 !       PIO_copy_att  ,       &   
        PIO_inquire_variable , &
diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90
index d3f863dbd797..6bc558198ef5 100644
--- a/src/flib/pio_nf.F90
+++ b/src/flib/pio_nf.F90
@@ -11,6 +11,7 @@ module pio_nf
 
   public :: &
        pio_def_var                                          ,   &
+       pio_def_var_deflate                                          ,   &
        pio_def_dim                                          ,  &
        pio_inq_attname                                      ,    & 
        pio_inq_att                                          ,        &
@@ -21,6 +22,7 @@ module pio_nf
        pio_inq_varndims                                     ,   &
        pio_inq_vardimid                                     ,   &
        pio_inq_varnatts                                     ,   &
+       pio_inq_vardeflate                                   ,    &       
        pio_inquire_variable                                 , &
        pio_inquire_dimension                                , &
        pio_inq_dimname                                      , &
@@ -39,6 +41,13 @@ module pio_nf
           def_var_0d_id                                     , &
           def_var_md_id 
   end interface
+  interface pio_def_var_deflate
+     module procedure &
+          def_var_deflate
+  end interface
+  ! interface pio_def_var_deflate
+  !    module procedure def_var_deflate
+  ! end interface
   interface pio_inq_attname
      module procedure &
           inq_attname_desc                                  , &
@@ -95,6 +104,12 @@ module pio_nf
           inq_varnatts_vid                                  ,    &
           inq_varnatts_id
   end interface
+  interface pio_inq_vardeflate
+     module procedure &
+          inq_vardeflate_desc                                 , &
+          inq_vardeflate_vid                                  , &
+          inq_vardeflate_id
+  end interface
   interface pio_inquire_dimension
      module procedure &
           inquire_dimension_desc                            , &
@@ -995,7 +1010,75 @@ end function PIOc_inq_varnatts
 
     ierr = PIOc_inq_varnatts(ncid                           ,varid-1,natts)
   end function inq_varnatts_id
-    
+
+!>
+!!  @defgroup PIO_inq_vardeflate PIO_inq_vardeflate
+!<
+!>
+!! @public 
+!! @ingroup PIO_inq_vardeflate
+!! @brief Gets metadata information for netcdf file.
+!! @details
+!! @param File @copydoc file_desc_t
+!! @param vardesc @copydoc var_desc_t
+!! @param type : The type of variable
+!! @retval ierr @copydoc error_return
+!<
+  integer function inq_vardeflate_desc(File, vardesc, shuffle, deflate, &
+       deflate_level) result(ierr)
+
+    type (File_desc_t), intent(in) :: File
+    type (Var_desc_t), intent(in) :: vardesc
+    integer, intent(out) :: shuffle
+    integer, intent(out) :: deflate
+    integer, intent(out) :: deflate_level
+
+    ierr = pio_inq_vardeflate(File%fh, vardesc%varid, shuffle, deflate, deflate_level)
+  end function inq_vardeflate_desc
+
+!>
+!! @public 
+!! @ingroup PIO_inq_vardeflate
+!! @brief Gets metadata information for netcdf file.
+!<
+  integer function inq_vardeflate_vid(File, varid, shuffle, deflate, deflate_level) result(ierr)
+
+    type (File_desc_t), intent(in) :: File
+    integer, intent(in) :: varid
+    integer, intent(out) :: shuffle
+    integer, intent(out) :: deflate
+    integer, intent(out) :: deflate_level
+
+    ierr = pio_inq_vardeflate(File%fh, varid, shuffle, deflate, deflate_level)
+  end function inq_vardeflate_vid
+!>
+!! @public 
+!! @ingroup PIO_inq_vardeflate
+!! @brief Gets metadata information for netcdf file.
+!<
+  integer function inq_vardeflate_id(ncid, varid, shuffle, deflate, &
+       deflate_level) result(ierr)
+    integer, intent(in) :: ncid
+    integer, intent(in) :: varid
+    integer, intent(out) :: shuffle
+    integer, intent(out) :: deflate
+    integer, intent(out) :: deflate_level
+
+    interface
+       integer(C_INT) function PIOc_inq_var_deflate(ncid, varid, shuffle, deflate, deflate_level) &
+            bind(C, name="PIOc_inq_var_deflate")
+         use iso_c_binding
+         integer(C_INT), value :: ncid
+         integer(C_INT), value :: varid
+         integer(C_INT) :: shuffle
+         integer(C_INT) :: deflate
+         integer(C_INT) :: deflate_level
+       end function PIOc_inq_var_deflate
+    end interface
+
+    ierr = PIOc_inq_var_deflate(ncid, varid-1, shuffle, deflate, deflate_level)
+  end function inq_vardeflate_id
+  
 !>
 !! @defgroup PIO_inq_varname
 !<
@@ -1413,5 +1496,31 @@ end function PIOc_def_var
     varid = varid+1
   end function def_var_md_id
 
+!> 
+!! @public 
+!! @ingroup PIO_def_var_deflate
+!! @brief Changes compression settings for a netCDF-4/HDF5 variable.
+!<
+  integer function def_var_deflate(file, vardesc, shuffle, deflate, deflate_level) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    type (var_desc_t), intent(in) :: vardesc
+    integer, intent(in) :: shuffle
+    integer, intent(in) :: deflate
+    integer, intent(in) :: deflate_level
+
+    interface
+       integer (C_INT) function PIOc_def_var_deflate(ncid, varid, shuffle, deflate, deflate_level) &
+            bind(c,name="PIOc_def_var_deflate")
+         use iso_c_binding
+         integer(c_int), value :: ncid
+         integer(c_int), value :: varid
+         integer(c_int), value :: shuffle
+         integer(c_int), value :: deflate
+         integer(c_int), value :: deflate_level
+       end function PIOc_def_var_deflate
+    end interface
+
+    ierr = PIOc_def_var_deflate(file%fh, vardesc%varid-1, shuffle, deflate, deflate_level)
+  end function def_var_deflate
 
 end module pio_nf
diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
index 251a2a220792..56958b319961 100644
--- a/tests/unit/ncdf_tests.F90
+++ b/tests/unit/ncdf_tests.F90
@@ -14,6 +14,7 @@ module ncdf_tests
 
   public :: test_redef
   public :: test_enddef
+  public :: test_nc4
 
 Contains
 
@@ -249,4 +250,184 @@ Subroutine test_enddef(test_id, err_msg)
 
   End Subroutine test_enddef
 
+  Subroutine test_nc4(test_id, err_msg)
+    ! test_nc4():
+    ! * Tests netCDF-4 function pio_def_var_deflate().
+    ! * Open file, enter define mode, add dimension / variable / attribute
+    !   * Set deflate on the variable.
+    ! * Leave define mode, close file
+    ! * Re-open file and check that deflate value is correct.
+    ! Routines used in test: PIO_initdecomp, PIO_openfile, PIO_def_dim,
+    !                        PIO_def_var, PIO_def_var_deflate, PIO_enddef,
+    !                        PIO_write_darray, PIO_closefile, PIO_freedecomp
+
+    ! Input / Output Vars
+    integer,                intent(in)  :: test_id
+    character(len=str_len), intent(out) :: err_msg
+
+    ! Local Vars
+    character(len=str_len) :: filename
+    integer                :: iotype, ret_val
+    integer                :: ret_val1
+
+    ! Data used to test writing
+    integer,          dimension(2) :: data_to_write, compdof
+    integer,          dimension(1) :: dims
+    type(io_desc_t)                :: iodesc_nCells
+    integer                        :: pio_dim
+    type(var_desc_t)               :: pio_var
+
+    integer :: shuffle
+    integer :: deflate
+    integer :: deflate_level
+
+    shuffle = 0
+    deflate = 1
+    deflate_level = 4
+    err_msg = "no_error"
+
+    dims(1) = 2*ntasks
+    compdof = 2*my_rank+(/1,2/)  ! Where in the global array each task writes
+    data_to_write = 1+my_rank
+
+    print *, 'calling PIO_initdecomp'
+    call PIO_initdecomp(pio_iosystem, PIO_int, dims, compdof, iodesc_nCells)
+
+    filename = fnames(test_id)
+    iotype   = iotypes(test_id)
+
+    ! Open existing file, write data to it
+    print *, 'calling PIO_openfile'
+    ret_val = PIO_openfile(pio_iosystem, pio_file, iotype, filename, PIO_write)
+    if (ret_val .ne. PIO_NOERR) then
+       ! Error in PIO_openfile
+       err_msg = "Could not open " // trim(filename) // " in write mode"
+       return
+    end if
+
+    ! Enter define mode
+    print *, 'calling PIO_redef'
+    ret_val = PIO_redef(pio_file)
+    if (ret_val .ne. PIO_NOERR) then
+       ! Error in PIO_redef
+       err_msg = "Could not enter define mode"
+       call PIO_closefile(pio_file)
+       return
+    end if
+
+    ! Define a new dimension M1.
+    print *, 'calling PIO_def_dim'
+    ret_val = PIO_def_dim(pio_file, 'M111222', int(2*ntasks,pio_offset_kind), pio_dim)
+    if (ret_val .ne. PIO_NOERR) then
+       err_msg = "Could not define dimension M111222"
+       print *, ret_val
+       call PIO_closefile(pio_file)
+       return
+    end if
+
+    ! Define a new variable
+    print *, 'calling PIO_def_va'
+    ret_val = PIO_def_var(pio_file, 'foo2222', PIO_int, &
+         (/pio_dim/), pio_var)
+    if (ret_val .ne. PIO_NOERR) then
+       ! Error in PIO_def_var
+       err_msg = "Could not define variable foo2222"
+       call PIO_closefile(pio_file)
+       return
+    end if
+
+    ! Try to turn on compression for this variable.
+    print *, 'calling PIO_def_var_deflate ', pio_file%fh, pio_var%varid, shuffle, deflate, deflate_level
+    ret_val = PIO_def_var_deflate(pio_file, pio_var, shuffle, deflate, &
+         deflate_level)
+    print *, 'PIO_def_var_deflate returned ', ret_val
+
+    ! Should not have worked except for netCDF-4/HDF5 serial.
+    if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
+       err_msg = "Could not turn on compression for variable foo2222"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to turn deflate on for pnetcdf file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to turn deflate on for netcdf classic file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to turn deflate on for parallel netcdf-4 file"
+       call PIO_closefile(pio_file)
+       return
+    end if
+
+    print *, 'calling PIO_put_att'
+    ret_val = PIO_put_att(pio_file, pio_var, "max_val", ntasks)
+    if (ret_val .ne. PIO_NOERR) then
+       ! Error in PIO_put_att
+       err_msg = "Could not define max_val attribute for foo2"
+       call PIO_closefile(pio_file)
+       return
+    end if
+
+    print *, 'calling PIO_put_att'
+    ret_val = PIO_put_att(pio_file, PIO_global, "created_by", "PIO unit tests")
+    if (ret_val .ne. PIO_NOERR) then
+       ! Error in PIO_put_att
+       err_msg = "Could not define global attribute"
+       call PIO_closefile(pio_file)
+       return
+    end if
+
+    ! Leave define mode
+    print *, 'calling PIO_enddef'
+    ret_val = PIO_enddef(pio_file)
+    if (ret_val .ne. PIO_NOERR) then
+       ! Error in PIO_enddef
+       print *,__FILE__,__LINE__,ret_val
+       err_msg = "Could not end define mode"
+       return
+    end if
+
+    ! Check the compression settings of the variables.
+    ! ret_val = PIO_inq_vardeflate(pio_file, pio_var, shuffle, deflate, deflate_level)
+
+    ! ! Should not have worked except for netCDF-4/HDF5 serial.
+    ! if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
+    !    err_msg = "Could not turn on compression for variable foo2222"
+    !    call PIO_closefile(pio_file)
+    !    return
+    ! else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
+    !    err_msg = "Did not get expected error when trying to turn deflate on for non-netcdf-4 file"
+    !    call PIO_closefile(pio_file)
+    !    return
+    ! else if (iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
+    !    err_msg = "Did not get expected error when trying to turn deflate on for non-netcdf-4 file"
+    !    call PIO_closefile(pio_file)
+    !    return
+    ! else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .ne. PIO_NOERR) then
+    !    err_msg = "Did not get expected error when trying to turn deflate on for parallel netcdf-4 file"
+    !    call PIO_closefile(pio_file)
+    !    return
+    ! end if
+
+    ! Write foo2
+    print *, 'calling PIO_write_darray'
+    call PIO_write_darray(pio_file, pio_var, iodesc_nCells, data_to_write, ret_val)
+    if (ret_val .ne. PIO_NOERR) then
+       ! Error in PIO_write_darray
+       err_msg = "Could not write data"
+       return
+    end if
+
+    ! Close file
+    print *, 'calling PIO_closefile'
+    call PIO_closefile(pio_file)
+
+    ! Free decomp
+    print *, 'calling PIO_freedecomp'
+    call PIO_freedecomp(pio_iosystem, iodesc_nCells)
+    call mpi_barrier(MPI_COMM_WORLD,ret_val)
+    
+  End Subroutine test_nc4
 end module ncdf_tests
diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index 2edfa4efaa28..59a0be94cc7b 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -299,21 +299,40 @@ main(int argc, char **argv)
 	    if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
 		ERR(ret);
 	}
+	if (verbose)
+	    printf("rank: %d Defining netCDF variable %s, ndims %d\n", my_rank, VAR_NAME, NDIM);
 	if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid)))
 	    ERR(ret);
 
 	/* For netCDF-4 files, set the chunksize to improve performance. */
 	if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P)
 	{
+	    if (verbose)
+		printf("rank: %d Defining chunksizes\n", my_rank);
 	    if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize)))
 		ERR(ret);
 
+	    /** Check that the inq_varname function works. */
+	    char varname[15];
+	    if (verbose)
+	    	printf("rank: %d Checking varname\n", my_rank);
+	    ret = PIOc_inq_varname(ncid, 0, &varname);
+	    printf("rank: %d ret: %d varname: %s\n", my_rank, ret, varname);
+	    
 	    /** Check that the inq_var_chunking function works. */
+	    if (verbose)
+		printf("rank: %d Checking chunksizes\n");
 	    if ((ret = PIOc_inq_var_chunking(ncid, 0, &storage, my_chunksize)))
 	    	ERR(ret);
+	    if (verbose)
+	    {
+		printf("rank: %d ret: %d storage: %d\n", my_rank, ret, storage);
+		for (d1 = 0; d1 < NDIM; d1++)
+		    printf("chunksize[%d]=%d\n", d1, my_chunksize[d1]);
+	    }
 	    
 	    /** For serial netCDF-4, only processor rank 0 gets the answers. */
-	    if (format[fmt] == PIO_IOTYPE_NETCDF4C && !my_rank ||
+	    if (format[fmt] == PIO_IOTYPE_NETCDF4C ||
 		format[fmt] == PIO_IOTYPE_NETCDF4P)
 	    {
 		if (storage != NC_CHUNKED)

From f09d69b75a900da3d1721bde65f72fe6b3fb4e15 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 9 Feb 2016 13:40:04 -0500
Subject: [PATCH 12/48] fixed some nc4 code

---
 src/clib/pio_nc4.c | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index ca96f09c7daf..be46d2056389 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -151,12 +151,6 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
 	case PIO_IOTYPE_NETCDF4C:
 	    if(ios->io_rank == 0)
 		ierr = nc_inq_var_deflate(file->fh, varid, shufflep, deflatep, deflate_levelp);
-	    if ((ret = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-		return PIO_EIO;
-	    if ((ret = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-		return PIO_EIO;
-	    if ((ret = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-		return PIO_EIO;
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
@@ -178,6 +172,15 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+    if (shufflep)
+	if((ret = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
+    if (deflatep)
+	if ((ret = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
+    if (deflate_levelp)
+	if ((ret = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
     if(errstr != NULL) free(errstr);
     return ierr;
 }    
@@ -573,12 +576,6 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizes
 		if ((ierr = nc_inq_varndims(file->fh, varid, &ndims)))
 		    return ierr;
 	    }
-	    if ((ierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-		return PIO_EIO;
-	    if ((ierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-		return PIO_EIO;
-	    if ((ierr = MPI_Bcast(chunksizesp, ndims, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
-		return PIO_EIO;
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
@@ -600,6 +597,13 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizes
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+    if ((ierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	return PIO_EIO;
+    if ((ierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	return PIO_EIO;
+    if ((ierr = MPI_Bcast(chunksizesp, ndims, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
+	return PIO_EIO;
+
     if(errstr != NULL) free(errstr);
     return ierr;
 }

From 29f0abda694584460a9da250d889a6d7666ae907 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 9 Feb 2016 14:50:33 -0500
Subject: [PATCH 13/48] removed unneeded test program

---
 src/clib/pio_nc4.c           |   8 +
 tests/general/CMakeLists.txt |  16 --
 tests/general/test_fillval.c | 366 -----------------------------------
 3 files changed, 8 insertions(+), 382 deletions(-)
 delete mode 100644 tests/general/test_fillval.c

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index be46d2056389..adca5cea2297 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -485,11 +485,13 @@ int PIOc_def_var_chunking(int ncid, int varid, int storage,
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
 	    ierr = nc_def_var_chunking(file->fh, varid, storage, chunksizesp);
+	    printf("parallel ierr = %d\n", ierr);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
 	    if(ios->io_rank==0){
 		ierr = nc_def_var_chunking(file->fh, varid, storage, chunksizesp);
 	    }
+	    printf("serial ierr = %d\n", ierr);
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
@@ -576,6 +578,12 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizes
 		if ((ierr = nc_inq_varndims(file->fh, varid, &ndims)))
 		    return ierr;
 	    }
+	    if ((ierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+		return PIO_EIO;
+	    if ((ierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+		return PIO_EIO;
+	    if ((ierr = MPI_Bcast(chunksizesp, ndims, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
+		return PIO_EIO;
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
diff --git a/tests/general/CMakeLists.txt b/tests/general/CMakeLists.txt
index d49819efc3f6..448918a1a77b 100644
--- a/tests/general/CMakeLists.txt
+++ b/tests/general/CMakeLists.txt
@@ -41,22 +41,6 @@ endif ()
 # Test Timeout (4 min = 240 sec)
 set (DEFAULT_TEST_TIMEOUT 240)
 
-add_executable (test_fillval EXCLUDE_FROM_ALL test_fillval.c)
-target_link_libraries (test_fillval pioc)
-add_dependencies (tests test_fillval)
-
-if (PIO_USE_MPISERIAL)
-    add_test(NAME test_fillval
-        COMMAND test_fillval)
-    set_tests_properties(test_fillval
-        PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
-else ()
-    add_mpi_test(test_fillval
-        EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_fillval
-        NUMPROCS 4
-        TIMEOUT ${DEFAULT_TEST_TIMEOUT})
-endif ()
-
 #===== pio_init_finalize =====
 add_executable (pio_init_finalize EXCLUDE_FROM_ALL 
     pio_init_finalize.F90
diff --git a/tests/general/test_fillval.c b/tests/general/test_fillval.c
deleted file mode 100644
index f7c4fc6ef79f..000000000000
--- a/tests/general/test_fillval.c
+++ /dev/null
@@ -1,366 +0,0 @@
-/**
- * @file 
- * Tests for handling of fillval.
- *
- * This test was added to track down memory leaks in
- * pio_decomp_fillval. Since that code is large, complex, and in
- * fortran, I have reproduced the error-inducing calls in a simple C
- * test.
- *
- */
-#include 
-#ifdef TIMING
-#include 
-#endif
-
-/** The number of possible output netCDF output flavors available to
- * the ParallelIO library. */
-#define NUM_NETCDF_FLAVORS 4
-
-/** The number of dimensions in the example data. In this example, we
- * are using three-dimensional data. */
-#define NDIM 3
-
-/** The length of our sample data along each dimension. There will be
- * a total of 16 integers in each timestep of our data, and
- * responsibilty for writing and reading them will be spread between
- * all the processors used to run this example. */
-/**@{*/
-#define X_DIM_LEN 400
-#define Y_DIM_LEN 400
-/**@}*/
-
-/** The number of timesteps of data to write. */
-#define NUM_TIMESTEPS 6
-
-/** The name of the variable in the netCDF output file. */
-#define VAR_NAME "foo"
-
-/** The meaning of life, the universe, and everything. */
-#define START_DATA_VAL 42
-
-/** Handle MPI errors. This should only be used with MPI library
- * function calls. */
-#define MPIERR(e) do {                                                  \
-	MPI_Error_string(e, err_buffer, &resultlen);			\
-	fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \
-	MPI_Finalize();							\
-	return 2;							\
-    } while (0) 
-
-/** Handle non-MPI errors by finalizing the MPI library and exiting
- * with an exit code. */
-#define ERR(e) do {				\
-        fprintf(stderr, "Error %d in %s, line %d\n", e, __FILE__, __LINE__); \
-	MPI_Finalize();				\
-	return e;				\
-    } while (0) 
-
-/** Global err buffer for MPI. When there is an MPI error, this buffer
- * is used to store the error message that is associated with the MPI
- * error. */
-char err_buffer[MPI_MAX_ERROR_STRING];
-
-/** This is the length of the most recent MPI error message, stored
- * int the global error string. */
-int resultlen;
-
-/** The dimension names. */
-char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"};
-
-/** Length of the dimensions in the sample data. */
-int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
-
-/** Length of chunksizes to use in netCDF-4 files. */
-size_t chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2};
-
-/** Error code for when things go wrong. */
-#define ERR_AWFUL 1111
-
-/** Run Tests for NetCDF-4 Functions.
- *
- * @param argc argument count
- * @param argv array of arguments
- */
-int
-main(int argc, char **argv)
-{
-    int verbose = 1;
-    
-    /** Zero-based rank of processor. */
-    int my_rank;
-
-    /** Number of processors involved in current execution. */
-    int ntasks;
-
-    /** Different output flavors. The example file is written (and
-     * then read) four times. The first two flavors,
-     * parallel-netcdf, and netCDF serial, both produce a netCDF
-     * classic format file (but with different libraries). The
-     * last two produce netCDF4/HDF5 format files, written with
-     * and without using netCDF-4 parallel I/O. */
-    int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, 
-				      PIO_IOTYPE_NETCDF,
-				      PIO_IOTYPE_NETCDF4C,
-				      PIO_IOTYPE_NETCDF4P};
-
-    /** Names for the output files. Two of them (pnetcdf and
-     * classic) will be in classic netCDF format, the others
-     * (serial4 and parallel4) will be in netCDF-4/HDF5
-     * format. All four can be read by the netCDF library, and all
-     * will contain the same contents. */
-    char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME + 1] = {"test_nc4_pnetcdf.nc",
-							  "test_nc4_classic.nc",
-							  "test_nc4_serial4.nc",
-							  "test_nc4_parallel4.nc"};
-	
-    /** Number of processors that will do IO. In this example we
-     * will do IO from all processors. */
-    int niotasks;
-
-    /** Stride in the mpi rank between io tasks. Always 1 in this
-     * example. */
-    int ioproc_stride = 1;
-
-    /** Number of the aggregator? Always 0 in this example. */
-    int numAggregator = 0;
-
-    /** Zero based rank of first processor to be used for I/O. */
-    int ioproc_start = 0;
-
-    /** Specifies the flavor of netCDF output format. */
-    int iotype;
-
-    /** The dimension IDs. */
-    int dimids[NDIM];
-
-    /** Array index per processing unit. This is the number of
-     * elements of the data array that will be handled by each
-     * processor. In this example there are 16 data elements. If the
-     * example is run on 4 processors, then arrIdxPerPe will be 4. */
-    PIO_Offset elements_per_pe;
-
-    /** The ID for the parallel I/O system. It is set by
-     * PIOc_Init_Intracomm(). It references an internal structure
-     * containing the general IO subsystem data and MPI
-     * structure. It is passed to PIOc_finalize() to free
-     * associated resources, after all I/O, but before
-     * MPI_Finalize is called. */
-    int iosysid;
-
-    /** The ncid of the netCDF file created in this example. */
-    int ncid = 0;
-
-    /** The ID of the netCDF varable in the example file. */
-    int varid;
-
-    /** The I/O description ID as passed back by PIOc_InitDecomp()
-     * and freed in PIOc_freedecomp(). */
-    int ioid;
-
-    /** A buffer for sample data.  The size of this array will
-     * vary depending on how many processors are involved in the
-     * execution of the example code. It's length will be the same
-     * as elements_per_pe.*/
-    float *buffer;
-
-    /** A buffer for reading data back from the file. The size of
-     * this array will vary depending on how many processors are
-     * involved in the execution of the example code. It's length
-     * will be the same as elements_per_pe.*/
-    int *read_buffer;
-
-    /** A 1-D array which holds the decomposition mapping for this
-     * example. The size of this array will vary depending on how
-     * many processors are involved in the execution of the
-     * example code. It's length will be the same as
-     * elements_per_pe. */
-    PIO_Offset *compdof;
-
-    /** Return code. */
-    int ret;
-
-    /* loop counters */
-    int i, fmt, d;
-
-
-    
-#ifdef TIMING    
-    /* Initialize the GPTL timing library. */
-    if ((ret = GPTLinitialize ()))
-	return ret;
-#endif    
-    
-    /* Initialize MPI. */
-    if ((ret = MPI_Init(&argc, &argv)))
-	MPIERR(ret);
-    if ((ret = MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
-	MPIERR(ret);
-
-    /* Learn my rank and the total number of processors. */
-    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
-	MPIERR(ret);
-    if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
-	MPIERR(ret);
-
-    /* Check that a valid number of processors was specified. */
-    if (!(ntasks == 1 || ntasks == 2 || ntasks == 4 ||
-	  ntasks == 8 || ntasks == 16))
-	fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n");
-    if (verbose)
-	printf("%d: ParallelIO Library example1 running on %d processors.\n",
-	       my_rank, ntasks);
-
-    /* keep things simple - 1 iotask per MPI process */    
-    niotasks = ntasks; 
-
-    /* Initialize the PIO IO system. This specifies how
-     * many and which processors are involved in I/O. */
-    if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, niotasks, ioproc_stride,
-				   ioproc_start, PIO_REARR_SUBSET, &iosysid)))
-	ERR(ret);
-
-    /* Describe the decomposition. This is a 1-based array, so add 1! */
-    elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks;
-    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
-	return PIO_ENOMEM;
-    for ( i = 0; i < elements_per_pe; i++) {
-	compdof[i] = my_rank * elements_per_pe + i + 1;
-    }
-	
-    /* Create the PIO decomposition for this test. */
-    if (verbose)
-	printf("rank: %d Creating decomposition...\n", my_rank);
-    if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, &dim_len[1], (PIO_Offset)elements_per_pe,
-			       compdof, &ioid, NULL, NULL, NULL)))
-	ERR(ret);
-    free(compdof);
-
-#ifdef HAVE_MPE
-    /* Log with MPE that we are done with INIT. */
-    if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init")))
-	MPIERR(ret);
-#endif /* HAVE_MPE */
-	
-    /* Use PIO to create the example file in each of the four
-     * available ways. */
-    for ( fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) 
-    {
-#ifdef HAVE_MPE
-	/* Log with MPE that we are starting CREATE. */
-	if ((ret = MPE_Log_event(event_num[START][CREATE_PNETCDF+fmt], 0, "start create")))
-	    MPIERR(ret);
-#endif /* HAVE_MPE */
-
-	/* Create the netCDF output file. */
-	if (verbose)
-	    printf("rank: %d Creating sample file %s with format %d...\n",
-		   my_rank, filename[fmt], format[fmt]);
-
-	PIOc_Set_IOSystem_Error_Handling(iosysid, PIO_BCAST_ERROR);
-
-	if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt],
-				   PIO_CLOBBER))){
-	  if(ret == PIO_EBADIOTYPE){
-	    printf("IOtype %d not defined in build - skipping test\n",fmt);
-	    continue;
-	  }else{
-	    ERR(ret);
-	  }
-	}
-	PIOc_Set_IOSystem_Error_Handling(iosysid, PIO_INTERNAL_ERROR);
-	
-	/* Define netCDF dimensions and variable. */
-	if (verbose)
-	    printf("rank: %d Defining netCDF metadata...\n", my_rank);
-	for ( d = 0; d < NDIM; d++) {
-	    if (verbose)
-		printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank,
-		       dim_name[d], dim_len[d]);
-	    if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
-		ERR(ret);
-	}
-	if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid)))
-	    ERR(ret);
-
-	/* For netCDF-4 files, set the chunksize to improve performance. */
-	if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P)
-	{
-	    if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize)))
-		ERR(ret);
-
-	    /** Check that the inq_var_chunking function works. */
-	    int storage;
-	    size_t my_chunksize[NDIM];
-	    if ((ret = PIOc_inq_var_chunking(ncid, 0, &storage, my_chunksize)))
-	    	ERR(ret);
-	    
-	    /** For serial netCDF-4, only processor rank 0 gets the answers. */
-	    if (format[fmt] == PIO_IOTYPE_NETCDF4C && !my_rank ||
-		format[fmt] == PIO_IOTYPE_NETCDF4P)
-	    {
-		if (storage != NC_CHUNKED)
-		    ERR(ERR_AWFUL);
-		for ( d = 0; d < NDIM; d++)
-		    if (my_chunksize[d] != chunksize[d])
-		    	ERR(ERR_AWFUL);
-	    }
-
-	    /* Check that the inv_var_deflate functions works. */
-	    int shuffle;
-	    int deflate;
-	    int deflate_level;
-	    if ((ret = PIOc_inq_var_deflate(ncid, 0, &shuffle, &deflate, &deflate_level)))
-	    	ERR(ret);
-
-	    /** For serial netCDF-4, only processor rank 0 gets the
-	     * answers. Also deflate is turned on by default */
-	    if (format[fmt] == PIO_IOTYPE_NETCDF4C && !my_rank)
-		if (shuffle || !deflate || deflate_level != 1)
-		    ERR(ERR_AWFUL);
-
-	    /* For parallel netCDF, no compression available. :-( */
-	    if (format[fmt] == PIO_IOTYPE_NETCDF4P)
-		if (shuffle || deflate)
-		    ERR(ERR_AWFUL);
-
-	} else {
-	    /* Trying to set chunking for non-netCDF-4 files results
-	     * in the PIO_ENOTNC4 error. */
-	    if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize)) != PIO_ENOTNC4)
-		ERR(ERR_AWFUL);
-	}	    
-	
-	if ((ret = PIOc_enddef(ncid)))
-	    ERR(ret);
-
-	/* Close the netCDF file. */
-	if (verbose)
-	    printf("rank: %d Closing the sample data file...\n", my_rank);
-	if ((ret = PIOc_closefile(ncid)))
-	    ERR(ret);
-    }
-	
-    /* Free the PIO decomposition. */
-    if (verbose)
-	printf("rank: %d Freeing PIO decomposition...\n", my_rank);
-    if ((ret = PIOc_freedecomp(iosysid, ioid)))
-	ERR(ret);
-	
-    /* Finalize the IO system. */
-    if (verbose)
-	printf("rank: %d Freeing PIO resources...\n", my_rank);
-    if ((ret = PIOc_finalize(iosysid)))
-	ERR(ret);
-
-    /* Finalize the MPI library. */
-    MPI_Finalize();
-
-#ifdef TIMING    
-    /* Finalize the GPTL timing library. */
-    if ((ret = GPTLfinalize ()))
-	return ret;
-#endif    
-    
-    return 0;
-}

From 912c392bc40976ce7ad06dee479396e82e413123 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 9 Feb 2016 14:54:38 -0500
Subject: [PATCH 14/48] removed redundant broadcast

---
 src/clib/pio_nc4.c | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index adca5cea2297..1612f2ca4b01 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -578,12 +578,6 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizes
 		if ((ierr = nc_inq_varndims(file->fh, varid, &ndims)))
 		    return ierr;
 	    }
-	    if ((ierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-		return PIO_EIO;
-	    if ((ierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-		return PIO_EIO;
-	    if ((ierr = MPI_Bcast(chunksizesp, ndims, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
-		return PIO_EIO;
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:

From 785cd2ac75fd45a559bb01c6805412dacfe2c4c7 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 9 Feb 2016 14:57:45 -0500
Subject: [PATCH 15/48] added c99 flags to unit test directory

---
 tests/unit/CMakeLists.txt | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt
index a6a6c69d69f1..66d8995f7af1 100644
--- a/tests/unit/CMakeLists.txt
+++ b/tests/unit/CMakeLists.txt
@@ -1,5 +1,22 @@
 include (LibMPI)
 
+# Compiler-specific compiler options
+if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -c99")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+endif()
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0")
+
 #==============================================================================
 #  PREPARE FOR TESTING
 #==============================================================================

From e649334aaba514a57b14ef600f58c494aa5e7ed5 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 9 Feb 2016 15:06:14 -0500
Subject: [PATCH 16/48] removed print statement

---
 src/clib/pio_nc4.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index 1612f2ca4b01..be46d2056389 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -485,13 +485,11 @@ int PIOc_def_var_chunking(int ncid, int varid, int storage,
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
 	    ierr = nc_def_var_chunking(file->fh, varid, storage, chunksizesp);
-	    printf("parallel ierr = %d\n", ierr);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
 	    if(ios->io_rank==0){
 		ierr = nc_def_var_chunking(file->fh, varid, storage, chunksizesp);
 	    }
-	    printf("serial ierr = %d\n", ierr);
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:

From d28ef0bb734b96d63cedc868130a2f4f99c14f6c Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 9 Feb 2016 15:10:51 -0500
Subject: [PATCH 17/48] better handling of null pointers

---
 src/clib/pio_nc4.c | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index be46d2056389..a5892a7c7af7 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -130,6 +130,10 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
     errstr = NULL;
     ierr = PIO_NOERR;
 
+    /* No null pointers please! */
+    if (!shufflep || !deflatep || !deflate_levelp)
+	return PIO_EINVAL;
+
     if (!(file = pio_get_file_from_id(ncid)))
 	return PIO_EBADID;
     ios = file->iosystem;
@@ -172,15 +176,12 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
-    if (shufflep)
-	if((ret = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
-    if (deflatep)
-	if ((ret = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
-    if (deflate_levelp)
-	if ((ret = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
+    if((ret = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	return PIO_EIO;
+    if ((ret = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	return PIO_EIO;
+    if ((ret = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	return PIO_EIO;
     if(errstr != NULL) free(errstr);
     return ierr;
 }    

From 748f357b4073199a9a308fb3df514adea37ca0cc Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 9 Feb 2016 15:12:28 -0500
Subject: [PATCH 18/48] even better handling of null pointers

---
 src/clib/pio_nc4.c | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index a5892a7c7af7..be46d2056389 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -130,10 +130,6 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
     errstr = NULL;
     ierr = PIO_NOERR;
 
-    /* No null pointers please! */
-    if (!shufflep || !deflatep || !deflate_levelp)
-	return PIO_EINVAL;
-
     if (!(file = pio_get_file_from_id(ncid)))
 	return PIO_EBADID;
     ios = file->iosystem;
@@ -176,12 +172,15 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
-    if((ret = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	return PIO_EIO;
-    if ((ret = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	return PIO_EIO;
-    if ((ret = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	return PIO_EIO;
+    if (shufflep)
+	if((ret = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
+    if (deflatep)
+	if ((ret = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
+    if (deflate_levelp)
+	if ((ret = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
     if(errstr != NULL) free(errstr);
     return ierr;
 }    

From a7e2b4de3d19b35f11d9f3b40c825f202ad230d0 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 9 Feb 2016 15:26:17 -0500
Subject: [PATCH 19/48] fixed other nc4 inq functions so now result is bcast

---
 src/clib/pio_nc4.c | 42 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 38 insertions(+), 4 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index be46d2056389..8e51fed2639f 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -262,6 +262,13 @@ int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp,
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+    if (options_maskp)
+	if((ierr = MPI_Bcast(options_maskp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
+    if (pixels_per_blockp)
+	if((ierr = MPI_Bcast(pixels_per_blockp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
+	
     if(errstr != NULL) free(errstr);
     return ierr;
 }    
@@ -427,6 +434,9 @@ int PIOc_inq_var_fletcher32(int ncid, int varid, int *fletcher32p)
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+    if (fletcher32p)
+	if((ierr = MPI_Bcast(fletcher32p, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
     if(errstr != NULL) free(errstr);
     return ierr;
 }    
@@ -599,10 +609,12 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizes
     }
     if ((ierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
 	return PIO_EIO;
-    if ((ierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	return PIO_EIO;
-    if ((ierr = MPI_Bcast(chunksizesp, ndims, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
-	return PIO_EIO;
+    if (storagep)
+	if ((ierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
+    if (chunksizesp)
+	if ((ierr = MPI_Bcast(chunksizesp, ndims, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
 
     if(errstr != NULL) free(errstr);
     return ierr;
@@ -850,6 +862,9 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+    if (endianp)
+	if((ierr = MPI_Bcast(endianp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
     if(errstr != NULL) free(errstr);
     return ierr;
 }    
@@ -1016,6 +1031,16 @@ int PIOc_get_chunk_cache(int iotype, int io_rank, size_t *sizep,
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+    /* if (sizep) */
+    /* 	if ((ierr = MPI_Bcast(sizep, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm))) */
+    /* 	    return PIO_EIO; */
+    /* if (nelemsp) */
+    /* 	if ((ierr = MPI_Bcast(nelemsp, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm))) */
+    /* 	    return PIO_EIO; */
+    /* if (preemptionp) */
+    /* 	if ((ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm))) */
+    /* 	    return PIO_EIO; */
+
     if(errstr != NULL) free(errstr);
     return ierr;
 }
@@ -1186,6 +1211,15 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+    if (sizep)
+	if ((ierr = MPI_Bcast(sizep, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
+    if (nelemsp)
+	if ((ierr = MPI_Bcast(nelemsp, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
+    if (preemptionp)
+	if ((ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm)))
+	    return PIO_EIO;
     if(errstr != NULL) free(errstr);
     return ierr;
 }    

From 778a2439014ea0effd4180d98f0f9651af03ba8f Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 9 Feb 2016 16:38:36 -0500
Subject: [PATCH 20/48] fixed potential memory leak in error condition, other
 error handling code

---
 src/clib/pio_nc4.c | 410 +++++++++++++++++++++++++++++++--------------
 1 file changed, 280 insertions(+), 130 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index 8e51fed2639f..2669344389cb 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -46,14 +46,17 @@ int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
     ios = file->iosystem;
     msg = PIO_MSG_DEF_VAR_DEFLATE;
 
-    if(ios->async_interface && ! ios->ioproc){
-	if(ios->compmaster) 
+    if (ios->async_interface && ! ios->ioproc)
+    {
+	if (ios->compmaster) 
 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
     }
 
-    if(ios->ioproc){
-	switch(file->iotype){
+    if (ios->ioproc)
+    {
+	switch (file->iotype)
+	{
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
@@ -83,12 +86,20 @@ int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
 	}
     }
 
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    if(ierr != PIO_NOERR){
+    /* If there is an error, allocate space for the error string. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
-    if(errstr != NULL) free(errstr);
+
+    /* Check the netCDF return code, and broadcast it to all tasks. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free the error string if there is no error. */
+    if (errstr != NULL)
+	free(errstr);
+
     return ierr;
 }    
 
@@ -135,21 +146,24 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
     ios = file->iosystem;
     msg = PIO_MSG_INQ_VAR_DEFLATE;
 
-    if(ios->async_interface && ! ios->ioproc){
-	if(ios->compmaster) 
+    if (ios->async_interface && ! ios->ioproc)
+    {
+	if (ios->compmaster) 
 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
     }
 
-    if(ios->ioproc){
-	switch(file->iotype){
+    if (ios->ioproc)
+    {
+	switch (file->iotype)
+	{
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
 	    ierr = nc_inq_var_deflate(file->fh, varid, shufflep, deflatep, deflate_levelp);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
-	    if(ios->io_rank == 0)
+	    if (ios->io_rank == 0)
 		ierr = nc_inq_var_deflate(file->fh, varid, shufflep, deflatep, deflate_levelp);
 	    break;
 #endif
@@ -167,21 +181,27 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
 	}
     }
 
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    if(ierr != PIO_NOERR){
+    /* If there is an error, allocate space for the error string. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+
+    /* Check the netCDF return code, and broadcast it to all tasks. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free the error string if there is no error. */
+    if (errstr != NULL)
+	free(errstr);
+
+    /* Broadcast results to all tasks. */
     if (shufflep)
-	if((ret = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
+	ierr = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm);
     if (deflatep)
-	if ((ret = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
+	ierr = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm);
     if (deflate_levelp)
-	if ((ret = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
-    if(errstr != NULL) free(errstr);
+	ierr = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm);
     return ierr;
 }    
 
@@ -225,19 +245,22 @@ int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp,
     ios = file->iosystem;
     msg = PIO_MSG_INQ_VAR_SZIP;
 
-    if(ios->async_interface && ! ios->ioproc){
-	if(ios->compmaster) 
+    if (ios->async_interface && ! ios->ioproc)
+    {
+	if (ios->compmaster) 
 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
     }
 
-    if(ios->ioproc){
-	switch(file->iotype){
+    if (ios->ioproc)
+    {
+	switch (file->iotype)
+	{
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
 	    ierr = nc_inq_var_szip(file->fh, varid, options_maskp, pixels_per_blockp);
-		break;
+	    break;
 	case PIO_IOTYPE_NETCDF4C:
 	    if (!ios->io_rank)
 		ierr = nc_inq_var_szip(file->fh, varid, options_maskp, pixels_per_blockp);
@@ -257,19 +280,26 @@ int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp,
 	}
     }
 
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    if(ierr != PIO_NOERR){
+    /* Allocate an error string if needed. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+
+    /* Check for netCDF error. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free unused error string. */
+    if (errstr != NULL)
+	free(errstr);
+
+    /* Broadcast results to all tasks. */
     if (options_maskp)
-	if((ierr = MPI_Bcast(options_maskp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
+	ierr = MPI_Bcast(options_maskp, 1, MPI_INT, ios->ioroot, ios->my_comm);
     if (pixels_per_blockp)
-	if((ierr = MPI_Bcast(pixels_per_blockp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
+	ierr = MPI_Bcast(pixels_per_blockp, 1, MPI_INT, ios->ioroot, ios->my_comm);
 	
-    if(errstr != NULL) free(errstr);
     return ierr;
 }    
 
@@ -316,21 +346,25 @@ int PIOc_def_var_fletcher32(int ncid, int varid, int fletcher32)
     ios = file->iosystem;
     msg = PIO_MSG_DEF_VAR_FLETCHER32;
 
-    if(ios->async_interface && ! ios->ioproc){
-	if(ios->compmaster) 
+    if (ios->async_interface && ! ios->ioproc)
+    {
+	if (ios->compmaster) 
 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
     }
 
-    if(ios->ioproc){
-	switch(file->iotype){
+    if (ios->ioproc)
+    {
+	switch (file->iotype)
+	{
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
 	    ierr = nc_def_var_fletcher32(file->fh, varid, fletcher32);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
-	    if(ios->io_rank==0){
+	    if (ios->io_rank==0)
+	    {
 		ierr = nc_def_var_fletcher32(file->fh, varid, fletcher32);
 	    }
 	    break;
@@ -349,12 +383,20 @@ int PIOc_def_var_fletcher32(int ncid, int varid, int fletcher32)
 	}
     }
 
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    if(ierr != PIO_NOERR){
+    /* Allocate an error string if needed. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
-    if(errstr != NULL) free(errstr);
+
+    /* Check for netCDF error. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free unused error string. */
+    if (errstr != NULL)
+	free(errstr);
+
     return ierr;
 }
 
@@ -397,14 +439,17 @@ int PIOc_inq_var_fletcher32(int ncid, int varid, int *fletcher32p)
     ios = file->iosystem;
     msg = PIO_MSG_INQ_VAR_FLETCHER32;
 
-    if(ios->async_interface && ! ios->ioproc){
-	if(ios->compmaster) 
+    if (ios->async_interface && ! ios->ioproc)
+    {
+	if (ios->compmaster) 
 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
     }
 
-    if(ios->ioproc){
-	switch(file->iotype){
+    if (ios->ioproc)
+    {
+	switch (file->iotype)
+	{
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
@@ -429,15 +474,24 @@ int PIOc_inq_var_fletcher32(int ncid, int varid, int *fletcher32p)
 	}
     }
 
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    if(ierr != PIO_NOERR){
+    /* If there is an error, allocate space for the error string. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+
+    /* Check the netCDF return code, and broadcast it to all tasks. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free the error string if there is no error. */
+    if (errstr != NULL)
+	free(errstr);
+
+    /* Broadcast results to all tasks. */
     if (fletcher32p)
-	if((ierr = MPI_Bcast(fletcher32p, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
-    if(errstr != NULL) free(errstr);
+	ierr = MPI_Bcast(fletcher32p, 1, MPI_INT, ios->ioroot, ios->my_comm);
+    
     return ierr;
 }    
 
@@ -483,21 +537,25 @@ int PIOc_def_var_chunking(int ncid, int varid, int storage,
     ios = file->iosystem;
     msg = PIO_MSG_DEF_VAR_CHUNKING;
 
-    if(ios->async_interface && ! ios->ioproc){
-	if(ios->compmaster) 
+    if (ios->async_interface && ! ios->ioproc)
+    {
+	if (ios->compmaster) 
 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
     }
 
-    if(ios->ioproc){
-	switch(file->iotype){
+    if (ios->ioproc)
+    {
+	switch (file->iotype)
+	{
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
 	    ierr = nc_def_var_chunking(file->fh, varid, storage, chunksizesp);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
-	    if(ios->io_rank==0){
+	    if (ios->io_rank==0)
+	    {
 		ierr = nc_def_var_chunking(file->fh, varid, storage, chunksizesp);
 	    }
 	    break;
@@ -516,12 +574,20 @@ int PIOc_def_var_chunking(int ncid, int varid, int storage,
 	}
     }
 
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    if(ierr != PIO_NOERR){
+    /* Allocate an error string if needed. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
-    if(errstr != NULL) free(errstr);
+
+    /* Check for netCDF error. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free unused error string. */
+    if (errstr != NULL)
+	free(errstr);
+
     return ierr;
 }
 
@@ -565,14 +631,17 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizes
     ios = file->iosystem;
     msg = PIO_MSG_INQ_VAR_CHUNKING;
 
-    if(ios->async_interface && ! ios->ioproc){
-	if(ios->compmaster) 
+    if (ios->async_interface && ! ios->ioproc)
+    {
+	if (ios->compmaster) 
 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
     }
 
-    if(ios->ioproc){
-	switch(file->iotype){
+    if (ios->ioproc)
+    {
+	switch (file->iotype)
+	{
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
@@ -602,21 +671,27 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizes
 	}
     }
 
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    if(ierr != PIO_NOERR){
+    /* If there is an error, allocate space for the error string. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
-    if ((ierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	return PIO_EIO;
+
+    /* Check the netCDF return code, and broadcast it to all tasks. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free the error string if there is no error. */
+    if (errstr != NULL)
+	free(errstr);
+
+    /* Broadcast results to all tasks. */
+    ierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm);
     if (storagep)
-	if ((ierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
+	ierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm);
     if (chunksizesp)
-	if ((ierr = MPI_Bcast(chunksizesp, ndims, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
+	ierr = MPI_Bcast(chunksizesp, ndims, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm);
 
-    if(errstr != NULL) free(errstr);
     return ierr;
 }
 
@@ -661,21 +736,25 @@ int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
     ios = file->iosystem;
     msg = PIO_MSG_SET_FILL;
 
-    if(ios->async_interface && ! ios->ioproc){
-	if(ios->compmaster) 
+    if (ios->async_interface && ! ios->ioproc)
+    {
+	if (ios->compmaster) 
 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
     }
 
-    if(ios->ioproc){
-	switch(file->iotype){
+    if (ios->ioproc)
+    {
+	switch (file->iotype)
+	{
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
 	    ierr = nc_def_var_fill(file->fh, varid, no_fill, fill_value);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
-	    if(ios->io_rank==0){
+	    if (ios->io_rank==0)
+	    {
 		ierr = nc_def_var_fill(file->fh, varid, no_fill, fill_value);
 	    }
 	    break;
@@ -694,12 +773,20 @@ int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
 	}
     }
 
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    if(ierr != PIO_NOERR){
+    /* Allocate an error string if needed. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
-    if(errstr != NULL) free(errstr);
+
+    /* Check for netCDF error. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free unused error string. */
+    if (errstr != NULL)
+	free(errstr);
+
     return ierr;
 }
 
@@ -744,21 +831,25 @@ int PIOc_def_var_endian(int ncid, int varid, int endian)
     ios = file->iosystem;
     msg = PIO_MSG_DEF_VAR_ENDIAN;
 
-    if(ios->async_interface && ! ios->ioproc){
-	if(ios->compmaster) 
+    if (ios->async_interface && ! ios->ioproc)
+    {
+	if (ios->compmaster) 
 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
     }
 
-    if(ios->ioproc){
-	switch(file->iotype){
+    if (ios->ioproc)
+    {
+	switch (file->iotype)
+	{
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
 	    ierr = nc_def_var_endian(file->fh, varid, endian);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
-	    if(ios->io_rank==0){
+	    if (ios->io_rank==0)
+	    {
 		ierr = nc_def_var_endian(file->fh, varid, endian);
 	    }
 	    break;
@@ -777,12 +868,20 @@ int PIOc_def_var_endian(int ncid, int varid, int endian)
 	}
     }
 
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    if(ierr != PIO_NOERR){
+    /* Allocate an error string if needed. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
-    if(errstr != NULL) free(errstr);
+
+    /* Check for netCDF error. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free unused error string. */
+    if (errstr != NULL)
+	free(errstr);
+
     return ierr;
 }
 
@@ -825,14 +924,17 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
     ios = file->iosystem;
     msg = PIO_MSG_INQ_VAR_CHUNKING;
 
-    if(ios->async_interface && ! ios->ioproc){
-	if(ios->compmaster) 
+    if (ios->async_interface && ! ios->ioproc)
+    {
+	if (ios->compmaster) 
 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
     }
 
-    if(ios->ioproc){
-	switch(file->iotype){
+    if (ios->ioproc)
+    {
+	switch (file->iotype)
+	{
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
@@ -857,15 +959,24 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
 	}
     }
 
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    if(ierr != PIO_NOERR){
+    /* If there is an error, allocate space for the error string. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+
+    /* Check the netCDF return code, and broadcast it to all tasks. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free the error string if there is no error. */
+    if (errstr != NULL)
+	free(errstr);
+
+    /* Broadcast results to all tasks. */
     if (endianp)
-	if((ierr = MPI_Bcast(endianp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
-    if(errstr != NULL) free(errstr);
+	ierr = MPI_Bcast(endianp, 1, MPI_INT, ios->ioroot, ios->my_comm);
+
     return ierr;
 }    
 
@@ -889,7 +1000,7 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
  * function.
  * 
  * @param iotype the iotype of files to be created or opened.
-`* @param io_rank the rank of the calling process.
+ `* @param io_rank the rank of the calling process.
  * @param size size of file cache.
  * @param nelems number of elements in file cache.
  * @param preemption preemption setting for file cache.
@@ -910,13 +1021,13 @@ int PIOc_set_chunk_cache(int iotype, int io_rank, size_t size,
     ierr = PIO_NOERR;
     /*msg = PIO_MSG_SET_CHUNK_CACHE;*/
 
-    /* if(ios->async_interface && ! ios->ioproc){ */
-    /* 	if(ios->compmaster)  */
+    /* if (ios->async_interface && ! ios->ioproc){ */
+    /* 	if (ios->compmaster)  */
     /* 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); */
     /* 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); */
     /* } */
 
-    switch(iotype)
+    switch (iotype)
     {
 #ifdef _NETCDF
 #ifdef _NETCDF4
@@ -941,12 +1052,20 @@ int PIOc_set_chunk_cache(int iotype, int io_rank, size_t size,
 	ierr = iotype_error(file->iotype,__FILE__,__LINE__);
     }
 
-    /* ierr = check_netcdf(file, ierr, errstr,__LINE__); */
-    if(ierr != PIO_NOERR){
+    /* Allocate an error string if needed. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
-    if(errstr != NULL) free(errstr);
+
+    /* Check for netCDF error. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free unused error string. */
+    if (errstr != NULL)
+	free(errstr);
+
     return ierr;
 }    
 
@@ -974,7 +1093,7 @@ int PIOc_set_chunk_cache(int iotype, int io_rank, size_t size,
  * performance check chunking against access patterns.
  *
  * @param iotype the iotype of files to be created or opened.
-`* @param io_rank the rank of the calling process.
+ `* @param io_rank the rank of the calling process.
  * @param sizep gets the size of file cache.
  * @param nelemsp gets the number of elements in file cache.
  * @param preemptionp gets the preemption setting for file cache.
@@ -997,13 +1116,14 @@ int PIOc_get_chunk_cache(int iotype, int io_rank, size_t *sizep,
      * now, comment it out. EJH */
     /* msg = PIO_MSG_INQ_VAR_FLETCHER32; */
 
-    /* if(ios->async_interface && ! ios->ioproc){ */
-    /* 	if(ios->compmaster)  */
+    /* if (ios->async_interface && ! ios->ioproc){ */
+    /* 	if (ios->compmaster)  */
     /* 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); */
     /* 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); */
     /* } */
 
-    switch(iotype){
+    switch (iotype)
+    {
 #ifdef _NETCDF
 #ifdef _NETCDF4
     case PIO_IOTYPE_NETCDF4P:
@@ -1027,10 +1147,20 @@ int PIOc_get_chunk_cache(int iotype, int io_rank, size_t *sizep,
 	ierr = iotype_error(iotype,__FILE__,__LINE__);
     }
 
-    if(ierr != PIO_NOERR){
+    /* Allocate an error string if needed. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+
+    /* Check for netCDF error. */
+    /* ierr = check_netcdf(file, ierr, errstr,__LINE__);*/
+
+    /* Free unused error string. */
+    if (errstr != NULL)
+	free(errstr);
+
     /* if (sizep) */
     /* 	if ((ierr = MPI_Bcast(sizep, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm))) */
     /* 	    return PIO_EIO; */
@@ -1041,7 +1171,6 @@ int PIOc_get_chunk_cache(int iotype, int io_rank, size_t *sizep,
     /* 	if ((ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm))) */
     /* 	    return PIO_EIO; */
 
-    if(errstr != NULL) free(errstr);
     return ierr;
 }
 
@@ -1087,21 +1216,25 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
     ios = file->iosystem;
     msg = PIO_MSG_SET_VAR_CHUNK_CACHE;
 
-    if(ios->async_interface && ! ios->ioproc){
-	if(ios->compmaster) 
+    if (ios->async_interface && ! ios->ioproc)
+    {
+	if (ios->compmaster) 
 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
     }
 
-    if(ios->ioproc){
-	switch(file->iotype){
+    if (ios->ioproc)
+    {
+	switch (file->iotype)
+	{
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
 	    ierr = nc_set_var_chunk_cache(file->fh, varid, size, nelems, preemption);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
-	    if(ios->io_rank==0){
+	    if (ios->io_rank==0)
+	    {
 		ierr = nc_set_var_chunk_cache(file->fh, varid, size, nelems, preemption);
 	    }
 	    break;
@@ -1120,12 +1253,20 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
 	}
     }
 
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    if(ierr != PIO_NOERR){
+    /* Allocate an error string if needed. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
-    if(errstr != NULL) free(errstr);
+
+    /* Check for netCDF error. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free unused error string. */
+    if (errstr != NULL)
+	free(errstr);
+
     return ierr;
 }    
 
@@ -1174,14 +1315,16 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp
      * now, comment it out. EJH */
     /* msg = PIO_MSG_INQ_VAR_FLETCHER32; */
 
-    /* if(ios->async_interface && ! ios->ioproc){ */
-    /* 	if(ios->compmaster)  */
+    /* if (ios->async_interface && ! ios->ioproc){ */
+    /* 	if (ios->compmaster)  */
     /* 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); */
     /* 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); */
     /* } */
 
-    if(ios->ioproc){
-	switch(file->iotype){
+    if (ios->ioproc)
+    {
+	switch (file->iotype)
+	{
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
@@ -1206,21 +1349,28 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp
 	}
     }
 
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    if(ierr != PIO_NOERR){
+    /* If there is an error, allocate space for the error string. */
+    if (ierr != PIO_NOERR)
+    {
 	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
 	sprintf(errstr,"in file %s",__FILE__);
     }
+
+    /* Check the netCDF return code, and broadcast it to all tasks. */
+    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+
+    /* Free the error string if there is no error. */
+    if (errstr != NULL)
+	free(errstr);
+
+    /* Broadcast results to all tasks. */
     if (sizep)
-	if ((ierr = MPI_Bcast(sizep, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
+	ierr = MPI_Bcast(sizep, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm);
     if (nelemsp)
-	if ((ierr = MPI_Bcast(nelemsp, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
+	ierr = MPI_Bcast(nelemsp, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm);
     if (preemptionp)
-	if ((ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm)))
-	    return PIO_EIO;
-    if(errstr != NULL) free(errstr);
+	ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm);
+
     return ierr;
 }    
 

From d6aeb17f655bb095be2a4398619ed53d3f7a9eb1 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 9 Feb 2016 16:52:01 -0500
Subject: [PATCH 21/48] fixed comment

---
 src/clib/pio_nc4.c | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index 2669344389cb..4bd7471ed12f 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -96,7 +96,7 @@ int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
     /* Check the netCDF return code, and broadcast it to all tasks. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free the error string if there is no error. */
+    /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -191,7 +191,7 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
     /* Check the netCDF return code, and broadcast it to all tasks. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free the error string if there is no error. */
+    /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -290,7 +290,7 @@ int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp,
     /* Check for netCDF error. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free unused error string. */
+    /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -393,7 +393,7 @@ int PIOc_def_var_fletcher32(int ncid, int varid, int fletcher32)
     /* Check for netCDF error. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free unused error string. */
+    /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -484,7 +484,7 @@ int PIOc_inq_var_fletcher32(int ncid, int varid, int *fletcher32p)
     /* Check the netCDF return code, and broadcast it to all tasks. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free the error string if there is no error. */
+    /* Free the error stringif it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -584,7 +584,7 @@ int PIOc_def_var_chunking(int ncid, int varid, int storage,
     /* Check for netCDF error. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free unused error string. */
+    /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -681,7 +681,7 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizes
     /* Check the netCDF return code, and broadcast it to all tasks. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free the error string if there is no error. */
+    /* Free the error stringif it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -783,7 +783,7 @@ int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
     /* Check for netCDF error. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free unused error string. */
+    /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -878,7 +878,7 @@ int PIOc_def_var_endian(int ncid, int varid, int endian)
     /* Check for netCDF error. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free unused error string. */
+    /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -969,7 +969,7 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
     /* Check the netCDF return code, and broadcast it to all tasks. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free the error string if there is no error. */
+    /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -1062,7 +1062,7 @@ int PIOc_set_chunk_cache(int iotype, int io_rank, size_t size,
     /* Check for netCDF error. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free unused error string. */
+    /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -1157,7 +1157,7 @@ int PIOc_get_chunk_cache(int iotype, int io_rank, size_t *sizep,
     /* Check for netCDF error. */
     /* ierr = check_netcdf(file, ierr, errstr,__LINE__);*/
 
-    /* Free unused error string. */
+    /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -1263,7 +1263,7 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
     /* Check for netCDF error. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free unused error string. */
+    /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
@@ -1359,7 +1359,7 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp
     /* Check the netCDF return code, and broadcast it to all tasks. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
 
-    /* Free the error string if there is no error. */
+    /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 

From f83901233352347df342445d9497708e598225d5 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Thu, 11 Feb 2016 11:49:17 -0500
Subject: [PATCH 22/48] fortran tests for def_var_deflate turned back on

---
 tests/unit/ncdf_tests.F90 | 40 +++++++++++++++++++--------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
index 56958b319961..382e22cafaca 100644
--- a/tests/unit/ncdf_tests.F90
+++ b/tests/unit/ncdf_tests.F90
@@ -390,26 +390,26 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Check the compression settings of the variables.
-    ! ret_val = PIO_inq_vardeflate(pio_file, pio_var, shuffle, deflate, deflate_level)
-
-    ! ! Should not have worked except for netCDF-4/HDF5 serial.
-    ! if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
-    !    err_msg = "Could not turn on compression for variable foo2222"
-    !    call PIO_closefile(pio_file)
-    !    return
-    ! else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
-    !    err_msg = "Did not get expected error when trying to turn deflate on for non-netcdf-4 file"
-    !    call PIO_closefile(pio_file)
-    !    return
-    ! else if (iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
-    !    err_msg = "Did not get expected error when trying to turn deflate on for non-netcdf-4 file"
-    !    call PIO_closefile(pio_file)
-    !    return
-    ! else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .ne. PIO_NOERR) then
-    !    err_msg = "Did not get expected error when trying to turn deflate on for parallel netcdf-4 file"
-    !    call PIO_closefile(pio_file)
-    !    return
-    ! end if
+    ret_val = PIO_inq_vardeflate(pio_file, pio_var, shuffle, deflate, deflate_level)
+
+    ! Should not have worked except for netCDF-4/HDF5 serial.
+    if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
+       err_msg = "Could not turn on compression for variable foo2222"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to turn deflate on for non-netcdf-4 file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to turn deflate on for non-netcdf-4 file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .ne. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to turn deflate on for parallel netcdf-4 file"
+       call PIO_closefile(pio_file)
+       return
+    end if
 
     ! Write foo2
     print *, 'calling PIO_write_darray'

From 452fd3ac39603a40538ac640d68b18a8ac8b140e Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Thu, 11 Feb 2016 12:16:19 -0500
Subject: [PATCH 23/48] more tests for nc4 functions

---
 tests/unit/ncdf_tests.F90 | 39 +++++++++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
index 382e22cafaca..ace9c0b10196 100644
--- a/tests/unit/ncdf_tests.F90
+++ b/tests/unit/ncdf_tests.F90
@@ -393,22 +393,37 @@ Subroutine test_nc4(test_id, err_msg)
     ret_val = PIO_inq_vardeflate(pio_file, pio_var, shuffle, deflate, deflate_level)
 
     ! Should not have worked except for netCDF-4/HDF5 serial.
-    if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
-       err_msg = "Could not turn on compression for variable foo2222"
-       call PIO_closefile(pio_file)
-       return
-    else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
-       err_msg = "Did not get expected error when trying to turn deflate on for non-netcdf-4 file"
+    if (iotype .eq. PIO_iotype_netcdf4c) then
+       if (ret_val .ne. PIO_NOERR) then
+          err_msg = "Got error trying to inquire about deflate on for serial netcdf-4 file"
+          call PIO_closefile(pio_file)
+          return
+       else
+          if (shuffle .ne. 0 .or. deflate .ne. 1 .or. deflate_level .ne. 4) then
+             err_msg = "Wrong values for deflate and shuffle for serial netcdf-4 file"
+             call PIO_closefile(pio_file)
+             return
+          end if
+       end if
+       err_msg = "Could not turn inquire about compression for variable foo2222"
        call PIO_closefile(pio_file)
        return
-    else if (iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
-       err_msg = "Did not get expected error when trying to turn deflate on for non-netcdf-4 file"
-       call PIO_closefile(pio_file)
-       return
-    else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .ne. PIO_NOERR) then
-       err_msg = "Did not get expected error when trying to turn deflate on for parallel netcdf-4 file"
+    else if (iotype .eq. PIO_iotype_pnetcdf .or. iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to check deflate for non-netcdf-4 file"
        call PIO_closefile(pio_file)
        return
+    else if (iotype .eq. PIO_iotype_netcdf4p) then
+       if (ret_val .ne. PIO_NOERR) then
+          err_msg = "Got error trying to inquire about deflate on for parallel netcdf-4 file"
+          call PIO_closefile(pio_file)
+          return
+       else
+          if (shuffle .ne. 0 .or. deflate .ne. 0) then
+             err_msg = "Wrong values for deflate and shuffle for parallel netcdf-4 file"
+             call PIO_closefile(pio_file)
+             return
+          end if
+       end if
     end if
 
     ! Write foo2

From 0056acea5ff04f4fefe51d534ca09fa9ff902987 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Thu, 11 Feb 2016 12:22:21 -0500
Subject: [PATCH 24/48] fixed name of inq_var_deflate in fortran API

---
 src/flib/pio.F90          |  2 +-
 src/flib/pio_nf.F90       | 34 +++++++++++++++++-----------------
 tests/unit/ncdf_tests.F90 |  2 +-
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/flib/pio.F90 b/src/flib/pio.F90
index d71297a33944..1941ffb20f8b 100644
--- a/src/flib/pio.F90
+++ b/src/flib/pio.F90
@@ -44,7 +44,7 @@ module pio
        PIO_inq_varndims ,&
        PIO_inq_vardimid ,&
        PIO_inq_varnatts ,&
-       PIO_inq_vardeflate ,&
+       PIO_inq_var_deflate ,&
        PIO_inq_dimid ,      &
        PIO_inq_dimname ,  &
        PIO_inq_dimlen ,    &
diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90
index 6bc558198ef5..fff48d783354 100644
--- a/src/flib/pio_nf.F90
+++ b/src/flib/pio_nf.F90
@@ -22,7 +22,7 @@ module pio_nf
        pio_inq_varndims                                     ,   &
        pio_inq_vardimid                                     ,   &
        pio_inq_varnatts                                     ,   &
-       pio_inq_vardeflate                                   ,    &       
+       pio_inq_var_deflate                                   ,    &       
        pio_inquire_variable                                 , &
        pio_inquire_dimension                                , &
        pio_inq_dimname                                      , &
@@ -104,11 +104,11 @@ module pio_nf
           inq_varnatts_vid                                  ,    &
           inq_varnatts_id
   end interface
-  interface pio_inq_vardeflate
+  interface pio_inq_var_deflate
      module procedure &
-          inq_vardeflate_desc                                 , &
-          inq_vardeflate_vid                                  , &
-          inq_vardeflate_id
+          inq_var_deflate_desc                                 , &
+          inq_var_deflate_vid                                  , &
+          inq_var_deflate_id
   end interface
   interface pio_inquire_dimension
      module procedure &
@@ -1012,11 +1012,11 @@ end function PIOc_inq_varnatts
   end function inq_varnatts_id
 
 !>
-!!  @defgroup PIO_inq_vardeflate PIO_inq_vardeflate
+!!  @defgroup PIO_inq_var_deflate PIO_inq_var_deflate
 !<
 !>
 !! @public 
-!! @ingroup PIO_inq_vardeflate
+!! @ingroup PIO_inq_var_deflate
 !! @brief Gets metadata information for netcdf file.
 !! @details
 !! @param File @copydoc file_desc_t
@@ -1024,7 +1024,7 @@ end function inq_varnatts_id
 !! @param type : The type of variable
 !! @retval ierr @copydoc error_return
 !<
-  integer function inq_vardeflate_desc(File, vardesc, shuffle, deflate, &
+  integer function inq_var_deflate_desc(File, vardesc, shuffle, deflate, &
        deflate_level) result(ierr)
 
     type (File_desc_t), intent(in) :: File
@@ -1033,15 +1033,15 @@ integer function inq_vardeflate_desc(File, vardesc, shuffle, deflate, &
     integer, intent(out) :: deflate
     integer, intent(out) :: deflate_level
 
-    ierr = pio_inq_vardeflate(File%fh, vardesc%varid, shuffle, deflate, deflate_level)
-  end function inq_vardeflate_desc
+    ierr = pio_inq_var_deflate(File%fh, vardesc%varid, shuffle, deflate, deflate_level)
+  end function inq_var_deflate_desc
 
 !>
 !! @public 
-!! @ingroup PIO_inq_vardeflate
+!! @ingroup PIO_inq_var_deflate
 !! @brief Gets metadata information for netcdf file.
 !<
-  integer function inq_vardeflate_vid(File, varid, shuffle, deflate, deflate_level) result(ierr)
+  integer function inq_var_deflate_vid(File, varid, shuffle, deflate, deflate_level) result(ierr)
 
     type (File_desc_t), intent(in) :: File
     integer, intent(in) :: varid
@@ -1049,14 +1049,14 @@ integer function inq_vardeflate_vid(File, varid, shuffle, deflate, deflate_level
     integer, intent(out) :: deflate
     integer, intent(out) :: deflate_level
 
-    ierr = pio_inq_vardeflate(File%fh, varid, shuffle, deflate, deflate_level)
-  end function inq_vardeflate_vid
+    ierr = pio_inq_var_deflate(File%fh, varid, shuffle, deflate, deflate_level)
+  end function inq_var_deflate_vid
 !>
 !! @public 
-!! @ingroup PIO_inq_vardeflate
+!! @ingroup PIO_inq_var_deflate
 !! @brief Gets metadata information for netcdf file.
 !<
-  integer function inq_vardeflate_id(ncid, varid, shuffle, deflate, &
+  integer function inq_var_deflate_id(ncid, varid, shuffle, deflate, &
        deflate_level) result(ierr)
     integer, intent(in) :: ncid
     integer, intent(in) :: varid
@@ -1077,7 +1077,7 @@ end function PIOc_inq_var_deflate
     end interface
 
     ierr = PIOc_inq_var_deflate(ncid, varid-1, shuffle, deflate, deflate_level)
-  end function inq_vardeflate_id
+  end function inq_var_deflate_id
   
 !>
 !! @defgroup PIO_inq_varname
diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
index ace9c0b10196..7312e3008281 100644
--- a/tests/unit/ncdf_tests.F90
+++ b/tests/unit/ncdf_tests.F90
@@ -390,7 +390,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Check the compression settings of the variables.
-    ret_val = PIO_inq_vardeflate(pio_file, pio_var, shuffle, deflate, deflate_level)
+    ret_val = PIO_inq_var_deflate(pio_file, pio_var, shuffle, deflate, deflate_level)
 
     ! Should not have worked except for netCDF-4/HDF5 serial.
     if (iotype .eq. PIO_iotype_netcdf4c) then

From c70d96e677e662f33b1aa51f2a0fb248286bbbe0 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Thu, 11 Feb 2016 12:55:30 -0500
Subject: [PATCH 25/48] added inq_szip fortran function

---
 src/flib/pio.F90    |  1 +
 src/flib/pio_nf.F90 | 69 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)

diff --git a/src/flib/pio.F90 b/src/flib/pio.F90
index 1941ffb20f8b..18f80c14d72c 100644
--- a/src/flib/pio.F90
+++ b/src/flib/pio.F90
@@ -45,6 +45,7 @@ module pio
        PIO_inq_vardimid ,&
        PIO_inq_varnatts ,&
        PIO_inq_var_deflate ,&
+       PIO_inq_var_szip ,&
        PIO_inq_dimid ,      &
        PIO_inq_dimname ,  &
        PIO_inq_dimlen ,    &
diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90
index fff48d783354..fbf277a0e638 100644
--- a/src/flib/pio_nf.F90
+++ b/src/flib/pio_nf.F90
@@ -23,6 +23,7 @@ module pio_nf
        pio_inq_vardimid                                     ,   &
        pio_inq_varnatts                                     ,   &
        pio_inq_var_deflate                                   ,    &       
+       pio_inq_var_szip                                   ,    &       
        pio_inquire_variable                                 , &
        pio_inquire_dimension                                , &
        pio_inq_dimname                                      , &
@@ -110,6 +111,12 @@ module pio_nf
           inq_var_deflate_vid                                  , &
           inq_var_deflate_id
   end interface
+  interface pio_inq_var_szip
+     module procedure &
+          inq_var_szip_desc                                 , &
+          inq_var_szip_vid                                  , &
+          inq_var_szip_id
+  end interface
   interface pio_inquire_dimension
      module procedure &
           inquire_dimension_desc                            , &
@@ -1079,6 +1086,68 @@ end function PIOc_inq_var_deflate
     ierr = PIOc_inq_var_deflate(ncid, varid-1, shuffle, deflate, deflate_level)
   end function inq_var_deflate_id
   
+!>
+!!  @defgroup PIO_inq_var PIO_inq_var_szip
+!<
+!>
+!! @public 
+!! @ingroup PIO_inq_var_szip
+!! @brief Gets metadata information for netcdf file.
+!! @details
+!! @param File @copydoc file_desc_t
+!! @param vardesc @copydoc var_desc_t
+!! @param type : The type of variable
+!! @retval ierr @copydoc error_return
+!<
+  integer function inq_var_szip_desc(File, vardesc, options_mask, pixels_per_block) result(ierr)
+
+    type (File_desc_t), intent(in) :: File
+    type (Var_desc_t), intent(in) :: vardesc
+    integer, intent(out) :: options_mask
+    integer, intent(out) :: pixels_per_block
+
+    ierr = pio_inq_var_szip(File%fh, vardesc%varid, options_mask, pixels_per_block)
+  end function inq_var_szip_desc
+
+!>
+!! @public 
+!! @ingroup PIO_inq_var_szip
+!! @brief Gets metadata information for netcdf file.
+!<
+  integer function inq_var_szip_vid(File, varid, options_mask, pixels_per_block) result(ierr)
+
+    type (File_desc_t), intent(in) :: File
+    integer, intent(in) :: varid
+    integer, intent(out) :: options_mask
+    integer, intent(out) :: pixels_per_block
+
+    ierr = pio_inq_var_szip(File%fh, varid, options_mask, pixels_per_block)
+  end function inq_var_szip_vid
+!>
+!! @public 
+!! @ingroup PIO_inq_var_szip
+!! @brief Gets metadata information for netcdf file.
+!<
+  integer function inq_var_szip_id(ncid, varid, options_mask, pixels_per_block) result(ierr)
+    integer, intent(in) :: ncid
+    integer, intent(in) :: varid
+    integer, intent(out) :: options_mask
+    integer, intent(out) :: pixels_per_block
+
+    interface
+       integer(C_INT) function PIOc_inq_var_szip(ncid, varid, options_mask, pixels_per_block) &
+            bind(C, name="PIOc_inq_var_szip")
+         use iso_c_binding
+         integer(C_INT), value :: ncid
+         integer(C_INT), value :: varid
+         integer(C_INT) :: options_mask
+         integer(C_INT) :: pixels_per_block
+       end function PIOc_inq_var_szip
+    end interface
+
+    ierr = PIOc_inq_var_szip(ncid, varid-1, options_mask, pixels_per_block)
+  end function inq_var_szip_id
+  
 !>
 !! @defgroup PIO_inq_varname
 !<

From 93c837520fef5165f20f5746131fda441412cbd3 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 16 Feb 2016 16:25:31 -0500
Subject: [PATCH 26/48] fixed bug in inq_var_deflate, got fortran version and
 tests working

---
 src/clib/pio_nc4.c        | 117 ++++------------------------------
 src/flib/pio.F90          |   1 -
 src/flib/pio_nf.F90       | 129 +++++++++++++-------------------------
 tests/unit/driver.F90     |   4 ++
 tests/unit/ncdf_tests.F90 |  80 ++++++++++++-----------
 tests/unit/test_nc4.c     |   8 ++-
 6 files changed, 103 insertions(+), 236 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index 4bd7471ed12f..894dd6ddbdea 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -168,12 +168,12 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
-	    return PIO_ENOTNC4;
+	    ierr = PIO_ENOTNC4;
 	    break;
 #endif
 #ifdef _PNETCDF
 	case PIO_IOTYPE_PNETCDF:
-	    return PIO_ENOTNC4;
+	    ierr = PIO_ENOTNC4;
 	    break;
 #endif
 	default:
@@ -189,117 +189,24 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
     }
 
     /* Check the netCDF return code, and broadcast it to all tasks. */
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
+    printf("about to check_netcdf ierr = %d errstr = %s\n", ierr, errstr);
+    ierr = check_netcdf(file, ierr, errstr, __LINE__);
+    printf("after check_netcdf ierr = %d errstr = %s\n", ierr, errstr);
 
     /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
     /* Broadcast results to all tasks. */
-    if (shufflep)
-	ierr = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm);
-    if (deflatep)
-	ierr = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm);
-    if (deflate_levelp)
-	ierr = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm);
-    return ierr;
-}    
-
-/**
- * @ingroup PIO_inq_var
- * Inquire about szip settings for a variable.
- *
- * This function only applies to netCDF-4 files. When used with netCDF
- * classic files, the error PIO_ENOTNC4 will be returned.
- *
- * Szip is a read-only compression format in netCDF-4. Only raw HDF5
- * can create szip files, but netcdf-4 can read them.
- *
- * See the netCDF
- * variable documentation for details about the operation of this
- * function.
- * 
- * @param ncid the ncid of the open file.
- * @param varid the ID of the variable to set chunksizes for.
- * @param options_maskp will get the options mask.
- * @param pixels_per_block will get the pixels per block.
- * 
- * @return PIO_NOERR for success, otherwise an error code.
- */
-int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp,
-		      int *pixels_per_blockp)
-{
-    int ierr;
-    int msg;
-    int mpierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    char *errstr;
-
-    errstr = NULL;
-    ierr = PIO_NOERR;
-
-    if (!(file = pio_get_file_from_id(ncid)))
-	return PIO_EBADID;
-    ios = file->iosystem;
-    msg = PIO_MSG_INQ_VAR_SZIP;
-
-    if (ios->async_interface && ! ios->ioproc)
-    {
-	if (ios->compmaster) 
-	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
-	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
-    }
-
-    if (ios->ioproc)
+    if (ierr == PIO_NOERR)
     {
-	switch (file->iotype)
-	{
-#ifdef _NETCDF
-#ifdef _NETCDF4
-	case PIO_IOTYPE_NETCDF4P:
-	    ierr = nc_inq_var_szip(file->fh, varid, options_maskp, pixels_per_blockp);
-	    break;
-	case PIO_IOTYPE_NETCDF4C:
-	    if (!ios->io_rank)
-		ierr = nc_inq_var_szip(file->fh, varid, options_maskp, pixels_per_blockp);
-	    break;
-#endif
-	case PIO_IOTYPE_NETCDF:
-	    return PIO_ENOTNC4;
-	    break;
-#endif
-#ifdef _PNETCDF
-	case PIO_IOTYPE_PNETCDF:
-	    return PIO_ENOTNC4;
-	    break;
-#endif
-	default:
-	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
-	}
+	if (shufflep)
+	    ierr = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm);
+	if (deflatep)
+	    ierr = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm);
+	if (deflate_levelp)
+	    ierr = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm);
     }
-
-    /* Allocate an error string if needed. */
-    if (ierr != PIO_NOERR)
-    {
-	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
-	sprintf(errstr,"in file %s",__FILE__);
-    }
-
-    /* Check for netCDF error. */
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-
-    /* Free the error string if it was allocated. */
-    if (errstr != NULL)
-	free(errstr);
-
-    /* Broadcast results to all tasks. */
-    if (options_maskp)
-	ierr = MPI_Bcast(options_maskp, 1, MPI_INT, ios->ioroot, ios->my_comm);
-    if (pixels_per_blockp)
-	ierr = MPI_Bcast(pixels_per_blockp, 1, MPI_INT, ios->ioroot, ios->my_comm);
-	
     return ierr;
 }    
 
diff --git a/src/flib/pio.F90 b/src/flib/pio.F90
index 18f80c14d72c..1941ffb20f8b 100644
--- a/src/flib/pio.F90
+++ b/src/flib/pio.F90
@@ -45,7 +45,6 @@ module pio
        PIO_inq_vardimid ,&
        PIO_inq_varnatts ,&
        PIO_inq_var_deflate ,&
-       PIO_inq_var_szip ,&
        PIO_inq_dimid ,      &
        PIO_inq_dimname ,  &
        PIO_inq_dimlen ,    &
diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90
index fbf277a0e638..01796f2ced41 100644
--- a/src/flib/pio_nf.F90
+++ b/src/flib/pio_nf.F90
@@ -10,26 +10,26 @@ module pio_nf
   private
 
   public :: &
-       pio_def_var                                          ,   &
-       pio_def_var_deflate                                          ,   &
-       pio_def_dim                                          ,  &
-       pio_inq_attname                                      ,    & 
-       pio_inq_att                                          ,        &
-       pio_inq_attlen                                       ,     &
-       pio_inq_varid                                        ,      &
-       pio_inq_varname                                      ,    &
-       pio_inq_vartype                                      ,    &
-       pio_inq_varndims                                     ,   &
-       pio_inq_vardimid                                     ,   &
-       pio_inq_varnatts                                     ,   &
-       pio_inq_var_deflate                                   ,    &       
-       pio_inq_var_szip                                   ,    &       
+       pio_def_var                                          , &
+       pio_def_var_deflate                                  , &
+       pio_def_var_chunking                                 , &
+       pio_def_dim                                          , &
+       pio_inq_attname                                      , & 
+       pio_inq_att                                          , &
+       pio_inq_attlen                                       , &
+       pio_inq_varid                                        , &
+       pio_inq_varname                                      , &
+       pio_inq_vartype                                      , &
+       pio_inq_varndims                                     , &
+       pio_inq_vardimid                                     , &
+       pio_inq_varnatts                                     , &
+       pio_inq_var_deflate                                  , &       
        pio_inquire_variable                                 , &
        pio_inquire_dimension                                , &
        pio_inq_dimname                                      , &
        pio_inq_dimlen                                       , &
        pio_inq_dimid                                        , &
-       pio_inq_unlimdim                                , &
+       pio_inq_unlimdim                                     , &
        pio_inquire                                          , &
        pio_enddef                                           , &
        pio_redef
@@ -46,9 +46,10 @@ module pio_nf
      module procedure &
           def_var_deflate
   end interface
-  ! interface pio_def_var_deflate
-  !    module procedure def_var_deflate
-  ! end interface
+  interface pio_def_var_chunking
+     module procedure &
+          def_var_chunking
+  end interface
   interface pio_inq_attname
      module procedure &
           inq_attname_desc                                  , &
@@ -111,12 +112,6 @@ module pio_nf
           inq_var_deflate_vid                                  , &
           inq_var_deflate_id
   end interface
-  interface pio_inq_var_szip
-     module procedure &
-          inq_var_szip_desc                                 , &
-          inq_var_szip_vid                                  , &
-          inq_var_szip_id
-  end interface
   interface pio_inquire_dimension
      module procedure &
           inquire_dimension_desc                            , &
@@ -1086,68 +1081,6 @@ end function PIOc_inq_var_deflate
     ierr = PIOc_inq_var_deflate(ncid, varid-1, shuffle, deflate, deflate_level)
   end function inq_var_deflate_id
   
-!>
-!!  @defgroup PIO_inq_var PIO_inq_var_szip
-!<
-!>
-!! @public 
-!! @ingroup PIO_inq_var_szip
-!! @brief Gets metadata information for netcdf file.
-!! @details
-!! @param File @copydoc file_desc_t
-!! @param vardesc @copydoc var_desc_t
-!! @param type : The type of variable
-!! @retval ierr @copydoc error_return
-!<
-  integer function inq_var_szip_desc(File, vardesc, options_mask, pixels_per_block) result(ierr)
-
-    type (File_desc_t), intent(in) :: File
-    type (Var_desc_t), intent(in) :: vardesc
-    integer, intent(out) :: options_mask
-    integer, intent(out) :: pixels_per_block
-
-    ierr = pio_inq_var_szip(File%fh, vardesc%varid, options_mask, pixels_per_block)
-  end function inq_var_szip_desc
-
-!>
-!! @public 
-!! @ingroup PIO_inq_var_szip
-!! @brief Gets metadata information for netcdf file.
-!<
-  integer function inq_var_szip_vid(File, varid, options_mask, pixels_per_block) result(ierr)
-
-    type (File_desc_t), intent(in) :: File
-    integer, intent(in) :: varid
-    integer, intent(out) :: options_mask
-    integer, intent(out) :: pixels_per_block
-
-    ierr = pio_inq_var_szip(File%fh, varid, options_mask, pixels_per_block)
-  end function inq_var_szip_vid
-!>
-!! @public 
-!! @ingroup PIO_inq_var_szip
-!! @brief Gets metadata information for netcdf file.
-!<
-  integer function inq_var_szip_id(ncid, varid, options_mask, pixels_per_block) result(ierr)
-    integer, intent(in) :: ncid
-    integer, intent(in) :: varid
-    integer, intent(out) :: options_mask
-    integer, intent(out) :: pixels_per_block
-
-    interface
-       integer(C_INT) function PIOc_inq_var_szip(ncid, varid, options_mask, pixels_per_block) &
-            bind(C, name="PIOc_inq_var_szip")
-         use iso_c_binding
-         integer(C_INT), value :: ncid
-         integer(C_INT), value :: varid
-         integer(C_INT) :: options_mask
-         integer(C_INT) :: pixels_per_block
-       end function PIOc_inq_var_szip
-    end interface
-
-    ierr = PIOc_inq_var_szip(ncid, varid-1, options_mask, pixels_per_block)
-  end function inq_var_szip_id
-  
 !>
 !! @defgroup PIO_inq_varname
 !<
@@ -1592,4 +1525,28 @@ end function PIOc_def_var_deflate
     ierr = PIOc_def_var_deflate(file%fh, vardesc%varid-1, shuffle, deflate, deflate_level)
   end function def_var_deflate
 
+!> 
+!! @public 
+!! @ingroup PIO_def_var_chunking
+!! @brief Changes chunking settings for a netCDF-4/HDF5 variable.
+!<
+  integer function def_var_chunking(file, vardesc, storage, chunksizes) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    type (var_desc_t), intent(in) :: vardesc
+    integer, intent(in) :: storage
+    integer, intent(in) :: chunksizes
+
+    interface
+       integer (C_INT) function PIOc_def_var_chunking(ncid, varid, storage, chunksizes) &
+            bind(c,name="PIOc_def_var_chunking")
+         use iso_c_binding
+         integer(c_int), value :: ncid
+         integer(c_int), value :: varid
+         integer(c_int), value :: storage
+         integer(c_int), value :: chunksizes
+       end function PIOc_def_var_chunking
+    end interface
+
+    ierr = PIOc_def_var_chunking(file%fh, vardesc%varid-1, storage, chunksizes)
+  end function def_var_chunking
 end module pio_nf
diff --git a/tests/unit/driver.F90 b/tests/unit/driver.F90
index 6622bd6bf126..87d8abc06201 100644
--- a/tests/unit/driver.F90
+++ b/tests/unit/driver.F90
@@ -167,6 +167,10 @@ Program pio_unit_test_driver
            if (master_task) write(*,"(3x,A,1x)", advance="no") "testing PIO_enddef..."
            call test_enddef(test_id, err_msg)
            call parse(err_msg, fail_cnt)
+
+           if (master_task) write(*,"(3x,A,1x)", advance="no") "testing PIO_enddef..."
+           call test_nc4(test_id, err_msg)
+           call parse(err_msg, fail_cnt)
         end if
 
 
diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
index 7312e3008281..ac95ade39514 100644
--- a/tests/unit/ncdf_tests.F90
+++ b/tests/unit/ncdf_tests.F90
@@ -280,6 +280,7 @@ Subroutine test_nc4(test_id, err_msg)
     integer :: shuffle
     integer :: deflate
     integer :: deflate_level
+    integer :: dim_len
 
     shuffle = 0
     deflate = 1
@@ -316,8 +317,9 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Define a new dimension M1.
-    print *, 'calling PIO_def_dim'
-    ret_val = PIO_def_dim(pio_file, 'M111222', int(2*ntasks,pio_offset_kind), pio_dim)
+    dim_len = int(2*ntasks,pio_offset_kind)
+    print *, 'calling PIO_def_dim, dim_len = ', dim_len
+    ret_val = PIO_def_dim(pio_file, 'M111222', dim_len, pio_dim)
     if (ret_val .ne. PIO_NOERR) then
        err_msg = "Could not define dimension M111222"
        print *, ret_val
@@ -326,11 +328,9 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Define a new variable
-    print *, 'calling PIO_def_va'
-    ret_val = PIO_def_var(pio_file, 'foo2222', PIO_int, &
-         (/pio_dim/), pio_var)
+    print *, 'calling PIO_def_var'
+    ret_val = PIO_def_var(pio_file, 'foo2222', PIO_int, (/pio_dim/), pio_var)
     if (ret_val .ne. PIO_NOERR) then
-       ! Error in PIO_def_var
        err_msg = "Could not define variable foo2222"
        call PIO_closefile(pio_file)
        return
@@ -383,51 +383,48 @@ Subroutine test_nc4(test_id, err_msg)
     print *, 'calling PIO_enddef'
     ret_val = PIO_enddef(pio_file)
     if (ret_val .ne. PIO_NOERR) then
-       ! Error in PIO_enddef
        print *,__FILE__,__LINE__,ret_val
        err_msg = "Could not end define mode"
        return
     end if
 
     ! Check the compression settings of the variables.
+    print *, 'calling PIO_inq_var_deflate'    
     ret_val = PIO_inq_var_deflate(pio_file, pio_var, shuffle, deflate, deflate_level)
 
-    ! Should not have worked except for netCDF-4/HDF5 serial.
-    if (iotype .eq. PIO_iotype_netcdf4c) then
-       if (ret_val .ne. PIO_NOERR) then
-          err_msg = "Got error trying to inquire about deflate on for serial netcdf-4 file"
-          call PIO_closefile(pio_file)
-          return
-       else
-          if (shuffle .ne. 0 .or. deflate .ne. 1 .or. deflate_level .ne. 4) then
-             err_msg = "Wrong values for deflate and shuffle for serial netcdf-4 file"
-             call PIO_closefile(pio_file)
-             return
-          end if
-       end if
-       err_msg = "Could not turn inquire about compression for variable foo2222"
-       call PIO_closefile(pio_file)
-       return
-    else if (iotype .eq. PIO_iotype_pnetcdf .or. iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
-       err_msg = "Did not get expected error when trying to check deflate for non-netcdf-4 file"
-       call PIO_closefile(pio_file)
-       return
-    else if (iotype .eq. PIO_iotype_netcdf4p) then
-       if (ret_val .ne. PIO_NOERR) then
-          err_msg = "Got error trying to inquire about deflate on for parallel netcdf-4 file"
-          call PIO_closefile(pio_file)
-          return
-       else
-          if (shuffle .ne. 0 .or. deflate .ne. 0) then
-             err_msg = "Wrong values for deflate and shuffle for parallel netcdf-4 file"
-             call PIO_closefile(pio_file)
-             return
-          end if
-       end if
-    end if
+    ! ! Should not have worked except for netCDF-4/HDF5 serial.
+    ! if (iotype .eq. PIO_iotype_netcdf4c) then
+    !    if (ret_val .ne. PIO_NOERR) then
+    !       err_msg = "Got error trying to inquire about deflate on for serial netcdf-4 file"
+    !       call PIO_closefile(pio_file)
+    !       return
+    !    else
+    !       if (shuffle .ne. 0 .or. deflate .ne. 1 .or. deflate_level .ne. 4) then
+    !          err_msg = "Wrong values for deflate and shuffle for serial netcdf-4 file"
+    !          call PIO_closefile(pio_file)
+    !          return
+    !       end if
+    !    end if
+    ! else if ((iotype .eq. PIO_iotype_pnetcdf .or. iotype .eq. PIO_iotype_netcdf) .and. ret_val .eq. PIO_NOERR) then
+    !    err_msg = "Did not get expected error when trying to check deflate for non-netcdf-4 file"
+    !    call PIO_closefile(pio_file)
+    !    return
+    ! else if (iotype .eq. PIO_iotype_netcdf4p) then
+    !    if (ret_val .ne. PIO_NOERR) then
+    !       err_msg = "Got error trying to inquire about deflate on for parallel netcdf-4 file"
+    !       call PIO_closefile(pio_file)
+    !       return
+    !    else
+    !       if (shuffle .ne. 0 .or. deflate .ne. 0) then
+    !          err_msg = "Wrong values for deflate and shuffle for parallel netcdf-4 file"
+    !          call PIO_closefile(pio_file)
+    !          return
+    !       end if
+    !    end if
+    ! end if
 
     ! Write foo2
-    print *, 'calling PIO_write_darray'
+    print *, 'calling PIO_write_darray, data_to_write = ', data_to_write, 'iodesc_nCells = ', iodesc_nCells
     call PIO_write_darray(pio_file, pio_var, iodesc_nCells, data_to_write, ret_val)
     if (ret_val .ne. PIO_NOERR) then
        ! Error in PIO_write_darray
@@ -444,5 +441,6 @@ Subroutine test_nc4(test_id, err_msg)
     call PIO_freedecomp(pio_iosystem, iodesc_nCells)
     call mpi_barrier(MPI_COMM_WORLD,ret_val)
     
+    print *, 'err_msg = ', err_msg
   End Subroutine test_nc4
 end module ncdf_tests
diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index 59a0be94cc7b..dc7b0e2bc517 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -288,6 +288,9 @@ main(int argc, char **argv)
 	if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt],
 				   PIO_CLOBBER)))
 	    ERR(ret);
+
+	/* Set error handling. */
+	PIOc_Set_File_Error_Handling(ncid, PIO_RETURN_ERROR);
 	
 	/* Define netCDF dimensions and variable. */
 	if (verbose)
@@ -346,9 +349,8 @@ main(int argc, char **argv)
 	    if ((ret = PIOc_inq_var_deflate(ncid, 0, &shuffle, &deflate, &deflate_level)))
 	    	ERR(ret);
 
-	    /** For serial netCDF-4, only processor rank 0 gets the
-	     * answers. Also deflate is turned on by default */
-	    if (format[fmt] == PIO_IOTYPE_NETCDF4C && !my_rank)
+	    /** For serial netCDF-4 deflate is turned on by default */
+	    if (format[fmt] == PIO_IOTYPE_NETCDF4C)
 		if (shuffle || !deflate || deflate_level != 1)
 		    ERR(ERR_AWFUL);
 

From 929a0911e6d08af504b432076543c6712af9fabd Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 16 Feb 2016 16:32:08 -0500
Subject: [PATCH 27/48] fixed test output

---
 tests/unit/driver.F90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/driver.F90 b/tests/unit/driver.F90
index 87d8abc06201..991946c9cafb 100644
--- a/tests/unit/driver.F90
+++ b/tests/unit/driver.F90
@@ -168,7 +168,7 @@ Program pio_unit_test_driver
            call test_enddef(test_id, err_msg)
            call parse(err_msg, fail_cnt)
 
-           if (master_task) write(*,"(3x,A,1x)", advance="no") "testing PIO_enddef..."
+           if (master_task) write(*,"(3x,A,1x)", advance="no") "testing PIO netCDF-4 functions..."
            call test_nc4(test_id, err_msg)
            call parse(err_msg, fail_cnt)
         end if

From 9410243f9f6f2dee13a6b8b5098d725a2cccac39 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 16 Feb 2016 16:40:05 -0500
Subject: [PATCH 28/48] test cleanup

---
 tests/unit/ncdf_tests.F90 | 82 +++++++++++++++------------------------
 1 file changed, 31 insertions(+), 51 deletions(-)

diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
index ac95ade39514..4dc2abcbe463 100644
--- a/tests/unit/ncdf_tests.F90
+++ b/tests/unit/ncdf_tests.F90
@@ -280,7 +280,6 @@ Subroutine test_nc4(test_id, err_msg)
     integer :: shuffle
     integer :: deflate
     integer :: deflate_level
-    integer :: dim_len
 
     shuffle = 0
     deflate = 1
@@ -291,14 +290,12 @@ Subroutine test_nc4(test_id, err_msg)
     compdof = 2*my_rank+(/1,2/)  ! Where in the global array each task writes
     data_to_write = 1+my_rank
 
-    print *, 'calling PIO_initdecomp'
     call PIO_initdecomp(pio_iosystem, PIO_int, dims, compdof, iodesc_nCells)
 
     filename = fnames(test_id)
     iotype   = iotypes(test_id)
 
     ! Open existing file, write data to it
-    print *, 'calling PIO_openfile'
     ret_val = PIO_openfile(pio_iosystem, pio_file, iotype, filename, PIO_write)
     if (ret_val .ne. PIO_NOERR) then
        ! Error in PIO_openfile
@@ -307,7 +304,6 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Enter define mode
-    print *, 'calling PIO_redef'
     ret_val = PIO_redef(pio_file)
     if (ret_val .ne. PIO_NOERR) then
        ! Error in PIO_redef
@@ -317,18 +313,14 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Define a new dimension M1.
-    dim_len = int(2*ntasks,pio_offset_kind)
-    print *, 'calling PIO_def_dim, dim_len = ', dim_len
-    ret_val = PIO_def_dim(pio_file, 'M111222', dim_len, pio_dim)
+    ret_val = PIO_def_dim(pio_file, 'M111222', int(2*ntasks,pio_offset_kind), pio_dim)
     if (ret_val .ne. PIO_NOERR) then
        err_msg = "Could not define dimension M111222"
-       print *, ret_val
        call PIO_closefile(pio_file)
        return
     end if
 
     ! Define a new variable
-    print *, 'calling PIO_def_var'
     ret_val = PIO_def_var(pio_file, 'foo2222', PIO_int, (/pio_dim/), pio_var)
     if (ret_val .ne. PIO_NOERR) then
        err_msg = "Could not define variable foo2222"
@@ -337,10 +329,8 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Try to turn on compression for this variable.
-    print *, 'calling PIO_def_var_deflate ', pio_file%fh, pio_var%varid, shuffle, deflate, deflate_level
     ret_val = PIO_def_var_deflate(pio_file, pio_var, shuffle, deflate, &
          deflate_level)
-    print *, 'PIO_def_var_deflate returned ', ret_val
 
     ! Should not have worked except for netCDF-4/HDF5 serial.
     if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
@@ -361,7 +351,6 @@ Subroutine test_nc4(test_id, err_msg)
        return
     end if
 
-    print *, 'calling PIO_put_att'
     ret_val = PIO_put_att(pio_file, pio_var, "max_val", ntasks)
     if (ret_val .ne. PIO_NOERR) then
        ! Error in PIO_put_att
@@ -370,7 +359,6 @@ Subroutine test_nc4(test_id, err_msg)
        return
     end if
 
-    print *, 'calling PIO_put_att'
     ret_val = PIO_put_att(pio_file, PIO_global, "created_by", "PIO unit tests")
     if (ret_val .ne. PIO_NOERR) then
        ! Error in PIO_put_att
@@ -380,67 +368,59 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Leave define mode
-    print *, 'calling PIO_enddef'
     ret_val = PIO_enddef(pio_file)
     if (ret_val .ne. PIO_NOERR) then
-       print *,__FILE__,__LINE__,ret_val
        err_msg = "Could not end define mode"
        return
     end if
 
     ! Check the compression settings of the variables.
-    print *, 'calling PIO_inq_var_deflate'    
     ret_val = PIO_inq_var_deflate(pio_file, pio_var, shuffle, deflate, deflate_level)
 
-    ! ! Should not have worked except for netCDF-4/HDF5 serial.
-    ! if (iotype .eq. PIO_iotype_netcdf4c) then
-    !    if (ret_val .ne. PIO_NOERR) then
-    !       err_msg = "Got error trying to inquire about deflate on for serial netcdf-4 file"
-    !       call PIO_closefile(pio_file)
-    !       return
-    !    else
-    !       if (shuffle .ne. 0 .or. deflate .ne. 1 .or. deflate_level .ne. 4) then
-    !          err_msg = "Wrong values for deflate and shuffle for serial netcdf-4 file"
-    !          call PIO_closefile(pio_file)
-    !          return
-    !       end if
-    !    end if
-    ! else if ((iotype .eq. PIO_iotype_pnetcdf .or. iotype .eq. PIO_iotype_netcdf) .and. ret_val .eq. PIO_NOERR) then
-    !    err_msg = "Did not get expected error when trying to check deflate for non-netcdf-4 file"
-    !    call PIO_closefile(pio_file)
-    !    return
-    ! else if (iotype .eq. PIO_iotype_netcdf4p) then
-    !    if (ret_val .ne. PIO_NOERR) then
-    !       err_msg = "Got error trying to inquire about deflate on for parallel netcdf-4 file"
-    !       call PIO_closefile(pio_file)
-    !       return
-    !    else
-    !       if (shuffle .ne. 0 .or. deflate .ne. 0) then
-    !          err_msg = "Wrong values for deflate and shuffle for parallel netcdf-4 file"
-    !          call PIO_closefile(pio_file)
-    !          return
-    !       end if
-    !    end if
-    ! end if
+    ! Should not have worked except for netCDF-4/HDF5 serial.
+    if (iotype .eq. PIO_iotype_netcdf4c) then
+       if (ret_val .ne. PIO_NOERR) then
+          err_msg = "Got error trying to inquire about deflate on for serial netcdf-4 file"
+          call PIO_closefile(pio_file)
+          return
+       else
+          if (shuffle .ne. 0 .or. deflate .ne. 1 .or. deflate_level .ne. 4) then
+             err_msg = "Wrong values for deflate and shuffle for serial netcdf-4 file"
+             call PIO_closefile(pio_file)
+             return
+          end if
+       end if
+    else if ((iotype .eq. PIO_iotype_pnetcdf .or. iotype .eq. PIO_iotype_netcdf) .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to check deflate for non-netcdf-4 file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf4p) then
+       if (ret_val .ne. PIO_NOERR) then
+          err_msg = "Got error trying to inquire about deflate on for parallel netcdf-4 file"
+          call PIO_closefile(pio_file)
+          return
+       else
+          if (shuffle .ne. 0 .or. deflate .ne. 0) then
+             err_msg = "Wrong values for deflate and shuffle for parallel netcdf-4 file"
+             call PIO_closefile(pio_file)
+             return
+          end if
+       end if
+    end if
 
     ! Write foo2
-    print *, 'calling PIO_write_darray, data_to_write = ', data_to_write, 'iodesc_nCells = ', iodesc_nCells
     call PIO_write_darray(pio_file, pio_var, iodesc_nCells, data_to_write, ret_val)
     if (ret_val .ne. PIO_NOERR) then
-       ! Error in PIO_write_darray
        err_msg = "Could not write data"
        return
     end if
 
     ! Close file
-    print *, 'calling PIO_closefile'
     call PIO_closefile(pio_file)
 
     ! Free decomp
-    print *, 'calling PIO_freedecomp'
     call PIO_freedecomp(pio_iosystem, iodesc_nCells)
     call mpi_barrier(MPI_COMM_WORLD,ret_val)
     
-    print *, 'err_msg = ', err_msg
   End Subroutine test_nc4
 end module ncdf_tests

From 1d15d18e916ec3a8c31a3cf71db5e6fed85eb613 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Wed, 17 Feb 2016 11:02:55 -0500
Subject: [PATCH 29/48] fixed fletcher32 filter inq and documentation

---
 src/clib/pio_nc4.c    | 11 ++++-------
 tests/unit/test_nc4.c |  2 +-
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index 894dd6ddbdea..f20f2cfe5c76 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -309,7 +309,7 @@ int PIOc_def_var_fletcher32(int ncid, int varid, int fletcher32)
 
 /**
  * @ingroup PIO_inq_var
- * Inquire about chunksizes for a variable.
+ * Inquire about fletcher32 checksums for a variable.
  *
  * This function only applies to netCDF-4 files. When used with netCDF
  * classic files, the error PIO_ENOTNC4 will be returned.
@@ -320,12 +320,9 @@ int PIOc_def_var_fletcher32(int ncid, int varid, int fletcher32)
  * function.
  * 
  * @param ncid the ncid of the open file.
- * @param varid the ID of the variable to set chunksizes for.
- * @param storagep pointer to int which will be set to either
- * NC_CONTIGUOUS or NC_CHUNKED.
- * @param chunksizep pointer to memory where chunksizes will be
- * set. There are the same number of chunksizes as there are
- * dimensions.
+ * @param varid the ID of the variable.
+ * @param fletcher32p pointer will get zero if checksums are not in
+ * use, non-zero otherwise.
  * 
  * @return PIO_NOERR for success, otherwise an error code.
  */
diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index dc7b0e2bc517..9d9c0cfc663a 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -363,7 +363,7 @@ main(int argc, char **argv)
 	     * work. Starts as off. */
 	    if ((ret = PIOc_inq_var_fletcher32(ncid, 0, &fletcher32)))
 	    	ERR(ret);
-	    if (format[fmt] == PIO_IOTYPE_NETCDF4C && !my_rank)
+	    if (format[fmt] == PIO_IOTYPE_NETCDF4C)
 		if (fletcher32)
 		    ERR(ERR_AWFUL);
 	    if (format[fmt] == PIO_IOTYPE_NETCDF4P)

From 791d9d11d3b39abe292bcee5ca822b9d19fdaaf1 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Wed, 17 Feb 2016 11:04:00 -0500
Subject: [PATCH 30/48] fixed fletcher32 filter inq and documentation

---
 tests/unit/test_nc4.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index 9d9c0cfc663a..ce076ac5bc73 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -370,20 +370,20 @@ main(int argc, char **argv)
 		if (fletcher32)
 		    ERR(ERR_AWFUL);
 
-	    /* Turn on (the off) fletcher32 filter for netCDF-4 serial. */
+	    /* Turn on (then off) fletcher32 filter for netCDF-4 serial. */
 	    if (format[fmt] == PIO_IOTYPE_NETCDF4C)
 	    {
 		if ((ret = PIOc_def_var_fletcher32(ncid, 0, 1)))
 		    ERR(ret);
 		if ((ret = PIOc_inq_var_fletcher32(ncid, 0, &fletcher32)))
 		    ERR(ret);
-		if (!fletcher32 && !my_rank)
+		if (!fletcher32)
 		    ERR(ERR_AWFUL);
 		if ((ret = PIOc_def_var_fletcher32(ncid, 0, 0)))
 		    ERR(ret);
 		if ((ret = PIOc_inq_var_fletcher32(ncid, 0, &fletcher32)))
 		    ERR(ret);
-		if (fletcher32 && !my_rank)
+		if (fletcher32)
 		    ERR(ERR_AWFUL);
 	    }		
 	    

From d97911211430a831dc05ac809fde797b8e4464fa Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Wed, 17 Feb 2016 11:18:11 -0500
Subject: [PATCH 31/48] test cleanup

---
 tests/unit/test_nc4.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index ce076ac5bc73..068f60c74be5 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -331,10 +331,12 @@ main(int argc, char **argv)
 	    {
 		printf("rank: %d ret: %d storage: %d\n", my_rank, ret, storage);
 		for (d1 = 0; d1 < NDIM; d1++)
+		{
 		    printf("chunksize[%d]=%d\n", d1, my_chunksize[d1]);
+		}
 	    }
 	    
-	    /** For serial netCDF-4, only processor rank 0 gets the answers. */
+	    /** Check the answers. */
 	    if (format[fmt] == PIO_IOTYPE_NETCDF4C ||
 		format[fmt] == PIO_IOTYPE_NETCDF4P)
 	    {

From 0d1f692524136e01eb6d299fd191e7ca8fab4df4 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Thu, 18 Feb 2016 15:25:15 -0500
Subject: [PATCH 32/48] further testing, other variations of fortran functions

---
 src/clib/pio_nc4.c        |   2 -
 src/flib/pio_nf.F90       |  39 +++++++++++---
 tests/unit/driver.F90     |   3 ++
 tests/unit/ncdf_tests.F90 | 108 ++++++++++++++++++++++++++++++++++++--
 4 files changed, 137 insertions(+), 15 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index f20f2cfe5c76..28adf44e63c1 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -189,9 +189,7 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
     }
 
     /* Check the netCDF return code, and broadcast it to all tasks. */
-    printf("about to check_netcdf ierr = %d errstr = %s\n", ierr, errstr);
     ierr = check_netcdf(file, ierr, errstr, __LINE__);
-    printf("after check_netcdf ierr = %d errstr = %s\n", ierr, errstr);
 
     /* Free the error string if it was allocated. */
     if (errstr != NULL)
diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90
index 01796f2ced41..e30cc6a8e130 100644
--- a/src/flib/pio_nf.F90
+++ b/src/flib/pio_nf.F90
@@ -44,7 +44,8 @@ module pio_nf
   end interface
   interface pio_def_var_deflate
      module procedure &
-          def_var_deflate
+          def_var_deflate_desc                              , &         
+          def_var_deflate_id
   end interface
   interface pio_def_var_chunking
      module procedure &
@@ -1053,6 +1054,7 @@ integer function inq_var_deflate_vid(File, varid, shuffle, deflate, deflate_leve
 
     ierr = pio_inq_var_deflate(File%fh, varid, shuffle, deflate, deflate_level)
   end function inq_var_deflate_vid
+
 !>
 !! @public 
 !! @ingroup PIO_inq_var_deflate
@@ -1503,9 +1505,9 @@ end function def_var_md_id
 !! @ingroup PIO_def_var_deflate
 !! @brief Changes compression settings for a netCDF-4/HDF5 variable.
 !<
-  integer function def_var_deflate(file, vardesc, shuffle, deflate, deflate_level) result(ierr)
+  integer function def_var_deflate_id(file, varid, shuffle, deflate, deflate_level) result(ierr)
     type (File_desc_t), intent(in)  :: file
-    type (var_desc_t), intent(in) :: vardesc
+    integer, intent(in) :: varid
     integer, intent(in) :: shuffle
     integer, intent(in) :: deflate
     integer, intent(in) :: deflate_level
@@ -1522,8 +1524,23 @@ integer (C_INT) function PIOc_def_var_deflate(ncid, varid, shuffle, deflate, def
        end function PIOc_def_var_deflate
     end interface
 
-    ierr = PIOc_def_var_deflate(file%fh, vardesc%varid-1, shuffle, deflate, deflate_level)
-  end function def_var_deflate
+    ierr = PIOc_def_var_deflate(file%fh, varid-1, shuffle, deflate, deflate_level)
+  end function def_var_deflate_id
+
+!> 
+!! @public 
+!! @ingroup PIO_def_var_deflate
+!! @brief Changes compression settings for a netCDF-4/HDF5 variable.
+!<
+  integer function def_var_deflate_desc(file, vardesc, shuffle, deflate, deflate_level) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    type (var_desc_t), intent(in) :: vardesc
+    integer, intent(in) :: shuffle
+    integer, intent(in) :: deflate
+    integer, intent(in) :: deflate_level
+
+    ierr = def_var_deflate_id(file, vardesc%varid, shuffle, deflate, deflate_level)
+  end function def_var_deflate_desc
 
 !> 
 !! @public 
@@ -1534,7 +1551,9 @@ integer function def_var_chunking(file, vardesc, storage, chunksizes) result(ier
     type (File_desc_t), intent(in)  :: file
     type (var_desc_t), intent(in) :: vardesc
     integer, intent(in) :: storage
-    integer, intent(in) :: chunksizes
+    integer, intent(in) :: chunksizes(:)
+    integer(C_INT) :: cchunksizes(PIO_MAX_VAR_DIMS)
+    integer :: ndims, i    
 
     interface
        integer (C_INT) function PIOc_def_var_chunking(ncid, varid, storage, chunksizes) &
@@ -1543,10 +1562,14 @@ integer (C_INT) function PIOc_def_var_chunking(ncid, varid, storage, chunksizes)
          integer(c_int), value :: ncid
          integer(c_int), value :: varid
          integer(c_int), value :: storage
-         integer(c_int), value :: chunksizes
+         integer(c_int) :: chunksizes(*)
        end function PIOc_def_var_chunking
     end interface
+    ndims = size(chunksizes)
+    do i=1,ndims
+       cchunksizes(i) = chunksizes(ndims-i+1)-1
+    enddo
 
-    ierr = PIOc_def_var_chunking(file%fh, vardesc%varid-1, storage, chunksizes)
+    ierr = PIOc_def_var_chunking(file%fh, vardesc%varid-1, storage, cchunksizes)
   end function def_var_chunking
 end module pio_nf
diff --git a/tests/unit/driver.F90 b/tests/unit/driver.F90
index 991946c9cafb..7dbcfada7eda 100644
--- a/tests/unit/driver.F90
+++ b/tests/unit/driver.F90
@@ -166,10 +166,13 @@ Program pio_unit_test_driver
 
            if (master_task) write(*,"(3x,A,1x)", advance="no") "testing PIO_enddef..."
            call test_enddef(test_id, err_msg)
+           print *, 'err_msg =', err_msg, ' fail_cnt = ', fail_cnt
            call parse(err_msg, fail_cnt)
 
            if (master_task) write(*,"(3x,A,1x)", advance="no") "testing PIO netCDF-4 functions..."
+           print *, 'err_msg =', err_msg, ' fail_cnt = ', fail_cnt
            call test_nc4(test_id, err_msg)
+           print *, 'err_msg =', err_msg, ' fail_cnt = ', fail_cnt
            call parse(err_msg, fail_cnt)
         end if
 
diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
index 4dc2abcbe463..3949a31e3ee5 100644
--- a/tests/unit/ncdf_tests.F90
+++ b/tests/unit/ncdf_tests.F90
@@ -279,23 +279,28 @@ Subroutine test_nc4(test_id, err_msg)
 
     integer :: shuffle
     integer :: deflate
-    integer :: deflate_level
+    integer :: my_deflate_level, deflate_level, deflate_level_2
+    integer :: storage
+    integer, dimension(1) :: chunksizes
 
     shuffle = 0
     deflate = 1
-    deflate_level = 4
+    deflate_level = 2
+    deflate_level_2 = 4
     err_msg = "no_error"
 
     dims(1) = 2*ntasks
     compdof = 2*my_rank+(/1,2/)  ! Where in the global array each task writes
     data_to_write = 1+my_rank
 
+    print*, ''
     call PIO_initdecomp(pio_iosystem, PIO_int, dims, compdof, iodesc_nCells)
 
     filename = fnames(test_id)
     iotype   = iotypes(test_id)
 
     ! Open existing file, write data to it
+    print*, 'PIO_openfile ', filename
     ret_val = PIO_openfile(pio_iosystem, pio_file, iotype, filename, PIO_write)
     if (ret_val .ne. PIO_NOERR) then
        ! Error in PIO_openfile
@@ -304,6 +309,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Enter define mode
+    print*, 'PIO_redef'
     ret_val = PIO_redef(pio_file)
     if (ret_val .ne. PIO_NOERR) then
        ! Error in PIO_redef
@@ -313,6 +319,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Define a new dimension M1.
+    print*, 'PIO_def_dim'
     ret_val = PIO_def_dim(pio_file, 'M111222', int(2*ntasks,pio_offset_kind), pio_dim)
     if (ret_val .ne. PIO_NOERR) then
        err_msg = "Could not define dimension M111222"
@@ -321,6 +328,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Define a new variable
+    print*, ''
     ret_val = PIO_def_var(pio_file, 'foo2222', PIO_int, (/pio_dim/), pio_var)
     if (ret_val .ne. PIO_NOERR) then
        err_msg = "Could not define variable foo2222"
@@ -329,6 +337,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Try to turn on compression for this variable.
+    print*, 'PIO_def_var_deflate'
     ret_val = PIO_def_var_deflate(pio_file, pio_var, shuffle, deflate, &
          deflate_level)
 
@@ -351,6 +360,31 @@ Subroutine test_nc4(test_id, err_msg)
        return
     end if
 
+    ! Try to set chunksizes for this variable.
+    ! storage = 0
+    ! chunksizes(1) = 42
+    ! ret_val = PIO_def_var_chunking(pio_file, pio_var, storage, chunksizes)
+
+    ! ! Should not have worked except for netCDF-4/HDF5 serial and netCDF-4/HDF5 parallel.
+    ! if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
+    !    err_msg = "Could not set chunksizes for variable foo2222 in netCDF-4 serial file"
+    !    call PIO_closefile(pio_file)
+    !    return
+    ! else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
+    !    err_msg = "Did not get expected error when trying to set chunksizes for pnetcdf file"
+    !    call PIO_closefile(pio_file)
+    !    return
+    ! else if (iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
+    !    err_msg = "Did not get expected error when trying to set chunksizes for netcdf classic file"
+    !    call PIO_closefile(pio_file)
+    !    return
+    ! else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .ne. PIO_NOERR) then
+    !    err_msg = "Could not set chunksizes for variable foo2222 in netCDF-4 parallel file"
+    !    call PIO_closefile(pio_file)
+    !    return
+    ! end if
+
+    print*, 'PIO_put_att'
     ret_val = PIO_put_att(pio_file, pio_var, "max_val", ntasks)
     if (ret_val .ne. PIO_NOERR) then
        ! Error in PIO_put_att
@@ -359,6 +393,7 @@ Subroutine test_nc4(test_id, err_msg)
        return
     end if
 
+    print*, 'PIO_put_att'
     ret_val = PIO_put_att(pio_file, PIO_global, "created_by", "PIO unit tests")
     if (ret_val .ne. PIO_NOERR) then
        ! Error in PIO_put_att
@@ -367,7 +402,67 @@ Subroutine test_nc4(test_id, err_msg)
        return
     end if
 
+    ! Check the compression settings of the variables.
+    print*, 'PIO_inq_var_deflate'
+    ret_val = PIO_inq_var_deflate(pio_file, pio_var, shuffle, deflate, my_deflate_level)
+
+    ! Should not have worked except for netCDF-4/HDF5 serial.
+    if (iotype .eq. PIO_iotype_netcdf4c) then
+       if (ret_val .ne. PIO_NOERR) then
+          err_msg = "Got error trying to inquire about deflate on for serial netcdf-4 file"
+          call PIO_closefile(pio_file)
+          return
+       else
+          if (shuffle .ne. 0 .or. deflate .ne. 1 .or. my_deflate_level .ne. deflate_level) then
+             err_msg = "Wrong values for deflate and shuffle for serial netcdf-4 file"
+             call PIO_closefile(pio_file)
+             return
+          end if
+       end if
+    else if ((iotype .eq. PIO_iotype_pnetcdf .or. iotype .eq. PIO_iotype_netcdf) .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to check deflate for non-netcdf-4 file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf4p) then
+       if (ret_val .ne. PIO_NOERR) then
+          err_msg = "Got error trying to inquire about deflate on for parallel netcdf-4 file"
+          call PIO_closefile(pio_file)
+          return
+       else
+          if (shuffle .ne. 0 .or. deflate .ne. 0) then
+             err_msg = "Wrong values for deflate and shuffle for parallel netcdf-4 file"
+             call PIO_closefile(pio_file)
+             return
+          end if
+       end if
+    end if
+
+    ! Try to turn on compression for this variable.
+    print*, 'PIO_def_var_deflate'
+    ret_val = PIO_def_var_deflate(pio_file, pio_var%varid, shuffle, deflate, &
+         deflate_level_2)
+
+    ! Should not have worked except for netCDF-4/HDF5 serial.
+    if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
+       err_msg = "Could not turn on compression for variable foo2222 second time"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to turn deflate on for pnetcdf file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to turn deflate on for netcdf classic file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to turn deflate on for parallel netcdf-4 file"
+       call PIO_closefile(pio_file)
+       return
+    end if
+
     ! Leave define mode
+    print*, 'PIO_enddef'
     ret_val = PIO_enddef(pio_file)
     if (ret_val .ne. PIO_NOERR) then
        err_msg = "Could not end define mode"
@@ -375,16 +470,17 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Check the compression settings of the variables.
-    ret_val = PIO_inq_var_deflate(pio_file, pio_var, shuffle, deflate, deflate_level)
+    print*, 'PIO_inq_var_deflate'
+    ret_val = PIO_inq_var_deflate(pio_file, pio_var%varid, shuffle, deflate, my_deflate_level)
 
-    ! Should not have worked except for netCDF-4/HDF5 serial.
+    ! Should not have worked except for netCDF-4/HDF5 serial and netcdf-4/HDF5 parallel.
     if (iotype .eq. PIO_iotype_netcdf4c) then
        if (ret_val .ne. PIO_NOERR) then
           err_msg = "Got error trying to inquire about deflate on for serial netcdf-4 file"
           call PIO_closefile(pio_file)
           return
        else
-          if (shuffle .ne. 0 .or. deflate .ne. 1 .or. deflate_level .ne. 4) then
+          if (shuffle .ne. 0 .or. deflate .ne. 1 .or. my_deflate_level .ne. deflate_level_2) then
              err_msg = "Wrong values for deflate and shuffle for serial netcdf-4 file"
              call PIO_closefile(pio_file)
              return
@@ -409,6 +505,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Write foo2
+    print*, 'PIO_write_darray'
     call PIO_write_darray(pio_file, pio_var, iodesc_nCells, data_to_write, ret_val)
     if (ret_val .ne. PIO_NOERR) then
        err_msg = "Could not write data"
@@ -416,6 +513,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Close file
+    print*, ' PIO_closefile'
     call PIO_closefile(pio_file)
 
     ! Free decomp

From cd1c5961768149239b263a76b7b6f42c5fa7a976 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Mon, 22 Feb 2016 13:06:13 -0500
Subject: [PATCH 33/48] set chunk cache function now working

---
 src/clib/pio.h        |  2 +-
 src/clib/pio_nc4.c    | 32 ++++++++++++----------------
 tests/unit/test_nc4.c | 49 +++++++++++++++++++++++++++++--------------
 3 files changed, 47 insertions(+), 36 deletions(-)

diff --git a/src/clib/pio.h b/src/clib/pio.h
index f4e8fe9251c2..4751be4cf49d 100644
--- a/src/clib/pio.h
+++ b/src/clib/pio.h
@@ -374,7 +374,7 @@ int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value);
 int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep);
 int PIOc_def_var_endian(int ncid, int varid, int endian);
 int PIOc_inq_var_endian(int ncid, int varid, int *endianp);
-int PIOc_set_chunk_cache(int iotype, int io_rank, size_t size, size_t nelems, float preemption);
+int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, size_t size, size_t nelems, float preemption);
 int PIOc_get_chunk_cache(int iotype, int io_rank, size_t *sizep, size_t *nelemsp, float *preemptionp);
 int PIOc_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
 			     float preemption);
diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index 28adf44e63c1..23870af888ce 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -902,14 +902,14 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
  * function.
  * 
  * @param iotype the iotype of files to be created or opened.
- `* @param io_rank the rank of the calling process.
+ * @param io_rank the rank of the calling process.
  * @param size size of file cache.
  * @param nelems number of elements in file cache.
  * @param preemption preemption setting for file cache.
  * 
  * @return PIO_NOERR for success, otherwise an error code.
  */
-int PIOc_set_chunk_cache(int iotype, int io_rank, size_t size,
+int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, size_t size,
 			 size_t nelems, float preemption)
 {
     int ierr;
@@ -921,10 +921,14 @@ int PIOc_set_chunk_cache(int iotype, int io_rank, size_t size,
 
     errstr = NULL;
     ierr = PIO_NOERR;
-    /*msg = PIO_MSG_SET_CHUNK_CACHE;*/
+    msg = PIO_MSG_SET_CHUNK_CACHE;
 
-    /* if (ios->async_interface && ! ios->ioproc){ */
-    /* 	if (ios->compmaster)  */
+    ios = pio_get_iosystem_from_id(iosysid);
+    if(ios == NULL)
+	return PIO_EBADID;
+
+  /* if (ios->async_interface && ! ios->ioproc){ */
+    /* 	if (ios->compmaster) */
     /* 	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm); */
     /* 	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm); */
     /* } */
@@ -942,31 +946,21 @@ int PIOc_set_chunk_cache(int iotype, int io_rank, size_t size,
 	break;
 #endif
     case PIO_IOTYPE_NETCDF:
-	return PIO_ENOTNC4;
+	ierr = PIO_ENOTNC4;
 	break;
 #endif
 #ifdef _PNETCDF
     case PIO_IOTYPE_PNETCDF:
-	return PIO_ENOTNC4;
+	ierr = PIO_ENOTNC4;
 	break;
 #endif
     default:
 	ierr = iotype_error(file->iotype,__FILE__,__LINE__);
     }
 
-    /* Allocate an error string if needed. */
-    if (ierr != PIO_NOERR)
-    {
-	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
-	sprintf(errstr,"in file %s",__FILE__);
-    }
-
     /* Check for netCDF error. */
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-
-    /* Free the error string if it was allocated. */
-    if (errstr != NULL)
-	free(errstr);
+    if (ierr)
+	MPI_Bcast(&ierr, 1, MPI_INTEGER, ios->ioroot, ios->my_comm);    
 
     return ierr;
 }    
diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index 068f60c74be5..c28451d91a33 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -163,15 +163,6 @@ main(int argc, char **argv)
     /** Endianness of variable. */
     int endianness;
 
-    /* Size of the file chunk cache. */
-    size_t chunk_cache_size;
-
-    /* Number of elements in file cache. */
-    size_t nelems;
-
-    /* File cache preemption. */
-    float preemption;
-
     /* Size of the var chunk cache. */
     size_t var_cache_size;
 
@@ -198,6 +189,13 @@ main(int argc, char **argv)
 
     /** Index for loops. */
     int fmt, d, d1, i;
+
+    /** For setting the chunk cache. */
+    size_t chunk_cache_size = 1024*1024;
+    size_t chunk_cache_nelems = 1024;
+    float chunk_cache_preemption = 0.5;
+    
+    char varname[15];
     
 #ifdef TIMING    
     /* Initialize the GPTL timing library. */
@@ -220,7 +218,7 @@ main(int argc, char **argv)
 	  ntasks == 8 || ntasks == 16))
 	fprintf(stderr, "Number of processors must be 1, 2, 4, 8, or 16!\n");
     if (verbose)
-	printf("%d: ParallelIO Library example1 running on %d processors.\n",
+	printf("%d: ParallelIO Library test_nc4 running on %d processors.\n",
 	       my_rank, ntasks);
 
     /* keep things simple - 1 iotask per MPI process */    
@@ -281,6 +279,26 @@ main(int argc, char **argv)
 	    MPIERR(ret);
 #endif /* HAVE_MPE */
 
+	if (verbose)
+	    printf("rank: %d Setting chunk cache for file %s with format %d...\n",
+		   my_rank, filename[fmt], format[fmt]);
+
+	/* Try to set the chunk cache. */
+	ret = PIOc_set_chunk_cache(iosysid, format[fmt], my_rank, chunk_cache_size,
+				   chunk_cache_nelems, chunk_cache_preemption);
+
+	/* Should only have worked for netCDF-4 iotypes. */
+	if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P)
+	{
+	    if (ret != PIO_NOERR)
+		ERR(ret);
+	}
+	else
+	{
+	    if (ret != PIO_ENOTNC4)
+		ERR(ERR_AWFUL);
+	}
+
 	/* Create the netCDF output file. */
 	if (verbose)
 	    printf("rank: %d Creating sample file %s with format %d...\n",
@@ -290,7 +308,7 @@ main(int argc, char **argv)
 	    ERR(ret);
 
 	/* Set error handling. */
-	PIOc_Set_File_Error_Handling(ncid, PIO_RETURN_ERROR);
+	PIOc_Set_File_Error_Handling(ncid, PIO_BCAST_ERROR);
 	
 	/* Define netCDF dimensions and variable. */
 	if (verbose)
@@ -316,10 +334,9 @@ main(int argc, char **argv)
 		ERR(ret);
 
 	    /** Check that the inq_varname function works. */
-	    char varname[15];
 	    if (verbose)
 	    	printf("rank: %d Checking varname\n", my_rank);
-	    ret = PIOc_inq_varname(ncid, 0, &varname);
+	    ret = PIOc_inq_varname(ncid, 0, varname);
 	    printf("rank: %d ret: %d varname: %s\n", my_rank, ret, varname);
 	    
 	    /** Check that the inq_var_chunking function works. */
@@ -413,11 +430,11 @@ main(int argc, char **argv)
 	    if ((ret = PIOc_get_var_chunk_cache(ncid, 0, &var_cache_size, &var_cache_nelems,
 						&var_cache_preemption)) != PIO_ENOTNC4)
 		ERR(ret);
-	    if ((ret = PIOc_set_chunk_cache(format[fmt], my_rank, chunk_cache_size, nelems,
-	    				    preemption)) != PIO_ENOTNC4)
+	    if ((ret = PIOc_set_chunk_cache(iosysid, format[fmt], my_rank, chunk_cache_size, chunk_cache_nelems,
+	    				    chunk_cache_preemption)) != PIO_ENOTNC4)
 	    	ERR(ret);
 	    if ((ret = PIOc_get_chunk_cache(format[fmt], my_rank, &chunk_cache_size,
-	    				    &nelems, &preemption)) != PIO_ENOTNC4)
+	    				    &chunk_cache_nelems, &chunk_cache_preemption)) != PIO_ENOTNC4)
 	    	ERR(ret);
 	}	    
 	

From 1a1c246b3042341f5c74b078a29d27a383b136cd Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 23 Feb 2016 12:44:52 -0500
Subject: [PATCH 34/48] more tests for nc4 performance functions

---
 src/clib/pio.h        |  2 +-
 src/clib/pio_nc4.c    | 42 ++++++++++++++++++++----------------------
 tests/unit/test_nc4.c | 25 +++++++++++++++++++++++--
 3 files changed, 44 insertions(+), 25 deletions(-)

diff --git a/src/clib/pio.h b/src/clib/pio.h
index 4751be4cf49d..f53f0e6e565f 100644
--- a/src/clib/pio.h
+++ b/src/clib/pio.h
@@ -375,7 +375,7 @@ int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep);
 int PIOc_def_var_endian(int ncid, int varid, int endian);
 int PIOc_inq_var_endian(int ncid, int varid, int *endianp);
 int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, size_t size, size_t nelems, float preemption);
-int PIOc_get_chunk_cache(int iotype, int io_rank, size_t *sizep, size_t *nelemsp, float *preemptionp);
+int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep, size_t *nelemsp, float *preemptionp);
 int PIOc_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
 			     float preemption);
 int PIOc_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp,
diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index 23870af888ce..df99d2557d23 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -996,17 +996,22 @@ int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, size_t size,
  * 
  * @return PIO_NOERR for success, otherwise an error code.
  */
-int PIOc_get_chunk_cache(int iotype, int io_rank, size_t *sizep,
+int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep,
 			 size_t *nelemsp, float *preemptionp)
 {
     int ierr;
     int msg;
     int mpierr;
+    iosystem_desc_t *ios;
     char *errstr;
 
     errstr = NULL;
     ierr = PIO_NOERR;
 
+    ios = pio_get_iosystem_from_id(iosysid);
+    if(ios == NULL)
+	return PIO_EBADID;
+
     /* Since this is a property of the running HDF5 instance, not the
      * file, it's not clear if this message passing will apply. For
      * now, comment it out. EJH */
@@ -1043,29 +1048,22 @@ int PIOc_get_chunk_cache(int iotype, int io_rank, size_t *sizep,
 	ierr = iotype_error(iotype,__FILE__,__LINE__);
     }
 
-    /* Allocate an error string if needed. */
-    if (ierr != PIO_NOERR)
-    {
-	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
-	sprintf(errstr,"in file %s",__FILE__);
-    }
 
     /* Check for netCDF error. */
-    /* ierr = check_netcdf(file, ierr, errstr,__LINE__);*/
-
-    /* Free the error string if it was allocated. */
-    if (errstr != NULL)
-	free(errstr);
-
-    /* if (sizep) */
-    /* 	if ((ierr = MPI_Bcast(sizep, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm))) */
-    /* 	    return PIO_EIO; */
-    /* if (nelemsp) */
-    /* 	if ((ierr = MPI_Bcast(nelemsp, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm))) */
-    /* 	    return PIO_EIO; */
-    /* if (preemptionp) */
-    /* 	if ((ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm))) */
-    /* 	    return PIO_EIO; */
+    if (ierr)
+	MPI_Bcast(&ierr, 1, MPI_INTEGER, ios->ioroot, ios->my_comm);
+    else
+    {
+	if (sizep)
+	    if ((ierr = MPI_Bcast(sizep, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
+		ierr = PIO_EIO;
+	if (nelemsp && !ierr)
+	    if ((ierr = MPI_Bcast(nelemsp, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
+		ierr = PIO_EIO;
+	if (preemptionp && !ierr)
+	    if ((ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm)))
+		ierr = PIO_EIO;
+    }
 
     return ierr;
 }
diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index c28451d91a33..459d58731d8b 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -194,7 +194,12 @@ main(int argc, char **argv)
     size_t chunk_cache_size = 1024*1024;
     size_t chunk_cache_nelems = 1024;
     float chunk_cache_preemption = 0.5;
-    
+
+    /* For reading the chunk cache. */
+    size_t chunk_cache_size_in;
+    size_t chunk_cache_nelems_in;
+    float chunk_cache_preemption_in;
+
     char varname[15];
     
 #ifdef TIMING    
@@ -299,6 +304,22 @@ main(int argc, char **argv)
 		ERR(ERR_AWFUL);
 	}
 
+	/* Now check the chunk cache. */
+	ret = PIOc_get_chunk_cache(iosysid, format[fmt], my_rank, &chunk_cache_size_in,
+				   &chunk_cache_nelems_in, &chunk_cache_preemption_in);
+
+	/* Should only have worked for netCDF-4 iotypes. */
+	if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P)
+	{
+	    if (ret != PIO_NOERR)
+		ERR(ret);
+	}
+	else
+	{
+	    if (ret != PIO_ENOTNC4)
+		ERR(ERR_AWFUL);
+	}
+
 	/* Create the netCDF output file. */
 	if (verbose)
 	    printf("rank: %d Creating sample file %s with format %d...\n",
@@ -433,7 +454,7 @@ main(int argc, char **argv)
 	    if ((ret = PIOc_set_chunk_cache(iosysid, format[fmt], my_rank, chunk_cache_size, chunk_cache_nelems,
 	    				    chunk_cache_preemption)) != PIO_ENOTNC4)
 	    	ERR(ret);
-	    if ((ret = PIOc_get_chunk_cache(format[fmt], my_rank, &chunk_cache_size,
+	    if ((ret = PIOc_get_chunk_cache(iosysid, format[fmt], my_rank, &chunk_cache_size,
 	    				    &chunk_cache_nelems, &chunk_cache_preemption)) != PIO_ENOTNC4)
 	    	ERR(ret);
 	}	    

From 854eb87696a1a163f44ea17608a857e4c7392ba2 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 23 Feb 2016 13:19:16 -0500
Subject: [PATCH 35/48] more testing

---
 tests/unit/test_nc4.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index 459d58731d8b..fd08f05a1e46 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -311,8 +311,14 @@ main(int argc, char **argv)
 	/* Should only have worked for netCDF-4 iotypes. */
 	if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P)
 	{
+	    /* Check that there was no error. */
 	    if (ret != PIO_NOERR)
 		ERR(ret);
+
+	    /* Check that we got the correct values. */
+	    if (chunk_cache_size_in != chunk_cache_size || chunk_cache_nelems_in != chunk_cache_nelems ||
+		chunk_cache_preemption_in != chunk_cache_preemption)
+		ERR(ERR_AWFUL);
 	}
 	else
 	{

From 3a98a06763eb6746aa6f7ada371f0b7b58a89f53 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 23 Feb 2016 13:59:41 -0500
Subject: [PATCH 36/48] more testing

---
 tests/unit/test_nc4.c | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index fd08f05a1e46..1daa68ef7a04 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -199,7 +199,7 @@ main(int argc, char **argv)
     size_t chunk_cache_size_in;
     size_t chunk_cache_nelems_in;
     float chunk_cache_preemption_in;
-
+    
     char varname[15];
     
 #ifdef TIMING    
@@ -431,8 +431,22 @@ main(int argc, char **argv)
 		    ERR(ret);
 		if (fletcher32)
 		    ERR(ERR_AWFUL);
-	    }		
-	    
+	    }
+
+	    /* Check setting the chunk cache for the variable. */
+	    if ((ret = PIOc_set_var_chunk_cache(ncid, 0, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
+						VAR_CACHE_PREEMPTION)) != PIO_ENOTNC4)
+	    	ERR(ret);
+
+	    /* Check getting the chunk cache values for the variable. */
+	    if ((ret = PIOc_get_var_chunk_cache(ncid, 0, &var_cache_size, &var_cache_nelems,
+						&var_cache_preemption)) != PIO_ENOTNC4)
+	    	ERR(ret);
+
+	    /* Check that we got expected values. */
+	    if (var_cache_size != VAR_CACHE_SIZE || var_cache_nelems != VAR_CACHE_NELEMS ||
+		var_cache_preemption != VAR_CACHE_PREEMPTION)
+		ERR(ERR_AWFUL);
 	} else {
 	    /* Trying to set or inq netCDF-4 settings for non-netCDF-4
 	     * files results in the PIO_ENOTNC4 error. */

From d60b54387cd3a6cdb49f4c42c11db740919f7bd8 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Wed, 24 Feb 2016 11:18:36 -0500
Subject: [PATCH 37/48] rest of nc4 functions in fortran API

---
 src/flib/pio_nf.F90 | 165 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 165 insertions(+)

diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90
index e30cc6a8e130..5f5608a78854 100644
--- a/src/flib/pio_nf.F90
+++ b/src/flib/pio_nf.F90
@@ -32,6 +32,10 @@ module pio_nf
        pio_inq_unlimdim                                     , &
        pio_inquire                                          , &
        pio_enddef                                           , &
+       pio_set_chunk_cache                                  , &
+       pio_get_chunk_cache                                  , &
+       pio_set_var_chunk_cache                              , &
+       pio_get_var_chunk_cache                              , &
        pio_redef
 !       pio_copy_att    to be done
 
@@ -189,6 +193,28 @@ module pio_nf
           inquire_id
   end interface
 
+  interface pio_set_chunk_cache
+     module procedure &
+          set_chunk_cache
+  end interface
+
+  interface pio_get_chunk_cache
+     module procedure &
+          get_chunk_cache
+  end interface
+
+  interface pio_set_var_chunk_cache
+     module procedure &
+          set_var_chunk_cache_desc                          , &
+          set_var_chunk_cache_id
+  end interface
+
+  interface pio_get_var_chunk_cache
+     module procedure &
+          get_var_chunk_cache_desc                          , &
+          get_var_chunk_cache_id
+  end interface
+
 contains
 
 !>
@@ -1572,4 +1598,143 @@ end function PIOc_def_var_chunking
 
     ierr = PIOc_def_var_chunking(file%fh, vardesc%varid-1, storage, cchunksizes)
   end function def_var_chunking
+
+!> 
+!! @public 
+!! @ingroup PIO_set_chunk_cache
+!! @brief Changes chunk cache settings for netCDF-4/HDF5 files created after this call.
+!<
+  integer function set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+    integer, intent(in) :: iosysid
+    integer, intent(in) :: iotype 
+    integer, intent(in) :: chunk_cache_size
+    integer, intent(in) :: chunk_cache_nelems
+    real, intent(in) :: chunk_cache_preemption
+
+    interface
+       integer (C_INT) function PIOc_set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
+            bind(c,name="PIOc_set_chunk_cache")
+         use iso_c_binding
+         integer(c_int), value :: iosysid
+         integer(c_int), value :: iotype
+         integer(c_int), value :: chunk_cache_size
+         integer(c_int), value :: chunk_cache_nelems
+         real(c_float), value :: chunk_cache_preemption
+       end function PIOc_set_chunk_cache
+    end interface
+
+    ierr = PIOc_set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+  end function set_chunk_cache
+
+!> 
+!! @public
+!! @ingroup PIO_set_chunk_cache  
+!! @brief Gets current settings for chunk cache (only relevant for netCDF4/HDF5 files.)
+!<
+  integer function get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+    integer, intent(in) :: iosysid
+    integer, intent(in) :: iotype
+    integer, intent(out) :: chunk_cache_size
+    integer, intent(out) :: chunk_cache_nelems
+    real, intent(out) :: chunk_cache_preemption
+
+    interface
+       integer (C_INT) function PIOc_get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
+            bind(c,name="PIOc_get_chunk_cache")
+         use iso_c_binding
+         integer(c_int), value :: iosysid
+         integer(c_int), value :: iotype
+         integer(c_int) :: chunk_cache_size
+         integer(c_int) :: chunk_cache_nelems
+         real(c_float) :: chunk_cache_preemption
+       end function PIOc_get_chunk_cache
+    end interface
+
+    ierr = PIOc_get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+  end function get_chunk_cache
+
+!> 
+!! @public 
+!! @ingroup PIO_set_chunk_cache
+!! @brief Changes chunk cache settings for a variable in a netCDF-4/HDF5 file.
+!<
+  integer function set_var_chunk_cache_id(file, varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    integer, intent(in) :: varid
+    integer, intent(in) :: chunk_cache_size
+    integer, intent(in) :: chunk_cache_nelems
+    real, intent(in) :: chunk_cache_preemption
+
+    interface
+       integer (C_INT) function PIOc_set_var_chunk_cache(ncid, varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
+            bind(c,name="PIOc_set_var_chunk_cache")
+         use iso_c_binding
+         integer(c_int), value :: ncid
+         integer(c_int), value :: varid
+         integer(c_int), value :: chunk_cache_size
+         integer(c_int), value :: chunk_cache_nelems
+         real(c_float), value :: chunk_cache_preemption
+       end function PIOc_set_var_chunk_cache
+    end interface
+
+    ierr = PIOc_set_var_chunk_cache(file%fh, varid-1, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+  end function set_var_chunk_cache_id
+
+  !> 
+!! @public 
+!! @ingroup PIO_set_var_chunk_cache
+!! @brief Changes chunk cacne for a variable.
+!<
+  integer function set_var_chunk_cache_desc(file, vardesc, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    type (var_desc_t), intent(in) :: vardesc
+    integer, intent(in) :: chunk_cache_size
+    integer, intent(in) :: chunk_cache_nelems
+    real, intent(in) :: chunk_cache_preemption
+
+    ierr = set_var_chunk_cache_id(file, vardesc%varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+  end function set_var_chunk_cache_desc
+
+!> 
+!! @public 
+!! @ingroup PIO_get_var_chunk_cache
+!! @brief Get the chunk cache settings for a variable.
+!<
+  integer function get_var_chunk_cache_desc(file, vardesc, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    type (var_desc_t), intent(in) :: vardesc
+    integer, intent(out) :: chunk_cache_size
+    integer, intent(out) :: chunk_cache_nelems
+    real, intent(out) :: chunk_cache_preemption
+
+    ierr = get_var_chunk_cache_id(file, vardesc%varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+  end function get_var_chunk_cache_desc
+
+!> 
+!! @public 
+!! @ingroup PIO_get_var_chunk_cache
+!! @brief Get the chunk cache settings for a variable.
+!<
+  integer function get_var_chunk_cache_id(file, varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+    type (File_desc_t), intent(in)  :: file
+    integer, intent(in) :: varid
+    integer, intent(out) :: chunk_cache_size
+    integer, intent(out) :: chunk_cache_nelems
+    real, intent(out) :: chunk_cache_preemption
+
+    interface
+       integer (C_INT) function PIOc_get_var_chunk_cache(ncid, varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
+            bind(c,name="PIOc_get_var_chunk_cache")
+         use iso_c_binding
+         integer(c_int), value :: ncid
+         integer(c_int), value :: varid
+         integer(c_int) :: chunk_cache_size
+         integer(c_int) :: chunk_cache_nelems
+         real(c_float) :: chunk_cache_preemption
+       end function PIOc_get_var_chunk_cache
+    end interface
+
+    ierr = PIOc_get_var_chunk_cache(file%fh, varid-1, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+  end function get_var_chunk_cache_id
+
 end module pio_nf

From d8a042f05687ca710230abba97efa063b88257d0 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Fri, 26 Feb 2016 11:22:59 -0500
Subject: [PATCH 38/48] further development of fortran set_chunk_cache tests

---
 src/flib/pio.F90          |  7 +++--
 tests/unit/ncdf_tests.F90 | 56 ++++++++++++++++++++++-----------------
 2 files changed, 36 insertions(+), 27 deletions(-)

diff --git a/src/flib/pio.F90 b/src/flib/pio.F90
index 1941ffb20f8b..2f3781f018e5 100644
--- a/src/flib/pio.F90
+++ b/src/flib/pio.F90
@@ -53,9 +53,12 @@ module pio
        PIO_def_var   ,        &
        PIO_def_var_deflate   ,        &
        PIO_redef     ,          &
-!       PIO_copy_att  ,       &   
        PIO_inquire_variable , &
-       PIO_inquire_dimension 
+       PIO_inquire_dimension, &
+       PIO_set_chunk_cache, &
+       PIO_get_chunk_cache, &
+       PIO_set_var_chunk_cache, &
+       PIO_get_var_chunk_cache
 
   use pionfatt_mod, only : PIO_put_att   => put_att,        &
        PIO_get_att   => get_att
diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
index 3949a31e3ee5..665a05351c8b 100644
--- a/tests/unit/ncdf_tests.F90
+++ b/tests/unit/ncdf_tests.F90
@@ -283,6 +283,10 @@ Subroutine test_nc4(test_id, err_msg)
     integer :: storage
     integer, dimension(1) :: chunksizes
 
+    integer :: chunk_cache_size
+    integer :: chunk_cache_nelems
+    real :: chunk_cache_preemption
+
     shuffle = 0
     deflate = 1
     deflate_level = 2
@@ -299,7 +303,33 @@ Subroutine test_nc4(test_id, err_msg)
     filename = fnames(test_id)
     iotype   = iotypes(test_id)
 
-    ! Open existing file, write data to it
+    ! Set the chunk cache for netCDF-4/HDF5 files.
+    chunk_cache_size = 1024 * 1024
+    chunk_cache_nelems = 3
+    chunk_cache_preemption = 0.1
+    ret_val = PIO_set_chunk_cache(pio_iosystem%iosysid, iotype, chunk_cache_size, chunk_cache_nelems, &
+         chunk_cache_preemption)
+    
+    ! Should not have worked except for netCDF-4/HDF5 iotypes.
+    if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
+       err_msg = "Could not set chunk cache"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to set chunk cache for pnetcdf file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to set chunk cache for netcdf classic file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .ne. PIO_NOERR) then
+       err_msg = "Could not set chunk cache"
+       call PIO_closefile(pio_file)
+       return
+    end if
+
+    ! Open existing file, write data to itn
     print*, 'PIO_openfile ', filename
     ret_val = PIO_openfile(pio_iosystem, pio_file, iotype, filename, PIO_write)
     if (ret_val .ne. PIO_NOERR) then
@@ -360,30 +390,6 @@ Subroutine test_nc4(test_id, err_msg)
        return
     end if
 
-    ! Try to set chunksizes for this variable.
-    ! storage = 0
-    ! chunksizes(1) = 42
-    ! ret_val = PIO_def_var_chunking(pio_file, pio_var, storage, chunksizes)
-
-    ! ! Should not have worked except for netCDF-4/HDF5 serial and netCDF-4/HDF5 parallel.
-    ! if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
-    !    err_msg = "Could not set chunksizes for variable foo2222 in netCDF-4 serial file"
-    !    call PIO_closefile(pio_file)
-    !    return
-    ! else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
-    !    err_msg = "Did not get expected error when trying to set chunksizes for pnetcdf file"
-    !    call PIO_closefile(pio_file)
-    !    return
-    ! else if (iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
-    !    err_msg = "Did not get expected error when trying to set chunksizes for netcdf classic file"
-    !    call PIO_closefile(pio_file)
-    !    return
-    ! else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .ne. PIO_NOERR) then
-    !    err_msg = "Could not set chunksizes for variable foo2222 in netCDF-4 parallel file"
-    !    call PIO_closefile(pio_file)
-    !    return
-    ! end if
-
     print*, 'PIO_put_att'
     ret_val = PIO_put_att(pio_file, pio_var, "max_val", ntasks)
     if (ret_val .ne. PIO_NOERR) then

From 243722066f5b214b0488ea641ce09011bb23c71c Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Fri, 26 Feb 2016 11:46:31 -0500
Subject: [PATCH 39/48] further development of fortran set_chunk_cache tests

---
 tests/unit/ncdf_tests.F90 | 51 +++++++++++++++++++++++++++++++++++----
 1 file changed, 46 insertions(+), 5 deletions(-)

diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
index 665a05351c8b..c9aff060a8f6 100644
--- a/tests/unit/ncdf_tests.F90
+++ b/tests/unit/ncdf_tests.F90
@@ -277,20 +277,27 @@ Subroutine test_nc4(test_id, err_msg)
     integer                        :: pio_dim
     type(var_desc_t)               :: pio_var
 
+    ! These will be used to test the compression settings for netCDF-4
+    ! files.
     integer :: shuffle
     integer :: deflate
     integer :: my_deflate_level, deflate_level, deflate_level_2
+
+    ! These will be used to test the chunksizes for netCDF-4 files.
     integer :: storage
     integer, dimension(1) :: chunksizes
 
+    ! These will be used to set chunk cache sizes in netCDF-4/HDF5
+    ! files.
     integer :: chunk_cache_size
     integer :: chunk_cache_nelems
     real :: chunk_cache_preemption
 
-    shuffle = 0
-    deflate = 1
-    deflate_level = 2
-    deflate_level_2 = 4
+    ! These will be used to check the settings of the chunk caches.
+    integer :: chunk_cache_size_in
+    integer :: chunk_cache_nelems_in
+    real :: chunk_cache_preemption_in
+
     err_msg = "no_error"
 
     dims(1) = 2*ntasks
@@ -329,6 +336,36 @@ Subroutine test_nc4(test_id, err_msg)
        return
     end if
 
+    ! Check the settings of the chunk cache for netCDF-4/HDF5 files.
+    ret_val = PIO_set_chunk_cache(pio_iosystem%iosysid, iotype, chunk_cache_size_in, chunk_cache_nelems_in, &
+         chunk_cache_preemption_in)
+    
+    ! Should not have worked except for netCDF-4/HDF5 iotypes.
+    if (iotype .eq. PIO_iotype_netcdf4c .or. iotype .eq. PIO_iotype_netcdf4p) then
+       if (ret_val .ne. PIO_NOERR) then
+          err_msg = "Could not set chunk cache"
+          call PIO_closefile(pio_file)
+       endif
+       if (chunk_cache_size_in .ne. chunk_cache_size .or. chunk_cache_nelems_in .ne. chunk_cache_nelems .or. &
+            chunk_cache_preemption_in .ne. chunk_cache_preemption) then
+          err_msg = "Incorrect chunk cache values!"
+          call PIO_closefile(pio_file)
+       endif
+       return
+    else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to set chunk cache for pnetcdf file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to set chunk cache for netcdf classic file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .ne. PIO_NOERR) then
+       err_msg = "Could not set chunk cache"
+       call PIO_closefile(pio_file)
+       return
+    end if
+
     ! Open existing file, write data to itn
     print*, 'PIO_openfile ', filename
     ret_val = PIO_openfile(pio_iosystem, pio_file, iotype, filename, PIO_write)
@@ -367,7 +404,11 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Try to turn on compression for this variable.
-    print*, 'PIO_def_var_deflate'
+    print*, 'PIO_def_var_deflate' 
+    shuffle = 0
+    deflate = 1
+    deflate_level = 2
+    deflate_level_2 = 4
     ret_val = PIO_def_var_deflate(pio_file, pio_var, shuffle, deflate, &
          deflate_level)
 

From 8edfbf53149e4648afbd39b7156745c926702667 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Mon, 29 Feb 2016 14:45:15 -0500
Subject: [PATCH 40/48] more fixes for chunk cache functions

---
 src/clib/pio.h            |   4 +-
 src/clib/pio_nc.c         |   1 +
 src/clib/pio_nc4.c        | 117 +++++++++++++++++++++++++++++---------
 src/flib/pio_nf.F90       |  56 +++++++++---------
 tests/unit/ncdf_tests.F90 | 104 +++++++++++++++++++++++++--------
 tests/unit/test_nc4.c     |  22 +++++--
 6 files changed, 220 insertions(+), 84 deletions(-)

diff --git a/src/clib/pio.h b/src/clib/pio.h
index f53f0e6e565f..575578f36fe9 100644
--- a/src/clib/pio.h
+++ b/src/clib/pio.h
@@ -376,9 +376,9 @@ int PIOc_def_var_endian(int ncid, int varid, int endian);
 int PIOc_inq_var_endian(int ncid, int varid, int *endianp);
 int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, size_t size, size_t nelems, float preemption);
 int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep, size_t *nelemsp, float *preemptionp);
-int PIOc_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
+int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems,
 			     float preemption);
-int PIOc_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp,
+int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp,
 			     float *preemptionp);
 int PIOc_inq_var (int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp, int *nattsp); 
 int PIOc_inq_varname (int ncid, int varid, char *name); 
diff --git a/src/clib/pio_nc.c b/src/clib/pio_nc.c
index f511475a4f6c..fdaddb4dc18f 100644
--- a/src/clib/pio_nc.c
+++ b/src/clib/pio_nc.c
@@ -3483,6 +3483,7 @@ int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp)
     sprintf(errstr,"in file %s",__FILE__);
   }
   ierr = check_netcdf(file, ierr, errstr,__LINE__);
+  printf("sizeof(MPI_OFFSET)=%d sizeof(PIO_Offset)=%d sizeof(*lenp)=%d \n", sizeof(MPI_OFFSET), sizeof(PIO_Offset), sizeof(*lenp));
     mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm);
   if(errstr != NULL) free(errstr);
   return ierr;
diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index df99d2557d23..6843891a3591 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -757,12 +757,12 @@ int PIOc_def_var_endian(int ncid, int varid, int endian)
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
-	    return PIO_ENOTNC4;
+	    ierr = PIO_ENOTNC4;
 	    break;
 #endif
 #ifdef _PNETCDF
 	case PIO_IOTYPE_PNETCDF:
-	    return PIO_ENOTNC4;
+	    ierr = PIO_ENOTNC4;
 	    break;
 #endif
 	default:
@@ -848,12 +848,12 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
-	    return PIO_ENOTNC4;
+	    ierr = PIO_ENOTNC4;
 	    break;
 #endif
 #ifdef _PNETCDF
 	case PIO_IOTYPE_PNETCDF:
-	    return PIO_ENOTNC4;
+	    ierr = PIO_ENOTNC4;
 	    break;
 #endif
 	default:
@@ -1012,6 +1012,12 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep,
     if(ios == NULL)
 	return PIO_EBADID;
 
+    int my_rank;
+    int ret;
+    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
+	return ret;
+    printf("rank %d PIOc_get_chunk_cache called iotype=%d\n", my_rank, iotype);
+
     /* Since this is a property of the running HDF5 instance, not the
      * file, it's not clear if this message passing will apply. For
      * now, comment it out. EJH */
@@ -1031,17 +1037,22 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep,
 	ierr = nc_get_chunk_cache(sizep, nelemsp, preemptionp);
 	break;
     case PIO_IOTYPE_NETCDF4C:
-	if (!io_rank)
+	if (!ios->io_rank)
+	{
 	    ierr = nc_get_chunk_cache(sizep, nelemsp, preemptionp);
+	    printf("rank 0 calling nc_get_chunk_cache returned *sizep = %d *nelemsp = %d *preemptionp=%g\n",
+		   *sizep, *nelemsp, *preemptionp);
+	}
 	break;
 #endif
     case PIO_IOTYPE_NETCDF:
-	return PIO_ENOTNC4;
+	ierr = PIO_ENOTNC4;
 	break;
 #endif
 #ifdef _PNETCDF
     case PIO_IOTYPE_PNETCDF:
-	return PIO_ENOTNC4;
+	ierr = PIO_ENOTNC4;
+	printf("rank %d PIOc_get_chunk_cache ierr = %d\n", my_rank, ierr);
 	break;
 #endif
     default:
@@ -1055,14 +1066,27 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep,
     else
     {
 	if (sizep)
-	    if ((ierr = MPI_Bcast(sizep, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
+	{
+	    int my_size = *sizep;
+	    printf("before bcast rank %d PIOc_get_chunk_cache my_size = %d ios->ioroot %d ios->io_rank=%d\n", my_rank, my_size, ios->ioroot, ios->io_rank);
+	    if ((ierr = MPI_Bcast(&my_size, 1, MPI_INT, ios->ioroot, ios->my_comm)))
 		ierr = PIO_EIO;
+	    printf("after bcast rank %d PIOc_get_chunk_cache my_size = %d *sizep = %d\n", my_rank, my_size, *sizep);
+	    *sizep = my_size;
+	    printf("rank %d PIOc_get_chunk_cache my_size = %d *sizep = %d\n", my_rank, my_size, *sizep);
+	}
 	if (nelemsp && !ierr)
-	    if ((ierr = MPI_Bcast(nelemsp, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm)))
+	{
+	    int my_nelems = *nelemsp;
+	    if ((ierr = MPI_Bcast(&my_nelems, 1, MPI_INT, ios->ioroot, ios->my_comm)))
 		ierr = PIO_EIO;
+	    *nelemsp = my_nelems;
+	}
 	if (preemptionp && !ierr)
+	{
 	    if ((ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm)))
 		ierr = PIO_EIO;
+	}
     }
 
     return ierr;
@@ -1092,7 +1116,7 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep,
  * 
  * @return PIO_NOERR for success, otherwise an error code.
  */
-int PIOc_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
+int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems,
 			     float preemption)
 {
     int ierr;
@@ -1110,6 +1134,12 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
     ios = file->iosystem;
     msg = PIO_MSG_SET_VAR_CHUNK_CACHE;
 
+    int my_rank;
+    int ret;
+    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
+	return ret;
+    printf("rank %d PIOc_set_var_chunk_cache called file->iotype=%d\n", my_rank, file->iotype);
+
     if (ios->async_interface && ! ios->ioproc)
     {
 	if (ios->compmaster) 
@@ -1127,19 +1157,20 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
 	    ierr = nc_set_var_chunk_cache(file->fh, varid, size, nelems, preemption);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
-	    if (ios->io_rank==0)
+	    if (!ios->io_rank)
 	    {
 		ierr = nc_set_var_chunk_cache(file->fh, varid, size, nelems, preemption);
 	    }
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
-	    return PIO_ENOTNC4;
+	    ierr = PIO_ENOTNC4;
 	    break;
 #endif
 #ifdef _PNETCDF
 	case PIO_IOTYPE_PNETCDF:
-	    return PIO_ENOTNC4;
+	    printf("rank %d PIOc_set_var_chunk_cache pnetcdf will return PIO_ENOTNC4\n", my_rank);
+	    ierr = PIO_ENOTNC4;
 	    break;
 #endif
 	default:
@@ -1187,7 +1218,7 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems,
  * 
  * @return PIO_NOERR for success, otherwise an error code.
  */
-int PIOc_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp,
+int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp,
 			     float *preemptionp)
 {
     int ierr;
@@ -1204,6 +1235,12 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp
 	return PIO_EBADID;
     ios = file->iosystem;
 
+    int my_rank;
+    int ret;
+    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
+	return ret;
+    printf("rank %d PIOc_get_var_chunk_cache called file->iotype=%d\n", my_rank, file->iotype);
+
     /* Since this is a property of the running HDF5 instance, not the
      * file, it's not clear if this message passing will apply. For
      * now, comment it out. EJH */
@@ -1222,27 +1259,34 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp
 #ifdef _NETCDF
 #ifdef _NETCDF4
 	case PIO_IOTYPE_NETCDF4P:
-	    ierr = nc_get_var_chunk_cache(file->fh, varid,  sizep, nelemsp, preemptionp);
+	    ierr = nc_get_var_chunk_cache(file->fh, varid,  (size_t *)sizep, (size_t *)nelemsp,
+					  preemptionp);
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
-	    if (!ios->io_rank)
-		ierr = nc_get_var_chunk_cache(file->fh, varid,  sizep, nelemsp, preemptionp);
+	    if (ios->io_rank == 0)
+	    {
+		ierr = nc_get_var_chunk_cache(file->fh, varid,  (size_t *)sizep, (size_t *)nelemsp,
+					      preemptionp);
+		printf("sizep=%d\n", *sizep);
+	    }
+	    
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
-	    return PIO_ENOTNC4;
+	    ierr = PIO_ENOTNC4;
 	    break;
 #endif
 #ifdef _PNETCDF
 	case PIO_IOTYPE_PNETCDF:
-	    return PIO_ENOTNC4;
+	    ierr = PIO_ENOTNC4;
+	    printf("rank %d PIOc_get_var_chunk_cache called file->iotype=%d ierr = %d\n", my_rank, file->iotype, ierr);
 	    break;
 #endif
 	default:
 	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
 	}
     }
-
+    printf("rank %d PIOc_get_var_chunk_cache called file->iotype=%d ierr = %d\n", my_rank, file->iotype, ierr);
     /* If there is an error, allocate space for the error string. */
     if (ierr != PIO_NOERR)
     {
@@ -1252,18 +1296,37 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, size_t *sizep, size_t *nelemsp
 
     /* Check the netCDF return code, and broadcast it to all tasks. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
-
+    printf("rank %d PIOc_get_var_chunk_cache called file->iotype=%d ierr = %d\n", my_rank, file->iotype, ierr);
     /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
     /* Broadcast results to all tasks. */
-    if (sizep)
-	ierr = MPI_Bcast(sizep, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm);
-    if (nelemsp)
-	ierr = MPI_Bcast(nelemsp, 1, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm);
-    if (preemptionp)
-	ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm);
+    printf("sizeof(PIO_Offset)=%d, sizeof(MPI_OFFSET)=%d, sizeof(MPI_Offset)=%d, sizeof(*sizep)= %d\n",
+	   sizeof(PIO_Offset), sizeof(MPI_OFFSET), sizeof(MPI_Offset), sizeof(*sizep));
+    if (sizep && !ierr)
+    {
+	int my_size = *sizep;
+	printf("rank %d sizep=%d\n", my_rank, *sizep);
+	printf("my_size = %d\n");
+	ierr = MPI_Bcast(&my_size, 1, MPI_INT, ios->ioroot, ios->my_comm);
+	printf("rank %d my_size = %d\n", my_rank, my_size);
+	*sizep = my_size;
+    }
+    printf("nelems\n");
+    if (nelemsp && !ierr)
+    {
+    	int my_nelems = *nelemsp;
+    	ierr = MPI_Bcast(&my_nelems, 1, MPI_INT, ios->ioroot, ios->my_comm);
+	printf("rank %d my_nelems = %d\n", my_rank, my_nelems);
+	*nelemsp = my_nelems;
+    }
+    printf("preemption\n");
+    if (preemptionp && !ierr)
+    {
+    	ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm);
+	printf("rank %d preemption = %g\n", my_rank, *preemptionp);	
+    }
 
     return ierr;
 }    
diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90
index 5f5608a78854..3f646ace99bc 100644
--- a/src/flib/pio_nf.F90
+++ b/src/flib/pio_nf.F90
@@ -1604,26 +1604,28 @@ end function def_var_chunking
 !! @ingroup PIO_set_chunk_cache
 !! @brief Changes chunk cache settings for netCDF-4/HDF5 files created after this call.
 !<
-  integer function set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+  integer function set_chunk_cache(iosysid, iotype, io_rank, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     integer, intent(in) :: iosysid
     integer, intent(in) :: iotype 
-    integer, intent(in) :: chunk_cache_size
-    integer, intent(in) :: chunk_cache_nelems
+    integer, intent(in) :: io_rank
+    integer(kind=PIO_OFFSET_KIND), intent(in) :: chunk_cache_size
+    integer(kind=PIO_OFFSET_KIND), intent(in) :: chunk_cache_nelems
     real, intent(in) :: chunk_cache_preemption
 
     interface
-       integer (C_INT) function PIOc_set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
+       integer (C_INT) function PIOc_set_chunk_cache(iosysid, iotype, io_rank, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
             bind(c,name="PIOc_set_chunk_cache")
          use iso_c_binding
          integer(c_int), value :: iosysid
          integer(c_int), value :: iotype
-         integer(c_int), value :: chunk_cache_size
-         integer(c_int), value :: chunk_cache_nelems
+         integer(c_int), value :: io_rank
+         integer(c_size_t), value :: chunk_cache_size
+         integer(c_size_t), value :: chunk_cache_nelems
          real(c_float), value :: chunk_cache_preemption
        end function PIOc_set_chunk_cache
     end interface
 
-    ierr = PIOc_set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+    ierr = PIOc_set_chunk_cache(iosysid, iotype, io_rank, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
   end function set_chunk_cache
 
 !> 
@@ -1631,26 +1633,28 @@ end function set_chunk_cache
 !! @ingroup PIO_set_chunk_cache  
 !! @brief Gets current settings for chunk cache (only relevant for netCDF4/HDF5 files.)
 !<
-  integer function get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+  integer function get_chunk_cache(iosysid, iotype, io_rank, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     integer, intent(in) :: iosysid
     integer, intent(in) :: iotype
-    integer, intent(out) :: chunk_cache_size
-    integer, intent(out) :: chunk_cache_nelems
+    integer, intent(in) :: io_rank
+    integer(kind=PIO_OFFSET_KIND), intent(out) :: chunk_cache_size
+    integer(kind=PIO_OFFSET_KIND), intent(out) :: chunk_cache_nelems
     real, intent(out) :: chunk_cache_preemption
 
     interface
-       integer (C_INT) function PIOc_get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
+       integer (C_INT) function PIOc_get_chunk_cache(iosysid, iotype, io_rank, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
             bind(c,name="PIOc_get_chunk_cache")
          use iso_c_binding
          integer(c_int), value :: iosysid
          integer(c_int), value :: iotype
-         integer(c_int) :: chunk_cache_size
-         integer(c_int) :: chunk_cache_nelems
+         integer(c_int), value :: io_rank
+         integer(c_size_t) :: chunk_cache_size
+         integer(c_size_t) :: chunk_cache_nelems
          real(c_float) :: chunk_cache_preemption
        end function PIOc_get_chunk_cache
     end interface
 
-    ierr = PIOc_get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+    ierr = PIOc_get_chunk_cache(iosysid, iotype, io_rank, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
   end function get_chunk_cache
 
 !> 
@@ -1661,8 +1665,8 @@ end function get_chunk_cache
   integer function set_var_chunk_cache_id(file, varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     type (File_desc_t), intent(in)  :: file
     integer, intent(in) :: varid
-    integer, intent(in) :: chunk_cache_size
-    integer, intent(in) :: chunk_cache_nelems
+    integer(PIO_OFFSET_KIND), intent(in) :: chunk_cache_size
+    integer(PIO_OFFSET_KIND), intent(in) :: chunk_cache_nelems
     real, intent(in) :: chunk_cache_preemption
 
     interface
@@ -1671,8 +1675,8 @@ integer (C_INT) function PIOc_set_var_chunk_cache(ncid, varid, chunk_cache_size,
          use iso_c_binding
          integer(c_int), value :: ncid
          integer(c_int), value :: varid
-         integer(c_int), value :: chunk_cache_size
-         integer(c_int), value :: chunk_cache_nelems
+         integer(c_size_t), value :: chunk_cache_size
+         integer(c_size_t), value :: chunk_cache_nelems
          real(c_float), value :: chunk_cache_preemption
        end function PIOc_set_var_chunk_cache
     end interface
@@ -1688,8 +1692,8 @@ end function set_var_chunk_cache_id
   integer function set_var_chunk_cache_desc(file, vardesc, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     type (File_desc_t), intent(in)  :: file
     type (var_desc_t), intent(in) :: vardesc
-    integer, intent(in) :: chunk_cache_size
-    integer, intent(in) :: chunk_cache_nelems
+    integer(PIO_OFFSET_KIND), intent(in) :: chunk_cache_size
+    integer(PIO_OFFSET_KIND), intent(in) :: chunk_cache_nelems
     real, intent(in) :: chunk_cache_preemption
 
     ierr = set_var_chunk_cache_id(file, vardesc%varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
@@ -1703,8 +1707,8 @@ end function set_var_chunk_cache_desc
   integer function get_var_chunk_cache_desc(file, vardesc, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     type (File_desc_t), intent(in)  :: file
     type (var_desc_t), intent(in) :: vardesc
-    integer, intent(out) :: chunk_cache_size
-    integer, intent(out) :: chunk_cache_nelems
+    integer(PIO_OFFSET_KIND), intent(out) :: chunk_cache_size
+    integer(PIO_OFFSET_KIND), intent(out) :: chunk_cache_nelems
     real, intent(out) :: chunk_cache_preemption
 
     ierr = get_var_chunk_cache_id(file, vardesc%varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
@@ -1718,8 +1722,8 @@ end function get_var_chunk_cache_desc
   integer function get_var_chunk_cache_id(file, varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     type (File_desc_t), intent(in)  :: file
     integer, intent(in) :: varid
-    integer, intent(out) :: chunk_cache_size
-    integer, intent(out) :: chunk_cache_nelems
+    integer(PIO_OFFSET_KIND), intent(out) :: chunk_cache_size
+    integer(PIO_OFFSET_KIND), intent(out) :: chunk_cache_nelems
     real, intent(out) :: chunk_cache_preemption
 
     interface
@@ -1728,8 +1732,8 @@ integer (C_INT) function PIOc_get_var_chunk_cache(ncid, varid, chunk_cache_size,
          use iso_c_binding
          integer(c_int), value :: ncid
          integer(c_int), value :: varid
-         integer(c_int) :: chunk_cache_size
-         integer(c_int) :: chunk_cache_nelems
+         integer(c_size_t) :: chunk_cache_size
+         integer(c_size_t) :: chunk_cache_nelems
          real(c_float) :: chunk_cache_preemption
        end function PIOc_get_var_chunk_cache
     end interface
diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
index c9aff060a8f6..89b5fabbec95 100644
--- a/tests/unit/ncdf_tests.F90
+++ b/tests/unit/ncdf_tests.F90
@@ -289,13 +289,13 @@ Subroutine test_nc4(test_id, err_msg)
 
     ! These will be used to set chunk cache sizes in netCDF-4/HDF5
     ! files.
-    integer :: chunk_cache_size
-    integer :: chunk_cache_nelems
+    integer(kind=PIO_OFFSET_KIND) :: chunk_cache_size
+    integer(kind=PIO_OFFSET_KIND) :: chunk_cache_nelems
     real :: chunk_cache_preemption
 
     ! These will be used to check the settings of the chunk caches.
-    integer :: chunk_cache_size_in
-    integer :: chunk_cache_nelems_in
+    integer(kind=PIO_OFFSET_KIND) :: chunk_cache_size_in
+    integer(kind=PIO_OFFSET_KIND) :: chunk_cache_nelems_in
     real :: chunk_cache_preemption_in
 
     err_msg = "no_error"
@@ -304,7 +304,7 @@ Subroutine test_nc4(test_id, err_msg)
     compdof = 2*my_rank+(/1,2/)  ! Where in the global array each task writes
     data_to_write = 1+my_rank
 
-    print*, ''
+    print*, 'PIO_initdecom'
     call PIO_initdecomp(pio_iosystem, PIO_int, dims, compdof, iodesc_nCells)
 
     filename = fnames(test_id)
@@ -314,8 +314,9 @@ Subroutine test_nc4(test_id, err_msg)
     chunk_cache_size = 1024 * 1024
     chunk_cache_nelems = 3
     chunk_cache_preemption = 0.1
-    ret_val = PIO_set_chunk_cache(pio_iosystem%iosysid, iotype, chunk_cache_size, chunk_cache_nelems, &
-         chunk_cache_preemption)
+    print*, 'PIO_set_chunk_cache'
+    ret_val = PIO_set_chunk_cache(pio_iosystem%iosysid, iotype, my_rank, chunk_cache_size, &
+         chunk_cache_nelems, chunk_cache_preemption)
     
     ! Should not have worked except for netCDF-4/HDF5 iotypes.
     if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
@@ -337,8 +338,11 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Check the settings of the chunk cache for netCDF-4/HDF5 files.
-    ret_val = PIO_set_chunk_cache(pio_iosystem%iosysid, iotype, chunk_cache_size_in, chunk_cache_nelems_in, &
-         chunk_cache_preemption_in)
+    print*, 'testing PIO_get_chunk_cache'
+    ret_val = PIO_get_chunk_cache(pio_iosystem%iosysid, iotype, my_rank, chunk_cache_size_in, &
+         chunk_cache_nelems_in, chunk_cache_preemption_in)
+    print*, 'PIO_get_chunk_cache returned ', chunk_cache_size_in, &
+         chunk_cache_nelems_in, chunk_cache_preemption_in
     
     ! Should not have worked except for netCDF-4/HDF5 iotypes.
     if (iotype .eq. PIO_iotype_netcdf4c .or. iotype .eq. PIO_iotype_netcdf4p) then
@@ -360,10 +364,6 @@ Subroutine test_nc4(test_id, err_msg)
        err_msg = "Did not get expected error when trying to set chunk cache for netcdf classic file"
        call PIO_closefile(pio_file)
        return
-    else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .ne. PIO_NOERR) then
-       err_msg = "Could not set chunk cache"
-       call PIO_closefile(pio_file)
-       return
     end if
 
     ! Open existing file, write data to itn
@@ -395,7 +395,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Define a new variable
-    print*, ''
+    print*, 'PIO_def_var'
     ret_val = PIO_def_var(pio_file, 'foo2222', PIO_int, (/pio_dim/), pio_var)
     if (ret_val .ne. PIO_NOERR) then
        err_msg = "Could not define variable foo2222"
@@ -403,8 +403,64 @@ Subroutine test_nc4(test_id, err_msg)
        return
     end if
 
+    ! Set the chunk cache for this variable in netCDF-4 files.
+    chunk_cache_size = 1024
+    chunk_cache_nelems = 4
+    chunk_cache_preemption = 0.2
+    print*, 'PIO_set_var_chunk_cache'
+    ret_val = PIO_set_var_chunk_cache(pio_file, pio_var, chunk_cache_size, chunk_cache_nelems, &
+         chunk_cache_preemption)
+    
+    ! Should not have worked except for netCDF-4/HDF5 iotypes.
+    if (iotype .eq. PIO_iotype_netcdf4c .and. ret_val .ne. PIO_NOERR) then
+       err_msg = "Could not set variable chunk cache"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to set variable chunk cache for pnetcdf file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to set variable chunk cache for netcdf classic file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf4p .and. ret_val .ne. PIO_NOERR) then
+       err_msg = "Could not set variable chunk cache"
+       call PIO_closefile(pio_file)
+       return
+    end if
+
+    ! Check the settings of the chunk cache for netCDF-4/HDF5 files.
+    print*, 'PIO_get_var_chunk_cache'
+    ret_val = PIO_get_var_chunk_cache(pio_file, pio_var, chunk_cache_size_in, &
+         chunk_cache_nelems_in, chunk_cache_preemption_in)
+    print*, 'PIO_get_var_chunk_cache ret_val=', ret_val
+    
+    ! Should not have worked except for netCDF-4/HDF5 iotypes.
+    if (iotype .eq. PIO_iotype_netcdf4c .or. iotype .eq. PIO_iotype_netcdf4p) then
+       if (ret_val .ne. PIO_NOERR) then
+          err_msg = "Could not set variable chunk cache"
+          call PIO_closefile(pio_file)
+       endif
+       if (chunk_cache_size_in .ne. chunk_cache_size .or. &
+            chunk_cache_nelems_in .ne. chunk_cache_nelems .or. &
+            chunk_cache_preemption_in .ne. chunk_cache_preemption) then
+          err_msg = "Incorrect variable chunk cache values!"
+          call PIO_closefile(pio_file)
+       endif
+       return
+    else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to get variable chunk cache for pnetcdf file"
+       call PIO_closefile(pio_file)
+       return
+    else if (iotype .eq. PIO_iotype_netcdf .and. ret_val .eq. PIO_NOERR) then
+       err_msg = "Did not get expected error when trying to get variable chunk cache for netcdf classic file"
+       call PIO_closefile(pio_file)
+       return
+    end if
+
     ! Try to turn on compression for this variable.
-    print*, 'PIO_def_var_deflate' 
+    print*, 'testing PIO_def_var_deflate' 
     shuffle = 0
     deflate = 1
     deflate_level = 2
@@ -431,7 +487,7 @@ Subroutine test_nc4(test_id, err_msg)
        return
     end if
 
-    print*, 'PIO_put_att'
+    print*, 'testing PIO_put_att'
     ret_val = PIO_put_att(pio_file, pio_var, "max_val", ntasks)
     if (ret_val .ne. PIO_NOERR) then
        ! Error in PIO_put_att
@@ -440,7 +496,7 @@ Subroutine test_nc4(test_id, err_msg)
        return
     end if
 
-    print*, 'PIO_put_att'
+    print*, 'testing PIO_put_att'
     ret_val = PIO_put_att(pio_file, PIO_global, "created_by", "PIO unit tests")
     if (ret_val .ne. PIO_NOERR) then
        ! Error in PIO_put_att
@@ -450,7 +506,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Check the compression settings of the variables.
-    print*, 'PIO_inq_var_deflate'
+    print*, 'testing PIO_inq_var_deflate'
     ret_val = PIO_inq_var_deflate(pio_file, pio_var, shuffle, deflate, my_deflate_level)
 
     ! Should not have worked except for netCDF-4/HDF5 serial.
@@ -485,7 +541,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Try to turn on compression for this variable.
-    print*, 'PIO_def_var_deflate'
+    print*, 'testing PIO_def_var_deflate'
     ret_val = PIO_def_var_deflate(pio_file, pio_var%varid, shuffle, deflate, &
          deflate_level_2)
 
@@ -509,7 +565,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Leave define mode
-    print*, 'PIO_enddef'
+    print*, 'testing PIO_enddef'
     ret_val = PIO_enddef(pio_file)
     if (ret_val .ne. PIO_NOERR) then
        err_msg = "Could not end define mode"
@@ -517,7 +573,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Check the compression settings of the variables.
-    print*, 'PIO_inq_var_deflate'
+    print*, 'testing PIO_inq_var_deflate'
     ret_val = PIO_inq_var_deflate(pio_file, pio_var%varid, shuffle, deflate, my_deflate_level)
 
     ! Should not have worked except for netCDF-4/HDF5 serial and netcdf-4/HDF5 parallel.
@@ -552,7 +608,7 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Write foo2
-    print*, 'PIO_write_darray'
+    print*, 'testing PIO_write_darray'
     call PIO_write_darray(pio_file, pio_var, iodesc_nCells, data_to_write, ret_val)
     if (ret_val .ne. PIO_NOERR) then
        err_msg = "Could not write data"
@@ -560,12 +616,14 @@ Subroutine test_nc4(test_id, err_msg)
     end if
 
     ! Close file
-    print*, ' PIO_closefile'
+    print*, 'testing  PIO_closefile'
     call PIO_closefile(pio_file)
 
     ! Free decomp
+    print*, 'testing  PIO_freedecomp'    
     call PIO_freedecomp(pio_iosystem, iodesc_nCells)
     call mpi_barrier(MPI_COMM_WORLD,ret_val)
     
+    print*, 'after testing  err_msg = '    , err_msg
   End Subroutine test_nc4
 end module ncdf_tests
diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index 1daa68ef7a04..ab522545a05f 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -9,6 +9,7 @@
  *
  */
 #include 
+
 #ifdef TIMING
 #include 
 #endif
@@ -164,10 +165,10 @@ main(int argc, char **argv)
     int endianness;
 
     /* Size of the var chunk cache. */
-    size_t var_cache_size;
+    PIO_Offset var_cache_size;
 
     /* Number of elements in var cache. */
-    size_t var_cache_nelems;
+    PIO_Offset var_cache_nelems;
 
     /* Var cache preemption. */    
     float var_cache_preemption;
@@ -434,18 +435,27 @@ main(int argc, char **argv)
 	    }
 
 	    /* Check setting the chunk cache for the variable. */
+	    printf("rank: %d PIOc_set_var_chunk_cache...\n", my_rank);
 	    if ((ret = PIOc_set_var_chunk_cache(ncid, 0, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
-						VAR_CACHE_PREEMPTION)) != PIO_ENOTNC4)
+						VAR_CACHE_PREEMPTION)))
 	    	ERR(ret);
 
 	    /* Check getting the chunk cache values for the variable. */
+	    printf("rank: %d PIOc_get_var_chunk_cache...\n", my_rank);	    
 	    if ((ret = PIOc_get_var_chunk_cache(ncid, 0, &var_cache_size, &var_cache_nelems,
-						&var_cache_preemption)) != PIO_ENOTNC4)
+						&var_cache_preemption)))
+	    	ERR(ret);
+	    PIO_Offset len;
+	    if ((ret = PIOc_inq_dimlen(ncid, 0, &len)))
 	    	ERR(ret);
 
 	    /* Check that we got expected values. */
-	    if (var_cache_size != VAR_CACHE_SIZE || var_cache_nelems != VAR_CACHE_NELEMS ||
-		var_cache_preemption != VAR_CACHE_PREEMPTION)
+	    printf("rank: %d var_cache_size = %d\n", my_rank, var_cache_size);	    
+	    if (var_cache_size != VAR_CACHE_SIZE)
+		ERR(ERR_AWFUL);
+	    if (var_cache_nelems != VAR_CACHE_NELEMS)
+		ERR(ERR_AWFUL);
+	    if (var_cache_preemption != VAR_CACHE_PREEMPTION)
 		ERR(ERR_AWFUL);
 	} else {
 	    /* Trying to set or inq netCDF-4 settings for non-netCDF-4

From 578919cd47830e9fb965f72cf9a08e9ea0302112 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Mon, 29 Feb 2016 17:05:21 -0500
Subject: [PATCH 41/48] got tests working with var and file chunk caches.

---
 src/clib/pio.h        |   2 -
 src/clib/pio_nc4.c    | 251 ++----------------------------------------
 tests/unit/test_nc4.c |  35 ------
 3 files changed, 7 insertions(+), 281 deletions(-)

diff --git a/src/clib/pio.h b/src/clib/pio.h
index 575578f36fe9..a6d9c8f9e731 100644
--- a/src/clib/pio.h
+++ b/src/clib/pio.h
@@ -366,8 +366,6 @@ int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
 int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
 			 int *deflatep, int *deflate_levelp);
 int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp);
-int PIOc_def_var_fletcher32(int ncid, int varid, int fletcher32);
-int PIOc_inq_var_fletcher32(int ncid, int varid, int *fletcher32p);
 int PIOc_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizesp);
 int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizesp);
 int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value);
diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index 6843891a3591..ad40f19fa0f3 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -208,195 +208,6 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
     return ierr;
 }    
 
-/**
- * @ingroup PIO_def_var
- * Turn on checksums for a variable (serial netCDF-4 only).
- * 
- * This function only applies to netCDF-4 files. When used with netCDF
- * classic files, the error PIO_ENOTNC4 will be returned.
- *
- * This function turns on the fletcher32 filter for this variable in
- * the HDF5 layer. This causes a 4-byte checksum to be written for
- * each chunk of data. These checksums are checked automatically when
- * the file is read, and an error thrown if they don't match. There
- * will be a performance penalty on read for checking the checksum.
- *
- * This function is only available for netCDF-4 Serial files. It is
- * not available for netCDF-4 parallel files.
- *
- * See the netCDF
- * documentation for details about the operation of this function.
- *
- * @param ncid the ncid of the open file.
- * @param varid the ID of the variable to set chunksizes for.
- * @param fletcher32 non-zero to turn on fletcher32 filter.
- * 
- * @return PIO_NOERR for success, otherwise an error code.
- */
-int PIOc_def_var_fletcher32(int ncid, int varid, int fletcher32)
-{
-    int ierr;
-    int msg;
-    int mpierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    char *errstr;
-
-    errstr = NULL;
-    ierr = PIO_NOERR;
-
-    if (!(file = pio_get_file_from_id(ncid)))
-	return PIO_EBADID;
-    ios = file->iosystem;
-    msg = PIO_MSG_DEF_VAR_FLETCHER32;
-
-    if (ios->async_interface && ! ios->ioproc)
-    {
-	if (ios->compmaster) 
-	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
-	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
-    }
-
-    if (ios->ioproc)
-    {
-	switch (file->iotype)
-	{
-#ifdef _NETCDF
-#ifdef _NETCDF4
-	case PIO_IOTYPE_NETCDF4P:
-	    ierr = nc_def_var_fletcher32(file->fh, varid, fletcher32);
-	    break;
-	case PIO_IOTYPE_NETCDF4C:
-	    if (ios->io_rank==0)
-	    {
-		ierr = nc_def_var_fletcher32(file->fh, varid, fletcher32);
-	    }
-	    break;
-#endif
-	case PIO_IOTYPE_NETCDF:
-	    return PIO_ENOTNC4;
-	    break;
-#endif
-#ifdef _PNETCDF
-	case PIO_IOTYPE_PNETCDF:
-	    return PIO_ENOTNC4;
-	    break;
-#endif
-	default:
-	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
-	}
-    }
-
-    /* Allocate an error string if needed. */
-    if (ierr != PIO_NOERR)
-    {
-	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
-	sprintf(errstr,"in file %s",__FILE__);
-    }
-
-    /* Check for netCDF error. */
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-
-    /* Free the error string if it was allocated. */
-    if (errstr != NULL)
-	free(errstr);
-
-    return ierr;
-}
-
-/**
- * @ingroup PIO_inq_var
- * Inquire about fletcher32 checksums for a variable.
- *
- * This function only applies to netCDF-4 files. When used with netCDF
- * classic files, the error PIO_ENOTNC4 will be returned.
- *
- * See the netCDF
- * variable documentation for details about the operation of this
- * function.
- * 
- * @param ncid the ncid of the open file.
- * @param varid the ID of the variable.
- * @param fletcher32p pointer will get zero if checksums are not in
- * use, non-zero otherwise.
- * 
- * @return PIO_NOERR for success, otherwise an error code.
- */
-int PIOc_inq_var_fletcher32(int ncid, int varid, int *fletcher32p)
-{
-    int ierr;
-    int msg;
-    int mpierr;
-    iosystem_desc_t *ios;
-    file_desc_t *file;
-    char *errstr;
-
-    errstr = NULL;
-    ierr = PIO_NOERR;
-
-    if (!(file = pio_get_file_from_id(ncid)))
-	return PIO_EBADID;
-    ios = file->iosystem;
-    msg = PIO_MSG_INQ_VAR_FLETCHER32;
-
-    if (ios->async_interface && ! ios->ioproc)
-    {
-	if (ios->compmaster) 
-	    mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
-	mpierr = MPI_Bcast(&(file->fh),1, MPI_INT, 0, ios->intercomm);
-    }
-
-    if (ios->ioproc)
-    {
-	switch (file->iotype)
-	{
-#ifdef _NETCDF
-#ifdef _NETCDF4
-	case PIO_IOTYPE_NETCDF4P:
-	    ierr = nc_inq_var_fletcher32(file->fh, varid, fletcher32p);
-	    break;
-	case PIO_IOTYPE_NETCDF4C:
-	    if (!ios->io_rank)
-		ierr = nc_inq_var_fletcher32(file->fh, varid, fletcher32p);
-	    break;
-#endif
-	case PIO_IOTYPE_NETCDF:
-	    return PIO_ENOTNC4;
-	    break;
-#endif
-#ifdef _PNETCDF
-	case PIO_IOTYPE_PNETCDF:
-	    return PIO_ENOTNC4;
-	    break;
-#endif
-	default:
-	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
-	}
-    }
-
-    /* If there is an error, allocate space for the error string. */
-    if (ierr != PIO_NOERR)
-    {
-	errstr = (char *) malloc((strlen(__FILE__) + 20)* sizeof(char));
-	sprintf(errstr,"in file %s",__FILE__);
-    }
-
-    /* Check the netCDF return code, and broadcast it to all tasks. */
-    ierr = check_netcdf(file, ierr, errstr,__LINE__);
-
-    /* Free the error stringif it was allocated. */
-    if (errstr != NULL)
-	free(errstr);
-
-    /* Broadcast results to all tasks. */
-    if (fletcher32p)
-	ierr = MPI_Bcast(fletcher32p, 1, MPI_INT, ios->ioroot, ios->my_comm);
-    
-    return ierr;
-}    
-
 /**
  * @ingroup PIO_def_var
  * Set chunksizes for a variable.
@@ -592,7 +403,7 @@ int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizes
     if (storagep)
 	ierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm);
     if (chunksizesp)
-	ierr = MPI_Bcast(chunksizesp, ndims, MPI_UNSIGNED_LONG, ios->ioroot, ios->my_comm);
+	ierr = MPI_Bcast(chunksizesp, ndims, MPI_OFFSET, ios->ioroot, ios->my_comm);
 
     return ierr;
 }
@@ -1016,7 +827,6 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep,
     int ret;
     if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
 	return ret;
-    printf("rank %d PIOc_get_chunk_cache called iotype=%d\n", my_rank, iotype);
 
     /* Since this is a property of the running HDF5 instance, not the
      * file, it's not clear if this message passing will apply. For
@@ -1038,11 +848,7 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep,
 	break;
     case PIO_IOTYPE_NETCDF4C:
 	if (!ios->io_rank)
-	{
 	    ierr = nc_get_chunk_cache(sizep, nelemsp, preemptionp);
-	    printf("rank 0 calling nc_get_chunk_cache returned *sizep = %d *nelemsp = %d *preemptionp=%g\n",
-		   *sizep, *nelemsp, *preemptionp);
-	}
 	break;
 #endif
     case PIO_IOTYPE_NETCDF:
@@ -1052,7 +858,6 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep,
 #ifdef _PNETCDF
     case PIO_IOTYPE_PNETCDF:
 	ierr = PIO_ENOTNC4;
-	printf("rank %d PIOc_get_chunk_cache ierr = %d\n", my_rank, ierr);
 	break;
 #endif
     default:
@@ -1066,27 +871,14 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep,
     else
     {
 	if (sizep)
-	{
-	    int my_size = *sizep;
-	    printf("before bcast rank %d PIOc_get_chunk_cache my_size = %d ios->ioroot %d ios->io_rank=%d\n", my_rank, my_size, ios->ioroot, ios->io_rank);
-	    if ((ierr = MPI_Bcast(&my_size, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    if ((ierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
 		ierr = PIO_EIO;
-	    printf("after bcast rank %d PIOc_get_chunk_cache my_size = %d *sizep = %d\n", my_rank, my_size, *sizep);
-	    *sizep = my_size;
-	    printf("rank %d PIOc_get_chunk_cache my_size = %d *sizep = %d\n", my_rank, my_size, *sizep);
-	}
 	if (nelemsp && !ierr)
-	{
-	    int my_nelems = *nelemsp;
-	    if ((ierr = MPI_Bcast(&my_nelems, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+	    if ((ierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
 		ierr = PIO_EIO;
-	    *nelemsp = my_nelems;
-	}
 	if (preemptionp && !ierr)
-	{
 	    if ((ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm)))
 		ierr = PIO_EIO;
-	}
     }
 
     return ierr;
@@ -1138,7 +930,6 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset ne
     int ret;
     if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
 	return ret;
-    printf("rank %d PIOc_set_var_chunk_cache called file->iotype=%d\n", my_rank, file->iotype);
 
     if (ios->async_interface && ! ios->ioproc)
     {
@@ -1158,9 +949,7 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset ne
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
 	    if (!ios->io_rank)
-	    {
 		ierr = nc_set_var_chunk_cache(file->fh, varid, size, nelems, preemption);
-	    }
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
@@ -1169,7 +958,6 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset ne
 #endif
 #ifdef _PNETCDF
 	case PIO_IOTYPE_PNETCDF:
-	    printf("rank %d PIOc_set_var_chunk_cache pnetcdf will return PIO_ENOTNC4\n", my_rank);
 	    ierr = PIO_ENOTNC4;
 	    break;
 #endif
@@ -1239,7 +1027,6 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset
     int ret;
     if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
 	return ret;
-    printf("rank %d PIOc_get_var_chunk_cache called file->iotype=%d\n", my_rank, file->iotype);
 
     /* Since this is a property of the running HDF5 instance, not the
      * file, it's not clear if this message passing will apply. For
@@ -1264,12 +1051,8 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset
 	    break;
 	case PIO_IOTYPE_NETCDF4C:
 	    if (ios->io_rank == 0)
-	    {
 		ierr = nc_get_var_chunk_cache(file->fh, varid,  (size_t *)sizep, (size_t *)nelemsp,
 					      preemptionp);
-		printf("sizep=%d\n", *sizep);
-	    }
-	    
 	    break;
 #endif
 	case PIO_IOTYPE_NETCDF:
@@ -1279,14 +1062,13 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset
 #ifdef _PNETCDF
 	case PIO_IOTYPE_PNETCDF:
 	    ierr = PIO_ENOTNC4;
-	    printf("rank %d PIOc_get_var_chunk_cache called file->iotype=%d ierr = %d\n", my_rank, file->iotype, ierr);
 	    break;
 #endif
 	default:
 	    ierr = iotype_error(file->iotype,__FILE__,__LINE__);
 	}
     }
-    printf("rank %d PIOc_get_var_chunk_cache called file->iotype=%d ierr = %d\n", my_rank, file->iotype, ierr);
+
     /* If there is an error, allocate space for the error string. */
     if (ierr != PIO_NOERR)
     {
@@ -1296,37 +1078,18 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset
 
     /* Check the netCDF return code, and broadcast it to all tasks. */
     ierr = check_netcdf(file, ierr, errstr,__LINE__);
-    printf("rank %d PIOc_get_var_chunk_cache called file->iotype=%d ierr = %d\n", my_rank, file->iotype, ierr);
+
     /* Free the error string if it was allocated. */
     if (errstr != NULL)
 	free(errstr);
 
     /* Broadcast results to all tasks. */
-    printf("sizeof(PIO_Offset)=%d, sizeof(MPI_OFFSET)=%d, sizeof(MPI_Offset)=%d, sizeof(*sizep)= %d\n",
-	   sizeof(PIO_Offset), sizeof(MPI_OFFSET), sizeof(MPI_Offset), sizeof(*sizep));
     if (sizep && !ierr)
-    {
-	int my_size = *sizep;
-	printf("rank %d sizep=%d\n", my_rank, *sizep);
-	printf("my_size = %d\n");
-	ierr = MPI_Bcast(&my_size, 1, MPI_INT, ios->ioroot, ios->my_comm);
-	printf("rank %d my_size = %d\n", my_rank, my_size);
-	*sizep = my_size;
-    }
-    printf("nelems\n");
+	ierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm);	
     if (nelemsp && !ierr)
-    {
-    	int my_nelems = *nelemsp;
-    	ierr = MPI_Bcast(&my_nelems, 1, MPI_INT, ios->ioroot, ios->my_comm);
-	printf("rank %d my_nelems = %d\n", my_rank, my_nelems);
-	*nelemsp = my_nelems;
-    }
-    printf("preemption\n");
+    	ierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm);
     if (preemptionp && !ierr)
-    {
     	ierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm);
-	printf("rank %d preemption = %g\n", my_rank, *preemptionp);	
-    }
 
     return ierr;
 }    
diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index ab522545a05f..89ea2052cf27 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -158,9 +158,6 @@ main(int argc, char **argv)
     /** The deflate level set for the variable in the netCDF-4 test file. */
     int deflate_level;
 
-    /** Non-zero if fletcher32 filter is used for variable. */
-    int fletcher32;
-
     /** Endianness of variable. */
     int endianness;
 
@@ -406,34 +403,6 @@ main(int argc, char **argv)
 		if (shuffle || deflate)
 		    ERR(ERR_AWFUL);
 
-	    /* Check that the def/inq_var_fletcher32 functions
-	     * work. Starts as off. */
-	    if ((ret = PIOc_inq_var_fletcher32(ncid, 0, &fletcher32)))
-	    	ERR(ret);
-	    if (format[fmt] == PIO_IOTYPE_NETCDF4C)
-		if (fletcher32)
-		    ERR(ERR_AWFUL);
-	    if (format[fmt] == PIO_IOTYPE_NETCDF4P)
-		if (fletcher32)
-		    ERR(ERR_AWFUL);
-
-	    /* Turn on (then off) fletcher32 filter for netCDF-4 serial. */
-	    if (format[fmt] == PIO_IOTYPE_NETCDF4C)
-	    {
-		if ((ret = PIOc_def_var_fletcher32(ncid, 0, 1)))
-		    ERR(ret);
-		if ((ret = PIOc_inq_var_fletcher32(ncid, 0, &fletcher32)))
-		    ERR(ret);
-		if (!fletcher32)
-		    ERR(ERR_AWFUL);
-		if ((ret = PIOc_def_var_fletcher32(ncid, 0, 0)))
-		    ERR(ret);
-		if ((ret = PIOc_inq_var_fletcher32(ncid, 0, &fletcher32)))
-		    ERR(ret);
-		if (fletcher32)
-		    ERR(ERR_AWFUL);
-	    }
-
 	    /* Check setting the chunk cache for the variable. */
 	    printf("rank: %d PIOc_set_var_chunk_cache...\n", my_rank);
 	    if ((ret = PIOc_set_var_chunk_cache(ncid, 0, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
@@ -467,10 +436,6 @@ main(int argc, char **argv)
 	    if ((ret = PIOc_inq_var_deflate(ncid, 0, &shuffle, &deflate, &deflate_level))
 		!= PIO_ENOTNC4)
 	    	ERR(ret);
-	    if ((ret = PIOc_def_var_fletcher32(ncid, 0, 1)) != PIO_ENOTNC4)
-		ERR(ret);
-	    if ((ret = PIOc_inq_var_fletcher32(ncid, 0, &fletcher32)) != PIO_ENOTNC4)
-	    	ERR(ret);
 	    if ((ret = PIOc_def_var_endian(ncid, 0, 1)) != PIO_ENOTNC4)
 		ERR(ret);
 	    if ((ret = PIOc_inq_var_endian(ncid, 0, &endianness)) != PIO_ENOTNC4)

From 7e0ea1926cf3e72db87a0585734d8b80f6d169a9 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Mon, 29 Feb 2016 17:10:39 -0500
Subject: [PATCH 42/48] using PIO_Offset instead of size_t in API

---
 src/clib/pio.h     |  8 ++++----
 src/clib/pio_nc4.c | 16 ++++++++--------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/clib/pio.h b/src/clib/pio.h
index a6d9c8f9e731..8c35c097c9ab 100644
--- a/src/clib/pio.h
+++ b/src/clib/pio.h
@@ -366,14 +366,14 @@ int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
 int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
 			 int *deflatep, int *deflate_levelp);
 int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp);
-int PIOc_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizesp);
-int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizesp);
+int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp);
+int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp);
 int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value);
 int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep);
 int PIOc_def_var_endian(int ncid, int varid, int endian);
 int PIOc_inq_var_endian(int ncid, int varid, int *endianp);
-int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, size_t size, size_t nelems, float preemption);
-int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep, size_t *nelemsp, float *preemptionp);
+int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset size, PIO_Offset nelems, float preemption);
+int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset *sizep, PIO_Offset *nelemsp, float *preemptionp);
 int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems,
 			     float preemption);
 int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp,
diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index ad40f19fa0f3..b79920020b17 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -233,7 +233,7 @@ int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep,
  * @return PIO_NOERR for success, otherwise an error code.
  */
 int PIOc_def_var_chunking(int ncid, int varid, int storage,
-			  const size_t *chunksizesp) 
+			  const PIO_Offset *chunksizesp) 
 {
     int ierr;
     int msg;
@@ -326,7 +326,7 @@ int PIOc_def_var_chunking(int ncid, int varid, int storage,
  * 
  * @return PIO_NOERR for success, otherwise an error code.
  */
-int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizesp)
+int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp)
 {
     int ierr;
     int msg;
@@ -720,8 +720,8 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
  * 
  * @return PIO_NOERR for success, otherwise an error code.
  */
-int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, size_t size,
-			 size_t nelems, float preemption)
+int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset size,
+			 PIO_Offset nelems, float preemption)
 {
     int ierr;
     int msg;
@@ -807,8 +807,8 @@ int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, size_t size,
  * 
  * @return PIO_NOERR for success, otherwise an error code.
  */
-int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep,
-			 size_t *nelemsp, float *preemptionp)
+int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset *sizep,
+			 PIO_Offset *nelemsp, float *preemptionp)
 {
     int ierr;
     int msg;
@@ -844,11 +844,11 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, size_t *sizep,
 #ifdef _NETCDF
 #ifdef _NETCDF4
     case PIO_IOTYPE_NETCDF4P:
-	ierr = nc_get_chunk_cache(sizep, nelemsp, preemptionp);
+	ierr = nc_get_chunk_cache((size_t *)sizep, (size_t *)nelemsp, preemptionp);
 	break;
     case PIO_IOTYPE_NETCDF4C:
 	if (!ios->io_rank)
-	    ierr = nc_get_chunk_cache(sizep, nelemsp, preemptionp);
+	    ierr = nc_get_chunk_cache((size_t *)sizep, (size_t *)nelemsp, preemptionp);
 	break;
 #endif
     case PIO_IOTYPE_NETCDF:

From 6d398bf12b2bfa214386d4a5227fa57891379b4c Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 1 Mar 2016 10:49:10 -0500
Subject: [PATCH 43/48] removed printf statement

---
 src/clib/pio_nc.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/clib/pio_nc.c b/src/clib/pio_nc.c
index fdaddb4dc18f..f511475a4f6c 100644
--- a/src/clib/pio_nc.c
+++ b/src/clib/pio_nc.c
@@ -3483,7 +3483,6 @@ int PIOc_inq_dimlen (int ncid, int dimid, PIO_Offset *lenp)
     sprintf(errstr,"in file %s",__FILE__);
   }
   ierr = check_netcdf(file, ierr, errstr,__LINE__);
-  printf("sizeof(MPI_OFFSET)=%d sizeof(PIO_Offset)=%d sizeof(*lenp)=%d \n", sizeof(MPI_OFFSET), sizeof(PIO_Offset), sizeof(*lenp));
     mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm);
   if(errstr != NULL) free(errstr);
   return ierr;

From f738fadb9dd118f183a0c8c1efd6a07b4a1ebbcd Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 1 Mar 2016 10:51:45 -0500
Subject: [PATCH 44/48] fixes based on Jim's comments to pull request

---
 src/clib/pio_nc4.c        | 15 ---------------
 tests/unit/ncdf_tests.F90 |  2 +-
 2 files changed, 1 insertion(+), 16 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index b79920020b17..d6ce46bddf11 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -823,11 +823,6 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset *sizep
     if(ios == NULL)
 	return PIO_EBADID;
 
-    int my_rank;
-    int ret;
-    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
-	return ret;
-
     /* Since this is a property of the running HDF5 instance, not the
      * file, it's not clear if this message passing will apply. For
      * now, comment it out. EJH */
@@ -926,11 +921,6 @@ int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset ne
     ios = file->iosystem;
     msg = PIO_MSG_SET_VAR_CHUNK_CACHE;
 
-    int my_rank;
-    int ret;
-    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
-	return ret;
-
     if (ios->async_interface && ! ios->ioproc)
     {
 	if (ios->compmaster) 
@@ -1023,11 +1013,6 @@ int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset
 	return PIO_EBADID;
     ios = file->iosystem;
 
-    int my_rank;
-    int ret;
-    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
-	return ret;
-
     /* Since this is a property of the running HDF5 instance, not the
      * file, it's not clear if this message passing will apply. For
      * now, comment it out. EJH */
diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
index 89b5fabbec95..8247a0e30684 100644
--- a/tests/unit/ncdf_tests.F90
+++ b/tests/unit/ncdf_tests.F90
@@ -304,7 +304,7 @@ Subroutine test_nc4(test_id, err_msg)
     compdof = 2*my_rank+(/1,2/)  ! Where in the global array each task writes
     data_to_write = 1+my_rank
 
-    print*, 'PIO_initdecom'
+    print*, 'PIO_initdecomp'
     call PIO_initdecomp(pio_iosystem, PIO_int, dims, compdof, iodesc_nCells)
 
     filename = fnames(test_id)

From 2fd4a49b2fc8fb415d19acbde02173c2703dec4d Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 1 Mar 2016 11:22:10 -0500
Subject: [PATCH 45/48] fixes based on Jim's comments to pull request

---
 src/clib/pio_nc4.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index d6ce46bddf11..654d9169cc41 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -752,7 +752,7 @@ int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset size,
 	ierr = nc_set_chunk_cache(size, nelems, preemption);
 	break;
     case PIO_IOTYPE_NETCDF4C:
-	if (!io_rank)
+	if (!ios->io_rank)
 	    ierr = nc_set_chunk_cache(size, nelems, preemption);
 	break;
 #endif
@@ -769,9 +769,8 @@ int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset size,
 	ierr = iotype_error(file->iotype,__FILE__,__LINE__);
     }
 
-    /* Check for netCDF error. */
-    if (ierr)
-	MPI_Bcast(&ierr, 1, MPI_INTEGER, ios->ioroot, ios->my_comm);    
+    /* Propogate error code to all processes. */
+    MPI_Bcast(&ierr, 1, MPI_INTEGER, ios->ioroot, ios->my_comm);    
 
     return ierr;
 }    
@@ -861,9 +860,8 @@ int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset *sizep
 
 
     /* Check for netCDF error. */
-    if (ierr)
-	MPI_Bcast(&ierr, 1, MPI_INTEGER, ios->ioroot, ios->my_comm);
-    else
+    MPI_Bcast(&ierr, 1, MPI_INTEGER, ios->ioroot, ios->my_comm);
+    if (!ierr)
     {
 	if (sizep)
 	    if ((ierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))

From 9751a59c67a4f98be20d8732932362bbfe325b80 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 1 Mar 2016 12:30:27 -0500
Subject: [PATCH 46/48] fixed handling of errors in set_chunk_cache functions

---
 tests/unit/test_nc4.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index 89ea2052cf27..55b539e938fd 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -286,7 +286,23 @@ main(int argc, char **argv)
 	    printf("rank: %d Setting chunk cache for file %s with format %d...\n",
 		   my_rank, filename[fmt], format[fmt]);
 
+	/* Try to set the chunk cache with invalid preemption to check error handling. */
+	chunk_cache_preemption = 50.0;
+	ret = PIOc_set_chunk_cache(iosysid, format[fmt], my_rank, chunk_cache_size,
+				   chunk_cache_nelems, chunk_cache_preemption);
+	if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P)
+	{
+	    if (ret != NC_EINVAL)
+		ERR(ERR_AWFUL);
+	}
+	else
+	{
+	    if (ret != NC_ENOTNC4)
+		ERR(ERR_AWFUL);
+	}
+
 	/* Try to set the chunk cache. */
+	chunk_cache_preemption = 0.5;
 	ret = PIOc_set_chunk_cache(iosysid, format[fmt], my_rank, chunk_cache_size,
 				   chunk_cache_nelems, chunk_cache_preemption);
 

From 70e680aeb00cc83299d5bdde569adb850e17848b Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Tue, 1 Mar 2016 12:38:00 -0500
Subject: [PATCH 47/48] fixed handling of errors in set_chunk_cache functions

---
 src/clib/pio.h            |  4 ++--
 src/clib/pio_nc4.c        |  6 ++----
 src/flib/pio_nf.F90       | 16 ++++++----------
 tests/unit/ncdf_tests.F90 |  4 ++--
 tests/unit/test_nc4.c     | 10 +++++-----
 5 files changed, 17 insertions(+), 23 deletions(-)

diff --git a/src/clib/pio.h b/src/clib/pio.h
index 8c35c097c9ab..5b8446bf21ef 100644
--- a/src/clib/pio.h
+++ b/src/clib/pio.h
@@ -372,8 +372,8 @@ int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value);
 int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep);
 int PIOc_def_var_endian(int ncid, int varid, int endian);
 int PIOc_inq_var_endian(int ncid, int varid, int *endianp);
-int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset size, PIO_Offset nelems, float preemption);
-int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset *sizep, PIO_Offset *nelemsp, float *preemptionp);
+int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems, float preemption);
+int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nelemsp, float *preemptionp);
 int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems,
 			     float preemption);
 int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp,
diff --git a/src/clib/pio_nc4.c b/src/clib/pio_nc4.c
index 654d9169cc41..735e3755a99d 100644
--- a/src/clib/pio_nc4.c
+++ b/src/clib/pio_nc4.c
@@ -713,14 +713,13 @@ int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
  * function.
  * 
  * @param iotype the iotype of files to be created or opened.
- * @param io_rank the rank of the calling process.
  * @param size size of file cache.
  * @param nelems number of elements in file cache.
  * @param preemption preemption setting for file cache.
  * 
  * @return PIO_NOERR for success, otherwise an error code.
  */
-int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset size,
+int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size,
 			 PIO_Offset nelems, float preemption)
 {
     int ierr;
@@ -799,14 +798,13 @@ int PIOc_set_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset size,
  * performance check chunking against access patterns.
  *
  * @param iotype the iotype of files to be created or opened.
- `* @param io_rank the rank of the calling process.
  * @param sizep gets the size of file cache.
  * @param nelemsp gets the number of elements in file cache.
  * @param preemptionp gets the preemption setting for file cache.
  * 
  * @return PIO_NOERR for success, otherwise an error code.
  */
-int PIOc_get_chunk_cache(int iosysid, int iotype, int io_rank, PIO_Offset *sizep,
+int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep,
 			 PIO_Offset *nelemsp, float *preemptionp)
 {
     int ierr;
diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90
index 3f646ace99bc..7d882c334ec1 100644
--- a/src/flib/pio_nf.F90
+++ b/src/flib/pio_nf.F90
@@ -1604,28 +1604,26 @@ end function def_var_chunking
 !! @ingroup PIO_set_chunk_cache
 !! @brief Changes chunk cache settings for netCDF-4/HDF5 files created after this call.
 !<
-  integer function set_chunk_cache(iosysid, iotype, io_rank, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+  integer function set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     integer, intent(in) :: iosysid
     integer, intent(in) :: iotype 
-    integer, intent(in) :: io_rank
     integer(kind=PIO_OFFSET_KIND), intent(in) :: chunk_cache_size
     integer(kind=PIO_OFFSET_KIND), intent(in) :: chunk_cache_nelems
     real, intent(in) :: chunk_cache_preemption
 
     interface
-       integer (C_INT) function PIOc_set_chunk_cache(iosysid, iotype, io_rank, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
+       integer (C_INT) function PIOc_set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
             bind(c,name="PIOc_set_chunk_cache")
          use iso_c_binding
          integer(c_int), value :: iosysid
          integer(c_int), value :: iotype
-         integer(c_int), value :: io_rank
          integer(c_size_t), value :: chunk_cache_size
          integer(c_size_t), value :: chunk_cache_nelems
          real(c_float), value :: chunk_cache_preemption
        end function PIOc_set_chunk_cache
     end interface
 
-    ierr = PIOc_set_chunk_cache(iosysid, iotype, io_rank, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+    ierr = PIOc_set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
   end function set_chunk_cache
 
 !> 
@@ -1633,28 +1631,26 @@ end function set_chunk_cache
 !! @ingroup PIO_set_chunk_cache  
 !! @brief Gets current settings for chunk cache (only relevant for netCDF4/HDF5 files.)
 !<
-  integer function get_chunk_cache(iosysid, iotype, io_rank, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+  integer function get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     integer, intent(in) :: iosysid
     integer, intent(in) :: iotype
-    integer, intent(in) :: io_rank
     integer(kind=PIO_OFFSET_KIND), intent(out) :: chunk_cache_size
     integer(kind=PIO_OFFSET_KIND), intent(out) :: chunk_cache_nelems
     real, intent(out) :: chunk_cache_preemption
 
     interface
-       integer (C_INT) function PIOc_get_chunk_cache(iosysid, iotype, io_rank, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
+       integer (C_INT) function PIOc_get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
             bind(c,name="PIOc_get_chunk_cache")
          use iso_c_binding
          integer(c_int), value :: iosysid
          integer(c_int), value :: iotype
-         integer(c_int), value :: io_rank
          integer(c_size_t) :: chunk_cache_size
          integer(c_size_t) :: chunk_cache_nelems
          real(c_float) :: chunk_cache_preemption
        end function PIOc_get_chunk_cache
     end interface
 
-    ierr = PIOc_get_chunk_cache(iosysid, iotype, io_rank, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+    ierr = PIOc_get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
   end function get_chunk_cache
 
 !> 
diff --git a/tests/unit/ncdf_tests.F90 b/tests/unit/ncdf_tests.F90
index 8247a0e30684..55459012de0d 100644
--- a/tests/unit/ncdf_tests.F90
+++ b/tests/unit/ncdf_tests.F90
@@ -315,7 +315,7 @@ Subroutine test_nc4(test_id, err_msg)
     chunk_cache_nelems = 3
     chunk_cache_preemption = 0.1
     print*, 'PIO_set_chunk_cache'
-    ret_val = PIO_set_chunk_cache(pio_iosystem%iosysid, iotype, my_rank, chunk_cache_size, &
+    ret_val = PIO_set_chunk_cache(pio_iosystem%iosysid, iotype, chunk_cache_size, &
          chunk_cache_nelems, chunk_cache_preemption)
     
     ! Should not have worked except for netCDF-4/HDF5 iotypes.
@@ -339,7 +339,7 @@ Subroutine test_nc4(test_id, err_msg)
 
     ! Check the settings of the chunk cache for netCDF-4/HDF5 files.
     print*, 'testing PIO_get_chunk_cache'
-    ret_val = PIO_get_chunk_cache(pio_iosystem%iosysid, iotype, my_rank, chunk_cache_size_in, &
+    ret_val = PIO_get_chunk_cache(pio_iosystem%iosysid, iotype, chunk_cache_size_in, &
          chunk_cache_nelems_in, chunk_cache_preemption_in)
     print*, 'PIO_get_chunk_cache returned ', chunk_cache_size_in, &
          chunk_cache_nelems_in, chunk_cache_preemption_in
diff --git a/tests/unit/test_nc4.c b/tests/unit/test_nc4.c
index 55b539e938fd..b748f7a420f5 100644
--- a/tests/unit/test_nc4.c
+++ b/tests/unit/test_nc4.c
@@ -288,7 +288,7 @@ main(int argc, char **argv)
 
 	/* Try to set the chunk cache with invalid preemption to check error handling. */
 	chunk_cache_preemption = 50.0;
-	ret = PIOc_set_chunk_cache(iosysid, format[fmt], my_rank, chunk_cache_size,
+	ret = PIOc_set_chunk_cache(iosysid, format[fmt], chunk_cache_size,
 				   chunk_cache_nelems, chunk_cache_preemption);
 	if (format[fmt] == PIO_IOTYPE_NETCDF4C || format[fmt] == PIO_IOTYPE_NETCDF4P)
 	{
@@ -303,7 +303,7 @@ main(int argc, char **argv)
 
 	/* Try to set the chunk cache. */
 	chunk_cache_preemption = 0.5;
-	ret = PIOc_set_chunk_cache(iosysid, format[fmt], my_rank, chunk_cache_size,
+	ret = PIOc_set_chunk_cache(iosysid, format[fmt], chunk_cache_size,
 				   chunk_cache_nelems, chunk_cache_preemption);
 
 	/* Should only have worked for netCDF-4 iotypes. */
@@ -319,7 +319,7 @@ main(int argc, char **argv)
 	}
 
 	/* Now check the chunk cache. */
-	ret = PIOc_get_chunk_cache(iosysid, format[fmt], my_rank, &chunk_cache_size_in,
+	ret = PIOc_get_chunk_cache(iosysid, format[fmt], &chunk_cache_size_in,
 				   &chunk_cache_nelems_in, &chunk_cache_preemption_in);
 
 	/* Should only have worked for netCDF-4 iotypes. */
@@ -462,10 +462,10 @@ main(int argc, char **argv)
 	    if ((ret = PIOc_get_var_chunk_cache(ncid, 0, &var_cache_size, &var_cache_nelems,
 						&var_cache_preemption)) != PIO_ENOTNC4)
 		ERR(ret);
-	    if ((ret = PIOc_set_chunk_cache(iosysid, format[fmt], my_rank, chunk_cache_size, chunk_cache_nelems,
+	    if ((ret = PIOc_set_chunk_cache(iosysid, format[fmt], chunk_cache_size, chunk_cache_nelems,
 	    				    chunk_cache_preemption)) != PIO_ENOTNC4)
 	    	ERR(ret);
-	    if ((ret = PIOc_get_chunk_cache(iosysid, format[fmt], my_rank, &chunk_cache_size,
+	    if ((ret = PIOc_get_chunk_cache(iosysid, format[fmt], &chunk_cache_size,
 	    				    &chunk_cache_nelems, &chunk_cache_preemption)) != PIO_ENOTNC4)
 	    	ERR(ret);
 	}	    

From 88a6eb68f858f94becbe287077e13556f43a44d6 Mon Sep 17 00:00:00 2001
From: Ed Hartnett 
Date: Mon, 7 Mar 2016 10:02:00 -0500
Subject: [PATCH 48/48] shortened lines so that NAG compiler will not complain

---
 src/flib/pio_nf.F90 | 58 +++++++++++++++++++++++++++++----------------
 1 file changed, 38 insertions(+), 20 deletions(-)

diff --git a/src/flib/pio_nf.F90 b/src/flib/pio_nf.F90
index 7d882c334ec1..5c798f834ec8 100644
--- a/src/flib/pio_nf.F90
+++ b/src/flib/pio_nf.F90
@@ -1531,7 +1531,8 @@ end function def_var_md_id
 !! @ingroup PIO_def_var_deflate
 !! @brief Changes compression settings for a netCDF-4/HDF5 variable.
 !<
-  integer function def_var_deflate_id(file, varid, shuffle, deflate, deflate_level) result(ierr)
+  integer function def_var_deflate_id(file, varid, shuffle, deflate, deflate_level) &
+       result(ierr)
     type (File_desc_t), intent(in)  :: file
     integer, intent(in) :: varid
     integer, intent(in) :: shuffle
@@ -1539,8 +1540,8 @@ integer function def_var_deflate_id(file, varid, shuffle, deflate, deflate_level
     integer, intent(in) :: deflate_level
 
     interface
-       integer (C_INT) function PIOc_def_var_deflate(ncid, varid, shuffle, deflate, deflate_level) &
-            bind(c,name="PIOc_def_var_deflate")
+       integer (C_INT) function PIOc_def_var_deflate(ncid, varid, shuffle, deflate, &
+            deflate_level) bind(c,name="PIOc_def_var_deflate")
          use iso_c_binding
          integer(c_int), value :: ncid
          integer(c_int), value :: varid
@@ -1558,7 +1559,8 @@ end function def_var_deflate_id
 !! @ingroup PIO_def_var_deflate
 !! @brief Changes compression settings for a netCDF-4/HDF5 variable.
 !<
-  integer function def_var_deflate_desc(file, vardesc, shuffle, deflate, deflate_level) result(ierr)
+  integer function def_var_deflate_desc(file, vardesc, shuffle, deflate, deflate_level) &
+       result(ierr)
     type (File_desc_t), intent(in)  :: file
     type (var_desc_t), intent(in) :: vardesc
     integer, intent(in) :: shuffle
@@ -1604,7 +1606,8 @@ end function def_var_chunking
 !! @ingroup PIO_set_chunk_cache
 !! @brief Changes chunk cache settings for netCDF-4/HDF5 files created after this call.
 !<
-  integer function set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+  integer function set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, &
+       chunk_cache_preemption) result(ierr)
     integer, intent(in) :: iosysid
     integer, intent(in) :: iotype 
     integer(kind=PIO_OFFSET_KIND), intent(in) :: chunk_cache_size
@@ -1612,7 +1615,8 @@ integer function set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_
     real, intent(in) :: chunk_cache_preemption
 
     interface
-       integer (C_INT) function PIOc_set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
+       integer (C_INT) function PIOc_set_chunk_cache(iosysid, iotype, chunk_cache_size, &
+            chunk_cache_nelems, chunk_cache_preemption) &
             bind(c,name="PIOc_set_chunk_cache")
          use iso_c_binding
          integer(c_int), value :: iosysid
@@ -1623,7 +1627,8 @@ integer (C_INT) function PIOc_set_chunk_cache(iosysid, iotype, chunk_cache_size,
        end function PIOc_set_chunk_cache
     end interface
 
-    ierr = PIOc_set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+    ierr = PIOc_set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, &
+         chunk_cache_preemption)
   end function set_chunk_cache
 
 !> 
@@ -1631,7 +1636,8 @@ end function set_chunk_cache
 !! @ingroup PIO_set_chunk_cache  
 !! @brief Gets current settings for chunk cache (only relevant for netCDF4/HDF5 files.)
 !<
-  integer function get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+  integer function get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, &
+       chunk_cache_preemption) result(ierr)
     integer, intent(in) :: iosysid
     integer, intent(in) :: iotype
     integer(kind=PIO_OFFSET_KIND), intent(out) :: chunk_cache_size
@@ -1639,7 +1645,8 @@ integer function get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_
     real, intent(out) :: chunk_cache_preemption
 
     interface
-       integer (C_INT) function PIOc_get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
+       integer (C_INT) function PIOc_get_chunk_cache(iosysid, iotype, chunk_cache_size, &
+            chunk_cache_nelems, chunk_cache_preemption) &
             bind(c,name="PIOc_get_chunk_cache")
          use iso_c_binding
          integer(c_int), value :: iosysid
@@ -1650,7 +1657,8 @@ integer (C_INT) function PIOc_get_chunk_cache(iosysid, iotype, chunk_cache_size,
        end function PIOc_get_chunk_cache
     end interface
 
-    ierr = PIOc_get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+    ierr = PIOc_get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, &
+         chunk_cache_preemption)
   end function get_chunk_cache
 
 !> 
@@ -1658,7 +1666,8 @@ end function get_chunk_cache
 !! @ingroup PIO_set_chunk_cache
 !! @brief Changes chunk cache settings for a variable in a netCDF-4/HDF5 file.
 !<
-  integer function set_var_chunk_cache_id(file, varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+  integer function set_var_chunk_cache_id(file, varid, chunk_cache_size, &
+       chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     type (File_desc_t), intent(in)  :: file
     integer, intent(in) :: varid
     integer(PIO_OFFSET_KIND), intent(in) :: chunk_cache_size
@@ -1666,7 +1675,8 @@ integer function set_var_chunk_cache_id(file, varid, chunk_cache_size, chunk_cac
     real, intent(in) :: chunk_cache_preemption
 
     interface
-       integer (C_INT) function PIOc_set_var_chunk_cache(ncid, varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
+       integer (C_INT) function PIOc_set_var_chunk_cache(ncid, varid, chunk_cache_size, &
+            chunk_cache_nelems, chunk_cache_preemption) &
             bind(c,name="PIOc_set_var_chunk_cache")
          use iso_c_binding
          integer(c_int), value :: ncid
@@ -1677,7 +1687,8 @@ integer (C_INT) function PIOc_set_var_chunk_cache(ncid, varid, chunk_cache_size,
        end function PIOc_set_var_chunk_cache
     end interface
 
-    ierr = PIOc_set_var_chunk_cache(file%fh, varid-1, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+    ierr = PIOc_set_var_chunk_cache(file%fh, varid-1, chunk_cache_size, &
+         chunk_cache_nelems, chunk_cache_preemption)
   end function set_var_chunk_cache_id
 
   !> 
@@ -1685,14 +1696,16 @@ end function set_var_chunk_cache_id
 !! @ingroup PIO_set_var_chunk_cache
 !! @brief Changes chunk cacne for a variable.
 !<
-  integer function set_var_chunk_cache_desc(file, vardesc, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+  integer function set_var_chunk_cache_desc(file, vardesc, chunk_cache_size, &
+       chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     type (File_desc_t), intent(in)  :: file
     type (var_desc_t), intent(in) :: vardesc
     integer(PIO_OFFSET_KIND), intent(in) :: chunk_cache_size
     integer(PIO_OFFSET_KIND), intent(in) :: chunk_cache_nelems
     real, intent(in) :: chunk_cache_preemption
 
-    ierr = set_var_chunk_cache_id(file, vardesc%varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+    ierr = set_var_chunk_cache_id(file, vardesc%varid, chunk_cache_size, &
+         chunk_cache_nelems, chunk_cache_preemption)
   end function set_var_chunk_cache_desc
 
 !> 
@@ -1700,14 +1713,16 @@ end function set_var_chunk_cache_desc
 !! @ingroup PIO_get_var_chunk_cache
 !! @brief Get the chunk cache settings for a variable.
 !<
-  integer function get_var_chunk_cache_desc(file, vardesc, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+  integer function get_var_chunk_cache_desc(file, vardesc, chunk_cache_size, &
+       chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     type (File_desc_t), intent(in)  :: file
     type (var_desc_t), intent(in) :: vardesc
     integer(PIO_OFFSET_KIND), intent(out) :: chunk_cache_size
     integer(PIO_OFFSET_KIND), intent(out) :: chunk_cache_nelems
     real, intent(out) :: chunk_cache_preemption
 
-    ierr = get_var_chunk_cache_id(file, vardesc%varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+    ierr = get_var_chunk_cache_id(file, vardesc%varid, chunk_cache_size, &
+         chunk_cache_nelems, chunk_cache_preemption)
   end function get_var_chunk_cache_desc
 
 !> 
@@ -1715,7 +1730,8 @@ end function get_var_chunk_cache_desc
 !! @ingroup PIO_get_var_chunk_cache
 !! @brief Get the chunk cache settings for a variable.
 !<
-  integer function get_var_chunk_cache_id(file, varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) result(ierr)
+  integer function get_var_chunk_cache_id(file, varid, chunk_cache_size, &
+       chunk_cache_nelems, chunk_cache_preemption) result(ierr)
     type (File_desc_t), intent(in)  :: file
     integer, intent(in) :: varid
     integer(PIO_OFFSET_KIND), intent(out) :: chunk_cache_size
@@ -1723,7 +1739,8 @@ integer function get_var_chunk_cache_id(file, varid, chunk_cache_size, chunk_cac
     real, intent(out) :: chunk_cache_preemption
 
     interface
-       integer (C_INT) function PIOc_get_var_chunk_cache(ncid, varid, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
+       integer (C_INT) function PIOc_get_var_chunk_cache(ncid, varid, &
+            chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption) &
             bind(c,name="PIOc_get_var_chunk_cache")
          use iso_c_binding
          integer(c_int), value :: ncid
@@ -1734,7 +1751,8 @@ integer (C_INT) function PIOc_get_var_chunk_cache(ncid, varid, chunk_cache_size,
        end function PIOc_get_var_chunk_cache
     end interface
 
-    ierr = PIOc_get_var_chunk_cache(file%fh, varid-1, chunk_cache_size, chunk_cache_nelems, chunk_cache_preemption)
+    ierr = PIOc_get_var_chunk_cache(file%fh, varid-1, chunk_cache_size, &
+         chunk_cache_nelems, chunk_cache_preemption)
   end function get_var_chunk_cache_id
 
 end module pio_nf