Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow user to turn on zlib, shuffle, and/or fletcher32 filters with parallel I/O for HDF5-1.10.2+ #1582

Merged
merged 29 commits into from
Jan 13, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
3e00967
allow parallel writes to use zlib
edwardhartnett Dec 19, 2019
cde58d2
allowing parallel write of gzipped data, plus added test
edwardhartnett Dec 19, 2019
2a415db
stop running tst_mode test
edwardhartnett Dec 20, 2019
d534b12
adding another zlib parallel I/O test
edwardhartnett Dec 20, 2019
6952eb7
documentation updates
edwardhartnett Dec 20, 2019
f86c0fb
now check that HDF5 version supports parallel zlib
edwardhartnett Dec 20, 2019
6d2d92d
added new tests to cmake, also additional test development
edwardhartnett Dec 20, 2019
663540b
adjusted tst_parallel_zlib to HDF5 version flag
edwardhartnett Dec 20, 2019
0c6641e
adjusted tst_parallel_zlib2 to HDF5 version flag
edwardhartnett Dec 20, 2019
f980743
now also can set fletcher32 in parallel I/O writes
edwardhartnett Dec 20, 2019
4b7f839
swtich to collective access when filters are applied
edwardhartnett Dec 20, 2019
2136063
better documentation
edwardhartnett Dec 20, 2019
8681b0d
more documentaiton
edwardhartnett Dec 20, 2019
8f8dfac
now prevent attempt to change to independent access when deflate, shu…
edwardhartnett Dec 20, 2019
accb83a
even more documentation updates
edwardhartnett Dec 20, 2019
b9b5a8f
fixed include issue
edwardhartnett Dec 20, 2019
a06df0e
fixing for non-parallel builds
edwardhartnett Dec 20, 2019
32a776b
getting parallel I/O with zlib working in cmake builds
edwardhartnett Dec 20, 2019
995cfda
merged master
edwardhartnett Dec 20, 2019
e6ded1a
more testing
edwardhartnett Dec 20, 2019
8e660e1
more tests
edwardhartnett Dec 20, 2019
9092124
more testing
edwardhartnett Dec 20, 2019
5227054
more test development
edwardhartnett Dec 20, 2019
c16e7ef
more test development
edwardhartnett Dec 20, 2019
680e44f
changed name of macro
edwardhartnett Dec 20, 2019
9ded9b1
restored tst_mode tests
edwardhartnett Dec 20, 2019
61477db
fixed comments
edwardhartnett Dec 20, 2019
c4a8671
fixed comments
edwardhartnett Dec 20, 2019
af0b28f
removed unneeded delete
edwardhartnett Dec 20, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -743,6 +743,9 @@ IF(USE_HDF5 OR ENABLE_NETCDF_4)
SET(HDF5_CC h5cc)
ENDIF()

# Check to see if this is hdf5-1.10.2 or later.
CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5DOread_chunk "" HDF5_SUPPORTS_PAR_FILTERS)

SET(H5_USE_16_API 1)
OPTION(NC_ENABLE_HDF_16_API "Enable HDF5 1.6.x Compatibility(Required)" ON)
IF(NOT NC_ENABLE_HDF_16_API)
Expand Down
11 changes: 9 additions & 2 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -1038,7 +1038,7 @@ if test "x$enable_hdf5" = xyes; then

# H5Pset_fapl_mpiposix and H5Pget_fapl_mpiposix have been removed since HDF5 1.8.12.
# Use H5Pset_fapl_mpio and H5Pget_fapl_mpio, instead.
AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5free_memory H5resize_memory H5allocate_memory H5Pset_libver_bounds H5Pset_all_coll_metadata_ops H5Z_SZIP])
AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5free_memory H5resize_memory H5allocate_memory H5Pset_libver_bounds H5Pset_all_coll_metadata_ops H5Z_SZIP H5DOread_chunk])

# Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0)
if test "x$ac_cv_func_H5Pset_all_coll_metadata_ops" = xyes; then
Expand All @@ -1053,7 +1053,14 @@ if test "x$enable_hdf5" = xyes; then
AC_MSG_CHECKING([whether parallel io is enabled in hdf5])
AC_MSG_RESULT([$hdf5_parallel])

# Check to see if we need to search for and link against szlib.
# Check to see if HDF5 library is 1.10.2 or greater. If so, allows parallel_zip.
if test "x$ac_cv_func_H5DOread_chunk" = xyes; then
AC_DEFINE([HDF5_SUPPORTS_PAR_FILTERS], [1], [if true, HDF5 is at least version 1.10.2 and allows parallel I/O with zip])
fi
AC_MSG_CHECKING([whether HDF5 is version 1.10.2 or greater])
AC_MSG_RESULT([$ac_cv_func_H5DOread_chunk])

# Check to see if we need to search for and link against szlib.
if test "x$ac_cv_func_H5Z_SZIP" = xyes; then
AC_SEARCH_LIBS([SZ_BufftoBuffCompress], [szip sz], [],
[AC_MSG_ERROR([libhdf5 installed with szip support, but cannot find or link to the szip library.])])
Expand Down
8 changes: 4 additions & 4 deletions include/netcdf_par.h
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,15 @@

#include <mpi.h>

#if defined(__cplusplus)
extern "C" {
#endif

/** Use with nc_var_par_access() to set parallel access to independent. */
#define NC_INDEPENDENT 0
/** Use with nc_var_par_access() to set parallel access to collective. */
#define NC_COLLECTIVE 1

#if defined(__cplusplus)
extern "C" {
#endif

/* Create a file and enable parallel I/O. */
EXTERNL int
nc_create_par(const char *path, int cmode, MPI_Comm comm, MPI_Info info,
Expand Down
10 changes: 9 additions & 1 deletion libdispatch/dparallel.c
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,13 @@ To obtain a good I/O performance, users are recommended to use collective mode.
In addition, switching between collective and independent I/O mode can be
expensive.

In netcdf-c-4.7.4 or later, using hdf5-1.10.2 or later, the zlib and
fletcher32 filters may be used when writing data with parallel
I/O. The use of these filters require collective access. Turning on
the zlib (deflate) or fletcher32 filter for a variable will
automatically set its access to collective. Attempts to set access to
independent will return ::NC_EINVAL.

\param ncid NetCDF or group ID, from a previous call to nc_open_par(),
nc_create_par(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
Expand All @@ -317,7 +324,8 @@ nc_inq_ncid().
\returns ::NC_EBADID Invalid ncid passed.
\returns ::NC_ENOTVAR Invalid varid passed.
\returns ::NC_ENOPAR File was not opened with nc_open_par/nc_create_var.
\returns ::NC_EINVAL Invalid par_access specified.
\returns ::NC_EINVAL Invalid par_access specified, or attempt to set
filtered variable to independent access.

<h1>Example</h1>

Expand Down
28 changes: 20 additions & 8 deletions libdispatch/dvar.c
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
}

/**
Set the compression settings for a netCDF-4/HDF5 variable.
Set the zlib compression settings for a netCDF-4/HDF5 variable.

This function must be called after nc_def_var and before nc_enddef
or any functions which writes data to the file.
Expand All @@ -324,15 +324,22 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)

If this function is called on a scalar variable, it is ignored.

@note Parallel I/O reads work with compressed data. Parallel I/O
writes work with compressed data in netcdf-c-4.7.4 and later
releases, using hdf5-1.10.2 and later releases. Using the zlib,
shuffle (or any other) filter requires that collective access be
used with the variable. Turning on deflate and/or shuffle for a
variable in a file opened for parallel I/O will automatically
switch the access for that variable to collective access.

@param ncid NetCDF or group ID, from a previous call to nc_open(),
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
@param varid Variable ID
@param shuffle True to turn on the shuffle filter. The shuffle
filter can assist with the compression of integer data by changing
the byte order in the data stream. It makes no sense to use the
shuffle filter without setting a deflate level, or to use shuffle
on non-integer data.
filter can assist with the compression of data by changing the byte
order in the data stream. It makes no sense to use the shuffle
filter without setting a deflate level.
@param deflate True to turn on deflation for this variable.
@param deflate_level A number between 0 (no compression) and 9
(maximum compression).
Expand All @@ -347,11 +354,8 @@ nc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value)
@return ::NC_ELATEDEF Too late to change settings for this variable.
@return ::NC_ENOTINDEFINE Not in define mode.
@return ::NC_EPERM File is read only.
@return ::NC_EMAXDIMS Classic model file exceeds ::NC_MAX_VAR_DIMS.
@return ::NC_ESTRICTNC3 Attempting to create netCDF-4 type var in
classic model file
@return ::NC_EBADTYPE Bad type.
@return ::NC_ENOMEM Out of memory.
@return ::NC_EHDFERR Error returned by HDF5 layer.
@return ::NC_EINVAL Invalid input. Deflate can't be set unless
variable storage is NC_CHUNK.
Expand Down Expand Up @@ -422,6 +426,14 @@ nc_def_var_deflate(int ncid, int varid, int shuffle, int deflate, int deflate_le
data, with default chunksizes. Use nc_def_var_chunking() to tune
performance with user-defined chunksizes.

@note Parallel I/O reads work with fletcher32 encoded
data. Parallel I/O writes work with fletcher32 in netcdf-c-4.7.4
and later releases, using hdf5-1.10.2 and later releases. Using the
fletcher32 (or any) filter requires that collective access be used
with the variable. Turning on fletcher32 for a variable in a file
opened for parallel I/O will automatically switch the access for
that variable to collective access.

@param ncid NetCDF or group ID, from a previous call to nc_open(),
nc_create(), nc_def_grp(), or associated inquiry functions such as
nc_inq_ncid().
Expand Down
20 changes: 17 additions & 3 deletions libhdf5/hdf5var.c
Original file line number Diff line number Diff line change
Expand Up @@ -661,10 +661,13 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
return NC_ENOTVAR;
assert(var && var->hdr.id == varid);

/* Can't turn on parallel and deflate/fletcher32/szip/shuffle (for now). */
/* Can't turn on parallel and deflate/fletcher32/szip/shuffle
* before HDF5 1.10.2. */
#ifndef HDF5_SUPPORTS_PAR_FILTERS
if (h5->parallel == NC_TRUE)
if (deflate || fletcher32 || shuffle)
return NC_EINVAL;
#endif

/* If the HDF5 dataset has already been created, then it is too
* late to set all the extra stuff. */
Expand All @@ -687,8 +690,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
if (!var->ndims)
return NC_NOERR;

/* Well, if we couldn't find any errors, I guess we have to take
* the users settings. Darn! */
/* Set the deflate settings. */
var->contiguous = NC_FALSE;
var->deflate = *deflate;
if (*deflate)
Expand All @@ -710,6 +712,18 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *deflate,
var->contiguous = NC_FALSE;
}

#ifdef USE_PARALLEL
/* If deflate, shuffle, or fletcher32 was turned on with
* parallel I/O writes, then switch to collective access. HDF5
* requires collevtive access for filter use with parallel
* I/O. */
if (deflate || shuffle || fletcher32)
{
if (h5->parallel && (var->deflate || var->shuffle || var->fletcher32))
var->parallel_access = NC_COLLECTIVE;
}
#endif /* USE_PARALLEL */

/* Handle storage settings. */
if (contiguous)
{
Expand Down
7 changes: 7 additions & 0 deletions libsrc4/nc4var.c
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,13 @@ NC4_var_par_access(int ncid, int varid, int par_access)
if (!var) return NC_ENOTVAR;
assert(var->hdr.id == varid);

/* If zlib, shuffle, or fletcher32 filters are in use, then access
* must be collective. Fail an attempt to set such a variable to
* independent access. */
if ((var->deflate || var->shuffle || var->fletcher32) &&
par_access == NC_INDEPENDENT)
return NC_EINVAL;

if (par_access)
var->parallel_access = NC_COLLECTIVE;
else
Expand Down
2 changes: 2 additions & 0 deletions nc_test4/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ IF(TEST_PARALLEL4)
build_bin_test(tst_parallel3)
build_bin_test(tst_parallel4)
build_bin_test(tst_parallel5)
build_bin_test(tst_parallel_zlib)
build_bin_test(tst_parallel_zlib2)
build_bin_test(tst_nc4perf)
build_bin_test(tst_mode)
build_bin_test(tst_simplerw_coll_r)
Expand Down
2 changes: 1 addition & 1 deletion nc_test4/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ endif # BUILD_UTILITIES
if TEST_PARALLEL4
check_PROGRAMS += tst_mpi_parallel tst_parallel tst_parallel3 \
tst_parallel4 tst_parallel5 tst_nc4perf tst_mode tst_simplerw_coll_r \
tst_mode
tst_mode tst_parallel_zlib tst_parallel_zlib2
TESTS += run_par_test.sh
endif

Expand Down
10 changes: 10 additions & 0 deletions nc_test4/run_par_test.sh.in
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,13 @@ echo "Parallel I/O test for Collective I/O, contributed by HDF Group."
@MPIEXEC@ -n 1 ./tst_simplerw_coll_r
@MPIEXEC@ -n 2 ./tst_simplerw_coll_r
@MPIEXEC@ -n 4 ./tst_simplerw_coll_r

echo
echo "Parallel I/O test with zlib."
@MPIEXEC@ -n 1 ./tst_parallel_zlib
@MPIEXEC@ -n 4 ./tst_parallel_zlib

echo
echo "Parallel I/O more tests with zlib."
@MPIEXEC@ -n 1 ./tst_parallel_zlib2
@MPIEXEC@ -n 4 ./tst_parallel_zlib2
78 changes: 43 additions & 35 deletions nc_test4/tst_mode.c
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
/*! \file

Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
2015, 2016, 2017, 2018
University Corporation for Atmospheric Research/Unidata.
Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002,
2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
2015, 2016, 2017, 2018
University Corporation for Atmospheric Research/Unidata.

See \ref copyright file for more info.
See \ref copyright file for more info.


* Test some illegal mode combinations
*
*/
* Test some illegal mode combinations
*
*/

#include "nc_tests.h"
#include "err_macros.h"
Expand All @@ -21,37 +21,45 @@ See \ref copyright file for more info.
int
main(int argc, char** argv)
{
int ncid,varid;
int retval;
int ncid,varid;
int retval;

printf("\n*** Testing illegal mode combinations\n");
printf("\n*** Testing illegal mode combinations\n");

MPI_Init(&argc,&argv);
MPI_Init(&argc,&argv);

printf("*** Testing create + MPIO + fletcher32\n");
if ((retval = nc_create_par(FILE_NAME, NC_CLOBBER|NC_NETCDF4, MPI_COMM_WORLD, MPI_INFO_NULL, &ncid))) ERR;
if ((retval = nc_def_var(ncid,"whatever",NC_INT,0,NULL,&varid))) ERR;
retval = nc_def_var_fletcher32(ncid,varid,NC_FLETCHER32);
if(retval != NC_EINVAL) ERR;
if ((retval = nc_abort(ncid)))
{
fprintf(stderr,"XXX: err=%d\n",retval);
fflush(stderr);
ERR;
}
printf("*** Testing create + MPIO + fletcher32\n");
if ((retval = nc_create_par(FILE_NAME, NC_CLOBBER|NC_NETCDF4, MPI_COMM_WORLD, MPI_INFO_NULL, &ncid))) ERR;
if ((retval = nc_def_var(ncid,"whatever",NC_INT,0,NULL,&varid))) ERR;
retval = nc_def_var_fletcher32(ncid,varid,NC_FLETCHER32);
#ifdef HDF5_SUPPORTS_PAR_FILTERS
if(retval != NC_NOERR) ERR;
#else
if(retval != NC_EINVAL) ERR;
#endif
if ((retval = nc_abort(ncid)))
{
fprintf(stderr,"XXX: err=%d\n",retval);
fflush(stderr);
ERR;
}

printf("*** Testing create + MPIO + deflation\n");
if ((retval = nc_create_par(FILE_NAME, NC_CLOBBER|NC_NETCDF4, MPI_COMM_WORLD, MPI_INFO_NULL, &ncid))) ERR;
if ((retval = nc_def_var(ncid,"whatever",NC_INT,0,NULL,&varid))) ERR;
retval = nc_def_var_deflate(ncid,varid, NC_NOSHUFFLE, 1, 1);
if(retval != NC_EINVAL) ERR;
if ((retval = nc_abort(ncid))) {
fprintf(stderr,"XXX: nc_abort: %d\n",retval); fflush(stderr);
ERR;
}
printf("*** Testing create + MPIO + deflation\n");
if ((retval = nc_create_par(FILE_NAME, NC_CLOBBER|NC_NETCDF4, MPI_COMM_WORLD, MPI_INFO_NULL, &ncid))) ERR;
if ((retval = nc_def_var(ncid,"whatever",NC_INT,0,NULL,&varid))) ERR;
retval = nc_def_var_deflate(ncid,varid, NC_NOSHUFFLE, 1, 1);
#ifdef HDF5_SUPPORTS_PAR_FILTERS
if(retval != NC_NOERR) ERR;
#else
if(retval != NC_EINVAL) ERR;
#endif
if ((retval = nc_abort(ncid))) {
fprintf(stderr,"XXX: nc_abort: %d\n",retval); fflush(stderr);
ERR;
}

MPI_Finalize();
MPI_Finalize();

SUMMARIZE_ERR;
FINAL_RESULTS;
SUMMARIZE_ERR;
FINAL_RESULTS;
}
Loading