Skip to content

Commit

Permalink
Merge branch 'format-compat-on-reopen' of https://github.com/Dave-All…
Browse files Browse the repository at this point in the history
…ured/netcdf-c into Dave-Allured-format-compat-on-reopen
  • Loading branch information
WardF committed Jan 21, 2022
2 parents 9b5e662 + 158f290 commit 41baf43
Show file tree
Hide file tree
Showing 6 changed files with 129 additions and 39 deletions.
4 changes: 3 additions & 1 deletion include/hdf5internal.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* Copyright 2018-2018 University Corporation for Atmospheric
/* Copyright 2018-2022 University Corporation for Atmospheric
Research/Unidata. */
/**
* @file This header file contains macros, types, and prototypes for
Expand Down Expand Up @@ -216,4 +216,6 @@ extern int nc4_find_default_chunksizes2(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var);
EXTERNL hid_t nc4_H5Fopen(const char *filename, unsigned flags, hid_t fapl_id);
EXTERNL hid_t nc4_H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id);

int hdf5set_format_compatibility(hid_t fapl_id);

#endif /* _HDF5INTERNAL_ */
6 changes: 3 additions & 3 deletions libhdf5/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
## This is a CMake file, part of Unidata's netCDF package.
# Copyright 2018, see the COPYRIGHT file for more information.
# Copyright 2018-2022, see the COPYRIGHT file for more information.
#
# This builds the HDF5 dispatch layer.
#
Expand All @@ -9,13 +9,13 @@
SET(libnchdf5_SOURCES nc4hdf.c nc4info.c hdf5file.c hdf5attr.c
hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c
hdf5var.c nc4mem.c nc4memcb.c hdf5dispatch.c hdf5filter.c
hdf5debug.c)
hdf5set_format_compatibility.c hdf5debug.c)

IF(ENABLE_BYTERANGE)
SET(libnchdf5_SOURCES ${libnchdf5_SOURCES} H5FDhttp.c)
ENDIF()

# Build the HDF4 dispatch layer as a library that will be included in
# Build the HDF5 dispatch layer as a library that will be included in
# the netCDF library.
add_library(netcdfhdf5 OBJECT ${libnchdf5_SOURCES})

Expand Down
11 changes: 3 additions & 8 deletions libhdf5/Makefile.am
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This is part of Unidata's netCDF package. Copyright 2018, see the
# COPYRIGHT file for more information.
# This is part of Unidata's netCDF package. Copyright 2018-2022,
# see the COPYRIGHT file for more information.

# Build the HDF5 dispatch layer.

Expand All @@ -16,16 +16,11 @@ noinst_LTLIBRARIES = libnchdf5.la
libnchdf5_la_SOURCES = nc4hdf.c nc4info.c hdf5file.c hdf5attr.c \
hdf5dim.c hdf5grp.c hdf5type.c hdf5internal.c hdf5create.c hdf5open.c \
hdf5var.c nc4mem.c nc4memcb.c hdf5dispatch.c hdf5filter.c \
hdf5debug.c hdf5debug.h hdf5err.h
hdf5set_format_compatibility.c hdf5debug.c hdf5debug.h hdf5err.h

if ENABLE_BYTERANGE
libnchdf5_la_SOURCES += H5FDhttp.c H5FDhttp.h
endif

# Package this for cmake build.
EXTRA_DIST = CMakeLists.txt





39 changes: 16 additions & 23 deletions libhdf5/hdf5create.c
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* Copyright 2003-2018, University Corporation for Atmospheric
/* Copyright 2003-2022, University Corporation for Atmospheric
* Research. See COPYRIGHT file for copying and redistribution
* conditions. */
/**
Expand Down Expand Up @@ -117,18 +117,19 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
}
}

/* Need this access plist to control how HDF5 handles open objects
* on file close. (Setting H5F_CLOSE_WEAK will cause H5Fclose not to
* fail if there are any open objects in the file. This may happen when virtual
/* Need this FILE ACCESS plist to control how HDF5 handles open
* objects on file close; as well as for other controls below.
* (Setting H5F_CLOSE_WEAK will cause H5Fclose not to fail if there
* are any open objects in the file. This may happen when virtual
* datasets are opened). */
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
BAIL(NC_EHDFERR);
if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_WEAK))
BAIL(NC_EHDFERR);

#ifdef USE_PARALLEL4
/* If this is a parallel file create, set up the file creation
property list. */
/* If this is a parallel file create, set up the file access
property list for MPI/IO. */
if (mpiinfo != NULL) {
nc4_info->parallel = NC_TRUE;
LOG((4, "creating parallel file with MPI/IO"));
Expand Down Expand Up @@ -164,31 +165,23 @@ nc4_create_file(const char *path, int cmode, size_t initialsz,
nc4_chunk_cache_preemption));
}

#if H5_VERSION_GE(1,10,2)
/* lib versions 1.10.2 and higher */
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_V18, H5F_LIBVER_LATEST) < 0)
#else
#if H5_VERSION_GE(1,10,0)
/* lib versions 1.10.0, 1.10.1 */
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0)
#else
/* all HDF5 1.8 lib versions */
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
#endif
#endif
BAIL(NC_EHDFERR);
/* Set HDF5 format compatibility in the FILE ACCESS property list.
* Compatibility is transient and must be reselected every time
* a file is opened for writing. */
retval = hdf5set_format_compatibility(fapl_id);
if (retval != NC_NOERR)
BAIL(retval);

/* Create the property list. */
/* Begin setup for the FILE CREATION property list. */
if ((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0)
BAIL(NC_EHDFERR);

/* RJ: this suppose to be FALSE that is defined in H5 private.h as 0 */
if (H5Pset_obj_track_times(fcpl_id,0)<0)
BAIL(NC_EHDFERR);

/* Set latest_format in access propertly list and
* H5P_CRT_ORDER_TRACKED in the creation property list. This turns
* on HDF5 creation ordering. */
/* Set H5P_CRT_ORDER_TRACKED in the creation property list.
* This turns on HDF5 creation ordering. */
if (H5Pset_link_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
H5P_CRT_ORDER_INDEXED)) < 0)
BAIL(NC_EHDFERR);
Expand Down
16 changes: 12 additions & 4 deletions libhdf5/hdf5open.c
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* Copyright 2003-2018, University Corporation for Atmospheric
/* Copyright 2003-2022, University Corporation for Atmospheric
* Research. See COPYRIGHT file for copying and redistribution
* conditions. */
/**
Expand Down Expand Up @@ -765,9 +765,10 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid)
mpiinfo = (NC_MPI_INFO *)parameters; /* assume, may be changed if inmemory is true */
#endif /* !USE_PARALLEL4 */

/* Need this access plist to control how HDF5 handles open objects
* on file close. (Setting H5F_CLOSE_WEAK will cause H5Fclose not to
* fail if there are any open objects in the file. This may happen when virtual
/* Need this FILE ACCESS plist to control how HDF5 handles open
* objects on file close; as well as for other controls below.
* (Setting H5F_CLOSE_WEAK will cause H5Fclose not to fail if there
* are any open objects in the file. This may happen when virtual
* datasets are opened). */
if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
BAIL(NC_EHDFERR);
Expand Down Expand Up @@ -820,6 +821,13 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid)
nc4_chunk_cache_preemption));
}

/* Set HDF5 format compatibility in the FILE ACCESS property list.
* Compatibility is transient and must be reselected every time
* a file is opened for writing. */
retval = hdf5set_format_compatibility(fapl_id);
if (retval != NC_NOERR)
BAIL(retval);

/* Process NC_INMEMORY */
if(nc4_info->mem.inmemory) {
NC_memio* memio;
Expand Down
92 changes: 92 additions & 0 deletions libhdf5/hdf5set_format_compatibility.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/* Copyright 2022, University Corporation for Atmospheric Research.
* See COPYRIGHT file for copying and redistribution conditions. */
/**
* @file
* @internal This function selects the best HDF5 file format options
* to create netCDF-4 files that can be read and written by older
* library versions.
*
* Format compatibility is transient, not baked in to an HDF5 file
* at creation time. Therefore the desired compatibilty options
* must be selected every time a file is opened for writing.
*
* This function should be called before every call to create a new
* netCDF-4 file, or to open an existing netCDF-4 file for writing.
* This function has no effect when opening a file for read only.
*
* This function should work correctly with all HDF5 library versions
* from 1.8.0 through 1.13.0 and beyond, with no further changes.
* This assumes that HDF5 versioning controls remain consistent
* into the future.
*
* The basic functionality is to select the traditional HDF5 v1.8
* format compatibility, whenever possible. The less desirable
* v1.6 compatibily is selected in a few strange cases when it is
* not possible to select v1.8.
*
* Files created or updated with v1.10 and higher compatibility are
* not legal netCDF-4 format, as of 2022 January. They are not
* readable by any netCDF library version linked with any HDF5 v1.8
* or older library version. However, it is possible for advanced
* or experimental software to deliberately override these default
* format settings, to create advanced format files for special
* purposes.
*
* Files created with v1.6 compatibility have superblock version 0.
* Files created with v1.8 compatibility have superblock version 2.
* Files created with v1.10 compatibility have superblock version 3,
* and are avoided by default. Et cetera.
*
* The superblock version is locked in when a file is first created.
* It is then possible to get a mix of v1.6 and v1.8 internal
* object versions, when an existing netCDF-4 file is modified by
* a different software version than the one that originally
* created the file. Mixed-object files of this nature are common
* and do not suffer any serious problems.
*
* See netcdf-c github issues #250 and #951 for more details about
* the rationale and evolution of netCDF-4 format compatibility.
*
* See HDF5 documentation for H5Pset_libver_bounds and related RFC's,
* for more details about HDF5 file object versioning.
*
* @author Dave Allured, NOAA/PSL/CIRES @date 2022 January 11
*/

#include "config.h"
#include "hdf5internal.h"

/**
* @internal Function to set HDF5 file access options for backward
* format compatibility. Call this before every call to H5Fcreate
* or H5Fopen.
*
* @param fapl_id Identifier for valid file access property list to
* be used in the next call to H5Fcreate or H5Fopen.
*
* @return ::NC_NOERR No error.
* @return ::NC_EHDFERR HDF5 returned error.
*
* @author Dave Allured, NOAA/PSL/CIRES @date 2022 January 11
*/
int
hdf5set_format_compatibility(hid_t fapl_id)
{
#if H5_VERSION_GE(1,10,2)
/* lib versions 1.10.2 and higher */
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_V18, H5F_LIBVER_LATEST) < 0)

#else
#if H5_VERSION_GE(1,10,0)
/* lib versions 1.10.0, 1.10.1 */
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0)

#else
/* all HDF5 1.8 lib versions */
if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
#endif
#endif
return NC_EHDFERR; /* failure exit */

return NC_NOERR; /* normal exit */
}

0 comments on commit 41baf43

Please sign in to comment.