Merge pull request #2238 from WardF/gh2206.wif

Conflict resolution for #2206

RELEASE_NOTES.md

@@ -9,6 +9,7 @@ This file contains a high-level description of this package's evolution. Release
 
 * [Bug Fix] Require that the type of the variable in nc_def_var_filter is not variable length. See [Github #/2231](https://github.com/Unidata/netcdf-c/pull/2231).
 * [File Change] Apply HDF5 v1.8 format compatibility when writing to previous files, as well as when creating new files. The superblock version remains at 2 for newly created files. Full backward read/write compatibility for netCDF-4 is maintained in all cases. See [Github #2176](https://github.com/Unidata/netcdf-c/issues/2176).
+* [Enhancement] Add ability to set dataset alignment for netcdf-4/HDF5 files. See [Github #2206](https://github.com/Unidata/netcdf-c/pull/2206).
 * [Enhancement] Add complete bitgroom support to NCZarr. See [Github #2197](https://github.com/Unidata/netcdf-c/pull/2197).
 * [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179](https://github.com/Unidata/netcdf-c/pull/2179).
 * [Bug Fix] Make sure that netcdf.h accurately defines the flags in the open/create mode flags. See [Github #2183](https://github.com/Unidata/netcdf-c/pull/2183).

acinclude.m4

@@ -384,4 +384,6 @@ MOSTLYCLEANFILES += $(valgrind_log_files)
  AC_SUBST([VALGRIND_CHECK_RULES])
  m4_ifdef([_AM_SUBST_NOTMAKE], [_AM_SUBST_NOTMAKE([VALGRIND_CHECK_RULES])])
 ])
+
 AC_DEFUN([_AC_FINALIZE],[])
+

docs/Doxyfile.in

@@ -781,7 +781,6 @@ INPUT = \
  @abs_top_srcdir@/libdispatch/derror.c \
  @abs_top_srcdir@/libdispatch/dv2i.c \
  @abs_top_srcdir@/libdispatch/dcopy.c \
- @abs_top_srcdir@/libdispatch/daux.c \
  @abs_top_srcdir@/libsrc4/nc4var.c \
  @abs_top_srcdir@/libhdf5/nc4hdf.c \
  @abs_top_srcdir@/libsrc4/nc4internal.c \

include/nc4internal.h

@@ -118,6 +118,7 @@ typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t;
 /* Forward declarations. */
 struct NC_GRP_INFO;
 struct NC_TYPE_INFO;
+struct NCRCinfo;
 
 /**
  * This struct provides indexed Access to Meta-data objects. See the
@@ -205,9 +206,11 @@ typedef struct NC_VAR_INFO
  int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent). */
  nc_bool_t shuffle; /**< True if var has shuffle filter applied. */
  nc_bool_t fletcher32; /**< True if var has fletcher32 filter applied. */
- size_t chunk_cache_size; /**< Size in bytes of the var chunk cache. */
- size_t chunk_cache_nelems; /**< Number of slots in var chunk cache. */
- float chunk_cache_preemption; /**< Chunk cache preemtion policy. */
+ struct ChunkCache {
+ size_t size; /**< Size in bytes of the var chunk cache. */
+ size_t nelems; /**< Number of slots in var chunk cache. */
+ float preemption; /**< Chunk cache preemtion policy. */
+ } chunkcache;
  int quantize_mode; /**< Quantize mode. NC_NOQUANTIZE is 0, and means no quantization. */
  int nsd; /**< Number of significant digits if quantization is used, 0 if not. */
  void *format_var_info; /**< Pointer to any binary format info. */
@@ -327,6 +330,24 @@ typedef struct NC_FILE_INFO
  } mem;
 } NC_FILE_INFO_T;
 
+/* Collect global state info in one place */
+typedef struct NCglobalstate {
+ int initialized;
+ char* tempdir; /* track a usable temp dir */
+ char* home; /* track $HOME */
+ char* cwd; /* track getcwd */
+ struct NCRCinfo* rcinfo; /* Currently only one rc file per session */
+ struct GlobalZarr { /* Zarr specific parameters */
+ char dimension_separator;
+ } zarr;
+ struct Alignment { /* H5Pset_alignment parameters */
+ int defined; /* 1 => threshold and alignment explicitly set */
+ int threshold;
+ int alignment;
+ } alignment;
+ struct ChunkCache chunkcache;
+} NCglobalstate;
+
 /** Variable Length Datatype struct in memory. Must be identical to
  * HDF5 hvl_t. (This is only used for VL sequences, not VL strings,
  * which are stored in char *'s) */
@@ -460,6 +481,10 @@ extern const char* nc4_atomic_name[NUM_ATOMIC_TYPES];
 /* Binary searcher for reserved attributes */
 extern const NC_reservedatt* NC_findreserved(const char* name);
 
+/* Global State Management */
+extern NCglobalstate* NC_getglobalstate(void);
+extern void NC_freeglobalstate(void);
+
 /* Generic reserved Attributes */
 #define NC_ATT_REFERENCE_LIST "REFERENCE_LIST"
 #define NC_ATT_CLASS "CLASS"

include/ncrc.h

@@ -33,40 +33,26 @@ typedef struct NCRCentry {
  char* value;
 } NCRCentry;
 
-/* collect all the relevant info around the rc file */
+struct AWSentry {
+ char* key;
+ char* value;
+};
+
+struct AWSprofile {
+ char* name;
+ NClist* entries; /* NClist<struct AWSentry*> */
+};
+
+/* collect all the relevant info around the rc file and AWS */
 typedef struct NCRCinfo {
  int ignore; /* if 1, then do not use any rc file */
  int loaded; /* 1 => already loaded */
  NClist* entries; /* the rc file entry store fields*/
  char* rcfile; /* specified rcfile; overrides anything else */
  char* rchome; /* Overrides $HOME when looking for .rc files */
+ NClist* s3profiles; /* NClist<struct AWSprofile*> */
 } NCRCinfo;
 
-/* Collect global state info in one place */
-typedef struct NCRCglobalstate {
- int initialized;
- char* tempdir; /* track a usable temp dir */
- char* home; /* track $HOME */
- char* cwd; /* track getcwd */
- NCRCinfo rcinfo; /* Currently only one rc file per session */
- struct GlobalZarr { /* Zarr specific parameters */
- char dimension_separator;
- } zarr;
- struct S3credentials {
- NClist* profiles; /* NClist<struct AWSprofile*> */
- } s3creds;
-} NCRCglobalstate;
-
-struct AWSprofile {
- char* name;
- NClist* entries; /* NClist<struct AWSentry*> */
-};
-
-struct AWSentry {
- char* key;
- char* value;
-};
-
 typedef struct NCS3INFO {
  char* host; /* non-null if other*/
  char* region; /* region */
@@ -80,9 +66,7 @@ extern "C" {
 #endif
 
 /* From drc.c */
-EXTERNL int ncrc_createglobalstate(void);
 EXTERNL void ncrc_initialize(void);
-EXTERNL void ncrc_freeglobalstate(void);
 EXTERNL int NC_rcfile_insert(const char* key, const char* value, const char* hostport, const char* path);
 EXTERNL char* NC_rclookup(const char* key, const char* hostport, const char* path);
 EXTERNL char* NC_rclookupx(NCURI* uri, const char* key);
@@ -94,7 +78,6 @@ EXTERNL size_t NC_rcfile_length(NCRCinfo*);
 EXTERNL NCRCentry* NC_rcfile_ith(NCRCinfo*,size_t);
 
 /* For internal use */
-EXTERNL NCRCglobalstate* ncrc_getglobalstate(void);
 EXTERNL void NC_rcclear(NCRCinfo* info);
 EXTERNL void NC_rcclear(NCRCinfo* info);
 

include/netcdf.h

@@ -560,6 +560,14 @@ nc_def_user_format(int mode_flag, NC_Dispatch *dispatch_table, char *magic_numbe
 EXTERNL int
 nc_inq_user_format(int mode_flag, NC_Dispatch **dispatch_table, char *magic_number);
 
+/* Set the global alignment property */
+EXTERNL int
+nc_set_alignment(int threshold, int alignment);
+
+/* Get the global alignment property */
+EXTERNL int
+nc_get_alignment(int* thresholdp, int* alignmentp);
+
 EXTERNL int
 nc__create(const char *path, int cmode, size_t initialsz,
  size_t *chunksizehintp, int *ncidp);

libdap4/d4file.c

@@ -367,7 +367,7 @@ set_curl_properties(NCD4INFO* d4info)
  char* newpath = NULL;
  int len;
  errno = 0;
- NCRCglobalstate* globalstate = ncrc_getglobalstate();
+ NCglobalstate* globalstate = NC_getglobalstate();
 
  /* Create the unique cookie file name */
  len =

libdispatch/ddispatch.c

@@ -12,6 +12,7 @@ See LICENSE.txt for license information.
 #include "ncoffsets.h"
 #include "ncpathmgr.h"
 #include "ncxml.h"
+#include "nc4internal.h"
 
 /* Required for getcwd, other functions. */
 #ifdef HAVE_UNISTD_H
@@ -48,16 +49,15 @@ NCDISPATCH_initialize(void)
 {
  int status = NC_NOERR;
  int i;
- NCRCglobalstate* globalstate = NULL;
+ NCglobalstate* globalstate = NULL;
 
  for(i=0;i<NC_MAX_VAR_DIMS;i++) {
  NC_coord_zero[i] = 0;
  NC_coord_one[i] = 1;
  NC_stride_one[i] = 1;
  }
 
- status = ncrc_createglobalstate(); /* will allocate and clear */
- globalstate = ncrc_getglobalstate(); /* will allocate and clear */
+ globalstate = NC_getglobalstate(); /* will allocate and clear */
 
  /* Capture temp dir*/
  {
@@ -123,7 +123,7 @@ int
 NCDISPATCH_finalize(void)
 {
  int status = NC_NOERR;
- ncrc_freeglobalstate();
+ NC_freeglobalstate();
 #if defined(ENABLE_BYTERANGE) || defined(ENABLE_DAP) || defined(ENABLE_DAP4)
  curl_global_cleanup();
 #endif