Merge pull request #1990 from DennisHeimbigner/nczarr_enhance1.dmh

Upgrade the nczarr code to match Zarr V2

CMakeLists.txt

@@ -1071,11 +1071,14 @@ IF(NOT FOUND_CURL)
   ENDIF()
 
   IF(ENABLE_DAP2 OR ENABLE_DAP4)
-    MESSAGE(FATAL_ERROR "DAP support specified, CURL libraries are not found.")
+    MESSAGE(WARNING "CURL libraries are not found; DAP support disabled")
+    SET(ENABLE_DAP2 OFF CACHE BOOL "Use DAP2" FORCE)
+    SET(ENABLE_DAP4 OFF CACHE BOOL "Use DAP4" FORCE)
   ENDIF()
 
   IF(ENABLE_HDF5_ROS3)
-    MESSAGE(FATAL_ERROR "ROS3 support specified, CURL libraries are not found.")
+    MESSAGE(WARNING "CURL libraries are not found; ROS3 support disabled.")
+    SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE)
   ENDIF()
 
   IF(ENABLE_NCZARR_S3)
@@ -2185,7 +2188,7 @@ is_enabled(STATUS_PNETCDF HAS_PNETCDF)
 is_enabled(STATUS_PARALLEL HAS_PARALLEL)
 is_enabled(ENABLE_PARALLEL4 HAS_PARALLEL4)
 is_enabled(ENABLE_DAP HAS_DAP)
-is_enabled(ENABLE_DAP HAS_DAP2)
+is_enabled(ENABLE_DAP2 HAS_DAP2)
 is_enabled(ENABLE_DAP4 HAS_DAP4)
 is_enabled(ENABLE_BYTERANGE HAS_BYTERANGE)
 is_enabled(ENABLE_DISKLESS HAS_DISKLESS)

RELEASE_NOTES.md

@@ -6,6 +6,7 @@ Release Notes       {#RELEASE_NOTES}
 This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries.
 ## 4.8.1 - TBD
 
+* [Enhancement] Add support for the new "dimension_separator" enhancement to Zarr v2. See [Github #1990](https://github.com/Unidata/netcdf-c/pull/1990) for more information.
 * [Bug Fix] Fix hack for handling failure of shell programs to properly handle escape characters. See [Github #1989](https://github.com/Unidata/netcdf-c/issues/1989).  
 * [Bug Fix] Allow some primitive type names to be used as identifiers depending on the file format. See [Github #1984](https://github.com/Unidata/netcdf-c/issues/1984).  
 * [Enhancement] Add support for reading/writing pure Zarr storage format that supports the XArray _ARRAY_DIMENSIONS attribute. See [Github #1952](https://github.com/Unidata/netcdf-c/pull/1952) for more information.

configure.ac

@@ -122,6 +122,12 @@ AC_ARG_ENABLE([pnetcdf], [AS_HELP_STRING([--enable-pnetcdf],
 test "x$enable_pnetcdf" = xyes || enable_pnetcdf=no
 AC_MSG_RESULT($enable_pnetcdf)
 
+# We need curl for remote operations
+AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no])
+if test "x$found_curl" = "xyes" ; then
+   AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll cygcurl.dll], [],[])
+fi
+
 ## Capture the state of the --enable-dap flag => enable dap2+dap4
 AC_MSG_CHECKING([whether DAP client(s) are to be built])
 AC_ARG_ENABLE([dap],
@@ -130,11 +136,16 @@ AC_ARG_ENABLE([dap],
 test "x$enable_dap" = xno || enable_dap=yes
 AC_MSG_RESULT($enable_dap)
 
-AC_MSG_CHECKING([whether netcdf zarr storage format should be enabled])
+if test "x$enable_dap" = xyes & test "x$found_curl" = xno ; then
+  AC_MSG_WARN([curl required for dap access. DAP support disabled.])
+  enable_dap=no
+fi
+
+AC_MSG_CHECKING([whether netcdf zarr storage format should be disabled])
 AC_ARG_ENABLE([nczarr],
-              [AS_HELP_STRING([--enable-nczarr],
-                              [enable netcdf zarr storage support; requires netCDF-4 and libcurl])])
-test "x$enable_nczarr" = xyes || enable_nczarr=no
+              [AS_HELP_STRING([--disable-nczarr],
+                              [disable netcdf zarr storage support])])
+test "x$enable_nczarr" = xno || enable_nczarr=yes
 AC_MSG_RESULT($enable_nczarr)
 
 # HDF5 | HDF4 | NCZarr => netcdf-4
@@ -395,12 +406,6 @@ if test "x$enable_set_log_level_func" = xyes -a "x$enable_netcdf_4" = xyes; then
 fi
 AC_MSG_RESULT($enable_set_log_level_func)
 
-# We need curl for DAP and byterange
-AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no])
-if test "x$found_curl" = "xyes" ; then
-   AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll cygcurl.dll], [],
-      [AC_MSG_ERROR([curl required for remote access. Install curl or disable relevant options.])])
-fi
 
 # CURLOPT_USERNAME is not defined until curl version 7.19.1
 # CURLOPT_PASSWORD is not defined until curl version 7.19.1
@@ -560,63 +565,86 @@ AC_MSG_RESULT([$enable_dap_long_tests])
 
 # Control zarr storage
 if test "x$enable_nczarr" = xyes ; then
-if test "x$enable_netcdf_4" = xno ; then
-AC_MSG_ERROR([netCDF-4 disabled, so you must not enable nczarr])
-enable_nczarr=no
-fi
+  if test "x$enable_netcdf_4" = xno ; then
+    AC_MSG_WARN([netCDF-4 disabled, so you must not enable nczarr])
+    enable_nczarr=no
+  fi
 fi
+
 if test "x$enable_nczarr" = xyes; then
    AC_DEFINE([ENABLE_NCZARR], [1], [if true, build NCZarr Client])
    AC_SUBST(ENABLE_NCZARR)
 fi
 AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes])
 
+# See if we have libzip
+AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no])
+if test "x$have_zip" = "xyes" ; then
+   AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [], [])
+fi
+AC_MSG_CHECKING([whether libzip library is available])
+AC_MSG_RESULT([${have_zip}])
+
+enable_nczarr_zip=${have_zip} # alias
+
+if test "x$enable_nczarr" = xno ; then
+enable_nczarr_zip=no
+fi
+
+AC_MSG_CHECKING([whether nczarr zip support is enabled])
+AC_MSG_RESULT([${enable_nczarr_zip}])
+
+if test "x$enable_nczarr_zip" = xyes ; then
+AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found])
+fi
+AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
+
 # Check for enabling of S3 support
 AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled])
 AC_ARG_ENABLE([nczarr-s3],
               [AS_HELP_STRING([--enable-nczarr-s3],
                               [enable netcdf zarr S3 support; make sure to set LDFLAGS])])
 test "x$enable_nczarr_s3" = xyes || enable_nczarr_s3=no
-AC_MSG_RESULT($enable_nczarr_s3)
-
-# Check for enabling S3 testing
-AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
-AC_ARG_ENABLE([nczarr-s3-tests],
-              [AS_HELP_STRING([--enable-nczarr-s3-tests],
-                              [enable netcdf zarr S3 testing])])
-test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no
-AC_MSG_RESULT($enable_nczarr_s3_tests)
-
-# Disable S3 tests if S3 support is disabled
-if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then
-    AC_MSG_ERROR([NCZarr S3 support is disabled; please specify option --disable-nczarr-s3-tests])
-    enable_nczarr_s3_tests=no
+if test "x$enable_nczarr" = xno ; then
+enable_nczarr_s3=no
 fi
+AC_MSG_RESULT($enable_nczarr_s3)
 
-# Set default
+# Note we check for the library after checking for enable_nczarr_s3
+# because for some reason this screws up if we unconditionally test for sdk
+# and it is not available. Fix someday
 have_aws=no
-
 if test "x$enable_nczarr_s3" = xyes ; then
 # See if we have the s3 aws library
 # Check for the AWS S3 SDK library
 AC_LANG_PUSH([C++])
 AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no])
 AC_LANG_POP
+fi
+
 AC_MSG_CHECKING([whether AWS S3 SDK library is available])
 AC_MSG_RESULT([$have_aws])
 
-if test "x$have_aws" = xyes ; then
+if test "x$have_aws" = xno ; then
+AC_MSG_WARN([AWS SDK not found; disabling S3 support])
+enable_nczarr_s3=no
+else
 AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
 fi
-fi
 AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes])
 
-if test "x$have_aws" = xno ; then
-  if test "x$enable_nczarr_s3" = xyes || test "x$enable_nczarr_s3_tests" = xyes ; then
-    AC_MSG_ERROR([AWS S3 libraries not found; please specify options --disable-nczarr-s3 and --disable-nczarr-s3-tests])
+# Check for enabling S3 testing
+AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
+AC_ARG_ENABLE([nczarr-s3-tests],
+              [AS_HELP_STRING([--enable-nczarr-s3-tests],
+                              [enable netcdf zarr S3 testing])])
+test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no
+AC_MSG_RESULT($enable_nczarr_s3_tests)
+
+# Disable S3 tests if S3 support is disabled
+if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then
+    AC_MSG_ERROR([NCZarr S3 support is disabled; please remove option --enable-nczarr-s3-tests])
     enable_nczarr_s3_tests=no
-    enable_nczarr_s3=no
-  fi
 fi
 
 if test "x$enable_nczarr_s3" = xyes ; then
@@ -634,22 +662,6 @@ if test "x$enable_nczarr_s3_tests" = xyes ; then
 fi
 
 # Set default
-# See if we have libzip
-AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no])
-if test "x$have_zip" = "xyes" ; then
-   AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [],
-      [AC_MSG_ERROR([libzip search failed.])])
-fi
-AC_MSG_CHECKING([whether libzip library is available])
-AC_MSG_RESULT([${have_zip}])
-
-enable_nczarr_zip=${have_zip} # alias
-
-if test "x$enable_nczarr_zip" = xyes ; then
-AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found])
-fi
-AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes])
-
 # Did the user specify a default cache size for NCZarr?
 AC_MSG_CHECKING([whether a default file cache size for NCZarr was specified])
 AC_ARG_WITH([chunk-cache-size-nczarr],

include/ncpathmgr.h

@@ -117,8 +117,7 @@ EXTERNL int NCaccess(const char* path, int mode);
 EXTERNL int NCremove(const char* path);
 EXTERNL int NCmkdir(const char* path, int mode);
 EXTERNL int NCrmdir(const char* path);
-EXTERNL char* NCcwd(char* cwdbuf, size_t len);
-EXTERNL char* NCcwd(char* cwdbuf, size_t len);
+EXTERNL char* NCgetcwd(char* cwdbuf, size_t len);
 #ifdef HAVE_SYS_STAT_H
 EXTERNL int NCstat(char* path, struct stat* buf);
 #endif

include/ncrc.h

@@ -16,6 +16,11 @@ and accessing rc files (e.g. .daprc).
 #include "nclist.h"
 #include "ncbytes.h"
 
+/* getenv() keys */
+#define NCRCENVIGNORE "NCRCENV_IGNORE"
+#define NCRCENVRC "NCRCENV_RC"
+
+
 typedef struct NCTriple {
 	char* host; /* combined host:port */
         char* key;
@@ -34,31 +39,40 @@ typedef struct NCRCinfo {
 typedef struct NCRCglobalstate {
     int initialized;
     char* tempdir; /* track a usable temp dir */
-    char* home; /* track $HOME for use in creating $HOME/.oc dir */
+    char* home; /* track $HOME */
+    char* cwd; /* track getcwd */
     NCRCinfo rcinfo; /* Currently only one rc file per session */
+    struct GlobalZarr { /* Zarr specific parameters */
+	char dimension_separator;
+    } zarr;
 } NCRCglobalstate;
 
 /* From drc.c */
-extern NCRCglobalstate* ncrc_getglobalstate(void);
-extern void ncrc_freeglobalstate(void);
+EXTERNL void ncrc_initialize(void);
+EXTERNL void ncrc_freeglobalstate(void);
 /* read and compile the rc file, if any */
-extern int NC_rcload(void);
-extern char* NC_rclookup(const char* key, const char* hostport);
-extern void NC_rcclear(NCRCinfo* info);
-extern int NC_set_rcfile(const char* rcfile);
-extern int NC_rcfile_insert(const char* key, const char* value, const char* hostport);
+EXTERNL int NC_rcload(void);
+EXTERNL char* NC_rclookup(const char* key, const char* hostport);
+EXTERNL int NC_rcfile_insert(const char* key, const char* value, const char* hostport);
+
+/* Following are primarily for debugging */
 /* Obtain the count of number of triples */
-extern size_t NC_rcfile_length(NCRCinfo*);
+EXTERNL size_t NC_rcfile_length(NCRCinfo*);
 /* Obtain the ith triple; return NULL if out of range */
-extern NCTriple* NC_rcfile_ith(NCRCinfo*,size_t);
+EXTERNL NCTriple* NC_rcfile_ith(NCRCinfo*,size_t);
+
+/* For internal use */
+EXTERNL NCRCglobalstate* ncrc_getglobalstate(void);
+EXTERNL void NC_rcclear(NCRCinfo* info);
+EXTERNL void NC_rcclear(NCRCinfo* info);
 
 /* From dutil.c (Might later move to e.g. nc.h */
-extern int NC__testurl(const char* path, char** basenamep);
-extern int NC_isLittleEndian(void);
-extern char* NC_entityescape(const char* s);
-extern int NC_readfile(const char* filename, NCbytes* content);
-extern int NC_writefile(const char* filename, size_t size, void* content);
-extern char* NC_mktmp(const char* base);
-extern int NC_getmodelist(const char* url, NClist** modelistp);
-extern int NC_testmode(const char* path, const char* tag);
+EXTERNL int NC__testurl(const char* path, char** basenamep);
+EXTERNL int NC_isLittleEndian(void);
+EXTERNL char* NC_entityescape(const char* s);
+EXTERNL int NC_readfile(const char* filename, NCbytes* content);
+EXTERNL int NC_writefile(const char* filename, size_t size, void* content);
+EXTERNL char* NC_mktmp(const char* base);
+EXTERNL int NC_getmodelist(const char* url, NClist** modelistp);
+EXTERNL int NC_testmode(const char* path, const char* tag);
 #endif /*NCRC_H*/

libdap2/dapattr.c

@@ -80,11 +80,15 @@ fprintf(stderr,"%s.maxstrlen=%d\n",node->ocname,(int)node->dodsspecial.maxstrlen
 		} else if(strcmp(ocname,"DODS.dimName")==0
 		   || strcmp(ocname,"DODS_EXTRA.dimName")==0) {
 		    if(values != NULL) {
+			nullfree(node->dodsspecial.dimname);
 		        node->dodsspecial.dimname = nulldup(values[0]);
 #ifdef DEBUG
 fprintf(stderr,"%s.dimname=%s\n",node->ocname,node->dodsspecial.dimname);
 #endif
-		    } else node->dodsspecial.dimname = NULL;
+		    } else {
+		        nullfree(node->dodsspecial.dimname);
+			node->dodsspecial.dimname = NULL;
+		    }
 		} else if(strcmp(ocname,"DODS.Unlimited_Dimension")==0
 		   || strcmp(ocname,"DODS_EXTRA.Unlimited_Dimension")==0) {
 		    char* val0 = NULL;

libdispatch/ddispatch.c

@@ -32,8 +32,6 @@ size_t NC_coord_zero[NC_MAX_VAR_DIMS] = {0};
 size_t NC_coord_one[NC_MAX_VAR_DIMS] = {1};
 ptrdiff_t NC_stride_one[NC_MAX_VAR_DIMS] = {1};
 
-NCRCglobalstate ncrc_globalstate;
-
 /*
 static nc_type longtype = (sizeof(long) == sizeof(int)?NC_INT:NC_INT64);
 static nc_type ulongtype = (sizeof(unsigned long) == sizeof(unsigned int)?NC_UINT:NC_UINT64);
@@ -81,6 +79,20 @@ NCDISPATCH_initialize(void)
         globalstate->home = strdup(home);
     }
 
+    /* Capture $CWD */
+    {
+        char cwdbuf[4096];
+
+        cwdbuf[0] = '\0';
+	(void)NCgetcwd(cwdbuf,sizeof(cwdbuf));
+
+        if(strlen(cwdbuf) == 0) {
+	    /* use tempdir */
+	    strcpy(cwdbuf, globalstate->tempdir);
+	}
+        globalstate->cwd = strdup(cwdbuf);
+    }
+
     /* Now load RC File */
     status = NC_rcload();
     ncloginit();

libdispatch/dinfermodel.c

@@ -123,6 +123,7 @@ static const struct MACRODEF {
 {"s3","mode","nczarr,s3"},
 {"bytes","mode","bytes"},
 {"xarray","mode","nczarr,zarr,xarray"},
+{"noxarray","mode","nczarr,zarr,noxarray"},
 {NULL,NULL,NULL}
 };
 
@@ -133,6 +134,7 @@ static const struct MODEINFER {
 } modeinferences[] = {
 {"zarr","nczarr"},
 {"xarray","zarr"},
+{"noxarray","zarr"},
 {NULL,NULL}
 };