diff --git a/CMakeLists.txt b/CMakeLists.txt index 32037f1e6e..797f868333 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1071,11 +1071,14 @@ IF(NOT FOUND_CURL) ENDIF() IF(ENABLE_DAP2 OR ENABLE_DAP4) - MESSAGE(FATAL_ERROR "DAP support specified, CURL libraries are not found.") + MESSAGE(WARNING "CURL libraries are not found; DAP support disabled") + SET(ENABLE_DAP2 OFF CACHE BOOL "Use DAP2" FORCE) + SET(ENABLE_DAP4 OFF CACHE BOOL "Use DAP4" FORCE) ENDIF() IF(ENABLE_HDF5_ROS3) - MESSAGE(FATAL_ERROR "ROS3 support specified, CURL libraries are not found.") + MESSAGE(WARNING "CURL libraries are not found; ROS3 support disabled.") + SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE) ENDIF() IF(ENABLE_NCZARR_S3) @@ -2185,7 +2188,7 @@ is_enabled(STATUS_PNETCDF HAS_PNETCDF) is_enabled(STATUS_PARALLEL HAS_PARALLEL) is_enabled(ENABLE_PARALLEL4 HAS_PARALLEL4) is_enabled(ENABLE_DAP HAS_DAP) -is_enabled(ENABLE_DAP HAS_DAP2) +is_enabled(ENABLE_DAP2 HAS_DAP2) is_enabled(ENABLE_DAP4 HAS_DAP4) is_enabled(ENABLE_BYTERANGE HAS_BYTERANGE) is_enabled(ENABLE_DISKLESS HAS_DISKLESS) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 995a481008..89372817fc 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -6,6 +6,7 @@ Release Notes {#RELEASE_NOTES} This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries. ## 4.8.1 - TBD +* [Enhancement] Add support for the new "dimension_separator" enhancement to Zarr v2. See [Github #1990](https://github.com/Unidata/netcdf-c/pull/1990) for more information. * [Bug Fix] Fix hack for handling failure of shell programs to properly handle escape characters. See [Github #1989](https://github.com/Unidata/netcdf-c/issues/1989). * [Bug Fix] Allow some primitive type names to be used as identifiers depending on the file format. See [Github #1984](https://github.com/Unidata/netcdf-c/issues/1984). * [Enhancement] Add support for reading/writing pure Zarr storage format that supports the XArray _ARRAY_DIMENSIONS attribute. See [Github #1952](https://github.com/Unidata/netcdf-c/pull/1952) for more information. diff --git a/configure.ac b/configure.ac index b6bc4fea8e..25598e9deb 100644 --- a/configure.ac +++ b/configure.ac @@ -122,6 +122,12 @@ AC_ARG_ENABLE([pnetcdf], [AS_HELP_STRING([--enable-pnetcdf], test "x$enable_pnetcdf" = xyes || enable_pnetcdf=no AC_MSG_RESULT($enable_pnetcdf) +# We need curl for remote operations +AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no]) +if test "x$found_curl" = "xyes" ; then + AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll cygcurl.dll], [],[]) +fi + ## Capture the state of the --enable-dap flag => enable dap2+dap4 AC_MSG_CHECKING([whether DAP client(s) are to be built]) AC_ARG_ENABLE([dap], @@ -130,11 +136,16 @@ AC_ARG_ENABLE([dap], test "x$enable_dap" = xno || enable_dap=yes AC_MSG_RESULT($enable_dap) -AC_MSG_CHECKING([whether netcdf zarr storage format should be enabled]) +if test "x$enable_dap" = xyes & test "x$found_curl" = xno ; then + AC_MSG_WARN([curl required for dap access. DAP support disabled.]) + enable_dap=no +fi + +AC_MSG_CHECKING([whether netcdf zarr storage format should be disabled]) AC_ARG_ENABLE([nczarr], - [AS_HELP_STRING([--enable-nczarr], - [enable netcdf zarr storage support; requires netCDF-4 and libcurl])]) -test "x$enable_nczarr" = xyes || enable_nczarr=no + [AS_HELP_STRING([--disable-nczarr], + [disable netcdf zarr storage support])]) +test "x$enable_nczarr" = xno || enable_nczarr=yes AC_MSG_RESULT($enable_nczarr) # HDF5 | HDF4 | NCZarr => netcdf-4 @@ -395,12 +406,6 @@ if test "x$enable_set_log_level_func" = xyes -a "x$enable_netcdf_4" = xyes; then fi AC_MSG_RESULT($enable_set_log_level_func) -# We need curl for DAP and byterange -AC_CHECK_LIB([curl],[curl_easy_setopt],[found_curl=yes],[found_curl=no]) -if test "x$found_curl" = "xyes" ; then - AC_SEARCH_LIBS([curl_easy_setopt],[curl curl.dll cygcurl.dll], [], - [AC_MSG_ERROR([curl required for remote access. Install curl or disable relevant options.])]) -fi # CURLOPT_USERNAME is not defined until curl version 7.19.1 # CURLOPT_PASSWORD is not defined until curl version 7.19.1 @@ -560,63 +565,86 @@ AC_MSG_RESULT([$enable_dap_long_tests]) # Control zarr storage if test "x$enable_nczarr" = xyes ; then -if test "x$enable_netcdf_4" = xno ; then -AC_MSG_ERROR([netCDF-4 disabled, so you must not enable nczarr]) -enable_nczarr=no -fi + if test "x$enable_netcdf_4" = xno ; then + AC_MSG_WARN([netCDF-4 disabled, so you must not enable nczarr]) + enable_nczarr=no + fi fi + if test "x$enable_nczarr" = xyes; then AC_DEFINE([ENABLE_NCZARR], [1], [if true, build NCZarr Client]) AC_SUBST(ENABLE_NCZARR) fi AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes]) +# See if we have libzip +AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no]) +if test "x$have_zip" = "xyes" ; then + AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [], []) +fi +AC_MSG_CHECKING([whether libzip library is available]) +AC_MSG_RESULT([${have_zip}]) + +enable_nczarr_zip=${have_zip} # alias + +if test "x$enable_nczarr" = xno ; then +enable_nczarr_zip=no +fi + +AC_MSG_CHECKING([whether nczarr zip support is enabled]) +AC_MSG_RESULT([${enable_nczarr_zip}]) + +if test "x$enable_nczarr_zip" = xyes ; then +AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found]) +fi +AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes]) + # Check for enabling of S3 support AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled]) AC_ARG_ENABLE([nczarr-s3], [AS_HELP_STRING([--enable-nczarr-s3], [enable netcdf zarr S3 support; make sure to set LDFLAGS])]) test "x$enable_nczarr_s3" = xyes || enable_nczarr_s3=no -AC_MSG_RESULT($enable_nczarr_s3) - -# Check for enabling S3 testing -AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled]) -AC_ARG_ENABLE([nczarr-s3-tests], - [AS_HELP_STRING([--enable-nczarr-s3-tests], - [enable netcdf zarr S3 testing])]) -test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no -AC_MSG_RESULT($enable_nczarr_s3_tests) - -# Disable S3 tests if S3 support is disabled -if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then - AC_MSG_ERROR([NCZarr S3 support is disabled; please specify option --disable-nczarr-s3-tests]) - enable_nczarr_s3_tests=no +if test "x$enable_nczarr" = xno ; then +enable_nczarr_s3=no fi +AC_MSG_RESULT($enable_nczarr_s3) -# Set default +# Note we check for the library after checking for enable_nczarr_s3 +# because for some reason this screws up if we unconditionally test for sdk +# and it is not available. Fix someday have_aws=no - if test "x$enable_nczarr_s3" = xyes ; then # See if we have the s3 aws library # Check for the AWS S3 SDK library AC_LANG_PUSH([C++]) AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no]) AC_LANG_POP +fi + AC_MSG_CHECKING([whether AWS S3 SDK library is available]) AC_MSG_RESULT([$have_aws]) -if test "x$have_aws" = xyes ; then +if test "x$have_aws" = xno ; then +AC_MSG_WARN([AWS SDK not found; disabling S3 support]) +enable_nczarr_s3=no +else AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found]) fi -fi AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes]) -if test "x$have_aws" = xno ; then - if test "x$enable_nczarr_s3" = xyes || test "x$enable_nczarr_s3_tests" = xyes ; then - AC_MSG_ERROR([AWS S3 libraries not found; please specify options --disable-nczarr-s3 and --disable-nczarr-s3-tests]) +# Check for enabling S3 testing +AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled]) +AC_ARG_ENABLE([nczarr-s3-tests], + [AS_HELP_STRING([--enable-nczarr-s3-tests], + [enable netcdf zarr S3 testing])]) +test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no +AC_MSG_RESULT($enable_nczarr_s3_tests) + +# Disable S3 tests if S3 support is disabled +if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then + AC_MSG_ERROR([NCZarr S3 support is disabled; please remove option --enable-nczarr-s3-tests]) enable_nczarr_s3_tests=no - enable_nczarr_s3=no - fi fi if test "x$enable_nczarr_s3" = xyes ; then @@ -634,22 +662,6 @@ if test "x$enable_nczarr_s3_tests" = xyes ; then fi # Set default -# See if we have libzip -AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no]) -if test "x$have_zip" = "xyes" ; then - AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [], - [AC_MSG_ERROR([libzip search failed.])]) -fi -AC_MSG_CHECKING([whether libzip library is available]) -AC_MSG_RESULT([${have_zip}]) - -enable_nczarr_zip=${have_zip} # alias - -if test "x$enable_nczarr_zip" = xyes ; then -AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found]) -fi -AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes]) - # Did the user specify a default cache size for NCZarr? AC_MSG_CHECKING([whether a default file cache size for NCZarr was specified]) AC_ARG_WITH([chunk-cache-size-nczarr], diff --git a/include/ncpathmgr.h b/include/ncpathmgr.h index 3cd27a9256..0245a40ad4 100644 --- a/include/ncpathmgr.h +++ b/include/ncpathmgr.h @@ -117,8 +117,7 @@ EXTERNL int NCaccess(const char* path, int mode); EXTERNL int NCremove(const char* path); EXTERNL int NCmkdir(const char* path, int mode); EXTERNL int NCrmdir(const char* path); -EXTERNL char* NCcwd(char* cwdbuf, size_t len); -EXTERNL char* NCcwd(char* cwdbuf, size_t len); +EXTERNL char* NCgetcwd(char* cwdbuf, size_t len); #ifdef HAVE_SYS_STAT_H EXTERNL int NCstat(char* path, struct stat* buf); #endif diff --git a/include/ncrc.h b/include/ncrc.h index b983743d29..da9e3dc34c 100644 --- a/include/ncrc.h +++ b/include/ncrc.h @@ -16,6 +16,11 @@ and accessing rc files (e.g. .daprc). #include "nclist.h" #include "ncbytes.h" +/* getenv() keys */ +#define NCRCENVIGNORE "NCRCENV_IGNORE" +#define NCRCENVRC "NCRCENV_RC" + + typedef struct NCTriple { char* host; /* combined host:port */ char* key; @@ -34,31 +39,40 @@ typedef struct NCRCinfo { typedef struct NCRCglobalstate { int initialized; char* tempdir; /* track a usable temp dir */ - char* home; /* track $HOME for use in creating $HOME/.oc dir */ + char* home; /* track $HOME */ + char* cwd; /* track getcwd */ NCRCinfo rcinfo; /* Currently only one rc file per session */ + struct GlobalZarr { /* Zarr specific parameters */ + char dimension_separator; + } zarr; } NCRCglobalstate; /* From drc.c */ -extern NCRCglobalstate* ncrc_getglobalstate(void); -extern void ncrc_freeglobalstate(void); +EXTERNL void ncrc_initialize(void); +EXTERNL void ncrc_freeglobalstate(void); /* read and compile the rc file, if any */ -extern int NC_rcload(void); -extern char* NC_rclookup(const char* key, const char* hostport); -extern void NC_rcclear(NCRCinfo* info); -extern int NC_set_rcfile(const char* rcfile); -extern int NC_rcfile_insert(const char* key, const char* value, const char* hostport); +EXTERNL int NC_rcload(void); +EXTERNL char* NC_rclookup(const char* key, const char* hostport); +EXTERNL int NC_rcfile_insert(const char* key, const char* value, const char* hostport); + +/* Following are primarily for debugging */ /* Obtain the count of number of triples */ -extern size_t NC_rcfile_length(NCRCinfo*); +EXTERNL size_t NC_rcfile_length(NCRCinfo*); /* Obtain the ith triple; return NULL if out of range */ -extern NCTriple* NC_rcfile_ith(NCRCinfo*,size_t); +EXTERNL NCTriple* NC_rcfile_ith(NCRCinfo*,size_t); + +/* For internal use */ +EXTERNL NCRCglobalstate* ncrc_getglobalstate(void); +EXTERNL void NC_rcclear(NCRCinfo* info); +EXTERNL void NC_rcclear(NCRCinfo* info); /* From dutil.c (Might later move to e.g. nc.h */ -extern int NC__testurl(const char* path, char** basenamep); -extern int NC_isLittleEndian(void); -extern char* NC_entityescape(const char* s); -extern int NC_readfile(const char* filename, NCbytes* content); -extern int NC_writefile(const char* filename, size_t size, void* content); -extern char* NC_mktmp(const char* base); -extern int NC_getmodelist(const char* url, NClist** modelistp); -extern int NC_testmode(const char* path, const char* tag); +EXTERNL int NC__testurl(const char* path, char** basenamep); +EXTERNL int NC_isLittleEndian(void); +EXTERNL char* NC_entityescape(const char* s); +EXTERNL int NC_readfile(const char* filename, NCbytes* content); +EXTERNL int NC_writefile(const char* filename, size_t size, void* content); +EXTERNL char* NC_mktmp(const char* base); +EXTERNL int NC_getmodelist(const char* url, NClist** modelistp); +EXTERNL int NC_testmode(const char* path, const char* tag); #endif /*NCRC_H*/ diff --git a/libdap2/dapattr.c b/libdap2/dapattr.c index 03a31e9391..f0dedfb8e4 100644 --- a/libdap2/dapattr.c +++ b/libdap2/dapattr.c @@ -80,11 +80,15 @@ fprintf(stderr,"%s.maxstrlen=%d\n",node->ocname,(int)node->dodsspecial.maxstrlen } else if(strcmp(ocname,"DODS.dimName")==0 || strcmp(ocname,"DODS_EXTRA.dimName")==0) { if(values != NULL) { + nullfree(node->dodsspecial.dimname); node->dodsspecial.dimname = nulldup(values[0]); #ifdef DEBUG fprintf(stderr,"%s.dimname=%s\n",node->ocname,node->dodsspecial.dimname); #endif - } else node->dodsspecial.dimname = NULL; + } else { + nullfree(node->dodsspecial.dimname); + node->dodsspecial.dimname = NULL; + } } else if(strcmp(ocname,"DODS.Unlimited_Dimension")==0 || strcmp(ocname,"DODS_EXTRA.Unlimited_Dimension")==0) { char* val0 = NULL; diff --git a/libdispatch/ddispatch.c b/libdispatch/ddispatch.c index 5b32691d83..999f6c3312 100644 --- a/libdispatch/ddispatch.c +++ b/libdispatch/ddispatch.c @@ -32,8 +32,6 @@ size_t NC_coord_zero[NC_MAX_VAR_DIMS] = {0}; size_t NC_coord_one[NC_MAX_VAR_DIMS] = {1}; ptrdiff_t NC_stride_one[NC_MAX_VAR_DIMS] = {1}; -NCRCglobalstate ncrc_globalstate; - /* static nc_type longtype = (sizeof(long) == sizeof(int)?NC_INT:NC_INT64); static nc_type ulongtype = (sizeof(unsigned long) == sizeof(unsigned int)?NC_UINT:NC_UINT64); @@ -81,6 +79,20 @@ NCDISPATCH_initialize(void) globalstate->home = strdup(home); } + /* Capture $CWD */ + { + char cwdbuf[4096]; + + cwdbuf[0] = '\0'; + (void)NCgetcwd(cwdbuf,sizeof(cwdbuf)); + + if(strlen(cwdbuf) == 0) { + /* use tempdir */ + strcpy(cwdbuf, globalstate->tempdir); + } + globalstate->cwd = strdup(cwdbuf); + } + /* Now load RC File */ status = NC_rcload(); ncloginit(); diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 57e04d3a70..a92c7a80af 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -123,6 +123,7 @@ static const struct MACRODEF { {"s3","mode","nczarr,s3"}, {"bytes","mode","bytes"}, {"xarray","mode","nczarr,zarr,xarray"}, +{"noxarray","mode","nczarr,zarr,noxarray"}, {NULL,NULL,NULL} }; @@ -133,6 +134,7 @@ static const struct MODEINFER { } modeinferences[] = { {"zarr","nczarr"}, {"xarray","zarr"}, +{"noxarray","zarr"}, {NULL,NULL} }; diff --git a/libdispatch/drc.c b/libdispatch/drc.c index 0283e237a0..5c8eafa5ea 100644 --- a/libdispatch/drc.c +++ b/libdispatch/drc.c @@ -20,7 +20,7 @@ See COPYRIGHT for license information. #include "nclog.h" #include "ncpathmgr.h" -#define RCFILEENV "DAPRCFILE" +#undef DRCDEBUG #define RTAG ']' #define LTAG '[' @@ -42,21 +42,41 @@ static void rcfreetriples(NClist* rc); static void storedump(char* msg, NClist* triples); #endif -/* Define default rc files and aliases, also defines search order*/ -static const char* rcfilenames[] = {".daprc",".dodsrc",".ncrc",NULL}; +/* Define default rc files and aliases, also defines load order*/ +static const char* rcfilenames[] = {".ncrc", ".daprc", ".dodsrc",NULL}; /**************************************************/ /* External Entry Points */ static NCRCglobalstate* ncrc_globalstate = NULL; -/* Get global state */ -NCRCglobalstate* -ncrc_getglobalstate(void) +static int NCRCinitialized = 0; + +/* Initialize defaults */ +void +ncrc_initialize(void) { + const char* tmp = NULL; + + if(NCRCinitialized) return; if(ncrc_globalstate == NULL) { ncrc_globalstate = calloc(1,sizeof(NCRCglobalstate)); } + /* Get environment variables */ + if(getenv(NCRCENVIGNORE) != NULL) + ncrc_globalstate->rcinfo.ignore = 1; + tmp = getenv(NCRCENVRC); + if(tmp != NULL && strlen(tmp) > 0) + ncrc_globalstate->rcinfo.rcfile = strdup(tmp); + NCRCinitialized = 1; +} + +/* Get global state */ +NCRCglobalstate* +ncrc_getglobalstate(void) +{ + if(!NCRCinitialized) + ncrc_initialize(); return ncrc_globalstate; } @@ -66,6 +86,7 @@ ncrc_freeglobalstate(void) if(ncrc_globalstate != NULL) { nullfree(ncrc_globalstate->tempdir); nullfree(ncrc_globalstate->home); + nullfree(ncrc_globalstate->cwd); NC_rcclear(&ncrc_globalstate->rcinfo); free(ncrc_globalstate); ncrc_globalstate = NULL; @@ -94,57 +115,67 @@ rcfreetriples(NClist* rc) nclistfree(rc); } -/* locate, read and compile the rc file, if any */ +/* locate, read and compile the rc files, if any */ int NC_rcload(void) { - int ret = NC_NOERR; + int i,ret = NC_NOERR; char* path = NULL; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); + NCRCglobalstate* globalstate = NULL; + NClist* rcfileorder = nclistnew(); + if(!NCRCinitialized) ncrc_initialize(); + globalstate = ncrc_getglobalstate(); + if(globalstate->rcinfo.ignore) { - nclog(NCLOGDBG,"No .daprc|.dodsrc runtime configuration file specified; continuing"); - return (NC_NOERR); + nclog(NCLOGDBG,".rc file loading suppressed"); + goto done; } - if(globalstate->rcinfo.loaded) return (NC_NOERR); - - /* locate the configuration files in the following order: - 1. specified by NC_set_rcfile - 2. set by DAPRCFILE env variable - 3. ./ (current directory) - 4. $HOME/ + if(globalstate->rcinfo.loaded) goto done; + + /* locate the configuration files in order of use: + 1. Specified by NCRCENV_RC environment variable. + 2. If NCRCENV_RC is not set then merge the set of rc files in this order: + 1. $HOME/.ncrc + 2. $HOME/.daprc + 3. $HOME/.docsrc + 4. $CWD/.ncrc + 5. $CWD/.daprc + 6. $CWD/.docsrc + Entries in later files override any of the earlier files */ if(globalstate->rcinfo.rcfile != NULL) { /* always use this */ - path = strdup(globalstate->rcinfo.rcfile); - } else if(getenv(RCFILEENV) != NULL && strlen(getenv(RCFILEENV)) > 0) { - path = strdup(getenv(RCFILEENV)); + nclistpush(rcfileorder,strdup(globalstate->rcinfo.rcfile)); } else { const char** rcname; - int found = 0; - for(rcname=rcfilenames;!found && *rcname;rcname++) { - ret = rcsearch(".",*rcname,&path); - if(ret == NC_NOERR && path == NULL) /* try $HOME */ - ret = rcsearch(globalstate->home,*rcname,&path); - if(ret != NC_NOERR) - goto done; - if(path != NULL) - found = 1; + const char* dirnames[3]; + const char** dir; + + dirnames[0] = globalstate->home; + dirnames[1] = globalstate->cwd; + dirnames[2] = NULL; + + for(dir=dirnames;*dir;dir++) { + for(rcname=rcfilenames;*rcname;rcname++) { + ret = rcsearch(*dir,*rcname,&path); + if(ret == NC_NOERR && path != NULL) + nclistpush(rcfileorder,path); + path = NULL; + } } } - if(path == NULL) { - nclog(NCLOGDBG,"No .daprc|.dodsrc runtime configuration file specified; continuing"); - } else { -#ifdef D4DEBUG - fprintf(stderr, "RC file: %s\n", path); -#endif - if((ret=rccompile(path))) { - nclog(NCLOGERR, "Error parsing %s\n",path); + for(i=0;ircinfo.loaded = 1; /* even if not exists */ - nullfree(path); + nclistfreeall(rcfileorder); return (ret); } @@ -155,10 +186,13 @@ NC_rcload(void) char* NC_rclookup(const char* key, const char* hostport) { - struct NCTriple* triple = rclocate(key,hostport); + struct NCTriple* triple = NULL; + if(!NCRCinitialized) ncrc_initialize(); + triple = rclocate(key,hostport); return (triple == NULL ? NULL : triple->value); } +#if 0 /*! Set the absolute path to use for the rc file. WARNING: this MUST be called before any other @@ -195,6 +229,7 @@ NC_set_rcfile(const char* rcfile) done: return stat; } +#endif /**************************************************/ /* RC processing functions */ @@ -224,6 +259,8 @@ rctrim(char* text) size_t len = 0; int i; + if(text == NULL) return; + /* locate first non-trimchar */ for(;*p;p++) { if(strchr(TRIMCHARS,*p) == NULL) break; /* hit non-trim char */ @@ -276,7 +313,7 @@ rcorder(NClist* rc) nclistfree(tmprc); } -/* Create a triple store from a file */ +/* Merge a triple store from a file*/ static int rccompile(const char* path) { @@ -289,24 +326,23 @@ rccompile(const char* path) NCRCglobalstate* globalstate = ncrc_getglobalstate(); if((ret=NC_readfile(path,tmp))) { - nclog(NCLOGERR, "Could not open configuration file: %s",path); + nclog(NCLOGWARN, "Could not open configuration file: %s",path); goto done; } contents = ncbytesextract(tmp); if(contents == NULL) contents = strdup(""); /* Either reuse or create new */ rc = globalstate->rcinfo.triples; - if(rc != NULL) - rcfreetriples(rc); /* clear out any old data */ - else { + if(rc == NULL) { rc = nclistnew(); globalstate->rcinfo.triples = rc; } nextline = contents; for(;;) { char* line; - char* key; - char* value; + char* key = NULL; + char* value = NULL; + char* host = NULL; size_t llen; NCTriple* triple; @@ -315,14 +351,11 @@ rccompile(const char* path) rctrim(line); /* trim leading and trailing blanks */ if(line[0] == '#') continue; /* comment */ if((llen=strlen(line)) == 0) continue; /* empty line */ - triple = (NCTriple*)calloc(1,sizeof(NCTriple)); - if(triple == NULL) {ret = NC_ENOMEM; goto done;} if(line[0] == LTAG) { char* url = ++line; char* rtag = strchr(line,RTAG); if(rtag == NULL) { nclog(NCLOGERR, "Malformed [url] in %s entry: %s",path,line); - free(triple); continue; } line = rtag + 1; @@ -331,7 +364,6 @@ rccompile(const char* path) if(uri) ncurifree(uri); if(ncuriparse(url,&uri)) { nclog(NCLOGERR, "Malformed [url] in %s entry: %s",path,line); - free(triple); continue; } ncbytesclear(tmp); @@ -341,9 +373,9 @@ rccompile(const char* path) ncbytescat(tmp,uri->port); } ncbytesnull(tmp); - triple->host = ncbytesextract(tmp); - if(strlen(triple->host)==0) - {free(triple->host); triple->host = NULL;} + host = ncbytesextract(tmp); + if(strlen(host)==0) + {free(host); host = NULL;} } /* split off key and value */ key=line; @@ -354,16 +386,30 @@ rccompile(const char* path) *value = '\0'; value++; } - triple->key = strdup(key); - triple->value = strdup(value); + /* See if key already exists */ + triple = rclocate(key,host); + if(triple != NULL) { + nullfree(triple->host); + nullfree(triple->key); + nullfree(triple->value); + } else { + triple = (NCTriple*)calloc(1,sizeof(NCTriple)); + if(triple == NULL) {ret = NC_ENOMEM; goto done;} + nclistpush(rc,triple); + } + triple->host = host; host = NULL; + triple->key = nulldup(key); + triple->value = nulldup(value); + rctrim(triple->host); rctrim(triple->key); rctrim(triple->value); -#ifdef D4DEBUG + +#ifdef DRCDEBUG fprintf(stderr,"rc: host=%s key=%s value=%s\n", (triple->host != NULL ? triple->host : ""), - triple->key,triple->valu); + triple->key,triple->value); #endif - nclistpush(rc,triple); + triple = NULL; } rcorder(rc); @@ -458,9 +504,13 @@ NC_rcfile_insert(const char* key, const char* value, const char* hostport) int ret = NC_NOERR; /* See if this key already defined */ struct NCTriple* triple = NULL; - NCRCglobalstate* globalstate = ncrc_getglobalstate(); - NClist* rc = globalstate->rcinfo.triples; + NCRCglobalstate* globalstate = NULL; + NClist* rc = NULL; + if(!NCRCinitialized) ncrc_initialize(); + globalstate = ncrc_getglobalstate(); + rc = globalstate->rcinfo.triples; + if(rc == NULL) { rc = nclistnew(); if(rc == NULL) {ret = NC_ENOMEM; goto done;} diff --git a/libnczarr/zcache.c b/libnczarr/obsolete/zcache.c similarity index 99% rename from libnczarr/zcache.c rename to libnczarr/obsolete/zcache.c index 4ca427e235..479af2be46 100644 --- a/libnczarr/zcache.c +++ b/libnczarr/obsolete/zcache.c @@ -388,7 +388,7 @@ From Zarr V2 Specification: a key formed from the index of the chunk within the grid of chunks representing the array. To form a string key for a chunk, the indices are converted to strings and concatenated -with the period character (".") separating each index. For +with the dimension_separator character ('/' or '.') separating each index. For example, given an array with shape (10000, 10000) and chunk shape (1000, 1000) there will be 100 chunks laid out in a 10 by 10 grid. The chunk with indices (0, 0) provides data for rows diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c index d524f228dc..b1ed766016 100644 --- a/libnczarr/zarr.c +++ b/libnczarr/zarr.c @@ -374,6 +374,7 @@ applycontrols(NCZ_FILE_INFO_T* zinfo) int i,stat = NC_NOERR; const char* value = NULL; NClist* modelist = nclistnew(); + int noflags = 0; /* track non-default negative flags */ if((value = controllookup((const char**)zinfo->envv_controls,"mode")) != NULL) { if((stat = NCZ_comma_parse(value,modelist))) goto done; @@ -382,12 +383,20 @@ applycontrols(NCZ_FILE_INFO_T* zinfo) zinfo->controls.mapimpl = NCZM_DEFAULT; for(i=0;icontrols.flags |= FLAG_PUREZARR; + if(strcasecmp(p,PUREZARRCONTROL)==0) zinfo->controls.flags |= (FLAG_PUREZARR|FLAG_XARRAYDIMS); else if(strcasecmp(p,XARRAYCONTROL)==0) zinfo->controls.flags |= (FLAG_XARRAYDIMS|FLAG_PUREZARR); /*xarray=>zarr*/ + else if(strcasecmp(p,NOXARRAYCONTROL)==0) { + noflags |= FLAG_XARRAYDIMS; + zinfo->controls.flags |= FLAG_PUREZARR; /*noxarray=>zarr*/ + } else if(strcasecmp(p,"zip")==0) zinfo->controls.mapimpl = NCZM_ZIP; else if(strcasecmp(p,"file")==0) zinfo->controls.mapimpl = NCZM_FILE; else if(strcasecmp(p,"s3")==0) zinfo->controls.mapimpl = NCZM_S3; } + /* Apply negative controls by turning off negative flags */ + /* This is necessary to avoid order dependence of mode flags when both positive and negative flags are defined */ + zinfo->controls.flags &= (~noflags); + /* Process other controls */ if((value = controllookup((const char**)zinfo->envv_controls,"log")) != NULL) { zinfo->controls.flags |= FLAG_LOGGING; diff --git a/libnczarr/zarr.h b/libnczarr/zarr.h index 8d1de0b309..497704ee92 100644 --- a/libnczarr/zarr.h +++ b/libnczarr/zarr.h @@ -12,6 +12,8 @@ #ifndef ZARR_H #define ZARR_H +struct ChunkKey; + /* zarr.c */ extern int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, const char** controls); extern int ncz_open_dataset(NC_FILE_INFO_T*, const char** controls); @@ -56,7 +58,7 @@ extern int NCZ_createobject(NCZMAP* zmap, const char* key, size64_t size); extern int NCZ_uploadjson(NCZMAP* zmap, const char* key, NCjson* json); extern int NCZ_downloadjson(NCZMAP* zmap, const char* key, NCjson** jsonp); extern int NCZ_isLittleEndian(void); -extern int NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, NClist* objlist); +extern int NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, char dimsep, NClist* objlist); extern int NCZ_grpname_full(int gid, char** pathp); extern int ncz_get_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var); extern int NCZ_comma_parse(const char* s, NClist* list); @@ -65,6 +67,9 @@ extern char** NCZ_clonestringvec(size_t len, const char** vec); extern void NCZ_freestringvec(size_t len, char** vec); extern int NCZ_create_fill_chunk(size64_t chunksize, size_t typesize, void* fill, void** fillchunkp); extern int NCZ_s3clear(ZS3INFO* s3); +extern int NCZ_ischunkname(const char* name,char dimsep); +extern char* NCZ_chunkpath(struct ChunkKey key,char dimsep); + /* Export */ EXTERNL int NCZ_s3urlprocess(NCURI* url, ZS3INFO* s3); diff --git a/libnczarr/zcache.h b/libnczarr/zcache.h index 3f8548535f..f54ae04a53 100644 --- a/libnczarr/zcache.h +++ b/libnczarr/zcache.h @@ -16,7 +16,10 @@ typedef struct NCZCacheEntry { struct List {void* next; void* prev; void* unused;} list; int modified; size64_t indices[NC_MAX_VAR_DIMS]; - char* key; + struct ChunkKey { + char* varkey; /* key to the containing variable */ + char* chunkkey; /* name of the chunk */ + } key; size64_t hashkey; void* data; } NCZCacheEntry; @@ -29,20 +32,20 @@ typedef struct NCZChunkCache { size_t maxentries; /* Max number of entries allowed */ NClist* mru; /* all cache entries in mru order */ struct NCxcache* xcache; + char dimension_separator; } NCZChunkCache; /**************************************************/ extern int NCZ_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, float preemption); extern int NCZ_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var); - -extern int NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t, NCZChunkCache** cachep); +extern int NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t, char dimsep, NCZChunkCache** cachep); extern void NCZ_free_chunk_cache(NCZChunkCache* cache); extern int NCZ_read_cache_chunk(NCZChunkCache* cache, const size64_t* indices, void** datap); extern int NCZ_flush_chunk_cache(NCZChunkCache* cache); extern size64_t NCZ_cache_entrysize(NCZChunkCache* cache); extern NCZCacheEntry* NCZ_cache_entry(NCZChunkCache* cache, const size64_t* indices); extern size64_t NCZ_cache_size(NCZChunkCache* cache); -extern int NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, char** keyp); +extern int NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, struct ChunkKey* key); #endif /*ZCACHE_H*/ diff --git a/libnczarr/zdebug.h b/libnczarr/zdebug.h index a7cbd5be2d..f6f12dc720 100644 --- a/libnczarr/zdebug.h +++ b/libnczarr/zdebug.h @@ -9,7 +9,7 @@ #undef ZDEBUG1 /* detailed debug */ #undef ZCATCH /* Warning: significant performance impact */ -#undef ZTRACING /* Warning: significant performance impact */ +#define ZTRACING /* Warning: significant performance impact */ #include "ncexternl.h" #include "nclog.h" diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index 486731103e..a1b4b53569 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -58,8 +58,26 @@ set_auto(void* func, void *client_data) int NCZ_initialize_internal(void) { + int stat = NC_NOERR; + char* dimsep = NULL; + NCRCglobalstate* ngs = NULL; + ncz_initialized = 1; - return NC_NOERR; + /* Load the .rc file */ + if((stat=NC_rcload())) goto done; + ngs = ncrc_getglobalstate(); + if(ngs != NULL) { + /* Defaults */ + ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR; + dimsep = NC_rclookup("ZARR.DIMENSION_SEPARATOR",NULL); + if(dimsep != NULL) { + /* Verify its value */ + if(dimsep != NULL && strlen(dimsep) == 1 && islegaldimsep(dimsep[0])) + ngs->zarr.dimension_separator = dimsep[0]; + } + } +done: + return stat; } /** diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 791490045f..b2dac0fba9 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -57,6 +57,13 @@ #define PUREZARRCONTROL "zarr" #define XARRAYCONTROL "xarray" +#define NOXARRAYCONTROL "noxarray" + +#define LEGAL_DIM_SEPARATORS "./" +#define DFALT_DIM_SEPARATOR '.' + +#define islegaldimsep(c) ((c) != '\0' && strchr(LEGAL_DIM_SEPARATORS,(c)) != NULL) + /* Mnemonics */ #define ZCLOSE 1 /* this is closeorabort as opposed to enddef */ @@ -112,7 +119,7 @@ typedef struct NCZ_DIM_INFO { NCZcommon common; } NCZ_DIM_INFO_T; -/** Strut to hold ZARR-specific info for attributes. */ +/** Struct to hold ZARR-specific info for attributes. */ typedef struct NCZ_ATT_INFO { NCZcommon common; } NCZ_ATT_INFO_T; @@ -143,6 +150,7 @@ typedef struct NCZ_VAR_INFO { size_t scalar; struct NCZChunkCache* cache; struct NClist* xarray; /* names from _ARRAY_DIMENSIONS */ + char dimension_separator; /* '.' | '/' */ } NCZ_VAR_INFO_T; /* Struct to hold ZARR-specific info for a field. */ diff --git a/libnczarr/zmap.c b/libnczarr/zmap.c index d864ea1255..19e19a2cbe 100644 --- a/libnczarr/zmap.c +++ b/libnczarr/zmap.c @@ -154,10 +154,27 @@ nczmap_write(NCZMAP* map, const char* key, size64_t start, size64_t count, const return map->api->write(map, key, start, count, content); } +/* Define a static qsort comparator for strings for use with qsort */ +static int +cmp_strings(const void* a1, const void* a2) +{ + const char** s1 = (const char**)a1; + const char** s2 = (const char**)a2; + return strcmp(*s1,*s2); +} + int nczmap_search(NCZMAP* map, const char* prefix, NClist* matches) { - return map->api->search(map, prefix, matches); + int stat = NC_NOERR; + if((stat = map->api->search(map, prefix, matches)) == NC_NOERR) { + /* sort the list */ + if(nclistlength(matches) > 1) { + void* base = nclistcontents(matches); + qsort(base, nclistlength(matches), sizeof(char*), cmp_strings); + } + } + return stat; } /**************************************************/ diff --git a/libnczarr/zmap.h b/libnczarr/zmap.h index 2d89c82b11..eb44cdbb1d 100644 --- a/libnczarr/zmap.h +++ b/libnczarr/zmap.h @@ -155,7 +155,7 @@ NCZM_S3=3, /* Amazon S3 implementation */ } NCZM_IMPL; /* Define the default map implementation */ -#define NCZM_DEFAULT NCZM_ZIP +#define NCZM_DEFAULT NCZM_FILE /* Define the per-implementation limitations flags */ typedef size64_t NCZM_FEATURES; diff --git a/libnczarr/zmap_file.c b/libnczarr/zmap_file.c index ebcc8df0fd..d73f0c1114 100755 --- a/libnczarr/zmap_file.c +++ b/libnczarr/zmap_file.c @@ -647,9 +647,11 @@ platformtestcontentbearing(ZFMAP* zfmap, const char* truepath) /* Localize */ if((ret = nczm_localize(truepath,&local,LOCALIZE))) goto done; - + errno = 0; - if((ret = stat(local, &buf)) < 0) { + ret = NCstat(local, &buf); + ZTRACEMORE(6,"stat: local=%s ret=%d, errno=%d st_mode=%d",local,ret,errno,buf.st_mode); + if(ret < 0) { ret = platformerr(errno); } else if(S_ISDIR(buf.st_mode)) { ret = NC_EEMPTY; @@ -926,7 +928,7 @@ platformdeleter(ZFMAP* zfmap, NClist* segments, int depth) if((path = NCpathcvt(tmp))==NULL) {ret = NC_ENOMEM; goto done;} errno = 0; - ret = stat(path, &statbuf); + ret = NCstat(path, &statbuf); if(ret < 0) { if(errno == ENOENT) {ret = NC_NOERR; goto done;} else {ret = platformerr(errno); goto done;} @@ -1184,7 +1186,7 @@ verify(const char* path, int isdir) ret = NCaccess(path,ACCESS_MODE_EXISTS); if(ret < 0) return 1; /* If it does not exist, then it can be anything */ - ret = stat(path,&buf); + ret = NCstat(path,&buf); if(ret < 0) abort(); if(isdir && S_ISDIR(buf.st_mode)) return 1; if(!isdir && S_ISREG(buf.st_mode)) return 1; @@ -1207,7 +1209,7 @@ testifdir(const char* path, int* isdirp, char** truepathp) if((truepath = NCpathcvt(tmp))==NULL) {ret = NC_ENOMEM; goto done;} errno = 0; - ret = stat(truepath, &statbuf); + ret = NCstat(truepath, &statbuf); if(ret < 0) { if(errno == ENOENT) ret = NC_ENOTFOUND; /* path does not exist */ diff --git a/libnczarr/zmap_zip.c b/libnczarr/zmap_zip.c index a077a47908..289b1e44f0 100755 --- a/libnczarr/zmap_zip.c +++ b/libnczarr/zmap_zip.c @@ -323,7 +323,7 @@ zipread(NCZMAP* map, const char* key, size64_t start, size64_t count, void* cont char* buffer = NULL; char* truekey = NULL; zip_int64_t red = 0; - + ZTRACE(6,"map=%s key=%s start=%llu count=%llu",map->url,key,start,count); switch(stat = zzlookupobj(zzmap,key,&zindex)) { @@ -382,7 +382,7 @@ zipwrite(NCZMAP* map, const char* key, size64_t start, size64_t count, const voi zip_int32_t compression = 0; zip_error_t zerror; void* localbuffer = NULL; - + ZTRACE(6,"map=%s key=%s start=%llu count=%llu",map->url,key,start,count); zip_error_init(&zerror); diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index 0cd95c8af8..7ed0dbb7ee 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -368,6 +368,17 @@ ncz_sync_var(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var) if((stat = NCJappend(jvar,jtmp))) goto done; jtmp = NULL; + /* dimension_separator key */ + /* Single char defining the separator in chunk keys */ + if(zvar->dimension_separator != DFALT_DIM_SEPARATOR) { + char sep[2]; + sep[0] = zvar->dimension_separator;/* make separator a string*/ + sep[1] = '\0'; + if((stat = NCJnewstring(NCJ_STRING,sep,&jtmp))) goto done; + if((stat = NCJinsert(jvar,"dimension_separator",jtmp))) goto done; + jtmp = NULL; + } + /* build .zarray path */ if((stat = nczm_concat(fullpath,ZARRAY,&key))) goto done; @@ -1380,6 +1391,22 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) {stat = THROW(NC_ENOMEM); goto done;} if((stat = decodeints(jvalue, shapes))) goto done; } + /* Capture dimension_separator (must precede chunk cache creation) */ + { + NCRCglobalstate* ngs = ncrc_getglobalstate(); + assert(ngs != NULL); + zvar->dimension_separator = 0; + if((stat = NCJdictget(jvar,"dimension_separator",&jvalue))) goto done; + if(jvalue != NULL) { + /* Verify its value */ + if(jvalue->sort == NCJ_STRING && jvalue->value != NULL && strlen(jvalue->value) == 1) + zvar->dimension_separator = jvalue->value[0]; + } + /* If value is invalid, then use global default */ + if(!islegaldimsep(zvar->dimension_separator)) + zvar->dimension_separator = ngs->zarr.dimension_separator; /* use global value */ + assert(islegaldimsep(zvar->dimension_separator)); /* we are hosed */ + } /* chunks */ { int rank; @@ -1406,7 +1433,7 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) } zvar->chunksize = zvar->chunkproduct * var->type_info->size; /* Create the cache */ - if((stat = NCZ_create_chunk_cache(var,var->type_info->size*zvar->chunkproduct,&zvar->cache))) + if((stat = NCZ_create_chunk_cache(var,var->type_info->size*zvar->chunkproduct,zvar->dimension_separator,&zvar->cache))) goto done; } } @@ -1970,17 +1997,18 @@ computedimrefs(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int purezarr, int xarra /* xarray => purezarr */ assert(!xarray || purezarr); - if(xarray) {/* Read in the attributes to get xarray dimdef attribute */ + if(xarray) {/* Read in the attributes to get xarray dimdef attribute; Note that it might not exist */ char zdimname[4096]; if(zvar->xarray == NULL) { assert(nclistlength(dimnames) == 0); if((stat = ncz_read_atts(file,(NC_OBJ*)var))) goto done; } - assert(zvar->xarray != NULL); - /* convert xarray to the dimnames */ - for(i=0;ixarray);i++) { - snprintf(zdimname,sizeof(zdimname),"/%s",(const char*)nclistget(zvar->xarray,i)); - nclistpush(dimnames,strdup(zdimname)); + if(zvar->xarray != NULL) { + /* convert xarray to the dimnames */ + for(i=0;ixarray);i++) { + snprintf(zdimname,sizeof(zdimname),"/%s",(const char*)nclistget(zvar->xarray,i)); + nclistpush(dimnames,strdup(zdimname)); + } } createdims = 1; /* may need to create them */ } diff --git a/libnczarr/zutil.c b/libnczarr/zutil.c index 29a7a19ae0..891fe86ee8 100644 --- a/libnczarr/zutil.c +++ b/libnczarr/zutil.c @@ -451,7 +451,7 @@ is an object in the map. Note: need to test with "/", "", and with and without trailing "/". */ int -NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, NClist* objlist) +NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, char dimsep, NClist* objlist) { int i,stat=NC_NOERR; NClist* matches = nclistnew(); @@ -460,7 +460,6 @@ NCZ_subobjects(NCZMAP* map, const char* prefix, const char* tag, NClist* objlist /* Get the list of names just below prefix */ if((stat = nczmap_search(map,prefix,matches))) goto done; for(i=0;i= 2 && name[0] == '.' && name[1] == 'z') continue; - for(p=name;*p;p++) { - if(*p != '.' && strchr("0123456789",*p) == NULL) break; - } - if(*p == '\0') continue; /* looks like a chunk name */ + if(NCZ_ischunkname(name,dimsep)) + continue; /* Create // and see if it exists */ ncbytesclear(path); ncbytescat(path,prefix); @@ -878,3 +875,32 @@ endswith(const char* s, const char* suffix) return 1; } +int +NCZ_ischunkname(const char* name,char dimsep) +{ + int stat = NC_NOERR; + const char* p; + if(strchr("0123456789",name[0])== NULL) + stat = NC_ENCZARR; + else for(p=name;*p;p++) { + if(*p != dimsep && strchr("0123456789",*p) == NULL) /* approximate */ + {stat = NC_ENCZARR; break;} + } + return stat; +} + +char* +NCZ_chunkpath(struct ChunkKey key,char dimsep) +{ + size_t plen = nulllen(key.varkey)+1+nulllen(key.chunkkey); + char* path = (char*)malloc(plen+1); + char sdimsep[2]; + + if(path == NULL) return NULL; + path[0] = '\0'; + strlcat(path,key.varkey,plen+1); + sdimsep[0] = dimsep; sdimsep[1] = '\0'; + strlcat(path,sdimsep,plen+1); + strlcat(path,key.chunkkey,plen+1); + return path; +} diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index 28ab00fbf2..7ba84f9683 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -381,6 +381,9 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims, zvar->common.file = h5; zvar->scalar = (ndims == 0 ? 1 : 0); + zvar->dimension_separator = ncrc_getglobalstate()->zarr.dimension_separator; + assert(zvar->dimension_separator != 0); + /* Set these state flags for the var. */ var->is_new_var = NC_TRUE; var->meta_read = NC_TRUE; @@ -455,7 +458,7 @@ var->type_info->rc++; var->chunk_cache_preemption = 1; /* not used */ /* Create the cache */ - if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,&zvar->cache))) + if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,zvar->dimension_separator,&zvar->cache))) BAIL(retval); /* Is this a variable with a chunksize greater than the current cache size? */ diff --git a/libnczarr/zxcache.c b/libnczarr/zxcache.c index 61418fa21a..4aec87241c 100644 --- a/libnczarr/zxcache.c +++ b/libnczarr/zxcache.c @@ -144,7 +144,7 @@ NCZ_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var) * @author Dennis Heimbigner, Ed Hartnett */ int -NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, NCZChunkCache** cachep) +NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, char dimsep, NCZChunkCache** cachep) { int stat = NC_NOERR; NCZChunkCache* cache = NULL; @@ -163,6 +163,7 @@ NCZ_create_chunk_cache(NC_VAR_INFO_T* var, size64_t chunksize, NCZChunkCache** c assert(cache->fillchunk == NULL); cache->fillchunk = NULL; cache->chunksize = chunksize; + cache->dimension_separator = dimsep; /* Figure out the actual cache size */ cachesize = var->chunk_cache_size; @@ -203,7 +204,7 @@ NCZ_free_chunk_cache(NCZChunkCache* cache) NCZCacheEntry* entry = nclistremove(cache->mru,0); (void)ncxcacheremove(cache->xcache,entry->hashkey,&ptr); assert(ptr == entry); - nullfree(entry->data); nullfree(entry->key); nullfree(entry); + nullfree(entry->data); nullfree(entry->key.varkey); nullfree(entry->key.chunkkey); nullfree(entry); } #ifdef DEBUG fprintf(stderr,"|cache.free|=%ld\n",nclistlength(cache->mru)); @@ -297,7 +298,7 @@ fprintf(stderr,"|cache.read.lru|=%ld\n",nclistlength(cache->mru)); done: if(created && stat == NC_NOERR) stat = NC_EEMPTY; /* tell upper layers */ - if(entry) {nullfree(entry->data); nullfree(entry->key);} + if(entry) {nullfree(entry->data); nullfree(entry->key.varkey); nullfree(entry->key.chunkkey);} nullfree(entry); return THROW(stat); } @@ -333,7 +334,7 @@ fprintf(stderr,"|cache.write|=%ld\n",nclistlength(cache->mru)); entry = NULL; done: - if(entry) {nullfree(entry->data); nullfree(entry->key);} + if(entry) {nullfree(entry->data); nullfree(entry->key.varkey); nullfree(entry->key.chunkkey);} nullfree(entry); return THROW(stat); } @@ -359,7 +360,7 @@ makeroom(NCZChunkCache* cache) if(e->modified) /* flush to file */ stat=put_chunk(cache,e); /* reclaim */ - nullfree(e->data); nullfree(e->key); nullfree(e); + nullfree(e->data); nullfree(e->key.varkey); nullfree(e->key.chunkkey); nullfree(e); } #ifdef DEBUG fprintf(stderr,"|cache.makeroom|=%ld\n",nclistlength(cache->mru)); @@ -423,32 +424,36 @@ From Zarr V2 Specification: a key formed from the index of the chunk within the grid of chunks representing the array. To form a string key for a chunk, the indices are converted to strings and concatenated -with the period character (".") separating each index. For -example, given an array with shape (10000, 10000) and chunk -shape (1000, 1000) there will be 100 chunks laid out in a 10 by -10 grid. The chunk with indices (0, 0) provides data for rows -0-1000 and columns 0-1000 and is stored under the key "0.0"; the -chunk with indices (2, 4) provides data for rows 2000-3000 and -columns 4000-5000 and is stored under the key "2.4"; etc." +with the dimension_separator character ('.' or '/') separating +each index. For example, given an array with shape (10000, +10000) and chunk shape (1000, 1000) there will be 100 chunks +laid out in a 10 by 10 grid. The chunk with indices (0, 0) +provides data for rows 0-1000 and columns 0-1000 and is stored +under the key "0.0"; the chunk with indices (2, 4) provides data +for rows 2000-3000 and columns 4000-5000 and is stored under the +key "2.4"; etc." */ /** * @param R Rank * @param chunkindices The chunk indices + * @param dimsep the dimension separator * @param keyp Return the chunk key string */ int -NCZ_buildchunkkey(size_t R, const size64_t* chunkindices, char** keyp) +NCZ_buildchunkkey(size_t R, const size64_t* chunkindices, char dimsep, char** keyp) { int stat = NC_NOERR; int r; NCbytes* key = ncbytesnew(); if(keyp) *keyp = NULL; + + assert(islegaldimsep(dimsep)); for(r=0;r 0) ncbytescat(key,"."); + if(r > 0) ncbytesappend(key,dimsep); /* Print as decimal with no leading zeros */ snprintf(sindex,sizeof(sindex),"%lu",(unsigned long)chunkindices[r]); ncbytescat(key,sindex); @@ -485,7 +490,11 @@ put_chunk(NCZChunkCache* cache, const NCZCacheEntry* entry) zfile = ((cache->var->container)->nc4_info)->format_file_info; map = zfile->map; - stat = nczmap_write(map,entry->key,0,cache->chunksize,entry->data); + { + char* path = NCZ_chunkpath(entry->key,cache->dimension_separator); + stat = nczmap_write(map,path,0,cache->chunksize,entry->data); + nullfree(path); + } switch(stat) { case NC_NOERR: break; @@ -514,7 +523,7 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry) NC_FILE_INFO_T* file = NULL; NCZ_FILE_INFO_T* zfile = NULL; - ZTRACE(5,"cache.var=%s entry.key=%s",cache->var->hdr.name,entry->key); + ZTRACE(5,"cache.var=%s entry.key=%s sep=%d",cache->var->hdr.name,entry->key,cache->dimension_separator); LOG((3, "%s: file: %p", __func__, file)); @@ -523,32 +532,33 @@ get_chunk(NCZChunkCache* cache, NCZCacheEntry* entry) map = zfile->map; assert(map && entry->data); - stat = nczmap_read(map,entry->key,0,cache->chunksize,(char*)entry->data); + { + char* path = NCZ_chunkpath(entry->key,cache->dimension_separator); + stat = nczmap_read(map,path,0,cache->chunksize,(char*)entry->data); + nullfree(path); + } return ZUNTRACE(stat); } - int -NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, char** keyp) +NCZ_buildchunkpath(NCZChunkCache* cache, const size64_t* chunkindices, struct ChunkKey* key) { int stat = NC_NOERR; char* chunkname = NULL; char* varkey = NULL; - char* key = NULL; + assert(key != NULL); /* Get the chunk object name */ - if((stat = NCZ_buildchunkkey(cache->ndims, chunkindices, &chunkname))) goto done; + if((stat = NCZ_buildchunkkey(cache->ndims, chunkindices, cache->dimension_separator, &chunkname))) goto done; /* Get the var object key */ if((stat = NCZ_varkey(cache->var,&varkey))) goto done; - /* Prefix the path to the containing variable object */ - if((stat=nczm_concat(varkey,chunkname,&key))) goto done; - if(keyp) {*keyp = key; key = NULL;} + key->varkey = varkey; varkey = NULL; + key->chunkkey = chunkname; chunkname = NULL; done: nullfree(chunkname); nullfree(varkey); - nullfree(key); return THROW(stat); } diff --git a/libnetcdf.settings.in b/libnetcdf.settings.in index 1f2e7ff188..0cc38514e1 100644 --- a/libnetcdf.settings.in +++ b/libnetcdf.settings.in @@ -31,7 +31,7 @@ HDF5 Support: @HAS_HDF5@ NetCDF-4 API: @HAS_NC4@ NC-4 Parallel Support: @HAS_PARALLEL4@ PnetCDF Support: @HAS_PNETCDF@ -DAP2 Support: @HAS_DAP@ +DAP2 Support: @HAS_DAP2@ DAP4 Support: @HAS_DAP4@ Byte-Range Support: @HAS_BYTERANGE@ Diskless Support: @HAS_DISKLESS@ diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index 0f94a8be5e..559e84deab 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -11,7 +11,7 @@ */ #include "config.h" -#include +#include "nc4internal.h" #include "nc4dispatch.h" #ifdef USE_HDF5 #include "hdf5internal.h" diff --git a/ncdap_test/Makefile.am b/ncdap_test/Makefile.am index b38999acca..4a03956efb 100644 --- a/ncdap_test/Makefile.am +++ b/ncdap_test/Makefile.am @@ -74,8 +74,7 @@ check_PROGRAMS += t_misc check_PROGRAMS += t_ncf330 if ENABLE_DAP_AUTH_TESTS -TESTS += testbasicauth.sh -TESTS += testcontainerauth.sh +TESTS += testauth.sh endif endif #ENABLE_DAP_REMOTE_TESTS diff --git a/ncdap_test/testauth.sh b/ncdap_test/testauth.sh index 02dd6769ca..8a16661fc9 100755 --- a/ncdap_test/testauth.sh +++ b/ncdap_test/testauth.sh @@ -3,64 +3,60 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -# Enable if using localhost -LOCAL=1 +# Since this involves a shared resource: the .rc files in current working directory, +# we need to isolate from any other test. -RCEMBED=1 -RCLOCAL=1 -RCHOME=1 -RCENV=1 -RCPREC=1 - -# Not currently testable in netcdf -#RCSPEC=1 +# Make sure execdir and srcdir absolute paths are available +WD=`pwd` +cd $srcdir ; abs_srcdir=`pwd` ; cd $WD +cd $execdir ; abs_execdir=`pwd` ; cd $WD + +# Now create a special directory +# And enter it to execute tests +rm -fr rcauthdir +mkdir rcauthdir +cd rcauthdir +WD=`pwd` #SHOW=1 #DBG=1 -# Choose at most 1 -#GDB=1 -#VG=1 +if test "x$NCAUTH_HOMETEST" != x ; then + RCHOME=1 +fi -NFL=1 +COOKIES="${WD}/.cookies_test" -WD=`pwd` +RC=.daprc -NETRCFILE=$WD/test_auth_netrc -# This is the control variable; set when needed -unset NETRC +if test "x$FPISMSVC" = x ; then +NETRC=.netrc_test +NETRCIMP=.netrc +else +NETRC=_netrc_test +NETRCIMP=_netrc +fi -COOKIES="${WD}/test_auth_cookies" +LOCALRCFILES="$WD/.dodsrc $WD/.daprc $WD/.ncrc $WD/$NETRC $WD/$NETRCIMP" +HOMERCFILES="$HOME/.dodsrc $HOME/.daprc $HOME/.ncrc $HOME/$NETRC $HOME/$NETRCIMP" -RC=.daprc +NETRCFILE=$WD/$NETRC +DAPRCFILE=$WD/$RC + +HOMENETRCFILE=$HOME/$NETRC +HOMEDAPRCFILE=$HOME/$RC -OCLOGFILE=stderr if test "x$DBG" = x1 ; then SHOW=1 fi # Major parameters -BASICCOMBO="tiggeUser:tigge" -BADCOMBO="tiggeUser:xxxxx" -URLPATH="thredds/dodsC/testRestrictedDataset/testData2.nc" -PROTO=http -if test "x$LOCAL" = x ; then -URLSERVER="remotetest.unidata.ucar.edu" -else -URLSERVER="localhost:8081" -fi - -# See if we need to override -if test "x$URS" != "x" ; then -#https://54.86.135.31/opendap/data/nc/fnoc1.nc.dds -URLSERVER="54.86.135.31" -URLPATH="opendap/data/nc/fnoc1.nc" -BASICCOMBO="$URS" -RCEMBED=0 -NETRC=$NETRCFILE +AUTHSERVER="thredds.ucar.edu" +BASICCOMBO="authtester:auth" +URLPATH="thredds/dodsC/test3/testData.nc" PROTO=https -fi +URLSERVER=${AUTHSERVER} if test "x$DBG" = x1 ; then URLPATH="${URLPATH}#log&show=fetch" @@ -70,230 +66,154 @@ fi BASICUSER=`echo $BASICCOMBO | cut -d: -f1` BASICPWD=`echo $BASICCOMBO | cut -d: -f2` -OUTPUT="./.output" - if test "x$TEMP" = x ; then TEMP="/tmp" fi TEMP=`echo "$TEMP" | sed -e "s|/$||"` -LOCALRC=./$RC +LOCALRC=${WD}/$RC +LOCALNETRC=${WD}/$NETRC HOMERC=${HOME}/$RC HOMERC=`echo "$HOMERC" | sed -e "s|//|/|g"` -SPECRC="$TEMP/temprc" -ENVRC="$WD/envrc" +HOMENETRC=${HOME}/$NETRC +HOMENETRC=`echo "$HOMENETRC" | sed -e "s|//|/|g"` createrc() { - RCP="$1" ; shift - unset NOPWD - unset BADPWD - while [[ $# > 0 ]] ; do - case "$1" in - nopwd) NOPWD=1 ;; - badpwd) BADPWD=1 ;; - *) ;; - esac - shift - done - if test "x$RCP" != x ; then - rm -f $RCP - echo "Creating rc file $RCP" - else + local RCP + local NETRCFILE + RCP="$1" + if test "x$RCP" = x ; then echo "createrc: no rc specified" exit 1 fi - if test "x${DBG}" != x ; then + shift + NETRCPATH="$1" + echo "Creating rc file $RCP" + if test "x${DBG}" = x1 ; then echo "HTTP.VERBOSE=1" >>$RCP fi - echo "HTTP.COOKIEJAR=${COOKIES}" >>$RCP - if test "x${URS}" = x ; then - if test "x${NOPWD}" = x ; then - if test "x${BADPWD}" = x ; then - echo "HTTP.CREDENTIALS.USERPASSWORD=${BASICCOMBO}" >>$RCP - else - echo "HTTP.CREDENTIALS.USERPASSWORD=${BADCOMBO}" >>$RCP - fi - fi - fi - if test "x${NETRC}" != x && test "x$NFL" = x ; then - echo "HTTP.NETRC=${NETRC}" >>$RCP + if test "x$NETRCPATH" = x ; then + echo "HTTP.CREDENTIALS.USERPASSWORD=${BASICCOMBO}" >>$RCP + elif test "x$NETRCPATH" != ximplicit ; then + echo "HTTP.NETRC=${NETRCPATH}" >>$RCP + elif test "x$NETRCPATH" = ximplicit ; then + echo "HTTP.NETRC=" >>$RCP fi + echo "HTTP.COOKIEJAR=${COOKIES}" >>$RCP } createnetrc() { - NCP="$1" ; shift - unset NOPWD - unset BADPWD - while [[ $# > 0 ]] ; do - case "$1" in - nopwd) NOPWD=1 ;; - badpwd) BADPWD=1 ;; - *) ;; - esac - shift - done - if test "x$NCP" != x ; then - rm -f $NCP - echo "Creating netrc file $NCP" - else - echo "createnetrc: no rc specified" - exit 1 - fi - if test "x$URS" != x ; then - echo "machine uat.urs.earthdata.nasa.gov login $BASICUSER password $BASICPWD" >>$NCP - #echo "machine 54.86.135.31 login $BASICUSER password $BASICPWD" >>$1 - else - echo -n "${PROTO}://$URLSERVER/$URLPATH" >>$NCP - if test "x$NOPWD" = x ; then - if test "x$BADPWD" = x ; then - echo -n " login $BASICUSER password $BASICPWD" >>$NCP - else - echo -n " login $BASICUSER password xxxxxx" >>$NCP - fi - fi - echo "" >>$NCP - fi + local NETRCPATH + NETRCPATH="$1"; # netrc file path + if test "x$NETRCPATH" = x ; then return; fi + echo "Creating netrc file $NETRCPATH" +# echo -n "${PROTO}://$URLSERVER/$URLPATH" >>$NETRCPATH + echo -n "machine $URLSERVER" >>$NETRCPATH + echo -n " login $BASICUSER password $BASICPWD" >>$NETRCPATH + echo "" >>$NETRCPATH + chmod go-rwx $NETRCPATH } -reset() { - for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC $OUTPUT ; do - rm -f ${f} - done - unset DAPRCFILE -} +# Test cases -restore() { +# Case: !daprc !netrc embedded usr:pwd +rcembed() { + echo "***Testing with embedded user:pwd" reset - for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC ; do - if test -f ${f}.save ; then - echo "restoring old ${f}" - cp ${f}.save ${f} - fi - done -} - -save() { - for f in ./$RC $HOME/$RC $SPECRC $ENVRC $COOKIES $NETRC ; do - if test -f $f ; then - if test -f ${f}.save ; then - ignore=1 - else - echo "saving $f" - cp ${f} ${f}.save - fi - fi - done -} - -show() { - if test "x$SHOW" = x1 ; then cat $OUTPUT; fi - if test "x$OUTPUT" != "x"; then rm -f $OUTPUT; fi -} - -# Assemble the ncdump command -if test "x$DBG" = x1; then -NCDUMP="$NCDUMP -D1" -fi - -if test "x$GDB" = x1 ; then - NCDUMP="gdb --args $NCDUMP" -fi -if test "x$VG" = x1 ; then -NCDUMP="valgrind --leak-check=full $NCDUMP" -fi - -# Initialize -save -reset - -if test "x$RCEMBED" = x1 ; then - echo "***Testing rc file with embedded user:pwd" URL="${PROTO}://${BASICCOMBO}@${URLSERVER}/$URLPATH" - unset NETRC # Invoke ncdump to extract a file the URL - echo "command: ${NCDUMP} -h ${URL} > $OUTPUT" - ${NCDUMP} -h "$URL" > $OUTPUT - show -fi - -# Rest of tests assume these defaults -URL="${PROTO}://${URLSERVER}/$URLPATH" -NETRC=$NETRCFILE + ${NCDUMP} -h "$URL" > testauthoutput +} -if test "x$RCLOCAL" = x1 ; then +# Case: local daprc no netrc no embed +rclocal1() { echo "***Testing rc file in local directory" - # Create the rc file and (optional) netrc fil in ./ reset - createnetrc $NETRC + # Create the rc file in ./ createrc $LOCALRC + # Invoke ncdump to extract a file using the URL + ${NCDUMP} -h "$URL" > testauthoutput +} +# Case: local daprc local netrc no embed +rclocal2() { + echo "***Testing rc file in local directory" + reset + # Create the rc file and (optional) netrc fil in ./ + createnetrc $LOCALNETRC + createrc $LOCALRC $LOCALNETRC # Invoke ncdump to extract a file using the URL - echo "command: ${NCDUMP} -h ${URL} > $OUTPUT" - ${NCDUMP} -h "$URL" > $OUTPUT - show -fi + ${NCDUMP} -h "$URL" > testauthoutput +} -if test "x$RCHOME" = x1 ; then - echo "***Testing rc file in home directory" - # Create the rc file and (optional) netrc file in ./ +# Case: home rc no netrc no embed +rchome1() { + echo "***Testing home rc file no netrc in home directory" reset - createnetrc $NETRC + # Create the rc file file in ./ createrc $HOMERC - # Invoke ncdump to extract a file the URL - echo "command: ${NCDUMP} -h ${URL} > $OUTPUT" - ${NCDUMP} -h "$URL" > $OUTPUT - show -fi + ${NCDUMP} -h "$URL" > testauthoutput +} -if test "x$RCSPEC" == x1 ; then - echo "*** Testing rc file in specified directory" - # Create the rc file and (optional) netrc file +# Case: home daprc implicit home netrc no embed +rchome2() { + echo "***Testing .netrc file in home directory" reset - createnetrc $NETRC - createrc $SPECRC + createnetrc $HOME/$NETRCIMP + createrc $HOMERC implicit + # Invoke ncdump to extract a file using the URL + ${NCDUMP} -h "$URL" > testauthoutput +} +# Case: local rc explicit netrc no embed +rchome3() { + echo "***Testing local rc file and .netrc explicit in home directory" + reset + # Create the rc file and (optional) netrc file in ./ + createnetrc $HOME/$NETRC + createrc $LOCALRC $HOME/$NETRC # Invoke ncdump to extract a file the URL - echo "command: ${NCDUMP} -h ${URL} > $OUTPUT" - ${NCDUMP} -h "$URL" > $OUTPUT - show -fi + ${NCDUMP} -h "$URL" > testauthoutput +} -if test "x$RCENV" = x1 ; then - echo "*** Testing rc file using env variable" - # Create the rc file and (optional) netrc file +# Case: local rc implicit netrc no embed +rchome4() { + echo "***Testing local rc file and .netrc implicit in home directory" reset - createnetrc $NETRC - echo "ENV: export DAPRCFILE=$ENVRC" - export DAPRCFILE=$ENVRC - createrc $DAPRCFILE - + # Create the rc file and (optional) netrc file in ./ + createnetrc $HOME/$NETRCIMP + createrc $LOCALRC implicit # Invoke ncdump to extract a file the URL - echo "command: ${NCDUMP} -h ${URL} > $OUTPUT" - ${NCDUMP} -h "$URL" > $OUTPUT - show - export DAPRCFILE= -fi + ${NCDUMP} -h "$URL" > testauthoutput +} -# Test that .daprc overrides netcrc for password +reset() { + if test "x$RCHOME" = x1 ; then + rm -f $HOMERCFILES + fi + rm -f $LOCALRCFILES + unset DAPRCFILE + rm -f ./testauthoutput +} + +rcembed + +# Next set tests assume these defaults URL="${PROTO}://${URLSERVER}/$URLPATH" -NETRC=$NETRCFILE -if test "x$RCPREC" = x1 ; then - echo "***Testing rc vs netrc file precedence" - # Create the rc file and (optional) netrc file in ./ - reset - createnetrc $NETRC badpwd - createrc $LOCALRC - # Invoke ncdump to extract a file using the URL - echo "command: ${NCDUMP} -h ${URL} > $OUTPUT" - ${NCDUMP} -h "$URL" > $OUTPUT - ${NCDUMP} -h "$URL" - show +rclocal1 +rclocal2 + +# Do not do this unless you know what you are doing +if test "x$RCHOME" = x1 ; then + rchome1 + rchome2 + rchome3 + rchome4 fi reset -restore exit diff --git a/ncdump/CMakeLists.txt b/ncdump/CMakeLists.txt index 828aa18d25..462231bb33 100644 --- a/ncdump/CMakeLists.txt +++ b/ncdump/CMakeLists.txt @@ -99,9 +99,12 @@ IF(ENABLE_TESTS) ADD_EXECUTABLE(bom bom.c) ADD_EXECUTABLE(tst_dimsizes tst_dimsizes.c) ADD_EXECUTABLE(nctrunc nctrunc.c) + ADD_EXECUTABLE(tst_rcmerge tst_rcmerge.c) TARGET_LINK_LIBRARIES(rewrite-scalar netcdf) TARGET_LINK_LIBRARIES(bom netcdf) TARGET_LINK_LIBRARIES(tst_dimsizes netcdf) + TARGET_LINK_LIBRARIES(nctrunc netcdf) + TARGET_LINK_LIBRARIES(tst_rcmerge netcdf) IF(USE_HDF5) ADD_EXECUTABLE(tst_fileinfo tst_fileinfo.c) @@ -137,6 +140,13 @@ IF(ENABLE_TESTS) SET_TARGET_PROPERTIES(nctrunc PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY + ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG + ${CMAKE_CURRENT_BINARY_DIR}) + SET_TARGET_PROPERTIES(tst_rcmerge PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE + ${CMAKE_CURRENT_BINARY_DIR}) + IF(USE_HDF5) SET_TARGET_PROPERTIES(tst_fileinfo PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) @@ -305,6 +315,8 @@ IF(ENABLE_TESTS) add_sh_test(ncdump test_keywords) ENDIF() + add_sh_test(ncdump test_rcmerge) + ENDIF(ENABLE_TESTS) #IF(MSVC) diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index fb382bb38a..b5cd0438b7 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -61,14 +61,14 @@ man_MANS = ncdump.1 nccopy.1 if BUILD_TESTSETS # C programs needed by shell scripts for classic tests. check_PROGRAMS = rewrite-scalar ref_ctest ref_ctest64 ncdump tst_utf8 \ -bom tst_dimsizes nctrunc +bom tst_dimsizes nctrunc tst_rcmerge # Tests for classic and 64-bit offset files. TESTS = tst_inttags.sh run_tests.sh tst_64bit.sh ref_ctest \ ref_ctest64 tst_output.sh tst_lengths.sh tst_calendars.sh \ run_utf8_tests.sh test_unicode_directory.sh tst_nccopy3.sh tst_nccopy3_subset.sh \ tst_charfill.sh tst_iter.sh tst_formatx3.sh tst_bom.sh \ -tst_dimsizes.sh run_ncgen_tests.sh tst_ncgen4_classic.sh test_radix.sh +tst_dimsizes.sh run_ncgen_tests.sh tst_ncgen4_classic.sh test_radix.sh test_rcmerge.sh # The tst_nccopy3.sh test uses output from a bunch of other # tests. This records the dependency so parallel builds work. @@ -164,7 +164,8 @@ ref_no_ncproperty.nc test_unicode_directory.sh \ ref_roman_szip_simple.cdl ref_roman_szip_unlim.cdl ref_tst_perdimspecs.cdl \ test_keywords.sh ref_keyword1.cdl ref_keyword2.cdl \ ref_keyword3.cdl ref_keyword4.cdl \ -ref_tst_nofilters.cdl test_unicode_path.sh +ref_tst_nofilters.cdl test_unicode_path.sh test_rcmerge.sh \ +ref_rcmerge1.txt ref_rcmerge2.txt ref_rcmerge3.txt # The L512.bin file is file containing exactly 512 bytes each of value 0. # It is used for creating hdf5 files with varying offsets for testing. @@ -176,7 +177,7 @@ EXTRA_DIST += tst_ctests.sh ref_ctest_small_3.c ref_ctest_small_4.c \ # CDL files and Expected results SUBDIRS = cdl expected -CLEANFILES = tst_*.nc tmp*.nc test*.nc iter.* tmp*.cdl \ +CLEANFILES = tst_*.nc tmp*.nc test*.nc iter.* tmp*.cdl tmp*.txt \ tst_output_*.cdl tst_output_*.c tst_utf8_*.cdl *.tmp tst_tst8.cdl \ tst_netcdf4_*.cdl test1_ncdump.cdl test2_ncdump.cdl test1.cdl \ ctest1.cdl test1_cdf5.cdl test2_cdf5.cdl test1_offset.cdl \ @@ -204,3 +205,6 @@ tst_roman_szip_unlim.cdl tst_perdimpspecs.nc tmppds.* \ keyword1.nc keyword2.nc keyword3.nc keyword4.nc \ tmp_keyword1.cdl tmp_keyword2.cdl tmp_keyword3.cdl tmp_keyword4.cdl +# Remove directories +clean-local: + rm -fr rcmergedir diff --git a/ncdump/ocprint.c b/ncdump/ocprint.c index c005a13ac4..270aba1c23 100755 --- a/ncdump/ocprint.c +++ b/ncdump/ocprint.c @@ -556,8 +556,7 @@ printdata_container(OClink link, OCdatanode datanode, NCbytes* buffer, int istop pushstack(field); FAIL(printdata_indices(link,field,buffer,istoplevel)); popstack(); - if(oc_data_free(link,field) != OC_NOERR) - break; + oc_data_free(link,field); } return stat; } diff --git a/ncdump/ref_rcmerge1.txt b/ncdump/ref_rcmerge1.txt new file mode 100644 index 0000000000..edc234b572 --- /dev/null +++ b/ncdump/ref_rcmerge1.txt @@ -0,0 +1,6 @@ +|ncrc_home|->|ncrc| +|daprc_home|->|daprc| +|dodsrc_home|->|dodsrc| +|ncrc_local|->|ncrc| +|daprc_local|->|daprc| +|dodsrc_local|->|dodsrc| diff --git a/ncdump/ref_rcmerge2.txt b/ncdump/ref_rcmerge2.txt new file mode 100644 index 0000000000..6aeae12971 --- /dev/null +++ b/ncdump/ref_rcmerge2.txt @@ -0,0 +1,3 @@ +|ncrc|->|ncrc| +|daprc|->|daprc| +|dodsrc|->|dodsrc| diff --git a/ncdump/ref_rcmerge3.txt b/ncdump/ref_rcmerge3.txt new file mode 100644 index 0000000000..2fd622d561 --- /dev/null +++ b/ncdump/ref_rcmerge3.txt @@ -0,0 +1,3 @@ +|ncrc|->|ncrc2| +|ncrcx|->|ncrcy| +|daprc|->|daprc| diff --git a/ncdump/test_rcmerge.sh b/ncdump/test_rcmerge.sh new file mode 100755 index 0000000000..3f97bb0b4a --- /dev/null +++ b/ncdump/test_rcmerge.sh @@ -0,0 +1,105 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +# Test that all available .rc files are merged and overrides are handled. +# The internal .rc is constructed as follows: +# +# 1. Use NCRCENV_RC environment variable exclusively if defined +# 2. If NCRCENV_RC is not defined then merge the set of rc files in this order: +# 1. $HOME/.ncrc +# 2. $HOME/.daprc +# 3. $HOME/.docsrc +# 4. $CWD/.ncrc +# 5. $CWD/.daprc +# 6. $CWD/.docsrc +# Entries in later files override any of the >earlier >files + +# Since this involves a shared resource: the .rc files in current working directory, +# we need to isolate from any other test. + +# Make sure execdir and srcdir absolute paths are available +WD=`pwd` +cd $srcdir ; abs_srcdir=`pwd` ; cd $WD +cd $execdir ; abs_execdir=`pwd` ; cd $WD + +# Now create a special directory +# And enter it to execute tests +rm -fr rcmergedir +mkdir rcmergedir +cd rcmergedir +WD=`pwd` + +if test "x$NCAUTH_HOMETEST" != x ; then + RCHOME=1 +fi + +HOMERCFILES="$HOME/.ncrc $HOME/.daprc $HOME/.dodsrc" +LOCALRCFILES="$WD/.ncrc $WD/.daprc $WD/.dodsrc" + +resetrc() { + if test "x$RCHOME" = x1 ; then + rm -f $HOMERCFILES + fi + rm -f $LOCALRCFILES + unset NCRCENV_RC +} + +mergecase1() { + # create everything with different keys to test merge + resetrc + rm -f tmp_rcmerge.txt tmpoutput.txt + for r in "ncrc" "daprc" "dodsrc" ; do + if test "x$RCHOME" = x1 ; then echo "${r}_home=${r}" >> $HOME/".${r}"; fi + echo "${r}_local=${r}" >> $WD/".${r}" + done; + ${abs_execdir}/tst_rcmerge > tmpoutput.txt + if test "x$RCHOME" = x1 ; then + cp ${abs_srcdir}/ref_rcmerge1.txt tmp_rcmerge1.txt + else + sed -e '/_local/p' -e d <${abs_srcdir}/ref_rcmerge1.txt > tmp_rcmerge1.txt + fi + diff -b tmp_rcmerge1.txt tmpoutput.txt +} + +mergecase2() { + # create with some same keys to test override + resetrc + rm -f tmp_rcmerge.txt tmpoutput.txt + for r in "ncrc" "daprc" "dodsrc" ; do + if test "x$RCHOME" = x1 ; then echo "${r}=${r}" >> $HOME/".${r}"; fi + echo "${r}=${r}" >> $WD/".${r}" + done; + ${abs_execdir}/tst_rcmerge > tmpoutput.txt + diff -b ${abs_srcdir}/ref_rcmerge2.txt tmpoutput.txt +} + +mergecase3() { + # Test cross file overrides + resetrc + rm -f tmp_rcmerge.txt tmpoutput.txt + if test "x$RCHOME" = x1 ; then + echo "ncrc=ncrc1" >> $HOME/.ncrc + echo "ncrcx=ncrcx" >> $HOME/.ncrc + echo "ncrc=ncrc2" >> $HOME/.dodsrc + echo "daprc=daprc" >> $HOME/.daprc + else + echo "ncrc=ncrc1" >> $WD/.ncrc + echo "ncrcx=ncrcx" >> $WD/.ncrc + echo "ncrc=ncrc2" >> $WD/.dodsrc + echo "daprc=daprc" >> $WD/.daprc + fi + echo "daprc=daprc" >> $WD/.dodsrc + echo "ncrcx=ncrcy" >> $WD/.dodsrc + ${abs_execdir}/tst_rcmerge > tmpoutput.txt + diff -b ${abs_srcdir}/ref_rcmerge3.txt tmpoutput.txt +} + +resetrc + +mergecase1 +mergecase2 +mergecase3 + +resetrc diff --git a/ncdump/tst_nccopy4.sh b/ncdump/tst_nccopy4.sh index 525982b308..4ffdb15502 100755 --- a/ncdump/tst_nccopy4.sh +++ b/ncdump/tst_nccopy4.sh @@ -109,8 +109,6 @@ test "x$STORAGE" = 'xtas:_Storage="chunked";' CHUNKSIZES=`cat tmppds.cdl | sed -e '/tas:_ChunkSizes/p' -ed | tr -d '\t \r'` test "x$CHUNKSIZES" = 'xtas:_ChunkSizes=10,15,20;' -set -x - echo "*** Test that nccopy -F var1,none works as intended " ${NCGEN} -4 -b -o tst_nofilters.nc $srcdir/ref_tst_nofilters.cdl ${NCCOPY} -M0 -4 -F var1,none -c // tst_nofilters.nc tmp_nofilters.nc diff --git a/ncdump/tst_rcmerge.c b/ncdump/tst_rcmerge.c new file mode 100644 index 0000000000..97610b75ff --- /dev/null +++ b/ncdump/tst_rcmerge.c @@ -0,0 +1,39 @@ +#include +#include +#include +#include "netcdf.h" +#include "ncrc.h" + +int +main(int argc, char** argv) +{ + size_t i,ntriples = 0; + NCRCglobalstate* ngs = ncrc_getglobalstate(); + NCRCinfo* info = NULL; + NCTriple* triple = NULL; + + /* Cause the .rc files to be read and merged */ + nc_initialize(); + + if((ngs = ncrc_getglobalstate())==NULL) abort(); + info = &ngs->rcinfo; + + if(info->ignore) { + fprintf(stderr,".rc ignored\n"); + exit(0); + } + + /* Print out the .rc triples */ + if((ntriples = NC_rcfile_length(info))==0) { + printf("\n"); + exit(0); + } + for(i=0;ihost != NULL) + printf("[%s] ",triple->host); + printf("|%s|->|%s|\n",triple->key,triple->value); + } + return 0; +} diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index c8afc687c2..9431be90a6 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -82,16 +82,19 @@ IF(ENABLE_TESTS) add_sh_test(nczarr_test run_fillonlyz) ENDIF() add_sh_test(nczarr_test run_ncgen4) + + BUILD_BIN_TEST(tst_chunkcases ${TSTCOMMONSRC}) + TARGET_INCLUDE_DIRECTORIES(tst_chunkcases PUBLIC ../libnczarr) + add_sh_test(nczarr_test run_chunkcases) + add_sh_test(nczarr_test run_purezarr) add_sh_test(nczarr_test run_interop) - - BUILD_BIN_TEST(tst_chunkcases ${TSTCOMMONSRC}) - TARGET_INCLUDE_DIRECTORIES(tst_chunkcases PUBLIC ../libnczarr) - add_sh_test(nczarr_test run_chunkcases) + add_sh_test(nczarr_test run_misc) if(ENABLE_NCZARR_S3) add_sh_test(nczarr_test run_s3_cleanup) ENDIF() + ENDIF(BUILD_UTILITIES) ENDIF(ENABLE_TESTS) diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index 3ee419b6dc..652b7de5c6 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -44,19 +44,21 @@ TESTS += run_ut_chunk.sh if BUILD_UTILITIES +TESTS += run_ncgen4.sh + if USE_HDF5 TESTS += run_nccopyz.sh TESTS += run_fillonlyz.sh endif -TESTS += run_ncgen4.sh -TESTS += run_purezarr.sh -TESTS += run_interop.sh - check_PROGRAMS += tst_chunkcases tst_chunkcases_SOURCES = tst_chunkcases.c ${tstcommonsrc} TESTS += run_chunkcases.sh +TESTS += run_purezarr.sh +TESTS += run_interop.sh +TESTS += run_misc.sh + endif if BUILD_BENCHMARKS @@ -99,7 +101,7 @@ ncdumpchunks_SOURCES = ncdumpchunks.c EXTRA_DIST = CMakeLists.txt \ run_ut_map.sh run_ut_mapapi.sh run_ut_misc.sh run_ut_chunk.sh run_ncgen4.sh \ run_nccopyz.sh run_fillonlyz.sh run_chunkcases.sh test_nczarr.sh run_perf_chunks1.sh run_s3_cleanup.sh \ -run_purezarr.sh run_interop.sh \ +run_purezarr.sh run_interop.sh run_misc.sh \ ref_ut_map_create.cdl ref_ut_map_writedata.cdl ref_ut_map_writemeta2.cdl ref_ut_map_writemeta.cdl \ ref_ut_map_readmeta.txt ref_ut_map_readmeta2.txt ref_ut_map_search.txt \ ref_ut_mapapi_create.cdl ref_ut_mapapi_data.cdl ref_ut_mapapi_meta.cdl ref_ut_mapapi_search.txt \ @@ -110,9 +112,9 @@ ref_perdimspecs.cdl ref_fillonly.cdl \ ref_whole.cdl ref_whole.txt \ ref_skip.cdl ref_skip.txt ref_skipw.cdl \ ref_rem.cdl ref_rem.dmp ref_ndims.cdl ref_ndims.dmp \ -ref_misc1.cdl ref_misc1.dmp \ +ref_misc1.cdl ref_misc1.dmp ref_misc2.cdl \ ref_avail1.cdl ref_avail1.dmp ref_avail1.txt \ -ref_xarray.cdl ref_purezarr.cdl ref_purezarr_base.cdl +ref_xarray.cdl ref_purezarr.cdl ref_purezarr_base.cdl ref_nczarr2zarr.cdl # Interoperability files EXTRA_DIST += power_901_constants.zip ref_power_901_constants.cdl @@ -123,3 +125,4 @@ CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.n clean-local: rm -fr tmp*.file results.file results.s3 results.zip rm -fr power_901_constants.file + rm -fr rcmiscdir diff --git a/nczarr_test/power_901_constants.zip b/nczarr_test/power_901_constants.zip index f8d58cda9f..2aa5b032bc 100644 Binary files a/nczarr_test/power_901_constants.zip and b/nczarr_test/power_901_constants.zip differ diff --git a/nczarr_test/ref_misc2.cdl b/nczarr_test/ref_misc2.cdl new file mode 100644 index 0000000000..cb662ae0de --- /dev/null +++ b/nczarr_test/ref_misc2.cdl @@ -0,0 +1,12 @@ +netcdf tmp_misc2 { +dimensions: + d0 = 2; + d1 = 2; +variables: + int v(d0, d1) ; +data: + + v = + 0, 1, + 2, 3 ; +} diff --git a/nczarr_test/ref_nczarr2zarr.cdl b/nczarr_test/ref_nczarr2zarr.cdl new file mode 100644 index 0000000000..90e24489a1 --- /dev/null +++ b/nczarr_test/ref_nczarr2zarr.cdl @@ -0,0 +1,18 @@ +netcdf nczarr2zarr { +dimensions: + .zdim_8 = 8 ; +variables: + int v(.zdim_8, .zdim_8) ; + v:_FillValue = -1 ; +data: + + v = + _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, + _, _, _, _, 0, 1, 2, 3, + _, _, _, _, 4, 5, 6, 7, + _, _, _, _, 8, 9, 10, 11, + _, _, _, _, 12, 13, 14, 15 ; +} diff --git a/nczarr_test/ref_power_901_constants.cdl b/nczarr_test/ref_power_901_constants.cdl index 6efa505698..04386efe6d 100644 --- a/nczarr_test/ref_power_901_constants.cdl +++ b/nczarr_test/ref_power_901_constants.cdl @@ -3,22 +3,6 @@ dimensions: lat = 361 ; lon = 576 ; variables: - float FROCEAN(lat, lon) ; - FROCEAN:fmissing_value = 999999986991104. ; - FROCEAN:long_name = "fraction_of_ocean" ; - FROCEAN:standard_name = "fraction_of_ocean" ; - FROCEAN:units = "1" ; - FROCEAN:valid_range = -999999986991104., 999999986991104. ; - FROCEAN:vmax = 999999986991104. ; - FROCEAN:vmin = -999999986991104. ; - float FRLAND(lat, lon) ; - FRLAND:fmissing_value = 999999986991104. ; - FRLAND:long_name = "fraction_of_land" ; - FRLAND:standard_name = "fraction_of_land" ; - FRLAND:units = "1" ; - FRLAND:valid_range = -999999986991104., 999999986991104. ; - FRLAND:vmax = 999999986991104. ; - FRLAND:vmin = -999999986991104. ; float FRLAKE(lat, lon) ; FRLAKE:fmissing_value = 999999986991104. ; FRLAKE:long_name = "fraction_of_lake" ; @@ -27,6 +11,14 @@ variables: FRLAKE:valid_range = -999999986991104., 999999986991104. ; FRLAKE:vmax = 999999986991104. ; FRLAKE:vmin = -999999986991104. ; + float FRLAND(lat, lon) ; + FRLAND:fmissing_value = 999999986991104. ; + FRLAND:long_name = "fraction_of_land" ; + FRLAND:standard_name = "fraction_of_land" ; + FRLAND:units = "1" ; + FRLAND:valid_range = -999999986991104., 999999986991104. ; + FRLAND:vmax = 999999986991104. ; + FRLAND:vmin = -999999986991104. ; float FRLANDICE(lat, lon) ; FRLANDICE:fmissing_value = 999999986991104. ; FRLANDICE:long_name = "fraction_of_land_ice" ; @@ -35,4 +27,12 @@ variables: FRLANDICE:valid_range = -999999986991104., 999999986991104. ; FRLANDICE:vmax = 999999986991104. ; FRLANDICE:vmin = -999999986991104. ; + float FROCEAN(lat, lon) ; + FROCEAN:fmissing_value = 999999986991104. ; + FROCEAN:long_name = "fraction_of_ocean" ; + FROCEAN:standard_name = "fraction_of_ocean" ; + FROCEAN:units = "1" ; + FROCEAN:valid_range = -999999986991104., 999999986991104. ; + FROCEAN:vmax = 999999986991104. ; + FROCEAN:vmin = -999999986991104. ; } diff --git a/nczarr_test/run_chunkcases.sh b/nczarr_test/run_chunkcases.sh index 88bcba22fe..6323e0219d 100755 --- a/nczarr_test/run_chunkcases.sh +++ b/nczarr_test/run_chunkcases.sh @@ -1,8 +1,5 @@ #!/bin/sh - -export SETX=1 - # Note that this test builds a special results. directory in # which to run the tests, where is the process id number of # the bash shell instance. The reason for doing this is so that @@ -13,8 +10,6 @@ export SETX=1 # directory for cmake. By running the tests in a separate # results. I can guarantee that isolation is preserved. - - if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh diff --git a/nczarr_test/run_interop.sh b/nczarr_test/run_interop.sh index 7108760fde..36e8acdc8e 100755 --- a/nczarr_test/run_interop.sh +++ b/nczarr_test/run_interop.sh @@ -16,7 +16,7 @@ testcasefile() { ref=$1 mode=$2 if test "x$3" = xmetaonly ; then flags="-h"; fi - fileargs ${srcdir}/$ref "mode=$mode,$zext" + fileargs ${execdir}/$ref "mode=$mode,$zext" rm -f tmp_${ref}_${zext}.cdl ${NCDUMP} $flags $fileurl > tmp_${ref}_${zext}.cdl diff -b ${srcdir}/ref_${ref}.cdl tmp_${ref}_${zext}.cdl @@ -27,7 +27,7 @@ testcasezip() { ref=$1 mode=$2 if test "x$3" = xmetaonly ; then flags="-h"; fi - fileargs ${srcdir}/$ref "mode=$mode,$zext" + fileargs ${execdir}/$ref "mode=$mode,$zext" rm -f tmp_${ref}_${zext}.cdl ${NCDUMP} $flags $fileurl > tmp_${ref}_${zext}.cdl diff -b ${srcdir}/ref_${ref}.cdl tmp_${ref}_${zext}.cdl @@ -41,16 +41,20 @@ case "$zext" in rm -fr power_901_constants power_901_constants.file unzip ${srcdir}/power_901_constants.zip > /dev/null mv power_901_constants power_901_constants.file - testcasefile power_901_constants xarray metaonly + testcasefile power_901_constants zarr metaonly; # test xarray as default ;; zip) + # Move into position + if test "x$srcdir" != "x$execdir" ; then + cp ${srcdir}/power_901_constants.zip ${execdir} + fi testcasezip power_901_constants xarray metaonly ;; *) echo "unimplemented kind: $1" ; exit 1;; esac } -#testallcases file +testallcases file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testallcases zip; fi #No examples yet: if test "x$FEATURE_S3TESTS" = xyes ; then testallcases s3; fi diff --git a/nczarr_test/run_misc.sh b/nczarr_test/run_misc.sh new file mode 100755 index 0000000000..065b676f28 --- /dev/null +++ b/nczarr_test/run_misc.sh @@ -0,0 +1,64 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "$srcdir/test_nczarr.sh" + +# This test uses a shared resource: the .rc files; so run in a special directory +# Create a special directory +# And enter it to execute tests +rm -fr rcmiscdir +mkdir rcmiscdir +cd rcmiscdir +WD=`pwd` + +# This shell script provides a miscellaneous set of tests + +set -e + +cleanup() { + resetrc +} + +# Setup the .rc files + +createrc() { + RCP="${WD}/.ncrc" + echo "Creating rc file $RCP" + echo "ZARR.DIMENSION_SEPARATOR=/" >>$RCP +} + +testcase1() { +zext=$1 +echo "*** Test: use '/' as dimension separator for write then read" +fileargs tmp_dimsep "mode=nczarr,$zext" +deletemap $zext $file +cleanup +createrc +${NCGEN} -4 -lb -o $fileurl ${abs_srcdir}/ref_misc1.cdl +${NCDUMP} -n tmp_misc1 $fileurl > tmp_misc1_$zext.cdl +diff -bw ${abs_srcdir}/ref_misc1.cdl tmp_misc1_$zext.cdl +} + +testcase2() { +zext=$1 +echo "*** Test: '/' as dimension separator creates extra groups" +fileargs tmp_extra "mode=nczarr,$zext" +deletemap $zext $file +cleanup +createrc +${NCGEN} -4 -lb -o "$fileurl" ${abs_srcdir}/ref_misc2.cdl +${NCDUMP} -n tmp_misc2 $fileurl > tmp_misc2_$zext.cdl +diff -wb ${abs_srcdir}/ref_misc2.cdl tmp_misc2_$zext.cdl +} + +testcase1 file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase1 zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcase1 s3; fi + +testcase2 file +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase2 zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcase2 s3; fi + +exit 0 diff --git a/nczarr_test/run_purezarr.sh b/nczarr_test/run_purezarr.sh index a9bf7e8a8c..1b3d34f47b 100755 --- a/nczarr_test/run_purezarr.sh +++ b/nczarr_test/run_purezarr.sh @@ -6,7 +6,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$srcdir/test_nczarr.sh" # This shell script tests support for: -# 1. pure zarr read/write +# 1. pure zarr (noxarray) read/write # 2. xarray read/write set -e @@ -14,19 +14,27 @@ set -e testcase() { zext=$1 -echo "*** Test: pure zarr write; format=$zext" -fileargs tmp_purezarr "mode=zarr,$zext" +echo "*** Test: pure zarr write then read; format=$zext" +fileargs tmp_purezarr "mode=noxarray,$zext" deletemap $zext $file ${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_purezarr_base.cdl ${NCDUMP} $fileurl > tmp_purezarr_${zext}.cdl diff -b ${srcdir}/ref_purezarr.cdl tmp_purezarr_${zext}.cdl -echo "*** Test: xarray zarr write; format=$zext" -fileargs tmp_xarray "mode=xarray,$zext" +echo "*** Test: xarray zarr write then read; format=$zext" +fileargs tmp_xarray "mode=zarr,$zext" deletemap $zext $file ${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_purezarr_base.cdl ${NCDUMP} $fileurl > tmp_xarray_${zext}.cdl diff -b ${srcdir}/ref_xarray.cdl tmp_xarray_${zext}.cdl + +echo "*** Test: pure zarr reading nczarr; format=$zext" +fileargs tmp_nczarr "mode=nczarr,$zext" +deletemap $zext $file +${NCGEN} -4 -b -o "$fileurl" $srcdir/ref_whole.cdl +fileargs tmp_nczarr "mode=zarr,$zext" +${NCDUMP} -n nczarr2zarr $fileurl > tmp_nczarr_${zext}.cdl +diff -b ${srcdir}/ref_nczarr2zarr.cdl tmp_nczarr_${zext}.cdl } testcase file diff --git a/nczarr_test/test_nczarr.sh b/nczarr_test/test_nczarr.sh index a4b6ccfa4b..94565c3139 100755 --- a/nczarr_test/test_nczarr.sh +++ b/nczarr_test/test_nczarr.sh @@ -125,3 +125,24 @@ for t in ${TESTS} ; do fi done } + +# Make sure execdir and srcdir absolute paths are available +WD=`pwd` +cd $srcdir ; abs_srcdir=`pwd` ; cd $WD +cd $execdir ; abs_execdir=`pwd` ; cd $WD + +# Clear out any existing .rc files +WD=`pwd` +if test "x$NCAUTH_HOMETEST" != x ; then RCHOME=1; fi + +resetrc() { + if test "x$RCHOME" = x1 ; then + rm -f ${HOME}/.dodsrc ${HOME}/.daprc ${HOME}/.ncrc + fi + rm -f ${WD}/.dodsrc ${WD}/.daprc ${WD}/.ncrc + unset NCRCENV_IGNORE + unset NCRCENV_RC + unset DAPRCFILE +} + +resetrc diff --git a/nczarr_test/ut_test.c b/nczarr_test/ut_test.c index 5bddcf76c6..9c376d7e48 100755 --- a/nczarr_test/ut_test.c +++ b/nczarr_test/ut_test.c @@ -107,6 +107,7 @@ ut_init(int argc, char** argv, struct UTOptions * options) return THROW(stat); } +#if 0 static void getpathcwd(char** cwdp) { @@ -114,15 +115,14 @@ getpathcwd(char** cwdp) (void)NCgetcwd(buf,sizeof(buf)); if(cwdp) *cwdp = strdup(buf); } +#endif static void canonicalfile(char** fp) { - size_t len, len2, offset; + size_t len; char* f = NULL; char* abspath = NULL; - char* p = NULL; - char* cwd = NULL; NCURI* uri = NULL; #ifdef _WIN32 int fwin32=0, cwd32=0; diff --git a/oc2/oc.c b/oc2/oc.c index fb19268d45..c82c46f7dc 100644 --- a/oc2/oc.c +++ b/oc2/oc.c @@ -1943,6 +1943,7 @@ oc_data_mode(OCobject link, OCobject datanode, OCDT* modep) return OC_NOERR; } +#if 0 /* Free up a datanode that is no longer being used; Currently does nothing */ @@ -1951,6 +1952,7 @@ oc_data_free(OCobject link, OCobject datanode) { return OCTHROW(OC_NOERR); } +#endif /* Free up a ddsnode that is no longer being used; Currently does nothing diff --git a/oc2/oc.h b/oc2/oc.h index eeb40161b5..cf3086a9fa 100644 --- a/oc2/oc.h +++ b/oc2/oc.h @@ -418,7 +418,8 @@ EXTERNL OCerror oc_data_ithelement(OClink, OCdatanode data, size_t* indices, OCd EXTERNL OCerror oc_data_ithrecord(OClink, OCdatanode data, size_t index, OCdatanode* recordp); /* Free up an data that is no longer being used */ -EXTERNL OCerror oc_data_free(OClink, OCdatanode data); +/* Currently does nothing */ +#define oc_data_free(link,datanode) /* Count the records associated with a sequence */ EXTERNL OCerror oc_data_recordcount(OClink, OCdatanode, size_t*);