Skip to content

Commit

Permalink
Merge pull request #2032 from DennisHeimbigner/newformat.dmh
Browse files Browse the repository at this point in the history
Move to Version 2 NCZarr Extended Meta-Data
  • Loading branch information
WardF authored Jul 19, 2021
2 parents 847c12e + 01718d7 commit 24330ed
Show file tree
Hide file tree
Showing 55 changed files with 2,865 additions and 475 deletions.
1 change: 1 addition & 0 deletions RELEASE_NOTES.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release

## 4.8.1 - TBD

* [Enhancement] Convert to a new representation of the NCZarr meta-data extensions: version 2. Read-only backward compatibility is provided. See [Github #2032](https://github.com/Unidata/netcdf-c/issues/2032).
* [Bug Fix] Fix dimension_separator bug in libnczarr. See [Github #2035](https://github.com/Unidata/netcdf-c/issues/2035).
* [Bug Fix] Fix bugs in libdap4. See [Github #2005](https://github.com/Unidata/netcdf-c/issues/2005).
* [Bug Fix] Store NCZarr fillvalue as a singleton instead of a 1-element array. See [Github #2017](https://github.com/Unidata/netcdf-c/issues/2017).
Expand Down
2 changes: 1 addition & 1 deletion docs/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,6 @@ obsolete/fan_utils.html bestpractices.md filters.md indexing.md
inmemory.md DAP2.dox attribute_conventions.md FAQ.md
file_format_specifications.md known_problems.md
COPYRIGHT.dox user_defined_formats.md DAP4.md DAP4.dox
testserver.dox byterange.dox)
testserver.dox byterange.dox filters.md nczarr.md)

ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")
2 changes: 2 additions & 0 deletions docs/Doxyfile.in
Original file line number Diff line number Diff line change
Expand Up @@ -751,6 +751,8 @@ INPUT = \
@abs_top_srcdir@/docs/byterange.dox \
@abs_top_srcdir@/docs/inmemory.md \
@abs_top_srcdir@/docs/auth.md \
@abs_top_srcdir@/docs/filters.md \
@abs_top_srcdir@/docs/nczarr.md \
@abs_top_srcdir@/docs/notes.md \
@abs_top_srcdir@/docs/all-error-codes.md \
@abs_top_srcdir@/docs/building-with-cmake.md \
Expand Down
3 changes: 2 additions & 1 deletion docs/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ obsolete/fan_utils.html indexing.dox \
inmemory.md attribute_conventions.md FAQ.md \
file_format_specifications.md known_problems.md COPYRIGHT.md \
inmeminternal.dox \
testserver.dox byterange.dox
testserver.dox byterange.dox \
nczarr.md filters.md

# Turn off parallel builds in this directory.
.NOTPARALLEL:
Expand Down
631 changes: 631 additions & 0 deletions docs/nczarr.md

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions include/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ INSTALL(FILES ${netCDF_BINARY_DIR}/include/netcdf_meta.h
INSTALL(FILES ${netCDF_SOURCE_DIR}/include/netcdf_filter.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
COMPONENT headers)
INSTALL(FILES ${netCDF_SOURCE_DIR}/include/netcdf_filter_build.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
COMPONENT headers)

INSTALL(FILES ${netCDF_BINARY_DIR}/include/netcdf_dispatch.h
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
Expand Down
5 changes: 3 additions & 2 deletions include/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# Ed Hartnett, Dennis Heimbigner, Ward Fisher

include_HEADERS = netcdf.h netcdf_meta.h netcdf_mem.h netcdf_aux.h \
netcdf_filter.h netcdf_dispatch.h
netcdf_filter.h netcdf_filter_build.h netcdf_dispatch.h

if BUILD_PARALLEL
include_HEADERS += netcdf_par.h
Expand All @@ -19,7 +19,8 @@ ncbytes.h nchashmap.h ceconstraints.h rnd.h nclog.h ncconfigure.h \
nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \
ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \
ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \
hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h ncfilter.h
hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h \
ncfilter.h ncjson.h

if USE_DAP
noinst_HEADERS += ncdap.h
Expand Down
1 change: 1 addition & 0 deletions include/nc4internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -461,5 +461,6 @@ extern const NC_reservedatt* NC_findreserved(const char* name);
#define NC_ATT_DIMID_NAME "_Netcdf4Dimid"
#define NC_ATT_NC3_STRICT_NAME "_nc3_strict"
#define NC_XARRAY_DIMS "_ARRAY_DIMENSIONS"
#define NC_NCZARR_ATTR "_NCZARR_ATTR"

#endif /* _NC4INTERNAL_ */
2 changes: 1 addition & 1 deletion include/nchttp.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ typedef struct NC_HTTP_STATE {
} NC_HTTP_STATE;

extern int nc_http_open(const char* objecturl, NC_HTTP_STATE** state, long long* lenp);
extern int nc_http_size(NC_HTTP_STATE* state, const char* url, size64_t* sizep);
extern int nc_http_size(NC_HTTP_STATE* state, const char* url, long long* sizep);
extern int nc_http_read(NC_HTTP_STATE* state, const char* url, size64_t start, size64_t count, NCbytes* buf);
extern int nc_http_close(NC_HTTP_STATE* state);
extern int nc_http_headers(NC_HTTP_STATE* state, const NClist** headersp); /* only if headerson */
Expand Down
126 changes: 126 additions & 0 deletions include/ncjson.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
/* Copyright 2018, UCAR/Unidata.
Copyright 2018 Unidata
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#ifndef NCJSON_H
#define NCJSON_H 1

/* Json object sorts (note use of term sort rather than e.g. type or discriminant) */
#define NCJ_UNDEF 0
#define NCJ_STRING 1
#define NCJ_INT 2
#define NCJ_DOUBLE 3
#define NCJ_BOOLEAN 4
#define NCJ_DICT 5
#define NCJ_ARRAY 6
#define NCJ_NULL 7

#define NCJ_NSORTS 8

/* No flags are currently defined, but the argument is a placeholder */


/* Define a struct to store primitive values
as unquoted strings. The sort will
provide more info.
Do not bother with a union since
the amount of saved space is minimal.
*/

typedef struct NCjson {
int sort; /* of this object */
char* string; /* sort != DICT|ARRAY */
struct NCjlist {
int len;
struct NCjson** contents;
} list; /* sort == DICT|ARRAY */
} NCjson;

/* Support Windows declspec */
#ifndef EXTERNL
# ifdef _WIN32
# ifdef NCJSON_INTERNAL /* define when compiling code */
# define EXTERNL __declspec(dllexport) extern
# else
# define EXTERNL __declspec(dllimport) extern
# endif
# else /* !_WIN32 */
# define EXTERNL extern
# endif
#endif /* !defined EXTERNL */

#if defined(__cplusplus)
extern "C" {
#endif

/* int return value is either 1 (ok) or 0 (failure) */

/* Parse */
EXTERNL int NCJparse(const char* text, unsigned flags, NCjson** jsonp);

/* Build */
EXTERNL int NCJnew(int sort, NCjson** object);

/* Recursively free NCjson instance */
EXTERNL void NCJreclaim(NCjson*);

/* Assign a nul terminated string value to an NCjson object as its contents */
EXTERNL int NCJnewstring(int sort, const char* value, NCjson** jsonp);

/* Assign a counted string value to an NCjson object as its contents */
EXTERNL int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);

/* Append value to an array or dict object. */
EXTERNL int NCJappend(NCjson* object, NCjson* value);

/* Insert key-value pair into a dict object. key will be copied */
EXTERNL int NCJinsert(NCjson* object, char* key, NCjson* value);

/* Unparser to convert NCjson object to text in buffer */
EXTERNL int NCJunparse(const NCjson* json, unsigned flags, char** textp);

/* Utilities */
EXTERNL int NCJaddstring(NCjson*, int sort, const char* s);
EXTERNL int NCJdictget(const NCjson* dict, const char* key, NCjson** valuep);

/* dump NCjson* object to output file */
EXTERNL void NCJdump(const NCjson* json, unsigned flags, FILE*);

/* Convert one json sort to value of another type; don't use union so we can know when to reclaim sval */
struct NCJconst {int bval; long long ival; double dval; char* sval;};
EXTERNL int NCJcvt(const NCjson* value, int outsort, struct NCJconst* output);

/* Deep clone a json object */
EXTERNL int NCJclone(const NCjson* json, NCjson** clonep);

/* Getters */
#define NCJsort(x) ((x)->sort)
#define NCJstring(x) ((x)->string)
#define NCJlength(x) ((x)==NULL ? 0 : (x)->list.len)
#define NCJcontents(x) ((x)->list.contents)
#define NCJith(x,i) ((x)->list.contents[i])

/* Setters */
#define NCJsetsort(x,s) (x)->sort=(s)
#define NCJsetstring(x,y) (x)->string=(y)
#define NCJsetcontents(x,c) (x)->list.contents=(c)
#define NCJsetlength(x,l) (x)->list.len=(l)

/* Misc */
#define NCJisatomic(j) ((j)->sort != NCJ_ARRAY && (j)->sort != NCJ_DICT && (j)->sort != NCJ_NULL && (j)->sort != NCJ_UNDEF)

#if defined(__cplusplus)
}
#endif

#endif /*NCJSON_H*/
16 changes: 16 additions & 0 deletions include/ncpathmgr.h
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,22 @@ This means it is ok to call it repeatedly with no harm.
*/
EXTERNL char* NCpathcvt(const char* path);

/**
It is often convenient to convert a path to some canonical format
that has some desirable properties:
1. All backslashes have been converted to forward slash
2. It can be suffixed or prefixed by simple concatenation
with a '/' separator. The exception being if the base part
may be absolute, in which case, suffixing only is allowed;
the user is responsible for getting this right.
To this end we choose the linux/cygwin format as our standard canonical form.
If the path has a windows drive letter, then it is represented
in the cygwin "/cygdrive/<drive-letter>" form. If it is on *nix* platform,
then this sequence will never appear and the canonical path will look
like a standard *nix* path.
*/
EXTERNL int NCpathcanonical(const char* srcpath, char** canonp);

EXTERNL int NChasdriveletter(const char* path);

/* Canonicalize and make absolute by prefixing the current working directory */
Expand Down
6 changes: 4 additions & 2 deletions include/netcdf.h
Original file line number Diff line number Diff line change
Expand Up @@ -480,9 +480,11 @@ by the desired type. */
#define NC_ENCZARR (-137) /**< Error at NCZarr layer. */
#define NC_ES3 (-138) /**< Generic S3 error */
#define NC_EEMPTY (-139) /**< Attempt to read empty NCZarr map key */
#define NC_EFOUND (-140) /**< Some object exists when it should not */
#define NC_EOBJECT (-140) /**< Some object exists when it should not */
#define NC_ENOOBJECT (-141) /**< Some object not found */
#define NC_EPLUGIN (-142) /**< Unclassified failure in accessing a dynamically loaded plugin> */

#define NC4_LAST_ERROR (-140) /**< @internal All netCDF errors > this. */
#define NC4_LAST_ERROR (-142) /**< @internal All netCDF errors > this. */

/* Errors for all remote access methods(e.g. DAP and CDMREMOTE)*/
#define NC_EURL (NC_EDAPURL) /**< Malformed URL */
Expand Down
Loading

0 comments on commit 24330ed

Please sign in to comment.