Skip to content

Commit

Permalink
Merge pull request #2936 from DennisHeimbigner/v2new.dmh
Browse files Browse the repository at this point in the history 
Convert NCzarr meta-data to use only Zarr attributes
  • Loading branch information
@WardF
WardF authored Jul 8, 2024
2 parents d694d38 + 06d2fe5 commit 18a98c4
Show file tree
Hide file tree
Showing 67 changed files with 1,539 additions and 1,330 deletions.
1 change: 1 addition & 0 deletions RELEASE_NOTES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release

## 4.9.3 - TBD

* Convert NCZarr V2 to store all netcdf-4 specific info as attributes. This improves interoperability with other Zarr implementations by no longer using non-standard keys. The price to be paid is that lazy attribute reading cannot be supported. See [Github #2836](https://github.com/Unidata/netcdf-c/issues/2936) for more information.
* Cleanup the option code for NETCDF_ENABLE_SET_LOG_LEVEL\[_FUNC\] See [Github #2931](https://github.com/Unidata/netcdf-c/issues/2931) for more information.
* Fix duplicate definition when using aws-sdk-cpp. See [Github #2928](https://github.com/Unidata/netcdf-c/issues/2928) for more information.
* Cleanup various obsolete options and do some code refactoring. See [Github #2926](https://github.com/Unidata/netcdf-c/issues/2926) for more information.
Expand Down
420 changes: 290 additions & 130 deletions docs/nczarr.md
View file

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion include/nc4internal.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,5 @@ extern void NC_initialize_reserved(void);
#define NC_NCZARR_GROUP "_nczarr_group"
#define NC_NCZARR_ARRAY "_nczarr_array"
#define NC_NCZARR_ATTR "_nczarr_attr"
#define NC_NCZARR_ATTR_UC "_NCZARR_ATTR" /* deprecated */

#endif /* _NC4INTERNAL_ */
15 changes: 12 additions & 3 deletions include/ncjson.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ typedef struct NCjson {
int sort; /* of this object */
char* string; /* sort != DICT|ARRAY */
struct NCjlist {
int len;
size_t len;
struct NCjson** contents;
} list; /* sort == DICT|ARRAY */
} NCjson;
Expand Down Expand Up @@ -96,7 +96,7 @@ OPTEXPORT int NCJnewstring(int sort, const char* value, NCjson** jsonp);
OPTEXPORT int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);

/* Get dict key value by name */
OPTEXPORT int NCJdictget(const NCjson* dict, const char* key, NCjson** valuep);
OPTEXPORT int NCJdictget(const NCjson* dict, const char* key, const NCjson** valuep);

/* Convert one json sort to value of another type; don't use union so we can know when to reclaim sval */
OPTEXPORT int NCJcvt(const NCjson* value, int outsort, struct NCJconst* output);
Expand All @@ -108,7 +108,14 @@ OPTEXPORT int NCJaddstring(NCjson* json, int sort, const char* s);
OPTEXPORT int NCJappend(NCjson* object, NCjson* value);

/* Insert key-value pair into a dict object. key will be copied */
OPTEXPORT int NCJinsert(NCjson* object, char* key, NCjson* value);
OPTEXPORT int NCJinsert(NCjson* object, const char* key, NCjson* value);

/* Insert key-value pair as strings into a dict object.
key and value will be copied */
OPTEXPORT int NCJinsertstring(NCjson* object, const char* key, const char* value);

/* Insert key-value pair where value is an int */
OPTEXPORT int NCJinsertint(NCjson* object, const char* key, long long ivalue);

/* Unparser to convert NCjson object to text in buffer */
OPTEXPORT int NCJunparse(const NCjson* json, unsigned flags, char** textp);
Expand All @@ -131,8 +138,10 @@ OPTEXPORT const char* NCJtotext(const NCjson* json);
#define NCJsort(x) ((x)->sort)
#define NCJstring(x) ((x)->string)
#define NCJlength(x) ((x)==NULL ? 0 : (x)->list.len)
#define NCJdictlength(x) ((x)==NULL ? 0 : (x)->list.len/2)
#define NCJcontents(x) ((x)->list.contents)
#define NCJith(x,i) ((x)->list.contents[i])
#define NCJdictith(x,i) ((x)->list.contents[2*i])

/* Setters */
#define NCJsetsort(x,s) (x)->sort=(s)
Expand Down
53 changes: 47 additions & 6 deletions include/netcdf_json.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ typedef struct NCjson {
int sort; /* of this object */
char* string; /* sort != DICT|ARRAY */
struct NCjlist {
int len;
size_t len;
struct NCjson** contents;
} list; /* sort == DICT|ARRAY */
} NCjson;
Expand Down Expand Up @@ -96,7 +96,7 @@ OPTEXPORT int NCJnewstring(int sort, const char* value, NCjson** jsonp);
OPTEXPORT int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);

/* Get dict key value by name */
OPTEXPORT int NCJdictget(const NCjson* dict, const char* key, NCjson** valuep);
OPTEXPORT int NCJdictget(const NCjson* dict, const char* key, const NCjson** valuep);

/* Convert one json sort to value of another type; don't use union so we can know when to reclaim sval */
OPTEXPORT int NCJcvt(const NCjson* value, int outsort, struct NCJconst* output);
Expand All @@ -108,7 +108,14 @@ OPTEXPORT int NCJaddstring(NCjson* json, int sort, const char* s);
OPTEXPORT int NCJappend(NCjson* object, NCjson* value);

/* Insert key-value pair into a dict object. key will be copied */
OPTEXPORT int NCJinsert(NCjson* object, char* key, NCjson* value);
OPTEXPORT int NCJinsert(NCjson* object, const char* key, NCjson* value);

/* Insert key-value pair as strings into a dict object.
key and value will be copied */
OPTEXPORT int NCJinsertstring(NCjson* object, const char* key, const char* value);

/* Insert key-value pair where value is an int */
OPTEXPORT int NCJinsertint(NCjson* object, const char* key, long long ivalue);

/* Unparser to convert NCjson object to text in buffer */
OPTEXPORT int NCJunparse(const NCjson* json, unsigned flags, char** textp);
Expand All @@ -131,8 +138,10 @@ OPTEXPORT const char* NCJtotext(const NCjson* json);
#define NCJsort(x) ((x)->sort)
#define NCJstring(x) ((x)->string)
#define NCJlength(x) ((x)==NULL ? 0 : (x)->list.len)
#define NCJdictlength(x) ((x)==NULL ? 0 : (x)->list.len/2)
#define NCJcontents(x) ((x)->list.contents)
#define NCJith(x,i) ((x)->list.contents[i])
#define NCJdictith(x,i) ((x)->list.contents[2*i])

/* Setters */
#define NCJsetsort(x,s) (x)->sort=(s)
Expand Down Expand Up @@ -278,7 +287,9 @@ static int NCJnewstring(int sort, const char* value, NCjson** jsonp);
static int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);
static int NCJclone(const NCjson* json, NCjson** clonep);
static int NCJaddstring(NCjson* json, int sort, const char* s);
static int NCJinsert(NCjson* object, char* key, NCjson* jvalue);
static int NCJinsert(NCjson* object, const char* key, NCjson* jvalue);
static int NCJinsertstring(NCjson* object, const char* key, const char* value);
static int NCJinsertint(NCjson* object, const char* key, long long ivalue);
static int NCJappend(NCjson* object, NCjson* value);
static int NCJunparse(const NCjson* json, unsigned flags, char** textp);
#else /*!NETCDF_JSON_H*/
Expand Down Expand Up @@ -764,7 +775,7 @@ NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp)
}

OPTSTATIC int
NCJdictget(const NCjson* dict, const char* key, NCjson** valuep)
NCJdictget(const NCjson* dict, const char* key, const NCjson** valuep)
{
int i,stat = NCJ_OK;

Expand Down Expand Up @@ -1050,7 +1061,7 @@ NCJaddstring(NCjson* json, int sort, const char* s)

/* Insert key-value pair into a dict object. key will be strdup'd */
OPTSTATIC int
NCJinsert(NCjson* object, char* key, NCjson* jvalue)
NCJinsert(NCjson* object, const char* key, NCjson* jvalue)
{
int stat = NCJ_OK;
NCjson* jkey = NULL;
Expand All @@ -1063,6 +1074,36 @@ NCJinsert(NCjson* object, char* key, NCjson* jvalue)
return NCJTHROW(stat);
}

/* Insert key-value pair as strings into a dict object.
key and value will be strdup'd */
OPTSTATIC int
NCJinsertstring(NCjson* object, const char* key, const char* value)
{
int stat = NCJ_OK;
NCjson* jvalue = NULL;
if(value == NULL)
NCJnew(NCJ_NULL,&jvalue);
else
NCJnewstring(NCJ_STRING,value,&jvalue);
NCJinsert(object,key,jvalue);
done:
return NCJTHROW(stat);
}

/* Insert key-value pair with value being an integer */
OPTSTATIC int
NCJinsertint(NCjson* object, const char* key, long long ivalue)
{
int stat = NCJ_OK;
NCjson* jvalue = NULL;
char digits[128];
snprintf(digits,sizeof(digits),"%lld",ivalue);
NCJnewstring(NCJ_STRING,digits,&jvalue);
NCJinsert(object,key,jvalue);
done:
return NCJTHROW(stat);
}

/* Append value to an array or dict object. */
OPTSTATIC int
NCJappend(NCjson* object, NCjson* value)
Expand Down
38 changes: 35 additions & 3 deletions libdispatch/ncjson.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,9 @@ static int NCJnewstring(int sort, const char* value, NCjson** jsonp);
static int NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp);
static int NCJclone(const NCjson* json, NCjson** clonep);
static int NCJaddstring(NCjson* json, int sort, const char* s);
static int NCJinsert(NCjson* object, char* key, NCjson* jvalue);
static int NCJinsert(NCjson* object, const char* key, NCjson* jvalue);
static int NCJinsertstring(NCjson* object, const char* key, const char* value);
static int NCJinsertint(NCjson* object, const char* key, long long ivalue);
static int NCJappend(NCjson* object, NCjson* value);
static int NCJunparse(const NCjson* json, unsigned flags, char** textp);
#else /*!NETCDF_JSON_H*/
Expand Down Expand Up @@ -614,7 +616,7 @@ NCJnewstringn(int sort, size_t len, const char* value, NCjson** jsonp)
}

OPTSTATIC int
NCJdictget(const NCjson* dict, const char* key, NCjson** valuep)
NCJdictget(const NCjson* dict, const char* key, const NCjson** valuep)
{
int i,stat = NCJ_OK;

Expand Down Expand Up @@ -900,7 +902,7 @@ NCJaddstring(NCjson* json, int sort, const char* s)

/* Insert key-value pair into a dict object. key will be strdup'd */
OPTSTATIC int
NCJinsert(NCjson* object, char* key, NCjson* jvalue)
NCJinsert(NCjson* object, const char* key, NCjson* jvalue)
{
int stat = NCJ_OK;
NCjson* jkey = NULL;
Expand All @@ -913,6 +915,36 @@ NCJinsert(NCjson* object, char* key, NCjson* jvalue)
return NCJTHROW(stat);
}

/* Insert key-value pair as strings into a dict object.
key and value will be strdup'd */
OPTSTATIC int
NCJinsertstring(NCjson* object, const char* key, const char* value)
{
int stat = NCJ_OK;
NCjson* jvalue = NULL;
if(value == NULL)
NCJnew(NCJ_NULL,&jvalue);
else
NCJnewstring(NCJ_STRING,value,&jvalue);
NCJinsert(object,key,jvalue);
done:
return NCJTHROW(stat);
}

/* Insert key-value pair with value being an integer */
OPTSTATIC int
NCJinsertint(NCjson* object, const char* key, long long ivalue)
{
int stat = NCJ_OK;
NCjson* jvalue = NULL;
char digits[128];
snprintf(digits,sizeof(digits),"%lld",ivalue);
NCJnewstring(NCJ_STRING,digits,&jvalue);
NCJinsert(object,key,jvalue);
done:
return NCJTHROW(stat);
}

/* Append value to an array or dict object. */
OPTSTATIC int
NCJappend(NCjson* object, NCjson* value)
Expand Down
2 changes: 1 addition & 1 deletion libdispatch/ncs3sdk_h5.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ NC_s3sdkinitialize(void)
}

/* Get environment information */
NC_s3sdkenvironment(void);
NC_s3sdkenvironment();

return NC_NOERR;
}
Expand Down
130 changes: 1 addition & 129 deletions libnczarr/zarr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,61 +239,6 @@ NCZ_get_superblock(NC_FILE_INFO_T* file, int* superblockp)
/**************************************************/
/* Utilities */

#if 0
/**
@internal Open the root group object
@param dataset - [in] the root dataset object
@param rootp - [out] created root group
@return NC_NOERR
@author Dennis Heimbigner
*/
static int
ncz_open_rootgroup(NC_FILE_INFO_T* dataset)
{
int stat = NC_NOERR;
int i;
NCZ_FILE_INFO_T* zfile = NULL;
NC_GRP_INFO_T* root = NULL;
void* content = NULL;
char* rootpath = NULL;
NCjson* json = NULL;

ZTRACE(3,"dataset=",dataset->hdr.name);

zfile = dataset->format_file_info;

/* Root should already be defined */
root = dataset->root_grp;

assert(root != NULL);

if((stat=nczm_concat(NULL,ZGROUP,&rootpath)))
goto done;
if((stat = NCZ_downloadjson(zfile->map, rootpath, &json)))
goto done;
/* Process the json */
for(i=0;i<nclistlength(json->contents);i+=2) {
const NCjson* key = nclistget(json->contents,i);
const NCjson* value = nclistget(json->contents,i+1);
if(strcmp(NCJstring(key),"zarr_format")==0) {
int zversion;
if(sscanf(NCJstring(value),"%d",&zversion)!=1)
{stat = NC_ENOTNC; goto done;}
/* Verify against the dataset */
if(zversion != zfile->zarr.zarr_version)
{stat = NC_ENOTNC; goto done;}
}
}

done:
if(json) NCJreclaim(json);
nullfree(rootpath);
nullfree(content);
return ZUNTRACE(stat);
}
#endif


static const char*
controllookup(NClist* controls, const char* key)
{
Expand All @@ -315,7 +260,7 @@ applycontrols(NCZ_FILE_INFO_T* zinfo)
int stat = NC_NOERR;
const char* value = NULL;
NClist* modelist = nclistnew();
int noflags = 0; /* track non-default negative flags */
size64_t noflags = 0; /* track non-default negative flags */

if((value = controllookup(zinfo->controllist,"mode")) != NULL) {
if((stat = NCZ_comma_parse(value,modelist))) goto done;
Expand Down Expand Up @@ -352,76 +297,3 @@ applycontrols(NCZ_FILE_INFO_T* zinfo)
nclistfreeall(modelist);
return stat;
}

#if 0
/**
@internal Rewrite attributes into a group or var
@param map - [in] the map object for storage
@param container - [in] the containing object
@param jattrs - [in] the json for .zattrs
@param jtypes - [in] the json for .ztypes
@return NC_NOERR
@author Dennis Heimbigner
*/
int
ncz_unload_jatts(NCZ_FILE_INFO_T* zinfo, NC_OBJ* container, NCjson* jattrs, NCjson* jtypes)
{
int stat = NC_NOERR;
char* fullpath = NULL;
char* akey = NULL;
char* tkey = NULL;
NCZMAP* map = zinfo->map;

assert((NCJsort(jattrs) == NCJ_DICT));
assert((NCJsort(jtypes) == NCJ_DICT));

if(container->sort == NCGRP) {
NC_GRP_INFO_T* grp = (NC_GRP_INFO_T*)container;
/* Get grp's fullpath name */
if((stat = NCZ_grpkey(grp,&fullpath)))
goto done;
} else {
NC_VAR_INFO_T* var = (NC_VAR_INFO_T*)container;
/* Get var's fullpath name */
if((stat = NCZ_varkey(var,&fullpath)))
goto done;
}

/* Construct the path to the .zattrs object */
if((stat = nczm_concat(fullpath,ZATTRS,&akey)))
goto done;

/* Always write as V2 */

{
NCjson* k = NULL;
NCjson* v = NULL;
/* remove any previous version */
if(!NCJremove(jattrs,NCZ_V2_ATTRS,1,&k,&v)) {
NCJreclaim(k); NCJreclaim(v);
}
}

if(!(zinfo->controls.flags & FLAG_PUREZARR)) {
/* Insert the jtypes into the set of attributes */
if((stat = NCJinsert(jattrs,NCZ_V2_ATTRS,jtypes))) goto done;
}

/* Upload the .zattrs object */
if((stat=NCZ_uploadjson(map,tkey,jattrs)))
goto done;

done:
if(stat) {
NCJreclaim(jattrs);
NCJreclaim(jtypes);
}
nullfree(fullpath);
nullfree(akey);
nullfree(tkey);
return stat;
}
#endif



Loading

0 comments on commit 18a98c4

Please sign in to comment.