Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Zarrv3b csl md #1

Open
wants to merge 5 commits into
base: zarrv3b.tmp
Choose a base branch
from
Open

Zarrv3b csl md #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e8bb351
Allow region and bucket inference with NCS3UNK S3 server type
mannreis Jul 9, 2024
80d886a
Recognize _unknown_ S3 server type
mannreis Jul 9, 2024
6a8bcfb
Add dispatch layer for consolidated ZARR access
mannreis Nov 18, 2024
a3de059
Use Zarr Metadata layer instead of sfetching zarr json files directly
mannreis Nov 18, 2024
4637956
Avoid S3 specif calls until really necessary - this allows broad HTT…
mannreis Nov 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion include/ncs3sdk.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
/* Track the server type, if known */
typedef enum NCS3SVC {NCS3UNK=0, /* unknown */
NCS3=1, /* s3.amazon.aws */
NCS3GS=0 /* storage.googleapis.com */
NCS3GS=2 /* storage.googleapis.com */
} NCS3SVC;

/* Opaque Handles */
Expand Down
50 changes: 33 additions & 17 deletions libdispatch/ds3util.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,17 +128,24 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp)
/* split the path by "/" */
if((stat = NC_split_delim(url->path,'/',pathsegments))) goto done;

/* Distinguish path-style from virtual-host style from s3: and from other.
Virtual: https://<bucket-name>.s3.<region>.amazonaws.com/<path> (1)
or: https://<bucket-name>.s3.amazonaws.com/<path> -- region defaults (to us-east-1) (2)
Path: https://s3.<region>.amazonaws.com/<bucket-name>/<path> (3)
or: https://s3.amazonaws.com/<bucket-name>/<path> -- region defaults to us-east-1 (4)
S3: s3://<bucket-name>/<path> (5)
Google: https://storage.googleapis.com/<bucket-name>/<path> (6)
or: gs3://<bucket-name>/<path> (7)
Other: https://<host>/<bucket-name>/<path> (8)
*/
if(url->host == NULL || strlen(url->host) == 0)
/* Distinguish path-style from virtual-host style from s3: and from other.
Virtual:
(1) https://<bucket-name>.s3.<region>.amazonaws.com/<path>
(2) https://<bucket-name>.s3.amazonaws.com/<path> -- region defaults (to us-east-1)
Path:
(3) https://s3.<region>.amazonaws.com/<bucket-name>/<path>
(4) https://s3.amazonaws.com/<bucket-name>/<path> -- region defaults to us-east-1
S3:
(5) s3://<bucket-name>/<path>
Google:
(6) https://storage.googleapis.com/<bucket-name>/<path>
(7) gs3://<bucket-name>/<path>
Other:
(8) https://<host>/<bucket-name>/<path>
(9) https://<bucket-name>.s3.<region>.domain.example.com/<path>
(10)https://s3.<region>.example.com/<bucket>/<path>
*/
if(url->host == NULL || strlen(url->host) == 0)
{stat = NC_EURL; goto done;}

/* Reduce the host to standard form such as s3.amazonaws.com by pulling out the
Expand Down Expand Up @@ -186,12 +193,21 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp)
/* region is unknown */
/* bucket is unknown at this point */
svc = NCS3GS;
} else { /* Presume Format (8) */
if((host = strdup(url->host))==NULL)
{stat = NC_ENOMEM; goto done;}
/* region is unknown */
/* bucket is unknown */
}
} else { /* Presume Formats (8),(9),(10) */
if (nclistlength(hostsegments) > 3 && strcasecmp(nclistget(hostsegments, 1), "s3") == 0){
bucket = nclistremove(hostsegments, 0);
region = nclistremove(hostsegments, 2);
host = strdup(url->host + sizeof(bucket) + 1);
}else{
if (nclistlength(hostsegments) > 2 && strcasecmp(nclistget(hostsegments, 0), "s3") == 0){
region = nclistremove(hostsegments, 1);
}
if ((host = strdup(url->host)) == NULL){
stat = NC_ENOMEM;
goto done;
}
}
}

/* region = (1) from url, (2) s3->region, (3) default */
if(region == NULL && s3 != NULL)
Expand Down
4 changes: 4 additions & 0 deletions libnczarr/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ zgrp.c
zinternal.c
zmap.c
zmap_file.c
zmetadata3.c
zmetadata2.c
zmetadata.c
zodom.c
zopen.c
zprov.c
Expand All @@ -44,6 +47,7 @@ zdispatch.h
zincludes.h
zinternal.h
zmap.h
zmetadata.h
zodom.h
zprovenance.h
zfilter.h
Expand Down
46 changes: 21 additions & 25 deletions libnczarr/zformat2.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1046,17 +1046,19 @@ read_grp_contents(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp)
purezarr = (zfile->flags & FLAG_PUREZARR);

/* Read .zgroup (might be NULL) */
if((stat = NCZ_grpkey(grp,&grppath))) goto done;
if((stat = nczm_concat(grppath,Z2GROUP,&key))) goto done;
stat=NCZ_downloadjson(zfile->map,key,&jgroup);
nullfree(key); key = NULL;
if(stat) goto done;

/* Read .zattrs (might be NULL) */
if((stat = nczm_concat(grppath,Z2ATTRS,&key))) goto done;
stat=NCZ_downloadjson(zfile->map,key,&jatts);
nullfree(key); key = NULL;
if(stat) goto done;
NCZMD_fetch_json_group(zfile, grp, NULL, &jgroup);
// if((stat = NCZ_grpkey(grp,&grppath))) goto done;
// if((stat = nczm_concat(grppath,Z2GROUP,&key))) goto done;
// stat=NCZ_downloadjson(zfile->map,key,&jgroup);
// nullfree(key); key = NULL;
// if(stat) goto done;

/* Read /.zattrs (might be NULL) */
NCZMD_fetch_json_attrs(zfile, grp, NULL, &jatts);
// if((stat = nczm_concat(grppath,Z2ATTRS,&key))) goto done;
// stat=NCZ_downloadjson(zfile->map,key,&jatts);
// nullfree(key); key = NULL;
// if(stat) goto done;

/* Extract jatts and _nczarr_XXX values */
if((stat = locate_nczarr_grp_info(file, grp, jgroup, jatts, &jzgroup, &jzatts, &jzsuper, &ncv21))) goto done;
Expand Down Expand Up @@ -1224,20 +1226,13 @@ read_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname)

purezarr = (zfile->flags & FLAG_PUREZARR)?1:0;

/* Construct var path */
if((stat = NCZ_grpkey(grp,&grppath))) goto done;
if((stat = nczm_concat(grppath,varname,&varpath))) goto done;
/* Construct the path to the zarray object */
if((stat = nczm_concat(varpath,Z2ARRAY,&key))) goto done;
/* Download the .zarray object */
if((stat=NCZ_readdict(zfile->map,key,&jvar))) goto done;
nullfree(key); key = NULL;
/* Download the <varname>/.zarray object */
if((stat=NCZMD_fetch_json_array(zfile,grp,varname,&jvar))) goto done;
if(jvar == NULL) {stat = NC_ENOTZARR; goto done;}
/* Download the .zattrs object */
/* Construct the path to .zattrs object */
if((stat = nczm_concat(varpath,Z2ATTRS,&key))) goto done;
if((stat=NCZ_readdict(zfile->map,key,&jatts))) goto done;
nullfree(key); key = NULL;

/* Construct the path to <varname>/.zattrs object */
if((stat=NCZMD_fetch_json_attrs(zfile,grp,varname,&jatts))) goto done;

/* locate the _nczarr_XXX values */
if((stat = locate_nczarr_array_info(file,grp,jvar,jatts,&jzarray,&jzatts,&nczv21))) goto done;
Expand Down Expand Up @@ -1873,9 +1868,10 @@ parse_group_content_pure(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varn
ZTRACE(3,"zfile=%s grp=%s |varnames|=%u |subgrps|=%u",zfile->common.file->controller->path,grp->hdr.name,(unsigned)nclistlength(varnames),(unsigned)nclistlength(subgrps));

nclistclear(varnames);
if((stat = searchvars(zfile,grp,varnames))) goto done;
if((stat = NCZMD_list_variables(zfile,grp,varnames))) goto done;
nclistclear(subgrps);
if((stat = searchsubgrps(zfile,grp,subgrps))) goto done;
if((stat = NCZMD_list_groups(zfile,grp,subgrps))) goto done;
//if((stat = searchsubgrps(zfile,grp,subgrps))) goto done;

done:
return ZUNTRACE(THROW(stat));
Expand Down
26 changes: 11 additions & 15 deletions libnczarr/zformat3.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -861,7 +861,8 @@ ZF3_readmeta(NC_FILE_INFO_T* file)
assert(zroot);

/* Read the root group's metadata */
if((stat = NCZ_downloadjson(zfile->map, Z3METAROOT, &jrootgrp))) goto done;
//if((stat = NCZ_downloadjson(zfile->map, Z3METAROOT, &jrootgrp))) goto done;
if((stat = NCZMD_fetch_json_group(zfile, NULL, NULL, &jrootgrp))) goto done;
if(jrootgrp == NULL) {/* not there */
zfile->flags |= FLAG_PUREZARR;
stat = NC_NOERR; /* reset */
Expand Down Expand Up @@ -1184,13 +1185,14 @@ read_grp_contents(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp)

/* build Z3GROUP path */
/* Construct grp path */
if((stat = NCZ_grpkey(grp,&grppath))) goto done;
/* and the grp's zarr.json */
if((stat = nczm_concat(grppath,Z3GROUP,&key))) goto done;
// if((stat = NCZ_grpkey(grp,&grppath))) goto done;
// /* and the grp's zarr.json */
// if((stat = nczm_concat(grppath,Z3GROUP,&key))) goto done;

/* Read zarr.json */
stat=NCZ_downloadjson(map,key,&jgroup);
nullfree(key); key = NULL;
//stat=NCZ_downloadjson(map,key,&jgroup);
stat=NCZMD_fetch_json_group(zfile,grp,key,&jgroup);
// nullfree(key); key = NULL;
if(stat) goto done;

/* Verify that group zarr.json exists */
Expand Down Expand Up @@ -1898,15 +1900,9 @@ static int
subobjects_pure(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames, NClist* grpnames)
{
int stat = NC_NOERR;
char* grpkey = NULL;

/* Compute the key for the grp */
if((stat = NCZ_grpkey(grp,&grpkey))) goto done;
/* Get the map and search group */
if((stat = getnextlevel(zfile,grp,varnames,grpnames))) goto done;

done:
nullfree(grpkey);
// Get names of variables and groups present in grp (from consolidated view or list storage)
stat = NCZMD_list_variables(zfile, grp, varnames);
stat = NCZMD_list_groups(zfile,grp,grpnames);
return stat;
}

Expand Down
1 change: 1 addition & 0 deletions libnczarr/zincludes.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ extern "C" {
#include "ncproplist.h"

#include "zmap.h"
#include "zmetadata.h"
#include "zinternal.h"
#include "zfilter.h"
#include "zformat.h"
Expand Down
29 changes: 6 additions & 23 deletions libnczarr/zinfer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,32 +118,15 @@ infer_open_format(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, int
const NCjson* jsupera = NULL;
struct TagParam param;

/* Probe the map for tell-tale objects and dict keys */

if(zarrformat == 0) {
/* We need to search subtree for a V2 or V3 tag */
param.zarrformat = 0; param.nczarrformat = 0;
switch(stat = nczmap_walk(map,"/",tagsearch, &param)) {
case NC_NOERR:
/* No tag was found, so its not a zarr file */
stat = NC_ENOTZARR;
goto done;
case NC_EOBJECT: /* Arbitrary error signaling found and format is in param */
stat = NC_NOERR;
switch(param.zarrformat) {
case ZARRFORMAT2: case ZARRFORMAT3: zarrformat = param.zarrformat; break;
default: stat = NC_ENOTZARR; goto done;
}
break;
default: stat = NC_ENOTZARR; goto done;
}
}
stat = NCZMD_set_metadata_handler(zfile, (const NCZ_Metadata **)&(zfile->metadata_handler));

stat = NCZMD_get_metadata_format(zfile, &zarrformat);

if(zarrformat == ZARRFORMAT2 && nczarrformat == 0) {
NCjson* jrootatts = NULL;
/* Download /.zattrs and /.zgroup */
if((stat = NCZ_downloadjson(zfile->map, Z2ATTSROOT, &jrootgrp))) goto done;
if((stat = NCZ_downloadjson(zfile->map, Z2METAROOT, &jrootatts))) goto done;
if((stat = NCZMD_fetch_json_attrs(zfile, NULL, NULL, &jrootgrp))) goto done;
if((stat = NCZMD_fetch_json_group(zfile, NULL,NULL, &jrootatts))) goto done;
/* Look for superblock */
if(jrootgrp != NULL) NCJdictget(jrootgrp,NCZ_V2_SUPERBLOCK,&jsuperg);
if(jrootatts != NULL) NCJdictget(jrootatts,NCZ_V2_SUPERBLOCK,&jsupera);
Expand All @@ -155,7 +138,7 @@ infer_open_format(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, int
if(zarrformat == ZARRFORMAT3 && nczarrformat == 0) {
const NCjson* jrootatts = NULL;
/* Look for "/zarr.json" */
if((stat = NCZ_downloadjson(zfile->map, Z3METAROOT, &jrootgrp))) goto done;
if((stat = NCZMD_fetch_json_group(zfile, NULL,NULL, &jrootgrp))) goto done;
if(jrootgrp == NULL || NCJsort(jrootgrp) != NCJ_DICT) {
nczarrformat = NCZARRFORMAT0;
} else {
Expand Down
2 changes: 2 additions & 0 deletions libnczarr/zinternal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
#define Z3METAROOT "/zarr.json"
#define Z3OBJECT "zarr.json"
#define Z3GROUP Z3OBJECT
#define Z3ATTRS Z3OBJECT
#define Z3ARRAY Z3OBJECT

/* Bytes codec name */
Expand Down Expand Up @@ -252,6 +253,7 @@ typedef struct NCZ_FILE_INFO {
# define FLAG_XARRAYDIMS 8
NCZM_IMPL mapimpl;
struct NCZ_Formatter* dispatcher;
struct NCZ_Metadata * metadata_handler;
struct NCZ_META_HDR* metastate; /* Hold per-format state */
} NCZ_FILE_INFO_T;

Expand Down
16 changes: 8 additions & 8 deletions libnczarr/zmap_s3sdk.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,14 +224,14 @@ zs3open(const char *path, mode_t mode, size64_t flags, void* parameters, NCZMAP*
z3map->s3client = NC_s3sdkcreateclient(&z3map->s3);

/* Search the root for content */
content = nclistnew();
if((stat = NC_s3sdklist(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey,&nkeys,NULL,&z3map->errmsg)))
goto done;
if(nkeys == 0) {
/* dataset does not actually exist; we choose to return ENOOBJECT instead of EEMPTY */
stat = NC_ENOOBJECT;
goto done;
}
//content = nclistnew();
//if((stat = NC_s3sdklist(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey,&nkeys,NULL,&z3map->errmsg)))
//goto done;
// if(nkeys == 0) {
// /* dataset does not actually exist; we choose to return ENOOBJECT instead of EEMPTY */
// stat = NC_ENOOBJECT;
// goto done;
// }
if(mapp) *mapp = (NCZMAP*)z3map;

done:
Expand Down
Loading