From b66392b9d9a46712ac14831dc196d49780866dd1 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Wed, 5 Jul 2023 22:16:57 -0600 Subject: [PATCH] test1 --- .github/workflows/run_tests_osx.yml | 2 +- .github/workflows/run_tests_s3.yml | 2 +- .github/workflows/run_tests_ubuntu.yml | 2 +- .github/workflows/run_tests_win_cygwin.yml | 2 +- .github/workflows/run_tests_win_mingw.yml | 2 +- dap4_test/test_meta.c | 2 +- dap4_test/test_parse.c | 2 +- libdap4/d4data.c | 14 ++++++----- libdap4/d4file.c | 2 +- libdap4/d4parser.c | 8 ++++--- libdap4/d4varx.c | 7 ++++-- libdap4/ncd4.h | 3 ++- libdap4/ncd4types.h | 1 + libnczarr/zarr.c | 27 +++++++++++----------- libnczarr/zarr.h | 4 ++-- libnczarr/zclose.c | 2 +- libnczarr/zcreate.c | 2 +- libnczarr/zinternal.h | 2 +- libnczarr/zopen.c | 2 +- 19 files changed, 49 insertions(+), 39 deletions(-) diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index c887c6b1e6..da1421729b 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -7,7 +7,7 @@ name: Run macOS-based netCDF Tests -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] jobs: diff --git a/.github/workflows/run_tests_s3.yml b/.github/workflows/run_tests_s3.yml index 0a1c942460..aca54f6eef 100644 --- a/.github/workflows/run_tests_s3.yml +++ b/.github/workflows/run_tests_s3.yml @@ -9,7 +9,7 @@ name: Run S3 netCDF Tests (under Ubuntu Linux) -on: [workflow_dispatch] +on: [push,workflow_dispatch] jobs: diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index fcd2eba7db..fa1c16e5c5 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -4,7 +4,7 @@ name: Run Ubuntu/Linux netCDF Tests -on: [pull_request, workflow_dispatch] +on: [push,pull_request, workflow_dispatch] jobs: diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index bc084a401f..5a2536b7cc 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -1,6 +1,6 @@ name: Run Cygwin-based tests -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] env: SHELLOPTS: igncr diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index 7b8bfef208..16264a9f46 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -9,7 +9,7 @@ name: Run MSYS2, MinGW64-based Tests env: CPPFLAGS: "-D_BSD_SOURCE" -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] jobs: diff --git a/dap4_test/test_meta.c b/dap4_test/test_meta.c index b9bbafc113..478fcfb62d 100644 --- a/dap4_test/test_meta.c +++ b/dap4_test/test_meta.c @@ -20,7 +20,7 @@ main(int argc, char** argv) fprintf(stderr,"t_dmrmeta %s -> %s\n",infile,outfile); #endif - if((ret = NCD4_parse(metadata,resp))) goto done; + if((ret = NCD4_parse(metadata,resp,0))) goto done; if((ret = NCD4_metabuild(metadata,ncid))) goto done; done: diff --git a/dap4_test/test_parse.c b/dap4_test/test_parse.c index f609ddbf5e..59afb74d5e 100644 --- a/dap4_test/test_parse.c +++ b/dap4_test/test_parse.c @@ -17,7 +17,7 @@ main(int argc, char** argv) setup(TDMR_PARSE,argc,argv); - if((ret = NCD4_parse(metadata,resp))) goto done; + if((ret = NCD4_parse(metadata,resp,0))) goto done; ret = NCD4_print(metadata,output); ncbytesnull(output); if(ret == NC_NOERR) { diff --git a/libdap4/d4data.c b/libdap4/d4data.c index 32930556ae..234f3e3776 100644 --- a/libdap4/d4data.c +++ b/libdap4/d4data.c @@ -29,7 +29,6 @@ static int fillopfixed(NCD4meta*, d4size_t opaquesize, NCD4offset* offset, void* static int fillopvar(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); static int fillstruct(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); static int fillseq(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); -static int NCD4_inferChecksums(NCD4response* resp, NClist* toplevel); static unsigned NCD4_computeChecksum(NCD4meta* meta, NCD4node* topvar); /***************************************************/ @@ -100,13 +99,11 @@ NCD4_processdata(NCD4meta* meta, NCD4response* resp) toplevel = nclistnew(); NCD4_getToplevelVars(meta,root,toplevel); - NCD4_inferChecksums(resp,toplevel); - /* Extract remote checksums */ for(i=0;iinferredchecksumming) { - /* Compute checksum of response data: must occur before any byte swapping */ + /* Compute checksum of response data: must occur before any byte swapping and after delimiting */ var->data.localchecksum = NCD4_computeChecksum(meta,var); #ifdef DUMPCHECKSUM fprintf(stderr,"var %s: remote-checksum = 0x%x\n",var->name,var->data.remotechecksum); @@ -394,11 +391,16 @@ fprintf(stderr,"toplevel: var=%s\n",node->name); return THROW(ret); } -static int -NCD4_inferChecksums(NCD4response* resp, NClist* toplevel) +int +NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp) { int ret = NC_NOERR; int i, attrfound; + NClist* toplevel = NULL; + + /* Get the toplevel vars */ + toplevel = nclistnew(); + NCD4_getToplevelVars(meta,meta->root,toplevel); /* First, look thru the DMR to see if there is a checksum attribute */ attrfound = 0; diff --git a/libdap4/d4file.c b/libdap4/d4file.c index f7ee43f55d..d6cb387b4b 100644 --- a/libdap4/d4file.c +++ b/libdap4/d4file.c @@ -188,7 +188,7 @@ NCD4_open(const char * path, int mode, } #endif - if((ret = NCD4_parse(meta,dmrresp))) goto done; + if((ret = NCD4_parse(meta,dmrresp,0))) goto done; #ifdef D4DEBUGMETA { diff --git a/libdap4/d4parser.c b/libdap4/d4parser.c index 506431b039..90b27e952c 100644 --- a/libdap4/d4parser.c +++ b/libdap4/d4parser.c @@ -154,7 +154,7 @@ static int defineBytestringType(NCD4parser*); /* API */ int -NCD4_parse(NCD4meta* metadata, NCD4response* resp) +NCD4_parse(NCD4meta* metadata, NCD4response* resp, int dapparse) { int ret = NC_NOERR; NCD4parser* parser = NULL; @@ -180,6 +180,7 @@ NCD4_parse(NCD4meta* metadata, NCD4response* resp) #ifdef D4DEBUG parser->debuglevel = 1; #endif + parser->dapparse = dapparse; /*Walk the DOM tree to build the DAP4 node tree*/ ret = traverse(parser,dom); @@ -1654,9 +1655,10 @@ parseForwards(NCD4parser* parser, NCD4node* root) const char* mapname = (const char*)nclistget(var->mapnames,j); /* Find the corresponding variable */ NCD4node* mapref = lookupFQN(parser,mapname,NCD4_VAR); - if(mapref == NULL) + if(mapref != NULL) + PUSH(var->maps,mapref); + else if(!parser->dapparse) FAIL(NC_ENOTVAR," name does not refer to a variable: %s",mapname); - PUSH(var->maps,mapref); } } diff --git a/libdap4/d4varx.c b/libdap4/d4varx.c index d53fbe1a55..1701718004 100644 --- a/libdap4/d4varx.c +++ b/libdap4/d4varx.c @@ -196,10 +196,13 @@ getvarx(int gid, int varid, NCD4INFO** infop, NCD4node** varp, /* Extract DMR and dechunk the data part */ if((ret=NCD4_dechunk(dapresp))) goto done; /* Process the dmr part */ - if((ret=NCD4_parse(dapmeta,dapresp))) goto done; - if((ret = NCD4_processdata(dapmeta,dapresp))) goto done; + if((ret=NCD4_parse(dapmeta,dapresp,1))) goto done; + /* See if we are checksumming */ + if((ret=NCD4_inferChecksums(dapmeta,dapresp))) goto done; /* connect variables and corresponding dap data */ if((ret = NCD4_parcelvars(dapmeta,dapresp))) goto done; + /* Process checksums and byte-order swapping */ + if((ret = NCD4_processdata(dapmeta,dapresp))) goto done; /* Transfer and process the data */ if((ret = mapvars(dapmeta,dmrmeta,dapresp->inferredchecksumming))) goto done; diff --git a/libdap4/ncd4.h b/libdap4/ncd4.h index fdd6917695..286406d771 100644 --- a/libdap4/ncd4.h +++ b/libdap4/ncd4.h @@ -89,7 +89,7 @@ EXTERNL int NCD4_readDAP(NCD4INFO* state, int flags, NCURI* ceuri, NCD4response* EXTERNL int NCD4_seterrormessage(NCD4response*, size_t len, char* msg); /* From d4parser.c */ -EXTERNL int NCD4_parse(NCD4meta*, NCD4response*); +EXTERNL int NCD4_parse(NCD4meta*, NCD4response*, int dapparse); EXTERNL NCD4node* NCD4_findAttr(NCD4node* container, const char* attrname); EXTERNL NCD4node* NCD4_groupFor(NCD4node* node); EXTERNL int NCD4_defineattr(NCD4meta* meta, NCD4node* parent, const char* aname, const char* typename, NCD4node** attrp); @@ -130,6 +130,7 @@ EXTERNL int NCD4_parcelvars(NCD4meta* meta, NCD4response* resp); EXTERNL int NCD4_processdata(NCD4meta*,NCD4response*); EXTERNL int NCD4_movetoinstance(NCD4meta*, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs); EXTERNL int NCD4_getToplevelVars(NCD4meta* meta, NCD4node* group, NClist* toplevel); +EXTERNL int NCD4_inferChecksums(NCD4meta* meta, NCD4response* resp); /* From d4util.c */ EXTERNL d4size_t NCD4_dimproduct(NCD4node* node); diff --git a/libdap4/ncd4types.h b/libdap4/ncd4types.h index 42c9d9cf85..ca0a5118ac 100644 --- a/libdap4/ncd4types.h +++ b/libdap4/ncd4types.h @@ -275,6 +275,7 @@ typedef struct NCD4parser { NCD4INFO* controller; char* input; int debuglevel; + int dapparse; /* 1 => we are parsing the DAP DMR */ NCD4meta* metadata; NCD4response* response; /* Capture useful subsets of dataset->allnodes */ diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c index 67232e74b8..cd40c1c2c7 100644 --- a/libnczarr/zarr.c +++ b/libnczarr/zarr.c @@ -22,7 +22,7 @@ static int applycontrols(NCZ_FILE_INFO_T* zinfo); */ int -ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** controls) +ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* controls) { int stat = NC_NOERR; NCZ_FILE_INFO_T* zinfo = NULL; @@ -32,7 +32,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr NCjson* json = NULL; char* key = NULL; - ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_envv(controls):"null")); + ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_env(controls):"null")); nc = (NC*)file->controller; @@ -52,7 +52,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr zinfo->created = 1; zinfo->common.file = file; zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG); - if((zinfo->envv_controls=NCZ_clonestringvec(0,controls)) == NULL) + if((zinfo->controllist=nclistclone(controls,1)) == NULL) {stat = NC_ENOMEM; goto done;} /* fill in some of the zinfo and zroot fields */ @@ -94,7 +94,7 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, const char** contr */ int -ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls) +ncz_open_dataset(NC_FILE_INFO_T* file, NClist* controls) { int stat = NC_NOERR; NC* nc = NULL; @@ -126,7 +126,7 @@ ncz_open_dataset(NC_FILE_INFO_T* file, const char** controls) zinfo->created = 0; zinfo->common.file = file; zinfo->native_endianness = (NCZ_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG); - if((zinfo->envv_controls = NCZ_clonestringvec(0,controls))==NULL) /*0=>envv style*/ + if((zinfo->controllist=nclistclone(controls,1)) == NULL) {stat = NC_ENOMEM; goto done;} zinfo->default_maxstrlen = NCZ_MAXSTR_DEFAULT; @@ -294,12 +294,13 @@ ncz_open_rootgroup(NC_FILE_INFO_T* dataset) static const char* -controllookup(const char** envv_controls, const char* key) +controllookup(NClist* controls, const char* key) { - const char** p; - for(p=envv_controls;*p;p+=2) { - if(strcasecmp(key,*p)==0) { - return p[1]; + int i; + for(i=0;ienvv_controls,"mode")) != NULL) { + if((value = controllookup(zinfo->controllist,"mode")) != NULL) { if((stat = NCZ_comma_parse(value,modelist))) goto done; } /* Process the modelist first */ @@ -337,11 +338,11 @@ applycontrols(NCZ_FILE_INFO_T* zinfo) zinfo->controls.flags &= (~noflags); /* Process other controls */ - if((value = controllookup((const char**)zinfo->envv_controls,"log")) != NULL) { + if((value = controllookup(zinfo->controllist,"log")) != NULL) { zinfo->controls.flags |= FLAG_LOGGING; ncsetlogging(1); } - if((value = controllookup((const char**)zinfo->envv_controls,"show")) != NULL) { + if((value = controllookup(zinfo->controllist,"show")) != NULL) { if(strcasecmp(value,"fetch")==0) zinfo->controls.flags |= FLAG_SHOWFETCH; } diff --git a/libnczarr/zarr.h b/libnczarr/zarr.h index 6957bdd144..32de709afa 100644 --- a/libnczarr/zarr.h +++ b/libnczarr/zarr.h @@ -26,8 +26,8 @@ struct ZCVT { #define zcvt_empty {0,0,0.0,NULL} /* zarr.c */ -EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, const char** controls); -EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, const char** controls); +EXTERNL int ncz_create_dataset(NC_FILE_INFO_T*, NC_GRP_INFO_T*, NClist* controls); +EXTERNL int ncz_open_dataset(NC_FILE_INFO_T*, NClist* controls); EXTERNL int ncz_del_attr(NC_FILE_INFO_T* file, NC_OBJ* container, const char* name); /* HDF5 Mimics */ diff --git a/libnczarr/zclose.c b/libnczarr/zclose.c index eda6fcc2b0..3d8d7d75a5 100644 --- a/libnczarr/zclose.c +++ b/libnczarr/zclose.c @@ -49,7 +49,7 @@ ncz_close_file(NC_FILE_INFO_T* file, int abort) if((stat = nczmap_close(zinfo->map,(abort && zinfo->created)?1:0))) goto done; - NCZ_freestringvec(0,zinfo->envv_controls); + nclistfreeall(zinfo->controllist); NC_authfree(zinfo->auth); nullfree(zinfo); diff --git a/libnczarr/zcreate.c b/libnczarr/zcreate.c index 02da9a3309..373490865b 100644 --- a/libnczarr/zcreate.c +++ b/libnczarr/zcreate.c @@ -30,7 +30,7 @@ static const int ILLEGAL_CREATE_FLAGS = (NC_NOWRITE|NC_MMAP|NC_DISKLESS|NC_64BIT * @author Dennis Heimbigner, Ed Hartnett */ static int -ncz_create_file(const char *path, int cmode, size_t initialsz, const char** controls, int ncid) +ncz_create_file(const char *path, int cmode, size_t initialsz, NClist* controls, int ncid) { int retval = NC_NOERR; NC_FILE_INFO_T* h5 = NULL; diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 4a441c49fd..c1a5a00133 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -144,7 +144,7 @@ typedef struct NCZ_FILE_INFO { } zarr; int created; /* 1=> created 0=>open */ int native_endianness; /* NC_ENDIAN_LITTLE | NC_ENDIAN_BIG */ - char** envv_controls; /* Envv format */ + NClist* controllist; /* Envv format */ struct Controls { size64_t flags; # define FLAG_PUREZARR 1 diff --git a/libnczarr/zopen.c b/libnczarr/zopen.c index 8d8644e0fa..2171f7b23d 100644 --- a/libnczarr/zopen.c +++ b/libnczarr/zopen.c @@ -64,7 +64,7 @@ check_for_classic_model(NC_GRP_INFO_T *root_grp, int *is_classic) * @author Dennis Heimbigner, Ed Hartnett */ static int -ncz_open_file(const char *path, int mode, const char** controls, int ncid) +ncz_open_file(const char *path, int mode, NClist* controls, int ncid) { int stat = NC_NOERR; NC_FILE_INFO_T *h5 = NULL;