ckp

dap4_test/test_common.h

@@ -21,6 +21,7 @@ typedef int TDMR;
 static NCbytes* input = NULL;
 static NCbytes* output = NULL;
 static NCD4meta* metadata = NULL;
+static NCD4response* resp = NULL;
 static char* infile = NULL;
 static char* outfile = NULL;
 static int ncid = 0;
@@ -85,16 +86,21 @@ setup(int tdmr, int argc, char** argv)
  if(translatenc4)
  controller->controls.translation = NCD4_TRANSNC4;
  NCD4_applyclientfragmentcontrols(controller);
- if((metadata=NCD4_newmeta(controller))==NULL)
- fail(NC_ENOMEM);
- metadata->mode = mode;
- NCD4_attachraw(metadata, ncbyteslength(input),ncbytescontents(input));
 
- if((ret=NCD4_dechunk(metadata))) /* ok for mode == DMR or mode == DAP */
+ if((ret=NCD4_newMeta(controller,&metadata)))
+ fail(ret);
+
+ if((ret=NCD4_newResponse(controller,&resp)))
+ fail(ret);
+ resp->raw.size = ncbyteslength(input);
+ resp->raw.memory = ncbytescontents(input);
+ resp->mode = mode;
+
+ if((ret=NCD4_dechunk(resp))) /* ok for mode == DMR or mode == DAP */
  fail(ret);
 #ifdef DEBUG
  {
- int swap = (metadata->serial.hostbigendian != metadata->serial.remotebigendian);
+ int swap = (controller->platform.hostlittleendian != resp->remotelittleendian);
  void* d = metadata->serial.dap;
  size_t sz = metadata->serial.dapsize;
  fprintf(stderr,"====================\n");

dap4_test/test_meta.c

@@ -20,7 +20,7 @@ main(int argc, char** argv)
  fprintf(stderr,"t_dmrmeta %s -> %s\n",infile,outfile);
 #endif
 
- if((ret = NCD4_parse(metadata))) goto done;
+ if((ret = NCD4_parse(metadata,resp))) goto done;
  if((ret = NCD4_metabuild(metadata,ncid))) goto done;
 
 done:

dap4_test/test_parse.c

@@ -17,7 +17,7 @@ main(int argc, char** argv)
 
  setup(TDMR_PARSE,argc,argv);
 
- if((ret = NCD4_parse(metadata))) goto done;
+ if((ret = NCD4_parse(metadata,resp))) goto done;
  ret = NCD4_print(metadata,output);
  ncbytesnull(output);
  if(ret == NC_NOERR) {

libdap4/d4chunk.c

@@ -28,15 +28,16 @@ static int processerrchunk(NCD4response*, void* errchunk, unsigned int count);
 int
 NCD4_dechunk(NCD4response* resp)
 {
- unsigned char *praw, *phdr, *pdap;
+ unsigned char *praw, *pdmr, *phdr, *pdap, *pappend, *pchunk;
  NCD4HDR hdr;
+ int firstchunk;
 
 #ifdef D4DUMPRAW
  NCD4_tagdump(resp->serial.raw.size,resp->serial.raw.data,0,"RAW");
 #endif
 
  /* Access the returned raw data */
- praw = (unsigned char*)resp->serial.raw.memory;
+ praw = (unsigned char*)resp->raw.memory;
 
  if(resp->mode == NCD4_DSR) {
  return THROW(NC_EDMR);
@@ -47,7 +48,7 @@ NCD4_dechunk(NCD4response* resp)
  size_t len = 0;
  /* setup as dmr only */
  /* Avoid strdup since rawdata might contain nul chars */
- len = resp->serial.raw.size;
+ len = resp->raw.size;
  if((resp->serial.dmr = malloc(len+1)) == NULL)
  return THROW(NC_ENOMEM); 
  memcpy(resp->serial.dmr,praw,len);
@@ -60,60 +61,70 @@ NCD4_dechunk(NCD4response* resp)
  return THROW(NC_EDAP);
 
  /* We must be processing a DAP mode packet */
- praw = (resp->serial.dap = resp->serial.raw.memory);
- NULLBLOB(resp->serial.raw);
+ praw = resp->raw.memory;
 
  /* If the raw data looks like xml, then we almost certainly have an error */
  if(memcmp(praw,"<?xml",strlen("<?xml"))==0
  || memcmp(praw,"<!doctype",strlen("<!doctype"))==0) {
  /* Set up to report the error */
- int stat = NCD4_seterrormessage(resp, resp->serial.raw.size, resp->serial.raw.memory);
+ int stat = NCD4_seterrormessage(resp, resp->raw.size, resp->raw.memory);
  return THROW(stat); /* slight lie */
  }
 
- /* Get the DMR chunk header*/
- phdr = NCD4_getheader(praw,&hdr,resp->controller->dmrmetadata->hostlittleendian);
+ /* Get the first header to get dmr content and endian flags*/
+ pdmr = NCD4_getheader(praw,&hdr,resp->controller->platform.hostlittleendian);
  if(hdr.count == 0)
  return THROW(NC_EDMR);
- if(hdr.flags & NCD4_ERR_CHUNK) {
- return processerrchunk(resp, (void*)phdr, hdr.count);
- }
-
+ if(hdr.flags & NCD4_ERR_CHUNK)
+ return processerrchunk(resp, (void*)pdmr, hdr.count);
  resp->remotelittleendian = ((hdr.flags & NCD4_LITTLE_ENDIAN_CHUNK) ? 1 : 0);
- /* Again, avoid strxxx operations on dmr */
+
+ /* avoid strxxx operations on dmr */
  if((resp->serial.dmr = malloc(hdr.count+1)) == NULL)
  return THROW(NC_ENOMEM); 
- memcpy(resp->serial.dmr,phdr,hdr.count);
+ memcpy(resp->serial.dmr,pdmr,hdr.count);
  resp->serial.dmr[hdr.count-1] = '\0';
  /* Suppress nuls */
  (void)NCD4_elidenuls(resp->serial.dmr,hdr.count);
 
+ /* See if there is any data after the DMR */
  if(hdr.flags & NCD4_LAST_CHUNK)
  return THROW(NC_ENODATA);
 
  /* Read and concat together the data chunks */
- phdr = phdr + hdr.count; /* point to data chunk header */
+ phdr = pdmr + hdr.count; /* point to data chunk header */
  /* Do a sanity check in case the server has shorted us with no data */
- if((hdr.count + CHUNKHDRSIZE) >= resp->serial.raw.size) {
+ if((hdr.count + CHUNKHDRSIZE) >= resp->raw.size) {
  /* Server only sent the DMR part */
  resp->serial.dapsize = 0;
  return THROW(NC_EDATADDS);
  }
- pdap = resp->serial.dap; 
- for(;;) {
- phdr = NCD4_getheader(phdr,&hdr,resp->controller->dmrmetadata->hostlittleendian);
- if(hdr.flags & NCD4_ERR_CHUNK) {
- return processerrchunk(resp, (void*)phdr, hdr.count);
+ /* walk all the data chunks */
+ /* invariants:
+ praw -- beginning of the raw response
+ pdmr -- beginning of the dmr in the raw data
+ pdap -- beginning of the dechunked dap data
+ phdr -- pointer to the hdr of the current chunk
+ pchunk -- pointer to the data part of the current chunk
+ pappend -- where to append next chunk to the growing dechunked data
+ */
+ for(firstchunk=1;;firstchunk=0) { 
+ pchunk = NCD4_getheader(phdr,&hdr,resp->controller->platform.hostlittleendian); /* Process first data chunk header */
+ if(firstchunk) {
+ pdap = phdr; /* remember start point of the dechunked data */
+ pappend = phdr; /* start appending here */
  }
+ if(hdr.flags & NCD4_ERR_CHUNK)
+ return processerrchunk(resp, (void*)pchunk, hdr.count);
  /* data chunk; possibly last; possibly empty */
- if(hdr.count > 0) {
- d4memmove(pdap,phdr,hdr.count); /* will overwrite the header */
- phdr += hdr.count;
- pdap += hdr.count;
- }
+ if(hdr.count > 0)
+ d4memmove(pappend,pchunk,hdr.count); /* overwrite the header; this the heart of dechunking */
+ pappend += hdr.count; /* next append point */
+ phdr = pchunk + hdr.count; /* point to header of next chunk */
  if(hdr.flags & NCD4_LAST_CHUNK) break;
  }
- resp->serial.dapsize = (size_t)DELTA(pdap,resp->serial.dap);
+ resp->serial.dap = pdap; /* start of dechunked data */
+ resp->serial.dapsize = (size_t)DELTA(pappend,pdap);
 
 #ifdef D4DUMPDMR
  fprintf(stderr,"%s\n",resp->serial.dmr);
@@ -143,8 +154,8 @@ Since DSR is not standardizes, it becomes the default.
 int
 NCD4_infermode(NCD4response* resp)
 {
- d4size_t size = resp->serial.raw.size;
- char* raw = resp->serial.raw.memory;
+ d4size_t size = resp->raw.size;
+ char* raw = resp->raw.memory;
 
  if(size < 16)
  return THROW(NC_EDAP); /* must have at least this to hold a hdr + partial dmr*/ 

libdap4/d4data.c

@@ -16,7 +16,7 @@ This code serves two purposes
  (NCD4_processdata)
 2. Walk a specified variable instance to convert to netcdf4
  memory representation.
- (NCD4_fillinstance)
+ (NCD4_movetoinstance)
 
 */
 
@@ -79,7 +79,7 @@ NCD4_parcelvars(NCD4meta* meta, NCD4response* resp)
  var->data.response = resp; /* cross link */ 
  }
 done:
- return ret;
+ return THROW(ret);
 }
 
 /* Process top level vars wrt checksums and swapping */
@@ -93,7 +93,7 @@ NCD4_processdata(NCD4meta* meta, NCD4response* resp)
  NCD4offset* offset = NULL;
 
  /* Do we need to swap the dap4 data? */
- meta->swap = (meta->hostlittleendian != resp->remotelittleendian);
+ meta->swap = (meta->controller->platform.hostlittleendian != resp->remotelittleendian);
 
  /* Recursively walk the tree in prefix order 
  to get the top-level variables; also mark as unvisited */
@@ -106,7 +106,7 @@ NCD4_processdata(NCD4meta* meta, NCD4response* resp)
  for(i=0;i<nclistlength(toplevel);i++) {
  NCD4node* var = (NCD4node*)nclistget(toplevel,i);
  if(resp->inferredchecksumming) {
- /* Compute remote checksum: must occur before any byte swapping */
+ /* Compute checksum of response data: must occur before any byte swapping */
  var->data.localchecksum = NCD4_computeChecksum(meta,var);
 #ifdef DUMPCHECKSUM
  fprintf(stderr,"var %s: remote-checksum = 0x%x\n",var->name,var->data.remotechecksum);
@@ -132,6 +132,7 @@ NCD4_processdata(NCD4meta* meta, NCD4response* resp)
  if((ret=NCD4_swapdata(resp,var,meta->swap)))
  FAIL(ret,"byte swapping failed");
  }
+ var->data.valid = 1; /* Everything should be in place */
  }
 
 done:
@@ -153,7 +154,7 @@ Assumes that NCD4_processdata has been called.
 */
 
 int
-NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs)
+NCD4_movetoinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist* blobs)
 {
  int ret = NC_NOERR;
  void* dst = *dstp;
@@ -169,30 +170,30 @@ NCD4_fillinstance(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dst
  } else switch(type->subsort) {
  case NC_STRING: /* oob strings */
  if((ret=fillstring(meta,offset,&dst,blobs)))
- FAIL(ret,"fillinstance");
+ FAIL(ret,"movetoinstance");
  break;
  case NC_OPAQUE:
  if(type->opaque.size > 0) {
  /* We know the size and its the same for all instances */
  if((ret=fillopfixed(meta,type->opaque.size,offset,&dst)))
- FAIL(ret,"fillinstance");
+ FAIL(ret,"movetoinstance");
  } else {
  /* Size differs per instance, so we need to convert each opaque to a vlen */
  if((ret=fillopvar(meta,type,offset,&dst,blobs)))
- FAIL(ret,"fillinstance");
+ FAIL(ret,"movetoinstance");
  }
  break;
  case NC_STRUCT:
  if((ret=fillstruct(meta,type,offset,&dst,blobs)))
- FAIL(ret,"fillinstance");
+ FAIL(ret,"movetoinstance");
  break;
  case NC_SEQ:
  if((ret=fillseq(meta,type,offset,&dst,blobs)))
- FAIL(ret,"fillinstance");
+ FAIL(ret,"movetoinstance");
  break;
  default:
  ret = NC_EINVAL;
- FAIL(ret,"fillinstance");
+ FAIL(ret,"movetoinstance");
  }
  *dstp = dst;
 
@@ -216,7 +217,7 @@ fillstruct(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NCli
  NCD4node* field = nclistget(type->vars,i);
  NCD4node* ftype = field->basetype;
  void* fdst = (((char*)dst) + field->meta.offset);
- if((ret=NCD4_fillinstance(meta,ftype,offset,&fdst,blobs)))
+ if((ret=NCD4_movetoinstance(meta,ftype,offset,&fdst,blobs)))
  FAIL(ret,"fillstruct");
  }
  dst = ((char*)dst) + type->meta.memsize;
@@ -251,7 +252,7 @@ fillseq(NCD4meta* meta, NCD4node* type, NCD4offset* offset, void** dstp, NClist*
  for(i=0;i<recordcount;i++) {
  /* Read each record instance */
  void* recdst = ((char*)(dst->p))+(recordsize * i);
- if((ret=NCD4_fillinstance(meta,vlentype,offset,&recdst,blobs)))
+ if((ret=NCD4_movetoinstance(meta,vlentype,offset,&recdst,blobs)))
  FAIL(ret,"fillseq");
  }
  dst++;

libdap4/d4debug.h

@@ -8,7 +8,7 @@
 #include <assert.h>
 #include <stdarg.h>
 
-#undef D4CATCH /* Warning: significant performance impact */
+#define D4CATCH /* Warning: significant performance impact */
 
 #undef D4DEBUG /* general debug */
 #undef D4DEBUGPARSER