More zlib/fwrite tracing for Solaris

R/test.data.table.R

@@ -147,6 +147,7 @@ test.data.table = function(script="tests.Rraw", verbose=FALSE, pkg=".", silent=F
     ", Sys.getlocale()=='", Sys.getlocale(), "'",
     ", l10n_info()=='", paste0(names(l10n_info()), "=", l10n_info(), collapse="; "), "'",
     ", getDTthreads()=='", paste0(gsub("[ ][ ]+","==",gsub("^[ ]+","",capture.output(invisible(getDTthreads(verbose=TRUE))))), collapse="; "), "'",
+    ", ", .Call(Cdt_zlib_version),
     "\n", sep="")
 
   if (inherits(err,"try-error")) {

src/fwrite.c

@@ -563,16 +563,50 @@ int init_stream(z_stream *stream) {
   return err;  // # nocov
 }
 
+void print_z_stream(const z_stream *s)   // temporary tracing function for #4099
+{
+  const char *byte = (char *)s;
+  DTPRINT("sizeof(z_stream)==%d: ", sizeof(z_stream));
+  for (int i=0; i<sizeof(z_stream); ++i) {
+    DTPRINT("%02x ", *(unsigned char *)byte++);  // not byte[i] is attempt to avoid valgrind's use-of-uninitialized, #4639, and z_stream={0} now too
+  }
+  // e.g. sizeof(z_stream)==56 on CRAN Solaris, 88 on Windows 64bit, and 112 on 64bit Linux
+  // trace z_stream->state->status which zlib:deflateStateCheck checks, #4826
+  // this structure is not exposed, so we'll get to it via memory offsets using the trace output we put in to show on CRAN's Solaris output
+  const char *pos = (char *)&s->msg + sizeof(char *); // the field after *msg (exposed) is internal_state *state (not exposed)
+  byte = *(char **)pos;  // byte now at start of internal_state pointed to by s->state
+  char *strm = *(char **)byte; // first 8 bytes (or 4 on 32bit) is strm labeled 'pointer back to this zlib stream'
+  DTPRINT("state: ");
+  for (int i=0; i<(sizeof(char *) + sizeof(int)); ++i) {
+    DTPRINT("%02x ", *(unsigned char *)byte++);
+  }
+  int status = *(int *)(byte-sizeof(int));
+  DTPRINT("strm==%p state->strm==%p state->status==%d", s, strm, status);  // two pointer values should be the same
+  DTPRINT(" zalloc==%p zfree==%p", s->zalloc, s->zfree); // checked to be !=0 by deflate.c:deflateStateCheck
+  DTPRINT(" (s->strm==strm)==%d", (char *)s==strm);     // mimics the s->strm==strm check in deflate.c:deflateStateCheck
+  DTPRINT(" s->next_out==%p s->avail_in=%d s->next_in=%p", s->next_out, s->avail_in, s->next_in); // top of deflate.c:deflate() after the call to deflateStateCheck
+  DTPRINT(" deflates()'s checks (excluding status) would %s here",
+    (s->zalloc==(alloc_func)0 || s->zfree==(free_func)0 || s==Z_NULL || (char *)s!=strm ||
+      s->next_out==Z_NULL || (s->avail_in!=0 && s->next_in==Z_NULL)) ?
+    "return -2" : "be ok");
+  DTPRINT("\n");
+}
+
 int compressbuff(z_stream *stream, void* dest, size_t *destLen, const void* source, size_t sourceLen)
 {
   stream->next_out = dest;
   stream->avail_out = *destLen;
   stream->next_in = (Bytef *)source; // don't use z_const anywhere; #3939
   stream->avail_in = sourceLen;
-  if (verbose) DTPRINT(_("deflate input stream: %p %d %p %d\n"), stream->next_out, (int)(stream->avail_out), stream->next_in, (int)(stream->avail_in));
-
+  if (verbose) {
+    DTPRINT(_("deflate input stream: %p %d %p %d z_stream: "), stream->next_out, (int)(stream->avail_out), stream->next_in, (int)(stream->avail_in));
+    print_z_stream(stream);
+  }
   int err = deflate(stream, Z_FINISH);
-  if (verbose) DTPRINT(_("deflate returned %d with stream->total_out==%d; Z_FINISH==%d, Z_OK==%d, Z_STREAM_END==%d\n"), err, (int)(stream->total_out), Z_FINISH, Z_OK, Z_STREAM_END);
+  if (verbose) {
+    DTPRINT(_("deflate returned %d with stream->total_out==%d; Z_FINISH==%d, Z_OK==%d, Z_STREAM_END==%d z_stream: "), err, (int)(stream->total_out), Z_FINISH, Z_OK, Z_STREAM_END);
+    print_z_stream(stream);
+  }
   if (err == Z_OK) {
     // with Z_FINISH, deflate must return Z_STREAM_END if correct, otherwise it's an error and we shouldn't return Z_OK (0)
     err = -9;  // # nocov
@@ -581,15 +615,6 @@ int compressbuff(z_stream *stream, void* dest, size_t *destLen, const void* sour
   return err == Z_STREAM_END ? Z_OK : err;
 }
 
-void print_z_stream(const z_stream *s)   // temporary tracing function for #4099
-{
-  const unsigned char *byte = (unsigned char *)s;
-  for (int i=0; i<sizeof(z_stream); ++i) {
-    DTPRINT("%02x ", *byte++);  // not byte[i] is attempt to avoid valgrind's use-of-uninitialized, #4639, and z_stream={0} now too
-  }
-  DTPRINT("\n");
-}
-
 void fwriteMain(fwriteMainArgs args)
 {
   double startTime = wallclock();
@@ -802,6 +827,7 @@ void fwriteMain(fwriteMainArgs args)
     if(init_stream(&stream))
       STOP(_("Can't allocate gzip stream structure")); // # nocov
     zbuffSize = deflateBound(&stream, buffSize);
+    if (verbose) DTPRINT("zbuffSize=%d returned from deflateBound\n", (int)zbuffSize);
     deflateEnd(&stream);
   }
 
@@ -986,3 +1012,5 @@ void fwriteMain(fwriteMainArgs args)
     // # nocov end
   }
 }
+
+

src/init.c

@@ -124,6 +124,7 @@ SEXP unlock();
 SEXP islockedR();
 SEXP allNAR();
 SEXP test_dt_win_snprintf();
+SEXP dt_zlib_version();
 
 // .Externals
 SEXP fastmean();
@@ -217,6 +218,7 @@ R_CallMethodDef callMethods[] = {
 {"CtestMsgR", (DL_FUNC) &testMsgR, -1},
 {"C_allNAR", (DL_FUNC) &allNAR, -1},
 {"Ctest_dt_win_snprintf", (DL_FUNC)&test_dt_win_snprintf, -1},
+{"Cdt_zlib_version", (DL_FUNC)&dt_zlib_version, -1},
 {NULL, NULL, 0}
 };
 

src/utils.c

@@ -374,3 +374,10 @@ SEXP coerceUtf8IfNeeded(SEXP x) {
   return(ans);
 }
 
+#include <zlib.h>
+SEXP dt_zlib_version() {
+  char out[51];
+  snprintf(out, 50, "zlibVersion()==%s ZLIB_VERSION==%s", zlibVersion(), ZLIB_VERSION);
+  return ScalarString(mkChar(out));
+}
+