Merge pull request #1155 from Unidata/fillmismatch.dmh

Modify DAP2 and DAP4 to optionally allow Fillvalue/Variable mismatch

.travis.yml

@@ -12,29 +12,29 @@ services:
 env:
     matrix:
 # Ubuntu
-        - DOCKIMG=unidata/nctests:serial   USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-fsigned-char --disable-netcdf-4 --disable-dap-remote-tests --enable-cdf5' COPTS='-DENABLE_NETCDF_4=OFF -DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-signed
+        - DOCKIMG=unidata/nctests:serial   USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-fsigned-char --disable-netcdf-4 --disable-dap-remote-tests --enable-cdf5' COPTS='-DENABLE_NETCDF_4=OFF -DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-signed TESTFILTER=FALSE
 
-        - DOCKIMG=unidata/nctests:serial32 USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-fsigned-char --disable-netcdf-4 --disable-dap-remote-tests' COPTS='-DENABLE_NETCDF_4=OFF -DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=OFF' USECP=FALSE CURHOST=docker-gcc-x86-signed
+        - DOCKIMG=unidata/nctests:serial32 USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-fsigned-char --disable-netcdf-4 --disable-dap-remote-tests' COPTS='-DENABLE_NETCDF_4=OFF -DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=OFF' USECP=FALSE CURHOST=docker-gcc-x86-signed TESTFILTER=FALSE
 
-        - DOCKIMG=unidata/nctests:serial   USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-funsigned-char --disable-netcdf-4 --disable-dap-remote-tests --enable-cdf5' COPTS='-DENABLE_NETCDF_4=OFF -DCMAKE_C_FLAGS=-funsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-unsigned
+        - DOCKIMG=unidata/nctests:serial   USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-funsigned-char --disable-netcdf-4 --disable-dap-remote-tests --enable-cdf5' COPTS='-DENABLE_NETCDF_4=OFF -DCMAKE_C_FLAGS=-funsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-unsigned TESTFILTER=FALSE
 
-        - DOCKIMG=unidata/nctests:serial32 USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-funsigned-char --disable-netcdf-4 --disable-dap-remote-tests' COPTS='-DENABLE_NETCDF_4=OFF -DCMAKE_C_FLAGS=-funsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=OFF' USECP=FALSE CURHOST=docker-gcc-x86-unsigned
+        - DOCKIMG=unidata/nctests:serial32 USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-funsigned-char --disable-netcdf-4 --disable-dap-remote-tests' COPTS='-DENABLE_NETCDF_4=OFF -DCMAKE_C_FLAGS=-funsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=OFF' USECP=FALSE CURHOST=docker-gcc-x86-unsigned TESTFILTER=FALSE
 
-        - DOCKIMG=unidata/nctests:serial   USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-fsigned-char --disable-dap-remote-tests --enable-cdf5' COPTS='-DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-signed
+        - DOCKIMG=unidata/nctests:serial   USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-fsigned-char --disable-dap-remote-tests --enable-cdf5' COPTS='-DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-signed TESTFILTER=TRUE
 
-        - DOCKIMG=unidata/nctests:serial32 USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-fsigned-char --disable-dap-remote-tests' COPTS='-DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=OFF' USECP=FALSE CURHOST=docker-gcc-x86-signed
+        - DOCKIMG=unidata/nctests:serial32 USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-fsigned-char --disable-dap-remote-tests' COPTS='-DCMAKE_C_FLAGS=-fsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=OFF' USECP=FALSE CURHOST=docker-gcc-x86-signed TESTFILTER=TRUE
 
-        - DOCKIMG=unidata/nctests:serial   USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-funsigned-char --disable-dap-remote-tests --enable-cdf5' COPTS='-DCMAKE_C_FLAGS=-funsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-unsigned
+        - DOCKIMG=unidata/nctests:serial   USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-funsigned-char --disable-dap-remote-tests --enable-cdf5' COPTS='-DCMAKE_C_FLAGS=-funsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=TRUE' USECP=FALSE CURHOST=docker-gcc-x64-unsigned TESTFILTER=TRUE
 
-        - DOCKIMG=unidata/nctests:serial32 USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-funsigned-char --disable-dap-remote-tests' COPTS='-DCMAKE_C_FLAGS=-funsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=OFF' USECP=FALSE CURHOST=docker-gcc-x86-unsigned
+        - DOCKIMG=unidata/nctests:serial32 USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc   AC_COPTS='CFLAGS=-funsigned-char --disable-dap-remote-tests' COPTS='-DCMAKE_C_FLAGS=-funsigned-char -DENABLE_DAP_REMOTE_TESTS=OFF -DENABLE_CDF5=OFF' USECP=FALSE CURHOST=docker-gcc-x86-unsigned TESTFILTER=TRUE
 
 # Centos, Fedora
-        - DOCKIMG=unidata/nctests:serial.centos USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc AC_COPTS='--disable-dap-remote-tests' COPTS='-DENABLE_DAP_REMOTE_TESTS=OFF' USECP=TRUE CURHOST=docker-gcc-x64-centos
-        - DOCKIMG=unidata/nctests:serial.fedora USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc AC_COPTS='--disable-dap-remote-tests' COPTS='-DENABLE_DAP_REMOTE_TESTS=OFF' USECP=TRUE CURHOST=docker-gcc-x64-fedora
+        - DOCKIMG=unidata/nctests:serial.centos USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc AC_COPTS='--disable-dap-remote-tests' COPTS='-DENABLE_DAP_REMOTE_TESTS=OFF' USECP=TRUE CURHOST=docker-gcc-x64-centos TESTFILTER=TRUE
+        - DOCKIMG=unidata/nctests:serial.fedora USECMAKE=TRUE USEAC=TRUE DISTCHECK=TRUE USE_CC=gcc AC_COPTS='--disable-dap-remote-tests' COPTS='-DENABLE_DAP_REMOTE_TESTS=OFF' USECP=TRUE CURHOST=docker-gcc-x64-fedora TESTFILTER=TRUE
 
 before_install:
     - docker pull $DOCKIMG > /dev/null
 
 script:
 
-    - docker run --rm -it -h "$CURHOST" -e USEDASH=FALSE -e RUNF=TRUE -e RUNCXX=OFF -e RUNP=OFF -e RUNNCO=OFF -e USECMAKE=$USECMAKE -e USEAC=$USEAC -e DISTCHECK=$DISTCHECK -e COPTS="$COPTS" -e AC_OPTS="$AC_OPTS" -e CTEST_OUTPUT_ON_FAILURE=1 -v $(pwd):/netcdf-c -e USE_LOCAL_CP=$USECP -e TESTPROC=100  $DOCKIMG
+    - docker run --rm -it -h "$CURHOST" -e USEDASH=FALSE -e RUNF=TRUE -e RUNCXX=OFF -e RUNP=OFF -e RUNNCO=OFF -e USECMAKE=$USECMAKE -e USEAC=$USEAC -e DISTCHECK=$DISTCHECK -e COPTS="$COPTS" -e AC_OPTS="$AC_OPTS" -e CTEST_OUTPUT_ON_FAILURE=1 -v $(pwd):/netcdf-c -e USE_LOCAL_CP=$USECP -e TESTPROC=100  -e ENABLE_FILTER_TESTING=$TESTFILTER $DOCKIMG

CMakeLists.txt

@@ -1996,8 +1996,8 @@ ENDIF()
 #####
 # Build ncdap_test|dap4_test/findtestserver[4].c
 #####
-configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_BINARY_DIR}/ncdap_test/findtestserver.c @ONLY NEWLINE_STYLE LF)
-configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_BINARY_DIR}/dap4_test/findtestserver4.c @ONLY NEWLINE_STYLE LF)
+configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c @ONLY NEWLINE_STYLE LF)
+configure_file(${CMAKE_SOURCE_DIR}/ncdap_test/findtestserver.c.in ${CMAKE_SOURCE_DIR}/dap4_test/findtestserver4.c @ONLY NEWLINE_STYLE LF)
 
 ####
 # Export files

cf

@@ -8,7 +8,7 @@ FAST=1
 
 HDF5=1
 DAP=1
-SZIP=1
+#SZIP=1
 #HDF4=1
 #PNETCDF=1
 #PAR4=1

dap4_test/CMakeLists.txt

@@ -9,6 +9,7 @@ ADD_SUBDIRECTORY(cdltestfiles)
 ADD_SUBDIRECTORY(daptestfiles)
 ADD_SUBDIRECTORY(dmrtestfiles)
 ADD_SUBDIRECTORY(nctestfiles)
+ADD_SUBDIRECTORY(misctestfiles)
 
 FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh)
 

dap4_test/Makefile.am

@@ -35,6 +35,7 @@ if BUILD_UTILITIES
 TESTS += test_raw.sh
 TESTS += test_meta.sh
 TESTS += test_data.sh
+TESTS += test_fillmismatch.sh
 endif
 
 # Note which tests depend on other tests. Necessary for make -j check.
@@ -56,9 +57,9 @@ endif
 endif #ENABLE_DAP4
 
 EXTRA_DIST = test_parse.sh test_meta.sh test_data.sh \
-             test_raw.sh test_remote.sh test_hyrax.sh \
+             test_raw.sh test_remote.sh test_hyrax.sh test_fillmismatch.sh \
              tst_curlopt.sh d4test_common.sh \
-	     daptestfiles dmrtestfiles cdltestfiles nctestfiles \
+	     daptestfiles dmrtestfiles cdltestfiles nctestfiles misctestfiles \
 	     baseline baselineraw baselineremote CMakeLists.txt
 
 CLEANFILES = *.exe

dap4_test/baselineraw/test_fillmismatch.nc.dmp

@@ -0,0 +1,10 @@
+netcdf test_fillmismatch {
+variables:
+	ubyte uv8 ;
+	short v16 ;
+	uint uv32 ;
+		uv32:_FillValue = 17U ;
+
+// global attributes:
+		:_DAP4_Little_Endian = 1UB ;
+}

dap4_test/d4test_common.sh

@@ -62,26 +62,17 @@ exit 0
 fi
 }
 
+# Return the complete set of arguments minus any found in $SUPPRESS
 suppress() {
-  F0="$1"
-  if test "x${SUPPRESS}" = x; then
-    RESULT="$F0"
-  else
-    RESULT=""
-    for f in ${F0} ; do
-      ignore=0
-      for s in ${SUPPRESS} ; do
-        if test "x$s" = "x$f" ; then
-  	  ignore=1;
-	  echo "Suppressing: $f"
-	  break;
-        fi       
-      done
-      if test "x$ignore" = x0 ; then RESULT="$f ${RESULT}" ; fi
-    done
-  fi
+  local args="$@"
+  suppress=
+  for a in $args; do
+  for s in $SUPPRESS; do if test "x$a" != "x$s" ; then suppress="$suppress $a"; fi;  done
+  done        
 }
 
 VG="valgrind --leak-check=full --error-exitcode=1 --num-callers=100"
 if test "x$USEVG" = x ; then VG=; fi
 
+DUMPFLAGS=
+

dap4_test/maketests.sh

@@ -40,7 +40,7 @@ if test "x$DEBUG" = x1 ; then
   F=`ls -1 *.dap | sed -e 's/[.]dap//' |tr '\r\n' '  '`
   popd
   for f in ${F} ; do
-    if ! diff -wBb ${SRC}/dmrtestfiles/${f}.dmr ./dmrtestfiles/${f}.dmr >& /dev/null ; then
+    if ! diff -wBb ${SRC}/dmrtestfiles/${f}.dmr ./dmrtestfiles/${f}.dmr > /dev/null 2>&1 ; then
       echo diff -wBb ${SRC}/dmrtestfiles/${f}.dmr ./dmrtestfiles/${f}.dmr
       diff -wBb ${SRC}/dmrtestfiles/${f}.dmr ./dmrtestfiles/${f}.dmr
     fi

dap4_test/misctestfiles/CMakeLists.txt

@@ -0,0 +1,6 @@
+FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*)
+FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE)
+
+FILE(GLOB CUR_EXTRA_DIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/*)
+SET(CUR_EXTRA_DIST ${CUR_EXTRA_DIST} CMakeLists.txt)
+ADD_EXTRA_DIST("${CUR_EXTRA_DIST}")

dap4_test/test_fillmismatch.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi
+. ../test_common.sh
+
+set -e
+. ${srcdir}/d4test_common.sh
+
+echo "test_fillmismatch.sh:"
+
+F="test_fillmismatch.nc"
+
+URL='[dap4]file://'
+URL="${URL}${srcdir}/misctestfiles/$F"
+
+# First check that without [fillmismatch], we get a failure
+rm -f ./tmp_dap4_mismatch
+if ${NCDUMP} -h "${URL}" > ./tmp_dap4_mismatch 2>&1 ; then
+echo "*** Fail: ${NCDUMP} ${URL} passed"
+exit 1
+else
+echo "*** XFail: ${NCDUMP} ${URL} failed"
+fi
+
+# Now check that with [fillmismatch], we get sucess
+URL="[fillmismatch]${URL}"
+rm -f ./tmp_dap4_mismatch
+if ${NCDUMP} -h "${URL}" > ./tmp_dap4_mismatch ; then
+echo "*** Pass: ${NCDUMP} ${URL} passed"
+else
+echo "*** Fail: ${NCDUMP} ${URL} failed"
+exit 1
+fi
+
+# Verify result
+diff -w ${srcdir}/baselineraw/$F.dmp ./tmp_dap4_mismatch
+#cleanup
+rm -f ./tmp_dap4_mismatch
+exit

dap4_test/test_meta.sh

@@ -7,6 +7,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi
 
 echo "test_meta.sh:"
 
+
 cd ${DMRTESTFILES}
 F=`ls -1 *.dmr | sed -e 's/[.]dmr//g' | tr '\r\n' '  '`
 cd $WD
@@ -30,7 +31,7 @@ for f in ${F} ; do
     if ! ${VG} ${execdir}/test_meta ${DMRTESTFILES}/${f}.dmr ./results_test_meta/${f} ; then
         failure "${execdir}/test_meta ${DMRTESTFILES}/${f}.dmr ./results_test_meta/${f}"
     fi
-    ${NCDUMP} -h ./results_test_meta/${f} > ./results_test_meta/${f}.d4m
+    ${NCDUMP} ${DUMPFLAGS} -h ./results_test_meta/${f} > ./results_test_meta/${f}.d4m
     if test "x${TEST}" = x1 ; then
 	if ! diff -wBb ${BASELINE}/${f}.d4m ./results_test_meta/${f}.d4m ; then
 	    failure "diff -wBb ${BASELINE}/${f}.ncdump ./results_test_meta/${f}.d4m"

dap4_test/test_remote.sh

@@ -73,7 +73,7 @@ for f in $F ; do
     if test "x$NOCSUM" = x1; then
 	URL="[ucar.checksummode=none]${URL}"
     fi
-    if ! ${NCDUMP} "${URL}" > ${builddir}/results_test_remote/${f}.dmp; then
+    if ! ${NCDUMP} ${DUMPFLAGS} "${URL}" > ${builddir}/results_test_remote/${f}.dmp; then
         failure "${URL}"
     fi
     if test "x${TEST}" = x1 ; then

dap4_test/test_test.sh

@@ -3,7 +3,7 @@
 if test "x$srcdir" = "x" ; then srcdir=`dirname $0`; fi; export srcdir
 
 rm -f log.log
-sh -x ${srcdir}/test_parse.sh >& ./parse.log >& log.log
+sh -x ${srcdir}/test_parse.sh > ./parse.log 2>&1
 cat log.log /dev/tty
 
 exit 0

dap4_test/tst_data.sh

@@ -15,7 +15,7 @@ for f in $F ; do
     if ! ${VG} ${execdir}/test_data ${DAPTESTFILES}/${f} ./results/${f}.nc ; then
         failure "${execdir}/test_data ${DAPTESTFILES}/${f} ./results/${f}.nc"
     fi
-    ${NCDUMP} ./results/${f}.nc > ./results/${f}.d4d
+    ${NCDUMP} ${DUMPFLAGS} ./results/${f}.nc > ./results/${f}.d4d
     if test "x${TEST}" = x1 ; then
 	if ! diff -wBb ${BASELINE}/${f}.d4d ./results/${f}.d4d ; then
 	    failure "diff -wBb ${BASELINE}/${f}.d4d ./results/${f}.d4d"

dap4_test/tst_meta.sh

@@ -26,7 +26,7 @@ for f in ${F} ; do
     if ! ${VG} ${execdir}/test_meta ${DMRTESTFILES}/${f}.dmr ./results/${f} ; then
         failure "${execdir}/test_meta ${DMRTESTFILES}/${f}.dmr ./results/${f}"
     fi
-    ${NCDUMP} -h ./results/${f} > ./results/${f}.d4m
+    ${NCDUMP} ${DUMPFLAGS} -h ./results/${f} > ./results/${f}.d4m
     if test "x${TEST}" = x1 ; then
 	if ! diff -wBb ${BASELINE}/${f}.d4m ./results/${f}.d4m ; then
 	    failure "diff -wBb ${BASELINE}/${f}.ncdump ./results/${f}.d4m"

dap4_test/tst_raw.sh

@@ -42,7 +42,7 @@ if test "x${RESET}" = x1 ; then rm -fr ${BASELINERAW}/*.dmp ; fi
 for f in $F ; do
     echo "testing: $f"
     URL="[dap4]file:${DAPTESTFILES}/${f}"
-    if ! ${VG} ${NCDUMP} "${URL}" > ./results/${f}.dmp; then
+    if ! ${VG} ${NCDUMP} ${DUMPFLAGS} "${URL}" > ./results/${f}.dmp; then
         failure "${URL}"
     fi
     if test "x${TEST}" = x1 ; then

docs/CMakeLists.txt

@@ -82,7 +82,7 @@ architecture.dox internal.dox windows-binaries.md
 building-with-cmake.md CMakeLists.txt groups.dox install.md notes.md
 install-fortran.md all-error-codes.md credits.md auth.md
 obsolete/fan_utils.html bestpractices.md filters.md indexing.md
-inmemory.md DAP4.dox OPeNDAP.dox attribute_conventions.md FAQ.md
+inmemory.md DAP2.dox attribute_conventions.md FAQ.md
 file_format_specifications.md known_problems.md
 COPYRIGHT.dox user_defined_formats.md)