Merge branch 'master' into ncpyfilter.dmh

CMakeLists.txt

@@ -1623,6 +1623,7 @@ ENDIF(USE_HDF5)
 
 IF(USE_HDF4)
   add_subdirectory(libhdf4)
+  add_subdirectory(hdf4_test)
 ENDIF(USE_HDF4)
 
 IF(ENABLE_DAP2)

Makefile.am

@@ -67,13 +67,14 @@ endif
 
 # Build HDF4 if desired.
 if USE_HDF4
+HDF4_TEST_DIR = hdf4_test
 LIBHDF4 = libhdf4
 endif
 
 # Define Test directories
 if BUILD_TESTSETS
-TESTDIRS = $(V2_TEST) nc_test $(NC_TEST4) $(NCDAP2TESTDIR)	\
-$(NCDAP4TESTDIR)
+TESTDIRS = $(V2_TEST) nc_test $(NC_TEST4) $(HDF4_TEST_DIR)	\
+$(NCDAP2TESTDIR) $(NCDAP4TESTDIR)
 endif
 
 # This is the list of subdirs for which Makefiles will be constructed

configure.ac

@@ -72,8 +72,8 @@ AC_CONFIG_LINKS([nc_test4/ref_hdf5_compat1.nc:nc_test4/ref_hdf5_compat1.nc])
 AC_CONFIG_LINKS([nc_test4/ref_hdf5_compat2.nc:nc_test4/ref_hdf5_compat2.nc])
 AC_CONFIG_LINKS([nc_test4/ref_hdf5_compat3.nc:nc_test4/ref_hdf5_compat3.nc])
 
-AC_CONFIG_LINKS([nc_test4/ref_chunked.hdf4:nc_test4/ref_chunked.hdf4])
-AC_CONFIG_LINKS([nc_test4/ref_contiguous.hdf4:nc_test4/ref_contiguous.hdf4])
+AC_CONFIG_LINKS([hdf4_test/ref_chunked.hdf4:hdf4_test/ref_chunked.hdf4])
+AC_CONFIG_LINKS([hdf4_test/ref_contiguous.hdf4:hdf4_test/ref_contiguous.hdf4])
 
 AM_INIT_AUTOMAKE([foreign dist-zip subdir-objects])
 
@@ -1430,6 +1430,7 @@ AC_CONFIG_FILES([Makefile
 		         include/netcdf_meta.h
                  include/Makefile
                  h5_test/Makefile
+                 hdf4_test/Makefile
                  libsrc/Makefile
                  libsrc4/Makefile
                  libsrcp/Makefile

hdf4_test/CMakeLists.txt

@@ -0,0 +1,16 @@
+# Copy some test files from current source dir to out-of-tree build dir.
+FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.hdf4)
+FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
+IF(MSVC)
+  FILE(COPY ${COPY_FILES} DESTINATION ${RUNTIME_OUTPUT_DIRECTORY}/)
+ENDIF()
+
+IF(USE_HDF4_FILE_TESTS AND NOT MSVC)
+  build_bin_test_no_prefix(tst_interops2)
+  build_bin_test_no_prefix(tst_interops3)
+  add_bin_test(hdf4_test tst_chunk_hdf4)
+  add_bin_test(hdf4_test tst_h4_lendian)
+  add_sh_test(hdf4_test run_get_hdf4_files)
+  add_sh_test(hdf4_test run_formatx_hdf4)
+ENDIF()
+

hdf4_test/Makefile.am

@@ -0,0 +1,37 @@
+# This is part of the netCDF package.
+# Copyright 2018 University Corporation for Atmospheric Research/Unidata
+# See COPYRIGHT file for conditions of use.
+#
+# This directory has tests for the HDF4 code. This directory will be
+# skipped if HDF4 is not enabled.
+#
+# Ed Hartnett
+
+# Put together AM_CPPFLAGS and AM_LDFLAGS.
+include $(top_srcdir)/lib_flags.am
+
+# Link to our assembled library.
+AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la
+
+# These are the C tests for HDF4.
+check_PROGRAMS = tst_chunk_hdf4 tst_h4_lendian
+TESTS = tst_chunk_hdf4 tst_h4_lendian
+
+# This test script depends on ncdump and tst_interops2.c.
+if BUILD_UTILITIES
+check_PROGRAMS += tst_interops2 
+TESTS += run_formatx_hdf4.sh
+endif # BUILD_UTILITIES
+
+# This test script fetches HDF4 files from an FTP server and uses
+# program tst_interops3.c to read them.
+if USE_HDF4_FILE_TESTS
+check_PROGRAMS += tst_interops3
+TESTS += run_get_hdf4_files.sh
+endif # USE_HDF4_FILE_TESTS
+
+EXTRA_DIST = CMakeLists.txt ref_contiguous.hdf4 ref_chunked.hdf4	\
+run_get_hdf4_files.sh run_formatx_hdf4.sh
+
+CLEANFILES = *.h4 *.hdf
+

hdf4_test/run_formatx_hdf4.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# This shell script runs tst_interops2 to create a HDF4 file, and read
+# it with netCDF. Then the script runs ncdump on the HDF4 file.
+
+# Ed Hartnett
+
+if test "x$srcdir" = x ; then srcdir=`pwd`; fi
+. ../test_common.sh
+
+FILE=tst_interops2.h4
+
+ECODE=0
+
+echo ""
+echo "*** Testing extended file format output."
+set -e
+
+echo "Creating HDF4 file"
+${execdir}/tst_interops2
+
+echo "Test extended format output for a HDF4 file"
+rm -f tmp_tst_formatx_hdf4
+${NCDUMP} -K $FILE >tmp_tst_formatx_hdf4
+if ! fgrep 'HDF4 mode=00001000' <tmp_tst_formatx_hdf4 ; then
+TMP=`cat tmp_tst_formatx_hdf4`
+echo "*** Fail: extended format for an HDF4 file: result=" $TMP
+ECODE=1
+fi
+
+rm -f tmp_tst_formatx_hdf4
+
+# Exit if there was a failure.
+if test $ECODE = 1 ; then
+    exit $ECODE
+fi
+
+echo ""
+echo "*** Testing reading an individual variable from an HDF4 file."
+
+${NCDUMP} -v hdf4_dataset_type_0 $FILE
+${NCDUMP} -v hdf4_dataset_type_1 $FILE
+${NCDUMP} -v hdf4_dataset_type_2 $FILE
+${NCDUMP} -v hdf4_dataset_type_3 $FILE
+${NCDUMP} -v hdf4_dataset_type_4 $FILE
+${NCDUMP} -v hdf4_dataset_type_5 $FILE
+${NCDUMP} -v hdf4_dataset_type_6 $FILE
+${NCDUMP} -v hdf4_dataset_type_7 $FILE
+
+echo "*** Success."
+

nc_test4/run_get_hdf4_files.sh → hdf4_test/run_get_hdf4_files.sh

@@ -21,7 +21,7 @@ echo "Getting HDF4 test files $file_list"
 for f1 in $file_list
 do
     if ! test -f $f1; then
-	curl -O "ftp://ftp.unidata.ucar.edu/pub/netcdf/sample_data/hdf4/$f1.gz"
+	wget "ftp://ftp.unidata.ucar.edu/pub/netcdf/sample_data/hdf4/$f1.gz"
 	gunzip $f1.gz
     fi
 done

nc_test4/tst_interops2.c → hdf4_test/tst_interops2.c

@@ -65,7 +65,7 @@ main(int argc, char **argv)
 
       /* Expected this to return NC_EPERM, but instead it returns
        * success. See github issue #744. */
-      /* if (nc_def_var_chunking_ints(ncid, 0, NC_CONTIGUOUS, NULL) != NC_EPERM) ERR; */
+      /* if (nc_def_var_chunking_ints(ncid, 0, NC_CONTIGUOUS, NULL)) ERR; */
 
       /* Read the data through a vara function from the netCDF API. */
       if (nc_get_vara(ncid, 0, nstart, ncount, data_in)) ERR;

nc_test4/tst_interops3.c → hdf4_test/tst_interops3.c

@@ -77,7 +77,6 @@ main(int argc, char **argv)
 							   "MOD29.A2000055.0005.005.2006267200024.hdf",
 							   "MYD29.A2002185.0000.005.2007160150627.hdf",
 							   "MYD29.A2009152.0000.005.2009153124331.hdf"};
-      size_t len_in;
       int expected_mode = NC_NETCDF4;
       int expected_extended_format = NC_FORMATX_NC_HDF4;
       int f;

libdap4/d4curlfunctions.c

@@ -200,12 +200,14 @@ set_curl_options(NCD4INFO* state)
     for(i=0;i<nclistlength(store);i++) {
         struct CURLFLAG* flag;
 	NCTriple* triple = (NCTriple*)nclistget(store,i);
-        size_t hostlen = strlen(triple->host);
+        size_t hostlen = (triple->host ? strlen(triple->host) : 0);
         const char* flagname;
         if(strncmp("CURL.",triple->key,5) != 0) continue; /* not a curl flag */
         /* do hostport prefix comparison */
         if(hostport != NULL) {
-          int t = strncmp(hostport,triple->host,hostlen);
+	  int t = 0;
+	  if(triple->host != NULL)
+              t = strncmp(hostport,triple->host,hostlen);
           if(t !=  0) continue;
         }
         flagname = triple->key+5; /* 5 == strlen("CURL."); */

libdispatch/drc.c

@@ -315,6 +315,8 @@ rccompile(const char* path)
 	    }
 	    ncbytesnull(tmp);
 	    triple->host = ncbytesextract(tmp);
+	    if(strlen(triple->host)==0)
+		{free(triple->host); triple->host = NULL;}
         }
         /* split off key and value */
         key=line;
@@ -331,7 +333,8 @@ rccompile(const char* path)
         rctrim(triple->value);
 #ifdef D4DEBUG
 	fprintf(stderr,"rc: host=%s key=%s value=%s\n",
-		triple->host,triple->key,triple->valu);
+		(triple->host != NULL ? triple->host : "<null>"),
+		triple->key,triple->valu);
 #endif
 	nclistpush(rc,triple);
 	triple = NULL;
@@ -371,7 +374,9 @@ rclocate(const char* key, const char* hostport)
            (because we have checked all other cases)*/
         if(hplen == 0) {found=1;break;}
         /* do hostport match */
-        t = strcmp(hostport,triple->host);
+	t = 0;
+	if(triple->host != NULL)
+            t = strcmp(hostport,triple->host);
         if(t ==  0) {found=1; break;}
     }
     return (found?triple:NULL);

nc_test4/CMakeLists.txt

@@ -72,7 +72,7 @@ IF(BUILD_BENCHMARKS)
 ENDIF()
 
 # Copy some test files from current source dir to out-of-tree build dir.
-FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/ref_bzip2.c ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.hdf4 ${CMAKE_CURRENT_SOURCE_DIR}/*.h5 ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl)
+FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/ref_bzip2.c ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.h5 ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl)
 FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/)
 IF(MSVC)
   FILE(COPY ${COPY_FILES} DESTINATION ${RUNTIME_OUTPUT_DIRECTORY}/)
@@ -83,20 +83,6 @@ FOREACH(CTEST ${NC4_TESTS})
   add_bin_test(nc_test4 ${CTEST})
 ENDFOREACH()
 
-# This must go after the 'foreach' stanza
-# immediately above this comment.
-IF(USE_HDF4_FILE_TESTS AND NOT MSVC)
-    add_bin_test_no_prefix(tst_interops2)
-    build_bin_test_no_prefix(tst_interops3)
-    add_sh_test(nc_test4 run_get_hdf4_files)
-    add_sh_test(nc_test4 tst_formatx_hdf4)
-    build_bin_test(tst_chunk_hdf4)
-    add_sh_test(nc_test4 run_chunk_hdf4)
-    add_bin_test(nc_test4 tst_h4_lendian)
-    add_sh_test(nc_test4 tst_hdf4_read_var)
-    SET_TESTS_PROPERTIES(nc_test4_tst_hdf4_read_var PROPERTIES DEPENDS tst_interops2)
-ENDIF()
-
 IF(TEST_PARALLEL4)
   build_bin_test(tst_mpi_parallel)
   build_bin_test(tst_parallel)

nc_test4/Makefile.am

@@ -95,26 +95,6 @@ benchmarks: check
 	./run_bm_ar4.sh
 endif # BUILD_BENCHMARKS
 
-# These are the tests for HDF4.
-if USE_HDF4
-check_PROGRAMS += tst_interops2 tst_chunk_hdf4 tst_h4_lendian
-
-if BUILD_UTILITIES
-# This test script depends on ncdump.
-TESTS += tst_interops2 tst_formatx_hdf4.sh
-tst_formatx_hdf4.log: tst_interops2.log
-endif # BUILD_UTILITIES
-
-TESTS += run_chunk_hdf4.sh tst_h4_lendian
-if USE_HDF4_FILE_TESTS
-check_PROGRAMS += tst_interops3
-TESTS += run_get_hdf4_files.sh tst_hdf4_read_var.sh
-
-tst_hdf4_read_var.log: tst_interops2.log
-
-endif # USE_HDF4_FILE_TESTS
-endif # USE_HDF4
-
 # Szip Tests (requires ncdump)
 if USE_SZIP
 if BUILD_UTILITIES
@@ -143,17 +123,15 @@ EXTRA_DIST = run_par_test.sh run_bm.sh run_bm_test1.sh			\
 run_bm_test2.sh run_bm_radar_2D.sh run_bm_radar_2D_compression1.sh	\
 run_par_bm_test.sh run_bm_elena.sh run_par_bm_radar_2D.sh		\
 run_bm_radar_2D_endianness1.sh run_tst_chunks.sh ref_chunks1.cdl	\
-ref_chunks2.cdl run_get_hdf4_files.sh run_bm_ar4.sh			\
-ref_tst_compounds.nc ref_tst_xplatform2_1.nc ref_tst_xplatform2_2.nc	\
-ref_tst_dims.nc ref_tst_interops4.nc run_get_knmi_files.sh		\
-CMakeLists.txt run_grp_rename.sh tst_formatx_hdf4.sh			\
-run_chunk_hdf4.sh tst_h5_endians.c tst_h4_lendian.c			\
-tst_atts_string_rewrite.c tst_put_vars_two_unlim_dim.c			\
-tst_empty_vlen_unlim.c run_empty_vlen_test.sh ref_hdf5_compat1.nc	\
-ref_hdf5_compat2.nc ref_hdf5_compat3.nc tst_misc.sh tdset.h5		\
-tst_hdf4_read_var.sh ref_contiguous.hdf4 ref_chunked.hdf4 tst_szip.sh	\
-ref_szip.h5 ref_szip.cdl tst_filter.sh bzip2.cdl filtered.cdl		\
-unfiltered.cdl ref_bzip2.c findplugin.in
+ref_chunks2.cdl run_bm_ar4.sh ref_tst_compounds.nc			\
+ref_tst_xplatform2_1.nc ref_tst_xplatform2_2.nc ref_tst_dims.nc		\
+ref_tst_interops4.nc run_get_knmi_files.sh CMakeLists.txt		\
+run_grp_rename.sh tst_h5_endians.c tst_atts_string_rewrite.c		\
+tst_put_vars_two_unlim_dim.c tst_empty_vlen_unlim.c			\
+run_empty_vlen_test.sh ref_hdf5_compat1.nc ref_hdf5_compat2.nc		\
+ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5	\
+ref_szip.cdl tst_filter.sh bzip2.cdl filtered.cdl unfiltered.cdl	\
+ref_bzip2.c findplugin.in
 
 CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc   \
 bm_radar.nc bm_radar1.nc radar_3d_compression_test.txt                  \
@@ -171,12 +149,4 @@ szip_dump.cdl
 
 DISTCLEANFILES = findplugin.sh
 
-if USE_HDF4_FILE_TESTS
-DISTCLEANFILES += AMSR_E_L2_Rain_V10_200905312326_A.hdf	\
-AMSR_E_L3_DailyLand_V06_20020619.hdf			\
-MYD29.A2009152.0000.005.2009153124331.hdf		\
-MYD29.A2002185.0000.005.2007160150627.hdf		\
-MOD29.A2000055.0005.005.2006267200024.hdf
-endif # HDF4_FILE_TESTS
-
 SUBDIRS=hdf5plugins