Merge branch 'main' into idint.dmh

CMakeLists.txt

@@ -1160,15 +1160,16 @@ set_std_filter(Szip)
 SET(HAVE_SZ ${Szip_FOUND})
 set_std_filter(Blosc)
 IF(Zstd_FOUND)
-set_std_filter(Zstd)
+ set_std_filter(Zstd)
+ SET(HAVE_ZSTD ON)
 ENDIF()
 IF(Bz2_FOUND)
  set_std_filter(Bz2)
 ELSE()
  # The reason we use a local version is to support a more comples test case
  MESSAGE("libbz2 not found using built-in version")
  SET(HAVE_LOCAL_BZ2 ON)
- SET(HAVE_BZ2 ON)
+ SET(HAVE_BZ2 ON CACHE BOOL "")
  set(STD_FILTERS "${STD_FILTERS} bz2")
 ENDIF()
 
@@ -1507,6 +1508,7 @@ ENDIF()
 
 # Enable Parallel IO with netCDF-4/HDF5 files using HDF5 parallel I/O.
 SET(STATUS_PARALLEL "OFF")
+set(IMPORT_MPI "")
 OPTION(ENABLE_PARALLEL4 "Build netCDF-4 with parallel IO" "${HDF5_PARALLEL}")
 IF(ENABLE_PARALLEL4 AND ENABLE_HDF5)
  IF(NOT HDF5_PARALLEL)
@@ -1528,6 +1530,7 @@ IF(ENABLE_PARALLEL4 AND ENABLE_HDF5)
  FILE(COPY "${netCDF_BINARY_DIR}/tmp/run_par_tests.sh"
  DESTINATION ${netCDF_BINARY_DIR}/h5_test
  FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
+ set(IMPORT_MPI "include(CMakeFindDependencyMacro)\nfind_dependency(MPI COMPONENTS C)")
  ENDIF()
 ENDIF()
 

RELEASE_NOTES.md

@@ -8,6 +8,10 @@ This file contains a high-level description of this package's evolution. Release
 ## 4.9.3 - TBD
 
 * Obviate a number of irrelevant warnings. See [Github #2781](https://github.com/Unidata/netcdf-c/pull/2781).
+* Improve the speed and data quantity for DAP4 queries. See [Github #2765](https://github.com/Unidata/netcdf-c/pull/2765).
+* Remove the use of execinfo to programmatically dump the stack; it never worked. See [Github #2789](https://github.com/Unidata/netcdf-c/pull/2789).
+* Update the internal copy of tinyxml2 to latest code. See [Github #2771](https://github.com/Unidata/netcdf-c/pull/2771).
+* Mitigate the problem of remote/nczarr-related test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755).
 * Fix DAP2 proxy problems. See [Github #2764](https://github.com/Unidata/netcdf-c/pull/2764).
 * Cleanup a number of misc issues. See [Github #2763](https://github.com/Unidata/netcdf-c/pull/2763).
 * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755).

config.h.cmake.in

@@ -118,6 +118,12 @@ are set when opening a binary file on Windows. */
 /* if true, build byte-range Client */
 #cmakedefine ENABLE_BYTERANGE 1
 
+/* if true, enable ERANGE fill */
+#cmakedefine ENABLE_ERANGE_FILL 1
+#ifdef ENABLE_ERANGE_FILL
+#define ERANGE_FILL 1
+#endif
+
 /* if true, use hdf5 S3 virtual file reader */
 #cmakedefine ENABLE_HDF5_ROS3 1
 
@@ -462,9 +468,6 @@ with zip */
 /* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */
 #cmakedefine HDF5_UTF8_PATHS 1
 
-/* if true, backtrace support will be used. */
-#cmakedefine HAVE_EXECINFO_H 1
-
 /* if true, include JNA bug fix */
 #cmakedefine JNA 1
 

configure.ac

@@ -857,129 +857,136 @@ AC_MSG_RESULT([${have_sz}])
 
 ##########
 
-# See if we have libzip for NCZarr
-AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll],[have_zip=yes],[have_zip=no])
-AC_MSG_CHECKING([whether libzip library is available])
-AC_MSG_RESULT([${have_zip}])
-enable_nczarr_zip=${have_zip} # alias
+## 
+# Check to see if we're using NCZarr. If not, we don't need to check for dependencies and such.
+##
 
 if test "x$enable_nczarr" = xno ; then
  enable_nczarr_zip=no
-fi
+else
 
-AC_MSG_CHECKING([whether nczarr zip support is enabled])
-AC_MSG_RESULT([${enable_nczarr_zip}])
+ # See if we have libzip for NCZarr
+ AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll],[have_zip=yes],[have_zip=no])
+ AC_MSG_CHECKING([whether libzip library is available])
+ AC_MSG_RESULT([${have_zip}])
+ enable_nczarr_zip=${have_zip} # alias
 
-if test "x$enable_nczarr_zip" = xyes ; then
-AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found])
-fi
 
-# Check for enabling of S3 support
-AC_MSG_CHECKING([whether netcdf S3 support should be enabled])
-AC_ARG_ENABLE([s3],
- [AS_HELP_STRING([--enable-s3],
- [enable netcdf S3 support])])
-test "x$enable_s3" = xyes || enable_s3=no
-AC_MSG_RESULT($enable_s3)
 
-if test "x$enable_remote_functionality" = xno ; then
- AC_MSG_WARN([--disable-remote-functionality => --disable-s3])
- enable_s3=no
-fi
-
-# --enable-nczarr-s3 is synonym for --enable-s3 (but...)
-AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled])
-AC_ARG_ENABLE([nczarr-s3],
- [AS_HELP_STRING([--enable-nczarr-s3],
- [(Deprecated) enable netcdf NCZarr S3 support; Deprecated in favor of --enable-s3])])
-AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)])
-
-# Set enable_s3 instead of enable_nczarr_s3
-if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes && test "x$enable_remote_functionality" = xyes; then
-enable_s3=yes # back compatibility
-fi
-unset enable_nczarr_s3
-
-# Note we check for the library after checking for enable_s3
-# because for some reason this fails if we unconditionally test for sdk
-# and it is not available. Fix someday
-S3LIBS=""
-if test "x$enable_s3" = xyes ; then
-# See if we have the s3 aws library
-# Check for the AWS S3 SDK library
- AC_LANG_PUSH([C++])
- AC_CHECK_LIB([aws-c-common], [aws_string_destroy], [enable_s3_aws=yes],[enable_s3_aws=no])
- if test "x$enable_s3_aws" = "xyes" ; then
- S3LIBS="-laws-cpp-sdk-core -laws-cpp-sdk-s3"
+ AC_MSG_CHECKING([whether nczarr zip support is enabled])
+ AC_MSG_RESULT([${enable_nczarr_zip}])
+
+ if test "x$enable_nczarr_zip" = xyes ; then
+ AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found])
  fi
- AC_LANG_POP
-else
- enable_s3_aws=no
-fi
 
-AC_MSG_CHECKING([whether AWS S3 SDK library is available])
-AC_MSG_RESULT([$enable_s3_aws])
+ # Check for enabling of S3 support
+ AC_MSG_CHECKING([whether netcdf S3 support should be enabled])
+ AC_ARG_ENABLE([s3],
+ [AS_HELP_STRING([--enable-s3],
+ [enable netcdf S3 support])])
+ test "x$enable_s3" = xyes || enable_s3=no
+ AC_MSG_RESULT($enable_s3)
+
+ if test "x$enable_remote_functionality" = xno ; then
+ AC_MSG_WARN([--disable-remote-functionality => --disable-s3])
+ enable_s3=no
+ fi
 
-# Check for enabling forced use of Internal S3 library
-AC_MSG_CHECKING([whether internal S3 support should be used])
-AC_ARG_ENABLE([s3-internal],
- [AS_HELP_STRING([--enable-s3-internal],
- [enable internal S3 support])])
-test "x$enable_s3_internal" = xyes || enable_s3_internal=no
-AC_MSG_RESULT($enable_s3_internal)
+ # --enable-nczarr-s3 is synonym for --enable-s3 (but...)
+ AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled])
+ AC_ARG_ENABLE([nczarr-s3],
+ [AS_HELP_STRING([--enable-nczarr-s3],
+ [(Deprecated) enable netcdf NCZarr S3 support; Deprecated in favor of --enable-s3])])
+ AC_MSG_RESULT([$enable_nczarr_s3 (Deprecated) Please use --enable-s3)])
 
-if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno ; then
-AC_MSG_WARN([No S3 library available => S3 support disabled])
-enable_s3=no
-fi
+ # Set enable_s3 instead of enable_nczarr_s3
+ if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes && test "x$enable_remote_functionality" = xyes; then
+ enable_s3=yes # back compatibility
+ fi
+ unset enable_nczarr_s3
+
+ # Note we check for the library after checking for enable_s3
+ # because for some reason this fails if we unconditionally test for sdk
+ # and it is not available. Fix someday
+ S3LIBS=""
+ if test "x$enable_s3" = xyes ; then
+ # See if we have the s3 aws library
+ # Check for the AWS S3 SDK library
+ AC_LANG_PUSH([C++])
+ AC_CHECK_LIB([aws-c-common], [aws_string_destroy], [enable_s3_aws=yes],[enable_s3_aws=no])
+ if test "x$enable_s3_aws" = "xyes" ; then
+ S3LIBS="-laws-cpp-sdk-core -laws-cpp-sdk-s3"
+ fi
+ AC_LANG_POP
+ else
+ enable_s3_aws=no
+ fi
 
-if test "x$enable_s3_aws" = xyes && test "x$enable_s3_internal" = xyes ; then
-AC_MSG_WARN([Both aws-sdk-cpp and s3-internal enabled => use s3-internal.])
-enable_s3_aws=no
-fi
+ AC_MSG_CHECKING([whether AWS S3 SDK library is available])
+ AC_MSG_RESULT([$enable_s3_aws])
 
-if test "x$enable_s3_internal" = xyes ; then
-if test "x$ISOSX" != xyes && test "x$ISMINGW" != xyes && test "x$ISMSVC" != xyes ; then
-# Find crypto libraries if using ssl
-AC_CHECK_LIB([ssl],[ssl_create_cipher_list])
-AC_CHECK_LIB([crypto],[SHA256])
-fi
-fi
+ # Check for enabling forced use of Internal S3 library
+ AC_MSG_CHECKING([whether internal S3 support should be used])
+ AC_ARG_ENABLE([s3-internal],
+ [AS_HELP_STRING([--enable-s3-internal],
+ [enable internal S3 support])])
+ test "x$enable_s3_internal" = xyes || enable_s3_internal=no
+ AC_MSG_RESULT($enable_s3_internal)
+
+ if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno ; then
+ AC_MSG_WARN([No S3 library available => S3 support disabled])
+ enable_s3=no
+ fi
 
-# Check for enabling S3 testing
-AC_MSG_CHECKING([what level of netcdf S3 testing should be enabled])
-AC_ARG_WITH([s3-testing],
- [AS_HELP_STRING([--with-s3-testing=yes|no|public],
- [control netcdf S3 testing])],
- [], [with_s3_testing=public])
-AC_MSG_RESULT($with_s3_testing)
+ if test "x$enable_s3_aws" = xyes && test "x$enable_s3_internal" = xyes ; then
+ AC_MSG_WARN([Both aws-sdk-cpp and s3-internal enabled => use s3-internal.])
+ enable_s3_aws=no
+ fi
 
-# Disable S3 tests if S3 support is disabled
-if test "x$enable_s3" = xno ; then
- if test "x$with_s3_testing" != xno ; then
-  AC_MSG_WARN([S3 support is disabled => no testing])
-  with_s3_testing=no
- fi
-fi
+ if test "x$enable_s3_internal" = xyes ; then
+ if test "x$ISOSX" != xyes && test "x$ISMINGW" != xyes && test "x$ISMSVC" != xyes ; then
+  # Find crypto libraries if using ssl
+ AC_CHECK_LIB([ssl],[ssl_create_cipher_list])
+ AC_CHECK_LIB([crypto],[SHA256])
+  fi
+ fi
 
-if test "x$enable_s3" = xyes ; then
- AC_DEFINE([ENABLE_S3], [1], [if true, build netcdf-c with S3 support enabled])
-fi
+ # Check for enabling S3 testing
+ AC_MSG_CHECKING([what level of netcdf S3 testing should be enabled])
+ AC_ARG_WITH([s3-testing],
+ [AS_HELP_STRING([--with-s3-testing=yes|no|public],
+ [control netcdf S3 testing])],
+ [], [with_s3_testing=public])
+ AC_MSG_RESULT($with_s3_testing)
+
+ # Disable S3 tests if S3 support is disabled
+ if test "x$enable_s3" = xno ; then
+ if test "x$with_s3_testing" != xno ; then
+ AC_MSG_WARN([S3 support is disabled => no testing])
+ with_s3_testing=no
+ fi
+ fi
 
-if test "x$enable_s3_aws" = xyes ; then
- LIBS="$LIBS$S3LIBS"
- AC_DEFINE([ENABLE_S3_AWS], [1], [If true, then use aws S3 library])
-fi
+ if test "x$enable_s3" = xyes ; then
+ AC_DEFINE([ENABLE_S3], [1], [if true, build netcdf-c with S3 support enabled])
+ fi
 
-if test "x$enable_s3_internal" = xyes ; then
- AC_DEFINE([ENABLE_S3_INTERNAL], [1], [If true, then use internal S3 library])
-fi
+ if test "x$enable_s3_aws" = xyes ; then
+ LIBS="$LIBS$S3LIBS"
+ AC_DEFINE([ENABLE_S3_AWS], [1], [If true, then use aws S3 library])
+ fi
 
-AC_DEFINE_UNQUOTED([WITH_S3_TESTING], [$with_s3_testing], [control S3 testing.])
+ if test "x$enable_s3_internal" = xyes ; then
+ AC_DEFINE([ENABLE_S3_INTERNAL], [1], [If true, then use internal S3 library])
+ fi
+
+ AC_DEFINE_UNQUOTED([WITH_S3_TESTING], [$with_s3_testing], [control S3 testing.])
 
-if test "x$with_s3_testing" = xyes ; then
- AC_MSG_WARN([*** DO NOT SPECIFY WITH_S3_TESTING=YES UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***])
- AC_DEFINE([ENABLE_S3_TESTALL], [yes], [control S3 testing.])
+ if test "x$with_s3_testing" = xyes ; then
+ AC_MSG_WARN([*** DO NOT SPECIFY WITH_S3_TESTING=YES UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***])
+ AC_DEFINE([ENABLE_S3_TESTALL], [yes], [control S3 testing.])
+ fi
 fi
 
 # Check whether we want to enable strict null byte header padding.
@@ -1305,9 +1312,6 @@ AC_CHECK_HEADERS([sys/resource.h])
 # See if we have ftw.h to walk directory trees
 AC_CHECK_HEADERS([ftw.h])
 
-# See if we can do stack tracing programmatically
-AC_CHECK_HEADERS([execinfo.h])
-
 # Check for these functions...
 AC_CHECK_FUNCS([strlcat snprintf strcasecmp fileno \
  strdup strtoll strtoull \

dap4_test/test_common.h

@@ -21,6 +21,7 @@ typedef int TDMR;
 static NCbytes* input = NULL;
 static NCbytes* output = NULL;
 static NCD4meta* metadata = NULL;
+static NCD4response* resp = NULL;
 static char* infile = NULL;
 static char* outfile = NULL;
 static int ncid = 0;
@@ -85,16 +86,21 @@ setup(int tdmr, int argc, char** argv)
  if(translatenc4)
  controller->controls.translation = NCD4_TRANSNC4;
  NCD4_applyclientfragmentcontrols(controller);
- if((metadata=NCD4_newmeta(controller))==NULL)
- fail(NC_ENOMEM);
- metadata->mode = mode;
- NCD4_attachraw(metadata, ncbyteslength(input),ncbytescontents(input));
 
- if((ret=NCD4_dechunk(metadata))) /* ok for mode == DMR or mode == DAP */
+ if((ret=NCD4_newMeta(controller,&metadata)))
+ fail(ret);
+
+ if((ret=NCD4_newResponse(controller,&resp)))
+ fail(ret);
+ resp->raw.size = ncbyteslength(input);
+ resp->raw.memory = ncbytescontents(input);
+ resp->mode = mode;
+
+ if((ret=NCD4_dechunk(resp))) /* ok for mode == DMR or mode == DAP */
  fail(ret);
 #ifdef DEBUG
  {
- int swap = (metadata->serial.hostbigendian != metadata->serial.remotebigendian);
+ int swap = (controller->platform.hostlittleendian != resp->remotelittleendian);
  void* d = metadata->serial.dap;
  size_t sz = metadata->serial.dapsize;
  fprintf(stderr,"====================\n");

dap4_test/test_meta.c

@@ -20,7 +20,7 @@ main(int argc, char** argv)
  fprintf(stderr,"t_dmrmeta %s -> %s\n",infile,outfile);
 #endif
 
- if((ret = NCD4_parse(metadata))) goto done;
+ if((ret = NCD4_parse(metadata,resp,0))) goto done;
  if((ret = NCD4_metabuild(metadata,ncid))) goto done;
 
 done:

dap4_test/test_parse.c

@@ -17,7 +17,7 @@ main(int argc, char** argv)
 
  setup(TDMR_PARSE,argc,argv);
 
- if((ret = NCD4_parse(metadata))) goto done;
+ if((ret = NCD4_parse(metadata,resp,0))) goto done;
  ret = NCD4_print(metadata,output);
  ncbytesnull(output);
  if(ret == NC_NOERR) {

docs/mainpage.dox

@@ -13,7 +13,7 @@ The NetCDF homepage may be found at <a href="https://www.unidata.ucar.edu/softwa
 
 You can find the documentation for netCDF-Fortran here:
 
-- <a href="https://www.unidata.ucar.edu/software/netcdf/fortran/docs"> The NetCDF-Fortran Developer's Guide</a>
+- <a href="https://docs.unidata.ucar.edu/netcdf-fortran/current/"> The NetCDF-Fortran Developer's Guide</a>
 
 \section this_release Learn more about the current NetCDF-C Release
 

examples/C/format.c

@@ -45,7 +45,7 @@ main()
  /* Create a bunch of phoney data so we have something to write in
  the example file. */
  for (fp=(float *)temp, i=0; i<LAT_LEN*LON_LEN; i++)
- *fp++ = 10. + i/10.;
+ *fp++ = 10.f + (float)i/10.f;
 
  /* Now create the file in both formats with the same code. */
  for (i=0; i<2; i++)