Improve the building of NCZARR S3 support in CMake and Autoconf

CMakeLists.txt

@@ -566,6 +566,9 @@ IF(ENABLE_NETCDF_4 AND NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC)
  SET(ENABLE_SET_LOG_LEVEL ON)
 ENDIF()
 
+# This has multiversion capability
+SET(HAS_MULTIFILTERS yes CACHE BOOL "")
+
 # Option to allow for strict null file padding.
 # See https://github.com/Unidata/netcdf-c/issues/657 for more information
 OPTION(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING "Enable strict null byte header padding." OFF)
@@ -779,7 +782,7 @@ IF(USE_HDF5)
  SET(ENABLE_NCDUMPCHUNKS ON)
  ENDIF()
 
- # Check to see if this is hdf5-1.10.6 or later.
+ # Record if ROS3 Driver is available
  IF(HAS_HDF5_ROS3)
  SET(ENABLE_HDF5_ROS3 ON)
  ENDIF()
@@ -821,6 +824,19 @@ IF(USE_HDF5)
  #option to include HDF5 High Level header file (hdf5_hl.h) in case we are not doing a make install
  INCLUDE_DIRECTORIES(${HDF5_HL_INCLUDE_DIR})
 
+ # Check to see if we have libcurl 7.66 or later
+ CHECK_C_SOURCE_COMPILES("
+ #include <H5public.h>
+ int main() {
+#if (H5_VERS_MAJOR*10000 + H5_VERS_MINOR*100 + H5_VERS_RELEASE < 11006)
+ choke me;
+#endif
+ }" HDF5_VERSION_1106)
+
+ IF(HDF5_VERSION_1106)
+ SET(HDF5_UTF8_PATHS ON)
+ ENDIF()
+
 ENDIF(USE_HDF5)
 
 # See if we have libcurl
@@ -962,24 +978,36 @@ IF(NOT WIN32)
 ENDIF()
 
 # Options to Extend NCZarr support
-OPTION(ENABLE_S3_SDK "Enable Search for AWS libraries." OFF)
-OPTION(ENABLE_S3_TESTS "Enable NCZarr S3 tests." OFF)
+OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support." OFF)
+OPTION(ENABLE_NCZARR_S3_TESTS "Enable NCZarr S3 tests." OFF)
 
-IF(NOT ENABLE_S3_SDK)
- IF(ENABLE_S3_TESTS)
- message(WARNING "DISABLE_S3_SDK => DISABLE_S3_TESTS")
- SET(ENABLE_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
- ENDIF()
+IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_NCZARR_S3)
+ message(FATAL_ERROR "NCZarr S3 support is disabled; please specify option -DENABLE_NCZARR_S3_TESTS=no")
+ SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCARR S3 TESTS" FORCE) 
 ENDIF()
 
-IF(ENABLE_S3_SDK)
 # See if aws-s3-sdk is available
+# But only if enabled
+IF(ENABLE_NCZARR_S3)
 find_package(AWSSDK REQUIRED)
 set(SERVICE s3)
 AWSSDK_DETERMINE_LIBS_TO_LINK(SERVICE AWSSDK_LINK_LIBRARIES)
-IF(NOT AWSSDK_FOUND)
+IF(AWSSDK_FOUND)
+ SET(ENABLE_S3_SDK ON CACHE BOOL "S3 SDK" FORCE) 
+ELSE()
  SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE)
 ENDIF()
+ELSE(ENABLE_NCZARR_S3)
+# Unconditionally disable
+SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE)
+ENDIF(ENABLE_NCZARR_S3)
+
+IF(NOT ENABLE_S3_SDK)
+ IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS)
+ message(FATAL_ERROR "AWS S3 libraries not found; please specify options DENABLE_NCZARR_S3=NO AND -DENABLE-NCZARR-S3-TESTS=NO")
+ SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "NCZARR S3 support" FORCE)
+ SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE)
+ ENDIF()
 ENDIF()
 
 ##
@@ -2076,7 +2104,7 @@ is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL)
 is_enabled(HAVE_H5Z_SZIP HAS_SZLIB)
 is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS)
 is_enabled(ENABLE_NCZARR HAS_NCZARR)
-is_enabled(ENABLE_S3_TESTS DO_S3_TESTS)
+is_enabled(ENABLE_NCZARR_S3_TESTS DO_S3_TESTS)
 
 # Generate file from template.
 CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
@@ -2120,6 +2148,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY
 # Build nc_test4/findplugin.sh
 #####
 SET(ISCMAKE "1")
+IF(MSVC)
+SET(ISMSVC "1")
+ENDIF()
+
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nc_test4/findplugin.sh @ONLY NEWLINE_STYLE LF)
 
 IF(ENABLE_EXAMPLES)

RELEASE_NOTES.md

@@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.8.0 - TBD
 
+* [Bug Fix] Cleanup the NCZarr S3 build options. See [Github #1869](https://github.com/Unidata/netcdf-c/pull/1869) for more information.
 * [Bug Fix] Support aligned access for selected ARM processors. See [Github #1871](https://github.com/Unidata/netcdf-c/pull/1871) for more information.
 * [Bug Fix] Fix time zone parser bug. See more information at [Github #1866](https://github.com/Unidata/netcdf-c/pull/1866) for more information.
 * [Documentation] Migrated the documents in the NUG/ directory to the dedicated NUG repository found at https://github.com/Unidata/netcdf

config.h.cmake.in

@@ -149,10 +149,13 @@ are set when opening a binary file on Windows. */
 #cmakedefine ENABLE_NCZARR 1
 
 /* if true, enable S3 support */
-#cmakedefine ENABLE_S3_SDK 1
+#cmakedefine ENABLE_NCZARR_S3 1
 
 /* if true, enable S3 testing*/
-#cmakedefine ENABLE_S3_TESTS 1
+#cmakedefine ENABLE_NCZARR_S3_TESTS 1
+
+/* if true, S3 SDK is available */
+#cmakedefine ENABLE_S3_SDK 1
 
 /* define the possible sources for remote test servers */
 #cmakedefine REMOTETESTSERVERS "${REMOTETESTSERVERS}"
@@ -413,6 +416,9 @@ are set when opening a binary file on Windows. */
 with zip */
 #cmakedefine HDF5_SUPPORTS_PAR_FILTERS 1
 
+/* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */
+#cmakedefine HDF5_UTF8_PATHS 1
+
 /* if true, include JNA bug fix */
 #cmakedefine JNA 1
 

configure.ac

@@ -572,42 +572,64 @@ if test "x$enable_nczarr" = xyes; then
 fi
 AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes])
 
-# Check for enabling S3 support
+# Check for enabling of S3 support
 AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled])
-AC_ARG_ENABLE([s3-sdk],
- [AS_HELP_STRING([--enable-s3-sdk],
+AC_ARG_ENABLE([nczarr-s3],
+ [AS_HELP_STRING([--enable-nczarr-s3],
  [enable netcdf zarr S3 support; make sure to set LDFLAGS])])
-test "x$enable_s3_sdk" = xyes || enable_s3_sdk=no
-have_aws=${enable_s3_sdk}
-AC_MSG_RESULT($enable_s3_sdk)
-if test "x$have_aws" = xyes ; then
-AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
-fi
-AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes])
+test "x$enable_nczarr_s3" = xyes || enable_nczarr_s3=no
+AC_MSG_RESULT($enable_nczarr_s3)
 
 # Check for enabling S3 testing
 AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled])
-AC_ARG_ENABLE([s3-tests],
- [AS_HELP_STRING([--enable-s3-tests],
+AC_ARG_ENABLE([nczarr-s3-tests],
+ [AS_HELP_STRING([--enable-nczarr-s3-tests],
  [enable netcdf zarr S3 testing])])
-test "x$enable_s3_tests" = xyes || enable_s3_tests=no
-AC_MSG_RESULT($enable_s3_tests)
+test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no
+AC_MSG_RESULT($enable_nczarr_s3_tests)
+
+# Disable S3 tests if S3 support is disabled
+if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then
+ AC_MSG_ERROR([NCZarr S3 support is disabled; please specify option --disable-nczarr-s3-tests])
+ enable_nczarr_s3_tests=no
+fi
+
+# See if we have the s3 aws library
+# Check for the AWS S3 SDK library
+AC_LANG_PUSH([C++])
+AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no])
+AC_LANG_POP
+AC_MSG_CHECKING([whether AWS S3 SDK library is available])
+AC_MSG_RESULT([$have_aws])
+
+if test "x$have_aws" = xyes ; then
+AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found])
+fi
+AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes])
 
 if test "x$have_aws" = xno ; then
- AC_MSG_WARN([aws libraries not found, so default to --disable-s3-tests])
- enable_s3_tests=no
+ if test "x$enable_nczarr_s3" = xyes || test "x$enable_nczarr_s3_tests" = xyes ; then
+ AC_MSG_ERROR([AWS S3 libraries not found; please specify options --disable-nczarr-s3 and --disable-nczarr-s3-tests])
+ enable_nczarr_s3_tests=no
+ enable_nczarr_s3=no
+ fi
 fi
 
-if test "x$enable_s3_tests" = xyes ; then
- AC_DEFINE([ENABLE_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled])
+if test "x$enable_nczarr_s3" = xyes ; then
+ AC_DEFINE([ENABLE_NCZARR_S3], [1], [if true, build libnczarr with S3 support enabled])
 fi
-AM_CONDITIONAL(ENABLE_S3_TESTS, [test "x$enable_s3_tests" = xyes])
+AM_CONDITIONAL(ENABLE_NCZARR_S3, [test "x$enable_nczarr_s3" = xyes])
+
+if test "x$enable_nczarr_s3_tests" = xyes ; then
+ AC_DEFINE([ENABLE_NCZARR_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled])
+fi
+AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes])
 
 # This has multiversion capability
 AC_MSG_CHECKING([whether multi-filter support is enabled])
-have_multifilters=yes
-AC_MSG_RESULT($have_multifilters)
-AC_DEFINE([HAVE_MULTIFILTERS], [1], [if true, multi-filter support enabled])
+has_multifilters=yes
+AC_MSG_RESULT($has_multifilters)
+AC_DEFINE([HAS_MULTIFILTERS], [1], [if true, multi-filter support enabled])
 
 # Check whether we want to enable strict null byte header padding.
 # See https://github.com/Unidata/netcdf-c/issues/657 for more information.
@@ -1020,18 +1042,23 @@ AC_TYPE_UINTPTR_T
 AC_C_CHAR_UNSIGNED
 AC_C_BIGENDIAN
 
+# Figure out platforms of special interest
+case "`uname`" in
+ CYGWIN*) ISCYGWIN=yes;;
+ WIN*) ISMSVC=yes;;
+esac
+AM_CONDITIONAL(ISCYGWIN, [test "x$ISCYGWIN" = xyes])
+AM_CONDITIONAL(ISMSVC, [test "x$ISMSVC" = xyes])
+
 ###
 # Crude hack to work around an issue
 # in Cygwin.
 ###
 SLEEPCMD=""
-PLTFORMOUT="$(uname | cut -d '_' -f 1)"
-if test "$PLTFORMOUT" = "CYGWIN"; then
- ISCYGWIN=yes
+if test "x$ISCYGWIN" = "xyes"; then
  SLEEPCMD="sleep 5"
  AC_MSG_NOTICE([Pausing between sizeof() checks to mitigate a Cygwin issue.])
 fi
-AM_CONDITIONAL(ISCYGWIN, [test "x$ISCYGWIN" = xyes])
 
 $SLEEPCMD
 AC_CHECK_SIZEOF(short)
@@ -1200,7 +1227,20 @@ if test "x$enable_hdf5" = xyes; then
  fi
  AC_MSG_RESULT([$enable_szlib])
 
+ # Check to see if HDF5 library is 1.10.6 or greater.
+ # Used to control path name conversion
+ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <H5public.h>]], [[
+#if (H5_VERS_MAJOR*10000 + H5_VERS_MINOR*100 + H5_VERS_RELEASE < 11006)
+ choke me
+#endif
+ ]])], [hdf5_version_1106=yes], [hdf5_version_1106=no])
+ AC_MSG_CHECKING([whether HDF5 library is version 1.10.6 or later])
+ AC_MSG_RESULT([$hdf5_version_1106])
+ if test "x$hdf5_version_1106" = xyes; then
+ AC_DEFINE([HDF5_UTF8_PATHS], [1], [if true, HDF5 paths can be utf-8])
+ fi
 fi
+
 AM_CONDITIONAL(ENABLE_NCDUMPCHUNKS, [test "x$has_readchunks" = xyes ])
 
 # If the user wants hdf4 built in, check it out.
@@ -1406,7 +1446,7 @@ AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes])
 AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ])
 AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes])
 AM_CONDITIONAL(HAVE_AWS, [test "x$have_aws" = xyes])
-AM_CONDITIONAL(HAVE_MULTIFILTERS, [test "x$have_multifilters" = xyes])
+AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes])
 
 # If the machine doesn't have a long long, and we want netCDF-4, then
 # we've got problems!
@@ -1529,8 +1569,8 @@ AC_SUBST(HAS_BYTERANGE,[$enable_byterange])
 AC_SUBST(RELAX_COORD_BOUND,[yes])
 AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters])
 AC_SUBST(HAS_NCZARR,[$enable_nczarr])
-AC_SUBST(DO_S3_TESTS,[$enable_s3_tests])
-AC_SUBST(HAS_MULTIFILTERS,[$have_multifilters])
+AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests])
+AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters])
 
 # Include some specifics for netcdf on windows.
 #AH_VERBATIM([_WIN32_STRICMP],
@@ -1602,7 +1642,7 @@ AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes])
 AX_SET_META([NC_HAS_PAR_FILTERS], [$hdf5_supports_par_filters],[yes])
 AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes])
 AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes])
-AX_SET_META([NC_HAS_MULTIFILTERS],[$have_multifilters],[yes])
+AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes])
 
 # This is the version of the dispatch table. If the dispatch table is
 # changed, this should be incremented, so that user-defined format
@@ -1615,11 +1655,13 @@ AC_SUBST([NC_DISPATCH_VERSION], [2])
 # End netcdf_meta.h definitions.
 #####
 
+# This would be true for a visual studio build.
+AC_SUBST([ISMSVC], [${ISMSVC}])
 # This would be true for a cmake build.
 AC_SUBST([ISCMAKE], [])
 
-# This would be true for a visual studio build.
-AC_SUBST([MSVC], [])
+# Provide conditional to temporarily suppress tests and such
+AM_CONDITIONAL([AX_IGNORE], [test xno = xyes])
 
 AC_MSG_NOTICE([generating header files and makefiles])
 AC_CONFIG_FILES(test_common.sh:test_common.in)

libhdf5/hdf5internal.c

@@ -1041,12 +1041,11 @@ nc4_ndf5_ansi_to_utf8(pathbuf_t *pb, const char *path)
  int n;
 
  if (hdf5_encoding == UNDEF) {
- uint majnum, minnum, relnum;
- H5get_libversion(&majnum, &minnum, &relnum);
- hdf5_encoding = (((majnum == UTF8_MAJNUM && minnum == UTF8_MINNUM && relnum >= UTF8_RELNUM)
- || (majnum == UTF8_MAJNUM && minnum > UTF8_MINNUM)
- || majnum > UTF8_MAJNUM)
- ? UTF8 : ANSI);
+#ifdef HDF5_UTF8_PATHS
+ hdf5_encoding = UTF8;
+#else
+ hdf5_encoding = ANSI;
+#endif
  }
  if (hdf5_encoding == ANSI) {
  pb->ptr = NULL;

nc-config.cmake.in

@@ -101,6 +101,13 @@ else
  has_parallel="yes"
 fi
 
+has_nczarr="@ENABLE_NCZARR@"
+if [ -z "$has_nczarr" -o "$has_nczarr" = "OFF" -o "$has_nczarr" = "FALSE" ]; then
+ has_nczarr="no"
+else
+ has_nczarr="yes"
+fi
+
 version="@PACKAGE@ @VERSION@"
 
 
@@ -164,6 +171,7 @@ Available values for OPTION include:
  --has-cdf5 whether cdf5 support is included in build
  --has-parallel4 whether has parallel IO support via HDF5
  --has-parallel whether has parallel IO support via HDF5 or PnetCDF
+ --has-nczarr whether nczarr enabled
  --libs library linking information for netcdf
  --static library linking information for statically-compiled netcdf
  --prefix Install prefix
@@ -245,6 +253,7 @@ fi
  echo " --has-cdf5 -> $has_cdf5"
  echo " --has-parallel4 -> $has_parallel4"
  echo " --has-parallel -> $has_parallel"
+ echo " --has-nczarr -> $has_nczarr"
  echo
  echo " --prefix -> $prefix"
  echo " --includedir -> $includedir"
@@ -346,6 +355,10 @@ while test $# -gt 0; do
  echo $has_parallel
  ;;
 
+ --has-nczarr)
+ echo $has_nczarr
+ ;;
+
  --libs)
  if [ "x$HAS_STATIC" = "xTRUE" ]; then
  echo $libs $libsprivate