diff --git a/CMakeLists.txt b/CMakeLists.txt index ee0953c8b4..67053aeeda 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -566,6 +566,9 @@ IF(ENABLE_NETCDF_4 AND NOT ENABLE_LOGGING AND ENABLE_SET_LOG_LEVEL_FUNC) SET(ENABLE_SET_LOG_LEVEL ON) ENDIF() +# This has multiversion capability +SET(HAS_MULTIFILTERS yes CACHE BOOL "") + # Option to allow for strict null file padding. # See https://github.com/Unidata/netcdf-c/issues/657 for more information OPTION(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING "Enable strict null byte header padding." OFF) @@ -779,7 +782,7 @@ IF(USE_HDF5) SET(ENABLE_NCDUMPCHUNKS ON) ENDIF() - # Check to see if this is hdf5-1.10.6 or later. + # Record if ROS3 Driver is available IF(HAS_HDF5_ROS3) SET(ENABLE_HDF5_ROS3 ON) ENDIF() @@ -821,6 +824,19 @@ IF(USE_HDF5) #option to include HDF5 High Level header file (hdf5_hl.h) in case we are not doing a make install INCLUDE_DIRECTORIES(${HDF5_HL_INCLUDE_DIR}) + # Check to see if we have libcurl 7.66 or later + CHECK_C_SOURCE_COMPILES(" + #include + int main() { +#if (H5_VERS_MAJOR*10000 + H5_VERS_MINOR*100 + H5_VERS_RELEASE < 11006) + choke me; +#endif + }" HDF5_VERSION_1106) + + IF(HDF5_VERSION_1106) + SET(HDF5_UTF8_PATHS ON) + ENDIF() + ENDIF(USE_HDF5) # See if we have libcurl @@ -962,24 +978,36 @@ IF(NOT WIN32) ENDIF() # Options to Extend NCZarr support -OPTION(ENABLE_S3_SDK "Enable Search for AWS libraries." OFF) -OPTION(ENABLE_S3_TESTS "Enable NCZarr S3 tests." OFF) +OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support." OFF) +OPTION(ENABLE_NCZARR_S3_TESTS "Enable NCZarr S3 tests." OFF) -IF(NOT ENABLE_S3_SDK) - IF(ENABLE_S3_TESTS) - message(WARNING "DISABLE_S3_SDK => DISABLE_S3_TESTS") - SET(ENABLE_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE) - ENDIF() +IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_NCZARR_S3) + message(FATAL_ERROR "NCZarr S3 support is disabled; please specify option -DENABLE_NCZARR_S3_TESTS=no") + SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCARR S3 TESTS" FORCE) ENDIF() -IF(ENABLE_S3_SDK) # See if aws-s3-sdk is available +# But only if enabled +IF(ENABLE_NCZARR_S3) find_package(AWSSDK REQUIRED) set(SERVICE s3) AWSSDK_DETERMINE_LIBS_TO_LINK(SERVICE AWSSDK_LINK_LIBRARIES) -IF(NOT AWSSDK_FOUND) +IF(AWSSDK_FOUND) + SET(ENABLE_S3_SDK ON CACHE BOOL "S3 SDK" FORCE) +ELSE() SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE) ENDIF() +ELSE(ENABLE_NCZARR_S3) +# Unconditionally disable +SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE) +ENDIF(ENABLE_NCZARR_S3) + +IF(NOT ENABLE_S3_SDK) + IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS) + message(FATAL_ERROR "AWS S3 libraries not found; please specify options DENABLE_NCZARR_S3=NO AND -DENABLE-NCZARR-S3-TESTS=NO") + SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "NCZARR S3 support" FORCE) + SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE) + ENDIF() ENDIF() ## @@ -2076,7 +2104,7 @@ is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL) is_enabled(HAVE_H5Z_SZIP HAS_SZLIB) is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS) is_enabled(ENABLE_NCZARR HAS_NCZARR) -is_enabled(ENABLE_S3_TESTS DO_S3_TESTS) +is_enabled(ENABLE_NCZARR_S3_TESTS DO_S3_TESTS) # Generate file from template. CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in" @@ -2120,6 +2148,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/test_common.in ${CMAKE_CURRENT_BINARY # Build nc_test4/findplugin.sh ##### SET(ISCMAKE "1") +IF(MSVC) +SET(ISMSVC "1") +ENDIF() + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nc_test4/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/nc_test4/findplugin.sh @ONLY NEWLINE_STYLE LF) IF(ENABLE_EXAMPLES) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 9bd2294400..cb6e42a00b 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.0 - TBD +* [Bug Fix] Cleanup the NCZarr S3 build options. See [Github #1869](https://github.com/Unidata/netcdf-c/pull/1869) for more information. * [Bug Fix] Support aligned access for selected ARM processors. See [Github #1871](https://github.com/Unidata/netcdf-c/pull/1871) for more information. * [Bug Fix] Fix time zone parser bug. See more information at [Github #1866](https://github.com/Unidata/netcdf-c/pull/1866) for more information. * [Documentation] Migrated the documents in the NUG/ directory to the dedicated NUG repository found at https://github.com/Unidata/netcdf diff --git a/config.h.cmake.in b/config.h.cmake.in index 59ded052fc..5165d7fcfc 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -149,10 +149,13 @@ are set when opening a binary file on Windows. */ #cmakedefine ENABLE_NCZARR 1 /* if true, enable S3 support */ -#cmakedefine ENABLE_S3_SDK 1 +#cmakedefine ENABLE_NCZARR_S3 1 /* if true, enable S3 testing*/ -#cmakedefine ENABLE_S3_TESTS 1 +#cmakedefine ENABLE_NCZARR_S3_TESTS 1 + +/* if true, S3 SDK is available */ +#cmakedefine ENABLE_S3_SDK 1 /* define the possible sources for remote test servers */ #cmakedefine REMOTETESTSERVERS "${REMOTETESTSERVERS}" @@ -413,6 +416,9 @@ are set when opening a binary file on Windows. */ with zip */ #cmakedefine HDF5_SUPPORTS_PAR_FILTERS 1 +/* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */ +#cmakedefine HDF5_UTF8_PATHS 1 + /* if true, include JNA bug fix */ #cmakedefine JNA 1 diff --git a/configure.ac b/configure.ac index 7051891951..2e166be5b0 100644 --- a/configure.ac +++ b/configure.ac @@ -572,42 +572,64 @@ if test "x$enable_nczarr" = xyes; then fi AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes]) -# Check for enabling S3 support +# Check for enabling of S3 support AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled]) -AC_ARG_ENABLE([s3-sdk], - [AS_HELP_STRING([--enable-s3-sdk], +AC_ARG_ENABLE([nczarr-s3], + [AS_HELP_STRING([--enable-nczarr-s3], [enable netcdf zarr S3 support; make sure to set LDFLAGS])]) -test "x$enable_s3_sdk" = xyes || enable_s3_sdk=no -have_aws=${enable_s3_sdk} -AC_MSG_RESULT($enable_s3_sdk) -if test "x$have_aws" = xyes ; then -AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found]) -fi -AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes]) +test "x$enable_nczarr_s3" = xyes || enable_nczarr_s3=no +AC_MSG_RESULT($enable_nczarr_s3) # Check for enabling S3 testing AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled]) -AC_ARG_ENABLE([s3-tests], - [AS_HELP_STRING([--enable-s3-tests], +AC_ARG_ENABLE([nczarr-s3-tests], + [AS_HELP_STRING([--enable-nczarr-s3-tests], [enable netcdf zarr S3 testing])]) -test "x$enable_s3_tests" = xyes || enable_s3_tests=no -AC_MSG_RESULT($enable_s3_tests) +test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no +AC_MSG_RESULT($enable_nczarr_s3_tests) + +# Disable S3 tests if S3 support is disabled +if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then + AC_MSG_ERROR([NCZarr S3 support is disabled; please specify option --disable-nczarr-s3-tests]) + enable_nczarr_s3_tests=no +fi + +# See if we have the s3 aws library +# Check for the AWS S3 SDK library +AC_LANG_PUSH([C++]) +AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [have_aws=yes],[have_aws=no]) +AC_LANG_POP +AC_MSG_CHECKING([whether AWS S3 SDK library is available]) +AC_MSG_RESULT([$have_aws]) + +if test "x$have_aws" = xyes ; then +AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found]) +fi +AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$have_aws" = xyes]) if test "x$have_aws" = xno ; then - AC_MSG_WARN([aws libraries not found, so default to --disable-s3-tests]) - enable_s3_tests=no + if test "x$enable_nczarr_s3" = xyes || test "x$enable_nczarr_s3_tests" = xyes ; then + AC_MSG_ERROR([AWS S3 libraries not found; please specify options --disable-nczarr-s3 and --disable-nczarr-s3-tests]) + enable_nczarr_s3_tests=no + enable_nczarr_s3=no + fi fi -if test "x$enable_s3_tests" = xyes ; then - AC_DEFINE([ENABLE_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled]) +if test "x$enable_nczarr_s3" = xyes ; then + AC_DEFINE([ENABLE_NCZARR_S3], [1], [if true, build libnczarr with S3 support enabled]) fi -AM_CONDITIONAL(ENABLE_S3_TESTS, [test "x$enable_s3_tests" = xyes]) +AM_CONDITIONAL(ENABLE_NCZARR_S3, [test "x$enable_nczarr_s3" = xyes]) + +if test "x$enable_nczarr_s3_tests" = xyes ; then + AC_DEFINE([ENABLE_NCZARR_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled]) +fi +AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes]) # This has multiversion capability AC_MSG_CHECKING([whether multi-filter support is enabled]) -have_multifilters=yes -AC_MSG_RESULT($have_multifilters) -AC_DEFINE([HAVE_MULTIFILTERS], [1], [if true, multi-filter support enabled]) +has_multifilters=yes +AC_MSG_RESULT($has_multifilters) +AC_DEFINE([HAS_MULTIFILTERS], [1], [if true, multi-filter support enabled]) # Check whether we want to enable strict null byte header padding. # See https://github.com/Unidata/netcdf-c/issues/657 for more information. @@ -1020,18 +1042,23 @@ AC_TYPE_UINTPTR_T AC_C_CHAR_UNSIGNED AC_C_BIGENDIAN +# Figure out platforms of special interest +case "`uname`" in + CYGWIN*) ISCYGWIN=yes;; + WIN*) ISMSVC=yes;; +esac +AM_CONDITIONAL(ISCYGWIN, [test "x$ISCYGWIN" = xyes]) +AM_CONDITIONAL(ISMSVC, [test "x$ISMSVC" = xyes]) + ### # Crude hack to work around an issue # in Cygwin. ### SLEEPCMD="" -PLTFORMOUT="$(uname | cut -d '_' -f 1)" -if test "$PLTFORMOUT" = "CYGWIN"; then - ISCYGWIN=yes +if test "x$ISCYGWIN" = "xyes"; then SLEEPCMD="sleep 5" AC_MSG_NOTICE([Pausing between sizeof() checks to mitigate a Cygwin issue.]) fi -AM_CONDITIONAL(ISCYGWIN, [test "x$ISCYGWIN" = xyes]) $SLEEPCMD AC_CHECK_SIZEOF(short) @@ -1200,7 +1227,20 @@ if test "x$enable_hdf5" = xyes; then fi AC_MSG_RESULT([$enable_szlib]) + # Check to see if HDF5 library is 1.10.6 or greater. + # Used to control path name conversion + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[ +#if (H5_VERS_MAJOR*10000 + H5_VERS_MINOR*100 + H5_VERS_RELEASE < 11006) + choke me +#endif + ]])], [hdf5_version_1106=yes], [hdf5_version_1106=no]) + AC_MSG_CHECKING([whether HDF5 library is version 1.10.6 or later]) + AC_MSG_RESULT([$hdf5_version_1106]) + if test "x$hdf5_version_1106" = xyes; then + AC_DEFINE([HDF5_UTF8_PATHS], [1], [if true, HDF5 paths can be utf-8]) + fi fi + AM_CONDITIONAL(ENABLE_NCDUMPCHUNKS, [test "x$has_readchunks" = xyes ]) # If the user wants hdf4 built in, check it out. @@ -1406,7 +1446,7 @@ AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes]) AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ]) AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes]) AM_CONDITIONAL(HAVE_AWS, [test "x$have_aws" = xyes]) -AM_CONDITIONAL(HAVE_MULTIFILTERS, [test "x$have_multifilters" = xyes]) +AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes]) # If the machine doesn't have a long long, and we want netCDF-4, then # we've got problems! @@ -1529,8 +1569,8 @@ AC_SUBST(HAS_BYTERANGE,[$enable_byterange]) AC_SUBST(RELAX_COORD_BOUND,[yes]) AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters]) AC_SUBST(HAS_NCZARR,[$enable_nczarr]) -AC_SUBST(DO_S3_TESTS,[$enable_s3_tests]) -AC_SUBST(HAS_MULTIFILTERS,[$have_multifilters]) +AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests]) +AC_SUBST(HAS_MULTIFILTERS,[$has_multifilters]) # Include some specifics for netcdf on windows. #AH_VERBATIM([_WIN32_STRICMP], @@ -1602,7 +1642,7 @@ AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes]) AX_SET_META([NC_HAS_PAR_FILTERS], [$hdf5_supports_par_filters],[yes]) AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes]) AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes]) -AX_SET_META([NC_HAS_MULTIFILTERS],[$have_multifilters],[yes]) +AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes]) # This is the version of the dispatch table. If the dispatch table is # changed, this should be incremented, so that user-defined format @@ -1615,11 +1655,13 @@ AC_SUBST([NC_DISPATCH_VERSION], [2]) # End netcdf_meta.h definitions. ##### +# This would be true for a visual studio build. +AC_SUBST([ISMSVC], [${ISMSVC}]) # This would be true for a cmake build. AC_SUBST([ISCMAKE], []) -# This would be true for a visual studio build. -AC_SUBST([MSVC], []) +# Provide conditional to temporarily suppress tests and such +AM_CONDITIONAL([AX_IGNORE], [test xno = xyes]) AC_MSG_NOTICE([generating header files and makefiles]) AC_CONFIG_FILES(test_common.sh:test_common.in) diff --git a/libhdf5/hdf5internal.c b/libhdf5/hdf5internal.c index a033492d8a..cd71e25708 100644 --- a/libhdf5/hdf5internal.c +++ b/libhdf5/hdf5internal.c @@ -1041,12 +1041,11 @@ nc4_ndf5_ansi_to_utf8(pathbuf_t *pb, const char *path) int n; if (hdf5_encoding == UNDEF) { - uint majnum, minnum, relnum; - H5get_libversion(&majnum, &minnum, &relnum); - hdf5_encoding = (((majnum == UTF8_MAJNUM && minnum == UTF8_MINNUM && relnum >= UTF8_RELNUM) - || (majnum == UTF8_MAJNUM && minnum > UTF8_MINNUM) - || majnum > UTF8_MAJNUM) - ? UTF8 : ANSI); +#ifdef HDF5_UTF8_PATHS + hdf5_encoding = UTF8; +#else + hdf5_encoding = ANSI; +#endif } if (hdf5_encoding == ANSI) { pb->ptr = NULL; diff --git a/nc-config.cmake.in b/nc-config.cmake.in index adc62c1a3b..56a9583bf1 100644 --- a/nc-config.cmake.in +++ b/nc-config.cmake.in @@ -101,6 +101,13 @@ else has_parallel="yes" fi +has_nczarr="@ENABLE_NCZARR@" +if [ -z "$has_nczarr" -o "$has_nczarr" = "OFF" -o "$has_nczarr" = "FALSE" ]; then + has_nczarr="no" +else + has_nczarr="yes" +fi + version="@PACKAGE@ @VERSION@" @@ -164,6 +171,7 @@ Available values for OPTION include: --has-cdf5 whether cdf5 support is included in build --has-parallel4 whether has parallel IO support via HDF5 --has-parallel whether has parallel IO support via HDF5 or PnetCDF + --has-nczarr whether nczarr enabled --libs library linking information for netcdf --static library linking information for statically-compiled netcdf --prefix Install prefix @@ -245,6 +253,7 @@ fi echo " --has-cdf5 -> $has_cdf5" echo " --has-parallel4 -> $has_parallel4" echo " --has-parallel -> $has_parallel" + echo " --has-nczarr -> $has_nczarr" echo echo " --prefix -> $prefix" echo " --includedir -> $includedir" @@ -346,6 +355,10 @@ while test $# -gt 0; do echo $has_parallel ;; + --has-nczarr) + echo $has_nczarr + ;; + --libs) if [ "x$HAS_STATIC" = "xTRUE" ]; then echo $libs $libsprivate diff --git a/nc-config.in b/nc-config.in index c31ace939b..1e0e4ea410 100644 --- a/nc-config.in +++ b/nc-config.in @@ -28,7 +28,6 @@ has_szlib="@HAS_SZLIB@" has_parallel="@HAS_PARALLEL@" has_parallel4="@HAS_PARALLEL4@" has_nczarr="@HAS_NCZARR@" -has_s3_sdk="@HAS_S3_SDK@" version="@PACKAGE_NAME@ @PACKAGE_VERSION@" HAS_STATIC="" @@ -94,7 +93,6 @@ Available values for OPTION include: --has-parallel4 whether has parallel IO support via HDF5 --has-parallel whether has parallel IO support via HDF5 or PnetCDF --has-nczarr whether nczarr enabled - --has-s3-sdk whether AWS S3 support enabled --libs library linking information for netcdf --static library linking information for statically-compiled netcdf --prefix Install prefix @@ -177,7 +175,6 @@ fi echo " --has-parallel4 -> $has_parallel4" echo " --has-parallel -> $has_parallel" echo " --has-nczarr -> $has_nczarr" - echo " --has-s3-sdk -> $has_s3_sdk" echo echo " --prefix -> $prefix" echo " --includedir -> $includedir" @@ -284,10 +281,6 @@ while test $# -gt 0; do echo $has_nczarr ;; - --has-s3-sdk) - echo $has_s3_sdk - ;; - --libs) if [ "x$HAS_STATIC" = "xTRUE" ]; then echo $libs $libsprivate diff --git a/nc_test4/findplugin.in b/nc_test4/findplugin.in index 1253f41bb0..aafe051de8 100644 --- a/nc_test4/findplugin.in +++ b/nc_test4/findplugin.in @@ -26,7 +26,7 @@ FP_NAME="$1" # Figure out the compiler (some values from ./configure) FP_ISCMAKE=@ISCMAKE@ -FP_ISMSVC=@MSVC@ +FP_ISMSVC=@ISMSVC@ # Are we operating under OS-X? (test using uname) FP_OS=`uname | cut -d '_' -f 1` diff --git a/test_common.in b/test_common.in index 3d9f7d8451..de0dfed2d5 100644 --- a/test_common.in +++ b/test_common.in @@ -11,7 +11,7 @@ TOPBUILDDIR='@abs_top_builddir@' # Define selected features of the build FEATURE_HDF5=@HAS_HDF5@ -FEATURE_S3TESTS=@DO_S3_TESTS@ +FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@ set -e