Merge branch 'HDFGroup:develop' into develop

.github/workflows/cve.yml

@@ -32,14 +32,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4.1.7
 
-      - name: Install Autotools Dependencies (Linux)
-        run: |
-          sudo apt update
-          sudo apt install automake autoconf libtool libtool-bin
       - name: Install HDF5
         run: |
-          ./autogen.sh
-          ./configure --prefix=/usr/local --disable-tests
+          mkdir "${{ runner.workspace }}/build"
+          cd "${{ runner.workspace }}/build"
+          cmake -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_TESTING:BOOL=OFF $GITHUB_WORKSPACE
           make
           sudo make install
       - name: Checkout CVE test repository

.github/workflows/main-cmake.yml

@@ -98,7 +98,7 @@ jobs:
             parallel: OFF
             mirror_vfd: ON
             direct_vfd: OFF
-            ros3_vfd: OFF
+            ros3_vfd: ON
             generator: "-G Ninja"
             run_tests: true
 
@@ -132,7 +132,7 @@ jobs:
       # CMake gets libaec from fetchcontent
 
       - name: Install Dependencies (macOS)
-        run: brew install ninja
+        run: brew install ninja curl
         if: ${{ matrix.os == 'macos-latest' }}
 
         # symlinks the compiler executables to a common location 

CMakeFilters.cmake

@@ -173,20 +173,26 @@ if (HDF5_ENABLE_SZIP_SUPPORT)
     set(libaec_USE_STATIC_LIBS ${HDF5_USE_LIBAEC_STATIC})
     set(SZIP_FOUND FALSE)
     # Search pure Config mode, there is not a FindSZIP module available
-    find_package (SZIP NAMES ${LIBAEC_PACKAGE_NAME}${HDF_PACKAGE_EXT} OPTIONAL_COMPONENTS ${LIBAEC_SEARCH_TYPE})
-    set(H5_SZIP_FOUND ${SZIP_FOUND})
+    find_package (${LIBAEC_PACKAGE_NAME} NAMES ${LIBAEC_PACKAGE_NAME}${HDF_PACKAGE_EXT} OPTIONAL_COMPONENTS ${LIBAEC_SEARCH_TYPE})
+    set(H5_SZIP_FOUND ${${LIBAEC_PACKAGE_NAME}_FOUND})
     if (H5_SZIP_FOUND)
       set (H5_SZIP_INCLUDE_DIR_GEN ${SZIP_INCLUDE_DIR})
       set (H5_SZIP_INCLUDE_DIRS ${H5_SZIP_INCLUDE_DIRS} ${SZIP_INCLUDE_DIR})
-      set (LINK_COMP_LIBS ${LINK_COMP_LIBS} ${SZIP_LIBRARIES})
+      if(LIBAEC_PACKAGE_NAME STREQUAL "libaec")
+        set (LINK_COMP_LIBS ${LINK_COMP_LIBS} libaec::sz libaec::aec)
+      else ()
+        set (LINK_COMP_LIBS ${LINK_COMP_LIBS} ${SZIP_LIBRARIES})
+      endif ()
     endif ()
+    message (VERBOSE "H5_SZIP_FOUND=${SZIP_FOUND} and LINK_COMP_LIBS=${LINK_COMP_LIBS}")
   else ()
     if (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "GIT" OR HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ")
       EXTERNAL_SZIP_LIBRARY (${HDF5_ALLOW_EXTERNAL_SUPPORT} ${HDF5_ENABLE_SZIP_ENCODING})
       message (VERBOSE "Filter SZIP is built using library AEC")
       set (LINK_COMP_LIBS ${LINK_COMP_LIBS} ${H5_SZIP_STATIC_LIBRARY})
     endif ()
   endif ()
+  message (VERBOSE "LINK_COMP_LIBS=${LINK_COMP_LIBS}")
   if (H5_SZIP_FOUND)
     set (H5_HAVE_FILTER_SZIP 1)
     set (H5_HAVE_SZLIB_H 1)

CMakeInstallation.cmake

@@ -465,11 +465,12 @@ The HDF5 data model, file format, API, library, and tools are open and distribut
         endif ()
       endif ()
       if (H5_SZIP_FOUND AND SZIP_USE_EXTERNAL)
+        set (SZIP_PROJNAME "${LIBAEC_PACKAGE_NAME}")
         if (WIN32)
-          set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${H5_SZIP_INCLUDE_DIR_GEN};SZIP;ALL;/")
+          set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${H5_SZIP_INCLUDE_DIR_GEN};${SZIP_PROJNAME};ALL;/")
         else ()
-          set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${H5_SZIP_INCLUDE_DIR_GEN};SZIP;libraries;/")
-          set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${H5_SZIP_INCLUDE_DIR_GEN};SZIP;configinstall;/")
+          set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${H5_SZIP_INCLUDE_DIR_GEN};${SZIP_PROJNAME};libraries;/")
+          set (CPACK_INSTALL_CMAKE_PROJECTS "${CPACK_INSTALL_CMAKE_PROJECTS};${H5_SZIP_INCLUDE_DIR_GEN};${SZIP_PROJNAME};configinstall;/")
         endif ()
       endif ()
       if (PLUGIN_FOUND AND PLUGIN_USE_EXTERNAL)

CMakeTests.cmake

@@ -26,7 +26,7 @@
   option (HDF5_DISABLE_TESTS_REGEX "Regex pattern to set execution of specific tests to DISABLED" "")
   mark_as_advanced (HDF5_DISABLE_TESTS_REGEX)
 
-  option (HDF5_TEST_API "Execute HDF5 API tests" OFF)
+  option (HDF5_TEST_API "Execute HDF5 API tests" ON)
   mark_as_advanced (HDF5_TEST_API)
   if (HDF5_TEST_API)
     option (HDF5_TEST_API_INSTALL "Install HDF5 API tests" OFF)

config/cmake/CTestCustom.cmake

@@ -192,3 +192,9 @@ set (CTEST_CUSTOM_MEMCHECK_IGNORE
     H5WATCH-clearall-objects
     H5WATCH-cleanall-objects
 )
+
+list (APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE
+    "/test/"
+    "/_deps/"
+)
+

config/cmake/scripts/HDF5options.cmake

@@ -65,6 +65,8 @@ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ALLOW_EXTERNAL_SUPPORT:STRIN
 #set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DZLIB_LIBRARY:FILEPATH=some_location/lib/zlib.lib -DZLIB_INCLUDE_DIR:PATH=some_location/include")
 #set(ENV{SZIP_ROOT} "some_location")
 #set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSZIP_LIBRARY:FILEPATH=some_location/lib/szlib.lib -DSZIP_INCLUDE_DIR:PATH=some_location/include")
+#set(ENV{libaec_ROOT} "some_location")
+#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -Dlibaec_LIBRARY:FILEPATH=some_location/lib/libaec.lib -Dlibaec_INCLUDE_DIR:PATH=some_location/include")
 
 ### disable using ext zlib
 #set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_ZLIB_SUPPORT:BOOL=OFF")

release_docs/INSTALL_CMake.txt

@@ -285,20 +285,26 @@ IV. Further considerations
       web site. The HDF5 2."X"."Y" product requires a minimum CMake version 3.18.
       If you are using VS2022, the CMake minimum version is 3.21.
 
-   2. If you plan to use Zlib or Szip:
+   2. If you plan to use Zlib or Szip (aka libaec):
       A. Download the binary packages and install them in a central location.
          For example on Windows, create a folder extlibs and install the
          packages there. Add the following CMake options:
             -DZLIB_LIBRARY:FILEPATH=some_location/lib/zlib.lib
             -DZLIB_INCLUDE_DIR:PATH=some_location/include
             -DZLIB_USE_EXTERNAL:BOOL=OFF
-            -DSZIP_LIBRARY:FILEPATH=some_location/lib/szlib.lib
+            -DSZIP_LIBRARY:FILEPATH=some_location/lib/libszaec.lib
             -DSZIP_INCLUDE_DIR:PATH=some_location/include
+            -Dlibaec_LIBRARY:FILEPATH=some_location/lib/libaec.lib
+            -Dlibaec_INCLUDE_DIR:PATH=some_location/include
             -DSZIP_USE_EXTERNAL:BOOL=OFF
          where "some_location" is the full path to the extlibs folder.
-         Also the appropriate environment variable must be set;
+         Also if the appropriate environment variable is set, the above options are not required;
             set(ENV{ZLIB_ROOT} "some_location")
             set(ENV{SZIP_ROOT} "some_location")
+            set(ENV{libaec_ROOT} "some_location")
+
+        Note that if there is a problem finding the libraries, try adding the
+        CMake variable CMAKE_FIND_DEBUG_MODE:BOOL=ON to the command line.
 
       B. Use source packages from an GIT server by adding the following CMake
          options:
@@ -307,6 +313,8 @@ IV. Further considerations
             ZLIB_GIT_BRANCH="some_branch"
             SZIP_GIT_URL:STRING="https://some_location/szip"
             SZIP_GIT_BRANCH="some_branch"
+            LIBAEC_GIT_URL:STRING="https://some_location/libaec"
+            LIBAEC_GIT_BRANCH="some_branch"
          where "some_location" is the URL to the GIT repository and "some_branch" is
          a branch in the repository, usually the default. Also set
          CMAKE_BUILD_TYPE to the configuration type.
@@ -505,6 +513,8 @@ These five steps are described in detail below.
         <options> is:
             * SZIP_INCLUDE_DIR:PATH=<path to szip includes directory>
             * SZIP_LIBRARY:FILEPATH=<path to szip/library file>
+            * libaec_INCLUDE_DIR:PATH=<path to libaec includes directory>
+            * libaec_LIBRARY:FILEPATH=<path to libaec/library file>
             * ZLIB_INCLUDE_DIR:PATH=<path to zlib includes directory>
             * ZLIB_LIBRARY:FILEPATH=<path to zlib/library file>
             * <HDF5OPTION>:BOOL=[ON | OFF]
@@ -729,6 +739,8 @@ These five steps are described in detail below.
                ZLIB_GIT_BRANCH="${git_branch}"
                SZIP_GIT_URL:STRING="https://${git_url}/szip"
                SZIP_GIT_BRANCH="${git_branch}"
+               LIBAEC_GIT_URL:STRING="https://${git_url}/libaec"
+               LIBAEC_GIT_BRANCH="${git_branch}"
                PLUGIN_GIT_URL:STRING="https://${git_url}/plugin"
                PLUGIN_GIT_BRANCH="${git_branch}"
            ${git_url} should be changed to your location and ${git_branch} is

release_docs/RELEASE.txt

@@ -546,6 +546,23 @@ Bug Fixes since HDF5-2.0.0 release
 
     Configuration
     -------------
+    - Use pre-installed libaec compression library
+
+      The CMake logic for finding the libaec compression library has been
+      modified for a system-installed version of the library. Two options
+      must be set;
+          HDF5_ALLOW_EXTERNAL_SUPPORT:STRING=NO
+          <LIB_PKG_NAME>_USE_EXTERNAL:BOOL=OFF
+      where <LIB_PKG_NAME> is one of ZLIB, ZLIBNG, SZIP, PLUGIN.
+      Note that HDF5_ALLOW_EXTERNAL_SUPPORT:STRING=NO disables building all plugins
+      and external libraries in-line with the HDF5 library.
+
+      In addition, the <LIB_PKG_NAME>_ROOT environment variables must be set,
+      where <LIB_PKG_NAME> is one of ZLIB, ZLIBNG, SZIP, libaec, PLUGIN.
+      Note that libaec is the expected name for using the libaec library in place of original szip.
+
+      See INSTALL_CMake.txt for more detailed information.
+
     - Changed the zlib/szip compression find message to FATAL ERROR
 
       The message was changed to indicate that zlib/szip compression was requested and

src/H5FDros3.c

@@ -299,7 +299,7 @@ H5Pset_fapl_ros3(hid_t fapl_id, const H5FD_ros3_fapl_t *fa)
     if (plist == NULL)
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
-    if (FAIL == H5FD__ros3_validate_config(fa))
+    if (H5FD__ros3_validate_config(fa) < 0)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid ros3 config");
 
     ret_value = H5P_set_driver(plist, H5FD_ROS3, (const void *)fa, NULL);
@@ -754,8 +754,8 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
         assert(now != NULL);
         if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1))
             HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while writing iso8601 timestamp");
-        if (FAIL == H5FD_s3comms_signing_key(signing_key, (const char *)fa->secret_key,
-                                             (const char *)fa->aws_region, (const char *)iso8601now))
+        if (H5FD_s3comms_make_aws_signing_key(signing_key, (const char *)fa->secret_key,
+                                              (const char *)fa->aws_region, (const char *)iso8601now) < 0)
             HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "problem while computing signing key");
 
         if (token_exists)
@@ -783,7 +783,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
     H5MM_memcpy(&(file->fa), fa, sizeof(H5FD_ros3_fapl_t));
 
 #ifdef ROS3_STATS
-    if (FAIL == H5FD__ros3_reset_stats(file))
+    if (H5FD__ros3_reset_stats(file) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics");
 #endif
 
@@ -804,7 +804,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
 done:
     if (ret_value == NULL) {
         if (handle != NULL)
-            if (FAIL == H5FD_s3comms_s3r_close(handle))
+            if (H5FD_s3comms_s3r_close(handle) < 0)
                 HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle");
         if (file != NULL) {
             H5MM_xfree(file->cache);
@@ -836,12 +836,12 @@ H5FD__ros3_close(H5FD_t H5_ATTR_UNUSED *_file)
     assert(file->s3r_handle != NULL);
 
 #ifdef ROS3_STATS
-    if (H5FD__ros3_print_stats(stdout, file) == FAIL)
+    if (H5FD__ros3_print_stats(stdout, file) < 0)
         HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics");
 #endif
 
     /* Close the underlying request handle */
-    if (FAIL == H5FD_s3comms_s3r_close(file->s3r_handle))
+    if (H5FD_s3comms_s3r_close(file->s3r_handle) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close S3 request handle");
 
     /* Release the file info */
@@ -1119,7 +1119,7 @@ H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU
         memcpy(buf, file->cache + addr, size);
     }
     else {
-        if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL)
+        if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) < 0)
             HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read");
 
 #ifdef ROS3_STATS