HDF5: Cross-compile

H/HDF5/build_tarballs.jl

@@ -1,90 +1,301 @@
-using BinaryBuilder
+# Note that this script can accept some limited command-line arguments, run
+# `julia build_tarballs.jl --help` to see a usage message.
+using BinaryBuilder, Pkg
 
-# Collection of sources required to build HDF5
 name = "HDF5"
-version = v"1.12.2"
+version = v"1.14.0"
 
+# Collection of sources required to complete build
 sources = [
- # 32-bit Windows from https://packages.msys2.org/package/mingw-w64-i686-hdf5
- ArchiveSource("https://mirror.msys2.org/mingw/mingw32/mingw-w64-i686-hdf5-1.12.2-1-any.pkg.tar.zst", "f37511b6208245b081a5b72a1f2021a73b67da07835ede1e092f7d7f20569319"; unpack_target="i686-w64-mingw32"),
- ArchiveSource("https://mirror.msys2.org/mingw/mingw32/mingw-w64-i686-libaec-1.0.6-2-any.pkg.tar.zst", "c6cff1a6f8a9f75e986589d8debc35e8076a7af38aa32cbda78bb6c2fbbbe58c"; unpack_target="i686-w64-mingw32"),
- ArchiveSource("https://mirror.msys2.org/mingw/mingw32/mingw-w64-i686-zlib-1.2.12-1-any.pkg.tar.zst", "74ace327d8e28cdb0c777fffc6003f5097836c247be40cf3b483bd9fd1c23183"; unpack_target="i686-w64-mingw32"),
- # We need some special compiler support libraries from mingw for i686 (libgcc_s_dw2)
- ArchiveSource("https://mirror.msys2.org/mingw/mingw32/mingw-w64-i686-gcc-libs-11.2.0-6-any.pkg.tar.zst", "bdc359047f61c8e96401ba25b17c80f5f8039c25a063c622e3680123bb0de9d1"; unpack_target="i686-w64-mingw32"),
-
- # 64-bit Windows from https://packages.msys2.org/package/mingw-w64-x86_64-hdf5
- ArchiveSource("https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-hdf5-1.12.2-1-any.pkg.tar.zst", "3ba6521d45368aabb334131e10282b25fab9891a20fb9129d897c65c8b6cdbda"; unpack_target="x86_64-w64-mingw32"),
- ArchiveSource("https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-libaec-1.0.6-2-any.pkg.tar.zst", "d970bd71e55fc5bd4a55e95ef22355d8c479631973860f2a9c37b49c931c5f35"; unpack_target="x86_64-w64-mingw32"),
- ArchiveSource("https://mirror.msys2.org/mingw/mingw64/mingw-w64-x86_64-zlib-1.2.12-1-any.pkg.tar.zst", "e728df08b4db7b291a52d8fd60b96f19016f059ab15170fc98120e5d580c86ac"; unpack_target="x86_64-w64-mingw32"),
-
- # x86_64 and aarch64 for Linux and macOS from https://anaconda.org/conda-forge/hdf5/files
- # NOTE: make sure to select those compatible with OpenSSL 1.1.1 (click info icon)
- # Unfortunately we cannot use conda-forge HDF 1.12.2 binaries since their libcurl is too new for us.
- # The MinGW 1.12.1 binaries work for HDF5.jl but not with libnetcdf. To have a HDF5_jll version
- # that works on all platforms, we resort to mixing two different patch versions.
- # See discussion following https://github.com/JuliaPackaging/Yggdrasil/issues/4511#issuecomment-1198134988
- ArchiveSource("https://anaconda.org/conda-forge/hdf5/1.12.1/download/linux-64/hdf5-1.12.1-nompi_h2750804_103.tar.bz2", "bd7bb0657d63acf52c9d30d1b89276356c6da4ff8a90dd5fcbd0cfde6578f317"; unpack_target="x86_64-linux-gnu"),
- ArchiveSource("https://anaconda.org/conda-forge/hdf5/1.12.1/download/linux-aarch64/hdf5-1.12.1-nompi_h774d4d8_103.tar.bz2", "8688cfc983962bf7a59a97becb0d67ee64eb4a7dd5793b915cf50dccd90bfa2d"; unpack_target="aarch64-linux-gnu"),
- ArchiveSource("https://anaconda.org/conda-forge/hdf5/1.12.1/download/osx-64/hdf5-1.12.1-nompi_h2f0ef1a_102.tar.bz2", "4a4640e44adea33833e7efb6ac3070dd4c80a8a156c2fd3aa7cfcac8865f5a26"; unpack_target="x86_64-apple-darwin14"),
- ArchiveSource("https://anaconda.org/conda-forge/hdf5/1.12.1/download/osx-arm64/hdf5-1.12.1-nompi_had0e5e0_103.tar.bz2", "ef48b684b22c6b0077bc9836e0cc6d15abb88868d7a6c842226666ebb8bbd449"; unpack_target="aarch64-apple-darwin20"),
+ ArchiveSource("https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.0/src/hdf5-1.14.0.tar.gz",
+ "a571cc83efda62e1a51a0a912dd916d01895801c5025af91669484a1575a6ef4"),
+ DirectorySource("./bundled"),
 ]
 
 # Bash recipe for building across all platforms
 script = raw"""
-cd ${WORKSPACE}/srcdir/
-mkdir -p ${libdir} ${includedir}
+cd ${WORKSPACE}/srcdir
+cd hdf5-*
 
-if [[ ${target} == *mingw* ]]; then
- cd ${target}/mingw${nbits}
+echo ${MACHTYPE}
+echo ${nbits}
+echo ${proc_family}
+echo ${target}
 
- rm -f lib/{*_cpp*,*fortran*,*f90*} # we do not need these
- rm -f bin/{*_cpp*,*fortran*,*f90*} # we do not need these
-
- mv -v lib/libhdf5*.dll.a ${prefix}/lib
- mv -v bin/*.dll ${libdir}
- mv -v include/* ${includedir}
+# Patch `CMakeLists.txt`:
+# - HDF5 would also try to build and run `H5detect` to collect ABI information.
+# We know this information, and thus can provide it manually.
+# - HDF5 would try to build and run `H5make_libsettings` to collect
+# build-time information. That information seems entirely optional, so
+# we do mostly nothing instead.
+atomic_patch -p1 ${WORKSPACE}/srcdir/patches/CMakeLists.txt.patch
+
+if [[ "${target}" == *-mingw* ]]; then
+ atomic_patch -p1 ${WORKSPACE}/srcdir/patches/H5timer.c.patch
+ atomic_patch -p1 ${WORKSPACE}/srcdir/patches/h5ls.c.patch
+ atomic_patch -p1 ${WORKSPACE}/srcdir/patches/mkdir.patch
+ atomic_patch -p1 ${WORKSPACE}/srcdir/patches/strncpy.patch
+fi
+
+mkdir build
+cd build
+
+# TODO:
+# - understand and fix long double / long configure tests
+# - -DHDF5_ENABLE_HDFS=ON
+# - -DHDF5_ENABLE_PARALLEL=ON
+# - -DDEFAULT_API_VERSION=...
+# - -DHDF5_ENABLE_THREADSAFE=ON
+# - build C++ and Fortran support
+# - -DHDF5_ENABLE_MAP_API=ON
+# - -DHDF5_BUILD_PARALLEL_TOOLS=ON
+# - do we actually need OpenMP? can we remove this dependency?
+# - the old HDF5 packages depends on OpenSSL and libCURL. why? what are we missing here?
+
+if false; then
+
+# Prepare the pre-generated file `H5Tinit.c` that cmake will expect:
+case "${target}" in
+ aarch64-apple-darwin*)
+ cat ../../files/H5Tinit-darwin-arm64v8.c
+ ;;
+ aarch64-linux-*)
+ cat ../../files/H5Tinit-debian-arm64v8.c
+ ;;
+ arm-linux-*)
+ cat ../../files/H5Tinit-debian-arm32v7.c
+ ;;
+ i686-linux-*)
+ cat ../../files/H5Tinit-debian-i386.c
+ ;;
+ i686-w64-mingw32)
+ # sizeof(long double) == 12
+ # layout seems to be 16-bit sign+exponent and 64-bit mantissa
+ # same as for Linux
+ cat ../../files/H5Tinit-debian-i386.c
+ ;;
+ powerpc64le-linux-*)
+ cat ../../files/H5Tinit-debian-ppc64le.c
+ ;;
+ x86_64-apple-darwin*)
+ cat ../../files/H5Tinit-darwin-amd64.c
+ ;;
+ x86_64-linux-* | x86_64-*-freebsd*)
+ cat ../../files/H5Tinit-debian-amd64.c
+ ;;
+ x86_64-w64-mingw32)
+ # sizeof(long double) == 16
+ # layout seems to be 16-bit sign+exponent and 64-bit mantissa
+ # same as for Linux
+ cat ../../files/H5Tinit-debian-amd64.c 
+ ;;
+ *)
+ echo "Unsupported target architecture ${target}" >&2
+ exit 1
+ ;;
+esac >../src/H5Tinit.c
+echo 'char H5libhdf5_settings[]="";' >../src/H5lib_settings.c
+
+cmake \
+ -DCMAKE_FIND_ROOT_PATH=${prefix} \
+ -DCMAKE_INSTALL_PREFIX=${prefix} \
+ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} \
+ -DBUILD_STATIC_LIBS=OFF \
+ -DBUILD_TESTING=OFF \
+ -DHDF5_BUILD_EXAMPLES=OFF \
+ -DHDF5_BUILD_HL_LIB=ON \
+ -DHDF5_BUILD_TOOLS=ON \
+ -DHAVE_IOEO_EXITCODE=0 \
+ -DTEST_LFS_WORKS_RUN=0 \
+ -DH5_LDOUBLE_TO_LONG_SPECIAL_RUN=1 \
+ -DH5_LDOUBLE_TO_LONG_SPECIAL_RUN__TRYRUN_OUTPUT= \
+ -DH5_LONG_TO_LDOUBLE_SPECIAL_RUN=1 \
+ -DH5_LONG_TO_LDOUBLE_SPECIAL_RUN__TRYRUN_OUTPUT= \
+ -DH5_LDOUBLE_TO_LLONG_ACCURATE_RUN=1 \
+ -DH5_LDOUBLE_TO_LLONG_ACCURATE_RUN__TRYRUN_OUTPUT= \
+ -DH5_LLONG_TO_LDOUBLE_CORRECT_RUN=1 \
+ -DH5_LLONG_TO_LDOUBLE_CORRECT_RUN__TRYRUN_OUTPUT= \
+ -DH5_DISABLE_SOME_LDOUBLE_CONV_RUN=1 \
+ -DH5_DISABLE_SOME_LDOUBLE_CONV_RUN__TRYRUN_OUTPUT= \
+ ..
+cmake --build . --config RelWithDebInfo --parallel ${nproc}
+cmake --build . --config RelWithDebInfo --parallel ${nproc} --target install
 
- install_license share/doc/hdf5/COPYING
 else
- cd ${target}
-
- # Delete the C++ and Fortran libraries, we can't use them because they'd restrict the
- # ABIs we can use this package for (e.g., *only* libgfortran5 and C++11 string ABI),
- # which would be overly restrictive, as Julia on Linux comes with libgfortran4, thus
- # making this package completely unusable there. The only way to use the C++ and
- # Fortran libraries is for BinaryBuilder to be able to actually build them:
- # <https://github.com/JuliaPackaging/Yggdrasil/issues/4760>.
- rm -f lib/{*_cpp*,*_fortran*}
- mv -v lib/* ${libdir}
- mv -v include/* ${includedir}
-
- install_license info/licenses/COPYING
+
+# Required for x86_64-linux-musl. Some HDF5 C code is C99, but configure only requests C89.
+# This might not be necessary if we switch to newer GCC versions.
+export CFLAGS="${CFLAGS} -std=c99"
+
+FLAGS=()
+if [[ "${target}" == *-mingw* ]]; then
+ FLAGS+=(LDFLAGS="-no-undefined")
 fi
+
+../configure \
+ --prefix=${prefix} \
+ --build=${MACHTYPE} \
+ --host=${target} \
+ --enable-hl=yes \
+ --enable-static=no \
+ --enable-tests=no \
+ --enable-tools=yes \
+ --with-examplesdir=/tmp \
+ hdf5_cv_ldouble_to_long_special=no \
+ hdf5_cv_long_to_ldouble_special=no \
+ hdf5_cv_ldouble_to_llong_accurate=no \
+ hdf5_cv_llong_to_ldouble_correct=no \
+ hdf5_cv_disable_some_ldouble_conv=yes
+
+# Patch the generated `Makefile`:
+# (We could instead patch `Makefile.in`, or maybe even `Makefile.am`.)
+# - HDF5 would also try to build and run `H5detect` to collect ABI information.
+# We know this information, and thus can provide it manually.
+# - HDF5 would try to build and run `H5make_libsettings` to collect
+# build-time information. That information seems entirely optional, so
+# we do mostly nothing instead.
+atomic_patch -p1 ${WORKSPACE}/srcdir/patches/Makefile.patch
+
+# Prepare the file `H5Tinit.c` that the patch above expects:
+case "${target}" in
+ aarch64-apple-darwin*)
+ cat ../../files/H5Tinit-darwin-arm64v8.c
+ ;;
+ aarch64-linux-*)
+ cat ../../files/H5Tinit-debian-arm64v8.c
+ ;;
+ arm-linux-*)
+ cat ../../files/H5Tinit-debian-arm32v7.c
+ ;;
+ i686-linux-*)
+ cat ../../files/H5Tinit-debian-i386.c
+ ;;
+ i686-w64-mingw32)
+ # sizeof(long double) == 12
+ # layout seems to be 16-bit sign+exponent and 64-bit mantissa
+ # same as for Linux
+ cat ../../files/H5Tinit-debian-i386.c
+ ;;
+ powerpc64le-linux-*)
+ cat ../../files/H5Tinit-debian-ppc64le.c
+ ;;
+ x86_64-apple-darwin*)
+ cat ../../files/H5Tinit-darwin-amd64.c
+ ;;
+ x86_64-linux-* | x86_64-*-freebsd*)
+ cat ../../files/H5Tinit-debian-amd64.c
+ ;;
+ x86_64-w64-mingw32)
+ # sizeof(long double) == 16
+ # layout seems to be 16-bit sign+exponent and 64-bit mantissa
+ # same as for Linux
+ cat ../../files/H5Tinit-debian-amd64.c 
+ ;;
+ *)
+ echo "Unsupported target architecture ${target}" >&2
+ exit 1
+ ;;
+esac >H5Tinit.c
+
+# `AM_V_P` is not defined. This must be a shell command that returns
+# true or false depending on whether `make` should be verbose. This is
+# probably caused by a bug in automake, or in how automake was used.
+make -j${nproc} AM_V_P=: "${FLAGS[@]}"
+
+make install
+
+fi
+
+# Create placeholders for missing executables
+if [[ "${target}" == *-mingw* ]]; then
+ cat >h5cc.c <<EOF
+#include <stdio.h>
+int main(int argc, char **argv) {
+ fprintf(stderr, "h5cc is not supported on this architecture\n");
+ return 1;
+}
+EOF
+ cc -c h5cc.c
+ cc -o "h5cc${exeext}" h5cc.o
+ install -Dvm 755 "h5cc${exeext}" "${bindir}/h5cc${exeext}"
+
+ cat >h5redeploy.c <<EOF
+#include <stdio.h>
+int main(int argc, char **argv) {
+ fprintf(stderr, "h5redeploy is not supported on this architecture\n");
+ return 1;
+}
+EOF
+ cc -c h5redeploy.c
+ cc -o "h5redeploy${exeext}" h5redeploy.o
+ install -Dvm 755 "h5redeploy${exeext}" "${bindir}/h5redeploy${exeext}"
+fi
+
+install_license ../COPYING
 """
 
 # These are the platforms we will build for by default, unless further
 # platforms are passed in on the command line
-platforms = [
- Platform("x86_64", "linux"),
- Platform("aarch64", "linux"; libc="glibc"),
- Platform("x86_64", "macos"),
- Platform("x86_64", "windows"),
- Platform("i686", "windows"),
- Platform("aarch64", "macos"),
-]
+platforms = supported_platforms()
+
+# Platforms:
+# aarch64-apple-darwin
+# aarch64-linux-gnu
+# aarch64-linux-musl
+# armv6l-linux-gnueabihf
+# armv6l-linux-musleabihf
+# armv7l-linux-gnueabihf
+# armv7l-linux-musleabihf
+# i686-linux-gnu
+# i686-linux-musl
+# i686-w64-mingw32
+# powerpc64le-linux-gnu
+# x86_64-apple-darwin
+# x86_64-linux-gnu
+# x86_64-linux-musl
+# x86_64-unknown-freebsd
+# x86_64-w64-mingw32
 
 # The products that we will ensure are always built
 products = [
+ # HDF5 tools
+ ExecutableProduct("h5cc", :h5cc),
+ ExecutableProduct("h5clear", :h5clear),
+ ExecutableProduct("h5copy", :h5copy),
+ ExecutableProduct("h5debug", :h5debug),
+ ExecutableProduct("h5delete", :h5delete),
+ ExecutableProduct("h5diff", :h5diff),
+ ExecutableProduct("h5dump", :h5dump),
+ ExecutableProduct("h5format_convert", :h5format_convert),
+ ExecutableProduct("h5import", :h5import),
+ ExecutableProduct("h5jam",:h5jam),
+ ExecutableProduct("h5ls", :h5ls),
+ ExecutableProduct("h5mkgrp", :h5mkgrp),
+ ExecutableProduct("h5perf_serial",:h5perf_serial),
+ ExecutableProduct("h5redeploy", :h5redeploy),
+ ExecutableProduct("h5repack", :h5repack),
+ ExecutableProduct("h5repart", :h5repart),
+ ExecutableProduct("h5stat", :h5stat),
+ ExecutableProduct("h5unjam", :h5unjam),
+ ExecutableProduct("h5watch", :h5watch),
+
+ # HDF5 libraries
  LibraryProduct("libhdf5", :libhdf5),
  LibraryProduct("libhdf5_hl", :libhdf5_hl),
 ]
 
 # Dependencies that must be installed before this package can be built
 dependencies = [
+ # For OpenMP we use libomp from `LLVMOpenMP_jll` where we use LLVM as compiler (BSD 
+ # systems), and libgomp from `CompilerSupportLibraries_jll` everywhere else. 
+ Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae");
+ platforms=filter(!Sys.isbsd, platforms)),
+ Dependency(PackageSpec(name="LLVMOpenMP_jll", uuid="1d63c593-3942-5779-bab2-d838dc0a180e");
+ platforms=filter(Sys.isbsd, platforms)),
  Dependency("Zlib_jll"),
- Dependency("OpenSSL_jll"; compat="1.1.10"),
- Dependency("LibCURL_jll"),
+ Dependency("dlfcn_win32_jll"),
 ]
 
 # Build the tarballs, and possibly a `build.jl` as well.
-build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6")
+build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies;
+ julia_compat="1.6")