For macOS, don't use brew (#342)

* for macos wheels and tests, build hdf5 ourselves
* (while I was here) use `actions/upload-artifact@v3` and make linux apt installs less likely to fail

.github/workflows/publish-sdist-wheels.yml

@@ -2,6 +2,8 @@ name: Publish sdist and wheels for macos, and manylinux, publish to pypi if a re
 on: [pull_request, push]
 
 env:
+ apt_options: -o Acquire::Retries=3
+
  CIBW_BUILD_VERBOSITY: 3
  CIBW_BUILD: 'cp*'
  CIBW_SKIP: 'cp35-* cp36-* cp37-* *-musllinux_* *-manylinux_i686'
@@ -32,6 +34,17 @@ jobs:
  run: |
  python -m pip install cibuildwheel
 
+ - name: Cache HDF5 On Linux/macOS
+ if: runner.os == 'Linux' || runner.os == 'macOS'
+ uses: actions/cache@v3
+ id: cache-hdf5-posix
+ env:
+ cache-name: cache-hdf5-posix
+ with:
+ path: src-cache/
+ key: ${{ runner.os }}-build-${{ env.cache-name }}
+
+
  - name: Build wheels on Linux
  if: runner.os == 'Linux'
  run: |
@@ -41,22 +54,35 @@ jobs:
 
  - name: Build wheels Mac OS
  if: runner.os == 'macOS'
+
  env:
- MACOSX_DEPLOYMENT_TARGET: '10.13'
+ CIBW_ENVIRONMENT_PASS: "HDF5_ROOT CMAKE_OSX_ARCHITECTURES MACOSX_DEPLOYMENT_TARGET"
  CIBW_BEFORE_BUILD: |
- # Unlink and re-link to prevent errors when GitHub macOS runner images
- # install Python outside of brew.
- brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done
-
- brew update
- brew --version
- brew install cmake
- brew install hdf5
+ # CMake complains if the dependencies come from within the same tree
+ # as the source, so we'll just pretend they are elsewhere
+ mkdir -p $PWD/src-cache
+ ln -s $PWD/src-cache /Users/runner/work/src-cache
+ bash ci/hdf5-build.sh /Users/runner/work/src-cache
+ UNIXY_AEC_VERSION: 1.0.4
+ UNIXY_HDF5_VERSION: 1.14.2
  run: |
+ # x86_64 macOS allows for cross compilation; first we do arm64,
+ # only for the 11.0 target; and store in the cache the compiled code...
+ export CIBW_ARCHS_MACOS="arm64"
+ export CMAKE_OSX_ARCHITECTURES="arm64"
+ export MACOSX_DEPLOYMENT_TARGET="11.0"
+ export HDF5_ROOT=/Users/runner/work/src-cache/install-$CIBW_ARCHS_MACOS
+ python -m cibuildwheel --output-dir dist
+
+ # ...and now we do both targets for x86_64
+ export CIBW_ARCHS_MACOS="x86_64"
+ export CMAKE_OSX_ARCHITECTURES="x86_64"
+ unset MACOSX_DEPLOYMENT_TARGET
+ export HDF5_ROOT=/Users/runner/work/src-cache/install-$CIBW_ARCHS_MACOS
  python -m cibuildwheel --output-dir dist
 
  - name: Store wheel as artifact
- uses: actions/upload-artifact@v2
+ uses: actions/upload-artifact@v3
  with:
  name: dist
  path: dist/*.whl
@@ -77,14 +103,16 @@ jobs:
  python-version: 3.11
 
  - name: Install packages
- run: sudo apt-get update && sudo apt-get install libhdf5-dev
+ run: |
+ sudo apt-get ${{env.apt_options}} update -y
+ sudo apt-get ${{env.apt_options}} install -y libhdf5-dev
 
  - name: Build a source tarball, check it installs
  run:
  ./ci/python_build_sdist.sh
 
  - name: Store sdist as artifact
- uses: actions/upload-artifact@v2
+ uses: actions/upload-artifact@v3
  with:
  name: dist
  path: dist/*.tar.gz
@@ -98,7 +126,7 @@ jobs:
 
  steps:
  - name: Download artifacts produced during the build_wheels and build_sdist jobs
- uses: actions/download-artifact@v2
+ uses: actions/download-artifact@v3
  with:
  name: dist
  path: dist/

.github/workflows/run-tests.yml

@@ -7,6 +7,9 @@ on:
  branches:
  - master
 
+env:
+ apt_options: -o Acquire::Retries=3
+
 jobs:
  build-linux-cpp:
  name: Run tests on ubuntu-20.04
@@ -25,7 +28,9 @@ jobs:
  submodules: 'true'
 
  - name: Install packages
- run: sudo apt-get update && sudo apt-get install libhdf5-dev
+ run: |
+ sudo apt-get ${{env.apt_options}} update -y
+ sudo apt-get ${{env.apt_options}} install -y libhdf5-dev
 
  - name: Build and run unittests
  env:
@@ -49,7 +54,9 @@ jobs:
  submodules: 'true'
 
  - name: Install packages
- run: sudo apt-get update && sudo apt-get install libhdf5-openmpi-dev
+ run: |
+ sudo apt-get ${{env.apt_options}} update -y
+ sudo apt-get ${{env.apt_options}} install -y libhdf5-openmpi-dev
 
  - name: Build and run unittests
  env:
@@ -78,7 +85,9 @@ jobs:
  python-version: ${{ matrix.python-version }}
 
  - name: Install packages
- run: sudo apt-get update && sudo apt-get install libhdf5-dev doxygen
+ run: |
+ sudo apt-get ${{env.apt_options}} update -y
+ sudo apt-get ${{env.apt_options}} install -y libhdf5-dev doxygen
 
  - name: Build and run unittests
  run: |
@@ -87,25 +96,36 @@ jobs:
  build-macos:
  name: Run tests on macos
  runs-on: macos-latest
+ env:
+ CIBW_ARCHS_MACOS: "x86_64"
+ CMAKE_OSX_ARCHITECTURES: "x86_64"
+ UNIXY_AEC_VERSION: 1.0.4
+ UNIXY_HDF5_VERSION: 1.14.2
 
  steps:
  - name: Checkout repository
  uses: actions/checkout@v3
  with:
  submodules: 'true'
 
+ - name: Cache HDF5 On Linux/macOS
+ if: runner.os == 'Linux' || runner.os == 'macOS'
+ uses: actions/cache@v3
+ id: cache-hdf5-posix
+ env:
+ cache-name: cache-hdf5-posix
+ with:
+ path: src-cache/
+ key: ${{ runner.os }}-build-${{ env.cache-name }}
+
  - name: Install packages on MacOS
  run: |
- # Unlink and re-link to prevent errors when GitHub macOS runner images
- # install Python outside of brew; See actions/setup-python#577 
- brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done
-
- brew update
- brew install cmake || true # macos image has cmake installed, but a new version may exist; ignore it if so
- brew install doxygen
- brew install hdf5
+ mkdir -p $PWD/src-cache
+ ln -s $PWD/src-cache /Users/runner/work/src-cache
+ bash ci/hdf5-build.sh /Users/runner/work/src-cache
 
  - name: Build and run unittests
  run: |
+ export HDF5_ROOT=/Users/runner/work/src-cache/install-$CIBW_ARCHS_MACOS
  ./ci/cpp_test.sh
  ./ci/python_test.sh

ci/hdf5-build.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+set -e -x
+
+
+export INPUT=$(cd $(dirname "$1") && pwd -P)/$(basename "$1")
+export OUTPUT="$INPUT/install-$CIBW_ARCHS_MACOS"
+
+: "${ZLIB_VERSION:=1.3}"
+: "${UNIXY_AEC_VERSION:=1.0.4}"
+: "${UNIXY_HDF5_VERSION:=1.14.2}"
+
+function download_unpack_zlib {
+ pushd "$INPUT"
+ rm -rf "zlib-$ZLIB_VERSION"
+ echo "Downloading & unpacking ZLIB ${ZLIB_VERSION}"
+ curl -sLO https://zlib.net/fossils/zlib-$ZLIB_VERSION.tar.gz
+ tar xzf zlib-$ZLIB_VERSION.tar.gz
+ popd
+}
+
+function download_unpack_libaec {
+ pushd "$INPUT"
+ rm -rf "libaec-$UNIXY_AEC_VERSION"
+ echo "Downloading & unpacking aec ${UNIXY_AEC_VERSION}"
+ curl -fsSLO "https://gitlab.dkrz.de/k202009/libaec/uploads/ea0b7d197a950b0c110da8dfdecbb71f/libaec-${UNIXY_AEC_VERSION}.tar.gz"
+ tar zxf "libaec-$UNIXY_AEC_VERSION.tar.gz"
+ popd
+}
+
+function download_unpack_hdf5 {
+ pushd "$INPUT"
+ rm -rf hdf5-$UNIXY_HDF5_VERSION
+ echo "Downloading & unpacking HDF5 ${UNIXY_HDF5_VERSION}"
+ curl -fsSLO "https://www.hdfgroup.org/ftp/HDF5/releases/hdf5-${UNIXY_HDF5_VERSION%.*}/hdf5-$UNIXY_HDF5_VERSION/src/hdf5-$UNIXY_HDF5_VERSION.tar.gz"
+ tar xzf "hdf5-$UNIXY_HDF5_VERSION.tar.gz"
+ popd
+}
+
+if [[ "$OSTYPE" == "darwin"* ]]; then
+ lib_name=libhdf5.dylib
+ NPROC=$(sysctl -n hw.ncpu)
+else
+ lib_name=libhdf5.so
+ NPROC=$(nproc)
+fi
+
+if [[ -f "$OUTPUT/lib/$lib_name" ]]; then
+ echo "using cached build"
+else
+ if [[ "$OSTYPE" == "darwin"* ]]; then
+ export CC="/usr/bin/clang"
+ export CXX="/usr/bin/clang"
+ export CFLAGS="$CFLAGS -arch $CIBW_ARCHS_MACOS"
+ export CPPFLAGS="$CPPFLAGS -arch $CIBW_ARCHS_MACOS"
+ export CXXFLAGS="$CXXFLAGS -arch $CIBW_ARCHS_MACOS"
+
+ download_unpack_zlib
+
+ pushd "$INPUT/zlib-$ZLIB_VERSION"
+ ./configure \
+ --archs="-arch $CIBW_ARCHS_MACOS" \
+ --prefix="$OUTPUT"
+ make
+ make install
+ popd
+
+ ZLIB_ARG="--with-zlib=$OUTPUT"
+
+ if [[ "$CIBW_ARCHS_MACOS" = "arm64" ]]; then
+ HOST_ARG="--host=aarch64-apple-darwin"
+ fi
+ fi
+
+ echo "Building & installing libaec"
+ download_unpack_libaec
+ pushd "$INPUT/libaec-$UNIXY_AEC_VERSION"
+ ./configure \
+ --prefix="$OUTPUT" \
+ ${HOST_ARG} \
+ --enable-static=no
+
+ make -j "$NPROC"
+ make install
+ popd
+
+ echo "Building & installing hdf5"
+ download_unpack_hdf5
+ pushd "$INPUT/hdf5-$UNIXY_HDF5_VERSION"
+ ./configure \
+ --prefix="$OUTPUT" \
+ --with-szlib="$OUTPUT" \
+ ${HOST_ARG} \
+ ${ZLIB_ARG} \
+ --enable-build-mode=production \
+ --enable-tools=no \
+ --enable-tests=no \
+ --enable-static=no
+
+ # {
+ # work around for crosscompile getting: `./H5detect: Bad CPU type in executable`
+ # so build it on the host architecture instead
+ pushd src
+ ( export ARCHFLAGS= ; make H5detect H5make_libsettings CFLAGS= CCLDFLAGS= CXXFLAGS= CPPFLAGS= )
+ popd
+ # }
+
+ make -j "$NPROC"
+ make install
+ popd
+
+ file "$OUTPUT"/lib/*
+fi