Add HDF5 support for building wheels (#5)

* Remove boost from sdist build, not needed

* Bump cibuildwheel and some cleanup

* Bump boost to 1.85.0

* Config updates

* Add hdf5 for macos and turn off fail fast

* Move some more config to pyproject.toml

* Update some build steps from h5py

* Star the python config

* Use boost install action on Linux

* Adapt windows builder from h5py

* Update macos build after h5py

* Checkout the repo

* D'oh

* Hopefully fixes for boost

* Fix boost locations

* Fix paths to script locations

* Boost is built-in to the Linux cibuildwheel image now

* Set MACOSX_DEPLOYMENT_TARGET for hdf5

We need to set MACOSX_DEPLOYMENT_TARGET when building HDF5, explained here https://github.com/h5py/h5py/pull/2444/files#r1679541011

* Fix boost arch for macos

* Cache HDF5 after building

* Try setting up HDF5 libs for macOS

* Fix boost paths on Windows

* More Boost fixes

* Move building hdf5 into macos script

* Set MACOSX_DEPLOYMENT_TARGET to support C++17

* Fix some bash syntax errors

* Patch libaec

* Debugging

* Fix dots and dashes in hdf5 version

* Fix building HDF5 on macOS

* Add container engine config to environment

* Bump MACOSX_DEPLOYMENT_TARGET

Latest error message says 10.15

* Export variables on macos to the environment

* Fix Windows test paths

* Simplify to a single job definition

* Rename the action file

* Add arch to boost installer

* Need to set boost arch for macos

* Fix download action

* Cache pip dependencies

* Fix build selectors on macOS

* Action garbage

* Quoting problems

* Windows updates

* Typo!

* Fix a macos arm misconfig

* Set HDF5 and ZLIB dirs so delvewheel config doesn't need to know the installation structure

* Clean up unused script and add README

* Try to clean up env var setting

* Set CMAKE parallel level on Windows

The other platforms should use Ninja which is automatically parallel

* Formatting and updates

* Address review comments

.github/actions/download-cantera-test/action.yml

@@ -0,0 +1,43 @@
+name: "Download Cantera Test Files"
+description: "Download and cache Cantera's Python test suite"
+inputs:
+  incoming-sha:
+    description: The hash of the commit that should be downloaded
+    required: true
+outputs:
+  test-root:
+    description: The root folder where the tests are located
+    value: ${{ steps.set-output.outputs.DESTINATION_PATH }}
+
+runs:
+  using: "composite"
+  steps:
+  - name: Sanitize the destination path
+    run: |
+      $DESTINATION_PATH = "${{ runner.temp }}" -replace "\\", "/"
+      echo "DESTINATION_PATH=$DESTINATION_PATH" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
+    if: runner.os == 'Windows'
+    shell: pwsh
+  - name: Sanitize the destination path
+    run: echo "DESTINATION_PATH=${{ runner.temp }}" >> $GITHUB_ENV
+    shell: bash
+    if: runner.os != 'Windows'
+  - name: Set the outputs
+    id: set-output
+    run: echo "DESTINATION_PATH=${{ env.DESTINATION_PATH }}" >> $GITHUB_OUTPUT
+    shell: bash
+  - name: Download and unpack the tarball
+    if: steps.restore-cache.outputs.cache-hit != 'true'
+    run: |
+      curl -fsSL "https://github.com/cantera/cantera/archive/${{ inputs.incoming-sha }}.tar.gz" -o cantera.tar.gz
+      tar -xzf cantera.tar.gz --strip-components=1 "cantera-${{ inputs.incoming-sha }}/test"
+      rm cantera.tar.gz
+    shell: bash
+    working-directory: ${{ steps.set-output.outputs.DESTINATION_PATH }}
+  - name: Save the test file cache
+    uses: actions/cache/save@v4
+    if: always() && steps.restore-cache.outputs.cache-hit != true
+    id: save-cache
+    with:
+      path: ${{ steps.set-output.outputs.DESTINATION_PATH }}/test
+      key: ${{ steps.restore-cache.outputs.cache-primary-key }}

.github/workflows/python-package.yml

@@ -19,9 +19,6 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  CIBW_BUILD_FRONTEND: build
-  CIBW_TEST_EXTRAS: pandas,units,graphviz
-  CIBW_TEST_REQUIRES: pytest
   ACTION_URL: "https://github.com/Cantera/pypi-packages/actions/runs/${{ github.run_id }}"
 
 jobs:
@@ -106,10 +103,6 @@ jobs:
     outputs:
       job-status: ${{ job.status }}
     steps:
-      - name: Install dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install libboost-dev
       - uses: actions/checkout@v4
         name: Checkout the repository
         with:
@@ -120,15 +113,13 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
+          cache: 'pip'
+          cache-dependency-path: interfaces/python_sdist/pyproject.toml.in
       - name: Install dependencies
-        run: python3 -m pip install -U pip scons build
+        run: python3 -m pip install scons build
       - name: Build the sdist
         run: |
-          python3 `which scons` sdist f90_interface=n python_package='none' \
-          system_blas_lapack=n system_sundials=n system_eigen=n system_fmt=n \
-          system_yamlcpp=n googletest=none env_vars='CYTHON_FORCE_REGEN'
-        env:
-          CYTHON_FORCE_REGEN: "1"
+          python3 `which scons` sdist
       - name: Archive the built sdist
         uses: actions/upload-artifact@v4
         with:
@@ -154,164 +145,153 @@ jobs:
           echo -e '\n</details>\n' >> $GITHUB_STEP_SUMMARY
           echo ----- End of Metadata  -----
 
-  linux-wheel:
-    name: Build ${{ matrix.libc }}linux_${{ matrix.arch }} for py${{ matrix.py }}
-    runs-on: ubuntu-22.04
+  build-wheels:
+    name: Build ${{ matrix.os }} ${{ matrix.arch }} for py${{ matrix.python || '-all' }}
+    runs-on: ${{ matrix.os }}
     needs: ["sdist", "post-pending-status"]
     outputs:
       job-status: ${{ job.status }}
     strategy:
       matrix:
-        py: ["38", "39", "310", "311", "312"]
-        arch: ["x86_64", "aarch64"]
-        libc: ["many"]
-      fail-fast: true
-    env:
-      BOOST_INCLUDE: include
-      BOOST_URL: https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.7z
+        # Wheel builds are fast except for aarch64, so split that into multiple jobs,
+        # one for each Python version
+        os: [ubuntu-latest]
+        arch: [aarch64]
+        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        include:
+          - os: ubuntu-latest
+            arch: x86_64
+          - os: windows-2022
+            arch: AMD64
+            boost-arch: x86
+            boost-toolset: msvc
+            boost-platform-version: 2022
+            boost-version: "1.85.0"
+          - os: macos-14
+            arch: arm64
+            boost-arch: aarch64
+            boost-toolset: clang
+            # Since we only use the headers, we can use the platform version for this
+            # macos version
+            boost-platform-version: "14"
+            boost-version: "1.85.0"
+          - os: macos-13
+            arch: x86_64
+            boost-arch: x86
+            boost-toolset: clang
+            # Since we only use the headers, we can use the platform version for this
+            # macos version
+            boost-platform-version: "13"
+            boost-version: "1.85.0"
+      fail-fast: false
     steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
       - name: Download pre-built sdist
         uses: actions/download-artifact@v4
         with:
           name: cibw-sdist
       - name: Extract the sdist tarball
         run: tar -xvf *.tar.gz --strip-components=1
-      - name: Restore Boost cache
-        uses: actions/cache@v4
-        id: cache-boost
+        shell: bash
+      - name: Download test files
+        id: download-test-files
+        uses: ./.github/actions/download-cantera-test
         with:
-          path: ${{ env.BOOST_INCLUDE }}/boost
-          key: boost-${{env.BOOST_URL}}
-      - name: Install Boost Headers
-        if: steps.cache-boost.outputs.cache-hit != 'true'
-        run: |
-          mkdir -p $BOOST_INCLUDE
-          curl --progress-bar --location --output $BOOST_INCLUDE/download.7z $BOOST_URL
-          7z -o$BOOST_INCLUDE x $BOOST_INCLUDE/download.7z -y -bd boost_1_78_0/boost
-          mv $BOOST_INCLUDE/boost_1_78_0/boost $BOOST_INCLUDE/boost
-          rm $BOOST_INCLUDE/download.7z
-          rm -r $BOOST_INCLUDE/boost_1_78_0
+          incoming-sha: ${{ needs.post-pending-status.outputs.incoming-sha }}
+
+      # Linux steps
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
         with:
           platforms: all
-      - name: Build wheels
-        uses: pypa/cibuildwheel@v2.19.1
-        env:
-          CIBW_ENVIRONMENT: BOOST_INCLUDE=${{ env.BOOST_INCLUDE }} CT_SKIP_SLOW=1 CYTHON_FORCE_REGEN=${{ matrix.py == '38' && '1' || '0' }}
-          CIBW_BUILD: cp${{ matrix.py }}-${{ matrix.libc }}linux*
-          CIBW_ARCHS: ${{ matrix.arch }}
-          # cibuildwheel on Linux uses a Docker container to run the build, so
-          # runner.temp is not available. cibuildwheel also uses the /tmp folder, so
-          # we should be pretty safe to also use that.
-          CIBW_TEST_COMMAND: pytest -vv --durations=100 /tmp/test/python
-          CIBW_BEFORE_TEST: |
-            curl -sL "https://github.com/cantera/cantera/archive/${{ needs.post-pending-status.outputs.incoming-sha }}.tar.gz" -o /tmp/cantera.tar.gz \
-            && tar -xzf /tmp/cantera.tar.gz --strip-components=1 -C /tmp "cantera-${{ needs.post-pending-status.outputs.incoming-sha }}/test"
-
-          # NumPy is generally not available for these platforms so testing takes a
-          # while. This just skips the tests on these
-          # combinations, the wheels are still built and uploaded.
-          CIBW_TEST_SKIP: "*-manylinux_{i686,ppc64le,s390x} *musl*"
+        if: matrix.arch != 'x86_64' && runner.os == 'Linux'
+      - name: Set up CIBW environment
+        run: |
+          PYTHON="${{ matrix.python }}"
+          if [[ $PYTHON == "" ]]; then PYTHON="*"; fi
+          CIBW_BUILD="cp${PYTHON//./}-*_${{ matrix.arch }}"
+          echo "CIBW_BUILD=${CIBW_BUILD}" | tee -a $GITHUB_ENV
+        if: runner.os == 'Linux'
 
-      - name: Archive the built wheels
-        uses: actions/upload-artifact@v4
+      - name: Install boost
+        # Our custom manylinux images already have Boost installed
+        if: runner.os != 'Linux'
+        uses: MarkusJx/install-boost@v2.4.5
+        id: install-boost
         with:
-          path: ./wheelhouse/*.whl
-          name: cibw-wheels-linux-${{ strategy.job-index }}
+            # REQUIRED: Specify the required boost version
+            # A list of supported versions can be found here:
+            # https://github.com/MarkusJx/prebuilt-boost/blob/main/versions-manifest.json
+            boost_version: ${{ matrix.boost-version }}
+            # OPTIONAL: Specify a custon install location
+            boost_install_dir: ${{ runner.temp }}
+            toolset: ${{ matrix.boost-toolset }}
+            platform_version: ${{ matrix.boost-platform-version }}
+            arch: ${{ matrix.boost-arch }}
 
-  windows-wheel:
-    name: Build Windows Wheels for py${{ matrix.py }}
-    runs-on: windows-2019
-    needs: ["sdist", "post-pending-status"]
-    outputs:
-      job-status: ${{ job.status }}
-    strategy:
-      matrix:
-        py: ["38", "39", "310", "311", "312"]
-      fail-fast: true
-    env:
-      BOOST_ROOT: ${{ github.workspace }}/3rdparty/boost
-      BOOST_URL: https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.7z
-    steps:
-      - name: Download pre-built sdist
-        uses: actions/download-artifact@v4
-        with:
-          name: cibw-sdist
-      - name: Extract the sdist tarball
-        run: tar -xvf *.tar.gz --strip-components=1
-        shell: bash
-      - name: Restore Boost cache
+      - name: Cache built libraries
+        id: cache-built-libraries
+        # Our custom manylinux images already have hdf5 installed
+        if: runner.os != 'Linux'
         uses: actions/cache@v4
-        id: cache-boost
         with:
-          path: ${{env.BOOST_ROOT}}
-          key: boost-${{env.BOOST_URL}}
-      - name: Install Boost Headers
-        if: steps.cache-boost.outputs.cache-hit != 'true'
-        run: |
-          BOOST_ROOT=$(echo $BOOST_ROOT | sed 's/\\/\//g')
-          mkdir -p $BOOST_ROOT
-          curl --progress-bar --location --output $BOOST_ROOT/download.7z $BOOST_URL
-          7z -o$BOOST_ROOT x $BOOST_ROOT/download.7z -y -bd boost_1_78_0/boost
-          mv $BOOST_ROOT/boost_1_78_0/boost $BOOST_ROOT/boost
-          rm $BOOST_ROOT/download.7z
-        shell: bash
-      - name: Build wheels
-        uses: pypa/cibuildwheel@v2.19.1
-        env:
-          CIBW_ENVIRONMENT: BOOST_INCLUDE=${BOOST_ROOT} CT_SKIP_SLOW=1 CYTHON_FORCE_REGEN=${{ matrix.py == '38' && '1' || '0' }}
-          CIBW_ARCHS: "AMD64"
-          CIBW_BUILD: cp${{ matrix.py }}-*
-          CIBW_TEST_COMMAND: pytest -vv --durations=100 ${{ runner.temp }}/test/python
-          CIBW_BEFORE_TEST: |
-            curl -sL "https://github.com/cantera/cantera/archive/${{ needs.post-pending-status.outputs.incoming-sha }}.tar.gz" -o ${{ runner.temp }}/cantera.tar.gz && tar -xzf ${{ runner.temp }}/cantera.tar.gz --strip-components=1 -C ${{ runner.temp }} "cantera-${{ needs.post-pending-status.outputs.incoming-sha }}/test"
-      - name: Archive the built wheels
-        uses: actions/upload-artifact@v4
+          key: ${{ matrix.os }}-${{ matrix.arch }}-0
+          path: ${{ runner.temp }}/cache
+
+      # Windows-only steps
+      - name: Set Up Nuget
+        uses: nuget/setup-nuget@v2
+        if: runner.os == 'Windows'
+      - uses: actions/setup-python@v5
         with:
-          path: ./wheelhouse/*.whl
-          name: cibw-wheels-windows-${{ strategy.job-index }}
+          python-version: 3.12
+        if: runner.os == 'Windows'
+      - run: bash ./cibw_before_all_windows.sh "${{ runner.temp }}"
+        if: runner.os == 'Windows'
+      - name: Set up CIBW environment
+        # On Windows, Boost_ROOT needs to have \ replaced by / because that's what
+        # cibuildwheel says. CANTERA_TEST_DIR doesn't need the replacement because
+        # it will be substituted in a cmd or pwsh session.
+        run: |
+          $BOOST_ROOT = "${{ steps.install-boost.outputs.BOOST_ROOT }}" -replace "\\", "/"
+          echo "Boost_ROOT=$BOOST_ROOT" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
+          echo "CANTERA_TEST_DIR=${{ steps.download-test-files.outputs.test-root }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
+          echo "CIBW_BUILD=cp*-*" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
+        shell: pwsh
+        if: runner.os == 'Windows'
 
-  macos-wheel:
-    name: Build ${{ matrix.macos-version }} Wheels for py${{ matrix.py }}
-    runs-on: ${{ matrix.macos-version }}
-    needs: ["sdist", "post-pending-status"]
-    outputs:
-      job-status: ${{ job.status }}
-    strategy:
-      matrix:
-        macos-version: [ "macos-13", "macos-14" ]
-        py: [ "39", "310", "311", "312" ]
-        deployment_target: [ "11.0" ]
-        include:
-          - py: "38"
-            deployment_target: "11.0"
-            macos-version: "macos-13"
-      fail-fast: true
-    env:
-      MACOSX_DEPLOYMENT_TARGET: ${{ matrix.deployment_target }}
-    steps:
-      - name: Download pre-built sdist
-        uses: actions/download-artifact@v4
+      # macOS-only steps
+      - name: Build required libraries
+        run: bash ./cibw_before_all_macos.sh "${{ runner.temp }}"
+        if: runner.os == 'macOS'
+      # Force installation to resolve Python 3.8 bug (https://github.com/pypa/cibuildwheel/pull/1871#issuecomment-2161613619)
+      - name: Hack for 3.8 bug
+        uses: actions/setup-python@v5
         with:
-          name: cibw-sdist
-      - name: Extract the sdist tarball
-        run: tar -xvf *.tar.gz --strip-components=1
-      - name: Install Brew dependencies
-        run: brew install boost
+          python-version: 3.8
+        if: runner.os == 'macOS' && matrix.arch == 'arm64'
+
+      - name: Set up CIBW environment
+        run: |
+          echo "Boost_ROOT=${{ steps.install-boost.outputs.BOOST_ROOT }}" >> $GITHUB_ENV
+          echo "CANTERA_TEST_DIR=${{ steps.download-test-files.outputs.test-root }}" >> $GITHUB_ENV
+          echo "CIBW_BUILD=cp*-*" >> $GITHUB_ENV
+        if: runner.os == 'macOS'
+
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.19.1
+        uses: pypa/cibuildwheel@v2.19.2
         env:
-          CIBW_ENVIRONMENT: BOOST_INCLUDE="$(brew --prefix)/include" RUNNER_TEMP=${{ runner.temp }} CT_SKIP_SLOW=1 CYTHON_FORCE_REGEN=${{ matrix.py == '38' && '1' || '0' }}
-          CIBW_BUILD: cp${{ matrix.py }}-*
-          CIBW_TEST_COMMAND: pytest -vv --durations=100 ${RUNNER_TEMP}/test/python
-          CIBW_BEFORE_TEST: |
-            curl -sL "https://github.com/cantera/cantera/archive/${{ needs.post-pending-status.outputs.incoming-sha }}.tar.gz" -o ${{ runner.temp }}/cantera.tar.gz && tar -xzf ${{ runner.temp }}/cantera.tar.gz --strip-components=1 -C ${{ runner.temp }} "cantera-${{ needs.post-pending-status.outputs.incoming-sha }}/test"
+          CIBW_ENVIRONMENT_LINUX: CT_SKIP_SLOW=1 CANTERA_TEST_DIR=/host/${{ steps.download-test-files.outputs.test-root }}
+          CIBW_ENVIRONMENT_WINDOWS: CT_SKIP_SLOW=1 CMAKE_BUILD_PARALLEL_LEVEL=4
+          CIBW_ENVIRONMNET_MACOS: CT_SKIP_SLOW=1
+          CIBW_BUILD: ${{ env.CIBW_BUILD }}
+          CIBW_ARCHS: ${{ matrix.arch }}
       - name: Archive the built wheels
         uses: actions/upload-artifact@v4
         with:
           path: ./wheelhouse/*.whl
-          name: cibw-wheels-macos-${{ strategy.job-index }}
+          name: cibw-wheels-${{ runner.os }}-${{ strategy.job-index }}
 
   publish-files-to-pypi:
     name: Publish distribution files to PyPI
@@ -320,9 +300,7 @@ jobs:
       job-status: ${{ job.status }}
     needs:
       - "sdist"
-      - "linux-wheel"
-      - "windows-wheel"
-      - "macos-wheel"
+      - "build-wheels"
     if: github.event.inputs.upload == 'true'
     permissions:
       id-token: write
@@ -343,9 +321,7 @@ jobs:
     needs:
       - "post-pending-status"
       - "sdist"
-      - "linux-wheel"
-      - "windows-wheel"
-      - "macos-wheel"
+      - "build-wheels"
       - "publish-files-to-pypi"
     if: always()
     steps:
@@ -355,9 +331,7 @@ jobs:
           import os
           statuses = {
               "sdist": "${{needs.sdist.outputs.job-status}}",
-              "linux": "${{needs.linux-wheel.outputs.job-status}}",
-              "windows": "${{needs.windows-wheel.outputs.job-status}}",
-              "macos": "${{needs.macos-wheel.outputs.job-status}}",
+              "wheels": "${{needs.build-wheels.outputs.job-status}}",
               "publish": "${{needs.publish-files-to-pypi.outputs.job-status}}",
           }
           # This is a deliberate comparison to the empty string.