BUG: GHA: Fix selection of Python versions for CUDA packages #358
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Package RTK with CUDA' | |
| on: [push,pull_request] | |
| env: | |
| itk-wheel-tag: 'v5.4.0' | |
| itk-python-package-tag: '45cfd02a22661731ca5b155f210cfa2f887d50e7' | |
| itk-python-package-org: 'SimonRit' | |
| itk-module-deps: "RTKConsortium/ITKCudaCommon@cae7c93c2f6d4c5b3cb5f6e5ce4a97686e626bf7" | |
| concurrency: | |
| group: '${{ github.workflow }}@${{ github.head_ref || github.run_id }}' | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| build-linux-cuda-python-packages: | |
| runs-on: self-hosted-linux | |
| strategy: | |
| max-parallel: 2 | |
| matrix: | |
| python3-minor-version: ${{ github.event_name == 'pull_request' && '["11"]' || fromJSON('["9","10","11"]') }} | |
| manylinux-platform: ["_2_28-x64","2014-x64"] | |
| cuda-version: ["116","121"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: 'Fetch build script' | |
| run: | | |
| IPP_DOWNLOAD_GIT_TAG=${{ env.itk-python-package-tag }} | |
| IPP_DOWNLOAD_ORG=${{ env.itk-python-package-org }} | |
| curl -L https://raw.githubusercontent.com/${IPP_DOWNLOAD_ORG:=InsightSoftwareConsortium}/ITKPythonPackage/${IPP_DOWNLOAD_GIT_TAG:=master}/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh -O | |
| chmod u+x dockcross-manylinux-download-cache-and-build-module-wheels.sh | |
| - name: 'Build 🐍 Python 📦 package' | |
| shell: bash | |
| run: | | |
| rm -rf dist | |
| export ITK_PACKAGE_VERSION=${{ env.itk-wheel-tag }} | |
| export ITKPYTHONPACKAGE_TAG=${{ env.itk-python-package-tag }} | |
| export ITKPYTHONPACKAGE_ORG=${{ env.itk-python-package-org }} | |
| export ITK_MODULE_PREQ=${{ env.itk-module-deps }} | |
| export CUDA_VERSION=${{ matrix.cuda-version }} | |
| export CUDA_VERSION_MAJOR=`(echo ${CUDA_VERSION} | cut -b1-2)` | |
| export CUDA_VERSION_MINOR=`(echo ${CUDA_VERSION} | cut -b3-)` | |
| CMAKE_OPTIONS="-DRTK_CUDA_VERSION=${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}" | |
| CMAKE_OPTIONS="${CMAKE_OPTIONS} -DRTK_BUILD_APPLICATIONS:BOOL=OFF" | |
| CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCUDAToolkit_ROOT=/usr/lib64/cuda${CUDA_VERSION}" | |
| CMAKE_OPTIONS="${CMAKE_OPTIONS} -DCMAKE_CUDA_COMPILER=/usr/lib64/cuda${CUDA_VERSION}/bin/nvcc" | |
| # The first two are not library paths but are included to be mounted in the | |
| # docker by dockcross-manylinux-build-module-wheels.sh | |
| export LD_LIBRARY_PATH="/home/srit/Downloads/cuda${CUDA_VERSION}" | |
| export LD_LIBRARY_PATH="/usr/lib64/libcuda.so.1:$LD_LIBRARY_PATH" | |
| export LD_LIBRARY_PATH="/home/srit/Downloads/cuda${CUDA_VERSION}/targets/x86_64-linux/lib:$LD_LIBRARY_PATH" | |
| if test -e ../../ITKPythonBuilds-linux-manylinux2014.tar.zst ; then | |
| mv ../../*zst . | |
| fi | |
| MANYLINUX_PLATFORM=${{ matrix.manylinux-platform }} | |
| if test ${MANYLINUX_PLATFORM} == "_2_28-x64" && test ${CUDA_VERSION} -lt 120; then | |
| export IMAGE_TAG=20230106-1aeaea0 | |
| CMAKE_OPTIONS="${CMAKE_OPTIONS} -DITK_USE_PYTHON_LIMITED_API=FALSE" | |
| fi | |
| CMAKE_OPTIONS=(--cmake_options "${CMAKE_OPTIONS}") | |
| echo "Manylinux platform ${MANYLINUX_PLATFORM}" | |
| rm -rf ITKPythonPackage | |
| export MANYLINUX_VERSION=`(echo ${MANYLINUX_PLATFORM} | cut -d '-' -f 1)` | |
| export TARGET_ARCH=`(echo ${MANYLINUX_PLATFORM} | cut -d '-' -f 2)` | |
| export ITK_MODULE_NO_CLEANUP=TRUE | |
| echo "Building for manylinux specialization ${MANYLINUX_VERSION} and target architecture ${TARGET_ARCH}" | |
| LIBCUDART=`basename /home/srit/Downloads/cuda${CUDA_VERSION}/targets/x86_64-linux/lib/libcudart.so.${CUDA_VERSION_MAJOR}*` | |
| LIBCUBLAS=`basename /home/srit/Downloads/cuda${CUDA_VERSION}/targets/x86_64-linux/lib/libcublas.so.${CUDA_VERSION_MAJOR}*` | |
| LIBCUBLASLT=`basename /home/srit/Downloads/cuda${CUDA_VERSION}/targets/x86_64-linux/lib/libcublasLt.so.${CUDA_VERSION_MAJOR}*` | |
| LIBCUFFT=`basename /home/srit/Downloads/cuda${CUDA_VERSION}/targets/x86_64-linux/lib/libcufft.so.$((${CUDA_VERSION_MAJOR}-1))*` | |
| sed -i "s/dependencies = \[/dependencies = [ \"itk-cudacommon-cuda${CUDA_VERSION} == 1.1.*\",/g" pyproject.toml | |
| sed -i "s/itk-rtk/itk-rtk-cuda${CUDA_VERSION}/g" pyproject.toml | |
| ./dockcross-manylinux-download-cache-and-build-module-wheels.sh "${CMAKE_OPTIONS[@]}" -x "libcuda.so.1;${LIBCUDART};${LIBCUBLAS};${LIBCUBLASLT};${LIBCUFFT}" cp3${{ matrix.python3-minor-version }} | |
| mv *zst ../.. | |
| - name: Validate build output | |
| shell: bash | |
| run: | | |
| /home/srit/miniconda3/envs/py310/bin/python -m pip install twine | |
| ls dist/ | |
| MANYLINUX_PLATFORM=${{ matrix.manylinux-platform }} | |
| MANYLINUX_VERSION=`(echo ${MANYLINUX_PLATFORM} | cut -d '-' -f 1)` | |
| TARGET_ARCH_NAME=`(echo ${MANYLINUX_PLATFORM} | cut -d '-' -f 2)` | |
| if [[ ${TARGET_ARCH_NAME} == "x64" ]]; then | |
| TARGET_ARCH_NAME="x86_64" # Match auditwheel naming convention | |
| fi | |
| WHEEL_PATTERN="dist/itk_*cp3${{ matrix.python3-minor-version }}*manylinux${MANYLINUX_VERSION}*${TARGET_ARCH_NAME}.whl" | |
| echo "Searching for wheels matching pattern ${WHEEL_PATTERN}" | |
| /home/srit/miniconda3/envs/py310/bin/python -m twine check ${WHEEL_PATTERN} | |
| - name: Publish Python package as GitHub Artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: LinuxWheel3${{ matrix.python3-minor-version }}${{ matrix.manylinux-platform }}-cuda${{ matrix.cuda-version }} | |
| path: dist/*.whl | |
| build-windows-cuda-python-packages: | |
| runs-on: self-hosted-windows | |
| strategy: | |
| max-parallel: 2 | |
| matrix: | |
| python3-minor-version: ${{ github.event_name == 'pull_request' && '["11"]' || fromJSON('["9","10","11"]') }} | |
| cuda-version: ["116","121"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| path: "im" | |
| - name: 'Reduce source path length' | |
| shell: bash | |
| run: | | |
| # Move ITKPythonBuilds archive to the checked-out source | |
| if test -f ../../im/ITKPythonBuilds-windows.zip; then | |
| mv ../../im/*.zip im | |
| fi | |
| rm -fr ../../im | |
| # Move checked-out source to a shorter path to avoid Windows path length issues | |
| mv im ../../ | |
| - name: 'Fetch build script' | |
| shell: pwsh | |
| run: | | |
| cd ../../im | |
| $ITKPYTHONPACKAGE_TAG = "${{ env.itk-python-package-tag }}" | |
| $ITKPYTHONPACKAGE_ORG = "${{ env.itk-python-package-org }}" | |
| $SCRIPT_UPSTREAM = "https://raw.githubusercontent.com/$ITKPYTHONPACKAGE_ORG/ITKPythonPackage/$ITKPYTHONPACKAGE_TAG/scripts/windows-download-cache-and-build-module-wheels.ps1" | |
| echo "Fetching $SCRIPT_UPSTREAM" | |
| (new-object net.webclient).DownloadString($SCRIPT_UPSTREAM) > windows-download-cache-and-build-module-wheels.ps1 | |
| - name: 'Build 🐍 Python 📦 package' | |
| shell: pwsh | |
| run: | | |
| if (Test-Path dist) { rm dist -r -fo } | |
| cd ../../im | |
| & "C:\Program Files\Microsoft Visual Studio\2022\Community\Common7\Tools\Launch-VsDevShell.ps1" -Arch amd64 -SkipAutomaticLocation | |
| $env:CC="cl.exe" | |
| $env:CXX="cl.exe" | |
| $env:ITK_PACKAGE_VERSION = "${{ env.itk-wheel-tag }}" | |
| $env:ITKPYTHONPACKAGE_TAG = "${{ env.itk-python-package-tag }}" | |
| $env:ITKPYTHONPACKAGE_ORG = "${{ env.itk-python-package-org }}" | |
| $env:ITK_MODULE_PREQ = "${{ env.itk-module-deps }}" | |
| $CUDA_VERSION = "${{ matrix.cuda-version }}" | |
| $CUDA_VERSION_MAJOR=$CUDA_VERSION.substring(0,2) | |
| $CUDA_VERSION_MINOR=$CUDA_VERSION.substring(2,$CUDA_VERSION.Length-2) | |
| $env:CUDA_PATH = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}" | |
| $env:CUDACXX = "$env:CUDA_PATH\bin\nvcc.exe" | |
| $env:Path = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}\bin;" + $env:Path | |
| $LIBCUDART= (Get-Item "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}\bin\cudart64*dll" ).Name | |
| $LIBCUBLAS= (Get-Item "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}\bin\cublas64*dll" ).Name | |
| $LIBCUBLASLT= (Get-Item "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}\bin\cublasLt64*dll" ).Name | |
| $LIBCUFFT= (Get-Item "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}\bin\cufft64*dll" ).Name | |
| ((Get-Content -Path pyproject.toml) -replace "itk-rtk","itk-rtk-cuda${CUDA_VERSION}") | Set-Content -Path pyproject.toml | |
| ((Get-Content -Path pyproject.toml) -replace "dependencies = \[","dependencies = [""itk-cudacommon-cuda${CUDA_VERSION} == 1.1.*"",") | Set-Content -Path pyproject.toml | |
| ./windows-download-cache-and-build-module-wheels.ps1 "${{ matrix.python3-minor-version }}" -setup_options "--lib-paths ""C:/Program\ Files/NVIDIA\ GPU\ Computing\ Toolkit/CUDA/v${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}/bin"" --exclude-libs ""nvcuda.dll;concrt140.dll;${LIBCUDART};${LIBCUBLAS};${LIBCUBLASLT};${LIBCUFFT}""" -cmake_options """-DRTK_CUDA_VERSION=${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}"" ""-DRTK_BUILD_APPLICATIONS:BOOL=OFF""" | |
| mkdir -p '${{ github.workspace }}\dist' | |
| cp 'dist\*.whl' '${{ github.workspace }}\dist' | |
| - name: Validate build output | |
| shell: pwsh | |
| run: | | |
| python -m pip install twine | |
| ls dist/ | |
| $WHEEL_PATTERN = "dist/itk_*cp3${{ matrix.python3-minor-version }}*win*.whl" | |
| echo "Searching for wheels matching pattern ${WHEEL_PATTERN}" | |
| python -m twine check ${WHEEL_PATTERN} | |
| - name: Publish Python package as GitHub Artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: WindowsWheel3${{ matrix.python3-minor-version }}-cuda${{ matrix.cuda-version }} | |
| path: dist/*.whl | |
| publish-python-packages-to-pypi: | |
| needs: | |
| - build-linux-cuda-python-packages | |
| - build-windows-cuda-python-packages | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Download Python Packages | |
| uses: actions/download-artifact@v4 | |
| - name: Prepare packages for upload | |
| run: | | |
| ls -R | |
| for d in */; do | |
| mv ${d}/*.whl . | |
| done | |
| mkdir dist | |
| mv *.whl dist/ | |
| ls dist | |
| - name: Publish 🐍 Python 📦 package to PyPI | |
| if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') | |
| uses: pypa/gh-action-pypi-publish@v1.5.1 | |
| with: | |
| skip_existing: true | |
| user: __token__ | |
| password: ${{ secrets.pypi_password }} |