Skip to content

Commit

Permalink
Use cibuildwheel to build binary distributions for PyPI (#1754)
Browse files Browse the repository at this point in the history
* Use `cibuildwheel` to build binary distributions for PyPI

* Fix env variable formatting for `cibuildwheel` action

* Split running CPU tests and building packages into two separate actions

* Rename the CPU and GPU test actions (for clarity)

* Stop building packages on every PR
  • Loading branch information
karlhigley authored and oliverholworthy committed Feb 3, 2023
1 parent 7a87ffd commit 1347885
Show file tree
Hide file tree
Showing 3 changed files with 201 additions and 135 deletions.
135 changes: 0 additions & 135 deletions .github/workflows/cpu-ci.yml

This file was deleted.

132 changes: 132 additions & 0 deletions .github/workflows/cpu-packages.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
name: Build NVTabular Packages (CPU)

on:
workflow_dispatch:
push:
branches: [main]
tags:
- v*

jobs:
build:
runs-on: ${{ matrix.os }}
strategy:
matrix:
python-version: [3.8]
os: [ubuntu-latest]

steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install Ubuntu packages
run: |
sudo apt-get update -y
sudo apt-get install -y protobuf-compiler
- name: Install and upgrade python packages
run: |
python -m pip install --upgrade pip setuptools==59.4.0 wheel tox pybind11
python -m pip uninstall protobuf -y
python -m pip install --no-binary=protobuf protobuf
- name: Build source package for PyPI
run: |
python setup.py sdist
- name: Check distribution is valid
run: |
./ci/check_dist.sh
- name: Build wheels for PyPI
uses: pypa/cibuildwheel@v2.12.0
env:
CIBW_SKIP: cp36-* cp37-* pp* # Don't build wheels for 3.6, 3.7, or PyPy
with:
output-dir: dist
- name: Upload PyPI artifacts to Github
uses: actions/upload-artifact@v3
with:
name: dist
path: dist
- name: Generate package for conda
id: conda_build
run: |
conda update conda
conda install conda-build pybind11
conda build --python ${{ matrix.python-version }} . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages
export CONDA_PACKAGE=$(conda build --python ${{ matrix.python-version }} . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages --output)
echo "conda_package : $CONDA_PACKAGE"
echo "conda_package=$CONDA_PACKAGE" >> $GITHUB_OUTPUT
- name: Upload conda artifacts to github
uses: actions/upload-artifact@v3
with:
name: conda
path: ${{ steps.conda_build.outputs.conda_package }}

# Build docs, treat warnings as errors
- name: Building docs
run: |
tox -e docs
- name: Upload HTML
uses: actions/upload-artifact@v3
with:
name: html-build-artifact
path: docs/build/html
if-no-files-found: error
retention-days: 1
- name: Store PR information
run: |
mkdir ./pr
echo ${{ github.event.number }} > ./pr/pr.txt
echo ${{ github.event.pull_request.merged }} > ./pr/merged.txt
echo ${{ github.event.action }} > ./pr/action.txt
- name: Upload PR information
uses: actions/upload-artifact@v3
with:
name: pr
path: pr/

release:
name: Release
runs-on: ubuntu-latest
if: "startsWith(github.ref, 'refs/tags/')"
needs: [build]
steps:
- uses: actions/download-artifact@v3
with:
name: dist
path: dist
- name: Create GitHub Release
uses: fnkr/github-action-ghr@v1.3
env:
GHR_PATH: ./dist
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Push to PyPi
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
pip install --upgrade wheel pip setuptools twine
twine upload dist/*
- uses: actions/download-artifact@v2
with:
name: conda
path: conda
- uses: conda-incubator/setup-miniconda@v2
with:
auto-update-conda: true
- name: Install conda dependencies
shell: bash -l {0}
run: |
conda install -y anaconda-client conda-build
- name: Push to anaconda
shell: bash -l {0}
env:
ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
run: |
anaconda -t $ANACONDA_TOKEN upload -u nvidia conda/*.tar.bz2
69 changes: 69 additions & 0 deletions .github/workflows/cpu-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
name: CPU Tests

on:
workflow_dispatch:
push:
branches: [main]
tags:
- v*
pull_request:
branches: [main]

jobs:
cpu-tests:
runs-on: ${{ matrix.os }}
strategy:
matrix:
python-version: [3.8, 3.9]
os: [ubuntu-latest]

steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install Ubuntu packages
run: |
sudo apt-get update -y
sudo apt-get install -y protobuf-compiler
- name: Install and upgrade python packages
run: |
python -m pip install --upgrade pip setuptools==59.4.0 wheel tox pybind11
python -m pip uninstall protobuf -y
python -m pip install --no-binary=protobuf protobuf
- name: Run tests
run: |
ref_type=${{ github.ref_type }}
branch=main
if [[ $ref_type == "tag"* ]]
then
raw=$(git branch -r --contains ${{ github.ref_name }})
branch=${raw/origin\/}
fi
tox -e test-cpu -- $branch
# Build docs, treat warnings as errors
- name: Building docs
run: |
tox -e docs
- name: Upload HTML
uses: actions/upload-artifact@v3
with:
name: html-build-artifact
path: docs/build/html
if-no-files-found: error
retention-days: 1
- name: Store PR information
run: |
mkdir ./pr
echo ${{ github.event.number }} > ./pr/pr.txt
echo ${{ github.event.pull_request.merged }} > ./pr/merged.txt
echo ${{ github.event.action }} > ./pr/action.txt
- name: Upload PR information
uses: actions/upload-artifact@v3
with:
name: pr
path: pr/

0 comments on commit 1347885

Please sign in to comment.