From 02b3e32d328159793245244e05f3a76f3d0c12ce Mon Sep 17 00:00:00 2001
From: Oliver Holworthy <oholworthy@nvidia.com>
Date: Tue, 2 May 2023 12:26:24 +0100
Subject: [PATCH] Update packages workflow, separating PyPI from conda build
 (#300)

* Update packages workflow, separating PyPI from conda

* Update workflow name in docs preview

* Add build-docs job as first step in Packages workflow

* Add docs build to preview workflow removing Packages workflow dep

* Update `WF_ID` env var

* Add branch to docs preview pr workflow

* ADd workflow-id input

* Use the run_id from the github context

* Remove workflow-id input

* Revert changes to docs-preview-pr.yaml

* Change build-docs job back to match original version

---------

Co-authored-by: Karl Higley <karlb@nvidia.com>
---
 .github/workflows/docs-preview-pr.yaml        |  2 +-
 .../{cpu-packages.yml => packages.yaml}       | 69 +++++++++++--------
 2 files changed, 42 insertions(+), 29 deletions(-)
 rename .github/workflows/{cpu-packages.yml => packages.yaml} (74%)

diff --git a/.github/workflows/docs-preview-pr.yaml b/.github/workflows/docs-preview-pr.yaml
index 4986e52c6..019107cc1 100644
--- a/.github/workflows/docs-preview-pr.yaml
+++ b/.github/workflows/docs-preview-pr.yaml
@@ -2,7 +2,7 @@ name: docs-preview-pr
 
 on:
   workflow_run:
-    workflows: ["Build Core Packages (CPU)"]
+    workflows: ["Packages"]
     types: [completed]
 
 env:
diff --git a/.github/workflows/cpu-packages.yml b/.github/workflows/packages.yaml
similarity index 74%
rename from .github/workflows/cpu-packages.yml
rename to .github/workflows/packages.yaml
index f7e0a06a4..415bd9151 100644
--- a/.github/workflows/cpu-packages.yml
+++ b/.github/workflows/packages.yaml
@@ -1,4 +1,4 @@
-name: Build Core Packages (CPU)
+name: Packages
 
 on:
   workflow_dispatch:
@@ -48,28 +48,18 @@ jobs:
           name: pr
           path: pr/
 
-  build-packages:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        python-version: [3.8]
-        os: [ubuntu-latest]
-
+  build-pypi:
+    name: Build PyPI Package
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install Ubuntu packages
-        run: |
-          sudo apt-get update -y
-          sudo apt-get install -y protobuf-compiler
+          python-version: 3.9
       - name: Install and upgrade python packages
         run: |
-          python -m pip install --upgrade pip setuptools==59.4.0 wheel tox
+          python -m pip install --upgrade pip setuptools==59.4.0 wheel
       - name: Generate package for pypi
         run: |
           python setup.py sdist
@@ -78,32 +68,45 @@ jobs:
         with:
           name: dist
           path: dist
+
+  build-conda:
+    name: Build Conda Package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
       - uses: conda-incubator/setup-miniconda@v2
         with:
           auto-update-conda: true
       - name: Generate package for conda
         id: conda_build
         run: |
-          conda update conda
+          echo "conda pkgs dir $CONDA_PKGS_DIRS"
           conda install -c conda-forge mamba
-          mamba install -c conda-forge conda-build boa
-          conda mambabuild . -c defaults -c conda-forge -c numba -c rapidsai --output-folder ./conda_packages
-          export CONDA_PACKAGE=$(find ./conda_packages/ -name merlin-core*.tar.bz2)
-          echo "conda_package : $CONDA_PACKAGE"
-          echo "conda_package=$CONDA_PACKAGE" >> $GITHUB_OUTPUT
+          mamba install -c conda-forge conda-build boa conda-verify
+          conda mambabuild . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages
+          conda_package=$(find ./conda_packages/ -name "*.tar.bz2")
+          export CONDA_PACKAGE=$conda_package
+          echo "conda_package : $conda_package"
+          echo "conda_package=$conda_package" >> "$GITHUB_OUTPUT"
       - name: Upload conda artifacts to github
         uses: actions/upload-artifact@v3
         with:
           name: conda
           path: ${{ steps.conda_build.outputs.conda_package }}
 
-  release:
-    name: Release
+  release-pypi:
+    name: Release PyPI Package
     runs-on: ubuntu-latest
     if: "startsWith(github.ref, 'refs/tags/')"
-    needs: [build-packages]
+    needs: [build-pypi]
     steps:
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v3
         with:
           name: dist
           path: dist
@@ -115,13 +118,23 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: 3.9
-      - name: Push to PyPi
+      - name: Push to PyPI
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
           pip install --upgrade wheel pip setuptools twine
           twine upload dist/*
+
+  release-conda:
+    name: Release Conda Package
+    runs-on: ubuntu-latest
+    if: "startsWith(github.ref, 'refs/tags/')"
+    needs: [build-conda]
+    steps:
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
       - uses: actions/download-artifact@v3
         with:
           name: conda