[CI] upgrade GitHub actions to be more robust and upgrade dpnp/dpctl …

…in CI (uxlfoundation#2147) * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update describe_system.sh * Update ci.yml * Update ci.yml * Update activate_components.sh * Update activate_components.bat * Update ci.yml
samir-nasibli · Nov 6, 2024 · 00482c2 · 00482c2
1 parent c39ca54
commit 00482c2
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 23 deletions.
diff --git a/.ci/scripts/describe_system.sh b/.ci/scripts/describe_system.sh
@@ -57,3 +57,8 @@ if [ -x "$(command -v icpx)" ]; then
     echo "ICPX:"
     icpx --version
 fi
+echo
+# SYCL devices
+if [ -x "$(command -v sycl-ls)" ]; then
+    sycl-ls
+fi
diff --git a/.github/scripts/activate_components.bat b/.github/scripts/activate_components.bat
@@ -19,9 +19,7 @@ rem %1 - dpcpp activate flag
 
 rem prepare vc
 call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall" x64
-rem prepare icx only if no parameter is given.
-if "%1"=="" call .\oneapi\compiler\latest\env\vars.bat
-rem prepare tbb
-call .\oneapi\tbb\latest\env\vars.bat
+rem prepare only TBB from oneAPI if a parameter is given.
+IF "%1"=="" (call .\oneapi\setvars.bat) ELSE (call .\oneapi\tbb\latest\env\vars.bat)
 rem prepare oneDAL
 call .\__release_win\daal\latest\env\vars.bat
diff --git a/.github/scripts/activate_components.sh b/.github/scripts/activate_components.sh
@@ -15,7 +15,10 @@
 # limitations under the License.
 #===============================================================================
 
-# if any parameter is given, then do not source the compiler
-if [ $# -eq 0 ]; then source /opt/intel/oneapi/compiler/latest/env/vars.sh; fi # prepare icpx
-source /opt/intel/oneapi/tbb/latest/env/vars.sh # prepare tbb
+# if any parameter is given then only source TBB from the oneAPI install
+if [ $# -eq 0 ]; then
+  source /opt/intel/oneapi/setvars.sh
+else
+  source /opt/intel/oneapi/tbb/latest/env/vars.sh # prepare tbb
+fi
 source ./__release_lnx/daal/latest/env/vars.sh # prepare oneDAL
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -28,8 +28,8 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  DPCTL_VERSION: 0.17.0
-  DPNP_VERSION: 0.15.0
+  DPCTL_VERSION: 0.18.1
+  DPNP_VERSION: 0.16.0
   DPCTL_PY_VERSIONS: '3.9\|3.11'
 
 jobs:
@@ -51,11 +51,6 @@ jobs:
     steps:
       - name: Checkout Scikit-learn-intelex
         uses: actions/checkout@v4
-      - name: Checkout oneDAL
-        uses: actions/checkout@v4
-        with:
-          repository: oneapi-src/oneDAL
-          path: oneDAL
       - name: Install Python
         uses: actions/setup-python@v5
         with:
@@ -65,7 +60,8 @@ jobs:
         run: |
           OTHER_REPO="oneapi-src/oneDAL"
           WF_NAME="Nightly-build"
-          RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId --status success --jq .[0].databaseId`
+          JQ_QUERY='map(select(.event == "workflow_dispatch" or .event == "schedule")) | .[0].databaseId'
+          RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId,event --status success --jq "${JQ_QUERY}"`
           echo "Detected latest run id of ${RUN_ID} for workflow ${WF_NAME}"
           echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT"
         env:
@@ -78,6 +74,14 @@ jobs:
           repository: oneapi-src/oneDAL
           run-id: ${{ steps.get-run-id.outputs.run-id }}
           path: ./__release_lnx
+      - name: Download oneDAL environment artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: oneDAL_env
+          github-token: ${{ github.token }}
+          repository: oneapi-src/oneDAL
+          run-id: ${{ steps.get-run-id.outputs.run-id }}
+          path: .ci/env
       - name: Set Environment Variables
         id: set-env
         run: |
@@ -92,9 +96,11 @@ jobs:
       - name: dpcpp installation
         run: |
           # This CI system yields oneAPI dependencies from the oneDAL repository
-          bash ./oneDAL/.ci/env/apt.sh dpcpp
+          bash .ci/env/apt.sh dpcpp
       - name: describe system
-        run: bash .ci/scripts/describe_system.sh
+        run: |
+          source /opt/intel/oneapi/setvars.sh
+          bash .ci/scripts/describe_system.sh
       - name: Install develop requirements
         run: |
           python -m venv venv
@@ -125,13 +131,15 @@ jobs:
         run: |
           source venv/bin/activate
           source .github/scripts/activate_components.sh
-          bash .ci/scripts/run_sklearn_tests.sh
+          if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then export CPU=cpu; fi
+          bash .ci/scripts/run_sklearn_tests.sh $CPU
       - name: Sklearn testing [preview]
         run: |
           source venv/bin/activate
           source .github/scripts/activate_components.sh
+          if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then export CPU=cpu; fi
           export SKLEARNEX_PREVIEW='YES'
-          bash .ci/scripts/run_sklearn_tests.sh
+          bash .ci/scripts/run_sklearn_tests.sh $CPU
 
   sklearn_win:
     strategy:
@@ -161,7 +169,8 @@ jobs:
         run: |
           OTHER_REPO="oneapi-src/oneDAL"
           WF_NAME="Nightly-build"
-          RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId --status success --jq .[0].databaseId`
+          JQ_QUERY='map(select(.event == "workflow_dispatch" or .event == "schedule")) | .[0].databaseId'
+          RUN_ID=`gh run --repo ${OTHER_REPO} list --workflow "${WF_NAME}" --json databaseId,event --status success --jq "${JQ_QUERY}"`
           echo "Detected latest run id of ${RUN_ID} for workflow ${WF_NAME}"
           echo "run-id=${RUN_ID}" >> "$GITHUB_OUTPUT"
         env:
@@ -224,7 +233,6 @@ jobs:
         run: |
           call .\venv\Scripts\activate.bat
           call .\oneapi\setvars.bat
-          call .\oneapi\compiler\latest\bin\sycl-ls.exe
           bash .ci/scripts/describe_system.sh
       - name: Build daal4py/sklearnex
         shell: cmd
@@ -258,11 +266,13 @@ jobs:
         run: |
           call .\venv\Scripts\activate.bat
           call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }}
-          bash .ci/scripts/run_sklearn_tests.sh
+          if "${{ steps.set-env.outputs.DPCFLAG }}"=="" set CPU=cpu
+          bash .ci/scripts/run_sklearn_tests.sh %CPU%
       - name: Sklearn testing [preview]
         shell: cmd
         run: |
           call .\venv\Scripts\activate.bat
           call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }}
+          if "${{ steps.set-env.outputs.DPCFLAG }}"=="" set CPU=cpu
           set SKLEARNEX_PREVIEW=YES
-          bash .ci/scripts/run_sklearn_tests.sh
+          bash .ci/scripts/run_sklearn_tests.sh %CPU%