[enh] Conda recipe update and CI OS version pinning (uxlfoundation#1992)

* Conda recipe update

* Remove compiler_c and icx-cl

* Add numpy matrix for conda-recipe CI

* Remove python req from build env for conda-recipe CI

* Reorder and extend testing deps

* Change numpy testing matrix

* Remove daal4py/sklearn from test files

* Update DPC compiler details for conda-build in INSTALL

* Update CI matrix and add comments

* Remove catboost; corrections for run_test.bat

* Add git tag as package version

* Add note about VS versions

* Change variable in run_test.bat

* Add recipe versioning comment

* Reorder MPI test command

* Remove extra dpcpp installation in CI

.ci/pipeline/build-and-test-win.yml

@@ -51,7 +51,7 @@ steps:
   - script: |
       call activate CB
       cd ..
-      call s\conda-recipe\run_test.bat s
+      call s\conda-recipe\run_test.bat s\
     displayName: 'Sklearnex testing'
   - script: |
       call activate CB

.ci/pipeline/ci.yml

@@ -62,6 +62,55 @@ jobs:
       pip install isort==${ISORT_VERSION} black[jupyter]==${BLACK_VERSION}
       isort --profile black --check . && black --check .
     displayName: 'Linting'
+- job: LinuxCondaRecipe
+  dependsOn: Lint
+  timeoutInMinutes: 120
+  strategy:
+    matrix:
+      Python3.12:
+        PYTHON_VERSION: '3.12'
+        NUMPY_VERSION: '2.1'
+  pool:
+    vmImage: 'ubuntu-22.04'
+  steps:
+  - script: |
+      conda config --add channels conda-forge
+      conda config --set channel_priority strict
+      conda update -y -q --all
+    displayName: "Conda update"
+  - script: |
+      conda create -y -q -n build-env conda-build conda-verify
+    displayName: "Conda create"
+  - script: |
+      . /usr/share/miniconda/etc/profile.d/conda.sh
+      conda activate build-env
+      conda build . --python $(PYTHON_VERSION) --numpy $(NUMPY_VERSION)
+    displayName: "Conda build and test"
+- job: WindowsCondaRecipe
+  dependsOn: Lint
+  timeoutInMinutes: 120
+  strategy:
+    matrix:
+      Python3.12:
+        PYTHON_VERSION: '3.12'
+        NUMPY_VERSION: '2.1'
+  pool:
+    vmImage: 'windows-2022'
+  steps:
+  - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+    displayName: Add conda to PATH
+  - script: |
+      conda config --add channels conda-forge
+      conda config --set channel_priority strict
+      conda update -y -q --all
+    displayName: "Conda update"
+  - script: |
+      conda create -y -q -n build-env conda-build conda-verify
+    displayName: "Conda create"
+  - script: |
+      call activate build-env
+      conda build . --python $(PYTHON_VERSION) --numpy $(NUMPY_VERSION)
+    displayName: "Conda build and test"
 - job: LinuxCondaEnv
   dependsOn: Lint
   timeoutInMinutes: 120
@@ -113,6 +162,6 @@ jobs:
         PYTHON_VERSION: '3.12'
         SKLEARN_VERSION: '1.5'
   pool:
-    vmImage: 'windows-latest'
+    vmImage: 'windows-2022'
   steps:
   - template: build-and-test-win.yml

.ci/pipeline/nightly.yml

@@ -34,7 +34,7 @@ variables:
 jobs:
 - job: Coverity
   pool:
-    vmImage: 'ubuntu-latest'
+    vmImage: 'ubuntu-22.04'
   steps:
   - script: |
       cd $(Agent.BuildDirectory)
@@ -58,7 +58,7 @@ jobs:
 - job: Jupyter
   timeoutInMinutes: 0
   pool:
-    vmImage: 'ubuntu-latest'
+    vmImage: 'ubuntu-22.04'
   steps:
   - script: |
       conda config --append channels conda-forge
@@ -110,6 +110,6 @@ jobs:
         PYTHON_VERSION: '3.11'
         SKLEARN_VERSION: 'main'
   pool:
-    vmImage: 'windows-latest'
+    vmImage: 'windows-2022'
   steps:
   - template: build-and-test-win.yml

.ci/scripts/gen_release_jobs.py

@@ -24,7 +24,9 @@
 
 CHANNELS = args.channels
 PYTHON_VERSIONS = ["3.9", "3.10", "3.11"]
-SYSTEMS = ["ubuntu-latest", "windows-latest"]
+# image versions are pinned to exact number instead of "latest"
+# to avoid unexpected failures when images are updated
+SYSTEMS = ["ubuntu-22.04", "windows-2022"]
 ACTIVATE = {
     "ubuntu-latest": "conda activate",
     "windows-latest": "call activate",

.github/workflows/ci.yml

@@ -245,7 +245,7 @@ jobs:
           call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }}
           set PYTHON=python
           cd ..
-          call scikit-learn-intelex\conda-recipe\run_test.bat scikit-learn-intelex
+          call scikit-learn-intelex\conda-recipe\run_test.bat scikit-learn-intelex\
       - name: Sklearn testing
         shell: cmd
         run: |

INSTALL.md

@@ -30,6 +30,9 @@ To install Intel(R) Extension for Scikit-learn*, use one of the following scenar
   - [Prerequisites](#prerequisites)
   - [Configure the Build with Environment Variables](#configure-the-build-with-environment-variables)
   - [Build Intel(R) Extension for Scikit-learn](#build-intelr-extension-for-scikit-learn)
+- [Build from Sources with `conda-build`](#build-from-sources-with-conda-build)
+  - [Prerequisites for `conda-build`](#prerequisites-for-conda-build)
+  - [Build Intel(R) Extension for Scikit-learn with `conda-build`](#build-intelr-extension-for-scikit-learn-with-conda-build)
 - [Next Steps](#next-steps)
 
 > **_NOTE:_** Intel(R) Extension for Scikit-learn* is also available as a part of [Intel® AI Tools](https://www.intel.com/content/www/us/en/developer/tools/oneapi/ai-analytics-toolkit.html). If you already have it installed, you do not need to separately install the extension.
@@ -212,6 +215,30 @@ back to the project source-code directory. That way, you can edit the source cod
 without reinstalling the package after a small change.
 * `--single-version-externally-managed` is an option for Python packages instructing the setup tools module to create a package the host's package manager can easily manage.
 
+## Build from Sources with `conda-build`
+
+Intel(R) Extension for Scikit-learn* is easily built from the sources using only one command and `conda-build` utility. 
+
+### Prerequisites for `conda-build`
+
+* any `conda` distribution (`miniforge` is recommended)
+* `conda-build` and `conda-verify` installed in a conda environment
+* (Windows only) Microsoft Visual Studio*
+* (optional) Intel(R) oneAPI DPC++/C++ Compiler
+
+`conda-build` config requires **2022** version of Microsoft Visual Studio* by default, you can specify another version in `conda-recipe/conda_build_config.yaml` if needed.
+
+In order to enable DPC++ interfaces support on Windows, you need to set `DPCPPROOT` environment variable pointing to DPC++/C++ Compiler distribution.
+Conda-forge distribution of DPC++ compiler is used by default on Linux, but you still can set your own distribution via `DPCPPROOT` variable.
+
+### Build Intel(R) Extension for Scikit-learn with `conda-build`
+
+Create and verify `scikit-learn-intelex` conda package with next command executed from root of sklearnex repo:
+
+```bash
+conda build .
+```
+
 ## Next Steps
 
 - [Learn what patching is and how to patch scikit-learn](https://intel.github.io/scikit-learn-intelex/latest/what-is-patching.html)

conda-recipe/bld.bat

@@ -15,21 +15,19 @@ rem See the License for the specific language governing permissions and
 rem limitations under the License.
 rem ============================================================================
 
-IF DEFINED PKG_VERSION (set DAAL4PY_VERSION=%PKG_VERSION%)
-
-set MPIROOT=%PREFIX%\Library
-
+IF NOT DEFINED PYTHON (set "PYTHON=python")
+IF DEFINED PKG_VERSION (set SKLEARNEX_VERSION=%PKG_VERSION%)
 IF NOT DEFINED DALROOT (set DALROOT=%PREFIX%)
+IF NOT DEFINED MPIROOT IF "%NO_DIST%"=="" (set MPIROOT=%PREFIX%\Library)
 
-set "BUILD_ARGS="
+rem reset preferred compilers to avoid usage of icx/icpx by default in all cases
+set CC=cl.exe
+set CXX=cl.exe
 
+rem source compiler if DPCPPROOT is set outside of conda-build
 IF DEFINED DPCPPROOT (
     echo "Sourcing DPCPPROOT"
     call "%DPCPPROOT%\env\vars.bat"
 )
 
-set PATH=%PATH%;%PREFIX%\Library\bin\libfabric
-
-%PYTHON% setup.py build %BUILD_ARGS%
-IF %ERRORLEVEL% neq 0 EXIT /b %ERRORLEVEL%
 %PYTHON% setup.py install --single-version-externally-managed --record record.txt

conda-recipe/build.sh

@@ -15,28 +15,29 @@
 # limitations under the License.
 #===============================================================================
 
-if [ "$PY3K" == "1" ]; then
-    ARGS=" --single-version-externally-managed --record=record.txt"
-else
-    ARGS="--old-and-unmanageable"
+if [ -z "${PYTHON}" ]; then
+    export PYTHON=python
 fi
 
-# if dpc++ vars path is specified
-if [ ! -z "${DPCPPROOT}" ]; then
-    source ${DPCPPROOT}/env/vars.sh
+if [ ! -z "${PKG_VERSION}" ]; then
+    export SKLEARNEX_VERSION=$PKG_VERSION
 fi
 
 if [ -z "${DALROOT}" ]; then
     export DALROOT=${PREFIX}
 fi
 
-if [ "$(uname)" == "Darwin" ]; then
-    export CC=gcc
-    export CXX=g++
+if [ -z "${MPIROOT}" ] && [ -z "${NO_DIST}" ]; then
+    export MPIROOT=${PREFIX}
 fi
-
-if [ ! -z "${PKG_VERSION}" ]; then
-    export DAAL4PY_VERSION=$PKG_VERSION
+# reset preferred compilers to avoid usage of icx/icpx by default in all cases
+if [ ! -z "${CC_FOR_BUILD}" ] && [ ! -z "${CXX_FOR_BUILD}" ]; then
+    export CC=$CC_FOR_BUILD
+    export CXX=$CXX_FOR_BUILD
 fi
-export MPIROOT=${PREFIX}
-${PYTHON} setup.py install $ARGS
+# source compiler if DPCPPROOT is set outside of conda-build
+if [ ! -z "${DPCPPROOT}" ]; then
+    source ${DPCPPROOT}/env/vars.sh
+fi
+
+${PYTHON} setup.py install --single-version-externally-managed --record record.txt

conda-recipe/conda_build_config.yaml

@@ -14,5 +14,7 @@
 # limitations under the License.
 #===============================================================================
 
-numpy:
-  - 1.19
+c_compiler:    # [win]
+- vs2022       # [win]
+cxx_compiler:  # [win]
+- vs2022       # [win]