From 66c58d37804510535e446a2def29aa48d7fad6a7 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Mon, 14 Oct 2024 10:15:54 +0100 Subject: [PATCH 01/16] Conda recipe update --- .ci/pipeline/build-and-test-win.yml | 2 +- .ci/pipeline/ci.yml | 53 ++++++++- .ci/pipeline/nightly.yml | 6 +- .ci/scripts/gen_release_jobs.py | 2 +- .github/workflows/ci.yml | 2 +- INSTALL.md | 24 ++++ conda-recipe/bld.bat | 16 ++- conda-recipe/build.sh | 31 +++--- conda-recipe/conda_build_config.yaml | 6 +- conda-recipe/meta.yaml | 161 ++++++++++++++------------- conda-recipe/run_test.bat | 30 ++--- conda-recipe/run_test.sh | 36 +++--- scripts/CMakeLists.txt | 4 + scripts/build_backend.py | 2 +- 14 files changed, 221 insertions(+), 154 deletions(-) diff --git a/.ci/pipeline/build-and-test-win.yml b/.ci/pipeline/build-and-test-win.yml index 0db4207670..c09beed84d 100644 --- a/.ci/pipeline/build-and-test-win.yml +++ b/.ci/pipeline/build-and-test-win.yml @@ -51,7 +51,7 @@ steps: - script: | call activate CB cd .. - call s\conda-recipe\run_test.bat s + call s\conda-recipe\run_test.bat s\ displayName: 'Sklearnex testing' - script: | call activate CB diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index bffbc08a0e..27e2a89547 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -57,6 +57,57 @@ jobs: pip install isort black[jupyter]==24.1.1 isort --profile black --check . && black --check . displayName: 'Linting' +- job: LinuxCondaRecipe + dependsOn: Lint + timeoutInMinutes: 120 + strategy: + matrix: + Python3.11: + PYTHON_VERSION: '3.11' + pool: + vmImage: 'ubuntu-22.04' + steps: + - script: | + bash .ci/scripts/install_dpcpp.sh + displayName: "dpcpp installation" + - script: | + conda config --add channels conda-forge + conda config --set channel_priority strict + conda update -y -q --all + displayName: "Conda update" + - script: | + conda create -y -q -n build-env python=$(PYTHON_VERSION) conda-build conda-verify + displayName: "Conda create" + - script: | + export DPCPPROOT=/opt/intel/oneapi/compiler/latest + . /usr/share/miniconda/etc/profile.d/conda.sh + conda activate build-env + conda build . --python $(PYTHON_VERSION) + displayName: "Conda build" +- job: WindowsCondaRecipe + dependsOn: Lint + timeoutInMinutes: 120 + strategy: + matrix: + Python3.11: + PYTHON_VERSION: '3.11' + pool: + vmImage: 'windows-2022' + steps: + - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" + displayName: Add conda to PATH + - script: | + conda config --add channels conda-forge + conda config --set channel_priority strict + conda update -y -q --all + displayName: "Conda update" + - script: | + conda create -y -q -n build-env python=$(PYTHON_VERSION) conda-build conda-verify + displayName: "Conda create" + - script: | + call activate build-env + conda build . --python $(PYTHON_VERSION) + displayName: "Conda build" - job: LinuxCondaEnv dependsOn: Lint timeoutInMinutes: 120 @@ -108,6 +159,6 @@ jobs: PYTHON_VERSION: '3.12' SKLEARN_VERSION: '1.5' pool: - vmImage: 'windows-latest' + vmImage: 'windows-2022' steps: - template: build-and-test-win.yml diff --git a/.ci/pipeline/nightly.yml b/.ci/pipeline/nightly.yml index b6f32c0ff4..998ccd934e 100644 --- a/.ci/pipeline/nightly.yml +++ b/.ci/pipeline/nightly.yml @@ -34,7 +34,7 @@ variables: jobs: - job: Coverity pool: - vmImage: 'ubuntu-latest' + vmImage: 'ubuntu-22.04' steps: - script: | cd $(Agent.BuildDirectory) @@ -58,7 +58,7 @@ jobs: - job: Jupyter timeoutInMinutes: 0 pool: - vmImage: 'ubuntu-latest' + vmImage: 'ubuntu-22.04' steps: - script: | conda config --append channels conda-forge @@ -110,6 +110,6 @@ jobs: PYTHON_VERSION: '3.11' SKLEARN_VERSION: 'main' pool: - vmImage: 'windows-latest' + vmImage: 'windows-2022' steps: - template: build-and-test-win.yml diff --git a/.ci/scripts/gen_release_jobs.py b/.ci/scripts/gen_release_jobs.py index f8de27f4f8..66bba66f6c 100644 --- a/.ci/scripts/gen_release_jobs.py +++ b/.ci/scripts/gen_release_jobs.py @@ -24,7 +24,7 @@ CHANNELS = args.channels PYTHON_VERSIONS = ["3.9", "3.10", "3.11"] -SYSTEMS = ["ubuntu-latest", "windows-latest"] +SYSTEMS = ["ubuntu-22.04", "windows-2022"] ACTIVATE = { "ubuntu-latest": "conda activate", "windows-latest": "call activate", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd5a059c5e..61d3687851 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -248,7 +248,7 @@ jobs: call .\.github\scripts\activate_components.bat ${{ env.DPCPP_VERSION }} ${{ env.TBB_VERSION }} ${{ steps.set-env.outputs.DPCFLAG }} set PYTHON=python cd .. - call scikit-learn-intelex\conda-recipe\run_test.bat scikit-learn-intelex + call scikit-learn-intelex\conda-recipe\run_test.bat scikit-learn-intelex\ - name: Sklearn testing shell: cmd run: | diff --git a/INSTALL.md b/INSTALL.md index 0dfc77691f..288790a7db 100755 --- a/INSTALL.md +++ b/INSTALL.md @@ -30,6 +30,9 @@ To install Intel(R) Extension for Scikit-learn*, use one of the following scenar - [Prerequisites](#prerequisites) - [Configure the Build with Environment Variables](#configure-the-build-with-environment-variables) - [Build Intel(R) Extension for Scikit-learn](#build-intelr-extension-for-scikit-learn) +- [Build from Sources with `conda-build`](#build-from-sources-with-conda-build) + - [Prerequisites for `conda-build`](#prerequisites-for-conda-build) + - [Build Intel(R) Extension for Scikit-learn with `conda-build`](#build-intelr-extension-for-scikit-learn-with-conda-build) - [Next Steps](#next-steps) > **_NOTE:_** Intel(R) Extension for Scikit-learn* is also available as a part of [IntelĀ® AI Tools](https://www.intel.com/content/www/us/en/developer/tools/oneapi/ai-analytics-toolkit.html). If you already have it installed, you do not need to separately install the extension. @@ -212,6 +215,27 @@ back to the project source-code directory. That way, you can edit the source cod without reinstalling the package after a small change. * `--single-version-externally-managed` is an option for Python packages instructing the setup tools module to create a package the host's package manager can easily manage. +## Build from Sources with `conda-build` + +Intel(R) Extension for Scikit-learn* is easily built from the sources using only one command and `conda-build` utility. + +### Prerequisites for `conda-build` + +* any `conda` distribution (`miniforge` is recommended) +* `conda-build` and `conda-verify` installed in a conda environment +* (Windows only) Microsoft Visual Studio* 2022 +* (optional) Intel(R) oneAPI DPC++/C++ Compiler + +In order to enable DPC++ interfaces support, you need to set `DPCPPROOT` environment variable pointing to DPC++/C++ Compiler distribution. + +### Build Intel(R) Extension for Scikit-learn with `conda-build` + +Create and verify `scikit-learn-intelex` conda package with next command executed from root of sklearnex repo: + +```bash +conda build . +``` + ## Next Steps - [Learn what patching is and how to patch scikit-learn](https://intel.github.io/scikit-learn-intelex/latest/what-is-patching.html) diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index e50a167864..f35aa1a49c 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -15,21 +15,19 @@ rem See the License for the specific language governing permissions and rem limitations under the License. rem ============================================================================ -IF DEFINED PKG_VERSION (set DAAL4PY_VERSION=%PKG_VERSION%) - -set MPIROOT=%PREFIX%\Library - +IF NOT DEFINED PYTHON (set PYTHON="python") +IF DEFINED PKG_VERSION (set SKLEARNEX_VERSION=%PKG_VERSION%) IF NOT DEFINED DALROOT (set DALROOT=%PREFIX%) +IF NOT DEFINED MPIROOT IF "%NO_DIST%"=="" (set MPIROOT=%PREFIX%\Library) -set "BUILD_ARGS=" +rem reset preferred compilers to avoid usage of icx/icpx by default in all cases +set CC=cl.exe +set CXX=cl.exe +rem source compiler if DPCPPROOT is set outside of conda-build IF DEFINED DPCPPROOT ( echo "Sourcing DPCPPROOT" call "%DPCPPROOT%\env\vars.bat" ) -set PATH=%PATH%;%PREFIX%\Library\bin\libfabric - -%PYTHON% setup.py build %BUILD_ARGS% -IF %ERRORLEVEL% neq 0 EXIT /b %ERRORLEVEL% %PYTHON% setup.py install --single-version-externally-managed --record record.txt diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index 5f8987deeb..b0b3136da7 100755 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -15,28 +15,29 @@ # limitations under the License. #=============================================================================== -if [ "$PY3K" == "1" ]; then - ARGS=" --single-version-externally-managed --record=record.txt" -else - ARGS="--old-and-unmanageable" +if [ -z "${PYTHON}" ]; then + export PYTHON=python fi -# if dpc++ vars path is specified -if [ ! -z "${DPCPPROOT}" ]; then - source ${DPCPPROOT}/env/vars.sh +if [ ! -z "${PKG_VERSION}" ]; then + export SKLEARNEX_VERSION=$PKG_VERSION fi if [ -z "${DALROOT}" ]; then export DALROOT=${PREFIX} fi -if [ "$(uname)" == "Darwin" ]; then - export CC=gcc - export CXX=g++ +if [ -z "${MPIROOT}" ] && [ -z "${NO_DIST}" ]; then + export MPIROOT=${PREFIX} fi - -if [ ! -z "${PKG_VERSION}" ]; then - export DAAL4PY_VERSION=$PKG_VERSION +# reset preferred compilers to avoid usage of icx/icpx by default in all cases +if [ ! -z "${CC_FOR_BUILD}" ] && [ ! -z "${CXX_FOR_BUILD}" ]; then + export CC=$CC_FOR_BUILD + export CXX=$CXX_FOR_BUILD fi -export MPIROOT=${PREFIX} -${PYTHON} setup.py install $ARGS +# source compiler if DPCPPROOT is set outside of conda-build +if [ ! -z "${DPCPPROOT}" ]; then + source ${DPCPPROOT}/env/vars.sh +fi + +${PYTHON} setup.py install --single-version-externally-managed --record record.txt diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index f52d3395a2..e86db7e307 100755 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -15,4 +15,8 @@ #=============================================================================== numpy: - - 1.19 + - 1.24 +c_compiler: # [win] +- vs2022 # [win] +cxx_compiler: # [win] +- vs2022 # [win] diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 98a6e537eb..b3280315fd 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -14,96 +14,101 @@ # limitations under the License. #=============================================================================== -{% set version = '2024.1' %} +# NB: this recipe should be synced between sklearnex repo and feedstocks + +{% set name = "scikit-learn-intelex" %} +{% set version = "2025.1" %} {% set buildnumber = 0 %} package: - name: daal4py - version: {{ version }} + name: {{ name|lower }} + version: {{ version }} +# Reminder: source should point to path for repo or github archive for feedstock +# source: +# url: https://github.com/intel/scikit-learn-intelex/archive/{{ version }}.tar.gz +# sha256: ... source: - path: .. + path: .. build: - number: {{ buildnumber }} - include_recipe: False - script_env: - - DPCPPROOT - - DALROOT - ignore_run_exports: - - python - - mpich + skip: true # [not (linux64 or win)] + number: {{ buildnumber }} + include_recipe: False + script_env: + - DPCPPROOT + - DALROOT + - NO_DIST=1 # [win] requirements: - build: - - {{ compiler('cxx') }} # [not osx] - - {{ compiler('c') }} # [not osx] - host: - - python - - setuptools - - numpy {{ numpy }} - - dal-devel ==2024.1.0 - - cython - - jinja2 - - mpich # [osx] - - impi-devel # [not osx] - - clang-tools - - pybind11 - - make - run: - - python - - dpcpp_cpp_rt ==2024.1.0 # [linux] - - dpcpp_cpp_rt ==2024.1.0 # [win or osx] - - dal ==2024.1.0 - ignore_run_exports: - - numpy + build: + - make # [linux] + - dpcpp_linux-64 # [linux64] + # - dpcpp_win-64 # [win] + - {{ compiler('cxx') }} # [linux64 or win] + - {{ compiler('c') }} # [linux64 or win] + # conda-forge feedstock specific + # - {{ stdlib("c") }} # [linux64 or win] + host: + - python + - setuptools + - cmake + - clang-format + - cython + - jinja2 + - pybind11 + - numpy {{ numpy }} + - impi-devel # [not win] + # dal-devel pinning depends on the recipe location (repo or feedstock) + - dal-devel + # - dal-devel =={{ version }} + run: + - python + - {{ pin_compatible('numpy') }} + - dpcpp-cpp-rt # [linux64] + # dal pinning depends on the recipe location (repo or feedstock) + - dal + # - dal =={{ version }} test: - requires: - - lightgbm # [not win] - - pandas - - scipy - - scikit-learn - - threadpoolctl - - xgboost # [not win] - - catboost - - pytest - - mpich # [osx] - - impi_rt # [not osx] - source_files: - - examples - - tests - - daal4py/sklearn - commands: - - python -c "import daal4py" - - mpirun -n 4 pytest --verbose -s tests/test*spmd*.py # [not win] - - mpiexec -localonly -n 4 pytest --verbose -s tests/test*spmd*.py # [win] - - pytest --pyargs --verbose -s tests - - pytest --pyargs daal4py/sklearn/ - - python tests/run_examples.py + requires: + - pandas + - scikit-learn + - xgboost + - lightgbm + - catboost + - pytest + - pyyaml + - impi_rt # [not win] + source_files: + - .ci + - examples + - tests + - daal4py/sklearn about: - about: - home: https://intelpython.github.io/daal4py/ - license: Apache-2.0 - license_file: - - LICENSE - - doc/third-party-programs.txt - summary: A convenient Python API to Intel (R) oneAPI Data Analytics Library - description: | - LEGAL NOTICE: Use of this software package is subject to the - software license agreement (as set forth above, in the license section of - the installed Conda package and/or the README file) and all notices, - disclaimers or license terms for third party or open source software - included in or with the software. -

- EULA: Apache-2.0 -

- dev_url: https://github.com/IntelPython/daal4py - doc_url: https://intelpython.github.io/daal4py + home: https://intel.github.io/scikit-learn-intelex + license: Apache-2.0 + license_file: + - LICENSE + - doc/third-party-programs-sklearnex.txt + summary: Intel(R) Extension for Scikit-learn* is a seamless way to speed up your Scikit-learn application. + description: | + LEGAL NOTICE: Use of this software package is subject to the + software license agreement (as set forth above, in the license section of + the installed Conda package and/or the README file) and all notices, + disclaimers or license terms for third party or open source software + included in or with the software. +

+ EULA: Apache-2.0 +

+ dev_url: https://github.com/intel/scikit-learn-intelex + doc_url: https://intel.github.io/scikit-learn-intelex extra: - recipe-maintainers: - # GitHub IDs for maintainers of the recipe. - - napetrov - - Alexsandruss + recipe-maintainers: + # GitHub IDs for maintainers of the recipe. + - napetrov + - Alexsandruss + - maria-Petrova + - ethanglaser diff --git a/conda-recipe/run_test.bat b/conda-recipe/run_test.bat index a65365cc5d..8557d0c9d7 100644 --- a/conda-recipe/run_test.bat +++ b/conda-recipe/run_test.bat @@ -15,34 +15,18 @@ rem See the License for the specific language governing permissions and rem limitations under the License. rem ============================================================================ -rem %1 - scikit-learn-intelex repo root +rem %1% - scikit-learn-intelex repo root (leave empty if it's PWD) -set MPIROOT=%PREFIX%\Library set exitcode=0 -IF DEFINED DPCPPROOT ( - echo "Sourcing DPCPPROOT" - call "%DPCPPROOT%\env\vars.bat" || set exitcode=1 - set "CC=dpcpp" - set "CXX=dpcpp" - dpcpp --version -) +IF NOT DEFINED PYTHON (set PYTHON="python") -IF DEFINED DALROOT ( - echo "Sourcing DALROOT" - call "%DALROOT%\env\vars.bat" || set exitcode=1 - echo "Finish sourcing DALROOT" -) +%PYTHON% -c "from sklearnex import patch_sklearn; patch_sklearn()" || set exitcode=1 -IF DEFINED TBBROOT ( - echo "Sourcing TBBROOT" - call "%TBBROOT%\env\vars.bat" || set exitcode=1 -) +%PYTHON% -m pytest --verbose -s %1%tests || set exitcode=1 -%PYTHON% -m pytest --verbose -s %1\tests || set exitcode=1 - -pytest --verbose --pyargs %1\daal4py\sklearn || set exitcode=1 +pytest --verbose --pyargs daal4py || set exitcode=1 pytest --verbose --pyargs sklearnex || set exitcode=1 -pytest --verbose --pyargs %1\onedal --deselect="onedal/common/tests/test_policy.py" || set exitcode=1 -pytest --verbose %1\.ci\scripts\test_global_patch.py || set exitcode=1 +pytest --verbose --pyargs onedal || set exitcode=1 +pytest --verbose %1%.ci\scripts\test_global_patch.py || set exitcode=1 EXIT /B %exitcode% diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh index bc8e557b9b..cc0cf4f6f2 100755 --- a/conda-recipe/run_test.sh +++ b/conda-recipe/run_test.sh @@ -15,55 +15,51 @@ # limitations under the License. #=============================================================================== -daal4py_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +sklex_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" count=3 while [[ count -ne 0 ]]; do - if [[ -d $daal4py_dir/daal4py/ && -d $daal4py_dir/tests/ && -d $daal4py_dir/examples/daal4py ]]; then + if [[ -d $sklex_root/.ci/ && -d $sklex_root/examples/ && -d $sklex_root/tests/ && -d $sklex_root/daal4py/sklearn ]]; then break fi - daal4py_dir="$( dirname "${daal4py_dir}" )" + sklex_root="$( dirname "${sklex_root}" )" count=$(($count - 1)) done -echo "daal4py_dir=$daal4py_dir" if [[ count -eq 0 ]]; then - echo "run_test.sh must be in daal4py repository" + echo "run_test.sh did not find the required testing directories" exit 1 fi -echo "Start testing ..." return_code=0 -python -c "import daal4py" -return_code=$(($return_code + $?)) +if [ -z "${PYTHON}" ]; then + export PYTHON=python +fi -echo "Pytest run of legacy unittest ..." -echo ${daal4py_dir} -pytest --verbose -s ${daal4py_dir}/tests +${PYTHON} -c "from sklearnex import patch_sklearn; patch_sklearn()" return_code=$(($return_code + $?)) echo "NO_DIST=$NO_DIST" if [[ ! $NO_DIST ]]; then - echo "MPI pytest run of legacy unittest ..." + echo "MPI pytest run" mpirun --version - mpirun -n 4 pytest --verbose -s ${daal4py_dir}/tests/test*spmd*.py + mpirun -n 4 pytest --verbose -s ${sklex_root}/tests/test*spmd*.py return_code=$(($return_code + $?)) fi -echo "Pytest of daal4py running ..." -pytest --verbose --pyargs ${daal4py_dir}/daal4py/sklearn +pytest --verbose -s ${sklex_root}/tests +return_code=$(($return_code + $?)) + +pytest --verbose --pyargs daal4py return_code=$(($return_code + $?)) -echo "Pytest of sklearnex running ..." pytest --verbose --pyargs sklearnex return_code=$(($return_code + $?)) -echo "Pytest of onedal running ..." -pytest --verbose --pyargs ${daal4py_dir}/onedal +pytest --verbose --pyargs onedal return_code=$(($return_code + $?)) -echo "Global patching test running ..." -pytest --verbose -s ${daal4py_dir}/.ci/scripts/test_global_patch.py +pytest --verbose -s ${sklex_root}/.ci/scripts/test_global_patch.py return_code=$(($return_code + $?)) exit $return_code diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 2ba4b7eb64..0b26468768 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -27,6 +27,10 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) if(WIN32) + # hint CMake to get python from PYTHON env. variable if defined + if(DEFINED ENV{PYTHON}) + set(PYTHON_EXECUTABLE $ENV{PYTHON}) + endif() set(SDL_FLAGS "-GS -DynamicBase") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /MD") elseif(UNIX) diff --git a/scripts/build_backend.py b/scripts/build_backend.py index a914adb807..8399429a89 100755 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -64,7 +64,7 @@ def custom_build_cmake_clib( if iface in ["dpc", "spmd_dpc"]: if IS_WIN: - cxx = "icx" + cxx = "icx-cl" else: cxx = "icpx" elif cxx is None: From 215648dd4653aac3f2f20406d3c618fd8cf30b07 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Mon, 14 Oct 2024 14:51:01 +0100 Subject: [PATCH 02/16] Remove compiler_c and icx-cl --- conda-recipe/meta.yaml | 1 - scripts/build_backend.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index b3280315fd..c962981c3a 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -46,7 +46,6 @@ requirements: - dpcpp_linux-64 # [linux64] # - dpcpp_win-64 # [win] - {{ compiler('cxx') }} # [linux64 or win] - - {{ compiler('c') }} # [linux64 or win] # conda-forge feedstock specific # - {{ stdlib("c") }} # [linux64 or win] host: diff --git a/scripts/build_backend.py b/scripts/build_backend.py index 8399429a89..a914adb807 100755 --- a/scripts/build_backend.py +++ b/scripts/build_backend.py @@ -64,7 +64,7 @@ def custom_build_cmake_clib( if iface in ["dpc", "spmd_dpc"]: if IS_WIN: - cxx = "icx-cl" + cxx = "icx" else: cxx = "icpx" elif cxx is None: From 973d712cf87f8391849a20f49a672e24da1020e8 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Mon, 14 Oct 2024 14:56:36 +0100 Subject: [PATCH 03/16] Add numpy matrix for conda-recipe CI --- .ci/pipeline/ci.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 27e2a89547..a0678d415b 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -62,8 +62,12 @@ jobs: timeoutInMinutes: 120 strategy: matrix: - Python3.11: - PYTHON_VERSION: '3.11' + Python3.12_Numpy2.1: + PYTHON_VERSION: '3.12' + NUMPY_VERSION: '2.1' + Python3.9_Numpy1.24: + PYTHON_VERSION: '3.9' + NUMPY_VERSION: '1.24' pool: vmImage: 'ubuntu-22.04' steps: @@ -82,15 +86,19 @@ jobs: export DPCPPROOT=/opt/intel/oneapi/compiler/latest . /usr/share/miniconda/etc/profile.d/conda.sh conda activate build-env - conda build . --python $(PYTHON_VERSION) + conda build . --python $(PYTHON_VERSION) --numpy $(NUMPY_VERSION) displayName: "Conda build" - job: WindowsCondaRecipe dependsOn: Lint timeoutInMinutes: 120 strategy: matrix: - Python3.11: - PYTHON_VERSION: '3.11' + Python3.12_Numpy2.1: + PYTHON_VERSION: '3.12' + NUMPY_VERSION: '2.1' + Python3.9_Numpy1.24: + PYTHON_VERSION: '3.9' + NUMPY_VERSION: '1.24' pool: vmImage: 'windows-2022' steps: @@ -106,7 +114,7 @@ jobs: displayName: "Conda create" - script: | call activate build-env - conda build . --python $(PYTHON_VERSION) + conda build . --python $(PYTHON_VERSION) --numpy $(NUMPY_VERSION) displayName: "Conda build" - job: LinuxCondaEnv dependsOn: Lint From 2ff9af3d58e148bc2b7d45c2063ddea1c1d16f73 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Mon, 14 Oct 2024 19:02:36 +0100 Subject: [PATCH 04/16] Remove python req from build env for conda-recipe CI --- .ci/pipeline/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index a0678d415b..f526782f76 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -80,7 +80,7 @@ jobs: conda update -y -q --all displayName: "Conda update" - script: | - conda create -y -q -n build-env python=$(PYTHON_VERSION) conda-build conda-verify + conda create -y -q -n build-env conda-build conda-verify displayName: "Conda create" - script: | export DPCPPROOT=/opt/intel/oneapi/compiler/latest @@ -110,7 +110,7 @@ jobs: conda update -y -q --all displayName: "Conda update" - script: | - conda create -y -q -n build-env python=$(PYTHON_VERSION) conda-build conda-verify + conda create -y -q -n build-env conda-build conda-verify displayName: "Conda create" - script: | call activate build-env From 846782f41f609c4ac555bb514219f39e0b52deff Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Mon, 14 Oct 2024 19:04:25 +0100 Subject: [PATCH 05/16] Reorder and extend testing deps --- conda-recipe/meta.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index c962981c3a..bc02af015e 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -71,14 +71,18 @@ requirements: test: requires: - - pandas + - pyyaml + - impi_rt # [not win] + # next deps are synced with requirements-test.txt + - pytest - scikit-learn + - pandas - xgboost - lightgbm - catboost - - pytest - - pyyaml - - impi_rt # [not win] + - shap + - array-api-compat + - array-api-strict source_files: - .ci - examples From 07caadd29ec61fdb34c8190f484f521d856330a6 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Mon, 14 Oct 2024 22:29:05 +0100 Subject: [PATCH 06/16] Change numpy testing matrix --- .ci/pipeline/ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index f526782f76..e827052b61 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -62,12 +62,12 @@ jobs: timeoutInMinutes: 120 strategy: matrix: - Python3.12_Numpy2.1: + Python3.12_Numpy1.26: PYTHON_VERSION: '3.12' - NUMPY_VERSION: '2.1' - Python3.9_Numpy1.24: + NUMPY_VERSION: '1.26' + Python3.9_Numpy1.22: PYTHON_VERSION: '3.9' - NUMPY_VERSION: '1.24' + NUMPY_VERSION: '1.22' pool: vmImage: 'ubuntu-22.04' steps: @@ -93,12 +93,12 @@ jobs: timeoutInMinutes: 120 strategy: matrix: - Python3.12_Numpy2.1: + Python3.12_Numpy1.26: PYTHON_VERSION: '3.12' - NUMPY_VERSION: '2.1' - Python3.9_Numpy1.24: + NUMPY_VERSION: '1.26' + Python3.9_Numpy1.22: PYTHON_VERSION: '3.9' - NUMPY_VERSION: '1.24' + NUMPY_VERSION: '1.22' pool: vmImage: 'windows-2022' steps: From e84cbe8376392d90523a39a1e8c4817faef0b7cb Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Tue, 15 Oct 2024 17:30:52 +0100 Subject: [PATCH 07/16] Remove daal4py/sklearn from test files --- conda-recipe/meta.yaml | 1 - conda-recipe/run_test.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index bc02af015e..f96c0220cd 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -87,7 +87,6 @@ test: - .ci - examples - tests - - daal4py/sklearn about: home: https://intel.github.io/scikit-learn-intelex diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh index cc0cf4f6f2..ab0babedc8 100755 --- a/conda-recipe/run_test.sh +++ b/conda-recipe/run_test.sh @@ -18,7 +18,7 @@ sklex_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" count=3 while [[ count -ne 0 ]]; do - if [[ -d $sklex_root/.ci/ && -d $sklex_root/examples/ && -d $sklex_root/tests/ && -d $sklex_root/daal4py/sklearn ]]; then + if [[ -d $sklex_root/.ci/ && -d $sklex_root/examples/ && -d $sklex_root/tests/ ]]; then break fi sklex_root="$( dirname "${sklex_root}" )" From 56f4a2c5fefc025e6cbe4a2d478c972d32c68d22 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Tue, 15 Oct 2024 17:53:37 +0100 Subject: [PATCH 08/16] Update DPC compiler details for conda-build in INSTALL --- INSTALL.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 288790a7db..0e956e7eb1 100755 --- a/INSTALL.md +++ b/INSTALL.md @@ -226,7 +226,8 @@ Intel(R) Extension for Scikit-learn* is easily built from the sources using only * (Windows only) Microsoft Visual Studio* 2022 * (optional) Intel(R) oneAPI DPC++/C++ Compiler -In order to enable DPC++ interfaces support, you need to set `DPCPPROOT` environment variable pointing to DPC++/C++ Compiler distribution. +In order to enable DPC++ interfaces support on Windows, you need to set `DPCPPROOT` environment variable pointing to DPC++/C++ Compiler distribution. +Conda-forge distribution of DPC++ compiler is used by default on Linux, but you still can set your own distribution via `DPCPPROOT` variable. ### Build Intel(R) Extension for Scikit-learn with `conda-build` From a1e6eeb75f5f4a8c3e9661050f537211e87fd5ab Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Thu, 17 Oct 2024 14:37:33 +0100 Subject: [PATCH 09/16] Update CI matrix and add comments --- .ci/pipeline/ci.yml | 18 ++++++------------ .ci/scripts/gen_release_jobs.py | 2 ++ conda-recipe/conda_build_config.yaml | 2 -- conda-recipe/meta.yaml | 6 +++++- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index e827052b61..0408963a6a 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -62,12 +62,9 @@ jobs: timeoutInMinutes: 120 strategy: matrix: - Python3.12_Numpy1.26: + Python3.12: PYTHON_VERSION: '3.12' - NUMPY_VERSION: '1.26' - Python3.9_Numpy1.22: - PYTHON_VERSION: '3.9' - NUMPY_VERSION: '1.22' + NUMPY_VERSION: '2.1' pool: vmImage: 'ubuntu-22.04' steps: @@ -87,18 +84,15 @@ jobs: . /usr/share/miniconda/etc/profile.d/conda.sh conda activate build-env conda build . --python $(PYTHON_VERSION) --numpy $(NUMPY_VERSION) - displayName: "Conda build" + displayName: "Conda build and test" - job: WindowsCondaRecipe dependsOn: Lint timeoutInMinutes: 120 strategy: matrix: - Python3.12_Numpy1.26: + Python3.12: PYTHON_VERSION: '3.12' - NUMPY_VERSION: '1.26' - Python3.9_Numpy1.22: - PYTHON_VERSION: '3.9' - NUMPY_VERSION: '1.22' + NUMPY_VERSION: '2.1' pool: vmImage: 'windows-2022' steps: @@ -115,7 +109,7 @@ jobs: - script: | call activate build-env conda build . --python $(PYTHON_VERSION) --numpy $(NUMPY_VERSION) - displayName: "Conda build" + displayName: "Conda build and test" - job: LinuxCondaEnv dependsOn: Lint timeoutInMinutes: 120 diff --git a/.ci/scripts/gen_release_jobs.py b/.ci/scripts/gen_release_jobs.py index 66bba66f6c..6dc6293454 100644 --- a/.ci/scripts/gen_release_jobs.py +++ b/.ci/scripts/gen_release_jobs.py @@ -24,6 +24,8 @@ CHANNELS = args.channels PYTHON_VERSIONS = ["3.9", "3.10", "3.11"] +# image versions are pinned to exact number instead of "latest" +# to avoid unexpected failures when images are updated SYSTEMS = ["ubuntu-22.04", "windows-2022"] ACTIVATE = { "ubuntu-latest": "conda activate", diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index e86db7e307..31ae4df64e 100755 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -14,8 +14,6 @@ # limitations under the License. #=============================================================================== -numpy: - - 1.24 c_compiler: # [win] - vs2022 # [win] cxx_compiler: # [win] diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index f96c0220cd..b047280b42 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -73,6 +73,9 @@ test: requires: - pyyaml - impi_rt # [not win] + # TODO: enable data parallel frameworks when they are available on conda-forge + # - dpctl + # - dpnp # next deps are synced with requirements-test.txt - pytest - scikit-learn @@ -80,7 +83,8 @@ test: - xgboost - lightgbm - catboost - - shap + # TODO: re-enable shap when conda-forge package has a numpy 2.* compatible version + # - shap - array-api-compat - array-api-strict source_files: From 8160909393c57334e65cf34ec23e769026e6b6cb Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Thu, 17 Oct 2024 17:29:38 +0100 Subject: [PATCH 10/16] Remove catboost; corrections for run_test.bat --- conda-recipe/meta.yaml | 4 ++-- conda-recipe/run_test.bat | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index b047280b42..686a694021 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -82,9 +82,9 @@ test: - pandas - xgboost - lightgbm - - catboost - # TODO: re-enable shap when conda-forge package has a numpy 2.* compatible version + # TODO: re-enable shap and catboost when conda-forge packages has a numpy 2.* compatibility # - shap + # - catboost - array-api-compat - array-api-strict source_files: diff --git a/conda-recipe/run_test.bat b/conda-recipe/run_test.bat index 8557d0c9d7..e410912ae4 100644 --- a/conda-recipe/run_test.bat +++ b/conda-recipe/run_test.bat @@ -15,11 +15,11 @@ rem See the License for the specific language governing permissions and rem limitations under the License. rem ============================================================================ -rem %1% - scikit-learn-intelex repo root (leave empty if it's PWD) +rem %1% - scikit-learn-intelex repo root (leave empty if it's %cd% / $PWD) set exitcode=0 -IF NOT DEFINED PYTHON (set PYTHON="python") +IF NOT DEFINED PYTHON (set "PYTHON=python") %PYTHON% -c "from sklearnex import patch_sklearn; patch_sklearn()" || set exitcode=1 From a33ec9925808e849d1acc140c3c815f396986dd6 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Thu, 17 Oct 2024 17:33:19 +0100 Subject: [PATCH 11/16] Add git tag as package version --- conda-recipe/meta.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 686a694021..5e93029bcb 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -17,12 +17,11 @@ # NB: this recipe should be synced between sklearnex repo and feedstocks {% set name = "scikit-learn-intelex" %} -{% set version = "2025.1" %} {% set buildnumber = 0 %} package: name: {{ name|lower }} - version: {{ version }} + version: {{ environ.get('GIT_DESCRIBE_TAG') }} # Reminder: source should point to path for repo or github archive for feedstock # source: From c9276247571f547c169fbfaff7244b1af1226588 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Fri, 18 Oct 2024 13:00:42 +0100 Subject: [PATCH 12/16] Add note about VS versions --- INSTALL.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 0e956e7eb1..2c4d22975c 100755 --- a/INSTALL.md +++ b/INSTALL.md @@ -223,9 +223,11 @@ Intel(R) Extension for Scikit-learn* is easily built from the sources using only * any `conda` distribution (`miniforge` is recommended) * `conda-build` and `conda-verify` installed in a conda environment -* (Windows only) Microsoft Visual Studio* 2022 +* (Windows only) Microsoft Visual Studio* * (optional) Intel(R) oneAPI DPC++/C++ Compiler +`conda-build` config requires **2022** version of Microsoft Visual Studio* by default, you can specify another version in `conda-recipe/conda_build_config.yaml` if needed. + In order to enable DPC++ interfaces support on Windows, you need to set `DPCPPROOT` environment variable pointing to DPC++/C++ Compiler distribution. Conda-forge distribution of DPC++ compiler is used by default on Linux, but you still can set your own distribution via `DPCPPROOT` variable. From 32d9316ac3d4007df097088bb2376a677eaf4319 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Fri, 18 Oct 2024 13:02:37 +0100 Subject: [PATCH 13/16] Change variable in run_test.bat --- conda-recipe/bld.bat | 2 +- conda-recipe/run_test.bat | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index f35aa1a49c..15321a39ec 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -15,7 +15,7 @@ rem See the License for the specific language governing permissions and rem limitations under the License. rem ============================================================================ -IF NOT DEFINED PYTHON (set PYTHON="python") +IF NOT DEFINED PYTHON (set "PYTHON=python") IF DEFINED PKG_VERSION (set SKLEARNEX_VERSION=%PKG_VERSION%) IF NOT DEFINED DALROOT (set DALROOT=%PREFIX%) IF NOT DEFINED MPIROOT IF "%NO_DIST%"=="" (set MPIROOT=%PREFIX%\Library) diff --git a/conda-recipe/run_test.bat b/conda-recipe/run_test.bat index e410912ae4..a9e27b0c91 100644 --- a/conda-recipe/run_test.bat +++ b/conda-recipe/run_test.bat @@ -15,7 +15,7 @@ rem See the License for the specific language governing permissions and rem limitations under the License. rem ============================================================================ -rem %1% - scikit-learn-intelex repo root (leave empty if it's %cd% / $PWD) +rem %1 - scikit-learn-intelex repo root (should end with '\', leave empty if it's %cd% / $PWD) set exitcode=0 @@ -23,10 +23,10 @@ IF NOT DEFINED PYTHON (set "PYTHON=python") %PYTHON% -c "from sklearnex import patch_sklearn; patch_sklearn()" || set exitcode=1 -%PYTHON% -m pytest --verbose -s %1%tests || set exitcode=1 +%PYTHON% -m pytest --verbose -s %1tests || set exitcode=1 pytest --verbose --pyargs daal4py || set exitcode=1 pytest --verbose --pyargs sklearnex || set exitcode=1 pytest --verbose --pyargs onedal || set exitcode=1 -pytest --verbose %1%.ci\scripts\test_global_patch.py || set exitcode=1 +pytest --verbose %1.ci\scripts\test_global_patch.py || set exitcode=1 EXIT /B %exitcode% From a1e02dd26acdecdf28d34520f57c4c4687a67572 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Fri, 18 Oct 2024 13:18:51 +0100 Subject: [PATCH 14/16] Add recipe versioning comment --- conda-recipe/meta.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 5e93029bcb..cef1bbba49 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -18,10 +18,13 @@ {% set name = "scikit-learn-intelex" %} {% set buildnumber = 0 %} +# version is set manually in feedstocks and through git tag in repo +# {% set version = "1.1.1" %} package: name: {{ name|lower }} version: {{ environ.get('GIT_DESCRIBE_TAG') }} + # version: {{ version }} # Reminder: source should point to path for repo or github archive for feedstock # source: From b54d69e883e47082dcced3b84a7be588c554feb7 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Fri, 18 Oct 2024 14:38:15 +0100 Subject: [PATCH 15/16] Reorder MPI test command --- conda-recipe/run_test.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh index ab0babedc8..af8df1a887 100755 --- a/conda-recipe/run_test.sh +++ b/conda-recipe/run_test.sh @@ -39,14 +39,6 @@ fi ${PYTHON} -c "from sklearnex import patch_sklearn; patch_sklearn()" return_code=$(($return_code + $?)) -echo "NO_DIST=$NO_DIST" -if [[ ! $NO_DIST ]]; then - echo "MPI pytest run" - mpirun --version - mpirun -n 4 pytest --verbose -s ${sklex_root}/tests/test*spmd*.py - return_code=$(($return_code + $?)) -fi - pytest --verbose -s ${sklex_root}/tests return_code=$(($return_code + $?)) @@ -62,4 +54,11 @@ return_code=$(($return_code + $?)) pytest --verbose -s ${sklex_root}/.ci/scripts/test_global_patch.py return_code=$(($return_code + $?)) +echo "NO_DIST=$NO_DIST" +if [[ ! $NO_DIST ]]; then + mpirun --version + mpirun -n 4 pytest --verbose -s ${sklex_root}/tests/test*spmd*.py + return_code=$(($return_code + $?)) +fi + exit $return_code From 853857dda42e614b2c956aa1e26e1251a5c092b9 Mon Sep 17 00:00:00 2001 From: Alexander Andreev Date: Fri, 18 Oct 2024 20:10:38 +0100 Subject: [PATCH 16/16] Remove extra dpcpp installation in CI --- .ci/pipeline/ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 0408963a6a..9d0d37598b 100644 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -68,9 +68,6 @@ jobs: pool: vmImage: 'ubuntu-22.04' steps: - - script: | - bash .ci/scripts/install_dpcpp.sh - displayName: "dpcpp installation" - script: | conda config --add channels conda-forge conda config --set channel_priority strict @@ -80,7 +77,6 @@ jobs: conda create -y -q -n build-env conda-build conda-verify displayName: "Conda create" - script: | - export DPCPPROOT=/opt/intel/oneapi/compiler/latest . /usr/share/miniconda/etc/profile.d/conda.sh conda activate build-env conda build . --python $(PYTHON_VERSION) --numpy $(NUMPY_VERSION)