diff --git a/.github/workflows/ci-checks-build.yml b/.github/workflows/ci-checks-build.yml index 08c66f3c6..17084546c 100644 --- a/.github/workflows/ci-checks-build.yml +++ b/.github/workflows/ci-checks-build.yml @@ -62,7 +62,7 @@ jobs: ~/.m2/repository /opt/intel/oneapi ~/opt - key: ${{ runner.os }}_spark-3.2.0_hadoop-3.2.0_oneapi-2021.4.0 + key: ${{ runner.os }}_spark-3.2.0_hadoop-3.2.0_oneapi-2022.1.2 restore-keys: | ${{ runner.os }}- - name: Build Test for CPU and GPU diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 59eb8b192..eedc2ea7d 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: local-test: - name: Local Test for Units + name: Local Test for Units (CPU) runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 @@ -20,7 +20,7 @@ jobs: ~/.m2/repository /opt/intel/oneapi ~/opt - key: ${{ runner.os }}_spark-3.2.0_hadoop-3.2.0_oneapi-2021.4.0 + key: ${{ runner.os }}_spark-3.2.0_hadoop-3.2.0_oneapi-2022.1.2 restore-keys: | ${{ runner.os }}- - name: Local Test @@ -28,7 +28,7 @@ jobs: ${{github.workspace}}/dev/ci/ci-local-test.sh cluster-test: - name: Cluster Test for Examples + name: Cluster Test for Examples (CPU) runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v2 @@ -44,7 +44,7 @@ jobs: ~/.m2/repository /opt/intel/oneapi ~/opt - key: ${{ runner.os }}_spark-3.2.0_hadoop-3.2.0_oneapi-2021.4.0 + key: ${{ runner.os }}_spark-3.2.0_hadoop-3.2.0_oneapi-2022.1.2 restore-keys: | ${{ runner.os }}- - name: Cluster Test diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index baa3db3b1..1f34b0ce5 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -9,7 +9,7 @@ "${JAVA_HOME}/include/**" ], "defines": [], - "compilerPath": "${CMPLR_ROOT}/linux/bin/clang", + "compilerPath": "${CMPLR_ROOT}/linux/bin/dpcpp", "cStandard": "c17", "cppStandard": "c++14", "intelliSenseMode": "clang-x64" diff --git a/README.md b/README.md index 6bfe76755..d14b0f864 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ You can find the all the OAP MLlib documents on the [project web page](https://o ### Java/Scala Users Preferred -Use a pre-built OAP MLlib JAR to get started. You can firstly download OAP package from [OAP-JARs-Tarball](https://github.com/oap-project/oap-tools/releases/download/v1.3.0/oap-1.3.0-bin.tar.gz) and extract this Tarball to get `oap-mllib-x.x.x.jar` under `oap-x.x.x-bin-spark-x.x.x/jars`. +Use a pre-built OAP MLlib JAR to get started. You can firstly download OAP package from [OAP-JARs-Tarball](https://github.com/oap-project/oap-tools/releases/download/v1.3.1/oap-1.3.1-bin.tar.gz) and extract this Tarball to get `oap-mllib-x.x.x.jar` under `oap-x.x.x-bin-spark-x.x.x/jars`. Then you can refer to the following [Running](#running) section to try out. @@ -118,7 +118,7 @@ We use [Apache Maven](https://maven.apache.org/) to manage and build source code * JDK 8.0+ * Apache Maven 3.6.2+ * GNU GCC 4.8.5+ -* Intel® oneAPI Base Toolkit (>=2021.4.0) Components : +* Intel® oneAPI Base Toolkit (>=2022.1) Components : - DPC++/C++ Compiler (dpcpp/clang++) - Data Analytics Library (oneDAL) - Threading Building Blocks (oneTBB) @@ -158,7 +158,7 @@ We suggest you to source `setvars.sh` script into current shell to setup buildin $ source /opt/intel/oneapi/setvars.sh ``` -You can also refer to [this CI script](dev/ci-test.sh) to setup the building environments. +You can also refer to [this CI script](dev/ci/ci-build-test.sh) to setup the building environments. If you prefer to buid your own open source [oneDAL](https://github.com/oneapi-src/oneDAL), [oneTBB](https://github.com/oneapi-src/oneTBB), [oneCCL](https://github.com/oneapi-src/oneCCL) versions rather than use the ones included in oneAPI Base Toolkit, you can refer to the related build instructions and manually source `setvars.sh` accordingly. diff --git a/RELEASE b/RELEASE index 7e9ffe4e3..ae20dcd65 100644 --- a/RELEASE +++ b/RELEASE @@ -1,3 +1,3 @@ -OAP_MLLIB_VERSION=1.3.0 +OAP_MLLIB_VERSION=1.3.1 SPARK_VERSION=3.2.0 PLATFORM_PROFILE=CPU_ONLY_PROFILE \ No newline at end of file diff --git a/dev/install-build-deps-centos.sh b/dev/install-build-deps-centos.sh index 877992228..02a7e1829 100755 --- a/dev/install-build-deps-centos.sh +++ b/dev/install-build-deps-centos.sh @@ -15,7 +15,7 @@ EOF sudo mv /tmp/oneAPI.repo /etc/yum.repos.d # sudo yum groupinstall -y "Development Tools" # sudo yum install -y cmake - sudo yum install -y intel-oneapi-dpcpp-cpp-2021.4.0 intel-oneapi-dal-devel-2021.4.0 intel-oneapi-tbb-devel-2021.4.0 intel-oneapi-ccl-devel-2021.4.0 intel-oneapi-mpi-devel-2021.4.0 + sudo yum install -y intel-oneapi-dpcpp-cpp-2022.0.2 intel-oneapi-dal-devel-2021.5.3 intel-oneapi-tbb-devel-2021.5.1 intel-oneapi-ccl-devel-2021.5.1 intel-oneapi-mpi-devel-2021.5.1 else echo "oneAPI components already installed!" fi diff --git a/dev/install-build-deps-ubuntu.sh b/dev/install-build-deps-ubuntu.sh index 027956b74..a745324b3 100755 --- a/dev/install-build-deps-ubuntu.sh +++ b/dev/install-build-deps-ubuntu.sh @@ -9,7 +9,7 @@ if [ ! -d /opt/intel/oneapi ]; then echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list sudo apt-get update # sudo apt-get install -y build-essential cmake - sudo apt-get install -y intel-oneapi-dpcpp-cpp-2021.4.0 intel-oneapi-dal-devel-2021.4.0 intel-oneapi-tbb-devel-2021.4.0 intel-oneapi-ccl-devel-2021.4.0 intel-oneapi-mpi-devel-2021.4.0 + sudo apt-get install -y intel-oneapi-dpcpp-cpp-2022.0.2 intel-oneapi-dal-devel-2021.5.3 intel-oneapi-tbb-devel-2021.5.1 intel-oneapi-ccl-devel-2021.5.1 intel-oneapi-mpi-devel-2021.5.1 else echo "oneAPI components already installed!" fi diff --git a/dev/prepare-build-deps-gpu.sh b/dev/prepare-build-deps-gpu.sh index abea6fd0f..1b5f3cab0 100755 --- a/dev/prepare-build-deps-gpu.sh +++ b/dev/prepare-build-deps-gpu.sh @@ -48,7 +48,7 @@ TARGET_DIR=./src/main/resources/lib rm -f $TARGET_DIR/*.so* -cp $CCL_ROOT/lib/cpu_icc/libccl.so.1.0 $TARGET_DIR/libccl.so.1 +cp $CCL_ROOT/lib/cpu/libccl.so.1.0 $TARGET_DIR/libccl.so.1 cp $I_MPI_ROOT/libfabric/lib/libfabric.so.1 $TARGET_DIR/libfabric.so.1 cp $I_MPI_ROOT/libfabric/lib/prov/libsockets-fi.so $TARGET_DIR @@ -61,8 +61,8 @@ cp $I_MPI_ROOT/lib/release_mt/libmpi.so.12.0.0 $TARGET_DIR/libmpi.so.12 cp $DAALROOT/lib/intel64/libJavaAPI.so.1.1 $TARGET_DIR/libJavaAPI.so -cp $TBBROOT/lib/intel64/gcc4.8/libtbb.so.12.4 $TARGET_DIR/libtbb.so.12 -cp $TBBROOT/lib/intel64/gcc4.8/libtbbmalloc.so.2.4 $TARGET_DIR/libtbbmalloc.so.2 +cp $TBBROOT/lib/intel64/gcc4.8/libtbb.so.12.5 $TARGET_DIR/libtbb.so.12 +cp $TBBROOT/lib/intel64/gcc4.8/libtbbmalloc.so.2.5 $TARGET_DIR/libtbbmalloc.so.2 # SYCL libs cp $CMPLR_ROOT/linux/compiler/lib/intel64_lin/libintlc.so.5 $TARGET_DIR diff --git a/dev/prepare-build-deps.sh b/dev/prepare-build-deps.sh index b3bbea9c7..fdbb7c0ab 100755 --- a/dev/prepare-build-deps.sh +++ b/dev/prepare-build-deps.sh @@ -48,7 +48,7 @@ TARGET_DIR=./src/main/resources/lib rm -f $TARGET_DIR/*.so* -cp $CCL_ROOT/lib/cpu_icc/libccl.so.1.0 $TARGET_DIR/libccl.so.1 +cp $CCL_ROOT/lib/cpu/libccl.so.1.0 $TARGET_DIR/libccl.so.1 cp $I_MPI_ROOT/libfabric/lib/libfabric.so.1 $TARGET_DIR/libfabric.so.1 cp $I_MPI_ROOT/libfabric/lib/prov/libsockets-fi.so $TARGET_DIR @@ -61,7 +61,7 @@ cp $I_MPI_ROOT/lib/release_mt/libmpi.so.12.0.0 $TARGET_DIR/libmpi.so.12 cp $DAALROOT/lib/intel64/libJavaAPI.so.1.1 $TARGET_DIR/libJavaAPI.so -cp $TBBROOT/lib/intel64/gcc4.8/libtbb.so.12.4 $TARGET_DIR/libtbb.so.12 -cp $TBBROOT/lib/intel64/gcc4.8/libtbbmalloc.so.2.4 $TARGET_DIR/libtbbmalloc.so.2 +cp $TBBROOT/lib/intel64/gcc4.8/libtbb.so.12.5 $TARGET_DIR/libtbb.so.12 +cp $TBBROOT/lib/intel64/gcc4.8/libtbbmalloc.so.2.5 $TARGET_DIR/libtbbmalloc.so.2 echo oneAPI Toolkit version: $(basename $CCL_ROOT) > $TARGET_DIR/VERSION diff --git a/examples/als/pom.xml b/examples/als/pom.xml index 39d207a95..3515af59f 100644 --- a/examples/als/pom.xml +++ b/examples/als/pom.xml @@ -4,7 +4,7 @@ com.intel.oap oap-mllib-examples - 1.3.0 + 1.3.1 jar ALSExample diff --git a/examples/correlation/pom.xml b/examples/correlation/pom.xml index a8af7cddc..a8ddcb0e8 100644 --- a/examples/correlation/pom.xml +++ b/examples/correlation/pom.xml @@ -4,7 +4,7 @@ com.intel.oap oap-mllib-examples - 1.3.0 + 1.3.1 jar CorrelationExample diff --git a/examples/kmeans/pom.xml b/examples/kmeans/pom.xml index e4437c51a..cf56f78e4 100644 --- a/examples/kmeans/pom.xml +++ b/examples/kmeans/pom.xml @@ -4,7 +4,7 @@ com.intel.oap oap-mllib-examples - 1.3.0 + 1.3.1 jar KMeansExample diff --git a/examples/linear-regression/pom.xml b/examples/linear-regression/pom.xml index 641562d85..c25e1cf6f 100644 --- a/examples/linear-regression/pom.xml +++ b/examples/linear-regression/pom.xml @@ -4,7 +4,7 @@ com.intel.oap oap-mllib-examples - 1.3.0 + 1.3.1 jar LinearRegressionExample diff --git a/examples/naive-bayes/pom.xml b/examples/naive-bayes/pom.xml index bb920cdeb..ee5e9af95 100644 --- a/examples/naive-bayes/pom.xml +++ b/examples/naive-bayes/pom.xml @@ -4,7 +4,7 @@ com.intel.oap oap-mllib-examples - 1.3.0 + 1.3.1 jar NaiveBayesExample diff --git a/examples/pca/pom.xml b/examples/pca/pom.xml index 263a984a9..a638f3470 100644 --- a/examples/pca/pom.xml +++ b/examples/pca/pom.xml @@ -4,7 +4,7 @@ com.intel.oap oap-mllib-examples - 1.3.0 + 1.3.1 jar PCAExample diff --git a/examples/summarizer/pom.xml b/examples/summarizer/pom.xml index bc6d70ae3..b75c43ced 100644 --- a/examples/summarizer/pom.xml +++ b/examples/summarizer/pom.xml @@ -4,7 +4,7 @@ com.intel.oap oap-mllib-examples - 1.3.0 + 1.3.1 jar SummaryStatisticsExample diff --git a/mllib-dal/build.sh b/mllib-dal/build.sh index d8fece3b8..4f9a51df1 100755 --- a/mllib-dal/build.sh +++ b/mllib-dal/build.sh @@ -128,7 +128,15 @@ echo DAALROOT=$DAALROOT echo TBBROOT=$TBBROOT echo CCL_ROOT=$CCL_ROOT echo Maven Version: $(mvn -v | head -n 1 | cut -f3 -d" ") -echo Clang Version: $(clang -dumpversion) + +if [[ $PLATFORM_PROFILE == CPU_ONLY_PROFILE ]] +then + echo GCC Version: $(gcc -dumpversion) +elif [[ $PLATFORM_PROFILE == CPU_GPU_PROFILE ]] +then + echo DPCPP Version: $(dpcpp -dumpversion) +fi + echo Spark Version: $SPARK_VERSION echo Platform Profile: $PLATFORM_PROFILE echo ============================= diff --git a/mllib-dal/pom.xml b/mllib-dal/pom.xml index c2b49e591..23f1b2a51 100644 --- a/mllib-dal/pom.xml +++ b/mllib-dal/pom.xml @@ -4,7 +4,7 @@ com.intel.oap oap-mllib - 1.3.0 + 1.3.1 OAP Project OAP MLlib jar https://github.com/oap-project/oap-mllib.git @@ -17,7 +17,7 @@ 2.12 3.2.9 3.2.0 - 2021.4.0.83 + 2021.5.0.91 src/assembly/assembly.xml diff --git a/mllib-dal/src/main/native/Makefile b/mllib-dal/src/main/native/Makefile index fe170d3ad..cfe516282 100644 --- a/mllib-dal/src/main/native/Makefile +++ b/mllib-dal/src/main/native/Makefile @@ -12,8 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -CC := clang -CXX := clang++ +# Use gcc for CPU and dpcpp for GPU +ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) + CC := gcc + CXX := g++ +else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) + CC := dpcpp + CXX := dpcpp +endif + RM := rm -rf PLATFORM_PROFILE ?= CPU_ONLY_PROFILE @@ -35,13 +42,13 @@ endif INCS := -I $(JAVA_HOME)/include \ -I $(JAVA_HOME)/include/linux \ - -I $(CCL_ROOT)/include/cpu_icc \ + -I $(CCL_ROOT)/include/cpu \ -I $(DAALROOT)/include \ -I ./javah \ -I ./ # Use static link if possible, TBB is only available as dynamic libs -LIBS_COMMON := -L$(CCL_ROOT)/lib/cpu_icc -lccl \ +LIBS_COMMON := -L$(CCL_ROOT)/lib/cpu -lccl \ -L$(CMPLR_ROOT)/linux/compiler/lib/intel64_lin -l:libirc.a \ -L$(DAALROOT)/lib/intel64 -l:libonedal_core.a -l:libonedal_thread.a \ -L$(TBBROOT)/lib/intel64/gcc4.8 -ltbb -ltbbmalloc