[ML-226] Migrate correlation daal to DPC++ (oap-project#215)

* 1. add JNI wrapper 2. modify compiler config 3. add unit test Signed-off-by: minmingzhu <minming.zhu@intel.com> * add ci test Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * java code style Signed-off-by: minmingzhu <minming.zhu@intel.com> * cpp code style Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update ci-local-test.sh * Update ci-local-test.sh * Update install-build-deps-ubuntu.sh * Update install-build-deps-ubuntu.sh * Update ci-local-test.sh * Update ci-local-test.sh * Update ci-local-test.sh * Update install-build-deps-ubuntu.sh * Update install-build-deps-ubuntu.sh * Update install-build-deps-ubuntu.sh * Update install-build-deps-ubuntu.sh * Update install-build-deps-ubuntu.sh * Update prepare-build-deps.sh * add convert array, vector to homogentable Signed-off-by: minmingzhu <minming.zhu@intel.com> * 1. rollback ci config 2. remove CI test becaue homogentable only support GPU Signed-off-by: minmingzhu <minming.zhu@intel.com> * 1. update prepare-build-deps.sh Signed-off-by: minmingzhu <minming.zhu@intel.com> * remove ci GPU test Signed-off-by: minmingzhu <minming.zhu@intel.com> * 1. remove GPU CI test Signed-off-by: minmingzhu <minming.zhu@intel.com> * update code style Signed-off-by: minmingzhu <minming.zhu@intel.com> * update code style Signed-off-by: minmingzhu <minming.zhu@intel.com> * fix comments Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Makefile Signed-off-by: minmingzhu <minming.zhu@intel.com> * fix comments Signed-off-by: minmingzhu <minming.zhu@intel.com> * 1. fix ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update header Signed-off-by: minmingzhu <minming.zhu@intel.com> * update header Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update ci-cluster-test.sh * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci conf Signed-off-by: minmingzhu <minming.zhu@intel.com> * update oneapi to 2022.1.2 Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci conf Signed-off-by: minmingzhu <minming.zhu@intel.com> * fix comment Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update build-jni.sh * update Makefile Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update test.sh * Update test.sh * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update test.sh * Update test.sh * update test.sh Signed-off-by: minmingzhu <minming.zhu@intel.com> * update test.sh Signed-off-by: minmingzhu <minming.zhu@intel.com> * update test.sh Signed-off-by: minmingzhu <minming.zhu@intel.com> * fix link libze_loader.so.1 Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci-local-dpc-test.sh Signed-off-by: minmingzhu <minming.zhu@intel.com> * update oneapi to 2022.1.2 Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update ci-local-dpc-test.sh * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Makefile Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update ci-local-dpc-test.sh * Update prepare-build-deps-gpu.sh * Update prepare-build-deps-gpu.sh * Update prepare-build-deps.sh * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update ci-local-dpc-test.sh * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update install-build-level-zero-deps-ubuntun.sh * Update install-build-level-zero-deps-ubuntun.sh * Update install-build-level-zero-deps-ubuntun.sh * Update ci-local-dpc-test.sh * Update GPU.cpp * Update GPU.cpp * Update ci-build-test.sh * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update HomogenTableImpl.cpp Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update ci-local-dpc-test.sh * Update ci-local-dpc-test.sh * Update ci-local-dpc-test.sh * update HomogenTable API Signed-off-by: minmingzhu <minming.zhu@intel.com> * update install-build-level-zero-deps-ubuntu.sh Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update code style Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update ci-local-dpc-test.sh * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update ci-local-dpc-test.sh * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * update ci Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update run.sh * Update build-jni.sh * 1. Merge branch 'make_homogen_table' into convert_homogentable 2. update ci 3. add a new parameters for HomogenTable class Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update pom.xml * Update test.sh * Update Makefile * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update ConvertHomogenTableSuite.scala * Update install-level-zero-ubuntu.sh * Update install-level-zero-ubuntu.sh * Update build.sh * Update test.sh * Update test.sh * Update HomogenTableImpl.cpp * Update install-build-deps-ubuntu.sh * Update HomogenTableImpl.cpp * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update code style Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update install-build-deps-ubuntu.sh * fix comments Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update list-compute-devices.cpp * Convert homogentable to array/vector/matrix (#4) * 1. convert homogenTable to array/vector/matrix 2. add unit test Signed-off-by: minmingzhu <minming.zhu@intel.com> * Merge branch 'make_homogen_table' into convert_homogentable Signed-off-by: minmingzhu <minming.zhu@intel.com> * update code style Signed-off-by: minmingzhu <minming.zhu@intel.com> * fix comments Signed-off-by: minmingzhu <minming.zhu@intel.com> * Convert RDD[Vector]/RDD[LabeledPoint] to HomogenTable (#8) * 1. convert rdd to HomogenTable * 2. add unit test Signed-off-by: minmingzhu <minming.zhu@intel.com> * add empty homogentable Signed-off-by: minmingzhu <minming.zhu@intel.com> * Mirgate kmeans daal to DPC++ (#7) Signed-off-by: minmingzhu <minming.zhu@intel.com> * 1. Mirgate kmeans daal to DPC++ * 2. add unit test * 1. migrate pca to DPC++ 2. add unit test Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * normalize data Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * 1. migrate correlation to DPC++ interface 2. add unit test Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * 1. move local host to distributed host 2. add cluster test 3. code style Signed-off-by: minmingzhu <minming.zhu@intel.com> * update code style Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update ci-cluster-test.sh * Update ci-cluster-test.sh * Update build.sh * Update build.sh * Update OneDAL.cpp * Update OneDAL.cpp * Update OneDAL.cpp * Update com_intel_oap_mllib_OneDAL__.h * update code style Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update NaiveBayesResult.java * 1. update code style 2. When using the Host/CPU device throws a fail Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * rollback code style Signed-off-by: minmingzhu <minming.zhu@intel.com> * rollback code style Signed-off-by: minmingzhu <minming.zhu@intel.com> * rollback checkstyle.xml Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * rollback checkstyle.xml Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update OneDAL.scala Signed-off-by: minmingzhu <minming.zhu@intel.com> * revert ALS.scala * revert checkstyle.xml * :Revert "revert checkstyle.xml" This reverts commit 4f74bac. * disable cluster test for gpu * disable cluster test for gpu * nit * migrate cpa to DPC++ Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update OutputHelpers.hpp * update com_intel_oap_mllib_feature_PCADALImpl.h Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update PCADALImpl.scala Signed-off-by: minmingzhu <minming.zhu@intel.com> * 1. update function name and variable name Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update stayHomogenTablePtrToVector to saveHomogenTablePtrToVector Signed-off-by: minmingzhu <minming.zhu@intel.com> * change compute_device to ComputeDevice Signed-off-by: minmingzhu <minming.zhu@intel.com> * 1. migrate pca to DPC++ 2. add unit test Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * normalize data Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * 1. migrate correlation to DPC++ interface 2. add unit test Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update prepare-build-deps-gpu.sh * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update Makefile * Update OutputHelpers.hpp * Update PCAOneAPIImpl.cpp * Update service.h * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * Update OneDAL.scala * Update PCADALImpl.scala * Update KmeansHomogenTableSuite.scala * Update TestCommon.scala * Update test.sh * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * fix comments Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update Signed-off-by: minmingzhu <minming.zhu@intel.com> * update code style Signed-off-by: minmingzhu <minming.zhu@intel.com> Signed-off-by: minmingzhu <minming.zhu@intel.com> Co-authored-by: minmingz <minming.zhu@inte.com> Co-authored-by: Wu, Xiaochang <xiaochang.wu@intel.com>
minmingzhu · May 11, 2023 · ad427e8 · ad427e8
1 parent 4fffd72
commit ad427e8
Show file tree

Hide file tree

Showing 5 changed files with 98 additions and 10,018 deletions.
diff --git a/mllib-dal/src/main/native/CorrelationOneAPIImpl.cpp b/mllib-dal/src/main/native/CorrelationOneAPIImpl.cpp
@@ -0,0 +1,89 @@
+/*******************************************************************************
+ * Copyright 2020 Intel Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *******************************************************************************/
+
+#include <chrono>
+
+#ifdef CPU_GPU_PROFILE
+#include "GPU.h"
+#ifndef ONEDAL_DATA_PARALLEL
+#define ONEDAL_DATA_PARALLEL
+#endif
+
+#include "Communicator.hpp"
+#include "OutputHelpers.hpp"
+#include "com_intel_oap_mllib_stat_CorrelationDALImpl.h"
+#include "oneapi/dal/algo/covariance.hpp"
+#include "oneapi/dal/table/homogen.hpp"
+#include "service.h"
+
+using namespace std;
+using namespace oneapi::dal;
+const int ccl_root = 0;
+
+static void doCorrelationOneAPICompute(JNIEnv *env, jint rankId,
+                                       jlong pNumTabData, jint executorNum,
+                                       const ccl::string &ipPort,
+                                       jint computeDeviceOrdinal,
+                                       jobject resultObj) {
+    std::cout << "oneDAL (native): compute start , rankid = " << rankId
+              << "; device = " << ComputeDeviceString[computeDeviceOrdinal]
+              << std::endl;
+    const bool isRoot = (rankId == ccl_root);
+    ComputeDevice device = getComputeDeviceByOrdinal(computeDeviceOrdinal);
+    homogen_table htable =
+        *reinterpret_cast<const homogen_table *>(pNumTabData);
+
+    const auto cor_desc = covariance::descriptor{}.set_result_options(
+        covariance::result_options::cor_matrix |
+        covariance::result_options::means);
+    auto queue = getQueue(device);
+    auto comm = preview::spmd::make_communicator<preview::spmd::backend::ccl>(
+        queue, executorNum, rankId, ipPort);
+    const auto result_train = preview::compute(comm, cor_desc, htable);
+    if (isRoot) {
+        std::cout << "Mean:\n" << result_train.get_means() << std::endl;
+        std::cout << "Correlation:\n"
+                  << result_train.get_cor_matrix() << std::endl;
+        // Return all covariance & mean
+        jclass clazz = env->GetObjectClass(resultObj);
+
+        // Get Field references
+        jfieldID correlationNumericTableField =
+            env->GetFieldID(clazz, "correlationNumericTable", "J");
+
+        HomogenTablePtr correlation =
+            std::make_shared<homogen_table>(result_train.get_cor_matrix());
+        saveHomogenTablePtrToVector(correlation);
+
+        env->SetLongField(resultObj, correlationNumericTableField,
+                          (jlong)correlation.get());
+    }
+}
+
+JNIEXPORT jlong JNICALL
+Java_com_intel_oap_mllib_stat_CorrelationDALImpl_cCorrelationTrainDAL(
+    JNIEnv *env, jobject obj, jlong pNumTabData, jint executorNum,
+    jint computeDeviceOrdinal, jint rankId, jstring ipPort, jobject resultObj) {
+    std::cout << "oneDAL (native): use DPC++ kernels " << std::endl;
+    const char *ipPortPtr = env->GetStringUTFChars(ipPort, 0);
+    std::string ipPortStr = std::string(ipPortPtr);
+    doCorrelationOneAPICompute(env, rankId, pNumTabData, executorNum, ipPortStr,
+                               computeDeviceOrdinal, resultObj);
+
+    env->ReleaseStringUTFChars(ipPort, ipPortPtr);
+    return 0;
+}
+#endif
diff --git a/mllib-dal/src/main/native/KMeansOneAPIImpl.cpp b/mllib-dal/src/main/native/KMeansOneAPIImpl.cpp
@@ -17,7 +17,6 @@
 #include <chrono>
 #include <iomanip>
 #include <iostream>
-#include <mutex>
 
 #ifdef CPU_GPU_PROFILE
 #include "GPU.h"
@@ -41,8 +40,8 @@ static jlong doKMeansOneAPICompute(JNIEnv *env, jint rankId, jlong pNumTabData,
                                    jint executorNum, const ccl::string &ipPort,
                                    jint computeDeviceOrdinal,
                                    jobject resultObj) {
-    std::cout << "oneDAL (native): GPU/CPU compute start , rankid = " << rankId
-              << "; device = " << computeDeviceOrdinal << "(0:HOST;1:GPU;2:CPU)"
+    std::cout << "oneDAL (native): compute start , rankid = " << rankId
+              << "; device = " << ComputeDeviceString[computeDeviceOrdinal]
               << std::endl;
     const bool isRoot = (rankId == ccl_root);
     ComputeDevice device = getComputeDeviceByOrdinal(computeDeviceOrdinal);
@@ -93,7 +92,7 @@ Java_com_intel_oap_mllib_clustering_KMeansDALImpl_cKMeansOneapiComputeWithInitCe
     JNIEnv *env, jobject obj, jlong pNumTabData, jlong pNumTabCenters,
     jint clusterNum, jdouble tolerance, jint iterationNum, jint executorNum,
     jint computeDeviceOrdinal, jint rankId, jstring ipPort, jobject resultObj) {
-    std::cout << "oneDAL (native): use GPU DPC++ kernels " << std::endl;
+    std::cout << "oneDAL (native): use DPC++ kernels " << std::endl;
     const char *ipPortPtr = env->GetStringUTFChars(ipPort, 0);
     std::string ipPortStr = std::string(ipPortPtr);
     jlong ret = 0L;

diff --git a/mllib-dal/src/main/native/PCAOneAPIImpl.cpp b/mllib-dal/src/main/native/PCAOneAPIImpl.cpp
@@ -22,6 +22,7 @@
 #ifndef ONEDAL_DATA_PARALLEL
 #define ONEDAL_DATA_PARALLEL
 #endif
+
 #include "Communicator.hpp"
 #include "OutputHelpers.hpp"
 #include "com_intel_oap_mllib_feature_PCADALImpl.h"
@@ -36,18 +37,19 @@ const int ccl_root = 0;
 static void doPCAOneAPICompute(JNIEnv *env, jint rankId, jlong pNumTabData,
                                jint executorNum, const ccl::string &ipPort,
                                jint computeDeviceOrdinal, jobject resultObj) {
+    std::cout << "oneDAL (native): compute start , rankid = " << rankId
+              << "; device = " << ComputeDeviceString[computeDeviceOrdinal]
+              << std::endl;
     const bool isRoot = (rankId == ccl_root);
     ComputeDevice device = getComputeDeviceByOrdinal(computeDeviceOrdinal);
-    std::cout << "oneDAL (native): GPU/CPU compute start , rankid = " << rankId
-              << "; device = " << computeDeviceOrdinal << "(0:HOST;1:GPU;2:CPU)"
-              << std::endl;
     homogen_table htable =
         *reinterpret_cast<const homogen_table *>(pNumTabData);
 
     const auto pca_desc = pca::descriptor{};
     auto queue = getQueue(device);
     auto comm = preview::spmd::make_communicator<preview::spmd::backend::ccl>(
         queue, executorNum, rankId, ipPort);
+
     pca::train_input local_input{htable};
     const auto result_train = preview::train(comm, pca_desc, local_input);
     if (isRoot) {
@@ -79,7 +81,7 @@ JNIEXPORT jlong JNICALL
 Java_com_intel_oap_mllib_feature_PCADALImpl_cPCATrainDAL(
     JNIEnv *env, jobject obj, jlong pNumTabData, jint executorNum,
     jint computeDeviceOrdinal, jint rankId, jstring ipPort, jobject resultObj) {
-    std::cout << "oneDAL (native): use GPU DPC++ kernels " << std::endl;
+    std::cout << "oneDAL (native): use DPC++ kernels " << std::endl;
     const char *ipPortPtr = env->GetStringUTFChars(ipPort, 0);
     std::string ipPortStr = std::string(ipPortPtr);
     doPCAOneAPICompute(env, rankId, pNumTabData, executorNum, ipPortStr,

diff --git a/mllib-dal/src/test/scala/data/kmeans_dense_train_centroids.csv b/mllib-dal/src/test/scala/data/kmeans_dense_train_centroids.csv