diff --git a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/ColumnAccessor.java b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/ColumnAccessor.java new file mode 100644 index 000000000..fa9e45939 --- /dev/null +++ b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/ColumnAccessor.java @@ -0,0 +1,43 @@ +package com.intel.oneapi.dal.table; + +public class ColumnAccessor { + private long cObject; + private Common.ComputeDevice cDevice; + + public ColumnAccessor(long cObject, Common.ComputeDevice device) { + this.cObject = cObject; + this.cDevice = device; + } + + public double[] pullDouble(long columnIndex){ + return this.cPullDouble(this.cObject, columnIndex, 0, -1, this.cDevice.ordinal()); + } + + public double[] pullDouble(long columnIndex, long rowStartIndex, long rowEndIndex){ + return this.cPullDouble(this.cObject, columnIndex, rowStartIndex, rowEndIndex, this.cDevice.ordinal()); + } + + public float[] pullFloat(long columnIndex){ + return this.cPullFloat(this.cObject, columnIndex, 0, -1, this.cDevice.ordinal()); + } + + public float[] pullFloat(long columnIndex, long rowStartIndex, long rowEndIndex){ + return this.cPullFloat(this.cObject, columnIndex, rowStartIndex, rowEndIndex, this.cDevice.ordinal()); + } + + public int[] pullInt(long columnIndex){ + return this.cPullInt(this.cObject, columnIndex, 0, -1, this.cDevice.ordinal()); + } + + public int[] pullInt(long columnIndex, long rowStartIndex, long rowEndIndex){ + return this.cPullInt(this.cObject, columnIndex, rowStartIndex, rowEndIndex, this.cDevice.ordinal()); + } + + private native double[] cPullDouble(long cObject, long cColumnIndex, + long cRowStartIndex, long cRowEndIndex, int computeDeviceIndex); + private native int[] cPullInt(long cObject, long cColumnIndex, long cRowStartIndex, + long cRowEndIndex, int computeDeviceIndex); + private native float[] cPullFloat(long cObject, long cColumnIndex, long cRowStartIndex, + long cRowEndIndex, int computeDeviceIndex); + +} diff --git a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/Common.java b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/Common.java index 87625dc24..a156ee823 100644 --- a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/Common.java +++ b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/Common.java @@ -30,7 +30,7 @@ public enum ComputeDevice { public static ComputeDevice get(int ordinal) { return values[ordinal]; } - public static int getOrdinalByName(String key){ + public static ComputeDevice getOrdinalByName(String key){ ComputeDevice device = null; switch (key.toUpperCase()){ case "HOST": @@ -43,7 +43,7 @@ public static int getOrdinalByName(String key){ device = GPU; break; } - return device.ordinal(); + return device; } } } diff --git a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/HomogenTable.java b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/HomogenTable.java index 5bdd58e76..191c33630 100644 --- a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/HomogenTable.java +++ b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/HomogenTable.java @@ -11,11 +11,10 @@ public HomogenTable(long cTable){ public HomogenTable(long rowCount, long colCount, int[] data, - int deviceIndex){ + Common.ComputeDevice device){ super(); // default Common.DataLayout dataLayout = Common.DataLayout.ROW_MAJOR; - Common.ComputeDevice device = Common.ComputeDevice.get(deviceIndex); impl = new HomogenTableImpl(rowCount, colCount, data, dataLayout, device); } @@ -23,11 +22,9 @@ public HomogenTable(long rowCount, public HomogenTable(long rowCount, long colCount, int[] data, - int layoutindex, - int deviceIndex){ + Common.DataLayout dataLayout, + Common.ComputeDevice device){ super(); - Common.DataLayout dataLayout = Common.DataLayout.get(layoutindex); - Common.ComputeDevice device = Common.ComputeDevice.get(deviceIndex); impl = new HomogenTableImpl(rowCount, colCount, data, dataLayout, device); @@ -36,11 +33,10 @@ public HomogenTable(long rowCount, public HomogenTable(long rowCount, long colCount, long[] data, - int deviceIndex){ + Common.ComputeDevice device){ super(); // default Common.DataLayout dataLayout = Common.DataLayout.ROW_MAJOR; - Common.ComputeDevice device = Common.ComputeDevice.get(deviceIndex); impl = new HomogenTableImpl(rowCount, colCount, data, dataLayout, device); } @@ -48,11 +44,9 @@ public HomogenTable(long rowCount, public HomogenTable(long rowCount, long colCount, long[] data, - int layoutIndex, - int deviceIndex){ + Common.DataLayout dataLayout, + Common.ComputeDevice device){ super(); - Common.DataLayout dataLayout = Common.DataLayout.get(layoutIndex); - Common.ComputeDevice device = Common.ComputeDevice.get(deviceIndex); impl = new HomogenTableImpl(rowCount, colCount, data, dataLayout, device); } @@ -60,11 +54,10 @@ public HomogenTable(long rowCount, public HomogenTable(long rowCount, long colCount, float[] data, - int deviceIndex){ + Common.ComputeDevice device){ super(); // default Common.DataLayout dataLayout = Common.DataLayout.ROW_MAJOR; - Common.ComputeDevice device = Common.ComputeDevice.get(deviceIndex); impl = new HomogenTableImpl(rowCount, colCount, data, dataLayout, device); } @@ -72,11 +65,9 @@ public HomogenTable(long rowCount, public HomogenTable(long rowCount, long colCount, float[] data, - int layoutIndex, - int deviceIndex){ + Common.DataLayout dataLayout, + Common.ComputeDevice device){ super(); - Common.DataLayout dataLayout = Common.DataLayout.get(layoutIndex); - Common.ComputeDevice device = Common.ComputeDevice.get(deviceIndex); impl = new HomogenTableImpl(rowCount, colCount, data, dataLayout, device); } @@ -84,11 +75,10 @@ public HomogenTable(long rowCount, public HomogenTable(long rowCount, long colCount, double[] data, - int deviceIndex){ + Common.ComputeDevice device){ super(); // default Common.DataLayout dataLayout = Common.DataLayout.ROW_MAJOR; - Common.ComputeDevice device = Common.ComputeDevice.get(deviceIndex); impl = new HomogenTableImpl(rowCount, colCount, data, dataLayout, device); } @@ -96,15 +86,12 @@ public HomogenTable(long rowCount, public HomogenTable(long rowCount, long colCount, double[] data, - int layoutIndex, - int deviceIndex){ + Common.DataLayout dataLayout, + Common.ComputeDevice device){ super(); - Common.DataLayout dataLayout = Common.DataLayout.get(layoutIndex); - Common.ComputeDevice device = Common.ComputeDevice.get(deviceIndex); impl = new HomogenTableImpl(rowCount, colCount, data, dataLayout, device); } - @Override public Long getColumnCount() { return impl.getColumnCount(); @@ -150,4 +137,8 @@ public long[] getLongData() { public double[] getDoubleData() { return impl.getDoubleData(); } + + public long getcObejct() { + return impl.getcObject(); + } } diff --git a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/HomogenTableImpl.java b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/HomogenTableImpl.java index 967a89d10..e09c87c98 100644 --- a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/HomogenTableImpl.java +++ b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/HomogenTableImpl.java @@ -12,8 +12,9 @@ public class HomogenTableImpl implements HomogenTableIface { e.printStackTrace(); } } - private transient long cObject; + private long cObject; private TableMetadata metadata; + private Common.ComputeDevice device; protected HomogenTableImpl() { super(); @@ -28,9 +29,8 @@ public HomogenTableImpl(long rowCount, int[] data, Common.DataLayout dataLayout, Common.ComputeDevice computeDevice) { - this.cObject = iInit(rowCount, colCount, data, dataLayout.ordinal(), computeDevice.ordinal()); - System.out.println(" HomogenTableImpl object : " + this.cObject); - + this.device = computeDevice; + this.cObject = iInit(rowCount, colCount, data, dataLayout.ordinal(), this.device.ordinal()); } public HomogenTableImpl(long rowCount, @@ -38,7 +38,8 @@ public HomogenTableImpl(long rowCount, float[] data, Common.DataLayout dataLayout, Common.ComputeDevice computeDevice) { - this.cObject = fInit(rowCount, colCount, data, dataLayout.ordinal(), computeDevice.ordinal()); + this.device = computeDevice; + this.cObject = fInit(rowCount, colCount, data, dataLayout.ordinal(), this.device.ordinal()); } @@ -47,7 +48,8 @@ public HomogenTableImpl(long rowCount, long[] data, Common.DataLayout dataLayout, Common.ComputeDevice computeDevice) { - this.cObject = lInit(rowCount, colCount, data, dataLayout.ordinal(), computeDevice.ordinal()); + this.device = computeDevice; + this.cObject = lInit(rowCount, colCount, data, dataLayout.ordinal(), this.device.ordinal()); } @@ -56,63 +58,33 @@ public HomogenTableImpl(long rowCount, double[] data, Common.DataLayout dataLayout, Common.ComputeDevice computeDevice) { - this.cObject = dInit(rowCount, colCount, data, dataLayout.ordinal(), computeDevice.ordinal()); + this.device = computeDevice; + this.cObject = dInit(rowCount, colCount, data, dataLayout.ordinal(), this.device.ordinal()); } - - private native long iInit(long rowCount, - long colCount, - int[] data, - int dataLayoutIndex, - int computeDeviceIndex); - - private native long fInit(long rowCount, - long colCount, - float[] data, - int dataLayoutIndex, - int computeDeviceIndex); - - private native long dInit(long rowCount, - long colCount, - double[] data, - int dataLayoutIndex, - int computeDeviceIndex); - - private native long lInit(long rowCount, - long colCount, - long[] data, - int dataLayoutIndex, - int computeDeviceIndex); - @Override public long getColumnCount() { return cGetColumnCount(this.cObject); } - private native long cGetColumnCount(long cObject); - @Override public long getRowCount() { return cGetRowCount(this.cObject); } - private native long cGetRowCount(long cObject); @Override public long getKind() { return this.cGetKind(this.cObject); } - private native long cGetKind(long cObject); - @Override public Common.DataLayout getDataLayout() { + return Common.DataLayout.get(cGetDataLayout(this.cObject)); } - private native int cGetDataLayout(long cObject); - @Override public TableMetadata getMetaData() { long cMetadata = cGetMetaData(this.cObject); @@ -120,61 +92,85 @@ public TableMetadata getMetaData() { return this.metadata; } - private native long cGetMetaData(long cObject); - @Override public long getPullRowsIface() { return 0; } - private native long cGetPullRowsIface(long cObject); @Override - public long getPullColumnIface() { - return 0; + public ColumnAccessor getPullColumnIface() { + ColumnAccessor accessor = new ColumnAccessor(cGetPullColumnIface(this.cObject), this.device); + return accessor; } - private native long cGetPullColumnIface(long cObject); - @Override public long getPullCSRBlockIface() { return 0; } - private native long cGetPullCSRBlockIface(long cObject); - @Override public boolean hasData() { return this.getColumnCount() > 0 && this.getRowCount() > 0; } - @Override public int[] getIntData() { return this.cGetIntData(this.cObject); } - private native int[] cGetIntData(long cObject); - @Override public long[] getLongData() { return this.cGetLongData(this.cObject); } - private native long[] cGetLongData(long cObject); - @Override public float[] getFloatData() { return this.cGetFloatData(this.cObject); } - private native float[] cGetFloatData(long cObject); - - @Override public double[] getDoubleData() { return cGetDoubleData(this.cObject); } + public long getcObject(){ + return this.cObject; + } + + private native long iInit(long rowCount, + long colCount, + int[] data, + int dataLayoutIndex, + int computeDeviceIndex); + + private native long fInit(long rowCount, + long colCount, + float[] data, + int dataLayoutIndex, + int computeDeviceIndex); + + private native long dInit(long rowCount, + long colCount, + double[] data, + int dataLayoutIndex, + int computeDeviceIndex); + + private native long lInit(long rowCount, + long colCount, + long[] data, + int dataLayoutIndex, + int computeDeviceIndex); + private native long cGetColumnCount(long cObject); + private native long cGetRowCount(long cObject); + private native long cGetKind(long cObject); + private native int cGetDataLayout(long cObject); + private native long cGetMetaData(long cObject); + private native long cGetPullRowsIface(long cObject); + private native long cGetPullColumnIface(long cObject); + private native long cGetPullCSRBlockIface(long cObject); + private native int[] cGetIntData(long cObject); + private native long[] cGetLongData(long cObject); + private native float[] cGetFloatData(long cObject); private native double[] cGetDoubleData(long cObject); } diff --git a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/RowAccessor.java b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/RowAccessor.java new file mode 100644 index 000000000..7b65c131a --- /dev/null +++ b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/RowAccessor.java @@ -0,0 +1,42 @@ +package com.intel.oneapi.dal.table; + +public class RowAccessor { + private long cObject; + private Common.ComputeDevice cDevice; + + public RowAccessor(long cObject, Common.ComputeDevice device) { + this.cObject = cObject; + this.cDevice =device; + } + + public double[] pullDouble(){ + return this.cPullDouble(this.cObject, 0, -1, this.cDevice.ordinal()); + } + + public double[] pullDouble(long rowStartIndex, long rowEndIndex){ + return this.cPullDouble(this.cObject, rowStartIndex, rowEndIndex, this.cDevice.ordinal()); + } + + public float[] pullFloat(){ + return this.cPullFloat(this.cObject, 0, -1, this.cDevice.ordinal()); + } + + public float[] pullFloat(long rowStartIndex, long rowEndIndex){ + return this.cPullFloat(this.cObject, rowStartIndex, rowEndIndex, this.cDevice.ordinal()); + } + + public int[] pullInt(){ + return this.cPullInt(this.cObject, 0, -1, this.cDevice.ordinal()); + } + + public int[] pullInt(long rowStartIndex, long rowEndIndex){ + return this.cPullInt(this.cObject, rowStartIndex, rowEndIndex, this.cDevice.ordinal()); + } + + private native double[] cPullDouble(long cObject, long cRowStartIndex, + long cRowEndIndex, int computeDeviceIndex); + private native float[] cPullFloat(long cObject, long cRowStartIndex, + long cRowEndIndex, int computeDeviceIndex); + private native int[] cPullInt(long cObject, long cRowStartIndex, + long cRowEndIndex, int computeDeviceIndex); +} diff --git a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/SimpleMetadataImpl.java b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/SimpleMetadataImpl.java index f41526c85..adfcc86c5 100644 --- a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/SimpleMetadataImpl.java +++ b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/SimpleMetadataImpl.java @@ -6,7 +6,7 @@ import java.util.ArrayList; public class SimpleMetadataImpl implements TableMetadataImpl{ - private transient long cObject; + private long cObject; private ArrayList dtypes; private ArrayList ftypes; static { @@ -33,20 +33,17 @@ public long getFeatureCount() { return cGetFeatureCount(this.cObject); } - private native long cGetFeatureCount(long cObject); - @Override public Common.FeatureType getFeatureType(int index) { return Common.FeatureType.get(cGetFeatureType(this.cObject, index)); } - private native int cGetFeatureType(long cObject, int index); - @Override public Common.DataType getDataType(int index) throws Exception { return Common.DataType.get(cGetDataType(this.cObject, index)); } private native int cGetDataType(long cObject, int index); - + private native long cGetFeatureCount(long cObject); + private native int cGetFeatureType(long cObject, int index); } diff --git a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/Table.java b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/Table.java index cabcdaeec..3fecd175c 100644 --- a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/Table.java +++ b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/Table.java @@ -2,7 +2,7 @@ import java.io.Serializable; -public abstract class Table implements Serializable { +public abstract class Table { public Table() {} protected abstract Long getColumnCount(); protected abstract Long getRowCount(); diff --git a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/TableIface.java b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/TableIface.java index b2dfde97b..83c153a3e 100644 --- a/mllib-dal/src/main/java/com/intel/oneapi/dal/table/TableIface.java +++ b/mllib-dal/src/main/java/com/intel/oneapi/dal/table/TableIface.java @@ -8,7 +8,7 @@ public interface TableIface { Common.DataLayout getDataLayout(); TableMetadata getMetaData(); long getPullRowsIface(); - long getPullColumnIface(); + ColumnAccessor getPullColumnIface(); long getPullCSRBlockIface(); boolean hasData(); } diff --git a/mllib-dal/src/main/native/GPU.cpp b/mllib-dal/src/main/native/GPU.cpp index e08ef9c00..8f744388d 100644 --- a/mllib-dal/src/main/native/GPU.cpp +++ b/mllib-dal/src/main/native/GPU.cpp @@ -3,14 +3,28 @@ #include #include "GPU.h" +typedef std::shared_ptr queuePtr; static std::mutex mtx; -static std::vector> cVector; -sycl::queue *getQue() { +static std::vector cVector; + +static void saveSyclQueue(const sycl::queue &queue) { + mtx.lock(); + cVector.push_back(queue); + mtx.unlock(); +} + +static sycl::queue &getSyclQueue(const sycl::device device) { + mtx.lock(); if (!cVector.empty()) { - return cVector[0].get(); + mtx.unlock(); + return cVector[0]; + } else { + sycl::queue queue{device}; + saveSyclQueue(queue); + mtx.unlock(); + return cVector[0]; } - return NULL; } static std::vector get_gpus() { @@ -72,35 +86,23 @@ sycl::device getAssignedGPU(ccl::communicator &comm, int size, int rankId, return rank_gpu; } -static void setVector(std::shared_ptr *ptr) { - mtx.lock(); - cVector.push_back(*ptr); - mtx.unlock(); -} - -sycl::queue *getQueue(const compute_device device) { +sycl::queue &getQueue(const compute_device device) { std::cout << "Get Queue" << std::endl; - std::shared_ptr *queuePtr; - sycl::queue *queue; + switch (device) { case compute_device::gpu: { std::cout << "selector GPU" << std::endl; auto device_gpu = sycl::gpu_selector{}.select_device(); - queue = new sycl::queue(device_gpu); - queuePtr = new std::shared_ptr(queue); - setVector(queuePtr); - return getQue(); + return getSyclQueue(device_gpu); } case compute_device::cpu: { std::cout << "selector CPU" << std::endl; auto device_cpu = sycl::cpu_selector{}.select_device(); - queue = new sycl::queue(device_cpu); - queuePtr = new std::shared_ptr(queue); - setVector(queuePtr); - return getQue(); + return getSyclQueue(device_cpu); } default: { - return getQue(); + std::cout << "No Device!" << std::endl; + exit(-1); } } } diff --git a/mllib-dal/src/main/native/GPU.h b/mllib-dal/src/main/native/GPU.h index c235f13d1..52db6ce6c 100644 --- a/mllib-dal/src/main/native/GPU.h +++ b/mllib-dal/src/main/native/GPU.h @@ -10,4 +10,4 @@ sycl::device getAssignedGPU(ccl::communicator &comm, int size, int rankId, jint *gpu_indices, int n_gpu); -sycl::queue *getQueue(const compute_device device); +sycl::queue &getQueue(const compute_device device); diff --git a/mllib-dal/src/main/native/Makefile b/mllib-dal/src/main/native/Makefile index 4ca676ee7..889636d9d 100644 --- a/mllib-dal/src/main/native/Makefile +++ b/mllib-dal/src/main/native/Makefile @@ -61,7 +61,6 @@ LIBS_COMMON := -L$(CCL_ROOT)/lib/cpu_gpu_dpcpp -lccl \ -L$(TBBROOT)/lib/intel64/gcc4.8 -ltbb -ltbbmalloc \ -L$(I_MPI_ROOT) - ifeq ($(PLATFORM_PROFILE),CPU_ONLY_PROFILE) LIBS := $(LIBS_COMMON) else ifeq ($(PLATFORM_PROFILE),CPU_GPU_PROFILE) @@ -78,7 +77,9 @@ CPP_SRCS += \ ./CorrelationDALImpl.cpp \ ./SummarizerDALImpl.cpp \ ./oneapi/dal/HomogenTableImpl.cpp \ - ./oneapi/dal/SimpleMetadataImpl.cpp + ./oneapi/dal/SimpleMetadataImpl.cpp \ + ./oneapi/dal/ColumnAccessorImpl.cpp \ + ./oneapi/dal/RowAccessorImpl.cpp OBJS += \ @@ -91,8 +92,9 @@ OBJS += \ ./CorrelationDALImpl.o \ ./SummarizerDALImpl.o \ ./oneapi/dal/HomogenTableImpl.o \ - ./oneapi/dal/SimpleMetadataImpl.o - + ./oneapi/dal/SimpleMetadataImpl.o \ + ./oneapi/dal/ColumnAccessorImpl.o \ + ./oneapi/dal/RowAccessorImpl.o DEFINES=-D$(PLATFORM_PROFILE) diff --git a/mllib-dal/src/main/native/build-jni.sh b/mllib-dal/src/main/native/build-jni.sh index 3d74b5f78..be6f3a191 100755 --- a/mllib-dal/src/main/native/build-jni.sh +++ b/mllib-dal/src/main/native/build-jni.sh @@ -39,5 +39,7 @@ javah -d $WORK_DIR/javah -classpath "$WORK_DIR/../../../target/classes:$DAAL_JAR com.intel.oap.mllib.stat.CorrelationDALImpl \ com.intel.oap.mllib.stat.SummarizerDALImpl \ com.intel.oneapi.dal.table.HomogenTableImpl \ - com.intel.oneapi.dal.table.SimpleMetadataImpl - + com.intel.oneapi.dal.table.SimpleMetadataImpl \ + com.intel.oneapi.dal.table.ColumnAccessor \ + com.intel.oneapi.dal.table.RowAccessor + diff --git a/mllib-dal/src/main/native/javah/com_intel_oneapi_dal_table_ColumnAccessor.h b/mllib-dal/src/main/native/javah/com_intel_oneapi_dal_table_ColumnAccessor.h new file mode 100644 index 000000000..9bfe74bb9 --- /dev/null +++ b/mllib-dal/src/main/native/javah/com_intel_oneapi_dal_table_ColumnAccessor.h @@ -0,0 +1,38 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class com_intel_oneapi_dal_table_ColumnAccessor */ + +#ifndef _Included_com_intel_oneapi_dal_table_ColumnAccessor +#define _Included_com_intel_oneapi_dal_table_ColumnAccessor +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: com_intel_oneapi_dal_table_ColumnAccessor + * Method: cPullDouble + * Signature: (JJJJ)[D + */ +JNIEXPORT jdoubleArray JNICALL Java_com_intel_oneapi_dal_table_ColumnAccessor_cPullDouble + (JNIEnv *, jobject, jlong, jlong, jlong, jlong, jint); + +/* + * Class: com_intel_oneapi_dal_table_ColumnAccessor + * Method: cPullFloat + * Signature: (JJJJ)[F + */ +JNIEXPORT jfloatArray JNICALL Java_com_intel_oneapi_dal_table_ColumnAccessor_cPullFloat + (JNIEnv *, jobject, jlong, jlong, jlong, jlong, jint); + +/* + * Class: com_intel_oneapi_dal_table_ColumnAccessor + * Method: cPullInt + * Signature: (JJJJ)[I + */ +JNIEXPORT jintArray JNICALL Java_com_intel_oneapi_dal_table_ColumnAccessor_cPullInt + (JNIEnv *, jobject, jlong, jlong, jlong, jlong, jint); + + +#ifdef __cplusplus +} +#endif +#endif diff --git a/mllib-dal/src/main/native/javah/com_intel_oneapi_dal_table_RowAccessor.h b/mllib-dal/src/main/native/javah/com_intel_oneapi_dal_table_RowAccessor.h new file mode 100644 index 000000000..4e583c87b --- /dev/null +++ b/mllib-dal/src/main/native/javah/com_intel_oneapi_dal_table_RowAccessor.h @@ -0,0 +1,38 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class com_intel_oneapi_dal_table_RowAccessor */ + +#ifndef _Included_com_intel_oneapi_dal_table_RowAccessor +#define _Included_com_intel_oneapi_dal_table_RowAccessor +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: com_intel_oneapi_dal_table_RowAccessor + * Method: cPullDouble + * Signature: (JJJ)[D + */ +JNIEXPORT jdoubleArray JNICALL Java_com_intel_oneapi_dal_table_RowAccessor_cPullDouble + (JNIEnv *, jobject, jlong, jlong, jlong, jint); + +/* + * Class: com_intel_oneapi_dal_table_RowAccessor + * Method: cPullFloat + * Signature: (JJJ)[F + */ +JNIEXPORT jfloatArray JNICALL Java_com_intel_oneapi_dal_table_RowAccessor_cPullFloat + (JNIEnv *, jobject, jlong, jlong, jlong, jint); + +/* + * Class: com_intel_oneapi_dal_table_RowAccessor + * Method: cPullInt + * Signature: (JJJ)[I + */ +JNIEXPORT jintArray JNICALL Java_com_intel_oneapi_dal_table_RowAccessor_cPullInt + (JNIEnv *, jobject, jlong, jlong, jlong, jint); + + +#ifdef __cplusplus +} +#endif +#endif diff --git a/mllib-dal/src/main/native/oneapi/dal/ColumnAccessorImpl.cpp b/mllib-dal/src/main/native/oneapi/dal/ColumnAccessorImpl.cpp new file mode 100644 index 000000000..6d089d6f4 --- /dev/null +++ b/mllib-dal/src/main/native/oneapi/dal/ColumnAccessorImpl.cpp @@ -0,0 +1,156 @@ +/******************************************************************************* + * Copyright 2020 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CPU_GPU_PROFILE +#include "GPU.h" +#endif +#ifndef ONEDAL_DATA_PARALLEL +#define ONEDAL_DATA_PARALLEL +#endif + +#include "com_intel_oneapi_dal_table_ColumnAccessor.h" +#include "oneapi/dal/table/homogen.hpp" +#include "oneapi/dal/table/column_accessor.hpp" +#include "service.h" + +using namespace std; +using namespace oneapi::dal; +typedef std::shared_ptr homogenPtr; + +/* + * Class: com_intel_oneapi_dal_table_ColumnAccessor + * Method: cPull + * Signature: (JJ)[D + */ +JNIEXPORT jdoubleArray JNICALL Java_com_intel_oneapi_dal_table_ColumnAccessor_cPullDouble + (JNIEnv *env, jobject, jlong cTableAddr, jlong cColumnIndex, jlong cRowStartIndex, + jlong cRowEndIndex, jint cComputeDevice) { + printf("ColumnAccessor PullDouble \n"); + homogen_table htable = *((homogen_table *)cTableAddr); + column_accessor acc{ htable }; + oneapi::dal::array col_values; + jdoubleArray newDoubleArray; + switch(getComputeDevice(cComputeDevice)) { + case compute_device::host:{ + col_values = acc.pull(cColumnIndex, {cRowStartIndex, cRowEndIndex}); + break; + } +#ifdef CPU_GPU_PROFILE + case compute_device::cpu:{ + sycl::queue cpu_queue = getQueue(compute_device::cpu); + col_values = acc.pull(cpu_queue, cColumnIndex, {cRowStartIndex, cRowEndIndex}); + break; + } + case compute_device::gpu:{ + sycl::queue gpu_queue = getQueue(compute_device::gpu); + col_values = acc.pull(gpu_queue, cColumnIndex, {cRowStartIndex, cRowEndIndex}); + break; + } +#endif + default: { + return newDoubleArray; + } + } + newDoubleArray = env->NewDoubleArray(col_values.get_count()); + env->SetDoubleArrayRegion(newDoubleArray, 0, col_values.get_count(), col_values.get_data()); + return newDoubleArray; +} + +/* + * Class: com_intel_oneapi_dal_table_ColumnAccessor + * Method: cPull + * Signature: (JJ)[D + */ +JNIEXPORT jfloatArray JNICALL Java_com_intel_oneapi_dal_table_ColumnAccessor_cPullFloat + (JNIEnv *env, jobject, jlong cTableAddr, jlong cColumnIndex, jlong cRowStartIndex, + jlong cRowEndIndex, jint cComputeDevice) { + printf("ColumnAccessor PullFloat \n"); + homogen_table htable = *((homogen_table *)cTableAddr); + column_accessor acc{ htable }; + oneapi::dal::array col_values; + jfloatArray newFloatArray; + switch(getComputeDevice(cComputeDevice)) { + case compute_device::host:{ + col_values = acc.pull(cColumnIndex, {cRowStartIndex, cRowEndIndex}); + break; + } +#ifdef CPU_GPU_PROFILE + case compute_device::cpu:{ + sycl::queue cpu_queue = getQueue(compute_device::cpu); + col_values = acc.pull(cpu_queue, cColumnIndex, {cRowStartIndex, cRowEndIndex}); + break; + } + case compute_device::gpu:{ + sycl::queue gpu_queue = getQueue(compute_device::gpu); + col_values = acc.pull(gpu_queue, cColumnIndex, {cRowStartIndex, cRowEndIndex}); + break; + } +#endif + default: { + return newFloatArray; + } + } + newFloatArray = env->NewFloatArray(col_values.get_count()); + env->SetFloatArrayRegion(newFloatArray, 0, col_values.get_count(), col_values.get_data()); + return newFloatArray; +} + +/* +* Class: com_intel_oneapi_dal_table_ColumnAccessor +* Method: cPull +* Signature: (JJ)[D +*/ +JNIEXPORT jintArray JNICALL Java_com_intel_oneapi_dal_table_ColumnAccessor_cPullInt +(JNIEnv *env, jobject, jlong cTableAddr, jlong cColumnIndex, jlong cRowStartIndex, + jlong cRowEndIndex, jint cComputeDevice) { +printf("ColumnAccessor PullInt \n"); +homogen_table htable = *((homogen_table *)cTableAddr); +column_accessor acc { htable }; +oneapi::dal::array col_values; +jintArray newIntArray; +switch(getComputeDevice(cComputeDevice)) { + case compute_device::host:{ + col_values = acc.pull(cColumnIndex, {cRowStartIndex, cRowEndIndex}); + break; + } +#ifdef CPU_GPU_PROFILE + case compute_device::cpu:{ + sycl::queue cpu_queue = getQueue(compute_device::cpu); + col_values = acc.pull(cpu_queue, cColumnIndex, {cRowStartIndex, cRowEndIndex}); + break; + } + case compute_device::gpu:{ + sycl::queue gpu_queue = getQueue(compute_device::gpu); + col_values = acc.pull(gpu_queue, cColumnIndex, {cRowStartIndex, cRowEndIndex}); + break; + } +#endif + default: { + return newIntArray; + } + } + newIntArray = env->NewIntArray(col_values.get_count()); + env->SetIntArrayRegion(newIntArray, 0, col_values.get_count(), col_values.get_data()); + return newIntArray; +} diff --git a/mllib-dal/src/main/native/oneapi/dal/HomogenTableImpl.cpp b/mllib-dal/src/main/native/oneapi/dal/HomogenTableImpl.cpp index bccac3b99..266ff9aff 100644 --- a/mllib-dal/src/main/native/oneapi/dal/HomogenTableImpl.cpp +++ b/mllib-dal/src/main/native/oneapi/dal/HomogenTableImpl.cpp @@ -37,12 +37,23 @@ using namespace std; using namespace oneapi::dal; +typedef std::shared_ptr homogenPtr; +typedef std::shared_ptr metadataPtr; + + std::mutex mtx; -std::vector> cVector; +std::vector cHomogenVector; +std::vector cMetaVector; -static void setVector(std::shared_ptr *ptr) { +static void saveShareHomogenPtrVector(const homogenPtr &ptr) { mtx.lock(); - cVector.push_back(*ptr); + cHomogenVector.push_back(ptr); + mtx.unlock(); +} + +static void saveShareMetaPtrVector(const metadataPtr &ptr) { + mtx.lock(); + cMetaVector.push_back(ptr); mtx.unlock(); } @@ -72,35 +83,39 @@ JNIEXPORT jlong JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_iInit( jint cLayout, jint cComputeDevice) { printf("HomogenTable int init \n"); jint *fData = env->GetIntArrayElements(cData, NULL); - homogen_table *h_table ; - std::shared_ptr *tablePtr ; + homogen_table *h_table; + homogenPtr tablePtr; switch(getComputeDevice(cComputeDevice)) { case compute_device::host:{ h_table = new homogen_table( fData, cRowCount, cColCount, detail::empty_delete(), getDataLayout(cLayout)); - tablePtr = new std::shared_ptr(h_table); - setVector(tablePtr); - return (jlong)tablePtr; + tablePtr = std::make_shared(*h_table); + saveShareHomogenPtrVector(tablePtr); + return (jlong)tablePtr.get(); } #ifdef CPU_GPU_PROFILE case compute_device::cpu:{ - sycl::queue *cpu_queue = getQueue(compute_device::cpu); - h_table = new homogen_table(*cpu_queue, - fData, cRowCount, cColCount, detail::empty_delete(), + sycl::queue cpu_queue = getQueue(compute_device::cpu); + auto cpu_data = malloc_shared(cRowCount * cColCount, cpu_queue); + cpu_queue.memcpy(cpu_data, fData, sizeof(int) * cRowCount * cColCount).wait(); + h_table = new homogen_table(cpu_queue, + cpu_data, cRowCount, cColCount, detail::make_default_delete(cpu_queue), {}, getDataLayout(cLayout)); - tablePtr = new std::shared_ptr(h_table); - setVector(tablePtr); - return (jlong)tablePtr; + tablePtr = std::make_shared(*h_table); + saveShareHomogenPtrVector(tablePtr); + return (jlong)tablePtr.get(); } case compute_device::gpu:{ - sycl::queue *gpu_queue = getQueue(compute_device::gpu); - h_table = new homogen_table(*gpu_queue, - fData, cRowCount, cColCount, detail::empty_delete(), + sycl::queue gpu_queue = getQueue(compute_device::gpu); + auto gpu_data = malloc_shared(cRowCount * cColCount, gpu_queue); + gpu_queue.memcpy(gpu_data, fData, sizeof(int) * cRowCount * cColCount).wait(); + h_table = new homogen_table(gpu_queue, + gpu_data, cRowCount, cColCount, detail::make_default_delete(gpu_queue), {}, getDataLayout(cLayout)); - tablePtr = new std::shared_ptr(h_table); - setVector(tablePtr); - return (jlong)tablePtr; + tablePtr = std::make_shared(*h_table); + saveShareHomogenPtrVector(tablePtr); + return (jlong)tablePtr.get(); } #endif default: { @@ -120,34 +135,38 @@ JNIEXPORT jlong JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_fInit( printf("HomogenTable float init \n"); jfloat *fData = env->GetFloatArrayElements(cData, NULL); homogen_table *h_table ; - std::shared_ptr *tablePtr ; + homogenPtr tablePtr ; switch(getComputeDevice(cComputeDevice)) { case compute_device::host:{ h_table = new homogen_table( fData, cRowCount, cColCount, detail::empty_delete(), getDataLayout(cLayout)); - tablePtr = new std::shared_ptr(h_table); - setVector(tablePtr); - return (jlong)tablePtr; + tablePtr = std::make_shared(*h_table); + saveShareHomogenPtrVector(tablePtr); + return (jlong)tablePtr.get(); } #ifdef CPU_GPU_PROFILE case compute_device::cpu:{ - sycl::queue *cpu_queue = getQueue(compute_device::cpu); - h_table = new homogen_table(*cpu_queue, - fData, cRowCount, cColCount, detail::empty_delete(), + sycl::queue cpu_queue = getQueue(compute_device::cpu); + auto cpu_data = malloc_shared(cRowCount * cColCount, cpu_queue); + cpu_queue.memcpy(cpu_data, fData, sizeof(float) * cRowCount * cColCount).wait(); + h_table = new homogen_table(cpu_queue, + cpu_data, cRowCount, cColCount, detail::make_default_delete(cpu_queue), {}, getDataLayout(cLayout)); - tablePtr = new std::shared_ptr(h_table); - setVector(tablePtr); - return (jlong)tablePtr; + tablePtr = std::make_shared(*h_table); + saveShareHomogenPtrVector(tablePtr); + return (jlong)tablePtr.get(); } case compute_device::gpu:{ - sycl::queue *gpu_queue = getQueue(compute_device::gpu); - h_table = new homogen_table(*gpu_queue, - fData, cRowCount, cColCount, detail::empty_delete(), + sycl::queue gpu_queue = getQueue(compute_device::gpu); + auto gpu_data = malloc_shared(cRowCount * cColCount, gpu_queue); + gpu_queue.memcpy(gpu_data, fData, sizeof(float) * cRowCount * cColCount).wait(); + h_table = new homogen_table(gpu_queue, + gpu_data, cRowCount, cColCount, detail::make_default_delete(gpu_queue), {}, getDataLayout(cLayout)); - tablePtr = new std::shared_ptr(h_table); - setVector(tablePtr); - return (jlong)tablePtr; + tablePtr = std::make_shared(*h_table); + saveShareHomogenPtrVector(tablePtr); + return (jlong)tablePtr.get(); } #endif default: { @@ -166,35 +185,38 @@ JNIEXPORT jlong JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_dInit( printf("HomogenTable double init \n"); jdouble *fData = env->GetDoubleArrayElements(cData, NULL); homogen_table *h_table ; - std::shared_ptr *tablePtr ; + homogenPtr tablePtr ; switch(getComputeDevice(cComputeDevice)) { case compute_device::host:{ - printf("HomogenTable double init host \n"); h_table = new homogen_table( fData, cRowCount, cColCount, detail::empty_delete(), getDataLayout(cLayout)); - tablePtr = new std::shared_ptr(h_table); - setVector(tablePtr); - return (jlong)tablePtr; + tablePtr = std::make_shared(*h_table); + saveShareHomogenPtrVector(tablePtr); + return (jlong)tablePtr.get(); } #ifdef CPU_GPU_PROFILE case compute_device::cpu:{ - sycl::queue *cpu_queue = getQueue(compute_device::cpu); - h_table = new homogen_table(*cpu_queue, - fData, cRowCount, cColCount, detail::empty_delete(), + sycl::queue cpu_queue = getQueue(compute_device::cpu); + auto cpu_data = malloc_shared(cRowCount * cColCount, cpu_queue); + cpu_queue.memcpy(cpu_data, fData, sizeof(double) * cRowCount * cColCount).wait(); + h_table = new homogen_table(cpu_queue, + cpu_data, cRowCount, cColCount, detail::make_default_delete(cpu_queue), {}, getDataLayout(cLayout)); - tablePtr = new std::shared_ptr(h_table); - setVector(tablePtr); - return (jlong)tablePtr; + tablePtr = std::make_shared(*h_table); + saveShareHomogenPtrVector(tablePtr); + return (jlong)tablePtr.get(); } case compute_device::gpu:{ - sycl::queue *gpu_queue = getQueue(compute_device::gpu); - h_table = new homogen_table(*gpu_queue, - fData, cRowCount, cColCount, detail::empty_delete(), + sycl::queue gpu_queue = getQueue(compute_device::gpu); + auto gpu_data = malloc_shared(cRowCount * cColCount, gpu_queue); + gpu_queue.memcpy(gpu_data, fData, sizeof(double) * cRowCount * cColCount).wait(); + h_table = new homogen_table(gpu_queue, + gpu_data, cRowCount, cColCount, detail::make_default_delete(gpu_queue), {}, getDataLayout(cLayout)); - tablePtr = new std::shared_ptr(h_table); - setVector(tablePtr); - return (jlong)tablePtr; + tablePtr = std::make_shared(*h_table); + saveShareHomogenPtrVector(tablePtr); + return (jlong)tablePtr.get(); } #endif default: { @@ -214,34 +236,38 @@ JNIEXPORT jlong JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_lInit( printf("HomogenTable long init \n"); jlong *fData = env->GetLongArrayElements(cData, NULL); homogen_table *h_table ; - std::shared_ptr *tablePtr ; + homogenPtr tablePtr ; switch(getComputeDevice(cComputeDevice)) { case compute_device::host:{ h_table = new homogen_table( fData, cRowCount, cColCount, detail::empty_delete(), getDataLayout(cLayout)); - tablePtr = new std::shared_ptr(h_table); - setVector(tablePtr); - return (jlong)tablePtr; + tablePtr = std::make_shared(*h_table); + saveShareHomogenPtrVector(tablePtr); + return (jlong)tablePtr.get(); } #ifdef CPU_GPU_PROFILE case compute_device::cpu:{ - sycl::queue *cpu_queue = getQueue(compute_device::cpu); - h_table = new homogen_table(*cpu_queue, - fData, cRowCount, cColCount, detail::empty_delete(), + sycl::queue cpu_queue = getQueue(compute_device::cpu); + auto cpu_data = malloc_shared(cRowCount * cColCount, cpu_queue); + cpu_queue.memcpy(cpu_data, fData, sizeof(long) * cRowCount * cColCount).wait(); + h_table = new homogen_table(cpu_queue, + cpu_data, cRowCount, cColCount, detail::make_default_delete(cpu_queue), {}, getDataLayout(cLayout)); - tablePtr = new std::shared_ptr(h_table); - setVector(tablePtr); - return (jlong)tablePtr; + tablePtr = std::make_shared(*h_table); + saveShareHomogenPtrVector(tablePtr); + return (jlong)tablePtr.get(); } case compute_device::gpu:{ - sycl::queue *gpu_queue = getQueue(compute_device::gpu); - h_table = new homogen_table(*gpu_queue, - fData, cRowCount, cColCount, detail::empty_delete(), + sycl::queue gpu_queue = getQueue(compute_device::gpu); + auto gpu_data = malloc_shared(cRowCount * cColCount, gpu_queue); + gpu_queue.memcpy(gpu_data, fData, sizeof(long) * cRowCount * cColCount).wait(); + h_table = new homogen_table(gpu_queue, + gpu_data, cRowCount, cColCount, detail::make_default_delete(gpu_queue), {}, getDataLayout(cLayout)); - tablePtr = new std::shared_ptr(h_table); - setVector(tablePtr); - return (jlong)tablePtr; + tablePtr = std::make_shared(*h_table); + saveShareHomogenPtrVector(tablePtr); + return (jlong)tablePtr.get(); } #endif default: { @@ -257,11 +283,11 @@ JNIEXPORT jlong JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_lInit( */ JNIEXPORT jlong JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetColumnCount( - JNIEnv *env, jobject, jlong ctableAddr) { - printf("HomogenTable getcolumncount \n"); - homogen_table *htable = - ((std::shared_ptr *)ctableAddr)->get(); - return htable->get_column_count(); + JNIEnv *env, jobject, jlong cTableAddr) { + printf("HomogenTable getcolumncount %ld \n", cTableAddr); + homogen_table htable = + *((homogen_table *)cTableAddr); + return htable.get_column_count(); } /* @@ -271,11 +297,11 @@ Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetColumnCount( */ JNIEXPORT jlong JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetRowCount( - JNIEnv *env, jobject, jlong ctableAddr) { + JNIEnv *env, jobject, jlong cTableAddr) { printf("HomogenTable getrowcount \n"); - homogen_table *htable = - ((std::shared_ptr *)ctableAddr)->get(); - return htable->get_row_count(); + homogen_table htable = + *((homogen_table *)cTableAddr); + return htable.get_row_count(); } /* @@ -285,11 +311,11 @@ Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetRowCount( */ JNIEXPORT jlong JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetKind(JNIEnv *env, jobject, - jlong ctableAddr) { + jlong cTableAddr) { printf("HomogenTable getkind \n"); - homogen_table *htable = - ((std::shared_ptr *)ctableAddr)->get(); - return htable->get_kind(); + homogen_table htable = + *((homogen_table *)cTableAddr); + return htable.get_kind(); } /* @@ -299,11 +325,11 @@ Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetKind(JNIEnv *env, jobject, */ JNIEXPORT jint JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetDataLayout( - JNIEnv *env, jobject, jlong ctableAddr) { + JNIEnv *env, jobject, jlong cTableAddr) { printf("HomogenTable getDataLayout \n"); - homogen_table *htable = - ((std::shared_ptr *)ctableAddr)->get(); - return (jint)htable->get_data_layout(); + homogen_table htable = + *((homogen_table *)cTableAddr); + return (jint)htable.get_data_layout(); } /* @@ -313,14 +339,13 @@ Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetDataLayout( */ JNIEXPORT jlong JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetMetaData( - JNIEnv *env, jobject, jlong ctableAddr) { + JNIEnv *env, jobject, jlong cTableAddr) { printf("HomogenTable getMetaData \n"); - homogen_table *htable = - ((std::shared_ptr *)ctableAddr)->get(); - table_metadata *mdata = (table_metadata *)&(htable->get_metadata()); - std::shared_ptr *metadataPtr = - new std::shared_ptr(mdata); - return (jlong)metadataPtr; + homogen_table htable = *((homogen_table *)cTableAddr); + table_metadata *mdata = (table_metadata *)&(htable.get_metadata()); + metadataPtr metaPtr = std::make_shared(*mdata); + saveShareMetaPtrVector(metaPtr); + return (jlong)metaPtr.get(); } /* @@ -331,12 +356,12 @@ Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetMetaData( JNIEXPORT jintArray JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetIntData(JNIEnv *env, jobject, - jlong ctableAddr) { + jlong cTableAddr) { printf("HomogenTable getIntData \n"); - homogen_table *htable = - ((std::shared_ptr *)ctableAddr)->get(); - const int *data = htable->get_data(); - const int datasize = htable->get_column_count() * htable->get_row_count(); + homogen_table htable = + *((homogen_table *)cTableAddr); + const int *data = htable.get_data(); + const int datasize = htable.get_column_count() * htable.get_row_count(); jintArray newIntArray = env->NewIntArray(datasize); env->SetIntArrayRegion(newIntArray, 0, datasize, data); @@ -350,12 +375,12 @@ Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetIntData(JNIEnv *env, */ JNIEXPORT jfloatArray JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetFloatData( - JNIEnv *env, jobject, jlong ctableAddr) { + JNIEnv *env, jobject, jlong cTableAddr) { printf("HomogenTable getFloatData \n"); - homogen_table *htable = - ((std::shared_ptr *)ctableAddr)->get(); - const float *data = htable->get_data(); - const int datasize = htable->get_column_count() * htable->get_row_count(); + homogen_table htable = + *((homogen_table *)cTableAddr); + const float *data = htable.get_data(); + const int datasize = htable.get_column_count() * htable.get_row_count(); jfloatArray newFloatArray = env->NewFloatArray(datasize); env->SetFloatArrayRegion(newFloatArray, 0, datasize, data); @@ -369,12 +394,12 @@ Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetFloatData( */ JNIEXPORT jlongArray JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetLongData( - JNIEnv *env, jobject, jlong ctableAddr) { + JNIEnv *env, jobject, jlong cTableAddr) { printf("HomogenTable getLongData \n"); - homogen_table *htable = - ((std::shared_ptr *)ctableAddr)->get(); - const long *data = htable->get_data(); - const int datasize = htable->get_column_count() * htable->get_row_count(); + homogen_table htable = + *((homogen_table *)cTableAddr); + const long *data = htable.get_data(); + const int datasize = htable.get_column_count() * htable.get_row_count(); jlongArray newLongArray = env->NewLongArray(datasize); env->SetLongArrayRegion(newLongArray, 0, datasize, data); @@ -388,12 +413,12 @@ Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetLongData( */ JNIEXPORT jdoubleArray JNICALL Java_com_intel_oneapi_dal_table_HomogenTableImpl_cGetDoubleData( - JNIEnv *env, jobject, jlong ctableAddr) { + JNIEnv *env, jobject, jlong cTableAddr) { printf("HomogenTable getDoubleData \n"); - homogen_table *htable = - ((std::shared_ptr *)ctableAddr)->get(); - const double *data = htable->get_data(); - const int datasize = htable->get_column_count() * htable->get_row_count(); + homogen_table htable = + *((homogen_table *)cTableAddr); + const double *data = htable.get_data(); + const int datasize = htable.get_column_count() * htable.get_row_count(); jdoubleArray newDoubleArray = env->NewDoubleArray(datasize); env->SetDoubleArrayRegion(newDoubleArray, 0, datasize, data); diff --git a/mllib-dal/src/main/native/oneapi/dal/RowAccessorImpl.cpp b/mllib-dal/src/main/native/oneapi/dal/RowAccessorImpl.cpp new file mode 100644 index 000000000..097d8966a --- /dev/null +++ b/mllib-dal/src/main/native/oneapi/dal/RowAccessorImpl.cpp @@ -0,0 +1,156 @@ +/******************************************************************************* + * Copyright 2020 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CPU_GPU_PROFILE +#include "GPU.h" +#endif +#ifndef ONEDAL_DATA_PARALLEL +#define ONEDAL_DATA_PARALLEL +#endif + +#include "com_intel_oneapi_dal_table_RowAccessor.h" +#include "oneapi/dal/table/homogen.hpp" +#include "oneapi/dal/table/row_accessor.hpp" +#include "service.h" + +using namespace std; +using namespace oneapi::dal; +typedef std::shared_ptr homogenPtr; + +/* + * Class: com_intel_oneapi_dal_table_RowAccessor + * Method: cPull + * Signature: (JJ)[D + */ +JNIEXPORT jdoubleArray JNICALL Java_com_intel_oneapi_dal_table_RowAccessor_cPullDouble + (JNIEnv *env, jobject, jlong cTableAddr, jlong cRowStartIndex, jlong cRowEndIndex, + jint cComputeDevice){ + printf("RowAccessor PullDouble \n"); + homogen_table htable = *((homogen_table *)cTableAddr); + row_accessor acc {htable}; + jdoubleArray newDoubleArray; + oneapi::dal::array row_values; + switch(getComputeDevice(cComputeDevice)) { + case compute_device::host:{ + row_values = acc.pull({cRowStartIndex, cRowEndIndex}); + break; + } +#ifdef CPU_GPU_PROFILE + case compute_device::cpu:{ + sycl::queue cpu_queue = getQueue(compute_device::cpu); + row_values = acc.pull(cpu_queue, {cRowStartIndex, cRowEndIndex}); + break; + } + case compute_device::gpu:{ + sycl::queue gpu_queue = getQueue(compute_device::gpu); + row_values = acc.pull(gpu_queue, {cRowStartIndex, cRowEndIndex}); + break; + } +#endif + default: { + return newDoubleArray; + } + } + newDoubleArray = env->NewDoubleArray(row_values.get_count()); + env->SetDoubleArrayRegion(newDoubleArray, 0, row_values.get_count(), row_values.get_data()); + return newDoubleArray; + } + +/* + * Class: com_intel_oneapi_dal_table_RowAccessor + * Method: cPull + * Signature: (JJ)[D + */ +JNIEXPORT jfloatArray JNICALL Java_com_intel_oneapi_dal_table_RowAccessor_cPullFloat + (JNIEnv *env, jobject, jlong cTableAddr, jlong cRowStartIndex, jlong cRowEndIndex, + jint cComputeDevice){ + printf("RowAccessor PullFloat \n"); + homogen_table htable = *((homogen_table *)cTableAddr); + row_accessor acc { htable }; + jfloatArray newFloatArray; + oneapi::dal::array row_values; + switch(getComputeDevice(cComputeDevice)) { + case compute_device::host:{ + row_values = acc.pull({cRowStartIndex, cRowEndIndex}); + break; + } +#ifdef CPU_GPU_PROFILE + case compute_device::cpu:{ + sycl::queue cpu_queue = getQueue(compute_device::cpu); + row_values = acc.pull(cpu_queue, {cRowStartIndex, cRowEndIndex}); + break; + } + case compute_device::gpu:{ + sycl::queue gpu_queue = getQueue(compute_device::gpu); + row_values = acc.pull(gpu_queue, {cRowStartIndex, cRowEndIndex}); + break; + } +#endif + default: { + return newFloatArray; + } + } + newFloatArray = env->NewFloatArray(row_values.get_count()); + env->SetFloatArrayRegion(newFloatArray, 0, row_values.get_count(), row_values.get_data()); + return newFloatArray; + } + +/* + * Class: com_intel_oneapi_dal_table_RowAccessor + * Method: cPull + * Signature: (JJ)[D + */ +JNIEXPORT jintArray JNICALL Java_com_intel_oneapi_dal_table_RowAccessor_cPullInt + (JNIEnv *env, jobject, jlong cTableAddr, jlong cRowStartIndex, jlong cRowEndIndex, + jint cComputeDevice){ + printf("RowAccessor PullInt \n"); + homogen_table htable = *((homogen_table *)cTableAddr); + row_accessor acc { htable }; + jintArray newIntArray; + oneapi::dal::array row_values; + switch(getComputeDevice(cComputeDevice)) { + case compute_device::host:{ + row_values = acc.pull({cRowStartIndex, cRowEndIndex}); + break; + } +#ifdef CPU_GPU_PROFILE + case compute_device::cpu:{ + sycl::queue cpu_queue = getQueue(compute_device::cpu); + row_values = acc.pull(cpu_queue, {cRowStartIndex, cRowEndIndex}); + break; + } + case compute_device::gpu:{ + sycl::queue gpu_queue = getQueue(compute_device::gpu); + row_values = acc.pull(gpu_queue, {cRowStartIndex, cRowEndIndex}); + break; + } +#endif + default: { + return newIntArray; + } + } + newIntArray = env->NewIntArray(row_values.get_count()); + env->SetIntArrayRegion(newIntArray, 0, row_values.get_count(), row_values.get_data()); + return newIntArray; + } diff --git a/mllib-dal/src/main/native/oneapi/dal/SimpleMetadataImpl.cpp b/mllib-dal/src/main/native/oneapi/dal/SimpleMetadataImpl.cpp index 6d004dd4a..5e2478db1 100644 --- a/mllib-dal/src/main/native/oneapi/dal/SimpleMetadataImpl.cpp +++ b/mllib-dal/src/main/native/oneapi/dal/SimpleMetadataImpl.cpp @@ -35,11 +35,11 @@ using namespace oneapi::dal; */ JNIEXPORT jlong JNICALL Java_com_intel_oneapi_dal_table_SimpleMetadataImpl_cGetFeatureCount( - JNIEnv *env, jobject, jlong ctableAddr) { + JNIEnv *env, jobject, jlong cTableAddr) { printf("SimpleMetadata getfeaturecount \n"); - table_metadata *mdata = - ((std::shared_ptr *)ctableAddr)->get(); - return (jlong)mdata->get_feature_count(); + table_metadata mdata = + *((table_metadata *)cTableAddr); + return (jlong)mdata.get_feature_count(); } /* @@ -49,11 +49,12 @@ Java_com_intel_oneapi_dal_table_SimpleMetadataImpl_cGetFeatureCount( */ JNIEXPORT jint JNICALL Java_com_intel_oneapi_dal_table_SimpleMetadataImpl_cGetFeatureType( - JNIEnv *env, jobject, jlong ctableAddr, jint cindex) { + JNIEnv *env, jobject, jlong cTableAddr, jint cindex) { printf("SimpleMetadata getfeaturetype \n"); - table_metadata *mdata = - ((std::shared_ptr *)ctableAddr)->get(); - return (jint)mdata->get_feature_type(cindex); + table_metadata mdata = + *((table_metadata *)cTableAddr); + + return (jint)mdata.get_feature_type(cindex); } /* @@ -63,9 +64,9 @@ Java_com_intel_oneapi_dal_table_SimpleMetadataImpl_cGetFeatureType( */ JNIEXPORT jint JNICALL Java_com_intel_oneapi_dal_table_SimpleMetadataImpl_cGetDataType( - JNIEnv *env, jobject, jlong ctableAddr, jint cindex) { + JNIEnv *env, jobject, jlong cTableAddr, jint cindex) { printf("SimpleMetadata getdatatype \n"); - table_metadata *mdata = - ((std::shared_ptr *)ctableAddr)->get(); - return (jint)mdata->get_data_type(cindex); + table_metadata mdata = + *((table_metadata *)cTableAddr); + return (jint)mdata.get_data_type(cindex); } diff --git a/mllib-dal/src/main/scala/com/intel/oap/mllib/OneDAL.scala b/mllib-dal/src/main/scala/com/intel/oap/mllib/OneDAL.scala index f3cda5cac..a43e91006 100644 --- a/mllib-dal/src/main/scala/com/intel/oap/mllib/OneDAL.scala +++ b/mllib-dal/src/main/scala/com/intel/oap/mllib/OneDAL.scala @@ -28,7 +28,7 @@ import java.lang import java.nio.DoubleBuffer import java.util.logging.{Level, Logger} -import com.intel.oneapi.dal.table.{Common, HomogenTable} +import com.intel.oneapi.dal.table.{ColumnAccessor, Common, HomogenTable, RowAccessor} import scala.collection.mutable.ArrayBuffer @@ -57,6 +57,17 @@ object OneDAL { matrix } + def homogenTableToMatrix(table: HomogenTable): Matrix = { + val numRows = table.getRowCount.toInt + val numCols = table.getColumnCount.toInt + + val arrayDouble = table.getDoubleData() + // Transpose as DAL numeric table is row-major and DenseMatrix is column major + val matrix = new DenseMatrix(numRows, numCols, arrayDouble, isTransposed = true) + + matrix + } + def numericTableToOldMatrix(table: NumericTable): OldMatrix = { val numRows = table.getNumberOfRows.toInt val numCols = table.getNumberOfColumns.toInt @@ -75,6 +86,17 @@ object OneDAL { OldMatrix } + def homogenTableToOldMatrix(table: HomogenTable): OldMatrix = { + val numRows = table.getRowCount.toInt + val numCols = table.getColumnCount.toInt + + val arrayDouble = table.getDoubleData() + // Transpose as DAL numeric table is row-major and DenseMatrix is column major + val OldMatrix = new OldDenseMatrix(numRows, numCols, arrayDouble, isTransposed = true) + + OldMatrix + } + def isDenseDataset(ds: Dataset[_], featuresCol: String): Boolean = { val row = ds.select(featuresCol).head() @@ -95,11 +117,17 @@ object OneDAL { Vectors.dense(arrayDouble) } + def homogenTableNx1ToVector(cTable: Long, device: Common.ComputeDevice ): Vector = { + val columnAcc = new ColumnAccessor(cTable, device) + val arrayDouble = columnAcc.pullDouble(0) + Vectors.dense(arrayDouble) + } + def numericTable1xNToVector(table: NumericTable): Vector = { val numCols = table.getNumberOfColumns.toInt var dataDouble: DoubleBuffer = null - // returned DoubleBuffer is ByteByffer, need to copy as double array + // returned DoubleBuffer is ByteBuffer, need to copy as double array dataDouble = table.getBlockOfRows(0, 1, dataDouble) val arrayDouble = new Array[Double](numCols.toInt) dataDouble.get(arrayDouble) @@ -109,6 +137,11 @@ object OneDAL { Vectors.dense(arrayDouble) } + def homogenTable1xNToVector(table: HomogenTable, device: Common.ComputeDevice): Vector = { + val rowAcc = new RowAccessor(table.getcObejct, device) + val arrayDouble = rowAcc.pullDouble(0, 1) + Vectors.dense(arrayDouble) + } // Convert DAL numeric table to array of vectors def numericTableToVectors(table: NumericTable): Array[Vector] = { val numRows = table.getNumberOfRows.toInt @@ -127,6 +160,20 @@ object OneDAL { resArray } + def homogenTableToVectors(table: HomogenTable, device: Common.ComputeDevice): Array[Vector] = { + val numRows = table.getRowCount.toInt + + val rowAcc = new RowAccessor(table.getcObejct(), device) + + val resArray = new Array[Vector](numRows.toInt) + + for (row <- 0 until numRows) { + val internArray = rowAcc.pullDouble( row, row + 1) + resArray(row) = Vectors.dense(internArray) + } + + resArray + } def makeNumericTable(cData: Long): NumericTable = { @@ -161,18 +208,6 @@ object OneDAL { table } - def homogenTable1xNToVector(table: HomogenTable): Vector = { - val numCols = table.getColumnCount - - var dataDouble: DoubleBuffer = null - // returned DoubleBuffer is ByteByffer, need to copy as double array - val arrayDouble = new Array[Double](numCols.toInt) - dataDouble.get(arrayDouble) - - - Vectors.dense(arrayDouble) - } - def rddDoubleToNumericTables(doubles: RDD[Double], executorNum: Int): RDD[Long] = { require(executorNum > 0) @@ -258,7 +293,7 @@ object OneDAL { private[mllib] def doubleArrayToHomogenTable(points: Array[Double], device: Common.ComputeDevice): HomogenTable = { - val table = new HomogenTable(1, points.length, points, device.ordinal()) + val table = new HomogenTable(1, points.length, points, device) table } @@ -278,7 +313,7 @@ object OneDAL { } } val table = new HomogenTable(numRows.toLong, numCols.toLong, arrayDouble, - device.ordinal()) + device) table } @@ -298,7 +333,7 @@ object OneDAL { } } val table = new HomogenTable(numRows.toLong, numCols.toLong, arrayDouble, - device.ordinal()) + device) table } diff --git a/mllib-dal/src/test/java/com/intel/oneapi/dal/table/ColumnAccessorTest.java b/mllib-dal/src/test/java/com/intel/oneapi/dal/table/ColumnAccessorTest.java new file mode 100644 index 000000000..b46e15653 --- /dev/null +++ b/mllib-dal/src/test/java/com/intel/oneapi/dal/table/ColumnAccessorTest.java @@ -0,0 +1,154 @@ +package com.intel.oneapi.dal.table; + +import org.junit.jupiter.api.Test; + +import static com.intel.oneapi.dal.table.Common.DataLayout.ROW_MAJOR; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +public class ColumnAccessorTest { + + @Test + public void getDoubleFirstColumnFromHomogenTable() throws Exception { + double[] data = {5.236359d, 8.718667d, 40.724176d, 10.770023d, 90.119887d, 3.815366d, + 53.620204d, 33.219769d, 85.208661d, 15.966239d}; + HomogenTable table = new HomogenTable(5, 2, + data, CommonTest.getDevice()); + + ColumnAccessor accessor = new ColumnAccessor(table.getcObejct(), CommonTest.getDevice()); + double[] columnData = accessor.pullDouble(0); + assertEquals(new Long(columnData.length), table.getRowCount()); + double[] tableData = table.getDoubleData(); + for (int i = 0; i < columnData.length; i++) { + assertEquals(columnData[i], tableData[i * table.getColumnCount().intValue()]); + } + } + + @Test + public void getDoubleSecondColumnFromHomogenTableWithConversion() throws Exception { + double[] data = {5.236359d, 8.718667d, 40.724176d, 10.770023d, 90.119887d, 3.815366d, + 53.620204d, 33.219769d, 85.208661d, 15.966239d}; + HomogenTable table = new HomogenTable(5, 2, + data, CommonTest.getDevice()); + + ColumnAccessor accessor = new ColumnAccessor(table.getcObejct(), CommonTest.getDevice()); + double[] columnData = accessor.pullDouble(1); + assertEquals(new Long(columnData.length), table.getRowCount()); + double[] tableData = table.getDoubleData(); + + for (int i = 0; i < columnData.length; i++) { + assertEquals(columnData[i], tableData[i * table.getColumnCount().intValue() + 1]); + } + } + + @Test + public void getDoubleSecondColumnFromHomogenTableWithSubsetOfRows() throws Exception { + double[] data = {5.236359d, 8.718667d, 40.724176d, 10.770023d, 90.119887d, 3.815366d, + 53.620204d, 33.219769d, 85.208661d, 15.966239d}; + HomogenTable table = new HomogenTable(5, 2, + data, CommonTest.getDevice()); + + ColumnAccessor accessor = new ColumnAccessor(table.getcObejct(), CommonTest.getDevice()); + double[] columnData = accessor.pullDouble(0, 1 , 3); + assertEquals(new Long(columnData.length), new Long(2)); + double[] tableData = table.getDoubleData(); + for (int i = 0; i < columnData.length; i++) { + assertEquals(columnData[i], tableData[2 + i * table.getColumnCount().intValue()]); + } + } + + @Test + public void getFloatFirstColumnFromHomogenTable() throws Exception { + float[] data = {5.236359f, 8.718667f, 40.724176f, 10.770023f, 90.119887f, 3.815366f, + 53.620204f, 33.219769f, 85.208661f, 15.966239f}; + HomogenTable table = new HomogenTable(5, 2, + data, CommonTest.getDevice()); + + ColumnAccessor accessor = new ColumnAccessor(table.getcObejct(), CommonTest.getDevice()); + float[] columnData = accessor.pullFloat(0); + assertEquals(new Long(columnData.length), table.getRowCount()); + float[] tableData = table.getFloatData(); + for (int i = 0; i < columnData.length; i++) { + assertEquals(columnData[i], tableData[i * table.getColumnCount().intValue()]); + } + } + + @Test + public void getFloatSecondColumnFromHomogenTableWithConversion() throws Exception { + float[] data = {5.236359f, 8.718667f, 40.724176f, 10.770023f, 90.119887f, 3.815366f, + 53.620204f, 33.219769f, 85.208661f, 15.966239f}; + HomogenTable table = new HomogenTable(5, 2, + data, CommonTest.getDevice()); + + ColumnAccessor accessor = new ColumnAccessor(table.getcObejct(), CommonTest.getDevice()); + float[] columnData = accessor.pullFloat(1); + assertEquals(new Long(columnData.length), table.getRowCount()); + float[] tableData = table.getFloatData(); + + for (int i = 0; i < columnData.length; i++) { + assertEquals(columnData[i], tableData[i * table.getColumnCount().intValue() + 1]); + } + } + + @Test + public void getFloatSecondColumnFromHomogenTableWithSubsetOfRows() throws Exception { + float[] data = {5.236359f, 8.718667f, 40.724176f, 10.770023f, 90.119887f, 3.815366f, + 53.620204f, 33.219769f, 85.208661f, 15.966239f}; + HomogenTable table = new HomogenTable(5, 2, + data, CommonTest.getDevice()); + + ColumnAccessor accessor = new ColumnAccessor(table.getcObejct(), CommonTest.getDevice()); + float[] columnData = accessor.pullFloat(0, 1 , 3); + assertEquals(new Long(columnData.length), new Long(2)); + float[] tableData = table.getFloatData(); + for (int i = 0; i < columnData.length; i++) { + assertEquals(columnData[i], tableData[2 + i * table.getColumnCount().intValue()]); + } + } + + @Test + public void getIntFirstColumnFromHomogenTable() throws Exception { + int[] data = {5, 8, 40, 10, 90, 3, 53, 33, 85, 15}; + HomogenTable table = new HomogenTable(5, 2, + data, CommonTest.getDevice()); + + ColumnAccessor accessor = new ColumnAccessor(table.getcObejct(), CommonTest.getDevice()); + int[] columnData = accessor.pullInt(0); + assertEquals(new Long(columnData.length), table.getRowCount()); + int[] tableData = table.getIntData(); + for (int i = 0; i < columnData.length; i++) { + assertEquals(columnData[i], tableData[i * table.getColumnCount().intValue()]); + } + } + + @Test + public void getIntSecondColumnFromHomogenTableWithConversion() throws Exception { + int[] data = {5, 8, 40, 10, 90, 3, 53, 33, 85, 15}; + HomogenTable table = new HomogenTable(5, 2, + data, CommonTest.getDevice()); + + ColumnAccessor accessor = new ColumnAccessor(table.getcObejct(), CommonTest.getDevice()); + int[] columnData = accessor.pullInt(1); + assertEquals(new Long(columnData.length), table.getRowCount()); + int[] tableData = table.getIntData(); + + for (int i = 0; i < columnData.length; i++) { + assertEquals(columnData[i], tableData[i * table.getColumnCount().intValue() + 1]); + } + } + + @Test + public void getIntSecondColumnFromHomogenTableWithSubsetOfRows() throws Exception { + int[] data = {5, 8, 40, 10, 90, 3, 53, 33, 85, 15}; + HomogenTable table = new HomogenTable(5, 2, + data, CommonTest.getDevice()); + + ColumnAccessor accessor = new ColumnAccessor(table.getcObejct(), CommonTest.getDevice()); + int[] columnData = accessor.pullInt(0, 1 , 3); + assertEquals(new Long(columnData.length), new Long(2)); + int[] tableData = table.getIntData(); + for (int i = 0; i < columnData.length; i++) { + assertEquals(columnData[i], tableData[2 + i * table.getColumnCount().intValue()]); + } + } +} diff --git a/mllib-dal/src/test/java/com/intel/oneapi/dal/table/CommonTest.java b/mllib-dal/src/test/java/com/intel/oneapi/dal/table/CommonTest.java new file mode 100644 index 000000000..3b86ce455 --- /dev/null +++ b/mllib-dal/src/test/java/com/intel/oneapi/dal/table/CommonTest.java @@ -0,0 +1,12 @@ +package com.intel.oneapi.dal.table; + +public class CommonTest { + public static Common.ComputeDevice getDevice(){ + String device = System.getProperty("computeDevice"); + if(device == null){ + device = "HOST"; + } + System.out.println("getDevice : " + device); + return Common.ComputeDevice.getOrdinalByName(device); + } +} diff --git a/mllib-dal/src/test/java/com/intel/oneapi/dal/table/HomogenTableTest.java b/mllib-dal/src/test/java/com/intel/oneapi/dal/table/HomogenTableTest.java index dae3cd333..e94f3306a 100644 --- a/mllib-dal/src/test/java/com/intel/oneapi/dal/table/HomogenTableTest.java +++ b/mllib-dal/src/test/java/com/intel/oneapi/dal/table/HomogenTableTest.java @@ -8,23 +8,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class HomogenTableTest { - - private String getDevice(){ - String device = System.getProperty("computeDevice"); - if(device == null){ - device = "CPU"; - } - System.out.println("getDevice : " + device); - return device; - } - + @Test // can construct rowmajor int table 5x2 public void createRowmajorIntTable() throws Exception { int[] data = {1, 2, 3, 4, 5, 6, 10, 80, 10, 11}; HomogenTable table = new HomogenTable(5, 2, - data, ROW_MAJOR.ordinal(), Common.ComputeDevice.getOrdinalByName(getDevice())); - + data, CommonTest.getDevice()); assertEquals(true, table.hasData()); assertEquals(new Long(2), table.getColumnCount()); assertEquals(new Long(5), table.getRowCount()); @@ -44,7 +34,7 @@ public void createRowmajorDoubleTable() throws Exception { double[] data = {5.236359d, 8.718667d, 40.724176d, 10.770023d, 90.119887d, 3.815366d, 53.620204d, 33.219769d, 85.208661d, 15.966239d}; HomogenTable table = new HomogenTable(5, 2, - data, ROW_MAJOR.ordinal(), Common.ComputeDevice.getOrdinalByName(getDevice())); + data, CommonTest.getDevice()); assertEquals(true, table.hasData()); assertEquals(new Long(2), table.getColumnCount()); @@ -64,7 +54,8 @@ public void createRowmajorDoubleTable() throws Exception { public void createRowmajorLongTable() throws Exception { long[] data = {1L, 2L, 3L, 4L, 5L, 6L, 10L, 80L, 10L, 11L}; HomogenTable table = new HomogenTable(5, 2, - data, ROW_MAJOR.ordinal(), Common.ComputeDevice.getOrdinalByName(getDevice())); + data, CommonTest.getDevice()); + assertEquals(true, table.hasData()); assertEquals(new Long(2), table.getColumnCount()); assertEquals(new Long(5), table.getRowCount()); @@ -84,7 +75,8 @@ public void createRowmajorFloatTable() throws Exception { float[] data = {5.236359f, 8.718667f, 40.724176f, 10.770023f, 90.119887f, 3.815366f, 53.620204f, 33.219769f, 85.208661f, 15.966239f}; HomogenTable table = new HomogenTable(5, 2, - data, ROW_MAJOR.ordinal(), Common.ComputeDevice.getOrdinalByName(getDevice())); + data, CommonTest.getDevice()); + assertEquals(true, table.hasData()); assertEquals(new Long(2), table.getColumnCount()); assertEquals(new Long(5), table.getRowCount()); @@ -105,7 +97,8 @@ public void createRowmajorFloatTable() throws Exception { public void createColmajorIntTable() throws Exception { int[] data = {1, 2, 3, 4, 5, 6, 10, 80, 10, 11}; HomogenTable table = new HomogenTable(5, 2, - data, COLUMN_MAJOR.ordinal(), Common.ComputeDevice.getOrdinalByName(getDevice())); + data, COLUMN_MAJOR, CommonTest.getDevice()); + assertEquals(true, table.hasData()); assertEquals(new Long(2), table.getColumnCount()); assertEquals(new Long(5), table.getRowCount()); @@ -125,7 +118,8 @@ public void createColmajorFloatTable() throws Exception { float[] data = {5.236359f, 8.718667f, 40.724176f, 10.770023f, 90.119887f, 3.815366f, 53.620204f, 33.219769f, 85.208661f, 15.966239f}; HomogenTable table = new HomogenTable(5, 2, - data, COLUMN_MAJOR.ordinal(), Common.ComputeDevice.getOrdinalByName(getDevice())); + data, COLUMN_MAJOR, CommonTest.getDevice()); + assertEquals(true, table.hasData()); assertEquals(new Long(2), table.getColumnCount()); assertEquals(new Long(5), table.getRowCount()); @@ -144,7 +138,8 @@ public void createColmajorFloatTable() throws Exception { public void createColmajorLongTable() throws Exception { long[] data = {1L, 2L, 3L, 4L, 5L, 6L, 10L, 80L, 10L, 11L}; HomogenTable table = new HomogenTable(5, 2, - data, COLUMN_MAJOR.ordinal(), Common.ComputeDevice.getOrdinalByName(getDevice())); + data, COLUMN_MAJOR, CommonTest.getDevice()); + assertEquals(true, table.hasData()); assertEquals(new Long(2), table.getColumnCount()); assertEquals(new Long(5), table.getRowCount()); @@ -164,7 +159,8 @@ public void createColmajorDoubleTable() throws Exception { double[] data = {5.236359d, 8.718667d, 40.724176d, 10.770023d, 90.119887d, 3.815366d, 53.620204d, 33.219769d, 85.208661d, 15.966239d}; HomogenTable table = new HomogenTable(5, 2, - data, COLUMN_MAJOR.ordinal(), Common.ComputeDevice.getOrdinalByName(getDevice())); + data, COLUMN_MAJOR, CommonTest.getDevice()); + assertEquals(true, table.hasData()); assertEquals(new Long(2), table.getColumnCount()); assertEquals(new Long(5), table.getRowCount()); diff --git a/mllib-dal/src/test/java/com/intel/oneapi/dal/table/RowAccessorTest.java b/mllib-dal/src/test/java/com/intel/oneapi/dal/table/RowAccessorTest.java new file mode 100644 index 000000000..106c2bd8b --- /dev/null +++ b/mllib-dal/src/test/java/com/intel/oneapi/dal/table/RowAccessorTest.java @@ -0,0 +1,55 @@ +package com.intel.oneapi.dal.table; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +public class RowAccessorTest { + + @Test + public void readDoubleTableDataFromRowAccessor() { + double[] data = {5.236359d, 8.718667d, 40.724176d, 10.770023d, 90.119887d, 3.815366d, + 53.620204d, 33.219769d, 85.208661d, 15.966239d}; + HomogenTable table = new HomogenTable(5, 2, + data, CommonTest.getDevice()); + + RowAccessor accessor = new RowAccessor(table.getcObejct(), CommonTest.getDevice()); + double[] rowData = accessor.pullDouble( 0 , table.getRowCount()); + assertEquals(new Long(rowData.length), new Long(table.getColumnCount() * table.getRowCount())); + assertArrayEquals(rowData, data); + for (int i = 0; i < rowData.length; i++) { + assertEquals(rowData[i], data[i]); + } + } + @Test + public void readFloatTableDataFromRowAccessor() { + float[] data = {5.236359f, 8.718667f, 40.724176f, 10.770023f, 90.119887f, 3.815366f, + 53.620204f, 33.219769f, 85.208661f, 15.966239f}; + HomogenTable table = new HomogenTable(5, 2, + data, CommonTest.getDevice()); + + RowAccessor accessor = new RowAccessor(table.getcObejct(), CommonTest.getDevice()); + float[] rowData = accessor.pullFloat( 0 , table.getRowCount()); + assertEquals(new Long(rowData.length), new Long(table.getColumnCount() * table.getRowCount())); + assertArrayEquals(rowData, data); + for (int i = 0; i < rowData.length; i++) { + assertEquals(rowData[i], data[i]); + } + } + @Test + public void readIntTableDataFromRowAccessor() { + int[] data = {5, 8, 40, 10, 90, 3, 53, 33, 85, 15}; + HomogenTable table = new HomogenTable(5, 2, + data, CommonTest.getDevice()); + + RowAccessor accessor = new RowAccessor(table.getcObejct(), CommonTest.getDevice()); + int[] rowData = accessor.pullInt( 0 , table.getRowCount()); + assertEquals(new Long(rowData.length), new Long(table.getColumnCount() * table.getRowCount())); + assertArrayEquals(rowData, data); + for (int i = 0; i < rowData.length; i++) { + assertEquals(rowData[i], data[i]); + } + } +} diff --git a/mllib-dal/src/test/scala/com/intel/oap/mllib/ConvertHomogenTableSuite.scala b/mllib-dal/src/test/scala/com/intel/oap/mllib/ConvertHomogenTableSuite.scala index f1000a624..6490dd53d 100644 --- a/mllib-dal/src/test/scala/com/intel/oap/mllib/ConvertHomogenTableSuite.scala +++ b/mllib-dal/src/test/scala/com/intel/oap/mllib/ConvertHomogenTableSuite.scala @@ -1,12 +1,16 @@ package com.intel.oap.mllib -import com.intel.oneapi.dal.table.Common +import com.intel.daal.data_management.data.{Matrix => DALMatrix, NumericTable} +import com.intel.daal.services.DaalContext +import com.intel.oneapi.dal.table.{Common, HomogenTable} import com.intel.oneapi.dal.table.Common.DataLayout.ROW_MAJOR import com.intel.oneapi.dal.table.Common.DataType.FLOAT64 import org.apache.spark.internal.Logging import org.apache.spark.ml.FunctionsSuite -import org.apache.spark.ml.linalg.{Matrices, Vector, Vectors} +import org.apache.spark.ml.linalg.{DenseMatrix, Matrices, Vector, Vectors} import org.apache.spark.mllib.linalg.{Vector => OldVector, Vectors => OldVectors} +import org.apache.spark.mllib.linalg.{DenseMatrix => OldDenseMatrix, Matrix => OldMatrix, Vector => OldVector} + class ConvertHomogenTableSuite extends FunctionsSuite with Logging { @@ -86,6 +90,93 @@ class ConvertHomogenTableSuite extends FunctionsSuite with Logging { assert(table.getDoubleData === convertArray(data)) } + test("test convert homogentable 1xN to vector "){ + val data = Array(5.308206,9.869278,1.018934,4.292158,6.081011,6.585723,2.411094,4.767308,-3.256320,-6.029562) + + val expectData = Array(5.308206,9.869278) + + val table = new HomogenTable(5, 2, data, getDevice) + val vector = OneDAL.homogenTable1xNToVector(table, getDevice) + + assert(expectData=== vector.toArray) + } + + test("test convert numerictable Nx1 to vector "){ + val data = Array( + Vectors.dense(5.308206,9.869278,1.018934,4.292158,6.081011,6.585723,2.411094,4.767308,-3.256320,-6.029562), + Vectors.dense(7.279464,0.390664,-9.619284,3.435376,-4.769490,-4.873188,-0.118791,-5.117316,-0.418655,-0.475422), + Vectors.dense(-6.615791,-6.191542,0.402459,-9.743521,-9.990568,9.105346,1.691312,-2.605659,9.534952,-7.829027), + Vectors.dense(-4.792007,-2.491098,-2.939393,8.086467,3.773812,-9.997300,0.222378,8.995244,-5.753282,6.091060), + Vectors.dense(7.700725,-6.414918,1.684476,-8.983361,4.284580,-9.017608,0.552379,-7.705741,2.589852,0.411561), + Vectors.dense(6.991900,-1.063721,9.321163,-0.429719,-2.167696,-1.736425,-0.919139,6.980681,-0.711914,3.414347), + Vectors.dense(5.794488,-1.062261,0.955322,0.389642,3.012921,-9.953994,-3.197309,3.992421,-6.935902,8.147622), + Vectors.dense(-2.486670,6.973242,-4.047004,-5.655629,5.081786,5.533859,7.821403,2.763313,-0.454056,6.554309), + Vectors.dense(2.204855,7.839522,7.381886,1.618749,-6.566877,7.584285,-8.355983,-5.501410,-8.191205,-2.608499), + Vectors.dense(-9.948613,-8.941953,-8.106389,4.863542,5.852806,-1.659259,6.342504,-8.190106,-3.110330,-7.484658), + ) + val expectData = Array(5.308206, 7.279464, -6.615791, -4.792007, 7.700725, 6.991900, 5.794488, -2.486670, 2.204855, -9.948613) + val matrix = OneDAL.makeNumericTable(data) + val vector = OneDAL.numericTableNx1ToVector(matrix) + assert(expectData === vector.toArray) + } + + test("test convert homogentable Nx1 to vector "){ + val data = Array(5.236359d, 8.718667d, + 40.724176d, 10.770023d, + 90.119887d, 3.815366d, + 53.620204d, 33.219769d, + 85.208661d, 15.966239d) + val expectData = Array(5.236359d, 40.724176d, 90.119887d, 53.620204d, 85.208661d) + + val table = new HomogenTable(5, 2, data, getDevice) + val vector = OneDAL.homogenTableNx1ToVector(table.getcObejct(), getDevice) + assert(expectData === vector.toArray) + } + + test("test convert homogentable to matrix "){ + val data = Array(5.236359d, 8.718667d, + 40.724176d, 10.770023d, + 90.119887d, 3.815366d, + 53.620204d, 33.219769d, + 85.208661d, 15.966239d) + val expectMatrix = new DenseMatrix(5, 2, data, isTransposed = true) + val table = new HomogenTable(5, 2, data, getDevice) + val matrix = OneDAL.homogenTableToMatrix(table) + assert(expectMatrix === matrix) + } + + test("test convert homogentable to oldmatrix "){ + val data = Array(5.236359d, 8.718667d, + 40.724176d, 10.770023d, + 90.119887d, 3.815366d, + 53.620204d, 33.219769d, + 85.208661d, 15.966239d) + val expectMatrix = new OldDenseMatrix(5, 2, data, isTransposed = true) + val table = new HomogenTable(5, 2, data, getDevice) + val matrix = OneDAL.homogenTableToOldMatrix(table) + + assert(expectMatrix === matrix) + } + + test("test convert homogentable to vector "){ + val data = Array( + Vectors.dense(5.308206,9.869278,1.018934,4.292158,6.081011,6.585723,2.411094,4.767308,-3.256320,-6.029562), + Vectors.dense(7.279464,0.390664,-9.619284,3.435376,-4.769490,-4.873188,-0.118791,-5.117316,-0.418655,-0.475422), + Vectors.dense(-6.615791,-6.191542,0.402459,-9.743521,-9.990568,9.105346,1.691312,-2.605659,9.534952,-7.829027), + Vectors.dense(-4.792007,-2.491098,-2.939393,8.086467,3.773812,-9.997300,0.222378,8.995244,-5.753282,6.091060), + Vectors.dense(7.700725,-6.414918,1.684476,-8.983361,4.284580,-9.017608,0.552379,-7.705741,2.589852,0.411561), + Vectors.dense(6.991900,-1.063721,9.321163,-0.429719,-2.167696,-1.736425,-0.919139,6.980681,-0.711914,3.414347), + Vectors.dense(5.794488,-1.062261,0.955322,0.389642,3.012921,-9.953994,-3.197309,3.992421,-6.935902,8.147622), + Vectors.dense(-2.486670,6.973242,-4.047004,-5.655629,5.081786,5.533859,7.821403,2.763313,-0.454056,6.554309), + Vectors.dense(2.204855,7.839522,7.381886,1.618749,-6.566877,7.584285,-8.355983,-5.501410,-8.191205,-2.608499), + Vectors.dense(-9.948613,-8.941953,-8.106389,4.863542,5.852806,-1.659259,6.342504,-8.190106,-3.110330,-7.484658), + ) + val arrayData = convertArray(data) + val table = new HomogenTable(10, 10, arrayData, getDevice) + val array = OneDAL.homogenTableToVectors(table, getDevice) + assert(data === array) + } + def convertArray(arrayVectors: Array[Vector]): Array[Double] = { val numCols = arrayVectors.head.size val numRows: Int = arrayVectors.size @@ -118,10 +209,9 @@ class ConvertHomogenTableSuite extends FunctionsSuite with Logging { arrayDouble } - private def getDevice = { + private def getDevice: Common.ComputeDevice = { val device = System.getProperty("computeDevice") var computeDevice: Common.ComputeDevice = Common.ComputeDevice.CPU - System.out.println("getDevice : " + device) if(device != null) { device.toUpperCase match { case "HOST" => computeDevice = Common.ComputeDevice.HOST diff --git a/mllib-dal/test.sh b/mllib-dal/test.sh index f3b31e9aa..039e92146 100755 --- a/mllib-dal/test.sh +++ b/mllib-dal/test.sh @@ -47,8 +47,10 @@ fi export OAP_MLLIB_TESTING=true suiteArray=( - "com.intel.oap.mllib.ConvertHomogenTableSuite" \ - "HomogenTableTest" + "HomogenTableTest" \ + "ColumnAccessorTest" \ + "RowAccessorTest" \ + "com.intel.oap.mllib.ConvertHomogenTableSuite" ) MVN_NO_TRANSFER_PROGRESS=