Merge branch 'develop' of https://github.com/GPUOpen-ProfessionalComp…

…ute-Libraries/MIVisionX into lk/rocalTensor

CMakeLists.txt

@@ -44,6 +44,7 @@ set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 option(ENHANCED_MESSAGE "MIVisionX Enhanced Message Option"        ON)
 option(NEURAL_NET       "Build MIVisionX with Neural Net Support"  ON)
 option(ROCAL            "Build MIVisionX with ROCAL Support"       ON)
+option(ROCAL_PYTHON     "Build rocAL Python Package"               ON)
 option(LOOM             "Build MIVisionX with LOOM Support"        ON)
 option(GPU_SUPPORT      "Build MIVisionX with GPU Support"         ON)
 option(MIGRAPHX         "Build MIVisionX with MIGraphX Support"    ON)
@@ -119,6 +120,7 @@ message("-- ${BoldBlue}MIVisionX Build Type -- ${CMAKE_BUILD_TYPE}${ColourReset}
 message("-- ${Cyan}MIVisionX Developer Options${ColourReset}")
 message("-- ${Cyan}     -D NEURAL_NET=${NEURAL_NET} [Turn ON/OFF Neural Net Modules (default:ON)]${ColourReset}")
 message("-- ${Cyan}     -D ROCAL=${ROCAL} [Turn ON/OFF ROCAL Modules (default:ON)]${ColourReset}")
+message("-- ${Cyan}     -D ROCAL_PYTHON=${ROCAL_PYTHON} [Turn ON/OFF ROCAL Modules (default:ON)]${ColourReset}")
 message("-- ${Cyan}     -D LOOM=${LOOM} [Turn ON/OFF LOOM Modules (default:ON)]${ColourReset}")
 message("-- ${Cyan}     -D GPU_SUPPORT=${GPU_SUPPORT} [Turn ON/OFF GPU support (default:ON)]${ColourReset}")
 message("-- ${Cyan}     -D MIGRAPHX=${MIGRAPHX} [Turn ON/OFF MIGraphX Module (default:ON)]${ColourReset}")
@@ -137,6 +139,11 @@ add_subdirectory(utilities)
 
 if(ROCAL)
   add_subdirectory(rocAL)
+  if(ROCAL_PYTHON)
+    add_subdirectory(rocAL/rocAL_pybind)
+  else()
+    message("-- ${Cyan}ROCAL Python Module turned OFF by user option -D ROCAL_PYTHON=OFF ${ColourReset}")
+  endif()
 else()
   message("-- ${Cyan}ROCAL Module turned OFF by user option -D ROCAL=OFF ${ColourReset}")
 endif()

README.md

@@ -269,10 +269,15 @@ macOS [build instructions](https://github.com/GPUOpen-ProfessionalCompute-Librar
     ```
     mkdir build-hip
     cd build-hip
-    cmake ../
-    make -j8
+    sudo cmake ../
+    sudo make -j8
+    sudo cmake --build . --target PyPackageInstall
     sudo make install
     ```
+    **Note:**
+    + `PyPackageInstall` used for rocal_pybind installation
+    + rocal_pybind not supported on windows.
+    + `sudo` required for pybind installation
 
   + Instructions for building MIVisionX with **OPENCL** GPU backend:
 
@@ -285,8 +290,9 @@ macOS [build instructions](https://github.com/GPUOpen-ProfessionalCompute-Librar
     ```
     mkdir build-ocl
     cd build-ocl
-    cmake -DBACKEND=OPENCL ../
-    make -j8
+    sudo cmake -DBACKEND=OPENCL ../
+    sudo make -j8
+    sudo cmake --build . --target PyPackageInstall
     sudo make install
     ```
 
@@ -295,6 +301,9 @@ macOS [build instructions](https://github.com/GPUOpen-ProfessionalCompute-Librar
   if an app interested in installing MIVisionX with both GPU backends, then add **-DCMAKE_INSTALL_PREFIX** in the cmake
   commands to install MIVisionX with OPENCL and HIP backends into two separate custom folders.
   + vx_winml is not supported on Linux
+  + ```PyPackageInstall``` used for rocal_pybind installation
+  + rocal_pybind not supported on windows
+  + `sudo` required for pybind installation
 
 ## Verify the Installation
 

rocAL/rocAL/include/pipeline/commons.h

@@ -126,7 +126,6 @@ struct Timing
     long long unsigned label_load_time= 0;
     long long unsigned bb_load_time= 0;
     long long unsigned mask_load_time = 0;
-    long long unsigned shuffle_time = 0;
     long long unsigned video_read_time= 0;
     long long unsigned video_decode_time= 0;
     long long unsigned video_process_time= 0;

rocAL/rocAL/include/readers/image/caffe2_lmdb_record_reader.h

@@ -67,7 +67,7 @@ class Caffe2LMDBRecordReader : public Reader
     int close() override;
 
     Caffe2LMDBRecordReader();
-    unsigned long long get_shuffle_time() override {return 0;}
+
 private:
     //! opens the folder containnig the images
     Reader::Status Caffe2_LMDB_reader();
@@ -108,7 +108,6 @@ class Caffe2LMDBRecordReader : public Reader
     void read_image(unsigned char* buff, std::string file_name);
     void read_image_names();
     std::map <std::string, uint> _image_record_starting;
-    TimingDBG _shuffle_time;
     int _open_env = 1;
     int rc;
     MDB_env* _read_mdb_env;

rocAL/rocAL/include/readers/image/caffe_lmdb_record_reader.h

@@ -66,7 +66,7 @@ class CaffeLMDBRecordReader : public Reader{
     int close() override;
 
     CaffeLMDBRecordReader();
-    unsigned long long get_shuffle_time() override {return 0;}
+
 private:
     //! opens the folder containnig the images
     Reader::Status folder_reading();
@@ -110,7 +110,6 @@ class CaffeLMDBRecordReader : public Reader{
     void read_image(unsigned char* buff, std::string _file_name);
     void read_image_names();
     std::map <std::string, uint> _image_record_starting;
-    TimingDBG _shuffle_time;
     int _open_env = 1;
     int rc;
     void open_env_for_read_image();

rocAL/rocAL/include/readers/image/cifar10_data_reader.h

@@ -62,7 +62,6 @@ class CIFAR10DataReader : public Reader {
     CIFAR10DataReader();
 
     unsigned get_file_index() { return _last_file_idx;}
-    unsigned long long get_shuffle_time() override {return 0;}
 
 private:
     //! opens the folder containing the images

rocAL/rocAL/include/readers/image/coco_file_source_reader.h

@@ -59,7 +59,6 @@ class COCOFileSourceReader : public Reader {
     std::string id() override { return _last_id;};
 
     unsigned count_items() override;
-    unsigned long long get_shuffle_time() override {return _shuffle_time.get_timing();};
 
     ~COCOFileSourceReader() override;
 
@@ -104,5 +103,4 @@ class COCOFileSourceReader : public Reader {
     void incremenet_file_id() { _file_id++; }
     void replicate_last_image_to_fill_last_shard();
     void replicate_last_batch_to_pad_partial_shard();
-    TimingDBG _shuffle_time;
 };

rocAL/rocAL/include/readers/image/file_source_reader.h

@@ -60,7 +60,6 @@ class FileSourceReader : public Reader {
     ~FileSourceReader() override;
 
     int close() override;
-    unsigned long long get_shuffle_time() override {return _shuffle_time.get_timing();};
 
     FileSourceReader();
 
@@ -97,6 +96,5 @@ class FileSourceReader : public Reader {
     void incremenet_file_id() { _file_id++; }
     void replicate_last_image_to_fill_last_shard();
     void replicate_last_batch_to_pad_partial_shard();
-    TimingDBG _shuffle_time;
 };
 

rocAL/rocAL/include/readers/image/mxnet_recordio_reader.h

@@ -63,7 +63,7 @@ class MXNetRecordIOReader : public Reader{
     int close() override;
 
     MXNetRecordIOReader();
-    unsigned long long get_shuffle_time() override {return 0;}
+
 private:
     //! opens the folder containnig the images
     Reader::Status record_reading();
@@ -108,6 +108,5 @@ class MXNetRecordIOReader : public Reader{
     const uint32_t _kMagic = 0xced7230a;
     int64_t _seek_pos, _data_size_to_read;
     ImageRecordIOHeader _hdr;
-    TimingDBG _shuffle_time;
 };
 

rocAL/rocAL/include/readers/image/reader.h

@@ -149,8 +149,5 @@ class Reader
     //! Returns the number of items remained in this resource
     virtual unsigned count_items() = 0;
 
-    //! return shuffle_time if applicable
-    virtual unsigned long long get_shuffle_time() = 0;
-
     virtual ~Reader() = default;
 };

rocAL/rocAL/include/readers/image/tf_record_reader.h

@@ -61,7 +61,6 @@ class TFRecordReader : public Reader
     std::string id() override { return _last_id;};
 
     unsigned count_items() override;
-    unsigned long long get_shuffle_time() override {return _shuffle_time.get_timing();};
 
     ~TFRecordReader() override;
 
@@ -115,5 +114,4 @@ class TFRecordReader : public Reader
     Reader::Status read_image(unsigned char* buff, std::string record_file_name, uint file_size);
     Reader::Status read_image_names(std::ifstream &file_contents, uint file_size);
     std::map <std::string, uint> _image_record_starting;
-    TimingDBG _shuffle_time;
 };

rocAL/rocAL/include/readers/video/sequence_file_source_reader.h

@@ -60,7 +60,6 @@ class SequenceFileSourceReader : public Reader {
     ~SequenceFileSourceReader() override;
 
     int close() override;
-    unsigned long long get_shuffle_time() override {return _shuffle_time.get_timing();};
 
     SequenceFileSourceReader();
 
@@ -105,6 +104,5 @@ class SequenceFileSourceReader : public Reader {
     void incremenet_sequence_id() { _sequence_id++; }
     void replicate_last_sequence_to_fill_last_shard();
     void replicate_last_batch_to_pad_partial_shard();
-    TimingDBG _shuffle_time;
 };
 

rocAL/rocAL/include/readers/video/video_file_source_reader.h

@@ -51,8 +51,6 @@ class VideoFileSourceReader : public VideoReader
 
     ~VideoFileSourceReader() override;
 
-    unsigned long long get_shuffle_time() override { return _shuffle_time.get_timing(); };
-
     VideoFileSourceReader();
 private:
     std::string _folder_path;
@@ -88,6 +86,5 @@ class VideoFileSourceReader : public VideoReader
     void replicate_last_sequence_to_fill_last_shard();
     void replicate_last_batch_to_pad_partial_shard();
     VideoReader::Status create_sequence_info();
-    TimingDBG _shuffle_time;
 };
 #endif

rocAL/rocAL/include/readers/video/video_reader.h

@@ -113,9 +113,6 @@ class VideoReader
     //! Returns the number of items remained in this resource
     virtual unsigned count_items() = 0;
 
-    //! return shuffle_time if applicable
-    virtual unsigned long long get_shuffle_time() = 0;
-
     virtual ~VideoReader() = default;
 };
 #endif

rocAL/rocAL/source/api/rocal_api_info.cpp

@@ -120,7 +120,6 @@ TimingInfo
 {
     auto context = static_cast<Context *>(p_context);
     auto info = context->timing();
-    // INFO("shuffle time "+ TOSTR(info.shuffle_time)); to display time taken for shuffling dataset
     // INFO("bbencode time "+ TOSTR(info.bb_process_time)); //to display time taken for bbox encoder
     if (context->master_graph->is_video_loader())
         return {info.video_read_time, info.video_decode_time, info.video_process_time, info.copy_to_output};

rocAL/rocAL/source/loaders/image/image_read_and_decode.cpp

@@ -51,7 +51,6 @@ ImageReadAndDecode::timing()
     Timing t;
     t.image_decode_time = _decode_time.get_timing();
     t.image_read_time = _file_load_time.get_timing();
-    t.shuffle_time = _reader->get_shuffle_time();
     return t;
 }
 

rocAL/rocAL/source/loaders/video/video_read_and_decode.cpp

@@ -51,7 +51,6 @@ VideoReadAndDecode::timing()
     Timing t;
     t.video_decode_time = _decode_time.get_timing();
     t.video_read_time = _file_load_time.get_timing();
-    t.shuffle_time = _video_reader->get_shuffle_time();
     return t;
 }
 

rocAL/rocAL/source/readers/image/caffe2_lmdb_record_reader.cpp

@@ -34,8 +34,7 @@ THE SOFTWARE.
 using namespace std;
 namespace filesys = boost::filesystem;
 
-Caffe2LMDBRecordReader::Caffe2LMDBRecordReader():
-_shuffle_time("shuffle_time", DBG_TIMING)
+Caffe2LMDBRecordReader::Caffe2LMDBRecordReader()
 {
     _src_dir = nullptr;
     _sub_dir = nullptr;
@@ -79,10 +78,8 @@ Reader::Status Caffe2LMDBRecordReader::initialize(ReaderConfig desc)
         }
     }
     //shuffle dataset if set
-    _shuffle_time.start();
     if( ret==Reader::Status::OK && _shuffle)
         std::random_shuffle(_file_names.begin(), _file_names.end());
-    _shuffle_time.end();
 
     return ret;
 
@@ -133,10 +130,8 @@ Caffe2LMDBRecordReader::release()
 
 void Caffe2LMDBRecordReader::reset()
 {
-    _shuffle_time.start();
     if(_shuffle)
         std::random_shuffle(_file_names.begin(), _file_names.end());
-    _shuffle_time.end();
     _read_counter = 0;
     _curr_file_idx = 0;
 }

rocAL/rocAL/source/readers/image/caffe_lmdb_record_reader.cpp

@@ -37,8 +37,7 @@ using caffe_protos::Datum;
 
 namespace filesys = boost::filesystem;
 
-CaffeLMDBRecordReader::CaffeLMDBRecordReader():
-_shuffle_time("shuffle_time", DBG_TIMING)
+CaffeLMDBRecordReader::CaffeLMDBRecordReader()
 {
     _sub_dir = nullptr;
     _curr_file_idx = 0;
@@ -82,10 +81,8 @@ Reader::Status CaffeLMDBRecordReader::initialize(ReaderConfig desc)
         }
     }
     //shuffle dataset if set
-    _shuffle_time.start();
     if( ret==Reader::Status::OK && _shuffle)
         std::random_shuffle(_file_names.begin(), _file_names.end());
-    _shuffle_time.end();
 
     return ret;
 
@@ -142,10 +139,8 @@ int CaffeLMDBRecordReader::release()
 
 void CaffeLMDBRecordReader::reset()
 {
-    _shuffle_time.start();
     if (_shuffle)
         std::random_shuffle(_file_names.begin(), _file_names.end());
-    _shuffle_time.end();
     _read_counter = 0;
     _curr_file_idx = 0;
 }

rocAL/rocAL/source/readers/image/coco_file_source_reader.cpp

@@ -32,8 +32,7 @@ THE SOFTWARE.
 namespace filesys = boost::filesystem;
 #define USE_STDIO_FILE 0
 
-COCOFileSourceReader::COCOFileSourceReader():
-_shuffle_time("shuffle_time", DBG_TIMING)
+COCOFileSourceReader::COCOFileSourceReader()
 {
     _src_dir = nullptr;
     _sub_dir = nullptr;
@@ -88,10 +87,8 @@ Reader::Status COCOFileSourceReader::initialize(ReaderConfig desc)
         }
     }
     //shuffle dataset if set
-    _shuffle_time.start();
     if (ret == Reader::Status::OK && _shuffle)
         std::random_shuffle(_file_names.begin(), _file_names.end());
-    _shuffle_time.end();
     return ret;
 }
 

rocAL/rocAL/source/readers/image/file_source_reader.cpp

@@ -28,8 +28,7 @@ THE SOFTWARE.
 
 namespace filesys = boost::filesystem;
 
-FileSourceReader::FileSourceReader():
-_shuffle_time("shuffle_time", DBG_TIMING)
+FileSourceReader::FileSourceReader()
 {
     _src_dir = nullptr;
     _sub_dir = nullptr;
@@ -73,12 +72,10 @@ Reader::Status FileSourceReader::initialize(ReaderConfig desc)
         }
     }
     //shuffle dataset if set
-    _shuffle_time.start();
     if( ret==Reader::Status::OK && _shuffle)
         std::random_shuffle(_file_names.begin(), _file_names.end());
-    _shuffle_time.end();
-    return ret;
 
+    return ret;
 }
 
 void FileSourceReader::incremenet_read_ptr()
@@ -152,9 +149,7 @@ FileSourceReader::release()
 
 void FileSourceReader::reset()
 {
-    _shuffle_time.start();
     if (_shuffle) std::random_shuffle(_file_names.begin(), _file_names.end());
-    _shuffle_time.end();
     _read_counter = 0;
     _curr_file_idx = 0;
 }