rocAL - Video_pipeline Support

.gitignore

@@ -1,6 +1,5 @@
 build*
 docker_build*
-hip_build*
 .vscode*
 opencv-3.4.0/*
 openvx_test_results*

amd_openvx_extensions/amd_rpp/CMakeLists.txt

@@ -116,6 +116,7 @@ list(APPEND SOURCES
         source/RotatebatchPD.cpp
         source/SaturationbatchPD.cpp
         source/ScalebatchPD.cpp
+        source/SequenceRearrange.cpp
         source/SnowbatchPD.cpp
         source/SobelbatchPD.cpp
         source/SubtractbatchPD.cpp

amd_openvx_extensions/amd_rpp/include/internal_publishKernels.h

@@ -102,6 +102,7 @@ vx_status ResizeCropMirrorPD_Register(vx_context);
 vx_status RotatebatchPD_Register(vx_context);
 vx_status SaturationbatchPD_Register(vx_context);
 vx_status ScalebatchPD_Register(vx_context);
+vx_status SequenceRearrange_Register(vx_context);
 vx_status SnowbatchPD_Register(vx_context);
 vx_status SobelbatchPD_Register(vx_context);
 vx_status SubtractbatchPD_Register(vx_context);
@@ -195,5 +196,6 @@ vx_status WarpPerspectivebatchPD_Register(vx_context);
 #define VX_KERNEL_RPP_CROPMIRRORNORMALIZEBATCHPD_NAME        	"org.rpp.CropMirrorNormalizebatchPD"
 #define VX_KERNEL_RPP_CROPPD_NAME   							"org.rpp.CropPD"
 #define VX_KERNEL_RPP_RESIZECROPMIRRORPD_NAME      				"org.rpp.ResizeCropMirrorPD"
+#define VX_KERNEL_RPP_SEQUENCEREARRANGE_NAME                    "org.rpp.SequenceRearrange"
 
 #endif //_AMDVX_EXT__PUBLISH_KERNELS_H_

amd_openvx_extensions/amd_rpp/include/kernels_rpp.h

@@ -102,15 +102,16 @@ extern "C"
         VX_KERNEL_RPP_SCALEBATCHPD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x45,
         VX_KERNEL_RPP_SATURATIONBATCHPD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x46,
         VX_KERNEL_RPP_SOBELBATCHPD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x47,
-        VX_KERNEL_RPP_THRESHOLDINGBATCHPD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x48,
-        VX_KERNEL_RPP_TENSORADD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x49,
-        VX_KERNEL_RPP_TENSORSUBTRACT = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x4a,
-        VX_KERNEL_RPP_TENSORMULTIPLY = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x4b,
-        VX_KERNEL_RPP_TENSORMATRIXMULTIPLY = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x4c,
-        VX_KERNEL_RPP_TENSORLOOKUP = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x4d,
-        VX_KERNEL_RPP_VIGNETTEBATCHPD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x4e,
-        VX_KERNEL_RPP_WARPAFFINEBATCHPD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x4f,
-        VX_KERNEL_RPP_WARPPERSPECTIVEBATCHPD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x50,
+        VX_KERNEL_RPP_SEQUENCEREARRANGE = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) +0x48,
+        VX_KERNEL_RPP_THRESHOLDINGBATCHPD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x49,
+        VX_KERNEL_RPP_TENSORADD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x4a,
+        VX_KERNEL_RPP_TENSORSUBTRACT = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x4b,
+        VX_KERNEL_RPP_TENSORMULTIPLY = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x4c,
+        VX_KERNEL_RPP_TENSORMATRIXMULTIPLY = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x4d,
+        VX_KERNEL_RPP_TENSORLOOKUP = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x4e,
+        VX_KERNEL_RPP_VIGNETTEBATCHPD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x4f,
+        VX_KERNEL_RPP_WARPAFFINEBATCHPD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x50,
+        VX_KERNEL_RPP_WARPPERSPECTIVEBATCHPD = VX_KERNEL_BASE(VX_ID_AMD, VX_LIBRARY_RPP) + 0x51
     };
 
 #ifdef __cplusplus

amd_openvx_extensions/amd_rpp/include/vx_ext_rpp.h

@@ -125,7 +125,7 @@ extern  "C" SHARED_PUBLIC vx_node VX_API_CALL vxExtrppNode_ThresholdingbatchPD(v
 extern  "C" SHARED_PUBLIC vx_node VX_API_CALL vxExtrppNode_VignettebatchPD(vx_graph graph,vx_image pSrc,vx_array srcImgWidth,vx_array srcImgHeight,vx_image pDst,vx_array stdDev,vx_uint32 nbatchSize);
 extern  "C" SHARED_PUBLIC vx_node VX_API_CALL vxExtrppNode_WarpAffinebatchPD(vx_graph graph,vx_image pSrc,vx_array srcImgWidth,vx_array srcImgHeight,vx_image pDst,vx_array dstImgWidth,vx_array dstImgHeight,vx_array affine,vx_uint32 nbatchSize);
 extern  "C" SHARED_PUBLIC vx_node VX_API_CALL vxExtrppNode_WarpPerspectivebatchPD(vx_graph graph,vx_image pSrc,vx_array srcImgWidth,vx_array srcImgHeight,vx_image pDst,vx_array dstImgWidth,vx_array dstImgHeight,vx_array perspective,vx_uint32 nbatchSize);
-
+extern  "C" SHARED_PUBLIC vx_node VX_API_CALL vxExtrppNode_SequenceRearrange(vx_graph graph,vx_image pSrc,vx_image pDst, vx_array newOrder,vx_uint32 newSequenceLength, vx_uint32 sequenceLength, vx_uint32 sequenceCount);
 #ifdef __cplusplus
 }
 #endif

amd_openvx_extensions/amd_rpp/source/internal_publishKernels.cpp

@@ -123,6 +123,7 @@ vx_status get_kernels_to_publish()
     STATUS_ERROR_CHECK(ADD_KERENEL(ResizeCropMirrorPD_Register));
     STATUS_ERROR_CHECK(ADD_KERENEL(Copy_Register));
     STATUS_ERROR_CHECK(ADD_KERENEL(Nop_Register));
+    STATUS_ERROR_CHECK(ADD_KERENEL(SequenceRearrange_Register));
     return status;
 }
 

amd_openvx_extensions/amd_rpp/source/kernel_rpp.cpp

@@ -1800,6 +1800,30 @@ VX_API_CALL vx_node VX_API_CALL vxExtrppNode_Nop(vx_graph graph, vx_image pSrc,
     return node;
 }
 
+VX_API_CALL vx_node VX_API_CALL  vxExtrppNode_SequenceRearrange(vx_graph graph,vx_image pSrc,vx_image pDst, vx_array newOrder, vx_uint32 newSequenceLength, vx_uint32 sequenceLength, vx_uint32 sequenceCount)
+{
+	vx_node node = NULL;
+	vx_context context = vxGetContext((vx_reference)graph);
+	if(vxGetStatus((vx_reference)context) == VX_SUCCESS) {
+		vx_uint32 dev_type = getGraphAffinity(graph);
+		vx_scalar DEV_TYPE = vxCreateScalar(vxGetContext((vx_reference)graph), VX_TYPE_UINT32, &dev_type);
+		vx_scalar NEWSEQUENCELENGTH = vxCreateScalar(vxGetContext((vx_reference)graph), VX_TYPE_UINT32, &newSequenceLength);
+		vx_scalar SEQUENCELENGTH = vxCreateScalar(vxGetContext((vx_reference)graph), VX_TYPE_UINT32, &sequenceLength);
+		vx_scalar SEQUENCECOUNT = vxCreateScalar(vxGetContext((vx_reference)graph), VX_TYPE_UINT32, &sequenceCount);
+		vx_reference params[] = {
+			(vx_reference) pSrc,
+			(vx_reference) pDst,
+			(vx_reference) newOrder,
+			(vx_reference) NEWSEQUENCELENGTH,
+			(vx_reference) SEQUENCELENGTH,
+			(vx_reference) SEQUENCECOUNT,
+			(vx_reference) DEV_TYPE
+		};
+		 node = createNode(graph, VX_KERNEL_RPP_SEQUENCEREARRANGE, params, 7);
+	}
+	return node;
+}
+
 // utility functions
 vx_node createNode(vx_graph graph, vx_enum kernelEnum, vx_reference params[], vx_uint32 num)
 {

apps/image_augmentation/image_augmentation.cpp

@@ -123,13 +123,15 @@ int main(int argc, const char ** argv)
 
     if(video_mode != 0)
     {
+        unsigned sequence_length = 3;
+        unsigned frame_step = 3;
+        unsigned frame_stride = 1;
         if (decode_height <= 0 || decode_width <= 0)
         {
             std::cout << "Output width and height is needed for video decode\n";
             return -1;
         }
-        input1 = raliVideoFileSource(handle, folderPath1, color_format, ((video_mode == 1) ? RaliDecodeDevice::RALI_HW_DECODE:RaliDecodeDevice::RALI_SW_DECODE)
-                , true, decode_width, decode_height, false);
+        input1 = raliVideoFileSource(handle, folderPath1, color_format, ((video_mode == 1) ? RaliDecodeDevice::RALI_HW_DECODE:RaliDecodeDevice::RALI_SW_DECODE), shard_count, sequence_length, frame_step, frame_stride, shuffle, true, false);
     }
     else
     {

rocAL/rocAL/CMakeLists.txt

@@ -33,7 +33,6 @@ if(GPU_SUPPORT)
     find_package(AMDRPP QUIET)
     find_package(TurboJpeg QUIET)
     find_package(Protobuf QUIET)
-    # If FFMPEG is found, vx_amd_media is expected to be built as well
     find_package(FFmpeg QUIET)
     find_package(OpenCV QUIET)
 else()
@@ -155,7 +154,19 @@ if(${BUILD_RALI})
         include/rali_api_meta_data.h
         DESTINATION ./include
         )
+    if(FFMPEG_FOUND)
+        find_path(AVUTIL_INCLUDE_DIR libavutil/avutil.h)
+        find_path(AVCODEC_INCLUDE_DIR libavcodec/avcodec.h)
+        find_path(SWSCALE_INCLUDE_DIR libswscale/swscale.h)
+        find_path(AVFORMAT_INCLUDE_DIR libavformat/avformat.h)
+    endif()
     include_directories(
+                ${OpenCL_INCLUDE_DIRS}
+                ${OpenCL_INCLUDE_DIRS}/Headers
+                ${AVUTIL_INCLUDE_DIR} 
+                ${AVCODEC_INCLUDE_DIR} 
+                ${SWSCALE_INCLUDE_DIR} 
+                ${AVFORMAT_INCLUDE_DIR}
                 ${CMAKE_CURRENT_BINARY_DIR}
                 ${TurboJpeg_INCLUDE_DIRS}
                 ${PROTOBUF_INCLUDE_DIRS}
@@ -203,17 +214,17 @@ if(${BUILD_RALI})
     else()
         target_compile_definitions(${PROJECT_NAME} PUBLIC ENABLE_OPENCV=0)
     endif()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fopenmp -msse4.2 -mavx2 -Wall  -fPIC -pg -pthread -std=c++14")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fopenmp -msse4.2 -mavx2 -Wall  -fPIC -pg -pthread -std=c++14 -Wno-deprecated-declarations")
     message("-- ${Green}rocAL - CMAKE_CXX_FLAGS:${CMAKE_CXX_FLAGS}")
 
     if(NOT FFMPEG_FOUND)
         message("-- ${Yellow}rocAL library is going to be built without video decode functionality ${ColourReset}")
-        target_link_libraries(${PROJECT_NAME} -fPIC ${PROTOBUF_LIBRARIES} lmdb boost_system boost_filesystem turbojpeg openvx vx_rpp )
+        target_link_libraries(${PROJECT_NAME} -fPIC ${PROTOBUF_LIBRARIES} lmdb boost_system boost_filesystem turbojpeg openvx vx_rpp)
     else()
         message("-- ${Green}rocAL library is going to be built with video decode functionality ${ColourReset}")
-        target_link_libraries(${PROJECT_NAME} -DRALI_VIDEO -fPIC ${PROTOBUF_LIBRARIES} lmdb turbojpeg openvx vx_rpp )
+        target_compile_definitions(${PROJECT_NAME} PUBLIC -DRALI_VIDEO)
+        target_link_libraries(${PROJECT_NAME} -fPIC ${PROTOBUF_LIBRARIES} ${FFMPEG_LIBRARIES} lmdb turbojpeg openvx vx_rpp)
     endif()
-
     if("${BACKEND}" STREQUAL "HIP" AND HIP_FOUND)
         target_link_libraries(${PROJECT_NAME} $<TARGET_OBJECTS:rocAL_hip>)
     endif()

rocAL/rocAL/include/caffe2_lmdb_record_reader.h

@@ -48,7 +48,7 @@ class Caffe2LMDBRecordReader : public Reader
      \param buf User's provided buffer to receive the loaded images
      \return Size of the loaded resource
     */
-    size_t read(unsigned char* buf, size_t max_size) override;
+    size_t read_data(unsigned char* buf, size_t max_size) override;
     //! Opens the next file in the folder
     /*!
      \return The size of the next file, 0 if couldn't access it
@@ -60,7 +60,7 @@ class Caffe2LMDBRecordReader : public Reader
     //! Returns the id of the latest file opened
     std::string id() override { return _last_id;};
 
-    unsigned count() override;
+    unsigned count_items() override;
 
     ~Caffe2LMDBRecordReader() override;
 

rocAL/rocAL/include/caffe2_meta_data_reader.h

@@ -40,6 +40,7 @@ public :
     void release(std::string image_name);
     void release() override;
     void print_map_contents();
+    bool set_timestamp_mode() override { return false; }
     MetaDataBatch * get_output() override { return _output; }
     Caffe2MetaDataReader();
     ~Caffe2MetaDataReader() override { delete _output; }

rocAL/rocAL/include/caffe2_meta_data_reader_detection.h

@@ -40,6 +40,7 @@ public :
     void release(std::string image_name);
     void release() override;
     void print_map_contents();
+    bool set_timestamp_mode() override { return false; }
     MetaDataBatch * get_output() override { return _output; }
     Caffe2MetaDataReaderDetection();
     ~Caffe2MetaDataReaderDetection() override { delete _output; }

rocAL/rocAL/include/caffe_lmdb_record_reader.h

@@ -47,7 +47,7 @@ class CaffeLMDBRecordReader : public Reader{
      \param buf User's provided buffer to receive the loaded images
      \return Size of the loaded resource
     */
-    size_t read(unsigned char* buf, size_t max_size) override;
+    size_t read_data(unsigned char* buf, size_t max_size) override;
     //! Opens the next file in the folder
     /*!
      \return The size of the next file, 0 if couldn't access it
@@ -59,7 +59,7 @@ class CaffeLMDBRecordReader : public Reader{
     //! Returns the id of the latest file opened
     std::string id() override { return _last_id;};
 
-    unsigned count() override;
+    unsigned count_items() override;
 
     ~CaffeLMDBRecordReader() override;
 

rocAL/rocAL/include/caffe_meta_data_reader.h

@@ -38,6 +38,7 @@ public :
     void read_all(const std::string& path) override;
     void release(std::string image_name);
     void release() override;
+    bool set_timestamp_mode() override { return false; }
     void print_map_contents();
     MetaDataBatch * get_output() override { return _output; }
     CaffeMetaDataReader();

rocAL/rocAL/include/caffe_meta_data_reader_detection.h

@@ -41,6 +41,7 @@ public :
     void read_all(const std::string& path) override;
     void release(std::string image_name);
     void release() override;
+    bool set_timestamp_mode() override { return false; }
     void print_map_contents();
     MetaDataBatch * get_output() override { return _output; }
     CaffeMetaDataReaderDetection();

rocAL/rocAL/include/cifar10_data_reader.h

@@ -40,7 +40,7 @@ class CIFAR10DataReader : public Reader {
      \param buf User's provided buffer to receive the loaded images
      \return Size of the loaded resource
     */
-    size_t read(unsigned char* buf, size_t max_size) override;
+    size_t read_data(unsigned char* buf, size_t max_size) override;
     //! Opens the next file in the folder
     /*!
      \return The size of the next file, 0 if couldn't access it
@@ -53,7 +53,7 @@ class CIFAR10DataReader : public Reader {
     //! Returns the name of the latest data_id opened
     std::string id() override { return _last_id;};
 
-    unsigned count() override;
+    unsigned count_items() override;
 
     ~CIFAR10DataReader() override;
 

rocAL/rocAL/include/cifar10_meta_data_reader.h

@@ -36,6 +36,7 @@ public :
     void release(std::string image_name);
     void release() override;
     void print_map_contents();
+    bool set_timestamp_mode() override { return false; }
     MetaDataBatch * get_output() override { return _output; }
     Cifar10MetaDataReader();
     ~Cifar10MetaDataReader() override { delete _output; }

rocAL/rocAL/include/coco_file_source_reader.h

@@ -23,7 +23,7 @@ class COCOFileSourceReader : public Reader {
      \param buf User's provided buffer to receive the loaded images
      \return Size of the loaded resource
     */
-    size_t read(unsigned char* buf, size_t max_size) override;
+    size_t read_data(unsigned char* buf, size_t max_size) override;
     //! Opens the next file in the folder
     /*!
      \return The size of the next file, 0 if couldn't access it
@@ -36,7 +36,7 @@ class COCOFileSourceReader : public Reader {
     //! Returns the name of the latest file opened
     std::string id() override { return _last_id;};
 
-    unsigned count() override;
+    unsigned count_items() override;
     unsigned long long get_shuffle_time() {return _shuffle_time.get_timing();};
 
     ~COCOFileSourceReader() override;

rocAL/rocAL/include/coco_meta_data_reader.h

@@ -36,6 +36,7 @@ class COCOMetaDataReader: public MetaDataReader
     void release(std::string image_name);
     void release() override;
     void print_map_contents();
+    bool set_timestamp_mode() override { return false; }
     MetaDataBatch * get_output() override { return _output; }
     std::map<std::string, std::shared_ptr<BoundingBox>> get_map_content() { return _map_content;}
     COCOMetaDataReader();

rocAL/rocAL/include/commons.h

@@ -71,6 +71,16 @@ enum class RaliMemType
     HIP
 };
 
+/*! \brief Decoder mode for Video decoding
+ * 
+ *  Currently supports Software decoding, will support Hardware decoding in future
+ */
+enum class DecodeMode
+{
+    HW_VAAPI = 0,
+    CPU = 1
+};
+
 struct Timing
 {
     // The following timings are accumulated timing not just the most recent activity
@@ -86,4 +96,7 @@ struct Timing
     long long unsigned bb_load_time= 0;
     long long unsigned mask_load_time = 0;
     long long unsigned shuffle_time = 0;
+    long long unsigned video_read_time= 0;
+    long long unsigned video_decode_time= 0;
+    long long unsigned video_process_time= 0;
 };

rocAL/rocAL/include/ffmpeg_video_decoder.h

@@ -0,0 +1,48 @@
+/*
+Copyright (c) 2019 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#include "video_decoder.h"
+
+#ifdef RALI_VIDEO
+class FFmpegVideoDecoder : public VideoDecoder
+{
+public:
+    //! Default constructor
+    FFmpegVideoDecoder();
+    VideoDecoder::Status Initialize(const char *src_filename) override;
+    VideoDecoder::Status Decode(unsigned char *output_buffer, unsigned seek_frame_number, size_t sequence_length, size_t stride, int out_width, int out_height, int out_stride, AVPixelFormat out_format) override;
+    int seek_frame(AVRational avg_frame_rate, AVRational time_base, unsigned frame_number) override;
+    void release() override;
+    ~FFmpegVideoDecoder() override;
+private:
+    const char *_src_filename = NULL;
+    AVFormatContext *_fmt_ctx = NULL;
+    AVCodecContext *_video_dec_ctx = NULL;
+    AVCodec *_decoder = NULL;
+    AVStream *_video_stream = NULL;
+    int _video_stream_idx = -1;
+    AVPixelFormat _dec_pix_fmt;
+    int _codec_width, _codec_height;
+};
+#endif

rocAL/rocAL/include/file_source_reader.h

@@ -42,7 +42,7 @@ class FileSourceReader : public Reader {
      \param buf User's provided buffer to receive the loaded images
      \return Size of the loaded resource
     */
-    size_t read(unsigned char* buf, size_t max_size) override;
+    size_t read_data(unsigned char* buf, size_t max_size) override;
     //! Opens the next file in the folder
     /*!
      \return The size of the next file, 0 if couldn't access it
@@ -55,7 +55,7 @@ class FileSourceReader : public Reader {
     //! Returns the name of the latest file opened
     std::string id() override { return _last_id;};
 
-    unsigned count() override;
+    unsigned count_items() override;
 
     ~FileSourceReader() override;
 

rocAL/rocAL/include/label_reader_folders.h

@@ -36,6 +36,7 @@ public :
     void release(std::string image_name);
     void release() override;
     void print_map_contents();
+    bool set_timestamp_mode() override { return false; }
     MetaDataBatch * get_output() override { return _output; }
     LabelReaderFolders();
     ~LabelReaderFolders() override { delete _output; }