Overlay module and updated MotionExtractor module to use OverlayModule (

#220)

* Basic commit

* overlaydatainfo class

* added FrameContainerQueueOverlayAdapter class

* Updated the module -  zaki

* added basic test and serialization

* updated the overlay

* serialization working - Not implemented deserialization completely

* serliazation and deserilization working without save and load

* updated the visitor and composite class

* Added the overlayFactory pattern for the overlay

* implemented Builder pattern

* added internal seriliaze function for composite

* seriliazation and deserialization working

* Updated the seriliazeVisitor pattern

* 1. Fixed the compositeOverlay primitiveType issue.
2. Serialization and deserialization working. Test passing.

* implemented draw visitor and did some refactoring

* refactored Overlay Module and tests

* draw function working

* Updated MotionExtrator module to use generic OverlayModule

* removed the adapter changes

* removed extra spaces

* diabled simultaneous read write test

* Updated the getSerialize function name

* Updated the test

---------

Co-authored-by: venkat0907 <venkatnallamothu0907@gmail.com>

base/CMakeLists.txt

@@ -197,7 +197,7 @@ SET(CORE_FILES
  src/RTSPClientSrc.cpp
  src/RTSPClientSrc.cpp
  src/MotionVectorExtractor.cpp
- src/OverlayMotionVectors.cpp
+ src/OverlayModule.cpp
 )
 
 SET(CORE_FILES_H
@@ -256,7 +256,7 @@ SET(CORE_FILES_H
  include/RTSPClientSrc.h
  include/H264Metadata.h
  include/MotionVectorExtractor.h
- include/OverlayMotionVectors.h
+ include/OverlayModule.h
 )
 
 IF(ENABLE_WINDOWS)
@@ -309,6 +309,9 @@ SET(IP_FILES
  src/ColorConversionStrategy.h
  src/AbsColorConversionFactory.h
  src/ArchiveSpaceManager.cpp
+ src/Overlay.cpp
+ src/OverlayFactory.h
+ src/OverlayFactory.cpp
 )
 
 
@@ -342,6 +345,7 @@ SET(IP_FILES_H
  include/FaceDetectorXform.h
  include/TextOverlayXForm.h
  include/ColorConversionXForm.h
+ include/Overlay.h
 )
 
 
@@ -619,6 +623,7 @@ SET(UT_FILES
  test/rtsp_client_tests.cpp
  test/rtsp_client_tests.cpp
  test/motionvector_extractor_and_overlay_tests.cpp
+ test/overlaymodule_tests.cpp
  ${ARM64_UT_FILES}
  ${CUDA_UT_FILES}
 )

base/include/FrameContainerQueue.h

@@ -149,5 +149,4 @@ class FrameContainerQueueAdapter : public FrameContainerQueue
  virtual frame_container on_pop_success(frame_container item) {
  return item;
  }
-};
-
+};

base/include/FrameMetadata.h

@@ -48,7 +48,8 @@ class FrameMetadata {
  BMP_IMAGE,
  MP4_VIDEO_METADATA,
  HEVC_DATA, //H265
- MOTION_VECTOR_DATA
+ MOTION_VECTOR_DATA,
+ OVERLAY_INFO_IMAGE
  };
 
  enum MemType

base/include/MotionVectorExtractor.h

@@ -15,15 +15,16 @@ class MotionVectorExtractorProps : public ModuleProps
  OPENH264
  };
 
- MotionVectorExtractorProps(MVExtractMethod _MVExtractMethod = MVExtractMethod::FFMPEG, bool _sendDecodedFrame = false) : MVExtract(_MVExtractMethod), sendDecodedFrame(_sendDecodedFrame)
+ MotionVectorExtractorProps(MVExtractMethod _MVExtractMethod = MVExtractMethod::FFMPEG, bool _sendDecodedFrame = false, int _motionVectorThreshold = 2) : MVExtract(_MVExtractMethod), sendDecodedFrame(_sendDecodedFrame), motionVectorThreshold(_motionVectorThreshold)
  {
  }
 
  size_t getSerializeSize()
  {
- return ModuleProps::getSerializeSize() + sizeof(sendDecodedFrame);
+ return ModuleProps::getSerializeSize() + sizeof(sendDecodedFrame) + sizeof(motionVectorThreshold);
  }
  bool sendDecodedFrame = false;
+ int motionVectorThreshold;
  MVExtractMethod MVExtract = MVExtractMethod::FFMPEG;
 private:
  friend class boost::serialization::access;
@@ -33,6 +34,7 @@ class MotionVectorExtractorProps : public ModuleProps
  {
  ar& boost::serialization::base_object<ModuleProps>(*this);
  ar& sendDecodedFrame;
+ ar& motionVectorThreshold;
  }
 };
 

base/include/Overlay.h

@@ -0,0 +1,244 @@
+#pragma once
+
+#include <boost/serialization/vector.hpp>
+#include <opencv2/core/types.hpp>
+#include <opencv2/opencv.hpp>
+#include "Utils.h"
+#include "Module.h"
+
+enum Primitive
+{
+ RECTANGLE,
+ CIRCLE,
+ LINE,
+ COMPOSITE,
+ DRAWING
+};
+
+class OverlayInfo;
+
+class OverlayInfoVisitor
+{
+public:
+ virtual ~OverlayInfoVisitor() {}
+ virtual void visit(OverlayInfo* overlay) {};
+};
+
+class OverlayInfo
+{
+public:
+ OverlayInfo(Primitive p) : primitiveType(p) {}
+ OverlayInfo() {}
+ virtual void serialize(boost::archive::binary_oarchive& oa) {}
+ virtual void deserialize(boost::archive::binary_iarchive& ia) {}
+ virtual size_t getSerializeSize() { return 0; }
+ virtual void accept(OverlayInfoVisitor* visitor) { visitor->visit(this); };
+ virtual void draw(cv::Mat matImg) {}
+ Primitive primitiveType;
+};
+
+class CircleOverlay : public OverlayInfo
+{
+public:
+ CircleOverlay() : OverlayInfo(Primitive::CIRCLE) {}
+ void serialize(boost::archive::binary_oarchive& oa);
+ size_t getSerializeSize();
+ void deserialize(boost::archive::binary_iarchive& ia);
+ void draw(cv::Mat matImg);
+
+ float x1, y1, radius;
+private:
+ friend class boost::serialization::access;
+ template <class Archive>
+ void save(Archive& ar, const unsigned int version /*file_version*/)
+ {
+ ar& primitiveType;
+ ar& x1& y1& radius;
+ }
+ template <class Archive>
+ void load(Archive& ar, const unsigned int version)
+ {
+ if (version > 0)
+ ar& primitiveType;
+ ar& x1& y1& radius;
+ }
+};
+
+class LineOverlay : public OverlayInfo
+{
+public:
+ LineOverlay() : OverlayInfo(Primitive::LINE) {}
+ void serialize(boost::archive::binary_oarchive& oa);
+ size_t getSerializeSize();
+ void deserialize(boost::archive::binary_iarchive& ia);
+ void draw(cv::Mat matImg);
+
+ float x1, y1, x2, y2;
+private:
+ friend class boost::serialization::access;
+ template <class Archive>
+ void save(Archive& ar, const unsigned int version /*file_version*/)
+ {
+ ar& primitiveType;
+ ar& x1& y1& x2& y2;
+ }
+ template <class Archive>
+ void load(Archive& ar, const unsigned int version /*file_version*/)
+ {
+ if (version > 0)
+ ar& primitiveType;
+ ar& x1& y1& x2& y2;
+ }
+};
+
+class RectangleOverlay : public OverlayInfo
+{
+public:
+ RectangleOverlay() : OverlayInfo(Primitive::RECTANGLE) {}
+ void serialize(boost::archive::binary_oarchive& oa);
+ size_t getSerializeSize();
+ void deserialize(boost::archive::binary_iarchive& ia);
+ void draw(cv::Mat matImg);
+
+ float x1, y1, x2, y2;
+private:
+ friend class boost::serialization::access;
+ template <class Archive>
+ void save(Archive& ar, const unsigned int version /*file_version*/)
+ {
+ ar& primitiveType;
+ ar& x1& y1& x2& y2;
+ }
+ template <class Archive>
+ void load(Archive& ar, const unsigned int version /*file_version*/)
+ {
+ if (version > 0)
+ ar& primitiveType;
+ ar& x1& y1& x2& y2;
+ }
+};
+
+// visitorsheirarchy
+class OverlayInfoSerializerVisitor : public OverlayInfoVisitor
+{
+public:
+ OverlayInfoSerializerVisitor(boost::archive::binary_oarchive& _oa) : oa(_oa) {}
+ void visit(OverlayInfo* overlay);
+private:
+ boost::archive::binary_oarchive& oa;
+};
+
+// visitor to estimate serialize size
+class OverlayInfoSerializeSizeVisitor : public OverlayInfoVisitor
+{
+public:
+ OverlayInfoSerializeSizeVisitor() : totalSize(0) {}
+ void visit(OverlayInfo* overlay);
+ size_t totalSize;
+};
+
+class OverlayInfoDrawingVisitor : public OverlayInfoVisitor
+{
+public:
+ OverlayInfoDrawingVisitor(frame_sp frame)
+ {
+ matImg = Utils::getMatHeader(FrameMetadataFactory::downcast<RawImageMetadata>(frame->getMetadata()));
+ matImg.data = static_cast<uchar*>(frame->data());
+ }
+ void visit(OverlayInfo* overlay);
+private:
+ cv::Mat matImg;
+};
+
+class CompositeOverlay : public OverlayInfo
+{
+public:
+ CompositeOverlay() : OverlayInfo(Primitive::COMPOSITE) {}
+ CompositeOverlay(Primitive primitiveType) : OverlayInfo(primitiveType) {}
+ void add(OverlayInfo* component);
+ // used by builder
+ void deserialize(boost::archive::binary_iarchive& ia);
+ void accept(OverlayInfoVisitor* visitor);
+ vector<OverlayInfo*> getList();
+
+protected:
+ vector<OverlayInfo*> gList; // used by DrawingOverlay
+ // used by visitor
+ void serialize(boost::archive::binary_oarchive& oa);
+
+private:
+ friend class boost::serialization::access;
+ template <class Archive>
+ void save(Archive& ar, const unsigned int version /*file_version*/)
+ {
+ ar& primitiveType;
+ ar& gList.size();
+ }
+ template <class Archive>
+ void load(Archive& ar, const unsigned int version /*file_version*/)
+ {
+ if (version > 0)
+ ar& primitiveType;
+ ar& gList.size();
+ }
+};
+
+// interface to be used externally
+// why is it required ? - put commment here
+class DrawingOverlay : public CompositeOverlay
+{
+public:
+ DrawingOverlay() : CompositeOverlay(Primitive::DRAWING) {}
+ void serialize(frame_sp frame);
+ void deserialize(frame_sp frame);
+ void draw(frame_sp frame);
+ size_t mGetSerializeSize();
+};
+
+// Builder heirarchy
+class DrawingOverlayBuilder
+{
+public:
+ DrawingOverlayBuilder() {}
+ virtual OverlayInfo* deserialize(boost::archive::binary_iarchive& ia) = 0;
+};
+
+class CompositeOverlayBuilder : public DrawingOverlayBuilder
+{
+public:
+ CompositeOverlayBuilder() : compositeOverlay(new CompositeOverlay()) {}
+ OverlayInfo* deserialize(boost::archive::binary_iarchive& ia);
+
+protected:
+ CompositeOverlay* compositeOverlay;
+};
+
+class LineOverlayBuilder : public DrawingOverlayBuilder
+{
+public:
+ LineOverlayBuilder() : lineOverlay(new LineOverlay()) {}
+ OverlayInfo* deserialize(boost::archive::binary_iarchive& ia);
+
+protected:
+ LineOverlay* lineOverlay;
+};
+
+class RectangleOverlayBuilder : public DrawingOverlayBuilder
+{
+public:
+ RectangleOverlayBuilder() : rectangleOverlay(new RectangleOverlay()) {}
+ OverlayInfo* deserialize(boost::archive::binary_iarchive& ia);
+
+protected:
+ RectangleOverlay* rectangleOverlay;
+};
+
+class CircleOverlayBuilder : public DrawingOverlayBuilder
+{
+public:
+ CircleOverlayBuilder() : circleOverlay(new CircleOverlay()) {}
+ OverlayInfo* deserialize(boost::archive::binary_iarchive& ia);
+
+protected:
+ CircleOverlay* circleOverlay;
+};

base/include/OverlayModule.h

@@ -0,0 +1,36 @@
+#pragma once
+#include <string>
+#include "Module.h"
+
+using namespace std;
+
+class OverlayModuleProps : public ModuleProps
+{
+public:
+ OverlayModuleProps() : ModuleProps() {}
+
+ size_t getSerializeSize()
+ {
+ return ModuleProps::getSerializeSize();
+ }
+};
+
+class OverlayCommand;
+
+class OverlayModule : public Module
+{
+public:
+ OverlayModule(OverlayModuleProps _props);
+ virtual ~OverlayModule() {};
+ bool init();
+ bool term();
+protected:
+ bool process(frame_container& frame);
+ void addInputPin(framemetadata_sp& metadata, string& pinId);
+ bool validateInputPins();
+ bool validateOutputPins();
+ bool shouldTriggerSOS();
+
+private:
+ std::string mOutputPinId;
+};