#48 add padding to frame parameters

docs/source/main_concepts/adapters.rst

@@ -31,7 +31,7 @@ Default ZeroMQ source and sink configuration in module config
 
 .. literalinclude:: ../../../savant/config/default.yml
   :language: YAML
-  :lines: 50-
+  :lines: 74-
 
 defines the following **default connection pattern**:
 

docs/source/main_concepts/module.rst

@@ -22,7 +22,7 @@ The following parameters are defined for a Savant module by default:
 
 .. literalinclude:: ../../../savant/config/default.yml
   :language: YAML
-  :lines: 1-66
+  :lines: 1-75
 
 .. note::
 

docs/source/main_concepts/pipeline.rst

@@ -32,7 +32,7 @@ and :py:attr:`~savant.config.schema.Pipeline.sink` nodes:
 
 .. literalinclude:: ../../../savant/config/default.yml
   :language: YAML
-  :lines: 50-
+  :lines: 74-
 
 It is possible to redefine these values, but it is important to remember that the main
 operation mode for a Savant module is with ZeroMQ source and sink.

savant/config/default.yml

@@ -33,6 +33,14 @@ parameters:
   frame:
     width: ${oc.decode:${oc.env:FRAME_WIDTH, 1280}}
     height: ${oc.decode:${oc.env:FRAME_HEIGHT, 720}}
+    # Add paddings to the frame before processing
+    # padding:
+    #   # Whether to keep paddings on the output frame
+    #   keep: false
+    #   left: 0
+    #   right: 0
+    #   top: 0
+    #   bottom: 0
 
   # FPS measurement period
   fps_period: ${oc.decode:${oc.env:FPS_PERIOD, 10000}}

savant/config/schema.py

@@ -7,6 +7,29 @@
 from savant.base.pyfunc import PyFunc
 
 
+@dataclass
+class FramePadding:
+    """Pipeline processing frame parameters"""
+
+    keep: bool = False
+    """Whether to keep paddings on the output frame"""
+
+    left: int = 0
+    """Size of the left padding"""
+
+    top: int = 0
+    """Size of the top padding"""
+
+    right: int = 0
+    """Size of the right padding"""
+
+    bottom: int = 0
+    """Size of the bottom padding"""
+
+    def __bool__(self):
+        return bool(self.left or self.top or self.right or self.bottom)
+
+
 @dataclass
 class FrameParameters:
     """Pipeline processing frame parameters"""
@@ -17,6 +40,21 @@ class FrameParameters:
     height: int
     """Pipeline processing frame height"""
 
+    padding: Optional[FramePadding] = None
+    """Add paddings to the frame before processing"""
+
+    @property
+    def total_width(self) -> int:
+        if self.padding is not None:
+            return self.width + self.padding.left + self.padding.right
+        return self.width
+
+    @property
+    def total_height(self) -> int:
+        if self.padding is not None:
+            return self.height + self.padding.top + self.padding.bottom
+        return self.height
+
 
 @dataclass
 class DynamicGstProperty:

savant/deepstream/buffer_processor.py

@@ -119,13 +119,19 @@ def prepare_input(self, buffer: Gst.Buffer):
                         model_uid,
                         class_id,
                     ) = self._model_object_registry.get_model_object_ids(obj_key)
+                    xc = obj_meta['bbox']['xc']
+                    yc = obj_meta['bbox']['yc']
+                    if self._frame_params.padding:
+                        xc += self._frame_params.padding.left / self._frame_params.width
+                        yc += self._frame_params.padding.top / self._frame_params.height
+
                     if obj_meta['bbox']['angle']:
                         scaled_bbox = scale_rbbox(
                             bboxes=np.array(
                                 [
                                     [
-                                        obj_meta['bbox']['xc'],
-                                        obj_meta['bbox']['yc'],
+                                        xc,
+                                        yc,
                                         obj_meta['bbox']['width'],
                                         obj_meta['bbox']['height'],
                                         obj_meta['bbox']['angle'],
@@ -138,8 +144,8 @@ def prepare_input(self, buffer: Gst.Buffer):
                         selection_type = ObjectSelectionType.ROTATED_BBOX
                     else:
                         scaled_bbox = (
-                            obj_meta['bbox']['xc'] * self._frame_params.width,
-                            obj_meta['bbox']['yc'] * self._frame_params.height,
+                            xc * self._frame_params.width,
+                            yc * self._frame_params.height,
                             obj_meta['bbox']['width'] * self._frame_params.width,
                             obj_meta['bbox']['height'] * self._frame_params.height,
                             obj_meta['bbox']['angle'],
@@ -163,6 +169,11 @@ def prepare_input(self, buffer: Gst.Buffer):
             model_uid, class_id = self._model_object_registry.get_model_object_ids(
                 obj_label
             )
+            xc = self._frame_params.width / 2
+            yc = self._frame_params.height / 2
+            if self._frame_params.padding:
+                xc += self._frame_params.padding.left
+                yc += self._frame_params.padding.top
             nvds_add_obj_meta_to_frame(
                 batch_meta=nvds_batch_meta,
                 frame_meta=nvds_frame_meta,
@@ -171,8 +182,8 @@ def prepare_input(self, buffer: Gst.Buffer):
                 gie_uid=model_uid,
                 # tuple(xc, yc, width, height, angle)
                 bbox=(
-                    self._frame_params.width / 2,
-                    self._frame_params.height / 2,
+                    xc,
+                    yc,
                     self._frame_params.width,
                     self._frame_params.height,
                     0,
@@ -263,6 +274,9 @@ def prepare_element_input(self, element: PipelineElement, buffer: Gst.Buffer):
                     else:
                         parent_bbox.left = 0
                         parent_bbox.top = 0
+                        if self._frame_params.padding:
+                            parent_bbox.left = self._frame_params.padding.left
+                            parent_bbox.top = self._frame_params.padding.top
                         parent_bbox.width = self._frame_params.width
                         parent_bbox.height = self._frame_params.height
 
@@ -298,6 +312,16 @@ def prepare_element_output(self, element: PipelineElement, buffer: Gst.Buffer):
         if not isinstance(element, ModelElement):
             return
 
+        frame_left = 0.0
+        frame_top = 0.0
+        frame_right = self._frame_params.width - 1.0
+        frame_bottom = self._frame_params.height - 1.0
+        if self._frame_params.padding:
+            frame_left += self._frame_params.padding.left
+            frame_top += self._frame_params.padding.top
+            frame_right += self._frame_params.padding.left
+            frame_bottom += self._frame_params.padding.top
+
         model_uid = self._model_object_registry.get_model_uid(element.name)
         model: Union[
             NvInferRotatedObjectDetector,
@@ -369,14 +393,18 @@ def prepare_element_output(self, element: PipelineElement, buffer: Gst.Buffer):
                                 bbox_tensor[:, 4] += bbox_tensor[:, 2]
                                 bbox_tensor[:, 5] += bbox_tensor[:, 3]
                                 # clip
-                                bbox_tensor[:, 2][bbox_tensor[:, 2] < 0.0] = 0.0
-                                bbox_tensor[:, 3][bbox_tensor[:, 3] < 0.0] = 0.0
+                                bbox_tensor[:, 2][
+                                    bbox_tensor[:, 2] < frame_left
+                                ] = frame_left
+                                bbox_tensor[:, 3][
+                                    bbox_tensor[:, 3] < frame_top
+                                ] = frame_top
                                 bbox_tensor[:, 4][
-                                    bbox_tensor[:, 4] > self._frame_params.width - 1.0
-                                ] = (self._frame_params.width - 1.0)
+                                    bbox_tensor[:, 4] > frame_right
+                                ] = frame_right
                                 bbox_tensor[:, 5][
-                                    bbox_tensor[:, 5] > self._frame_params.height - 1.0
-                                ] = (self._frame_params.height - 1.0)
+                                    bbox_tensor[:, 5] > frame_bottom
+                                ] = frame_bottom
 
                                 # right to width, bottom to height
                                 bbox_tensor[:, 4] -= bbox_tensor[:, 2]

savant/deepstream/metadata.py

@@ -30,23 +30,35 @@ def nvds_obj_meta_output_converter(
         if nvds_obj_meta.tracker_confidence < 1.0:  # specified confidence
             confidence = nvds_obj_meta.tracker_confidence
 
+    if frame_params.padding and not frame_params.padding.keep:
+        frame_width = frame_params.width
+        frame_height = frame_params.height
+    else:
+        frame_width = frame_params.total_width
+        frame_height = frame_params.total_height
+
     # scale bbox to [0..1]
     if rect_params.width == 0:
         rbbox = nvds_get_rbbox(nvds_obj_meta)
+        x_center = rbbox.x_center
+        y_center = rbbox.y_center
+        if frame_params.padding and not frame_params.padding.keep:
+            x_center -= frame_params.padding.left
+            y_center -= frame_params.padding.top
         scaled_bbox = scale_rbbox(
             bboxes=np.array(
                 [
                     [
-                        rbbox.x_center,
-                        rbbox.y_center,
+                        x_center,
+                        y_center,
                         rbbox.width,
                         rbbox.height,
                         rbbox.angle,
                     ]
                 ]
             ),
-            scale_factor_x=1 / frame_params.width,
-            scale_factor_y=1 / frame_params.height,
+            scale_factor_x=1 / frame_width,
+            scale_factor_y=1 / frame_height,
         )[0]
         bbox = dict(
             xc=scaled_bbox[0],
@@ -56,11 +68,16 @@ def nvds_obj_meta_output_converter(
             angle=scaled_bbox[4],
         )
     else:
-        obj_width = rect_params.width / frame_params.width
-        obj_height = rect_params.height / frame_params.height
+        obj_left = rect_params.left
+        obj_top = rect_params.top
+        if frame_params.padding and not frame_params.padding.keep:
+            obj_left -= frame_params.padding.left
+            obj_top -= frame_params.padding.top
+        obj_width = rect_params.width / frame_width
+        obj_height = rect_params.height / frame_height
         bbox = dict(
-            xc=rect_params.left / frame_params.width + obj_width / 2,
-            yc=rect_params.top / frame_params.height + obj_height / 2,
+            xc=obj_left / frame_width + obj_width / 2,
+            yc=obj_top / frame_height + obj_height / 2,
             width=obj_width,
             height=obj_height,
         )

savant/deepstream/source_output.py

@@ -6,7 +6,7 @@
 import pyds
 from pygstsavantframemeta import add_convert_savant_frame_meta_pad_probe
 
-from savant.config.schema import PipelineElement
+from savant.config.schema import PipelineElement, FrameParameters
 from savant.gstreamer import Gst  # noqa:F401
 from savant.gstreamer.codecs import CodecInfo
 from savant.gstreamer.pipeline import GstPipeline
@@ -70,11 +70,19 @@ class SourceOutputWithFrame(SourceOutput):
     Output contains frames along with metadata.
     """
 
+    def __init__(self, frame_params: FrameParameters):
+        super().__init__()
+        self._frame_params = frame_params
+
     def dest_resolution(self, source_info: SourceInfo) -> Resolution:
-        return Resolution(
-            width=source_info.src_resolution.width,
-            height=source_info.src_resolution.height,
-        )
+        width = source_info.src_resolution.width
+        height = source_info.src_resolution.height
+        if self._frame_params.padding and self._frame_params.padding.keep:
+            width = width * self._frame_params.total_width // self._frame_params.width
+            height = (
+                height * self._frame_params.total_height // self._frame_params.height
+            )
+        return Resolution(width=width, height=height)
 
     def add_output(
         self,
@@ -87,14 +95,26 @@ def add_output(
         self._logger.debug(
             'Added pad probe to convert savant frame meta from NvDsMeta to GstMeta'
         )
+        output_converter_props = {}
+        if not is_aarch64():
+            output_converter_props['nvbuf-memory-type'] = int(
+                pyds.NVBUF_MEM_CUDA_UNIFIED
+            )
+        if self._frame_params.padding and not self._frame_params.padding.keep:
+            output_converter_props['src_crop'] = ':'.join(
+                str(x)
+                for x in [
+                    self._frame_params.padding.left,
+                    self._frame_params.padding.top,
+                    self._frame_params.width,
+                    self._frame_params.height,
+                ]
+            )
+        self._logger.debug('Output converter properties: %s', output_converter_props)
         output_converter = pipeline._add_element(
             PipelineElement(
                 'nvvideoconvert',
-                properties=(
-                    {}
-                    if is_aarch64()
-                    else {'nvbuf-memory-type': int(pyds.NVBUF_MEM_CUDA_UNIFIED)}
-                ),
+                properties=output_converter_props,
             ),
         )
         source_info.after_demuxer.append(output_converter)
@@ -151,13 +171,14 @@ def __init__(
         self,
         codec: CodecInfo,
         params: Optional[Dict[str, Any]],
+        frame_params: FrameParameters,
     ):
         """
         :param codec: Codec for output frames.
         :param params: Parameters of the encoder.
         """
 
-        super().__init__()
+        super().__init__(frame_params=frame_params)
         self._codec = codec
         self._params = params or {}
 
@@ -213,7 +234,8 @@ class SourceOutputPng(SourceOutputEncoded):
 
     def dest_resolution(self, source_info: SourceInfo) -> Resolution:
         # Rounding resolution to the multiple of 8 to avoid cuda errors.
+        dest_resolution = super().dest_resolution(source_info)
         return Resolution(
-            width=round(source_info.src_resolution.width / 8) * 8,
-            height=round(source_info.src_resolution.height / 8) * 8,
+            width=round(dest_resolution.width / 8) * 8,
+            height=round(dest_resolution.height / 8) * 8,
         )