Frame RoI change

savant/base/model.py

@@ -4,6 +4,7 @@
 from typing import List, Optional, Tuple
 from omegaconf import MISSING
 from savant.base.pyfunc import PyFunc
+from savant.meta.constants import PRIMARY_OBJECT_LABEL
 from savant.remote_file.schema import RemoteFile
 
 
@@ -74,7 +75,7 @@ class ModelInput:
                 maintain_aspect_ratio: True
     """
 
-    object: str = 'frame'
+    object: str = PRIMARY_OBJECT_LABEL
     """A text label in the form of ``model_name.object_label``.
     Indicates objects that will be used as input data.
     Special value `frame` is used to specify the entire frame as model input.

savant/deepstream/buffer_processor.py

@@ -11,8 +11,9 @@
 from pysavantboost import ObjectsPreprocessing
 
 from savant.base.model import ObjectModel, ComplexModel
+from savant.meta.constants import PRIMARY_OBJECT_LABEL
 from savant.config.schema import PipelineElement, ModelElement, FrameParameters
-from savant.converter.scale import scale_rbbox
+from savant.meta.bbox import BBox, RBBox
 from savant.deepstream.nvinfer.model import (
     NvInferRotatedObjectDetector,
     NvInferDetector,
@@ -28,7 +29,7 @@
     nvds_infer_tensor_meta_to_outputs,
     nvds_add_obj_meta_to_frame,
     nvds_add_attr_meta_to_obj,
-    nvds_set_selection_type,
+    nvds_set_obj_selection_type,
     nvds_set_obj_uid,
 )
 from savant.gstreamer import Gst  # noqa:F401
@@ -105,13 +106,33 @@ def prepare_input(self, buffer: Gst.Buffer):
 
             frame_meta = metadata_get_frame_meta(source_id, frame_idx, frame_pts)
 
+            # full frame primary object by default
+            primary_bbox = BBox(
+                x_center=self._frame_params.width / 2,
+                y_center=self._frame_params.height / 2,
+                width=self._frame_params.width,
+                height=self._frame_params.height,
+            )
+
             # add external objects to nvds meta
             for obj_meta in frame_meta.metadata['objects']:
                 obj_key = self._model_object_registry.model_object_key(
                     obj_meta['model_name'], obj_meta['label']
                 )
-                # skip frame, will be added with proper width/height
-                if obj_key == 'frame':
+                # skip primary object for now, will be added later
+                if obj_key == PRIMARY_OBJECT_LABEL:
+                    bbox = (
+                        obj_meta['bbox']['xc'],
+                        obj_meta['bbox']['yc'],
+                        obj_meta['bbox']['width'],
+                        obj_meta['bbox']['height'],
+                    )
+                    # if not a full frame then correct primary object
+                    if bbox != (0.5, 0.5, 1, 1):
+                        primary_bbox = BBox(*bbox)
+                        primary_bbox.scale(
+                            self._frame_params.width, self._frame_params.height
+                        )
                     continue
                 # obj_key was only registered if
                 # it was required by the pipeline model elements (this case)
@@ -122,72 +143,68 @@ def prepare_input(self, buffer: Gst.Buffer):
                         class_id,
                     ) = self._model_object_registry.get_model_object_ids(obj_key)
                     if obj_meta['bbox']['angle']:
-                        scaled_bbox = scale_rbbox(
-                            bboxes=np.array(
-                                [
-                                    [
-                                        obj_meta['bbox']['xc'],
-                                        obj_meta['bbox']['yc'],
-                                        obj_meta['bbox']['width'],
-                                        obj_meta['bbox']['height'],
-                                        obj_meta['bbox']['angle'],
-                                    ]
-                                ]
-                            ),
-                            scale_factor_x=self._frame_params.width,
-                            scale_factor_y=self._frame_params.height,
-                        )[0]
+                        bbox = RBBox(
+                            x_center=obj_meta['bbox']['xc'],
+                            y_center=obj_meta['bbox']['yc'],
+                            width=obj_meta['bbox']['width'],
+                            height=obj_meta['bbox']['height'],
+                            angle=obj_meta['bbox']['angle'],
+                        )
                         selection_type = ObjectSelectionType.ROTATED_BBOX
                     else:
-                        scaled_bbox = (
-                            obj_meta['bbox']['xc'] * self._frame_params.width,
-                            obj_meta['bbox']['yc'] * self._frame_params.height,
-                            obj_meta['bbox']['width'] * self._frame_params.width,
-                            obj_meta['bbox']['height'] * self._frame_params.height,
-                            obj_meta['bbox']['angle'],
+                        bbox = BBox(
+                            x_center=obj_meta['bbox']['xc'],
+                            y_center=obj_meta['bbox']['yc'],
+                            width=obj_meta['bbox']['width'],
+                            height=obj_meta['bbox']['height'],
                         )
                         selection_type = ObjectSelectionType.REGULAR_BBOX
+
+                    bbox.scale(self._frame_params.width, self._frame_params.height)
                     if self._frame_params.padding:
-                        scaled_bbox = (
-                            scaled_bbox[0] + self._frame_params.padding.left,
-                            scaled_bbox[1] + self._frame_params.padding.top,
-                        ) + scaled_bbox[2:]
+                        bbox.left += self._frame_params.padding.left
+                        bbox.top += self._frame_params.padding.top
 
                     nvds_add_obj_meta_to_frame(
                         batch_meta=nvds_batch_meta,
                         frame_meta=nvds_frame_meta,
                         selection_type=selection_type,
                         class_id=class_id,
                         gie_uid=model_uid,
-                        bbox=scaled_bbox,
+                        bbox=(
+                            bbox.x_center,
+                            bbox.y_center,
+                            bbox.width,
+                            bbox.height,
+                            bbox.angle
+                            if selection_type == ObjectSelectionType.ROTATED_BBOX
+                            else 0.0,
+                        ),
                         object_id=obj_meta['object_id'],
                         obj_label=obj_key,
                         confidence=obj_meta['confidence'],
                     )
 
             # add primary frame object
-            obj_label = 'frame'
+            obj_label = PRIMARY_OBJECT_LABEL
             model_uid, class_id = self._model_object_registry.get_model_object_ids(
                 obj_label
             )
-            xc = self._frame_params.width / 2
-            yc = self._frame_params.height / 2
             if self._frame_params.padding:
-                xc += self._frame_params.padding.left
-                yc += self._frame_params.padding.top
+                primary_bbox.x_center += self._frame_params.padding.left
+                primary_bbox.y_center += self._frame_params.padding.top
             nvds_add_obj_meta_to_frame(
                 batch_meta=nvds_batch_meta,
                 frame_meta=nvds_frame_meta,
                 selection_type=ObjectSelectionType.REGULAR_BBOX,
                 class_id=class_id,
                 gie_uid=model_uid,
-                # tuple(xc, yc, width, height, angle)
                 bbox=(
-                    xc,
-                    yc,
-                    self._frame_params.width,
-                    self._frame_params.height,
-                    0,
+                    primary_bbox.x_center,
+                    primary_bbox.y_center,
+                    primary_bbox.width,
+                    primary_bbox.height,
+                    0.0,
                 ),
                 obj_label=obj_label,
                 # confidence should be bigger than tracker minDetectorConfidence
@@ -491,7 +508,7 @@ def prepare_element_output(self, element: PipelineElement, buffer: Gst.Buffer):
                     if nvds_obj_meta.unique_component_id == model_uid:
                         for obj in model.output.objects:
                             if nvds_obj_meta.class_id == obj.class_id:
-                                nvds_set_selection_type(
+                                nvds_set_obj_selection_type(
                                     obj_meta=nvds_obj_meta,
                                     selection_type=ObjectSelectionType.REGULAR_BBOX,
                                 )

savant/deepstream/drawfunc.py

@@ -61,7 +61,7 @@ def draw_on_frame(self, frame_meta: NvDsFrameMeta, artist: Artist):
         :param artist: Cairo context drawer to drawing primitives and directly on frame.
         """
         for obj_meta in frame_meta.objects:
-            if not obj_meta.element_name and obj_meta.label == 'frame':
+            if obj_meta.is_primary:
                 continue
 
             if self.rendered_objects is None or (

savant/deepstream/meta/frame.py

@@ -1,9 +1,10 @@
 """Wrapper of deepstream frame meta information."""
-from typing import Iterator
+from typing import Iterator, Optional
 import pyds
 from savant.meta.errors import MetaValueError
 from savant.deepstream.meta.iterators import NvDsObjectMetaIterator
 from savant.deepstream.meta.object import _NvDsObjectMetaImpl
+from savant.meta.bbox import BBox
 from savant.meta.object import ObjectMeta
 from savant.utils.source_info import SourceInfoRegistry
 
@@ -21,6 +22,7 @@ def __init__(
         super().__init__()
         self.batch_meta = frame_meta.base_meta.batch_meta
         self.frame_meta = frame_meta
+        self._primary_obj: Optional[ObjectMeta] = None
 
     @property
     def source_id(self) -> str:
@@ -40,6 +42,22 @@ def objects(self) -> Iterator[ObjectMeta]:
         """
         return NvDsObjectMetaIterator(self.frame_meta)
 
+    @property
+    def roi(self) -> BBox:
+        if not self._primary_obj:
+            for obj_meta in self.objects:
+                if obj_meta.is_primary:
+                    self._primary_obj = obj_meta
+                    break
+        return self._primary_obj.bbox
+
+    @roi.setter
+    def roi(self, value: BBox):
+        self._primary_obj.bbox.x_center = value.x_center
+        self._primary_obj.bbox.y_center = value.y_center
+        self._primary_obj.bbox.width = value.width
+        self._primary_obj.bbox.height = value.height
+
     @property
     def objects_number(self) -> int:
         """Returns number of objects in frame meta.

savant/deepstream/meta/object.py

@@ -8,11 +8,11 @@
 from savant.deepstream.meta.bbox import NvDsBBox, NvDsRBBox
 from savant.deepstream.meta.constants import MAX_LABEL_SIZE
 from savant.deepstream.utils import (
-    nvds_get_selection_type,
+    nvds_get_obj_selection_type,
     nvds_get_obj_attr_meta,
     nvds_get_obj_attr_meta_list,
     nvds_add_attr_meta_to_obj,
-    nvds_set_selection_type,
+    nvds_set_obj_selection_type,
     nvds_set_obj_uid,
     nvds_get_obj_uid,
 )
@@ -96,7 +96,7 @@ def __init__(
             rect_params.top = bbox.top
             rect_params.width = bbox.width
             rect_params.height = bbox.height
-            nvds_set_selection_type(
+            nvds_set_obj_selection_type(
                 obj_meta=self.ds_object_meta,
                 selection_type=ObjectSelectionType.REGULAR_BBOX,
             )
@@ -111,7 +111,7 @@ def __init__(
             add_rbbox_to_object_meta(
                 self._frame_meta.batch_meta, self.ds_object_meta, rbbox_coords
             )
-            nvds_set_selection_type(
+            nvds_set_obj_selection_type(
                 obj_meta=self.ds_object_meta,
                 selection_type=ObjectSelectionType.ROTATED_BBOX,
             )
@@ -136,7 +136,7 @@ def bbox(self) -> Optional[Union[BBox, RBBox]]:
             It can be :py:class:`~savant.meta.bbox.RegularBoundingBox` or
             :py:class:`~savant.meta.bbox.RotatedBoundingBox`.
         """
-        bbox_type = nvds_get_selection_type(self.ds_object_meta)
+        bbox_type = nvds_get_obj_selection_type(self.ds_object_meta)
         if bbox_type == ObjectSelectionType.REGULAR_BBOX:
             return NvDsBBox(self.ds_object_meta)
         if bbox_type == ObjectSelectionType.ROTATED_BBOX:

savant/deepstream/metadata.py

@@ -1,12 +1,12 @@
 """Convert deepstream object meta to output format."""
 from typing import Any, Dict
-import numpy as np
 import pyds
 
 from savant.config.schema import FrameParameters
-from savant.converter.scale import scale_rbbox
-from savant.deepstream.utils import nvds_get_rbbox
+from savant.deepstream.meta.bbox import NvDsRBBox
+from savant.deepstream.utils import nvds_get_obj_bbox
 from savant.meta.attribute import AttributeMeta
+from savant.meta.constants import PRIMARY_OBJECT_LABEL
 from savant.utils.model_registry import ModelObjectRegistry
 
 
@@ -23,12 +23,6 @@ def nvds_obj_meta_output_converter(
     model_name, label = ModelObjectRegistry.parse_model_object_key(
         nvds_obj_meta.obj_label
     )
-    rect_params = nvds_obj_meta.detector_bbox_info.org_bbox_coords
-    confidence = nvds_obj_meta.confidence
-    if nvds_obj_meta.tracker_bbox_info.org_bbox_coords.width > 0:
-        rect_params = nvds_obj_meta.tracker_bbox_info.org_bbox_coords
-        if nvds_obj_meta.tracker_confidence < 1.0:  # specified confidence
-            confidence = nvds_obj_meta.tracker_confidence
 
     if frame_params.padding and frame_params.padding.keep:
         frame_width = frame_params.total_width
@@ -37,48 +31,26 @@ def nvds_obj_meta_output_converter(
         frame_width = frame_params.width
         frame_height = frame_params.height
 
-    # scale bbox to [0..1]
-    # TODO: use a function to check bbox type explicitly
-    if rect_params.width == 0:
-        rbbox = nvds_get_rbbox(nvds_obj_meta)
-        scaled_bbox = scale_rbbox(
-            bboxes=np.array(
-                [
-                    [
-                        rbbox.x_center,
-                        rbbox.y_center,
-                        rbbox.width,
-                        rbbox.height,
-                        rbbox.angle,
-                    ]
-                ]
-            ),
-            scale_factor_x=1 / frame_width,
-            scale_factor_y=1 / frame_height,
-        )[0]
-        bbox = dict(
-            xc=scaled_bbox[0],
-            yc=scaled_bbox[1],
-            width=scaled_bbox[2],
-            height=scaled_bbox[3],
-            angle=scaled_bbox[4],
-        )
-    else:
-        obj_width = rect_params.width / frame_width
-        obj_height = rect_params.height / frame_height
-        bbox = dict(
-            xc=rect_params.left / frame_width + obj_width / 2,
-            yc=rect_params.top / frame_height + obj_height / 2,
-            width=obj_width,
-            height=obj_height,
-        )
+    confidence = nvds_obj_meta.confidence
+    if 0.0 < nvds_obj_meta.tracker_confidence < 1.0:  # specified confidence
+        confidence = nvds_obj_meta.tracker_confidence
+
+    nvds_bbox = nvds_get_obj_bbox(nvds_obj_meta)
     if frame_params.padding and not frame_params.padding.keep:
-        bbox['xc'] -= frame_params.padding.left / frame_width
-        bbox['yc'] -= frame_params.padding.top / frame_height
+        nvds_bbox.x_center -= frame_params.padding.left
+        nvds_bbox.y_center -= frame_params.padding.top
+    nvds_bbox.scale(1 / frame_width, 1 / frame_height)
+    bbox = dict(
+        xc=nvds_bbox.x_center,
+        yc=nvds_bbox.y_center,
+        width=nvds_bbox.width,
+        height=nvds_bbox.height,
+        angle=nvds_bbox.angle if isinstance(nvds_bbox, NvDsRBBox) else 0.0,
+    )
 
     # parse parent object
     parent_model_name, parent_label, parent_object_id = None, None, None
-    if nvds_obj_meta.parent and nvds_obj_meta.parent.obj_label != 'frame':
+    if nvds_obj_meta.parent and nvds_obj_meta.parent.obj_label != PRIMARY_OBJECT_LABEL:
         parent_model_name, parent_label = ModelObjectRegistry.parse_model_object_key(
             nvds_obj_meta.parent.obj_label
         )