NvDsFrameMeta is extended and returns frame tags (#62)

The "NvDsFrameMeta" has been extended to include frame tags and other video frame metadata information. The pipeline metadata now includes the source metadata, and the source video adapter reads and adds frame metadata to the sent frames.

adapters/gst/gst_plugins/python/video_to_avro_serializer.py

@@ -1,4 +1,7 @@
+import json
+from copy import deepcopy
 from fractions import Fraction
+from pathlib import Path
 from typing import Any, NamedTuple, Optional
 
 from savant.api import serialize, ENCODING_REGISTRY
@@ -81,6 +84,14 @@ class VideoToAvroSerializer(LoggerMixin, GstBase.BaseTransform):
             True,
             GObject.ParamFlags.READWRITE,
         ),
+        'read-metadata': (
+            bool,
+            'Read metadata',
+            'Attempt to read the metadata of objects from the JSON file that has the identical name '
+            'as the source file with `json` extension, and then send it to the module.',
+            False,
+            GObject.ParamFlags.READWRITE,
+        ),
     }
 
     def __init__(self):
@@ -94,11 +105,13 @@ def __init__(self):
         # will be set after caps negotiation
         self.frame_params: Optional[FrameParams] = None
         self.last_frame_params: Optional[FrameParams] = None
-        self.location: Optional[str] = None
-        self.last_location: Optional[str] = None
+        self.location: Optional[Path] = None
+        self.last_location: Optional[Path] = None
         self.default_framerate: str = DEFAULT_FRAMERATE
 
         self.stream_in_progress = False
+        self.read_metadata: bool = False
+        self.json_metadata = None
 
     def do_set_caps(  # pylint: disable=unused-argument
         self, in_caps: Gst.Caps, out_caps: Gst.Caps
@@ -142,6 +155,8 @@ def do_get_property(self, prop: GObject.GParamSpec):
             return self.eos_on_location_change
         if prop.name == 'eos-on-frame-params-change':
             return self.eos_on_frame_params_change
+        if prop.name == 'read-metadata':
+            return self.read_metadata
         raise AttributeError(f'Unknown property {prop.name}.')
 
     def do_set_property(self, prop: GObject.GParamSpec, value: Any):
@@ -164,6 +179,8 @@ def do_set_property(self, prop: GObject.GParamSpec, value: Any):
             self.eos_on_location_change = value
         elif prop.name == 'eos-on-frame-params-change':
             self.eos_on_frame_params_change = value
+        elif prop.name == 'read-metadata':
+            self.read_metadata = value
         else:
             raise AttributeError(f'Unknown property {prop.name}.')
 
@@ -185,6 +202,9 @@ def do_prepare_output_buffer(self, in_buf: Gst.Buffer):
                 or self.eos_on_frame_params_change
                 and self.frame_params != self.last_frame_params
             ):
+                self.json_metadata = self.read_json_metadata_file(
+                    self.location.parent / f"{self.location.stem}.json"
+                )
                 self.send_end_message()
         self.last_location = self.location
         self.last_frame_params = self.frame_params
@@ -216,11 +236,32 @@ def do_sink_event(self, event: Gst.Event):
             has_location, location = tag_list.get_string(Gst.TAG_LOCATION)
             if has_location:
                 self.logger.info('Set location to %s', location)
-                self.location = location
+                self.location = Path(location)
+                self.json_metadata = self.read_json_metadata_file(
+                    self.location.parent / f"{self.location.stem}.json"
+                )
 
         # Cannot use `super()` since it is `self`
         return GstBase.BaseTransform.do_sink_event(self, event)
 
+    def read_json_metadata_file(self, location: Path):
+        json_metadata = None
+        if self.read_metadata:
+            if location.is_file():
+                with open(location, 'r') as fp:
+                    json_metadata = dict(
+                        map(
+                            lambda x: (x["pts"], x),
+                            filter(
+                                lambda x: x["schema"] == "VideoFrame",
+                                map(json.loads, fp.readlines()),
+                            ),
+                        )
+                    )
+            else:
+                self.logger.warning('JSON file `%s` not found', location.absolute())
+        return json_metadata
+
     def send_end_message(self):
         data = serialize(
             self.eos_schema,
@@ -241,6 +282,9 @@ def build_message(
         if pts == Gst.CLOCK_TIME_NONE:
             # TODO: support CLOCK_TIME_NONE in schema
             pts = 0
+        frame_metadata = None
+        if self.read_metadata and self.json_metadata:
+            frame_metadata = self.json_metadata[pts]
         message = {
             'source_id': self.source_id,
             'framerate': self.frame_params.framerate,
@@ -251,10 +295,11 @@ def build_message(
             'dts': dts,
             'duration': duration,
             'frame': frame,
+            'metadata': frame_metadata["metadata"] if self.read_metadata else None,
             **kwargs,
         }
         if self.location:
-            message['tags'] = {'location': self.location}
+            message['tags'] = {'location': str(self.location)}
         return message
 
 

adapters/gst/sources/media_files.sh

@@ -45,7 +45,8 @@ gst-launch-1.0 --eos-on-shutdown \
     media_files_src_bin location="${LOCATION}" file-type="${FILE_TYPE}" framerate="${FRAMERATE}" sort-by-time="${SORT_BY_TIME}" ! \
     fps_meter "${FPS_PERIOD}" output="${FPS_OUTPUT}" measure-per-file="${MEASURE_PER_FILE}" ! \
     adjust_timestamps ! \
-    video_to_avro_serializer source-id="${SOURCE_ID}" eos-on-location-change="${EOS_ON_LOCATION_CHANGE}" eos-on-frame-params-change=true ! \
+    video_to_avro_serializer source-id="${SOURCE_ID}" eos-on-location-change="${EOS_ON_LOCATION_CHANGE}" \
+    eos-on-frame-params-change=true read-metadata="${READ_METADATA}" ! \
     zeromq_sink socket="${ZMQ_ENDPOINT}" socket-type="${ZMQ_SOCKET_TYPE}" bind="${ZMQ_SOCKET_BIND}" sync="${SYNC_OUTPUT}" \
     &
 

savant/deepstream/buffer_processor.py

@@ -35,7 +35,7 @@
 from savant.gstreamer import Gst  # noqa:F401
 from savant.gstreamer.buffer_processor import GstBufferProcessor
 from savant.gstreamer.codecs import CodecInfo, Codec
-from savant.gstreamer.metadata import metadata_get_frame_meta, metadata_pop_frame_meta
+from savant.gstreamer.metadata import get_source_frame_meta, metadata_pop_frame_meta
 from savant.meta.type import ObjectSelectionType
 from savant.utils.fps_meter import FPSMeter
 from savant.utils.model_registry import ModelObjectRegistry
@@ -104,7 +104,7 @@ def prepare_input(self, buffer: Gst.Buffer):
                 frame_pts,
             )
 
-            frame_meta = metadata_get_frame_meta(source_id, frame_idx, frame_pts)
+            frame_meta = get_source_frame_meta(source_id, frame_idx, frame_pts)
 
             # full frame primary object by default
             primary_bbox = BBox(
@@ -137,54 +137,63 @@ def prepare_input(self, buffer: Gst.Buffer):
                 # obj_key was only registered if
                 # it was required by the pipeline model elements (this case)
                 # or equaled the output object of one of the pipeline model elements
-                if self._model_object_registry.is_model_object_key_registered(obj_key):
-                    (
-                        model_uid,
-                        class_id,
-                    ) = self._model_object_registry.get_model_object_ids(obj_key)
-                    if obj_meta['bbox']['angle']:
-                        bbox = RBBox(
-                            x_center=obj_meta['bbox']['xc'],
-                            y_center=obj_meta['bbox']['yc'],
-                            width=obj_meta['bbox']['width'],
-                            height=obj_meta['bbox']['height'],
-                            angle=obj_meta['bbox']['angle'],
-                        )
-                        selection_type = ObjectSelectionType.ROTATED_BBOX
-                    else:
-                        bbox = BBox(
-                            x_center=obj_meta['bbox']['xc'],
-                            y_center=obj_meta['bbox']['yc'],
-                            width=obj_meta['bbox']['width'],
-                            height=obj_meta['bbox']['height'],
-                        )
-                        selection_type = ObjectSelectionType.REGULAR_BBOX
-
-                    bbox.scale(self._frame_params.width, self._frame_params.height)
-                    if self._frame_params.padding:
-                        bbox.left += self._frame_params.padding.left
-                        bbox.top += self._frame_params.padding.top
-
-                    nvds_add_obj_meta_to_frame(
-                        batch_meta=nvds_batch_meta,
+                (
+                    model_uid,
+                    class_id,
+                ) = self._model_object_registry.get_model_object_ids(obj_key)
+                if obj_meta['bbox']['angle']:
+                    bbox = RBBox(
+                        x_center=obj_meta['bbox']['xc'],
+                        y_center=obj_meta['bbox']['yc'],
+                        width=obj_meta['bbox']['width'],
+                        height=obj_meta['bbox']['height'],
+                        angle=obj_meta['bbox']['angle'],
+                    )
+                    selection_type = ObjectSelectionType.ROTATED_BBOX
+                else:
+                    bbox = BBox(
+                        x_center=obj_meta['bbox']['xc'],
+                        y_center=obj_meta['bbox']['yc'],
+                        width=obj_meta['bbox']['width'],
+                        height=obj_meta['bbox']['height'],
+                    )
+                    selection_type = ObjectSelectionType.REGULAR_BBOX
+
+                bbox.scale(self._frame_params.width, self._frame_params.height)
+                if self._frame_params.padding:
+                    bbox.left += self._frame_params.padding.left
+                    bbox.top += self._frame_params.padding.top
+
+                nvds_obj_meta = nvds_add_obj_meta_to_frame(
+                    batch_meta=nvds_batch_meta,
+                    frame_meta=nvds_frame_meta,
+                    selection_type=selection_type,
+                    class_id=class_id,
+                    gie_uid=model_uid,
+                    bbox=(
+                        bbox.x_center,
+                        bbox.y_center,
+                        bbox.width,
+                        bbox.height,
+                        bbox.angle
+                        if selection_type == ObjectSelectionType.ROTATED_BBOX
+                        else 0.0,
+                    ),
+                    object_id=obj_meta['object_id'],
+                    obj_label=obj_key,
+                    confidence=obj_meta['confidence'],
+                )
+                for attr in obj_meta['attributes']:
+                    nvds_add_attr_meta_to_obj(
                         frame_meta=nvds_frame_meta,
-                        selection_type=selection_type,
-                        class_id=class_id,
-                        gie_uid=model_uid,
-                        bbox=(
-                            bbox.x_center,
-                            bbox.y_center,
-                            bbox.width,
-                            bbox.height,
-                            bbox.angle
-                            if selection_type == ObjectSelectionType.ROTATED_BBOX
-                            else 0.0,
-                        ),
-                        object_id=obj_meta['object_id'],
-                        obj_label=obj_key,
-                        confidence=obj_meta['confidence'],
+                        obj_meta=nvds_obj_meta,
+                        element_name=attr['element_name'],
+                        name=attr['name'],
+                        value=attr['value'],
+                        confidence=attr['confidence'],
                     )
 
+            frame_meta.metadata['objects'] = []
             # add primary frame object
             obj_label = PRIMARY_OBJECT_LABEL
             model_uid, class_id = self._model_object_registry.get_model_object_ids(

savant/deepstream/meta/bbox.py

@@ -1,4 +1,6 @@
 """Deepstream bounding boxes."""
+import logging
+
 import pyds
 from pysavantboost import get_rbbox
 
@@ -89,6 +91,17 @@ def left(self, value: float):
         self._nv_ds_bbox.left = value
         self._nv_ds_rect_meta.left = value
 
+    def scale(self, scale_x: float, scale_y: float):
+        """Scales BBox.
+
+        :param scale_x: The scaling factor applied along the x-axis.
+        :param scale_y: The scaling factor applied along the y-axis.
+        """
+        self.left *= scale_x
+        self.top *= scale_y
+        self.width *= scale_x
+        self.height *= scale_y
+
 
 class NvDsRBBox(RBBox):
     """Deepstream rotated bounding box wrapper.

savant/deepstream/meta/frame.py

@@ -1,12 +1,19 @@
 """Wrapper of deepstream frame meta information."""
-from typing import Iterator, Optional
+from typing import Iterator, Any, Dict, Optional
 import pyds
+
+from savant.gstreamer.metadata import (
+    get_source_frame_meta,
+    SourceFrameMeta,
+    OnlyExtendedDict,
+)
 from savant.meta.errors import MetaValueError
 from savant.deepstream.meta.iterators import NvDsObjectMetaIterator
 from savant.deepstream.meta.object import _NvDsObjectMetaImpl
 from savant.meta.bbox import BBox
 from savant.meta.object import ObjectMeta
 from savant.utils.source_info import SourceInfoRegistry
+from pygstsavantframemeta import nvds_frame_meta_get_nvds_savant_frame_meta
 
 
 class NvDsFrameMeta:
@@ -22,6 +29,7 @@ def __init__(
         super().__init__()
         self.batch_meta = frame_meta.base_meta.batch_meta
         self.frame_meta = frame_meta
+        self._source_frame_meta: Optional[SourceFrameMeta] = None
         self._primary_obj: Optional[ObjectMeta] = None
 
     @property
@@ -66,6 +74,59 @@ def objects_number(self) -> int:
         """
         return self.frame_meta.num_obj_meta
 
+    @property
+    def tags(self) -> OnlyExtendedDict:
+        """Returns tags of frame. These tags are part of the meta information about
+        the frame that comes with the frames in the module.
+
+        :return: Dictionary with tags
+        """
+        if self._source_frame_meta is None:
+            self._set_source_frame_meta()
+        return self._source_frame_meta.tags
+
+    def _set_source_frame_meta(self):
+        """Set the source frame metadata.
+
+        :return: None
+        """
+        savant_frame_meta = nvds_frame_meta_get_nvds_savant_frame_meta(self.frame_meta)
+        self._source_frame_meta = get_source_frame_meta(
+            source_id=self.source_id,
+            frame_idx=savant_frame_meta.idx if savant_frame_meta else None,
+            frame_pts=self.frame_meta.buf_pts,
+        )
+
+    @property
+    def pts(self) -> int:
+        """Get the presentation time stamp (PTS) of the current frame.
+
+        :return: The PTS of the current frame, if available; None otherwise.
+        """
+        if self._source_frame_meta is None:
+            self._set_source_frame_meta()
+        return self._source_frame_meta.pts
+
+    @property
+    def duration(self) -> Optional[int]:
+        """Get the duration of the current frame.
+
+        :returns: The duration of the current frame, if available; None otherwise.
+        """
+        if self._source_frame_meta is None:
+            self._set_source_frame_meta()
+        return self._source_frame_meta.duration
+
+    @property
+    def framerate(self) -> str:
+        """Get the framerate of the current frame.
+
+        returns: The framerate of the current frame as a string.
+        """
+        if self._source_frame_meta is None:
+            self._set_source_frame_meta()
+        return self._source_frame_meta.framerate
+
     def add_obj_meta(self, object_meta: ObjectMeta):
         """Add an object meta to frame meta.
 

savant/deepstream/pipeline.py

@@ -33,7 +33,7 @@
     nvds_obj_meta_output_converter,
     nvds_attr_meta_output_converter,
 )
-from savant.gstreamer.metadata import metadata_add_frame_meta, metadata_get_frame_meta
+from savant.gstreamer.metadata import metadata_add_frame_meta, get_source_frame_meta
 from savant.gstreamer.utils import on_pad_event, pad_to_source_id
 from savant.deepstream.utils import (
     gst_nvevent_new_stream_eos,
@@ -504,7 +504,7 @@ def update_frame_meta(self, pad: Gst.Pad, info: Gst.PadProbeInfo):
             )
             frame_idx = savant_frame_meta.idx if savant_frame_meta else None
             frame_pts = nvds_frame_meta.buf_pts
-            frame_meta = metadata_get_frame_meta(source_id, frame_idx, frame_pts)
+            frame_meta = get_source_frame_meta(source_id, frame_idx, frame_pts)
 
             # second iteration to collect module objects
             for nvds_obj_meta in nvds_obj_meta_iterator(nvds_frame_meta):