Update project requirements

Makefile

@@ -25,6 +25,9 @@ publish-local: build build-adapters-deepstream build-adapters-gstreamer build-ad
 	docker tag savant-adapters-gstreamer$(PLATFORM_SUFFIX) ghcr.io/insight-platform/savant-adapters-gstreamer$(PLATFORM_SUFFIX)
 	docker tag savant-adapters-py$(PLATFORM_SUFFIX) ghcr.io/insight-platform/savant-adapters-py$(PLATFORM_SUFFIX)
 
+publish-local-extra: build-extra
+	docker tag savant-deepstream$(PLATFORM_SUFFIX)-extra ghcr.io/insight-platform/savant-deepstream$(PLATFORM_SUFFIX)-extra
+
 build:
 	docker buildx build \
 		--platform $(PLATFORM) \
@@ -135,6 +138,7 @@ run-tests:
 		savant-deepstream$(PLATFORM_SUFFIX)-extra \
 		 -s $(PROJECT_PATH)/tests
 
+# run jupyter inside `jupyter lab --allow-root`
 run-dev:
 	#xhost +local:docker
 	docker run -it --rm $(RUNTIME) \

adapters/requirements.txt

@@ -1,4 +1,3 @@
-cachetools~=5.3.0
 pretty-traceback==2023.1019
 # ClientSDK JpegSource, PngSource
 python-magic~=0.4.27

docker/Dockerfile.deepstream

@@ -140,13 +140,8 @@ RUN wget -qO- \
     dpkg -i OpenCV* && \
     rm OpenCV*
 
-COPY requirements/base.txt requirements/base.txt
-RUN python -m pip install --no-cache-dir -r requirements/base.txt
-
-COPY requirements/ext.txt requirements/ext.txt
-RUN python -m pip install --no-cache-dir -r requirements/ext.txt
-
-RUN rm -r requirements
+COPY requirements/base.txt requirements.txt
+RUN python -m pip install --no-cache-dir -r requirements.txt && rm requirements.txt
 
 COPY --from=savant-boost-builder /libs/savanboost/dist /libs/savanboost/dist
 RUN python -m pip install --no-cache-dir /libs/savanboost/dist/*.whl
@@ -178,7 +173,9 @@ COPY savant/VERSION .
 COPY gst_plugins gst_plugins
 
 COPY adapters/gst/gst_plugins adapters/gst/gst_plugins
-RUN rm -f adapters/gst/gst_plugins/python/video_files_sink.py
+RUN rm -f \
+    adapters/gst/gst_plugins/python/video_files_sink.py \
+    adapters/gst/gst_plugins/python/zeromq_sink.py
 COPY adapters/gst/sources adapters/gst/sources
 
 COPY scripts/uri-input.py scripts/
@@ -293,13 +290,9 @@ RUN wget -nv -O ${PYCUDA_WHL} ${PACKAGES_URL}/${PYCUDA_WHL} && \
 ARG ORT_VERSION=1.17.1
 RUN python -m pip install --no-cache-dir onnx onnxruntime-gpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/
 
-# cython, polars, scikit-learn, jupyter
-RUN python -m pip install --no-cache-dir \
-    Cython \
-    polars[all] \
-    scikit-learn \
-    jupyterlab \
-    pytest
+# cython, polars, scikit-learn, jupyter, etc.
+COPY requirements/extra.txt requirements.txt
+RUN python -m pip install --no-cache-dir -r requirements.txt && rm requirements.txt
 
 
 # Extra l4t container
@@ -356,10 +349,6 @@ ARG ORT_WHL="onnxruntime_gpu-1.17.0-cp310-cp310-linux_aarch64.whl"
 RUN wget -nv -O ${ORT_WHL} ${ORT_URL} && \
     python -m pip install --no-cache-dir onnx ${ORT_WHL}
 
-# cython, polars, scikit-learn, jupyter
-RUN python -m pip install --no-cache-dir \
-    Cython \
-    polars[all] \
-    scikit-learn \
-    jupyterlab \
-    pytest
+# cython, polars, scikit-learn, jupyter, etc.
+COPY requirements/extra.txt requirements.txt
+RUN python -m pip install --no-cache-dir -r requirements.txt && rm requirements.txt

requirements/base.txt

@@ -1,20 +1,15 @@
 numpy~=1.22.4
 cupy-cuda12x
-
 numba~=0.57
 scipy~=1.10
 
-pyzmq~=22.2.1
-cachetools~=5.3.0
-
-splitstream==1.2.6
-click~=8.1.6
-
+# yaml config
 omegaconf~=2.3
-# for omegaconf arithmetic resolver
+# omegaconf arithmetic resolver
 simpleeval~=0.9.12
-# for dynamic pyfunc reloading
+# dynamic pyfunc reloading
 inotify-simple~=1.3.5
+
 # pretty error output
 pretty-traceback==2023.1019
 
@@ -24,6 +19,15 @@ boto3~=1.23
 tqdm~=4.64
 
 # ClientSDK JpegSource, PngSource
+click~=8.1.6
 python-magic~=0.4.27
 
 prometheus-client~=0.19.0
+
+# pyds
+pyds @ https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/releases/download/v1.1.10/pyds-1.1.10-py3-none-linux_x86_64.whl ; platform_machine=='x86_64'
+pyds @ https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/releases/download/v1.1.10/pyds-1.1.10-py3-none-linux_aarch64.whl ; platform_machine=='aarch64'
+
+# ffmpeg-input
+ffmpeg-input @ https://github.com/insight-platform/FFmpeg-Input/releases/download/0.1.23/ffmpeg_input-0.1.23-cp310-cp310-manylinux_2_28_x86_64.whl ; platform_machine=='x86_64'
+ffmpeg-input @ https://github.com/insight-platform/FFmpeg-Input/releases/download/0.1.23/ffmpeg_input-0.1.23-cp310-cp310-manylinux_2_28_aarch64.whl ; platform_machine=='aarch64'

requirements/extra.txt

@@ -0,0 +1,7 @@
+Cython
+polars[all]
+scikit-learn
+jupyterlab
+ipywidgets
+matplotlib
+pytest

samples/nvidia_car_classification/flavors/module-etlt-config.yml

@@ -13,13 +13,12 @@ pipeline:
     - element: nvinfer@detector
       name: DashCamNet
       model:
-        format: etlt
         remote:
-          url: "https://api.ngc.nvidia.com/v2/models/nvidia/tao/dashcamnet/versions/pruned_v1.0.2/zip"
-        model_file: resnet18_dashcamnet_pruned.etlt
+          url: "https://api.ngc.nvidia.com/v2/models/nvidia/tao/dashcamnet/versions/pruned_v1.0.4/zip"
+        model_file: resnet18_dashcamnet_pruned.onnx
         # label_file: labels.txt
         precision: int8
-        int8_calib_file: dashcamnet_int8.txt
+        int8_calib_file: resnet18_dashcamnet_pruned_int8.txt
         batch_size: 1
         input:
           layer_name: input_1

savant/VERSION

@@ -1,3 +1,3 @@
 SAVANT=0.4.0
-SAVANT_RS=0.2.19
+SAVANT_RS=0.2.22
 DEEPSTREAM=6.4

savant/converter/yolo.py

@@ -73,6 +73,10 @@ def __call__(
             confidences = det_scores[:num]
             class_ids = det_classes[:num]
 
+            # [0..1] -> model.input
+            bboxes[:, [0, 2]] *= model.input.width
+            bboxes[:, [1, 3]] *= model.input.height
+
             # (left, top, right, bottom) -> (xc, yc, width, height)
             bboxes[:, 2] -= bboxes[:, 0]
             bboxes[:, 3] -= bboxes[:, 1]

savant/deepstream/nvinfer/file_config.py

@@ -189,6 +189,9 @@ class _FieldMap:
         _FieldMap('num-detected-classes', 'output.num_detected_classes', int),
         _FieldMap('gpu-id', 'gpu_id', int),
         _FieldMap('secondary-reinfer-interval', 'interval', int),
+        _FieldMap(
+            'layer-device-precision', 'layer_device_precision', lambda v: v.split(';')
+        ),
     ]
 
     _CLASS_ATTR_MAP = [
@@ -198,7 +201,7 @@ class _FieldMap:
         _FieldMap('detected-min-h', 'min_height', int, 32),
         _FieldMap('detected-max-w', 'max_width', int),
         _FieldMap('detected-max-h', 'max_height', int),
-        # FieldMap('topk', 'topk', int),
+        _FieldMap('topk', 'top_k', int),
     ]
 
     @staticmethod

savant/deepstream/nvinfer/model.py

@@ -1,11 +1,9 @@
 """Gst-nvinfer model configuration templates."""
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
 from typing import List, Optional
 
-from omegaconf import MISSING
-
 from savant.base.model import (
     AttributeModel,
     ComplexModel,
@@ -173,6 +171,11 @@ class NvInferModel(Model):
     engine_create_func_name: Optional[str] = None
     """Name of the custom TensorRT CudaEngine creation function."""
 
+    layer_device_precision: List[str] = field(default_factory=list)
+    """Specifies the device type and precision for any layer in the network.
+    List of items of format ``<layer1-name>:<precision>:<device-type>``.
+    """
+
 
 NVINFER_DEFAULT_OBJECT_SELECTOR = PyFunc(
     module='savant.selector.detector',

savant/deepstream/nvinfer/processor.py

@@ -374,14 +374,15 @@ def _process_custom_model_output(self, buffer: Gst.Buffer):
                         # object or complex model with non-empty output
                         if bbox_tensor.shape[1] == 6:  # no angle
                             selection_type = ObjectSelectionType.REGULAR_BBOX
+
                             # xc -> left, yc -> top
                             bbox_tensor[:, 2] -= bbox_tensor[:, 4] / 2
                             bbox_tensor[:, 3] -= bbox_tensor[:, 5] / 2
 
-                            # clip
                             # width to right, height to bottom
                             bbox_tensor[:, 4] += bbox_tensor[:, 2]
                             bbox_tensor[:, 5] += bbox_tensor[:, 3]
+
                             # clip
                             bbox_tensor[:, 2][
                                 bbox_tensor[:, 2] < self._frame_rect[0]
@@ -487,14 +488,10 @@ def _process_custom_model_output(self, buffer: Gst.Buffer):
                                 )
                                 selected_bboxes.append((int(bbox[7]), _nvds_obj_meta))
 
+                    # attribute or complex model
                     if values:
-                        # attribute or complex model
                         if self._is_complex_model:
-                            values = [
-                                v
-                                for i, v in enumerate(values)
-                                if i in {i for i, _ in selected_bboxes}
-                            ]
+                            values = [values[i] for i, _ in selected_bboxes]
                         else:
                             selected_bboxes = [(0, nvds_obj_meta)]
                             values = [values]

savant/deepstream/pipeline.py

@@ -811,6 +811,9 @@ def _update_meta_for_single_frame(
                 frame_pts,
             )
 
+        def _nvds_obj_id(_obj_meta: pyds.NvDsObjectMeta) -> str:
+            return f'{_obj_meta.obj_label}#{_obj_meta.object_id}'
+
         # collect frame objects
         nvds_object_id_map = {}  # nvds_obj_meta.object_id -> video_object.id
         parents = {}  # video_object.id -> nvds_obj_meta.parent.object_id
@@ -855,12 +858,12 @@ def _update_meta_for_single_frame(
             obj_meta = nvds_obj_meta_output_converter(
                 nvds_frame_meta, nvds_obj_meta, self._frame_params, video_frame
             )
-            nvds_object_id_map[nvds_obj_meta.object_id] = obj_meta.id
+            nvds_object_id_map[_nvds_obj_id(nvds_obj_meta)] = obj_meta.id
             if (
                 not nvds_is_empty_object_meta(nvds_obj_meta.parent)
                 and nvds_obj_meta.parent.obj_label != PRIMARY_OBJECT_KEY
             ):
-                parents[obj_meta.id] = nvds_obj_meta.parent.object_id
+                parents[obj_meta.id] = _nvds_obj_id(nvds_obj_meta.parent)
             # add obj attributes
             for attr in attributes:
                 obj_meta.set_attribute(attr)

savant/selector/detector.py

@@ -87,11 +87,12 @@ def __call__(self, bbox_tensor: np.ndarray) -> np.ndarray:
         )
 
 
-@nb.njit('f4[:, :](f4[:, :], f4, f4, u2, u2, u2, u2)', nogil=True, cache=True)
+@nb.njit('f4[:, :](f4[:, :], f4, f4, u2, u2, u2, u2, u2)', nogil=True, cache=True)
 def default_selector(
     bbox_tensor: np.ndarray,
     confidence_threshold: float = 0.0,
     nms_iou_threshold: float = 0.0,
+    top_k: int = 0,
     min_width: int = 0,
     min_height: int = 0,
     max_width: int = 0,
@@ -102,6 +103,7 @@ def default_selector(
     :param bbox_tensor: tensor(class_id, confidence, left, top, width, height)
     :param confidence_threshold: confidence threshold
     :param nms_iou_threshold: nms iou threshold
+    :param top_k: top k bboxes to keep
     :param min_width: minimal bbox width
     :param min_height: minimal bbox height
     :param max_width: maximum bbox width
@@ -129,7 +131,8 @@ def default_selector(
             selected_bbox_tensor[:, 2:6],
             selected_bbox_tensor[:, 1],
             nms_iou_threshold,
-            selected_bbox_tensor.shape[0],  # should specify default with numba.njit
+            # should specify default with numba.njit
+            top_k if top_k > 0 else selected_bbox_tensor.shape[0],
         )
         selected_bbox_tensor = selected_bbox_tensor[keep]
 
@@ -141,6 +144,7 @@ class BBoxSelector(BaseSelector):
 
     :param confidence_threshold: confidence threshold
     :param nms_iou_threshold: nms iou threshold
+    :param top_k: top k bboxes to keep
     :param min_width: minimal bbox width
     :param min_height: minimal bbox height
     :param max_width: maximum bbox width
@@ -151,6 +155,7 @@ def __init__(
         self,
         confidence_threshold: float = 0.5,
         nms_iou_threshold: float = 0.5,
+        top_k: int = 0,
         min_width: int = 0,
         min_height: int = 0,
         max_width: int = 0,
@@ -160,6 +165,7 @@ def __init__(
         super().__init__(**kwargs)
         self.confidence_threshold = confidence_threshold
         self.nms_iou_threshold = nms_iou_threshold
+        self.top_k = top_k
         self.min_width = min_width
         self.min_height = min_height
         self.max_width = max_width
@@ -175,6 +181,7 @@ def __call__(self, bbox_tensor: np.ndarray) -> np.ndarray:
             bbox_tensor=bbox_tensor,
             confidence_threshold=self.confidence_threshold,
             nms_iou_threshold=self.nms_iou_threshold,
+            top_k=self.top_k,
             min_width=self.min_width,
             min_height=self.min_height,
             max_width=self.max_width,

savant/utils/file_types.py

@@ -19,12 +19,15 @@ def from_mime_type(mime_type: Optional[str]) -> Optional['FileType']:
 
 
 def parse_mime_types(files: List[Path]) -> List[Tuple[Path, str]]:
-    output = subprocess.check_output(
-        ['file', '--no-pad', '--mime-type'] + [str(x) for x in files]
-    )
     mime_types = []
-    for line in output.decode().strip().split('\n'):
-        path, mime_type = line.rsplit(': ', 1)
-        mime_types.append((Path(path), mime_type))
-
+    # use chunks to avoid `Argument list too long` error
+    chunk_size = 1000
+    for i in range(0, len(files), chunk_size):
+        chunk = files[i : i + chunk_size]
+        output = subprocess.check_output(
+            ['file', '--no-pad', '--mime-type'] + [str(x) for x in chunk]
+        )
+        for line in output.decode().strip().split('\n'):
+            path, mime_type = line.rsplit(': ', 1)
+            mime_types.append((Path(path), mime_type))
     return mime_types