diff --git a/Makefile b/Makefile index 4ff54be92..e5d2b7abd 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,9 @@ publish-local: build build-adapters-deepstream build-adapters-gstreamer build-ad docker tag savant-adapters-gstreamer$(PLATFORM_SUFFIX) ghcr.io/insight-platform/savant-adapters-gstreamer$(PLATFORM_SUFFIX) docker tag savant-adapters-py$(PLATFORM_SUFFIX) ghcr.io/insight-platform/savant-adapters-py$(PLATFORM_SUFFIX) +publish-local-extra: build-extra + docker tag savant-deepstream$(PLATFORM_SUFFIX)-extra ghcr.io/insight-platform/savant-deepstream$(PLATFORM_SUFFIX)-extra + build: docker buildx build \ --platform $(PLATFORM) \ @@ -135,6 +138,7 @@ run-tests: savant-deepstream$(PLATFORM_SUFFIX)-extra \ -s $(PROJECT_PATH)/tests +# run jupyter inside `jupyter lab --allow-root` run-dev: #xhost +local:docker docker run -it --rm $(RUNTIME) \ diff --git a/adapters/requirements.txt b/adapters/requirements.txt index 8a70c7457..d23dd4ee3 100644 --- a/adapters/requirements.txt +++ b/adapters/requirements.txt @@ -1,4 +1,3 @@ -cachetools~=5.3.0 pretty-traceback==2023.1019 # ClientSDK JpegSource, PngSource python-magic~=0.4.27 diff --git a/docker/Dockerfile.deepstream b/docker/Dockerfile.deepstream index 855c4302b..45e4db23c 100644 --- a/docker/Dockerfile.deepstream +++ b/docker/Dockerfile.deepstream @@ -140,13 +140,8 @@ RUN wget -qO- \ dpkg -i OpenCV* && \ rm OpenCV* -COPY requirements/base.txt requirements/base.txt -RUN python -m pip install --no-cache-dir -r requirements/base.txt - -COPY requirements/ext.txt requirements/ext.txt -RUN python -m pip install --no-cache-dir -r requirements/ext.txt - -RUN rm -r requirements +COPY requirements/base.txt requirements.txt +RUN python -m pip install --no-cache-dir -r requirements.txt && rm requirements.txt COPY --from=savant-boost-builder /libs/savanboost/dist /libs/savanboost/dist RUN python -m pip install --no-cache-dir /libs/savanboost/dist/*.whl @@ -178,7 +173,9 @@ COPY savant/VERSION . COPY gst_plugins gst_plugins COPY adapters/gst/gst_plugins adapters/gst/gst_plugins -RUN rm -f adapters/gst/gst_plugins/python/video_files_sink.py +RUN rm -f \ + adapters/gst/gst_plugins/python/video_files_sink.py \ + adapters/gst/gst_plugins/python/zeromq_sink.py COPY adapters/gst/sources adapters/gst/sources COPY scripts/uri-input.py scripts/ @@ -293,13 +290,9 @@ RUN wget -nv -O ${PYCUDA_WHL} ${PACKAGES_URL}/${PYCUDA_WHL} && \ ARG ORT_VERSION=1.17.1 RUN python -m pip install --no-cache-dir onnx onnxruntime-gpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/onnxruntime-cuda-12/pypi/simple/ -# cython, polars, scikit-learn, jupyter -RUN python -m pip install --no-cache-dir \ - Cython \ - polars[all] \ - scikit-learn \ - jupyterlab \ - pytest +# cython, polars, scikit-learn, jupyter, etc. +COPY requirements/extra.txt requirements.txt +RUN python -m pip install --no-cache-dir -r requirements.txt && rm requirements.txt # Extra l4t container @@ -356,10 +349,6 @@ ARG ORT_WHL="onnxruntime_gpu-1.17.0-cp310-cp310-linux_aarch64.whl" RUN wget -nv -O ${ORT_WHL} ${ORT_URL} && \ python -m pip install --no-cache-dir onnx ${ORT_WHL} -# cython, polars, scikit-learn, jupyter -RUN python -m pip install --no-cache-dir \ - Cython \ - polars[all] \ - scikit-learn \ - jupyterlab \ - pytest +# cython, polars, scikit-learn, jupyter, etc. +COPY requirements/extra.txt requirements.txt +RUN python -m pip install --no-cache-dir -r requirements.txt && rm requirements.txt diff --git a/requirements/base.txt b/requirements/base.txt index 7c838c356..0fc5dfbb7 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,20 +1,15 @@ numpy~=1.22.4 cupy-cuda12x - numba~=0.57 scipy~=1.10 -pyzmq~=22.2.1 -cachetools~=5.3.0 - -splitstream==1.2.6 -click~=8.1.6 - +# yaml config omegaconf~=2.3 -# for omegaconf arithmetic resolver +# omegaconf arithmetic resolver simpleeval~=0.9.12 -# for dynamic pyfunc reloading +# dynamic pyfunc reloading inotify-simple~=1.3.5 + # pretty error output pretty-traceback==2023.1019 @@ -24,6 +19,15 @@ boto3~=1.23 tqdm~=4.64 # ClientSDK JpegSource, PngSource +click~=8.1.6 python-magic~=0.4.27 prometheus-client~=0.19.0 + +# pyds +pyds @ https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/releases/download/v1.1.10/pyds-1.1.10-py3-none-linux_x86_64.whl ; platform_machine=='x86_64' +pyds @ https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/releases/download/v1.1.10/pyds-1.1.10-py3-none-linux_aarch64.whl ; platform_machine=='aarch64' + +# ffmpeg-input +ffmpeg-input @ https://github.com/insight-platform/FFmpeg-Input/releases/download/0.1.23/ffmpeg_input-0.1.23-cp310-cp310-manylinux_2_28_x86_64.whl ; platform_machine=='x86_64' +ffmpeg-input @ https://github.com/insight-platform/FFmpeg-Input/releases/download/0.1.23/ffmpeg_input-0.1.23-cp310-cp310-manylinux_2_28_aarch64.whl ; platform_machine=='aarch64' diff --git a/requirements/ext.txt b/requirements/ext.txt deleted file mode 100644 index c9f8c23f6..000000000 --- a/requirements/ext.txt +++ /dev/null @@ -1,7 +0,0 @@ -# pyds -pyds @ https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/releases/download/v1.1.10/pyds-1.1.10-py3-none-linux_x86_64.whl; platform_machine=='x86_64' -pyds @ https://github.com/NVIDIA-AI-IOT/deepstream_python_apps/releases/download/v1.1.10/pyds-1.1.10-py3-none-linux_aarch64.whl; platform_machine=='aarch64' - -# ffmpeg-input -ffmpeg-input @ https://github.com/insight-platform/FFmpeg-Input/releases/download/0.1.23/ffmpeg_input-0.1.23-cp310-cp310-manylinux_2_28_x86_64.whl; platform_machine=='x86_64' -ffmpeg-input @ https://github.com/insight-platform/FFmpeg-Input/releases/download/0.1.23/ffmpeg_input-0.1.23-cp310-cp310-manylinux_2_28_aarch64.whl; platform_machine=='aarch64' diff --git a/requirements/extra.txt b/requirements/extra.txt new file mode 100644 index 000000000..615cec1c2 --- /dev/null +++ b/requirements/extra.txt @@ -0,0 +1,7 @@ +Cython +polars[all] +scikit-learn +jupyterlab +ipywidgets +matplotlib +pytest diff --git a/samples/nvidia_car_classification/flavors/module-etlt-config.yml b/samples/nvidia_car_classification/flavors/module-etlt-config.yml index 9e6f07970..2303a9386 100644 --- a/samples/nvidia_car_classification/flavors/module-etlt-config.yml +++ b/samples/nvidia_car_classification/flavors/module-etlt-config.yml @@ -13,13 +13,12 @@ pipeline: - element: nvinfer@detector name: DashCamNet model: - format: etlt remote: - url: "https://api.ngc.nvidia.com/v2/models/nvidia/tao/dashcamnet/versions/pruned_v1.0.2/zip" - model_file: resnet18_dashcamnet_pruned.etlt + url: "https://api.ngc.nvidia.com/v2/models/nvidia/tao/dashcamnet/versions/pruned_v1.0.4/zip" + model_file: resnet18_dashcamnet_pruned.onnx # label_file: labels.txt precision: int8 - int8_calib_file: dashcamnet_int8.txt + int8_calib_file: resnet18_dashcamnet_pruned_int8.txt batch_size: 1 input: layer_name: input_1 diff --git a/savant/VERSION b/savant/VERSION index 0a00e0d42..302510725 100644 --- a/savant/VERSION +++ b/savant/VERSION @@ -1,3 +1,3 @@ SAVANT=0.4.0 -SAVANT_RS=0.2.19 +SAVANT_RS=0.2.22 DEEPSTREAM=6.4 diff --git a/savant/converter/yolo.py b/savant/converter/yolo.py index 0bbdc55b8..d3bf02077 100644 --- a/savant/converter/yolo.py +++ b/savant/converter/yolo.py @@ -73,6 +73,10 @@ def __call__( confidences = det_scores[:num] class_ids = det_classes[:num] + # [0..1] -> model.input + bboxes[:, [0, 2]] *= model.input.width + bboxes[:, [1, 3]] *= model.input.height + # (left, top, right, bottom) -> (xc, yc, width, height) bboxes[:, 2] -= bboxes[:, 0] bboxes[:, 3] -= bboxes[:, 1] diff --git a/savant/deepstream/nvinfer/file_config.py b/savant/deepstream/nvinfer/file_config.py index 80181e524..ddd9b3636 100644 --- a/savant/deepstream/nvinfer/file_config.py +++ b/savant/deepstream/nvinfer/file_config.py @@ -189,6 +189,9 @@ class _FieldMap: _FieldMap('num-detected-classes', 'output.num_detected_classes', int), _FieldMap('gpu-id', 'gpu_id', int), _FieldMap('secondary-reinfer-interval', 'interval', int), + _FieldMap( + 'layer-device-precision', 'layer_device_precision', lambda v: v.split(';') + ), ] _CLASS_ATTR_MAP = [ @@ -198,7 +201,7 @@ class _FieldMap: _FieldMap('detected-min-h', 'min_height', int, 32), _FieldMap('detected-max-w', 'max_width', int), _FieldMap('detected-max-h', 'max_height', int), - # FieldMap('topk', 'topk', int), + _FieldMap('topk', 'top_k', int), ] @staticmethod diff --git a/savant/deepstream/nvinfer/model.py b/savant/deepstream/nvinfer/model.py index b535b3da6..ed3c3193a 100644 --- a/savant/deepstream/nvinfer/model.py +++ b/savant/deepstream/nvinfer/model.py @@ -1,11 +1,9 @@ """Gst-nvinfer model configuration templates.""" -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum from typing import List, Optional -from omegaconf import MISSING - from savant.base.model import ( AttributeModel, ComplexModel, @@ -173,6 +171,11 @@ class NvInferModel(Model): engine_create_func_name: Optional[str] = None """Name of the custom TensorRT CudaEngine creation function.""" + layer_device_precision: List[str] = field(default_factory=list) + """Specifies the device type and precision for any layer in the network. + List of items of format ``::``. + """ + NVINFER_DEFAULT_OBJECT_SELECTOR = PyFunc( module='savant.selector.detector', diff --git a/savant/deepstream/nvinfer/processor.py b/savant/deepstream/nvinfer/processor.py index 51c622f0b..3a86b2a7a 100644 --- a/savant/deepstream/nvinfer/processor.py +++ b/savant/deepstream/nvinfer/processor.py @@ -374,14 +374,15 @@ def _process_custom_model_output(self, buffer: Gst.Buffer): # object or complex model with non-empty output if bbox_tensor.shape[1] == 6: # no angle selection_type = ObjectSelectionType.REGULAR_BBOX + # xc -> left, yc -> top bbox_tensor[:, 2] -= bbox_tensor[:, 4] / 2 bbox_tensor[:, 3] -= bbox_tensor[:, 5] / 2 - # clip # width to right, height to bottom bbox_tensor[:, 4] += bbox_tensor[:, 2] bbox_tensor[:, 5] += bbox_tensor[:, 3] + # clip bbox_tensor[:, 2][ bbox_tensor[:, 2] < self._frame_rect[0] @@ -487,14 +488,10 @@ def _process_custom_model_output(self, buffer: Gst.Buffer): ) selected_bboxes.append((int(bbox[7]), _nvds_obj_meta)) + # attribute or complex model if values: - # attribute or complex model if self._is_complex_model: - values = [ - v - for i, v in enumerate(values) - if i in {i for i, _ in selected_bboxes} - ] + values = [values[i] for i, _ in selected_bboxes] else: selected_bboxes = [(0, nvds_obj_meta)] values = [values] diff --git a/savant/deepstream/pipeline.py b/savant/deepstream/pipeline.py index 79ef9cb70..7305158c6 100644 --- a/savant/deepstream/pipeline.py +++ b/savant/deepstream/pipeline.py @@ -811,6 +811,9 @@ def _update_meta_for_single_frame( frame_pts, ) + def _nvds_obj_id(_obj_meta: pyds.NvDsObjectMeta) -> str: + return f'{_obj_meta.obj_label}#{_obj_meta.object_id}' + # collect frame objects nvds_object_id_map = {} # nvds_obj_meta.object_id -> video_object.id parents = {} # video_object.id -> nvds_obj_meta.parent.object_id @@ -855,12 +858,12 @@ def _update_meta_for_single_frame( obj_meta = nvds_obj_meta_output_converter( nvds_frame_meta, nvds_obj_meta, self._frame_params, video_frame ) - nvds_object_id_map[nvds_obj_meta.object_id] = obj_meta.id + nvds_object_id_map[_nvds_obj_id(nvds_obj_meta)] = obj_meta.id if ( not nvds_is_empty_object_meta(nvds_obj_meta.parent) and nvds_obj_meta.parent.obj_label != PRIMARY_OBJECT_KEY ): - parents[obj_meta.id] = nvds_obj_meta.parent.object_id + parents[obj_meta.id] = _nvds_obj_id(nvds_obj_meta.parent) # add obj attributes for attr in attributes: obj_meta.set_attribute(attr) diff --git a/savant/selector/detector.py b/savant/selector/detector.py index c1b2c5fbd..eb7047467 100644 --- a/savant/selector/detector.py +++ b/savant/selector/detector.py @@ -87,11 +87,12 @@ def __call__(self, bbox_tensor: np.ndarray) -> np.ndarray: ) -@nb.njit('f4[:, :](f4[:, :], f4, f4, u2, u2, u2, u2)', nogil=True, cache=True) +@nb.njit('f4[:, :](f4[:, :], f4, f4, u2, u2, u2, u2, u2)', nogil=True, cache=True) def default_selector( bbox_tensor: np.ndarray, confidence_threshold: float = 0.0, nms_iou_threshold: float = 0.0, + top_k: int = 0, min_width: int = 0, min_height: int = 0, max_width: int = 0, @@ -102,6 +103,7 @@ def default_selector( :param bbox_tensor: tensor(class_id, confidence, left, top, width, height) :param confidence_threshold: confidence threshold :param nms_iou_threshold: nms iou threshold + :param top_k: top k bboxes to keep :param min_width: minimal bbox width :param min_height: minimal bbox height :param max_width: maximum bbox width @@ -129,7 +131,8 @@ def default_selector( selected_bbox_tensor[:, 2:6], selected_bbox_tensor[:, 1], nms_iou_threshold, - selected_bbox_tensor.shape[0], # should specify default with numba.njit + # should specify default with numba.njit + top_k if top_k > 0 else selected_bbox_tensor.shape[0], ) selected_bbox_tensor = selected_bbox_tensor[keep] @@ -141,6 +144,7 @@ class BBoxSelector(BaseSelector): :param confidence_threshold: confidence threshold :param nms_iou_threshold: nms iou threshold + :param top_k: top k bboxes to keep :param min_width: minimal bbox width :param min_height: minimal bbox height :param max_width: maximum bbox width @@ -151,6 +155,7 @@ def __init__( self, confidence_threshold: float = 0.5, nms_iou_threshold: float = 0.5, + top_k: int = 0, min_width: int = 0, min_height: int = 0, max_width: int = 0, @@ -160,6 +165,7 @@ def __init__( super().__init__(**kwargs) self.confidence_threshold = confidence_threshold self.nms_iou_threshold = nms_iou_threshold + self.top_k = top_k self.min_width = min_width self.min_height = min_height self.max_width = max_width @@ -175,6 +181,7 @@ def __call__(self, bbox_tensor: np.ndarray) -> np.ndarray: bbox_tensor=bbox_tensor, confidence_threshold=self.confidence_threshold, nms_iou_threshold=self.nms_iou_threshold, + top_k=self.top_k, min_width=self.min_width, min_height=self.min_height, max_width=self.max_width, diff --git a/savant/utils/file_types.py b/savant/utils/file_types.py index 49cb52e82..3e18b2b39 100644 --- a/savant/utils/file_types.py +++ b/savant/utils/file_types.py @@ -19,12 +19,15 @@ def from_mime_type(mime_type: Optional[str]) -> Optional['FileType']: def parse_mime_types(files: List[Path]) -> List[Tuple[Path, str]]: - output = subprocess.check_output( - ['file', '--no-pad', '--mime-type'] + [str(x) for x in files] - ) mime_types = [] - for line in output.decode().strip().split('\n'): - path, mime_type = line.rsplit(': ', 1) - mime_types.append((Path(path), mime_type)) - + # use chunks to avoid `Argument list too long` error + chunk_size = 1000 + for i in range(0, len(files), chunk_size): + chunk = files[i : i + chunk_size] + output = subprocess.check_output( + ['file', '--no-pad', '--mime-type'] + [str(x) for x in chunk] + ) + for line in output.decode().strip().split('\n'): + path, mime_type = line.rsplit(': ', 1) + mime_types.append((Path(path), mime_type)) return mime_types