[Refactor] update Det3DLocalVisualization and visualization documen…

…tation (#1857)

* update vis doc

* support bev visualization and update proj_3d visualization

* fix bugs

* fix comments and add visualizer in dataset config

* fix comments

* add draw_points_on_image sample

* fix ci

* remove mkdir_or_exist

* update vis doc

configs/_base_/datasets/kitti-3d-3class.py

@@ -126,3 +126,7 @@
  ann_file=data_root + 'kitti_infos_val.pkl',
  metric='bbox')
 test_evaluator = val_evaluator
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/datasets/kitti-3d-car.py

@@ -124,3 +124,7 @@
  ann_file=data_root + 'kitti_infos_val.pkl',
  metric='bbox')
 test_evaluator = val_evaluator
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/datasets/kitti-mono3d.py

@@ -82,3 +82,7 @@
  pred_box_type_3d='Camera')
 
 test_evaluator = val_evaluator
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/datasets/lyft-3d.py

@@ -124,3 +124,7 @@
  type='LyftMetric',
  ann_file=data_root + 'lyft_infos_val.pkl',
  metric='bbox')
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/datasets/nus-3d.py

@@ -126,3 +126,7 @@
  ann_file=data_root + 'nuscenes_infos_val.pkl',
  metric='bbox')
 test_evaluator = val_evaluator
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/datasets/nus-mono3d.py

@@ -108,3 +108,7 @@
  metric='bbox')
 
 test_evaluator = val_evaluator
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/datasets/s3dis-seg.py

@@ -142,3 +142,7 @@
 
 val_evaluator = dict(type='SegMetric')
 test_evaluator = val_evaluator
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/datasets/scannet-3d.py

@@ -125,3 +125,7 @@
  box_type_3d='Depth'))
 val_evaluator = dict(type='IndoorMetric')
 test_evaluator = val_evaluator
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/datasets/scannet-seg.py

@@ -137,3 +137,7 @@
 
 val_evaluator = dict(type='SegMetric')
 test_evaluator = val_evaluator
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/datasets/sunrgbd-3d.py

@@ -111,3 +111,7 @@
  box_type_3d='Depth'))
 val_evaluator = dict(type='IndoorMetric')
 test_evaluator = val_evaluator
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/datasets/waymoD5-3d-3class.py

@@ -138,3 +138,7 @@
  waymo_bin_file='./data/waymo/waymo_format/gt.bin',
  data_root='./data/waymo/waymo_format')
 test_evaluator = val_evaluator
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/datasets/waymoD5-3d-car.py

@@ -136,3 +136,7 @@
  waymo_bin_file='./data/waymo/waymo_format/gt.bin',
  data_root='./data/waymo/waymo_format')
 test_evaluator = val_evaluator
+
+vis_backends = [dict(type='LocalVisBackend')]
+visualizer = dict(
+ type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')

configs/_base_/default_runtime.py

@@ -14,9 +14,6 @@
  dist_cfg=dict(backend='nccl'),
 )
 
-vis_backends = [dict(type='LocalVisBackend')]
-visualizer = dict(
- type='Det3DLocalVisualizer', vis_backends=vis_backends, name='visualizer')
 log_processor = dict(type='LogProcessor', window_size=50, by_epoch=True)
 
 log_level = 'INFO'

docs/en/user_guides/index.rst

@@ -11,3 +11,4 @@
  index.rst
  model_deployment.md
  useful_tools.md
+ visualization.md

docs/en/user_guides/visualization.md

@@ -0,0 +1,197 @@
+# Visualization
+
+MMDetection3D provides a `Det3DLocalVisualizer` to visualize and store the state of the model during training and testing, as well as results, with the following features.
+
+1. Support the basic drawing interface for multi-modality data and multi-task.
+2. Support multiple backends such as local, TensorBoard, to write training status such as `loss`, `lr`, or performance evaluation metrics and to a specified single or multiple backends.
+3. Support ground truth visualization on multimodal data, and cross-modal visualization of 3D detection results.
+
+## Basic Drawing Interface
+
+Inherited from `DetLocalVisualizer`, `Det3DLocalVisualizer` provides an interface for drawing common objects on 2D images, such as drawing detection boxes, points, text, lines, circles, polygons, and binary masks. More details about 2D drawing can refer to the [visualization documentation](https://mmengine.readthedocs.io/zh_CN/latest/advanced_tutorials/visualization.html) in MMDetection. Here we introduce the 3D drawing interface:
+
+### Drawing point cloud on the image
+
+We support drawing point cloud on the image by using `draw_points_on_image`.
+
+```python
+import mmcv
+from mmengine import load
+from mmdet3d.visualization import Det3DLocalVisualizer
+import numpy as np
+info_file = load('demo/data/kitti/000008.pkl')
+points = np.fromfile('demo/data/kitti/000008.bin', dtype=np.float32)
+points = points.reshape(-1, 4)[:, :3]
+lidar2img = np.array(info_file['data_list'][0]['images']['CAM2']['lidar2img'], dtype=np.float32)
+
+visualizer = Det3DLocalVisualizer()
+img = mmcv.imread('demo/data/kitti/000008.png')
+img = mmcv.imconvert(img, 'bgr', 'rgb')
+visualizer.set_image(img)
+visualizer.draw_points_on_image(points, lidar2img)
+visualizer.show()
+```
+
+![points_on_image](../../../resources/points_on_image.png)
+
+### Drawing 3D Boxes on Point Cloud
+
+We support drawing 3D boxes on point cloud by using `draw_bboxes_3d`.
+
+```python
+import torch
+from mmdet3d.visualization import Det3DLocalVisualizer
+from mmdet3d.structures import LiDARInstance3DBoxes
+
+points = np.fromfile('tests/data/kitti/training/velodyne/000000.bin', dtype=np.float32)
+points = points.reshape(-1, 4)
+visualizer = Det3DLocalVisualizer()
+# set point cloud in visualizer
+visualizer.set_points(points)
+bboxes_3d = LiDARInstance3DBoxes(torch.tensor(
+ [[8.7314, -1.8559, -1.5997, 1.2000, 0.4800, 1.8900,
+ -1.5808]])),
+# Draw 3D bboxes
+visualizer.draw_bboxes_3d(bboxes_3d)
+visualizer.show()
+```
+
+![mono3d](../../../resources/pcd.png)
+
+### Drawing Projected 3D Boxes on Image
+
+We support drawing projected 3D boxes on image by using `draw_proj_bboxes_3d`.
+
+```python
+import mmcv
+import numpy as np
+from mmengine import load
+from mmdet3d.visualization import Det3DLocalVisualizer
+from mmdet3d.structures import CameraInstance3DBoxes
+info_file = load('demo/data/kitti/000008.pkl')
+cam2img = np.array(info_file['data_list'][0]['images']['CAM2']['cam2img'], dtype=np.float32)
+bboxes_3d = []
+for instance in info_file['data_list'][0]['instances']:
+ bboxes_3d.append(instance['bbox_3d'])
+gt_bboxes_3d = np.array(bboxes_3d, dtype=np.float32)
+gt_bboxes_3d = CameraInstance3DBoxes(gt_bboxes_3d)
+input_meta = {'cam2img': cam2img}
+
+visualizer = Det3DLocalVisualizer()
+
+img = mmcv.imread('demo/data/kitti/000008.png')
+img = mmcv.imconvert(img, 'bgr', 'rgb')
+visualizer.set_image(img)
+# project 3D bboxes to image
+visualizer.draw_proj_bboxes_3d(gt_bboxes_3d, input_meta)
+visualizer.show()
+```
+
+![mono3d](../../../resources/mono3d.png)
+
+### Drawing BEV Boxes
+
+We support drawing projected 3D boxes on image by using `draw_proj_bboxes_3d`.
+
+```python
+import numpy as np
+from mmengine import load
+from mmdet3d.visualization import Det3DLocalVisualizer
+from mmdet3d.structures import CameraInstance3DBoxes
+
+info_file = load('demo/data/kitti/000008.pkl')
+bboxes_3d = []
+for instance in info_file['data_list'][0]['instances']:
+ bboxes_3d.append(instance['bbox_3d'])
+gt_bboxes_3d = np.array(bboxes_3d, dtype=np.float32)
+gt_bboxes_3d = CameraInstance3DBoxes(gt_bboxes_3d)
+
+visualizer = Det3DLocalVisualizer()
+# set bev image in visualizer
+visualizer.set_bev_image()
+# draw bev bboxes
+visualizer.draw_bev_bboxes(gt_bboxes_3d, edge_colors='orange')
+visualizer.show()
+```
+
+<img src="../../../resources/bev.png" width = "50%" />
+
+### Drawing 3D Semantic Mask
+
+We support draw segmentation mask via per-point colorization by using `draw_seg_mask`.
+
+```python
+import torch
+from mmdet3d.visualization import Det3DLocalVisualizer
+
+points = np.fromfile('tests/data/s3dis/points/Area_1_office_2.bin', dtype=np.float32)
+points = points.reshape(-1, 3)
+visualizer = Det3DLocalVisualizer()
+mask = np.random.rand(points.shape[0], 3)
+points_with_mask = np.concatenate((points, mask), axis=-1)
+# Draw 3D points with mask
+visualizer.draw_seg_mask(points_with_mask)
+visualizer.show()
+```
+
+## Results
+
+To see the prediction results of trained models, you can run the following command
+
+```bash
+python tools/test.py ${CONFIG_FILE} ${CKPT_PATH} --show --show-dir ${SHOW_DIR}
+```
+
+After running this command, plotted results including input data and the output of networks visualized on the input will be saved in `${SHOW_DIR}`.
+
+After running this command, you will obtain the input data, the output of networks and ground-truth labels visualized on the input (e.g. `***_gt.png` and `***_pred.png` in multi-modality detection task and vision-based detection task) in `${SHOW_DIR}`. When `show` is enabled, [Open3D](http://www.open3d.org/) will be used to visualize the results online. If you are running test in remote server without GUI, the online visualization is not supported, you can download the `results.pkl` from the remote server, and visualize the prediction results offline in your local machine.
+
+To visualize the results with `Open3D` backend offline, you can run the following command
+
+```bash
+python tools/misc/visualize_results.py ${CONFIG_FILE} --result ${RESULTS_PATH} --show-dir ${SHOW_DIR}
+```
+
+![](../../../resources/open3d_visual.gif)
+
+This allows the inference and results generation to be done in remote server and the users can open them on their host with GUI.
+
+## Dataset
+
+We also provide scripts to visualize the dataset without inference. You can use `tools/misc/browse_dataset.py` to show loaded data and ground-truth online and save them on the disk. Currently we support single-modality 3D detection and 3D segmentation on all the datasets, multi-modality 3D detection on KITTI and SUN RGB-D, as well as monocular 3D detection on nuScenes. To browse the KITTI dataset, you can run the following command
+
+```shell
+python tools/misc/browse_dataset.py configs/_base_/datasets/kitti-3d-3class.py --task det --output-dir ${OUTPUT_DIR}
+```
+
+**Notice**: Once specifying `--output-dir`, the images of views specified by users will be saved when pressing `_ESC_` in open3d window.
+
+To verify the data consistency and the effect of data augmentation, you can also add `--aug` flag to visualize the data after data augmentation using the command as below:
+
+```shell
+python tools/misc/browse_dataset.py configs/_base_/datasets/kitti-3d-3class.py --task det --aug --output-dir ${OUTPUT_DIR}
+```
+
+If you also want to show 2D images with 3D bounding boxes projected onto them, you need to find a config that supports multi-modality data loading, and then change the `--task` args to `multi_modality-det`. An example is showed below
+
+```shell
+python tools/misc/browse_dataset.py configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py --task multi-modality_det --output-dir ${OUTPUT_DIR}
+```
+
+![](../../../resources/browse_dataset_multi_modality.png)
+
+You can simply browse different datasets using different configs, e.g. visualizing the ScanNet dataset in 3D semantic segmentation task
+
+```shell
+python tools/misc/browse_dataset.py configs/_base_/datasets/scannet_seg-3d-20class.py --task seg --output-dir ${OUTPUT_DIR} --online
+```
+
+![](../../../resources/browse_dataset_seg.png)
+
+And browsing the nuScenes dataset in monocular 3D detection task
+
+```shell
+python tools/misc/browse_dataset.py configs/_base_/datasets/nus-mono3d.py --task mono_det --output-dir ${OUTPUT_DIR} --online
+```
+
+![](../../../resources/browse_dataset_mono.png)

mmdet3d/engine/hooks/visualization_hook.py

@@ -165,7 +165,8 @@ def after_test_iter(self, runner: Runner, batch_idx: int, data_batch: dict,
 
  out_file = None
  if self.test_out_dir is not None:
- out_file = self.test_out_dir
+ out_file = osp.basename(img_path)
+ out_file = osp.join(self.test_out_dir, out_file)
 
  self._visualizer.add_datasample(
  'test sample',