[Feature] Support monocular 3D detection on nuScenes

mmdet3d/core/bbox/box_np_ops.py

@@ -335,21 +335,40 @@ def rotation_points_single_angle(points, angle, axis=0):
  return points @ rot_mat_T, rot_mat_T
 
 
-def points_cam2img(points_3d, proj_mat):
+def points_cam2img(points_3d, proj_mat, with_depth=False):
  """Project points in camera coordinates to image coordinates.
 
  Args:
  points_3d (np.ndarray): Points in shape (N, 3)
  proj_mat (np.ndarray): Transformation matrix between coordinates.
+ with_depth (bool): Whether to keep depth in the output.
 
  Returns:
  np.ndarray: Points in image coordinates with shape [N, 2].
  """
  points_shape = list(points_3d.shape)
  points_shape[-1] = 1
+
+ assert len(proj_mat.shape) == 2, 'The dimension of the projection'\
+ f' matrix should be 2 instead of {len(proj_mat.shape)}.'
+ d1, d2 = proj_mat.shape[:2]
+ assert (d1 == 3 and d2 == 3) or (d1 == 3 and d2 == 4) or (
+ d1 == 4 and d2 == 4), 'The shape of the projection matrix'\
+ f' ({d1}*{d2}) is not supported.'
+ if d1 == 3:
+ proj_mat_expanded = np.eye(4, dtype=proj_mat.dtype)
+ proj_mat_expanded[:d1, :d2] = proj_mat
+ proj_mat = proj_mat_expanded
+
  points_4 = np.concatenate([points_3d, np.ones(points_shape)], axis=-1)
  point_2d = points_4 @ proj_mat.T
  point_2d_res = point_2d[..., :2] / point_2d[..., 2:3]
+
+ if with_depth:
+ points_2d_depth = np.concatenate([point_2d_res, point_2d[..., 2:3]],
+ axis=-1)
+ return points_2d_depth
+
  return point_2d_res
 
 

mmdet3d/core/bbox/structures/utils.py

@@ -124,11 +124,11 @@ def points_cam2img(points_3d, proj_mat):
  points_num = list(points_3d.shape)[:-1]
 
  points_shape = np.concatenate([points_num, [1]], axis=0).tolist()
- assert len(proj_mat.shape) == 2, f'The dimension of the projection'\
- f'matrix should be 2 instead of {len(proj_mat.shape)}.'
+ assert len(proj_mat.shape) == 2, 'The dimension of the projection'\
+ f' matrix should be 2 instead of {len(proj_mat.shape)}.'
  d1, d2 = proj_mat.shape[:2]
  assert (d1 == 3 and d2 == 3) or (d1 == 3 and d2 == 4) or (
- d1 == 4 and d2 == 4), f'The shape of the projection matrix'\
+ d1 == 4 and d2 == 4), 'The shape of the projection matrix'\
  f' ({d1}*{d2}) is not supported.'
  if d1 == 3:
  proj_mat_expanded = torch.eye(

mmdet3d/core/bbox/transforms.py

@@ -46,22 +46,30 @@ def bbox3d2roi(bbox_list):
  return rois
 
 
-def bbox3d2result(bboxes, scores, labels):
+def bbox3d2result(bboxes, scores, labels, attrs=None):
  """Convert detection results to a list of numpy arrays.
 
  Args:
  bboxes (torch.Tensor): Bounding boxes with shape of (n, 5).
  labels (torch.Tensor): Labels with shape of (n, ).
  scores (torch.Tensor): Scores with shape of (n, ).
+ attrs (torch.Tensor, optional): Attributes with shape of (n, ). \
+ Defaults to None.
 
  Returns:
  dict[str, torch.Tensor]: Bounding box results in cpu mode.
 
  - boxes_3d (torch.Tensor): 3D boxes.
  - scores (torch.Tensor): Prediction scores.
  - labels_3d (torch.Tensor): Box labels.
+ - attrs_3d (torch.Tensor, optional): Box attributes.
  """
- return dict(
+ result_dict = dict(
  boxes_3d=bboxes.to('cpu'),
  scores_3d=scores.cpu(),
  labels_3d=labels.cpu())
+
+ if attrs is not None:
+ result_dict['attrs_3d'] = attrs.cpu()
+
+ return result_dict

mmdet3d/core/post_processing/box3d_nms.py

@@ -11,7 +11,9 @@ def box3d_multiclass_nms(mlvl_bboxes,
  score_thr,
  max_num,
  cfg,
- mlvl_dir_scores=None):
+ mlvl_dir_scores=None,
+ mlvl_attr_scores=None,
+ mlvl_bboxes2d=None):
  """Multi-class nms for 3D boxes.
 
  Args:
@@ -27,10 +29,15 @@ def box3d_multiclass_nms(mlvl_bboxes,
  cfg (dict): Configuration dict of NMS.
  mlvl_dir_scores (torch.Tensor, optional): Multi-level scores
  of direction classifier. Defaults to None.
+ mlvl_attr_scores (torch.Tensor, optional): Multi-level scores
+ of attribute classifier. Defaults to None.
+ mlvl_bboxes2d (torch.Tensor, optional): Multi-level 2D bounding
+ boxes. Defaults to None.
 
  Returns:
  tuple[torch.Tensor]: Return results after nms, including 3D \
- bounding boxes, scores, labels and direction scores.
+ bounding boxes, scores, labels, direction scores, attribute \
+ scores (optional) and 2D bounding boxes (optional).
  """
  # do multi class nms
  # the fg class id range: [0, num_classes-1]
@@ -39,6 +46,8 @@ def box3d_multiclass_nms(mlvl_bboxes,
  scores = []
  labels = []
  dir_scores = []
+ attr_scores = []
+ bboxes2d = []
  for i in range(0, num_classes):
  # get bboxes and scores of this class
  cls_inds = mlvl_scores[:, i] > score_thr
@@ -65,13 +74,23 @@ def box3d_multiclass_nms(mlvl_bboxes,
  if mlvl_dir_scores is not None:
  _mlvl_dir_scores = mlvl_dir_scores[cls_inds]
  dir_scores.append(_mlvl_dir_scores[selected])
+ if mlvl_attr_scores is not None:
+ _mlvl_attr_scores = mlvl_attr_scores[cls_inds]
+ attr_scores.append(_mlvl_attr_scores[selected])
+ if mlvl_bboxes2d is not None:
+ _mlvl_bboxes2d = mlvl_bboxes2d[cls_inds]
+ bboxes2d.append(_mlvl_bboxes2d[selected])
 
  if bboxes:
  bboxes = torch.cat(bboxes, dim=0)
  scores = torch.cat(scores, dim=0)
  labels = torch.cat(labels, dim=0)
  if mlvl_dir_scores is not None:
  dir_scores = torch.cat(dir_scores, dim=0)
+ if mlvl_attr_scores is not None:
+ attr_scores = torch.cat(attr_scores, dim=0)
+ if mlvl_bboxes2d is not None:
+ bboxes2d = torch.cat(bboxes2d, dim=0)
  if bboxes.shape[0] > max_num:
  _, inds = scores.sort(descending=True)
  inds = inds[:max_num]
@@ -80,12 +99,31 @@ def box3d_multiclass_nms(mlvl_bboxes,
  scores = scores[inds]
  if mlvl_dir_scores is not None:
  dir_scores = dir_scores[inds]
+ if mlvl_attr_scores is not None:
+ attr_scores = attr_scores[inds]
+ if mlvl_bboxes2d is not None:
+ bboxes2d = bboxes2d[inds]
  else:
  bboxes = mlvl_scores.new_zeros((0, mlvl_bboxes.size(-1)))
  scores = mlvl_scores.new_zeros((0, ))
  labels = mlvl_scores.new_zeros((0, ), dtype=torch.long)
- dir_scores = mlvl_scores.new_zeros((0, ))
- return bboxes, scores, labels, dir_scores
+ if mlvl_dir_scores is not None:
+ dir_scores = mlvl_scores.new_zeros((0, ))
+ if mlvl_attr_scores is not None:
+ attr_scores = mlvl_scores.new_zeros((0, ))
+ if mlvl_bboxes2d is not None:
+ bboxes2d = mlvl_scores.new_zeros((0, 4))
+
+ results = (bboxes, scores, labels)
+
+ if mlvl_dir_scores is not None:
+ results = results + (dir_scores, )
+ if mlvl_attr_scores is not None:
+ results = results + (attr_scores, )
+ if mlvl_bboxes2d is not None:
+ results = results + (bboxes2d, )
+
+ return results
 
 
 def aligned_3d_nms(boxes, scores, classes, thresh):

mmdet3d/datasets/__init__.py

@@ -4,6 +4,7 @@
 from .kitti_dataset import KittiDataset
 from .lyft_dataset import LyftDataset
 from .nuscenes_dataset import NuScenesDataset
+from .nuscenes_mono_dataset import NuScenesMonoDataset
 from .pipelines import (BackgroundPointsFilter, GlobalRotScaleTrans,
  IndoorPointSample, LoadAnnotations3D,
  LoadPointsFromFile, LoadPointsFromMultiSweeps,
@@ -18,9 +19,9 @@
 __all__ = [
  'KittiDataset', 'GroupSampler', 'DistributedGroupSampler',
  'build_dataloader', 'RepeatFactorDataset', 'DATASETS', 'build_dataset',
- 'CocoDataset', 'NuScenesDataset', 'LyftDataset', 'ObjectSample',
- 'RandomFlip3D', 'ObjectNoise', 'GlobalRotScaleTrans', 'PointShuffle',
- 'ObjectRangeFilter', 'PointsRangeFilter', 'Collect3D',
+ 'CocoDataset', 'NuScenesDataset', 'NuScenesMonoDataset', 'LyftDataset',
+ 'ObjectSample', 'RandomFlip3D', 'ObjectNoise', 'GlobalRotScaleTrans',
+ 'PointShuffle', 'ObjectRangeFilter', 'PointsRangeFilter', 'Collect3D',
  'LoadPointsFromFile', 'NormalizePointsColor', 'IndoorPointSample',
  'LoadAnnotations3D', 'SUNRGBDDataset', 'ScanNetDataset',
  'SemanticKITTIDataset', 'Custom3DDataset', 'LoadPointsFromMultiSweeps',