From 891b1bbb6cc9b1951e98e68776ba8d40e18add0a Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Wed, 23 Jun 2021 00:16:01 +0800 Subject: [PATCH 01/13] [Enhance] Sampling points based on distance metric --- mmdet3d/core/points/base_points.py | 6 ++ mmdet3d/datasets/__init__.py | 20 ++-- mmdet3d/datasets/pipelines/__init__.py | 17 ++-- mmdet3d/datasets/pipelines/transforms_3d.py | 98 +++++++++++++++---- .../test_datasets/test_scannet_dataset.py | 2 +- .../test_datasets/test_sunrgbd_dataset.py | 4 +- .../test_test_augment_utils.py | 2 +- .../test_pipelines/test_indoor_pipeline.py | 4 +- .../test_pipelines/test_indoor_sample.py | 9 +- 9 files changed, 115 insertions(+), 47 deletions(-) diff --git a/mmdet3d/core/points/base_points.py b/mmdet3d/core/points/base_points.py index 8e82dd9c6f..288e3babc6 100644 --- a/mmdet3d/core/points/base_points.py +++ b/mmdet3d/core/points/base_points.py @@ -281,6 +281,8 @@ def __getitem__(self, item): Nonzero elements in the vector will be selected. 4. `new_points = points[3:11, vector]`: return a slice of points and attribute dims. + 5. `new_points = points[4:12, 2]`: + return a slice of points with single attribute Note that the returned Points might share storage with this Points, subject to Pytorch's indexing semantics. @@ -303,6 +305,10 @@ def __getitem__(self, item): item = list(item) item[1] = list(range(start, stop, step)) item = tuple(item) + elif isinstance(item[1], int): + item = list(item) + item[1] = [item[1]] + item = tuple(item) p = self.tensor[item[0], item[1]] keep_dims = list( diff --git a/mmdet3d/datasets/__init__.py b/mmdet3d/datasets/__init__.py index 6bc252edf9..ec63956d60 100644 --- a/mmdet3d/datasets/__init__.py +++ b/mmdet3d/datasets/__init__.py @@ -7,14 +7,17 @@ from .lyft_dataset import LyftDataset from .nuscenes_dataset import NuScenesDataset from .nuscenes_mono_dataset import NuScenesMonoDataset +# yapf: disable from .pipelines import (BackgroundPointsFilter, GlobalAlignment, GlobalRotScaleTrans, IndoorPatchPointSample, IndoorPointSample, LoadAnnotations3D, LoadPointsFromFile, LoadPointsFromMultiSweeps, NormalizePointsColor, ObjectNameFilter, ObjectNoise, - ObjectRangeFilter, ObjectSample, PointShuffle, - PointsRangeFilter, RandomDropPointsColor, RandomFlip3D, - RandomJitterPoints, VoxelBasedPointSampler) + ObjectRangeFilter, ObjectSample, PointSample, + PointShuffle, PointsRangeFilter, RandomDropPointsColor, + RandomFlip3D, RandomJitterPoints, + VoxelBasedPointSampler) +# yapf: enable from .s3dis_dataset import S3DISSegDataset from .scannet_dataset import ScanNetDataset, ScanNetSegDataset from .semantickitti_dataset import SemanticKITTIDataset @@ -30,9 +33,10 @@ 'ObjectNoise', 'GlobalRotScaleTrans', 'PointShuffle', 'ObjectRangeFilter', 'PointsRangeFilter', 'Collect3D', 'LoadPointsFromFile', 'S3DISSegDataset', 'NormalizePointsColor', 'IndoorPatchPointSample', 'IndoorPointSample', - 'LoadAnnotations3D', 'GlobalAlignment', 'SUNRGBDDataset', 'ScanNetDataset', - 'ScanNetSegDataset', 'SemanticKITTIDataset', 'Custom3DDataset', - 'Custom3DSegDataset', 'LoadPointsFromMultiSweeps', 'WaymoDataset', - 'BackgroundPointsFilter', 'VoxelBasedPointSampler', 'get_loading_pipeline', - 'RandomDropPointsColor', 'RandomJitterPoints', 'ObjectNameFilter' + 'PointSample', 'LoadAnnotations3D', 'GlobalAlignment', 'SUNRGBDDataset', + 'ScanNetDataset', 'ScanNetSegDataset', 'SemanticKITTIDataset', + 'Custom3DDataset', 'Custom3DSegDataset', 'LoadPointsFromMultiSweeps', + 'WaymoDataset', 'BackgroundPointsFilter', 'VoxelBasedPointSampler', + 'get_loading_pipeline', 'RandomDropPointsColor', 'RandomJitterPoints', + 'ObjectNameFilter' ] diff --git a/mmdet3d/datasets/pipelines/__init__.py b/mmdet3d/datasets/pipelines/__init__.py index 4cf398ab28..f70114a063 100644 --- a/mmdet3d/datasets/pipelines/__init__.py +++ b/mmdet3d/datasets/pipelines/__init__.py @@ -9,10 +9,10 @@ from .transforms_3d import (BackgroundPointsFilter, GlobalAlignment, GlobalRotScaleTrans, IndoorPatchPointSample, IndoorPointSample, ObjectNameFilter, ObjectNoise, - ObjectRangeFilter, ObjectSample, PointShuffle, - PointsRangeFilter, RandomDropPointsColor, - RandomFlip3D, RandomJitterPoints, - VoxelBasedPointSampler) + ObjectRangeFilter, ObjectSample, PointSample, + PointShuffle, PointsRangeFilter, + RandomDropPointsColor, RandomFlip3D, + RandomJitterPoints, VoxelBasedPointSampler) __all__ = [ 'ObjectSample', 'RandomFlip3D', 'ObjectNoise', 'GlobalRotScaleTrans', @@ -20,8 +20,9 @@ 'Compose', 'LoadMultiViewImageFromFiles', 'LoadPointsFromFile', 'DefaultFormatBundle', 'DefaultFormatBundle3D', 'DataBaseSampler', 'NormalizePointsColor', 'LoadAnnotations3D', 'IndoorPointSample', - 'PointSegClassMapping', 'MultiScaleFlipAug3D', 'LoadPointsFromMultiSweeps', - 'BackgroundPointsFilter', 'VoxelBasedPointSampler', 'GlobalAlignment', - 'IndoorPatchPointSample', 'LoadImageFromFileMono3D', 'ObjectNameFilter', - 'RandomDropPointsColor', 'RandomJitterPoints' + 'PointSample', 'PointSegClassMapping', 'MultiScaleFlipAug3D', + 'LoadPointsFromMultiSweeps', 'BackgroundPointsFilter', + 'VoxelBasedPointSampler', 'GlobalAlignment', 'IndoorPatchPointSample', + 'LoadImageFromFileMono3D', 'ObjectNameFilter', 'RandomDropPointsColor', + 'RandomJitterPoints' ] diff --git a/mmdet3d/datasets/pipelines/transforms_3d.py b/mmdet3d/datasets/pipelines/transforms_3d.py index e75e63fb6e..05ede45f23 100644 --- a/mmdet3d/datasets/pipelines/transforms_3d.py +++ b/mmdet3d/datasets/pipelines/transforms_3d.py @@ -818,24 +818,26 @@ def __repr__(self): @PIPELINES.register_module() -class IndoorPointSample(object): - """Indoor point sample. +class PointSample(object): + """Point sample. Sampling data to a certain number. Args: - name (str): Name of the dataset. num_points (int): Number of points to be sampled. + dist_metric (int, optional): The indicator to the near/far boundary """ - def __init__(self, num_points): + def __init__(self, num_points, dist_metric=None): self.num_points = num_points - - def points_random_sampling(self, - points, - num_samples, - replace=None, - return_choices=False): + self.dist_metric = dist_metric + + def _points_random_sampling(self, + points, + num_samples, + dist_metric=None, + replace=None, + return_choices=False): """Points random sampling. Sample points to a certain number. @@ -843,20 +845,34 @@ def points_random_sampling(self, Args: points (np.ndarray | :obj:`BasePoints`): 3D Points. num_samples (int): Number of samples to be sampled. - replace (bool): Whether the sample is with or without replacement. - Defaults to None. - return_choices (bool): Whether return choice. Defaults to False. - + dist_metric (int, optional): Indicator to the near/far boundary. + Once given, only the near points will be sampled. + Defaults to None. + replace (bool, optional): Sampling with or without replacement. + Defaults to None. + return_choices (bool, optional): Whether return choice. + Defaults to False. Returns: tuple[np.ndarray] | np.ndarray: - - points (np.ndarray | :obj:`BasePoints`): 3D Points. - choices (np.ndarray, optional): The generated random samples. """ if replace is None: replace = (points.shape[0] < num_samples) - choices = np.random.choice( - points.shape[0], num_samples, replace=replace) + sample_range = range(len(points)) + if dist_metric: + depth = points.coord[:, 2] + far_inds = np.where(depth > dist_metric)[0] + near_inds = np.where(depth <= dist_metric)[0] + if not replace: + # Only sampling the near points when len(points) >= num_samples + sample_range = near_inds + num_samples -= len(far_inds) + choices = np.random.choice(sample_range, num_samples, replace=replace) + if dist_metric and not replace: + choices = np.concatenate((far_inds, choices)) + # Shuffle points after sampling + np.random.shuffle(choices) if return_choices: return points[choices], choices else: @@ -867,14 +883,19 @@ def __call__(self, results): Args: input_dict (dict): Result dict from loading pipeline. - Returns: dict: Results after sampling, 'points', 'pts_instance_mask' \ and 'pts_semantic_mask' keys are updated in the result dict. """ + from mmdet3d.core.points import CameraPoints points = results['points'] - points, choices = self.points_random_sampling( - points, self.num_points, return_choices=True) + # Points in Camera coord can provide the depth information. + # TODO: Need to suport distance-based sampling for other coord system. + if self.dist_metric: + assert isinstance(points, CameraPoints), \ + 'Sampling based on distance is only appliable for CAMERA coord' + points, choices = self._points_random_sampling( + points, self.num_points, self.dist_metric, return_choices=True) results['points'] = points pts_instance_mask = results.get('pts_instance_mask', None) @@ -890,6 +911,43 @@ def __call__(self, results): return results + def __repr__(self): + """str: Return a string that describes the module.""" + repr_str = self.__class__.__name__ + repr_str += f'(num_points={self.num_points},' + repr_str += f' dist_metric={self.dist_metric})' + + return repr_str + + +@PIPELINES.register_module() +class IndoorPointSample(PointSample): + """Indoor point sample. + + Sampling data to a certain number. + NOTE: IndoorPointSample is deprecated in favor of PointSample + + Args: + num_points (int): Number of points to be sampled. + """ + + def __init__(self, num_points): + warnings.warn( + 'IndoorPointSample is deprecated in favor of PointSample') + super(IndoorPointSample, self).__init__(num_points) + self.num_points = num_points + + def __call__(self, results): + """Call function to sample points to in indoor scenes. + + Args: + input_dict (dict): Result dict from loading pipeline. + Returns: + dict: Results after sampling, 'points', 'pts_instance_mask' \ + and 'pts_semantic_mask' keys are updated in the result dict. + """ + return super(IndoorPointSample, self).__call__(results) + def __repr__(self): """str: Return a string that describes the module.""" repr_str = self.__class__.__name__ diff --git a/tests/test_data/test_datasets/test_scannet_dataset.py b/tests/test_data/test_datasets/test_scannet_dataset.py index 10aac8a3dd..f098d2d983 100644 --- a/tests/test_data/test_datasets/test_scannet_dataset.py +++ b/tests/test_data/test_datasets/test_scannet_dataset.py @@ -32,7 +32,7 @@ def test_getitem(): type='PointSegClassMapping', valid_cat_ids=(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34, 36, 39)), - dict(type='IndoorPointSample', num_points=5), + dict(type='PointSample', num_points=5), dict( type='RandomFlip3D', sync_2d=False, diff --git a/tests/test_data/test_datasets/test_sunrgbd_dataset.py b/tests/test_data/test_datasets/test_sunrgbd_dataset.py index ebcfc044a3..1c8e480185 100644 --- a/tests/test_data/test_datasets/test_sunrgbd_dataset.py +++ b/tests/test_data/test_datasets/test_sunrgbd_dataset.py @@ -28,7 +28,7 @@ def _generate_sunrgbd_dataset_config(): rot_range=[-0.523599, 0.523599], scale_ratio_range=[0.85, 1.15], shift_height=True), - dict(type='IndoorPointSample', num_points=5), + dict(type='PointSample', num_points=5), dict(type='DefaultFormatBundle3D', class_names=class_names), dict( type='Collect3D', @@ -73,7 +73,7 @@ def _generate_sunrgbd_multi_modality_dataset_config(): rot_range=[-0.523599, 0.523599], scale_ratio_range=[0.85, 1.15], shift_height=True), - dict(type='IndoorPointSample', num_points=5), + dict(type='PointSample', num_points=5), dict(type='DefaultFormatBundle3D', class_names=class_names), dict( type='Collect3D', diff --git a/tests/test_data/test_pipelines/test_augmentations/test_test_augment_utils.py b/tests/test_data/test_pipelines/test_augmentations/test_test_augment_utils.py index 37d9771e15..946c52998c 100644 --- a/tests/test_data/test_pipelines/test_augmentations/test_test_augment_utils.py +++ b/tests/test_data/test_pipelines/test_augmentations/test_test_augment_utils.py @@ -17,7 +17,7 @@ def test_multi_scale_flip_aug_3D(): 'sync_2d': False, 'flip_ratio_bev_horizontal': 0.5 }, { - 'type': 'IndoorPointSample', + 'type': 'PointSample', 'num_points': 5 }, { 'type': diff --git a/tests/test_data/test_pipelines/test_indoor_pipeline.py b/tests/test_data/test_pipelines/test_indoor_pipeline.py index 91f87a942a..058c8aff5c 100644 --- a/tests/test_data/test_pipelines/test_indoor_pipeline.py +++ b/tests/test_data/test_pipelines/test_indoor_pipeline.py @@ -32,7 +32,7 @@ def test_scannet_pipeline(): type='PointSegClassMapping', valid_cat_ids=(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34, 36, 39)), - dict(type='IndoorPointSample', num_points=5), + dict(type='PointSample', num_points=5), dict( type='RandomFlip3D', sync_2d=False, @@ -278,7 +278,7 @@ def test_sunrgbd_pipeline(): rot_range=[-0.523599, 0.523599], scale_ratio_range=[0.85, 1.15], shift_height=True), - dict(type='IndoorPointSample', num_points=5), + dict(type='PointSample', num_points=5), dict(type='DefaultFormatBundle3D', class_names=class_names), dict( type='Collect3D', keys=['points', 'gt_bboxes_3d', 'gt_labels_3d']), diff --git a/tests/test_data/test_pipelines/test_indoor_sample.py b/tests/test_data/test_pipelines/test_indoor_sample.py index d529f7b10d..998d489a19 100644 --- a/tests/test_data/test_pipelines/test_indoor_sample.py +++ b/tests/test_data/test_pipelines/test_indoor_sample.py @@ -1,14 +1,13 @@ import numpy as np from mmdet3d.core.points import DepthPoints -from mmdet3d.datasets.pipelines import (IndoorPatchPointSample, - IndoorPointSample, +from mmdet3d.datasets.pipelines import (IndoorPatchPointSample, PointSample, PointSegClassMapping) def test_indoor_sample(): np.random.seed(0) - scannet_sample_points = IndoorPointSample(5) + scannet_sample_points = PointSample(5) scannet_results = dict() scannet_points = np.array([[1.0719866, -0.7870435, 0.8408122, 0.9196809], [1.103661, 0.81065744, 2.6616862, 2.7405548], @@ -39,7 +38,7 @@ def test_indoor_sample(): scannet_semantic_labels_result) np.random.seed(0) - sunrgbd_sample_points = IndoorPointSample(5) + sunrgbd_sample_points = PointSample(5) sunrgbd_results = dict() sunrgbd_point_cloud = np.array( [[-1.8135729e-01, 1.4695230e+00, -1.2780589e+00, 7.8938007e-03], @@ -58,7 +57,7 @@ def test_indoor_sample(): sunrgbd_choices = np.array([2, 8, 4, 9, 1]) sunrgbd_points_result = sunrgbd_results['points'].tensor.numpy() repr_str = repr(sunrgbd_sample_points) - expected_repr_str = 'IndoorPointSample(num_points=5)' + expected_repr_str = 'PointSample(num_points=5)' assert repr_str == expected_repr_str assert np.allclose(sunrgbd_point_cloud[sunrgbd_choices], sunrgbd_points_result) From 1469c6fae66cb056bc6dc5d73b3f98bf1f3c77f9 Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Wed, 23 Jun 2021 00:22:25 +0800 Subject: [PATCH 02/13] fix typo --- mmdet3d/core/points/base_points.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmdet3d/core/points/base_points.py b/mmdet3d/core/points/base_points.py index 288e3babc6..95b754cb76 100644 --- a/mmdet3d/core/points/base_points.py +++ b/mmdet3d/core/points/base_points.py @@ -282,7 +282,7 @@ def __getitem__(self, item): 4. `new_points = points[3:11, vector]`: return a slice of points and attribute dims. 5. `new_points = points[4:12, 2]`: - return a slice of points with single attribute + return a slice of points with single attribute. Note that the returned Points might share storage with this Points, subject to Pytorch's indexing semantics. From 25c0c342627fac7e5ce73b591d4c812fdc20ca95 Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Wed, 23 Jun 2021 13:32:32 +0800 Subject: [PATCH 03/13] refine unittest --- tests/test_data/test_pipelines/test_indoor_sample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_data/test_pipelines/test_indoor_sample.py b/tests/test_data/test_pipelines/test_indoor_sample.py index 998d489a19..a950373c3c 100644 --- a/tests/test_data/test_pipelines/test_indoor_sample.py +++ b/tests/test_data/test_pipelines/test_indoor_sample.py @@ -57,7 +57,7 @@ def test_indoor_sample(): sunrgbd_choices = np.array([2, 8, 4, 9, 1]) sunrgbd_points_result = sunrgbd_results['points'].tensor.numpy() repr_str = repr(sunrgbd_sample_points) - expected_repr_str = 'PointSample(num_points=5)' + expected_repr_str = 'PointSample(num_points=5), dist_metric=None)' assert repr_str == expected_repr_str assert np.allclose(sunrgbd_point_cloud[sunrgbd_choices], sunrgbd_points_result) From ae2de141d7fcb89b1e7a9f5aaa865acf253e4007 Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Wed, 23 Jun 2021 16:26:35 +0800 Subject: [PATCH 04/13] refine unittest --- tests/test_data/test_pipelines/test_indoor_sample.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_data/test_pipelines/test_indoor_sample.py b/tests/test_data/test_pipelines/test_indoor_sample.py index a950373c3c..ec7fbebd99 100644 --- a/tests/test_data/test_pipelines/test_indoor_sample.py +++ b/tests/test_data/test_pipelines/test_indoor_sample.py @@ -57,7 +57,7 @@ def test_indoor_sample(): sunrgbd_choices = np.array([2, 8, 4, 9, 1]) sunrgbd_points_result = sunrgbd_results['points'].tensor.numpy() repr_str = repr(sunrgbd_sample_points) - expected_repr_str = 'PointSample(num_points=5), dist_metric=None)' + expected_repr_str = 'PointSample(num_points=5, dist_metric=None)' assert repr_str == expected_repr_str assert np.allclose(sunrgbd_point_cloud[sunrgbd_choices], sunrgbd_points_result) From 1c35deb5b013204b90aee9f0b0c0c77050a67f47 Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Tue, 6 Jul 2021 10:49:18 +0800 Subject: [PATCH 05/13] refine details & add unittest & refine configs --- configs/3dssd/3dssd_4x4_kitti-3d-car.py | 4 ++-- configs/_base_/datasets/sunrgbd-3d-10class.py | 4 ++-- .../imvotenet_stage2_16x8_sunrgbd-3d-10class.py | 4 ++-- docs/tutorials/config.md | 10 +++++----- mmdet3d/datasets/pipelines/transforms_3d.py | 16 ++-------------- tests/test_utils/test_points.py | 8 ++++++++ 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/configs/3dssd/3dssd_4x4_kitti-3d-car.py b/configs/3dssd/3dssd_4x4_kitti-3d-car.py index fead2b5ed5..1e705db182 100644 --- a/configs/3dssd/3dssd_4x4_kitti-3d-car.py +++ b/configs/3dssd/3dssd_4x4_kitti-3d-car.py @@ -51,7 +51,7 @@ rot_range=[-0.78539816, 0.78539816], scale_ratio_range=[0.9, 1.1]), dict(type='BackgroundPointsFilter', bbox_enlarge_range=(0.5, 2.0, 0.5)), - dict(type='IndoorPointSample', num_points=16384), + dict(type='PointSample', num_points=16384), dict(type='DefaultFormatBundle3D', class_names=class_names), dict(type='Collect3D', keys=['points', 'gt_bboxes_3d', 'gt_labels_3d']) ] @@ -77,7 +77,7 @@ dict(type='RandomFlip3D'), dict( type='PointsRangeFilter', point_cloud_range=point_cloud_range), - dict(type='IndoorPointSample', num_points=16384), + dict(type='PointSample', num_points=16384), dict( type='DefaultFormatBundle3D', class_names=class_names, diff --git a/configs/_base_/datasets/sunrgbd-3d-10class.py b/configs/_base_/datasets/sunrgbd-3d-10class.py index 6e40e3d2a6..7121b75bbf 100644 --- a/configs/_base_/datasets/sunrgbd-3d-10class.py +++ b/configs/_base_/datasets/sunrgbd-3d-10class.py @@ -20,7 +20,7 @@ rot_range=[-0.523599, 0.523599], scale_ratio_range=[0.85, 1.15], shift_height=True), - dict(type='IndoorPointSample', num_points=20000), + dict(type='PointSample', num_points=20000), dict(type='DefaultFormatBundle3D', class_names=class_names), dict(type='Collect3D', keys=['points', 'gt_bboxes_3d', 'gt_labels_3d']) ] @@ -47,7 +47,7 @@ sync_2d=False, flip_ratio_bev_horizontal=0.5, ), - dict(type='IndoorPointSample', num_points=20000), + dict(type='PointSample', num_points=20000), dict( type='DefaultFormatBundle3D', class_names=class_names, diff --git a/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py b/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py index bb8f314132..1dce5483f1 100644 --- a/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py +++ b/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py @@ -187,7 +187,7 @@ rot_range=[-0.523599, 0.523599], scale_ratio_range=[0.85, 1.15], shift_height=True), - dict(type='IndoorPointSample', num_points=20000), + dict(type='PointSample', num_points=20000), dict(type='DefaultFormatBundle3D', class_names=class_names), dict( type='Collect3D', @@ -225,7 +225,7 @@ sync_2d=False, flip_ratio_bev_horizontal=0.5, ), - dict(type='IndoorPointSample', num_points=20000), + dict(type='PointSample', num_points=20000), dict( type='DefaultFormatBundle3D', class_names=class_names, diff --git a/docs/tutorials/config.md b/docs/tutorials/config.md index caf31dfb87..c58287f459 100644 --- a/docs/tutorials/config.md +++ b/docs/tutorials/config.md @@ -203,7 +203,7 @@ train_pipeline = [ # Training pipeline, refer to mmdet3d.datasets.pipelines for valid_cat_ids=(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34, 36, 39), # all valid categories ids max_cat_id=40), # max possible category id in input segmentation mask - dict(type='IndoorPointSample', # Sample indoor points, refer to mmdet3d.datasets.pipelines.indoor_sample for more details + dict(type='PointSample', # Sample points, refer to mmdet3d.datasets.pipelines.transforms_3d for more details num_points=40000), # Number of points to be sampled dict(type='IndoorFlipData', # Augmentation pipeline that flip points and 3d boxes flip_ratio_yz=0.5, # Probability of being flipped along yz plane @@ -232,7 +232,7 @@ test_pipeline = [ # Testing pipeline, refer to mmdet3d.datasets.pipelines for m shift_height=True, # Whether to use shifted height load_dim=6, # The dimension of the loaded points use_dim=[0, 1, 2]), # Which dimensions of the points to be used - dict(type='IndoorPointSample', # Sample indoor points, refer to mmdet3d.datasets.pipelines.indoor_sample for more details + dict(type='PointSample', # Sample points, refer to mmdet3d.datasets.pipelines.transforms_3d for more details num_points=40000), # Number of points to be sampled dict( type='DefaultFormatBundle3D', # Default format bundle to gather data in the pipeline, refer to mmdet3d.datasets.pipelines.formating for more details @@ -286,7 +286,7 @@ data = dict( valid_cat_ids=(3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34, 36, 39), max_cat_id=40), - dict(type='IndoorPointSample', num_points=40000), + dict(type='PointSample', num_points=40000), dict( type='IndoorFlipData', flip_ratio_yz=0.5, @@ -325,7 +325,7 @@ data = dict( shift_height=True, load_dim=6, use_dim=[0, 1, 2]), - dict(type='IndoorPointSample', num_points=40000), + dict(type='PointSample', num_points=40000), dict( type='DefaultFormatBundle3D', class_names=('cabinet', 'bed', 'chair', 'sofa', 'table', @@ -350,7 +350,7 @@ data = dict( shift_height=True, load_dim=6, use_dim=[0, 1, 2]), - dict(type='IndoorPointSample', num_points=40000), + dict(type='PointSample', num_points=40000), dict( type='DefaultFormatBundle3D', class_names=('cabinet', 'bed', 'chair', 'sofa', 'table', diff --git a/mmdet3d/datasets/pipelines/transforms_3d.py b/mmdet3d/datasets/pipelines/transforms_3d.py index 05ede45f23..484efbe31a 100644 --- a/mmdet3d/datasets/pipelines/transforms_3d.py +++ b/mmdet3d/datasets/pipelines/transforms_3d.py @@ -931,22 +931,10 @@ class IndoorPointSample(PointSample): num_points (int): Number of points to be sampled. """ - def __init__(self, num_points): + def __init__(self, *args, **kwargs): warnings.warn( 'IndoorPointSample is deprecated in favor of PointSample') - super(IndoorPointSample, self).__init__(num_points) - self.num_points = num_points - - def __call__(self, results): - """Call function to sample points to in indoor scenes. - - Args: - input_dict (dict): Result dict from loading pipeline. - Returns: - dict: Results after sampling, 'points', 'pts_instance_mask' \ - and 'pts_semantic_mask' keys are updated in the result dict. - """ - return super(IndoorPointSample, self).__call__(results) + super(IndoorPointSample, self).__init__(*args, **kwargs) def __repr__(self): """str: Return a string that describes the module.""" diff --git a/tests/test_utils/test_points.py b/tests/test_utils/test_points.py index cfd5ef0dbe..aaa4325623 100644 --- a/tests/test_utils/test_points.py +++ b/tests/test_utils/test_points.py @@ -193,6 +193,8 @@ def test_base_points(): [[9.0722, 47.3678, -2.5382, 0.6666, 0.1956, 0.4974, 0.9409], [6.8547, 42.2509, -2.5955, 0.6565, 0.6248, 0.6954, 0.2538]]) assert torch.allclose(expected_tensor, base_points[mask].tensor, 1e-4) + expected_tensor = torch.tensor([[0.6666], [0.1502], [0.6565], [0.2803]]) + assert torch.allclose(expected_tensor, base_points[:, 3].tensor, 1e-4) # test length assert len(base_points) == 4 @@ -451,6 +453,8 @@ def test_cam_points(): [[9.0722, 47.3678, -2.5382, 0.6666, 0.1956, 0.4974, 0.9409], [6.8547, 42.2509, -2.5955, 0.6565, 0.6248, 0.6954, 0.2538]]) assert torch.allclose(expected_tensor, cam_points[mask].tensor, 1e-4) + expected_tensor = torch.tensor([[0.6666], [0.1502], [0.6565], [0.2803]]) + assert torch.allclose(expected_tensor, cam_points[:, 3].tensor, 1e-4) # test length assert len(cam_points) == 4 @@ -725,6 +729,8 @@ def test_lidar_points(): [[9.0722, 47.3678, -2.5382, 0.6666, 0.1956, 0.4974, 0.9409], [6.8547, 42.2509, -2.5955, 0.6565, 0.6248, 0.6954, 0.2538]]) assert torch.allclose(expected_tensor, lidar_points[mask].tensor, 1e-4) + expected_tensor = torch.tensor([[0.6666], [0.1502], [0.6565], [0.2803]]) + assert torch.allclose(expected_tensor, lidar_points[:, 3].tensor, 1e-4) # test length assert len(lidar_points) == 4 @@ -999,6 +1005,8 @@ def test_depth_points(): [[9.0722, 47.3678, -2.5382, 0.6666, 0.1956, 0.4974, 0.9409], [6.8547, 42.2509, -2.5955, 0.6565, 0.6248, 0.6954, 0.2538]]) assert torch.allclose(expected_tensor, depth_points[mask].tensor, 1e-4) + expected_tensor = torch.tensor([[0.6666], [0.1502], [0.6565], [0.2803]]) + assert torch.allclose(expected_tensor, depth_points[:, 3].tensor, 1e-4) # test length assert len(depth_points) == 4 From 04e2f2c8fa5feadea9c976477e22db34d3830269 Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Wed, 7 Jul 2021 10:00:01 +0800 Subject: [PATCH 06/13] remove __repr__ & rename arg --- mmdet3d/datasets/pipelines/transforms_3d.py | 32 ++++++++----------- .../test_pipelines/test_indoor_sample.py | 2 +- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/mmdet3d/datasets/pipelines/transforms_3d.py b/mmdet3d/datasets/pipelines/transforms_3d.py index 484efbe31a..bd9b079017 100644 --- a/mmdet3d/datasets/pipelines/transforms_3d.py +++ b/mmdet3d/datasets/pipelines/transforms_3d.py @@ -825,17 +825,17 @@ class PointSample(object): Args: num_points (int): Number of points to be sampled. - dist_metric (int, optional): The indicator to the near/far boundary + sample_range (int, optional): The range where to sample points. """ - def __init__(self, num_points, dist_metric=None): + def __init__(self, num_points, sample_range=None): self.num_points = num_points - self.dist_metric = dist_metric + self.sample_range = sample_range def _points_random_sampling(self, points, num_samples, - dist_metric=None, + sample_range=None, replace=None, return_choices=False): """Points random sampling. @@ -845,8 +845,8 @@ def _points_random_sampling(self, Args: points (np.ndarray | :obj:`BasePoints`): 3D Points. num_samples (int): Number of samples to be sampled. - dist_metric (int, optional): Indicator to the near/far boundary. - Once given, only the near points will be sampled. + sample_range (int, optional): Indicating the range where the points + will be sampled. Defaults to None. replace (bool, optional): Sampling with or without replacement. Defaults to None. @@ -860,16 +860,16 @@ def _points_random_sampling(self, if replace is None: replace = (points.shape[0] < num_samples) sample_range = range(len(points)) - if dist_metric: + if sample_range: depth = points.coord[:, 2] - far_inds = np.where(depth > dist_metric)[0] - near_inds = np.where(depth <= dist_metric)[0] + far_inds = np.where(depth > sample_range)[0] + near_inds = np.where(depth <= sample_range)[0] if not replace: # Only sampling the near points when len(points) >= num_samples sample_range = near_inds num_samples -= len(far_inds) choices = np.random.choice(sample_range, num_samples, replace=replace) - if dist_metric and not replace: + if sample_range and not replace: choices = np.concatenate((far_inds, choices)) # Shuffle points after sampling np.random.shuffle(choices) @@ -891,11 +891,11 @@ def __call__(self, results): points = results['points'] # Points in Camera coord can provide the depth information. # TODO: Need to suport distance-based sampling for other coord system. - if self.dist_metric: + if self.sample_range: assert isinstance(points, CameraPoints), \ 'Sampling based on distance is only appliable for CAMERA coord' points, choices = self._points_random_sampling( - points, self.num_points, self.dist_metric, return_choices=True) + points, self.num_points, self.sample_range, return_choices=True) results['points'] = points pts_instance_mask = results.get('pts_instance_mask', None) @@ -915,7 +915,7 @@ def __repr__(self): """str: Return a string that describes the module.""" repr_str = self.__class__.__name__ repr_str += f'(num_points={self.num_points},' - repr_str += f' dist_metric={self.dist_metric})' + repr_str += f' sample_range={self.sample_range})' return repr_str @@ -936,12 +936,6 @@ def __init__(self, *args, **kwargs): 'IndoorPointSample is deprecated in favor of PointSample') super(IndoorPointSample, self).__init__(*args, **kwargs) - def __repr__(self): - """str: Return a string that describes the module.""" - repr_str = self.__class__.__name__ - repr_str += f'(num_points={self.num_points})' - return repr_str - @PIPELINES.register_module() class IndoorPatchPointSample(object): diff --git a/tests/test_data/test_pipelines/test_indoor_sample.py b/tests/test_data/test_pipelines/test_indoor_sample.py index ec7fbebd99..527cafc6cb 100644 --- a/tests/test_data/test_pipelines/test_indoor_sample.py +++ b/tests/test_data/test_pipelines/test_indoor_sample.py @@ -57,7 +57,7 @@ def test_indoor_sample(): sunrgbd_choices = np.array([2, 8, 4, 9, 1]) sunrgbd_points_result = sunrgbd_results['points'].tensor.numpy() repr_str = repr(sunrgbd_sample_points) - expected_repr_str = 'PointSample(num_points=5, dist_metric=None)' + expected_repr_str = 'PointSample(num_points=5, sample_range=None)' assert repr_str == expected_repr_str assert np.allclose(sunrgbd_point_cloud[sunrgbd_choices], sunrgbd_points_result) From fc4a4e9764044d693526c58645ce6a0372198da9 Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Wed, 7 Jul 2021 15:51:13 +0800 Subject: [PATCH 07/13] fix unittest --- mmdet3d/datasets/pipelines/transforms_3d.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mmdet3d/datasets/pipelines/transforms_3d.py b/mmdet3d/datasets/pipelines/transforms_3d.py index bd9b079017..034c6a1ca8 100644 --- a/mmdet3d/datasets/pipelines/transforms_3d.py +++ b/mmdet3d/datasets/pipelines/transforms_3d.py @@ -859,16 +859,16 @@ def _points_random_sampling(self, """ if replace is None: replace = (points.shape[0] < num_samples) - sample_range = range(len(points)) + point_range = range(len(points)) if sample_range: depth = points.coord[:, 2] far_inds = np.where(depth > sample_range)[0] near_inds = np.where(depth <= sample_range)[0] if not replace: # Only sampling the near points when len(points) >= num_samples - sample_range = near_inds + point_range = near_inds num_samples -= len(far_inds) - choices = np.random.choice(sample_range, num_samples, replace=replace) + choices = np.random.choice(point_range, num_samples, replace=replace) if sample_range and not replace: choices = np.concatenate((far_inds, choices)) # Shuffle points after sampling From 442621f4aa7efc1e0b6c34b6e588a74b051adbd9 Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Wed, 21 Jul 2021 12:41:29 +0800 Subject: [PATCH 08/13] add unitest --- .../test_augmentations/test_transforms_3d.py | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/test_data/test_pipelines/test_augmentations/test_transforms_3d.py b/tests/test_data/test_pipelines/test_augmentations/test_transforms_3d.py index 1de4437376..f5d32d3681 100644 --- a/tests/test_data/test_pipelines/test_augmentations/test_transforms_3d.py +++ b/tests/test_data/test_pipelines/test_augmentations/test_transforms_3d.py @@ -5,11 +5,12 @@ from mmdet3d.core import (Box3DMode, CameraInstance3DBoxes, DepthInstance3DBoxes, LiDARInstance3DBoxes) +from mmdet3d.core.bbox import Coord3DMode from mmdet3d.core.points import DepthPoints, LiDARPoints from mmdet3d.datasets import (BackgroundPointsFilter, GlobalAlignment, GlobalRotScaleTrans, ObjectNameFilter, ObjectNoise, ObjectRangeFilter, ObjectSample, - PointShuffle, PointsRangeFilter, + PointSample, PointShuffle, PointsRangeFilter, RandomDropPointsColor, RandomFlip3D, RandomJitterPoints, VoxelBasedPointSampler) @@ -702,3 +703,36 @@ def test_voxel_based_point_filter(): assert pts_instance_mask.min() >= 0 assert pts_semantic_mask.max() < 6 assert pts_semantic_mask.min() >= 0 + + +def test_points_sample(): + np.random.seed(0) + points = np.fromfile( + './tests/data/kitti/training/velodyne_reduced/000000.bin', + np.float32).reshape(-1, 4) + annos = mmcv.load('./tests/data/kitti/kitti_infos_train.pkl') + info = annos[0] + rect = torch.tensor(info['calib']['R0_rect'].astype(np.float32)) + Trv2c = torch.tensor(info['calib']['Tr_velo_to_cam'].astype(np.float32)) + + points = LiDARPoints( + points.copy(), points_dim=4).convert_to(Coord3DMode.CAM, rect @ Trv2c) + num_points = 20 + sample_range = 40 + input_dict = dict(points=points) + + point_sample = PointSample( + num_points=num_points, sample_range=sample_range) + sampled_pts = point_sample(input_dict)['points'] + + select_idx = np.array([ + 622, 146, 231, 444, 504, 533, 80, 401, 379, 2, 707, 562, 176, 491, 496, + 464, 15, 590, 194, 449 + ]) + expected_pts = points.tensor.numpy()[select_idx] + assert np.allclose(sampled_pts.tensor.numpy(), expected_pts) + + repr_str = repr(point_sample) + expected_repr_str = f'PointSample(num_points={num_points},'\ + ' sample_range={sample_range})' + assert repr_str == expected_repr_str From 1bd3d3a4a8d660530a0e9d5cec2624c58fee7e55 Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Wed, 21 Jul 2021 14:15:33 +0800 Subject: [PATCH 09/13] refine unittest --- .../test_pipelines/test_augmentations/test_transforms_3d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_data/test_pipelines/test_augmentations/test_transforms_3d.py b/tests/test_data/test_pipelines/test_augmentations/test_transforms_3d.py index f5d32d3681..f4c52eb6df 100644 --- a/tests/test_data/test_pipelines/test_augmentations/test_transforms_3d.py +++ b/tests/test_data/test_pipelines/test_augmentations/test_transforms_3d.py @@ -734,5 +734,5 @@ def test_points_sample(): repr_str = repr(point_sample) expected_repr_str = f'PointSample(num_points={num_points},'\ - ' sample_range={sample_range})' + + f' sample_range={sample_range})' assert repr_str == expected_repr_str From a55287e92cd9dbbf171d3f74fe7fe2cc231412bb Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Wed, 21 Jul 2021 21:23:37 +0800 Subject: [PATCH 10/13] refine code --- mmdet3d/datasets/pipelines/transforms_3d.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mmdet3d/datasets/pipelines/transforms_3d.py b/mmdet3d/datasets/pipelines/transforms_3d.py index 034c6a1ca8..e3bd22c897 100644 --- a/mmdet3d/datasets/pipelines/transforms_3d.py +++ b/mmdet3d/datasets/pipelines/transforms_3d.py @@ -860,16 +860,15 @@ def _points_random_sampling(self, if replace is None: replace = (points.shape[0] < num_samples) point_range = range(len(points)) - if sample_range: + if sample_range is not None and not replace: + # Only sampling the near points when len(points) >= num_samples depth = points.coord[:, 2] far_inds = np.where(depth > sample_range)[0] near_inds = np.where(depth <= sample_range)[0] - if not replace: - # Only sampling the near points when len(points) >= num_samples - point_range = near_inds - num_samples -= len(far_inds) + point_range = near_inds + num_samples -= len(far_inds) choices = np.random.choice(point_range, num_samples, replace=replace) - if sample_range and not replace: + if sample_range is not None and not replace: choices = np.concatenate((far_inds, choices)) # Shuffle points after sampling np.random.shuffle(choices) From fa6c1a0a5eae5330860dc00b205f02ff499ee0c2 Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Thu, 22 Jul 2021 11:07:19 +0800 Subject: [PATCH 11/13] refine code --- mmdet3d/datasets/pipelines/transforms_3d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmdet3d/datasets/pipelines/transforms_3d.py b/mmdet3d/datasets/pipelines/transforms_3d.py index e3bd22c897..efea04472e 100644 --- a/mmdet3d/datasets/pipelines/transforms_3d.py +++ b/mmdet3d/datasets/pipelines/transforms_3d.py @@ -890,7 +890,7 @@ def __call__(self, results): points = results['points'] # Points in Camera coord can provide the depth information. # TODO: Need to suport distance-based sampling for other coord system. - if self.sample_range: + if self.sample_range is not None: assert isinstance(points, CameraPoints), \ 'Sampling based on distance is only appliable for CAMERA coord' points, choices = self._points_random_sampling( From 0fde94e2a582bc2b1fd4e02935f58b1a9c34a164 Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Fri, 23 Jul 2021 20:21:36 +0800 Subject: [PATCH 12/13] refine depth calculation --- mmdet3d/datasets/pipelines/transforms_3d.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mmdet3d/datasets/pipelines/transforms_3d.py b/mmdet3d/datasets/pipelines/transforms_3d.py index efea04472e..a0ede950fc 100644 --- a/mmdet3d/datasets/pipelines/transforms_3d.py +++ b/mmdet3d/datasets/pipelines/transforms_3d.py @@ -825,7 +825,7 @@ class PointSample(object): Args: num_points (int): Number of points to be sampled. - sample_range (int, optional): The range where to sample points. + sample_range (float, optional): The range where to sample points. """ def __init__(self, num_points, sample_range=None): @@ -845,8 +845,8 @@ def _points_random_sampling(self, Args: points (np.ndarray | :obj:`BasePoints`): 3D Points. num_samples (int): Number of samples to be sampled. - sample_range (int, optional): Indicating the range where the points - will be sampled. + sample_range (float, optional): Indicating the range where the + points will be sampled. Defaults to None. replace (bool, optional): Sampling with or without replacement. Defaults to None. @@ -862,7 +862,7 @@ def _points_random_sampling(self, point_range = range(len(points)) if sample_range is not None and not replace: # Only sampling the near points when len(points) >= num_samples - depth = points.coord[:, 2] + depth = np.linalg.norm(points.tensor, axis=1) far_inds = np.where(depth > sample_range)[0] near_inds = np.where(depth <= sample_range)[0] point_range = near_inds From d1e7cd064551745a2ba03355fe10e4271f2d2300 Mon Sep 17 00:00:00 2001 From: wHao-Wu Date: Mon, 2 Aug 2021 17:47:29 +0800 Subject: [PATCH 13/13] refine code --- mmdet3d/datasets/pipelines/transforms_3d.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mmdet3d/datasets/pipelines/transforms_3d.py b/mmdet3d/datasets/pipelines/transforms_3d.py index a0ede950fc..c7dd9fbc56 100644 --- a/mmdet3d/datasets/pipelines/transforms_3d.py +++ b/mmdet3d/datasets/pipelines/transforms_3d.py @@ -828,15 +828,16 @@ class PointSample(object): sample_range (float, optional): The range where to sample points. """ - def __init__(self, num_points, sample_range=None): + def __init__(self, num_points, sample_range=None, replace=False): self.num_points = num_points self.sample_range = sample_range + self.replace = replace def _points_random_sampling(self, points, num_samples, sample_range=None, - replace=None, + replace=False, return_choices=False): """Points random sampling. @@ -857,7 +858,7 @@ def _points_random_sampling(self, - points (np.ndarray | :obj:`BasePoints`): 3D Points. - choices (np.ndarray, optional): The generated random samples. """ - if replace is None: + if not replace: replace = (points.shape[0] < num_samples) point_range = range(len(points)) if sample_range is not None and not replace: @@ -894,7 +895,11 @@ def __call__(self, results): assert isinstance(points, CameraPoints), \ 'Sampling based on distance is only appliable for CAMERA coord' points, choices = self._points_random_sampling( - points, self.num_points, self.sample_range, return_choices=True) + points, + self.num_points, + self.sample_range, + self.replace, + return_choices=True) results['points'] = points pts_instance_mask = results.get('pts_instance_mask', None)