[Enhance] Benchmark PN++ on ScanNet and S3DIS

.dev_scripts/gather_models.py

@@ -21,7 +21,11 @@
  '_3x_': 36,
  '_4x_': 48,
  '_24e_': 24,
- '_6x_': 73
+ '_6x_': 73,
+ '_50e_': 50,
+ '_80e_': 80,
+ '_200e_': 200,
+ '_250e_': 250
 }
 
 # TODO: add support for lyft dataset
@@ -34,6 +38,8 @@
  ],
  'kitti-3d-car': ['KITTI/Car_3D_moderate_strict', 'Car_3D_moderate_strict'],
  'lyft': ['score'],
+ 'scannet_seg': ['miou'],
+ 's3dis_seg': ['miou'],
  'scannet': ['mAR_0.50'],
  'sunrgbd': ['mAR_0.50']
 }

README.md

@@ -90,6 +90,7 @@ Support methods
 - [x] [SSN (ECCV'2020)](configs/ssn/README.md)
 - [x] [ImVoteNet (CVPR'2020)](configs/imvotenet/README.md)
 - [x] [FCOS3D (Arxiv'2021)](configs/fcos3d/README.md)
+- [x] [PointNet++ (NeurIPS'2017)](configs/pointnet2/README.md)
 
 | | ResNet | ResNeXt | SENet |PointNet++ | HRNet | RegNetX | Res2Net |
 |--------------------|:--------:|:--------:|:--------:|:---------:|:-----:|:--------:|:-----:|
@@ -105,6 +106,7 @@ Support methods
 | SSN | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ |
 | ImVoteNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ |
 | FCOS3D | ✓ | ☐ | ☐ | ✗ | ☐ | ☐ | ☐ |
+| PointNet++ | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ |
 
 Other features
 - [x] [Dynamic Voxelization](configs/dynamic_voxelization/README.md)

README_zh-CN.md

@@ -89,6 +89,7 @@ MMDetection3D 是一个基于 PyTorch 的目标检测开源工具箱, 下一代
 - [x] [SSN (ECCV'2020)](configs/ssn/README.md)
 - [x] [ImVoteNet (CVPR'2020)](configs/imvotenet/README.md)
 - [x] [FCOS3D (Arxiv'2021)](configs/fcos3d/README.md)
+- [x] [PointNet++ (NeurIPS'2017)](configs/pointnet2/README.md)
 
 | | ResNet | ResNeXt | SENet |PointNet++ | HRNet | RegNetX | Res2Net |
 |--------------------|:--------:|:--------:|:--------:|:---------:|:-----:|:--------:|:-----:|
@@ -104,6 +105,7 @@ MMDetection3D 是一个基于 PyTorch 的目标检测开源工具箱, 下一代
 | SSN | ☐ | ☐ | ☐ | ✗ | ☐ | ✓ | ☐ |
 | ImVoteNet | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ |
 | FCOS3D | ✓ | ☐ | ☐ | ✗ | ☐ | ☐ | ☐ |
+| PointNet++ | ✗ | ✗ | ✗ | ✓ | ✗ | ✗ | ✗ |
 
 其他特性
 - [x] [Dynamic Voxelization](configs/dynamic_voxelization/README.md)

configs/_base_/schedules/seg_cosine_200e.py

@@ -0,0 +1,9 @@
+# optimizer
+# This schedule is mainly used on ScanNet dataset in segmentation task
+optimizer = dict(type='Adam', lr=0.001, weight_decay=0.01)
+optimizer_config = dict(grad_clip=None)
+lr_config = dict(policy='CosineAnnealing', warmup=None, min_lr=1e-5)
+momentum_config = None
+
+# runtime settings
+runner = dict(type='EpochBasedRunner', max_epochs=200)

configs/_base_/schedules/seg_cosine_50e.py

@@ -0,0 +1,9 @@
+# optimizer
+# This schedule is mainly used on S3DIS dataset in segmentation task
+optimizer = dict(type='Adam', lr=0.001, weight_decay=0.001)
+optimizer_config = dict(grad_clip=None)
+lr_config = dict(policy='CosineAnnealing', warmup=None, min_lr=1e-5)
+momentum_config = None
+
+# runtime settings
+runner = dict(type='EpochBasedRunner', max_epochs=50)

configs/pointnet2/README.md

@@ -0,0 +1,58 @@
+# PointNet++: Deep Hierarchical Feature Learning on Point Sets in a Metric Space
+
+## Introduction
+
+<!-- [ALGORITHM] -->
+
+We implement PointNet++ and provide the result and checkpoints on ScanNet and S3DIS datasets.
+
+```
+@inproceedings{qi2017pointnet++,
+ title={PointNet++ deep hierarchical feature learning on point sets in a metric space},
+ author={Qi, Charles R and Yi, Li and Su, Hao and Guibas, Leonidas J},
+ booktitle={Proceedings of the 31st International Conference on Neural Information Processing Systems},
+ pages={5105--5114},
+ year={2017}
+}
+```
+
+**Notice**: The original PointNet++ paper used step learning rate schedule. We discovered that cosine schedule achieves much better results and adopt it in our implementations. We also use a larger `weight_decay` factor because we find it consistently improves the performance.
+
+## Results
+
+### ScanNet
+
+| Method | Input | Lr schd | Mem (GB) | Inf time (fps) | mIoU (Val set) | mIoU (Test set) | Download |
+| :-------------------------------------------------------------------------------------: | :-------: | :---------: | :------: | :------------: | :------------: | :-------------: | ------------------------ |
+| [PointNet++ (SSG)](./pointnet2_ssg_xyz-only_16x2_cosine_200e_scannet_seg-3d-20class.py) | XYZ | cosine 200e | 1.9 | | 53.91 | | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/pointnet2/pointnet2_ssg_xyz-only_16x2_cosine_200e_scannet_seg-3d-20class/pointnet2_ssg_xyz-only_16x2_cosine_200e_scannet_seg-3d-20class_20210514_143628-4e341a48.pth) &#124; [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/pointnet2/pointnet2_ssg_xyz-only_16x2_cosine_200e_scannet_seg-3d-20class/pointnet2_ssg_xyz-only_16x2_cosine_200e_scannet_seg-3d-20class_20210514_143628.log.json) |
+| [PointNet++ (SSG)](./pointnet2_ssg_16x2_cosine_200e_scannet_seg-3d-20class.py) | XYZ+Color | cosine 200e | 1.9 | | 54.44 | | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/pointnet2/pointnet2_ssg_16x2_cosine_200e_scannet_seg-3d-20class/pointnet2_ssg_16x2_cosine_200e_scannet_seg-3d-20class_20210514_143644-ee73704a.pth) &#124; [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/pointnet2/pointnet2_ssg_16x2_cosine_200e_scannet_seg-3d-20class/pointnet2_ssg_16x2_cosine_200e_scannet_seg-3d-20class_20210514_143644.log.json) |
+| [PointNet++ (MSG)](./pointnet2_msg_xyz-only_16x2_cosine_250e_scannet_seg-3d-20class.py) | XYZ | cosine 250e | 2.4 | | 54.26 | | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/pointnet2/pointnet2_msg_xyz-only_16x2_cosine_250e_scannet_seg-3d-20class/pointnet2_msg_xyz-only_16x2_cosine_250e_scannet_seg-3d-20class_20210514_143838-b4a3cf89.pth) &#124; [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/pointnet2/pointnet2_msg_xyz-only_16x2_cosine_250e_scannet_seg-3d-20class/pointnet2_msg_xyz-only_16x2_cosine_250e_scannet_seg-3d-20class_20210514_143838.log.json) |
+| [PointNet++ (MSG)](./pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py) | XYZ+Color | cosine 250e | 2.4 | | 55.05 | | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class_20210514_144009-24477ab1.pth) &#124; [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class_20210514_144009.log.json) |
+
+**Notes:**
+
+- The original PointNet++ paper conducted experiments on the ScanNet V1 dataset, while later point cloud segmentor papers often used ScanNet V2. Following common practice, we report results on the ScanNet V2 dataset.
+- Since ScanNet dataset doesn't provide ground-truth labels for the test set, users can only evaluate test set performance by submitting to its online benchmark [website](http://kaldir.vc.in.tum.de/scannet_benchmark/). However, users are only allowed to submit once every two weeks. Therefore, we currently report val set mIoU. Test set performance may be added in the future.
+- To generate submission file for ScanNet online benchmark, you need to modify the ScanNet dataset's [config](https://github.com/open-mmlab/mmdetection3d/blob/master/configs/_base_/datasets/scannet_seg-3d-20class.py#L126). Change `ann_file=data_root + 'scannet_infos_val.pkl'` to `ann_file=data_root + 'scannet_infos_test.pkl'`, and then simply run:
+
+ ```shell
+ python tools/test.py ${CONFIG_FILE} ${CHECKPOINT_FILE} --format-only --options 'txt_prefix=exps/pointnet2_scannet_results'
+ ```
+
+ This will save the prediction results as `txt` files in `exps/pointnet2_scannet_results/`. Then, go to this folder and zip all files into `pn2_scannet.zip`. Now you can submit it to the online benchmark and wait for the test set result. More instructions can be found at their official [website](http://kaldir.vc.in.tum.de/scannet_benchmark/documentation#submission-policy).
+
+### S3DIS
+
+| Method | Split | Lr schd | Mem (GB) | Inf time (fps) | mIoU (Val set) | Download |
+| :-------------------------------------------------------------------------: | :----: | :--------: | :------: | :------------: | :------------: | :----------------------: |
+| [PointNet++ (SSG)](./pointnet2_ssg_16x2_cosine_50e_s3dis_seg-3d-13class.py) | Area_5 | cosine 50e | 3.6 | | 56.93 | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/pointnet2/pointnet2_ssg_16x2_cosine_50e_s3dis_seg-3d-13class/pointnet2_ssg_16x2_cosine_50e_s3dis_seg-3d-13class_20210514_144205-995d0119.pth) &#124; [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/pointnet2/pointnet2_ssg_16x2_cosine_50e_s3dis_seg-3d-13class/pointnet2_ssg_16x2_cosine_50e_s3dis_seg-3d-13class_20210514_144205.log.json) |
+| [PointNet++ (MSG)](./pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py) | Area_5 | cosine 80e | 3.6 | | 58.04 | [model](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class_20210514_144307-b2059817.pth) &#124; [log](https://download.openmmlab.com/mmdetection3d/v0.1.0_models/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class_20210514_144307.log.json) |
+
+**Notes:**
+
+- We use XYZ+Color+Normalized_XYZ as input in all the experiments on S3DIS datasets.
+- `Area_5` Split means training the model on Area_1, 2, 3, 4, 6 and testing on Area_5.
+
+## Indeterminism
+
+Since PointNet++ testing adopts sliding patch inference which involves random point sampling, and the test script uses fixed random seeds while the random seeds of validation in training are not fixed, the test results may be slightly different from the results reported above.

configs/pointnet2/pointnet2_msg_16x2_scannet-3d-20class.py → configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py

@@ -1,6 +1,7 @@
 _base_ = [
  '../_base_/datasets/scannet_seg-3d-20class.py',
- '../_base_/models/pointnet2_msg.py', '../_base_/default_runtime.py'
+ '../_base_/models/pointnet2_msg.py',
+ '../_base_/schedules/seg_cosine_200e.py', '../_base_/default_runtime.py'
 ]
 
 # data settings
@@ -29,12 +30,7 @@
  use_normalized_coord=False,
  batch_size=24))
 
-# optimizer
-lr = 0.001 # max learning rate
-optimizer = dict(type='Adam', lr=lr, weight_decay=1e-4)
-optimizer_config = dict(grad_clip=None)
-lr_config = dict(policy='CosineAnnealing', warmup=None, min_lr=1e-5)
-
 # runtime settings
 checkpoint_config = dict(interval=5)
-runner = dict(type='EpochBasedRunner', max_epochs=150)
+# PointNet2-MSG needs longer training time than PointNet2-SSG
+runner = dict(type='EpochBasedRunner', max_epochs=250)

configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py

@@ -0,0 +1,27 @@
+_base_ = [
+ '../_base_/datasets/s3dis_seg-3d-13class.py',
+ '../_base_/models/pointnet2_msg.py',
+ '../_base_/schedules/seg_cosine_50e.py', '../_base_/default_runtime.py'
+]
+
+# data settings
+data = dict(samples_per_gpu=16)
+evaluation = dict(interval=2)
+
+# model settings
+model = dict(
+ backbone=dict(in_channels=9), # [xyz, rgb, normalized_xyz]
+ decode_head=dict(
+ num_classes=13, ignore_index=13,
+ loss_decode=dict(class_weight=None)), # S3DIS doesn't use class_weight
+ test_cfg=dict(
+ num_points=4096,
+ block_size=1.0,
+ sample_rate=0.5,
+ use_normalized_coord=True,
+ batch_size=24))
+
+# runtime settings
+checkpoint_config = dict(interval=2)
+# PointNet2-MSG needs longer training time than PointNet2-SSG
+runner = dict(type='EpochBasedRunner', max_epochs=80)

configs/pointnet2/pointnet2_msg_xyz-only_16x2_cosine_250e_scannet_seg-3d-20class.py

@@ -0,0 +1,166 @@
+_base_ = [
+ '../_base_/datasets/scannet_seg-3d-20class.py',
+ '../_base_/models/pointnet2_msg.py',
+ '../_base_/schedules/seg_cosine_200e.py', '../_base_/default_runtime.py'
+]
+
+# dataset settings
+# in this setting, we only use xyz as network input
+# so we need to re-write all the data pipeline
+dataset_type = 'ScanNetSegDataset'
+data_root = './data/scannet/'
+class_names = ('wall', 'floor', 'cabinet', 'bed', 'chair', 'sofa', 'table',
+ 'door', 'window', 'bookshelf', 'picture', 'counter', 'desk',
+ 'curtain', 'refrigerator', 'showercurtrain', 'toilet', 'sink',
+ 'bathtub', 'otherfurniture')
+num_points = 8192
+train_pipeline = [
+ dict(
+ type='LoadPointsFromFile',
+ coord_type='DEPTH',
+ shift_height=False,
+ use_color=False,
+ load_dim=6,
+ use_dim=[0, 1, 2]), # only load xyz coordinates
+ dict(
+ type='LoadAnnotations3D',
+ with_bbox_3d=False,
+ with_label_3d=False,
+ with_mask_3d=False,
+ with_seg_3d=True),
+ dict(
+ type='PointSegClassMapping',
+ valid_cat_ids=(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28,
+ 33, 34, 36, 39),
+ max_cat_id=40),
+ dict(
+ type='IndoorPatchPointSample',
+ num_points=num_points,
+ block_size=1.5,
+ sample_rate=1.0,
+ ignore_index=len(class_names),
+ use_normalized_coord=False),
+ dict(type='DefaultFormatBundle3D', class_names=class_names),
+ dict(type='Collect3D', keys=['points', 'pts_semantic_mask'])
+]
+test_pipeline = [
+ dict(
+ type='LoadPointsFromFile',
+ coord_type='DEPTH',
+ shift_height=False,
+ use_color=False,
+ load_dim=6,
+ use_dim=[0, 1, 2]),
+ dict(
+ # a wrapper in order to successfully call test function
+ # actually we don't perform test-time-aug
+ type='MultiScaleFlipAug3D',
+ img_scale=(1333, 800),
+ pts_scale_ratio=1,
+ flip=False,
+ transforms=[
+ dict(
+ type='GlobalRotScaleTrans',
+ rot_range=[0, 0],
+ scale_ratio_range=[1., 1.],
+ translation_std=[0, 0, 0]),
+ dict(
+ type='RandomFlip3D',
+ sync_2d=False,
+ flip_ratio_bev_horizontal=0.0,
+ flip_ratio_bev_vertical=0.0),
+ dict(
+ type='DefaultFormatBundle3D',
+ class_names=class_names,
+ with_label=False),
+ dict(type='Collect3D', keys=['points'])
+ ])
+]
+# construct a pipeline for data and gt loading in show function
+# please keep its loading function consistent with test_pipeline (e.g. client)
+# we need to load gt seg_mask!
+eval_pipeline = [
+ dict(
+ type='LoadPointsFromFile',
+ coord_type='DEPTH',
+ shift_height=False,
+ use_color=False,
+ load_dim=6,
+ use_dim=[0, 1, 2]),
+ dict(
+ type='LoadAnnotations3D',
+ with_bbox_3d=False,
+ with_label_3d=False,
+ with_mask_3d=False,
+ with_seg_3d=True),
+ dict(
+ type='PointSegClassMapping',
+ valid_cat_ids=(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28,
+ 33, 34, 36, 39),
+ max_cat_id=40),
+ dict(
+ type='DefaultFormatBundle3D',
+ with_label=False,
+ class_names=class_names),
+ dict(type='Collect3D', keys=['points', 'pts_semantic_mask'])
+]
+
+data = dict(
+ samples_per_gpu=16,
+ workers_per_gpu=4,
+ train=dict(
+ type=dataset_type,
+ data_root=data_root,
+ ann_file=data_root + 'scannet_infos_train.pkl',
+ pipeline=train_pipeline,
+ classes=class_names,
+ test_mode=False,
+ ignore_index=len(class_names),
+ scene_idxs=data_root + 'seg_info/train_resampled_scene_idxs.npy',
+ label_weight=data_root + 'seg_info/train_label_weight.npy'),
+ val=dict(
+ type=dataset_type,
+ data_root=data_root,
+ ann_file=data_root + 'scannet_infos_val.pkl',
+ pipeline=test_pipeline,
+ classes=class_names,
+ test_mode=True,
+ ignore_index=len(class_names)),
+ test=dict(
+ type=dataset_type,
+ data_root=data_root,
+ ann_file=data_root + 'scannet_infos_val.pkl',
+ pipeline=test_pipeline,
+ classes=class_names,
+ test_mode=True,
+ ignore_index=len(class_names)))
+
+evaluation = dict(pipeline=eval_pipeline, interval=5)
+
+# model settings
+model = dict(
+ backbone=dict(in_channels=3), # only [xyz]
+ decode_head=dict(
+ num_classes=20,
+ ignore_index=20,
+ # `class_weight` is generated in data pre-processing, saved in
+ # `data/scannet/seg_info/train_label_weight.npy`
+ # you can copy paste the values here, or input the file path as
+ # `class_weight=data/scannet/seg_info/train_label_weight.npy`
+ loss_decode=dict(class_weight=[
+ 2.389689, 2.7215734, 4.5944676, 4.8543367, 4.096086, 4.907941,
+ 4.690836, 4.512031, 4.623311, 4.9242644, 5.358117, 5.360071,
+ 5.019636, 4.967126, 5.3502126, 5.4023647, 5.4027233, 5.4169416,
+ 5.3954206, 4.6971426
+ ])),
+ test_cfg=dict(
+ num_points=8192,
+ block_size=1.5,
+ sample_rate=0.5,
+ use_normalized_coord=False,
+ batch_size=24))
+
+# runtime settings
+checkpoint_config = dict(interval=5)
+# PointNet2-MSG needs longer training time than PointNet2-SSG
+runner = dict(type='EpochBasedRunner', max_epochs=250)

configs/pointnet2/pointnet2_ssg_16x2_scannet-3d-20class.py → configs/pointnet2/pointnet2_ssg_16x2_cosine_200e_scannet_seg-3d-20class.py

@@ -1,6 +1,7 @@
 _base_ = [
  '../_base_/datasets/scannet_seg-3d-20class.py',
- '../_base_/models/pointnet2_ssg.py', '../_base_/default_runtime.py'
+ '../_base_/models/pointnet2_ssg.py',
+ '../_base_/schedules/seg_cosine_200e.py', '../_base_/default_runtime.py'
 ]
 
 # data settings
@@ -29,12 +30,5 @@
  use_normalized_coord=False,
  batch_size=24))
 
-# optimizer
-lr = 0.001 # max learning rate
-optimizer = dict(type='Adam', lr=lr, weight_decay=1e-4)
-optimizer_config = dict(grad_clip=None)
-lr_config = dict(policy='CosineAnnealing', warmup=None, min_lr=1e-5)
-
 # runtime settings
 checkpoint_config = dict(interval=5)
-runner = dict(type='EpochBasedRunner', max_epochs=150)

configs/pointnet2/pointnet2_ssg_16x2_cosine_50e_s3dis_seg-3d-13class.py

@@ -0,0 +1,25 @@
+_base_ = [
+ '../_base_/datasets/s3dis_seg-3d-13class.py',
+ '../_base_/models/pointnet2_ssg.py',
+ '../_base_/schedules/seg_cosine_50e.py', '../_base_/default_runtime.py'
+]
+
+# data settings
+data = dict(samples_per_gpu=16)
+evaluation = dict(interval=2)
+
+# model settings
+model = dict(
+ backbone=dict(in_channels=9), # [xyz, rgb, normalized_xyz]
+ decode_head=dict(
+ num_classes=13, ignore_index=13,
+ loss_decode=dict(class_weight=None)), # S3DIS doesn't use class_weight
+ test_cfg=dict(
+ num_points=4096,
+ block_size=1.0,
+ sample_rate=0.5,
+ use_normalized_coord=True,
+ batch_size=24))
+
+# runtime settings
+checkpoint_config = dict(interval=2)