[Enhancement] Add Dev tools to boost develop (#798)

* Modify default work dir when training.

* Refactor gather_models.py.

* Add train and test matching list.

* Regression benchmark list.

* lower readme name to upper readme name.

* Add url check tool and model inference test tool.

* Modify tool name.

* Support duplicate mode of log json url check.

* Add regression benchmark evaluation automatic tool.

* Add train script generator.

* Only Support script running.

* Add evaluation results gather.

* Add exec Authority.

* Automatically make checkpoint root folder.

* Modify gather results save path.

* Coarse-grained train results gather tool.

* Complete benchmark train script.

* Make some little modifications.

* Fix checkpoint urls.

* Fix unet checkpoint urls.

* Fix fast scnn & fcn checkpoint url.

* Fix fast scnn checkpoint urls.

* Fix fast scnn url.

* Add differential results calculation.

* Add differential results of regression benchmark train results.

* Add an extra argument to select model.

* Update nonlocal_net & hrnet checkpoint url.

* Fix checkpoint url of hrnet and Fix some tta evaluation results and modify gather models tool.

* Modify fast scnn checkpoint url.

* Resolve new comments.

* Fix url check status code bug.

* Resolve some comments.

* Modify train scripts generator.

* Modify work_dir of regression benchmark results.

* model gather tool modification.

.dev/batch_test_list.py

@@ -0,0 +1,133 @@
+# yapf: disable
+# Inference Speed is tested on NVIDIA V100
+hrnet = [
+    dict(
+        config='configs/hrnet/fcn_hr18s_512x512_160k_ade20k.py',
+        checkpoint='fcn_hr18s_512x512_160k_ade20k_20200614_214413-870f65ac.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=33.0),
+    ),
+    dict(
+        config='configs/hrnet/fcn_hr18s_512x1024_160k_cityscapes.py',
+        checkpoint='fcn_hr18s_512x1024_160k_cityscapes_20200602_190901-4a0797ea.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=76.31),
+    ),
+    dict(
+        config='configs/hrnet/fcn_hr48_512x512_160k_ade20k.py',
+        checkpoint='fcn_hr48_512x512_160k_ade20k_20200614_214407-a52fc02c.pth',
+        eval='mIoU',
+        metric=dict(mIoU=42.02),
+    ),
+    dict(
+        config='configs/hrnet/fcn_hr48_512x1024_160k_cityscapes.py',
+        checkpoint='fcn_hr48_512x1024_160k_cityscapes_20200602_190946-59b7973e.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=80.65),
+    ),
+]
+pspnet = [
+    dict(
+        config='configs/pspnet/pspnet_r50-d8_512x1024_80k_cityscapes.py',
+        checkpoint='pspnet_r50-d8_512x1024_80k_cityscapes_20200606_112131-2376f12b.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=78.55),
+    ),
+    dict(
+        config='configs/pspnet/pspnet_r101-d8_512x1024_80k_cityscapes.py',
+        checkpoint='pspnet_r101-d8_512x1024_80k_cityscapes_20200606_112211-e1e1100f.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=79.76),
+    ),
+    dict(
+        config='configs/pspnet/pspnet_r101-d8_512x512_160k_ade20k.py',
+        checkpoint='pspnet_r101-d8_512x512_160k_ade20k_20200615_100650-967c316f.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=44.39),
+    ),
+    dict(
+        config='configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py',
+        checkpoint='pspnet_r50-d8_512x512_160k_ade20k_20200615_184358-1890b0bd.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=42.48),
+    ),
+]
+resnest = [
+    dict(
+        config='configs/resnest/pspnet_s101-d8_512x512_160k_ade20k.py',
+        checkpoint='pspnet_s101-d8_512x512_160k_ade20k_20200807_145416-a6daa92a.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=45.44),
+    ),
+    dict(
+        config='configs/resnest/pspnet_s101-d8_512x1024_80k_cityscapes.py',
+        checkpoint='pspnet_s101-d8_512x1024_80k_cityscapes_20200807_140631-c75f3b99.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=78.57),
+    ),
+]
+fastscnn = [
+    dict(
+        config='configs/fastscnn/fast_scnn_lr0.12_8x4_160k_cityscapes.py',
+        checkpoint='fast_scnn_8x4_160k_lr0.12_cityscapes-0cec9937.pth',
+        eval='mIoU',
+        metric=dict(mIoU=70.96),
+    )
+]
+deeplabv3plus = [
+    dict(
+        config='configs/deeplabv3plus/deeplabv3plus_r101-d8_769x769_80k_cityscapes.py', # noqa
+        checkpoint='deeplabv3plus_r101-d8_769x769_80k_cityscapes_20200607_000405-a7573d20.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=80.98),
+    ),
+    dict(
+        config='configs/deeplabv3plus/deeplabv3plus_r101-d8_512x1024_80k_cityscapes.py', # noqa
+        checkpoint='deeplabv3plus_r101-d8_512x1024_80k_cityscapes_20200606_114143-068fcfe9.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=80.97),
+    ),
+    dict(
+        config='configs/deeplabv3plus/deeplabv3plus_r50-d8_512x1024_80k_cityscapes.py', # noqa
+        checkpoint='deeplabv3plus_r50-d8_512x1024_80k_cityscapes_20200606_114049-f9fb496d.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=80.09),
+    ),
+    dict(
+        config='configs/deeplabv3plus/deeplabv3plus_r50-d8_769x769_80k_cityscapes.py', # noqa
+        checkpoint='deeplabv3plus_r50-d8_769x769_80k_cityscapes_20200606_210233-0e9dfdc4.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=79.83),
+    ),
+]
+vit = [
+    dict(
+        config='configs/vit/upernet_vit-b16_ln_mln_512x512_160k_ade20k.py',
+        checkpoint='upernet_vit-b16_ln_mln_512x512_160k_ade20k-f444c077.pth',
+        eval='mIoU',
+        metric=dict(mIoU=47.73),
+    ),
+    dict(
+        config='configs/vit/upernet_deit-s16_ln_mln_512x512_160k_ade20k.py',
+        checkpoint='upernet_deit-s16_ln_mln_512x512_160k_ade20k-c0cd652f.pth',
+        eval='mIoU',
+        metric=dict(mIoU=43.52),
+    ),
+]
+fp16 = [
+    dict(
+        config='configs/fp16/deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes.py', # noqa
+        checkpoint='deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes-cc58bc8d.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=80.46),
+    )
+]
+swin = [
+    dict(
+        config='configs/swin/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K.py', # noqa
+        checkpoint='upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210531_112542-e380ad3e.pth', # noqa
+        eval='mIoU',
+        metric=dict(mIoU=44.41),
+    )
+]
+# yapf: enable

.dev/batch_train_list.txt

@@ -0,0 +1,19 @@
+configs/hrnet/fcn_hr18s_512x512_160k_ade20k.py
+configs/hrnet/fcn_hr18s_512x1024_160k_cityscapes.py
+configs/hrnet/fcn_hr48_512x512_160k_ade20k.py
+configs/hrnet/fcn_hr48_512x1024_160k_cityscapes.py
+configs/pspnet/pspnet_r50-d8_512x1024_80k_cityscapes.py
+configs/pspnet/pspnet_r101-d8_512x1024_80k_cityscapes.py
+configs/pspnet/pspnet_r101-d8_512x512_160k_ade20k.py
+configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py
+configs/resnest/pspnet_s101-d8_512x512_160k_ade20k.py
+configs/resnest/pspnet_s101-d8_512x1024_80k_cityscapes.py
+configs/fastscnn/fast_scnn_lr0.12_8x4_160k_cityscapes.py
+configs/deeplabv3plus/deeplabv3plus_r101-d8_769x769_80k_cityscapes.py
+configs/deeplabv3plus/deeplabv3plus_r101-d8_512x1024_80k_cityscapes.py
+configs/deeplabv3plus/deeplabv3plus_r50-d8_512x1024_80k_cityscapes.py
+configs/deeplabv3plus/deeplabv3plus_r50-d8_769x769_80k_cityscapes.py
+configs/vit/upernet_vit-b16_ln_mln_512x512_160k_ade20k.py
+configs/vit/upernet_deit-s16_ln_mln_512x512_160k_ade20k.py
+configs/fp16/deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes.py
+configs/swin/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K.py

.dev/benchmark_evaluation.sh

@@ -0,0 +1,41 @@
+PARTITION=$1
+CHECKPOINT_DIR=$2
+
+echo 'configs/hrnet/fcn_hr18s_512x512_160k_ade20k.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION fcn_hr18s_512x512_160k_ade20k configs/hrnet/fcn_hr18s_512x512_160k_ade20k.py $CHECKPOINT_DIR/fcn_hr18s_512x512_160k_ade20k_20200614_214413-870f65ac.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/fcn_hr18s_512x512_160k_ade20k --options dist_params.port=28171 &
+echo 'configs/hrnet/fcn_hr18s_512x1024_160k_cityscapes.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION fcn_hr18s_512x1024_160k_cityscapes configs/hrnet/fcn_hr18s_512x1024_160k_cityscapes.py $CHECKPOINT_DIR/fcn_hr18s_512x1024_160k_cityscapes_20200602_190901-4a0797ea.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/fcn_hr18s_512x1024_160k_cityscapes --options dist_params.port=28172 &
+echo 'configs/hrnet/fcn_hr48_512x512_160k_ade20k.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION fcn_hr48_512x512_160k_ade20k configs/hrnet/fcn_hr48_512x512_160k_ade20k.py $CHECKPOINT_DIR/fcn_hr48_512x512_160k_ade20k_20200614_214407-a52fc02c.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/fcn_hr48_512x512_160k_ade20k --options dist_params.port=28173 &
+echo 'configs/hrnet/fcn_hr48_512x1024_160k_cityscapes.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION fcn_hr48_512x1024_160k_cityscapes configs/hrnet/fcn_hr48_512x1024_160k_cityscapes.py $CHECKPOINT_DIR/fcn_hr48_512x1024_160k_cityscapes_20200602_190946-59b7973e.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/fcn_hr48_512x1024_160k_cityscapes --options dist_params.port=28174 &
+echo 'configs/pspnet/pspnet_r50-d8_512x1024_80k_cityscapes.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION pspnet_r50-d8_512x1024_80k_cityscapes configs/pspnet/pspnet_r50-d8_512x1024_80k_cityscapes.py $CHECKPOINT_DIR/pspnet_r50-d8_512x1024_80k_cityscapes_20200606_112131-2376f12b.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/pspnet_r50-d8_512x1024_80k_cityscapes --options dist_params.port=28175 &
+echo 'configs/pspnet/pspnet_r101-d8_512x1024_80k_cityscapes.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION pspnet_r101-d8_512x1024_80k_cityscapes configs/pspnet/pspnet_r101-d8_512x1024_80k_cityscapes.py $CHECKPOINT_DIR/pspnet_r101-d8_512x1024_80k_cityscapes_20200606_112211-e1e1100f.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/pspnet_r101-d8_512x1024_80k_cityscapes --options dist_params.port=28176 &
+echo 'configs/pspnet/pspnet_r101-d8_512x512_160k_ade20k.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION pspnet_r101-d8_512x512_160k_ade20k configs/pspnet/pspnet_r101-d8_512x512_160k_ade20k.py $CHECKPOINT_DIR/pspnet_r101-d8_512x512_160k_ade20k_20200615_100650-967c316f.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/pspnet_r101-d8_512x512_160k_ade20k --options dist_params.port=28177 &
+echo 'configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION pspnet_r50-d8_512x512_160k_ade20k configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py $CHECKPOINT_DIR/pspnet_r50-d8_512x512_160k_ade20k_20200615_184358-1890b0bd.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/pspnet_r50-d8_512x512_160k_ade20k --options dist_params.port=28178 &
+echo 'configs/resnest/pspnet_s101-d8_512x512_160k_ade20k.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION pspnet_s101-d8_512x512_160k_ade20k configs/resnest/pspnet_s101-d8_512x512_160k_ade20k.py $CHECKPOINT_DIR/pspnet_s101-d8_512x512_160k_ade20k_20200807_145416-a6daa92a.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/pspnet_s101-d8_512x512_160k_ade20k --options dist_params.port=28179 &
+echo 'configs/resnest/pspnet_s101-d8_512x1024_80k_cityscapes.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION pspnet_s101-d8_512x1024_80k_cityscapes configs/resnest/pspnet_s101-d8_512x1024_80k_cityscapes.py $CHECKPOINT_DIR/pspnet_s101-d8_512x1024_80k_cityscapes_20200807_140631-c75f3b99.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/pspnet_s101-d8_512x1024_80k_cityscapes --options dist_params.port=28180 &
+echo 'configs/fastscnn/fast_scnn_lr0.12_8x4_160k_cityscapes.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION fast_scnn_lr0.12_8x4_160k_cityscapes configs/fastscnn/fast_scnn_lr0.12_8x4_160k_cityscapes.py $CHECKPOINT_DIR/fast_scnn_8x4_160k_lr0.12_cityscapes-0cec9937.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/fast_scnn_lr0.12_8x4_160k_cityscapes --options dist_params.port=28181 &
+echo 'configs/deeplabv3plus/deeplabv3plus_r101-d8_769x769_80k_cityscapes.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION deeplabv3plus_r101-d8_769x769_80k_cityscapes configs/deeplabv3plus/deeplabv3plus_r101-d8_769x769_80k_cityscapes.py $CHECKPOINT_DIR/deeplabv3plus_r101-d8_769x769_80k_cityscapes_20200607_000405-a7573d20.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/deeplabv3plus_r101-d8_769x769_80k_cityscapes --options dist_params.port=28182 &
+echo 'configs/deeplabv3plus/deeplabv3plus_r101-d8_512x1024_80k_cityscapes.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION deeplabv3plus_r101-d8_512x1024_80k_cityscapes configs/deeplabv3plus/deeplabv3plus_r101-d8_512x1024_80k_cityscapes.py $CHECKPOINT_DIR/deeplabv3plus_r101-d8_512x1024_80k_cityscapes_20200606_114143-068fcfe9.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/deeplabv3plus_r101-d8_512x1024_80k_cityscapes --options dist_params.port=28183 &
+echo 'configs/deeplabv3plus/deeplabv3plus_r50-d8_512x1024_80k_cityscapes.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION deeplabv3plus_r50-d8_512x1024_80k_cityscapes configs/deeplabv3plus/deeplabv3plus_r50-d8_512x1024_80k_cityscapes.py $CHECKPOINT_DIR/deeplabv3plus_r50-d8_512x1024_80k_cityscapes_20200606_114049-f9fb496d.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/deeplabv3plus_r50-d8_512x1024_80k_cityscapes --options dist_params.port=28184 &
+echo 'configs/deeplabv3plus/deeplabv3plus_r50-d8_769x769_80k_cityscapes.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION deeplabv3plus_r50-d8_769x769_80k_cityscapes configs/deeplabv3plus/deeplabv3plus_r50-d8_769x769_80k_cityscapes.py $CHECKPOINT_DIR/deeplabv3plus_r50-d8_769x769_80k_cityscapes_20200606_210233-0e9dfdc4.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/deeplabv3plus_r50-d8_769x769_80k_cityscapes --options dist_params.port=28185 &
+echo 'configs/vit/upernet_vit-b16_ln_mln_512x512_160k_ade20k.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION upernet_vit-b16_ln_mln_512x512_160k_ade20k configs/vit/upernet_vit-b16_ln_mln_512x512_160k_ade20k.py $CHECKPOINT_DIR/upernet_vit-b16_ln_mln_512x512_160k_ade20k-f444c077.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/upernet_vit-b16_ln_mln_512x512_160k_ade20k --options dist_params.port=28186 &
+echo 'configs/vit/upernet_deit-s16_ln_mln_512x512_160k_ade20k.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION upernet_deit-s16_ln_mln_512x512_160k_ade20k configs/vit/upernet_deit-s16_ln_mln_512x512_160k_ade20k.py $CHECKPOINT_DIR/upernet_deit-s16_ln_mln_512x512_160k_ade20k-c0cd652f.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/upernet_deit-s16_ln_mln_512x512_160k_ade20k --options dist_params.port=28187 &
+echo 'configs/fp16/deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes configs/fp16/deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes.py $CHECKPOINT_DIR/deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes-cc58bc8d.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes --options dist_params.port=28188 &
+echo 'configs/swin/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K.py' &
+GPUS=4  GPUS_PER_NODE=4  CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K configs/swin/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K.py $CHECKPOINT_DIR/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210531_112542-e380ad3e.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K --options dist_params.port=28189 &