Skip to content

Commit d35fbbd

Browse files
clownrat6clownrat6
authored
[Enhancement] Add Dev tools to boost develop (#798)
* Modify default work dir when training. * Refactor gather_models.py. * Add train and test matching list. * Regression benchmark list. * lower readme name to upper readme name. * Add url check tool and model inference test tool. * Modify tool name. * Support duplicate mode of log json url check. * Add regression benchmark evaluation automatic tool. * Add train script generator. * Only Support script running. * Add evaluation results gather. * Add exec Authority. * Automatically make checkpoint root folder. * Modify gather results save path. * Coarse-grained train results gather tool. * Complete benchmark train script. * Make some little modifications. * Fix checkpoint urls. * Fix unet checkpoint urls. * Fix fast scnn & fcn checkpoint url. * Fix fast scnn checkpoint urls. * Fix fast scnn url. * Add differential results calculation. * Add differential results of regression benchmark train results. * Add an extra argument to select model. * Update nonlocal_net & hrnet checkpoint url. * Fix checkpoint url of hrnet and Fix some tta evaluation results and modify gather models tool. * Modify fast scnn checkpoint url. * Resolve new comments. * Fix url check status code bug. * Resolve some comments. * Modify train scripts generator. * Modify work_dir of regression benchmark results. * model gather tool modification.
1 parent 4981ff6 commit d35fbbd

30 files changed

+1051
-142
lines changed

‎.dev/batch_test_list.py

+133
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
# yapf: disable
2+
# Inference Speed is tested on NVIDIA V100
3+
hrnet = [
4+
dict(
5+
config='configs/hrnet/fcn_hr18s_512x512_160k_ade20k.py',
6+
checkpoint='fcn_hr18s_512x512_160k_ade20k_20200614_214413-870f65ac.pth', # noqa
7+
eval='mIoU',
8+
metric=dict(mIoU=33.0),
9+
),
10+
dict(
11+
config='configs/hrnet/fcn_hr18s_512x1024_160k_cityscapes.py',
12+
checkpoint='fcn_hr18s_512x1024_160k_cityscapes_20200602_190901-4a0797ea.pth', # noqa
13+
eval='mIoU',
14+
metric=dict(mIoU=76.31),
15+
),
16+
dict(
17+
config='configs/hrnet/fcn_hr48_512x512_160k_ade20k.py',
18+
checkpoint='fcn_hr48_512x512_160k_ade20k_20200614_214407-a52fc02c.pth',
19+
eval='mIoU',
20+
metric=dict(mIoU=42.02),
21+
),
22+
dict(
23+
config='configs/hrnet/fcn_hr48_512x1024_160k_cityscapes.py',
24+
checkpoint='fcn_hr48_512x1024_160k_cityscapes_20200602_190946-59b7973e.pth', # noqa
25+
eval='mIoU',
26+
metric=dict(mIoU=80.65),
27+
),
28+
]
29+
pspnet = [
30+
dict(
31+
config='configs/pspnet/pspnet_r50-d8_512x1024_80k_cityscapes.py',
32+
checkpoint='pspnet_r50-d8_512x1024_80k_cityscapes_20200606_112131-2376f12b.pth', # noqa
33+
eval='mIoU',
34+
metric=dict(mIoU=78.55),
35+
),
36+
dict(
37+
config='configs/pspnet/pspnet_r101-d8_512x1024_80k_cityscapes.py',
38+
checkpoint='pspnet_r101-d8_512x1024_80k_cityscapes_20200606_112211-e1e1100f.pth', # noqa
39+
eval='mIoU',
40+
metric=dict(mIoU=79.76),
41+
),
42+
dict(
43+
config='configs/pspnet/pspnet_r101-d8_512x512_160k_ade20k.py',
44+
checkpoint='pspnet_r101-d8_512x512_160k_ade20k_20200615_100650-967c316f.pth', # noqa
45+
eval='mIoU',
46+
metric=dict(mIoU=44.39),
47+
),
48+
dict(
49+
config='configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py',
50+
checkpoint='pspnet_r50-d8_512x512_160k_ade20k_20200615_184358-1890b0bd.pth', # noqa
51+
eval='mIoU',
52+
metric=dict(mIoU=42.48),
53+
),
54+
]
55+
resnest = [
56+
dict(
57+
config='configs/resnest/pspnet_s101-d8_512x512_160k_ade20k.py',
58+
checkpoint='pspnet_s101-d8_512x512_160k_ade20k_20200807_145416-a6daa92a.pth', # noqa
59+
eval='mIoU',
60+
metric=dict(mIoU=45.44),
61+
),
62+
dict(
63+
config='configs/resnest/pspnet_s101-d8_512x1024_80k_cityscapes.py',
64+
checkpoint='pspnet_s101-d8_512x1024_80k_cityscapes_20200807_140631-c75f3b99.pth', # noqa
65+
eval='mIoU',
66+
metric=dict(mIoU=78.57),
67+
),
68+
]
69+
fastscnn = [
70+
dict(
71+
config='configs/fastscnn/fast_scnn_lr0.12_8x4_160k_cityscapes.py',
72+
checkpoint='fast_scnn_8x4_160k_lr0.12_cityscapes-0cec9937.pth',
73+
eval='mIoU',
74+
metric=dict(mIoU=70.96),
75+
)
76+
]
77+
deeplabv3plus = [
78+
dict(
79+
config='configs/deeplabv3plus/deeplabv3plus_r101-d8_769x769_80k_cityscapes.py', # noqa
80+
checkpoint='deeplabv3plus_r101-d8_769x769_80k_cityscapes_20200607_000405-a7573d20.pth', # noqa
81+
eval='mIoU',
82+
metric=dict(mIoU=80.98),
83+
),
84+
dict(
85+
config='configs/deeplabv3plus/deeplabv3plus_r101-d8_512x1024_80k_cityscapes.py', # noqa
86+
checkpoint='deeplabv3plus_r101-d8_512x1024_80k_cityscapes_20200606_114143-068fcfe9.pth', # noqa
87+
eval='mIoU',
88+
metric=dict(mIoU=80.97),
89+
),
90+
dict(
91+
config='configs/deeplabv3plus/deeplabv3plus_r50-d8_512x1024_80k_cityscapes.py', # noqa
92+
checkpoint='deeplabv3plus_r50-d8_512x1024_80k_cityscapes_20200606_114049-f9fb496d.pth', # noqa
93+
eval='mIoU',
94+
metric=dict(mIoU=80.09),
95+
),
96+
dict(
97+
config='configs/deeplabv3plus/deeplabv3plus_r50-d8_769x769_80k_cityscapes.py', # noqa
98+
checkpoint='deeplabv3plus_r50-d8_769x769_80k_cityscapes_20200606_210233-0e9dfdc4.pth', # noqa
99+
eval='mIoU',
100+
metric=dict(mIoU=79.83),
101+
),
102+
]
103+
vit = [
104+
dict(
105+
config='configs/vit/upernet_vit-b16_ln_mln_512x512_160k_ade20k.py',
106+
checkpoint='upernet_vit-b16_ln_mln_512x512_160k_ade20k-f444c077.pth',
107+
eval='mIoU',
108+
metric=dict(mIoU=47.73),
109+
),
110+
dict(
111+
config='configs/vit/upernet_deit-s16_ln_mln_512x512_160k_ade20k.py',
112+
checkpoint='upernet_deit-s16_ln_mln_512x512_160k_ade20k-c0cd652f.pth',
113+
eval='mIoU',
114+
metric=dict(mIoU=43.52),
115+
),
116+
]
117+
fp16 = [
118+
dict(
119+
config='configs/fp16/deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes.py', # noqa
120+
checkpoint='deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes-cc58bc8d.pth', # noqa
121+
eval='mIoU',
122+
metric=dict(mIoU=80.46),
123+
)
124+
]
125+
swin = [
126+
dict(
127+
config='configs/swin/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K.py', # noqa
128+
checkpoint='upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210531_112542-e380ad3e.pth', # noqa
129+
eval='mIoU',
130+
metric=dict(mIoU=44.41),
131+
)
132+
]
133+
# yapf: enable

‎.dev/batch_train_list.txt

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
configs/hrnet/fcn_hr18s_512x512_160k_ade20k.py
2+
configs/hrnet/fcn_hr18s_512x1024_160k_cityscapes.py
3+
configs/hrnet/fcn_hr48_512x512_160k_ade20k.py
4+
configs/hrnet/fcn_hr48_512x1024_160k_cityscapes.py
5+
configs/pspnet/pspnet_r50-d8_512x1024_80k_cityscapes.py
6+
configs/pspnet/pspnet_r101-d8_512x1024_80k_cityscapes.py
7+
configs/pspnet/pspnet_r101-d8_512x512_160k_ade20k.py
8+
configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py
9+
configs/resnest/pspnet_s101-d8_512x512_160k_ade20k.py
10+
configs/resnest/pspnet_s101-d8_512x1024_80k_cityscapes.py
11+
configs/fastscnn/fast_scnn_lr0.12_8x4_160k_cityscapes.py
12+
configs/deeplabv3plus/deeplabv3plus_r101-d8_769x769_80k_cityscapes.py
13+
configs/deeplabv3plus/deeplabv3plus_r101-d8_512x1024_80k_cityscapes.py
14+
configs/deeplabv3plus/deeplabv3plus_r50-d8_512x1024_80k_cityscapes.py
15+
configs/deeplabv3plus/deeplabv3plus_r50-d8_769x769_80k_cityscapes.py
16+
configs/vit/upernet_vit-b16_ln_mln_512x512_160k_ade20k.py
17+
configs/vit/upernet_deit-s16_ln_mln_512x512_160k_ade20k.py
18+
configs/fp16/deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes.py
19+
configs/swin/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K.py

‎.dev/benchmark_evaluation.sh

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
PARTITION=$1
2+
CHECKPOINT_DIR=$2
3+
4+
echo 'configs/hrnet/fcn_hr18s_512x512_160k_ade20k.py' &
5+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION fcn_hr18s_512x512_160k_ade20k configs/hrnet/fcn_hr18s_512x512_160k_ade20k.py $CHECKPOINT_DIR/fcn_hr18s_512x512_160k_ade20k_20200614_214413-870f65ac.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/fcn_hr18s_512x512_160k_ade20k --options dist_params.port=28171 &
6+
echo 'configs/hrnet/fcn_hr18s_512x1024_160k_cityscapes.py' &
7+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION fcn_hr18s_512x1024_160k_cityscapes configs/hrnet/fcn_hr18s_512x1024_160k_cityscapes.py $CHECKPOINT_DIR/fcn_hr18s_512x1024_160k_cityscapes_20200602_190901-4a0797ea.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/fcn_hr18s_512x1024_160k_cityscapes --options dist_params.port=28172 &
8+
echo 'configs/hrnet/fcn_hr48_512x512_160k_ade20k.py' &
9+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION fcn_hr48_512x512_160k_ade20k configs/hrnet/fcn_hr48_512x512_160k_ade20k.py $CHECKPOINT_DIR/fcn_hr48_512x512_160k_ade20k_20200614_214407-a52fc02c.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/fcn_hr48_512x512_160k_ade20k --options dist_params.port=28173 &
10+
echo 'configs/hrnet/fcn_hr48_512x1024_160k_cityscapes.py' &
11+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION fcn_hr48_512x1024_160k_cityscapes configs/hrnet/fcn_hr48_512x1024_160k_cityscapes.py $CHECKPOINT_DIR/fcn_hr48_512x1024_160k_cityscapes_20200602_190946-59b7973e.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/fcn_hr48_512x1024_160k_cityscapes --options dist_params.port=28174 &
12+
echo 'configs/pspnet/pspnet_r50-d8_512x1024_80k_cityscapes.py' &
13+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION pspnet_r50-d8_512x1024_80k_cityscapes configs/pspnet/pspnet_r50-d8_512x1024_80k_cityscapes.py $CHECKPOINT_DIR/pspnet_r50-d8_512x1024_80k_cityscapes_20200606_112131-2376f12b.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/pspnet_r50-d8_512x1024_80k_cityscapes --options dist_params.port=28175 &
14+
echo 'configs/pspnet/pspnet_r101-d8_512x1024_80k_cityscapes.py' &
15+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION pspnet_r101-d8_512x1024_80k_cityscapes configs/pspnet/pspnet_r101-d8_512x1024_80k_cityscapes.py $CHECKPOINT_DIR/pspnet_r101-d8_512x1024_80k_cityscapes_20200606_112211-e1e1100f.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/pspnet_r101-d8_512x1024_80k_cityscapes --options dist_params.port=28176 &
16+
echo 'configs/pspnet/pspnet_r101-d8_512x512_160k_ade20k.py' &
17+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION pspnet_r101-d8_512x512_160k_ade20k configs/pspnet/pspnet_r101-d8_512x512_160k_ade20k.py $CHECKPOINT_DIR/pspnet_r101-d8_512x512_160k_ade20k_20200615_100650-967c316f.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/pspnet_r101-d8_512x512_160k_ade20k --options dist_params.port=28177 &
18+
echo 'configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py' &
19+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION pspnet_r50-d8_512x512_160k_ade20k configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py $CHECKPOINT_DIR/pspnet_r50-d8_512x512_160k_ade20k_20200615_184358-1890b0bd.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/pspnet_r50-d8_512x512_160k_ade20k --options dist_params.port=28178 &
20+
echo 'configs/resnest/pspnet_s101-d8_512x512_160k_ade20k.py' &
21+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION pspnet_s101-d8_512x512_160k_ade20k configs/resnest/pspnet_s101-d8_512x512_160k_ade20k.py $CHECKPOINT_DIR/pspnet_s101-d8_512x512_160k_ade20k_20200807_145416-a6daa92a.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/pspnet_s101-d8_512x512_160k_ade20k --options dist_params.port=28179 &
22+
echo 'configs/resnest/pspnet_s101-d8_512x1024_80k_cityscapes.py' &
23+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION pspnet_s101-d8_512x1024_80k_cityscapes configs/resnest/pspnet_s101-d8_512x1024_80k_cityscapes.py $CHECKPOINT_DIR/pspnet_s101-d8_512x1024_80k_cityscapes_20200807_140631-c75f3b99.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/pspnet_s101-d8_512x1024_80k_cityscapes --options dist_params.port=28180 &
24+
echo 'configs/fastscnn/fast_scnn_lr0.12_8x4_160k_cityscapes.py' &
25+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION fast_scnn_lr0.12_8x4_160k_cityscapes configs/fastscnn/fast_scnn_lr0.12_8x4_160k_cityscapes.py $CHECKPOINT_DIR/fast_scnn_8x4_160k_lr0.12_cityscapes-0cec9937.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/fast_scnn_lr0.12_8x4_160k_cityscapes --options dist_params.port=28181 &
26+
echo 'configs/deeplabv3plus/deeplabv3plus_r101-d8_769x769_80k_cityscapes.py' &
27+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION deeplabv3plus_r101-d8_769x769_80k_cityscapes configs/deeplabv3plus/deeplabv3plus_r101-d8_769x769_80k_cityscapes.py $CHECKPOINT_DIR/deeplabv3plus_r101-d8_769x769_80k_cityscapes_20200607_000405-a7573d20.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/deeplabv3plus_r101-d8_769x769_80k_cityscapes --options dist_params.port=28182 &
28+
echo 'configs/deeplabv3plus/deeplabv3plus_r101-d8_512x1024_80k_cityscapes.py' &
29+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION deeplabv3plus_r101-d8_512x1024_80k_cityscapes configs/deeplabv3plus/deeplabv3plus_r101-d8_512x1024_80k_cityscapes.py $CHECKPOINT_DIR/deeplabv3plus_r101-d8_512x1024_80k_cityscapes_20200606_114143-068fcfe9.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/deeplabv3plus_r101-d8_512x1024_80k_cityscapes --options dist_params.port=28183 &
30+
echo 'configs/deeplabv3plus/deeplabv3plus_r50-d8_512x1024_80k_cityscapes.py' &
31+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION deeplabv3plus_r50-d8_512x1024_80k_cityscapes configs/deeplabv3plus/deeplabv3plus_r50-d8_512x1024_80k_cityscapes.py $CHECKPOINT_DIR/deeplabv3plus_r50-d8_512x1024_80k_cityscapes_20200606_114049-f9fb496d.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/deeplabv3plus_r50-d8_512x1024_80k_cityscapes --options dist_params.port=28184 &
32+
echo 'configs/deeplabv3plus/deeplabv3plus_r50-d8_769x769_80k_cityscapes.py' &
33+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION deeplabv3plus_r50-d8_769x769_80k_cityscapes configs/deeplabv3plus/deeplabv3plus_r50-d8_769x769_80k_cityscapes.py $CHECKPOINT_DIR/deeplabv3plus_r50-d8_769x769_80k_cityscapes_20200606_210233-0e9dfdc4.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/deeplabv3plus_r50-d8_769x769_80k_cityscapes --options dist_params.port=28185 &
34+
echo 'configs/vit/upernet_vit-b16_ln_mln_512x512_160k_ade20k.py' &
35+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION upernet_vit-b16_ln_mln_512x512_160k_ade20k configs/vit/upernet_vit-b16_ln_mln_512x512_160k_ade20k.py $CHECKPOINT_DIR/upernet_vit-b16_ln_mln_512x512_160k_ade20k-f444c077.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/upernet_vit-b16_ln_mln_512x512_160k_ade20k --options dist_params.port=28186 &
36+
echo 'configs/vit/upernet_deit-s16_ln_mln_512x512_160k_ade20k.py' &
37+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION upernet_deit-s16_ln_mln_512x512_160k_ade20k configs/vit/upernet_deit-s16_ln_mln_512x512_160k_ade20k.py $CHECKPOINT_DIR/upernet_deit-s16_ln_mln_512x512_160k_ade20k-c0cd652f.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/upernet_deit-s16_ln_mln_512x512_160k_ade20k --options dist_params.port=28187 &
38+
echo 'configs/fp16/deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes.py' &
39+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes configs/fp16/deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes.py $CHECKPOINT_DIR/deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes-cc58bc8d.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/deeplabv3plus_r101-d8_512x1024_80k_fp16_cityscapes --options dist_params.port=28188 &
40+
echo 'configs/swin/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K.py' &
41+
GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=2 tools/slurm_test.sh $PARTITION upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K configs/swin/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K.py $CHECKPOINT_DIR/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K_20210531_112542-e380ad3e.pth --eval mIoU --work-dir work_dirs/benchmark_evaluation/upernet_swin_tiny_patch4_window7_512x512_160k_ade20k_pretrain_224x224_1K --options dist_params.port=28189 &

0 commit comments

Comments
 (0)