diff --git a/configs/apcnet/apcnet_r50-d8_512x512_160k_ade20k.py b/configs/apcnet/apcnet_r50-d8_512x512_160k_ade20k.py
index aa45e35d30..f7821c559d 100644
--- a/configs/apcnet/apcnet_r50-d8_512x512_160k_ade20k.py
+++ b/configs/apcnet/apcnet_r50-d8_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/apcnet/apcnet_r50-d8_512x512_80k_ade20k.py b/configs/apcnet/apcnet_r50-d8_512x512_80k_ade20k.py
index 6b40d1f7ad..daafa5fbc1 100644
--- a/configs/apcnet/apcnet_r50-d8_512x512_80k_ade20k.py
+++ b/configs/apcnet/apcnet_r50-d8_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/deeplabv3/deeplabv3_r50-d8_512x512_160k_ade20k.py b/configs/deeplabv3/deeplabv3_r50-d8_512x512_160k_ade20k.py
index 742e17d749..b4a9d4e1b9 100644
--- a/configs/deeplabv3/deeplabv3_r50-d8_512x512_160k_ade20k.py
+++ b/configs/deeplabv3/deeplabv3_r50-d8_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/deeplabv3/deeplabv3_r50-d8_512x512_80k_ade20k.py b/configs/deeplabv3/deeplabv3_r50-d8_512x512_80k_ade20k.py
index 5ddef212f7..78f4d0d9de 100644
--- a/configs/deeplabv3/deeplabv3_r50-d8_512x512_80k_ade20k.py
+++ b/configs/deeplabv3/deeplabv3_r50-d8_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_160k_ade20k.py b/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_160k_ade20k.py
index e734880956..1491e3b824 100644
--- a/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_160k_ade20k.py
+++ b/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_80k_ade20k.py b/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_80k_ade20k.py
index 8705972631..352d870bc8 100644
--- a/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_80k_ade20k.py
+++ b/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/dmnet/dmnet_r50-d8_512x512_160k_ade20k.py b/configs/dmnet/dmnet_r50-d8_512x512_160k_ade20k.py
index 69f4165c7c..a8fbd9beb1 100644
--- a/configs/dmnet/dmnet_r50-d8_512x512_160k_ade20k.py
+++ b/configs/dmnet/dmnet_r50-d8_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/dmnet/dmnet_r50-d8_512x512_80k_ade20k.py b/configs/dmnet/dmnet_r50-d8_512x512_80k_ade20k.py
index 513f58cbe4..74f6d6a85a 100644
--- a/configs/dmnet/dmnet_r50-d8_512x512_80k_ade20k.py
+++ b/configs/dmnet/dmnet_r50-d8_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/fcn/fcn_r50-d8_512x512_160k_ade20k.py b/configs/fcn/fcn_r50-d8_512x512_160k_ade20k.py
index db272d6b5b..9ca7fd23ce 100644
--- a/configs/fcn/fcn_r50-d8_512x512_160k_ade20k.py
+++ b/configs/fcn/fcn_r50-d8_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/fcn/fcn_r50-d8_512x512_80k_ade20k.py b/configs/fcn/fcn_r50-d8_512x512_80k_ade20k.py
index 64997c26f7..ef194cb594 100644
--- a/configs/fcn/fcn_r50-d8_512x512_80k_ade20k.py
+++ b/configs/fcn/fcn_r50-d8_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/psanet/psanet_r50-d8_512x512_160k_ade20k.py b/configs/psanet/psanet_r50-d8_512x512_160k_ade20k.py
index d177d17e17..9c6364eb43 100644
--- a/configs/psanet/psanet_r50-d8_512x512_160k_ade20k.py
+++ b/configs/psanet/psanet_r50-d8_512x512_160k_ade20k.py
@@ -5,4 +5,3 @@
 model = dict(
     decode_head=dict(mask_size=(66, 66), num_classes=150),
     auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/psanet/psanet_r50-d8_512x512_80k_ade20k.py b/configs/psanet/psanet_r50-d8_512x512_80k_ade20k.py
index 58a18a043a..0141a6d092 100644
--- a/configs/psanet/psanet_r50-d8_512x512_80k_ade20k.py
+++ b/configs/psanet/psanet_r50-d8_512x512_80k_ade20k.py
@@ -5,4 +5,3 @@
 model = dict(
     decode_head=dict(mask_size=(66, 66), num_classes=150),
     auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py b/configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py
index c34b66aaf8..86584573a3 100644
--- a/configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py
+++ b/configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/pspnet/pspnet_r50-d8_512x512_80k_ade20k.py b/configs/pspnet/pspnet_r50-d8_512x512_80k_ade20k.py
index 6922cc6d1f..52efdf51d7 100644
--- a/configs/pspnet/pspnet_r50-d8_512x512_80k_ade20k.py
+++ b/configs/pspnet/pspnet_r50-d8_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/upernet/upernet_r50_512x512_160k_ade20k.py b/configs/upernet/upernet_r50_512x512_160k_ade20k.py
index f259165fca..f5dd9aa4ed 100644
--- a/configs/upernet/upernet_r50_512x512_160k_ade20k.py
+++ b/configs/upernet/upernet_r50_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/upernet/upernet_r50_512x512_80k_ade20k.py b/configs/upernet/upernet_r50_512x512_80k_ade20k.py
index ce5d71f56d..f561e309e3 100644
--- a/configs/upernet/upernet_r50_512x512_80k_ade20k.py
+++ b/configs/upernet/upernet_r50_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/mmseg/core/evaluation/metrics.py b/mmseg/core/evaluation/metrics.py
index 86475a8983..95b096e7a3 100644
--- a/mmseg/core/evaluation/metrics.py
+++ b/mmseg/core/evaluation/metrics.py
@@ -34,15 +34,15 @@ def intersect_and_union(pred_label,
 
     if isinstance(label, str):
         label = mmcv.imread(label, flag='unchanged', backend='pillow')
-        # modify if custom classes
-        if label_map is not None:
-            for old_id, new_id in label_map.items():
-                label[label == old_id] = new_id
-        if reduce_zero_label:
-            # avoid using underflow conversion
-            label[label == 0] = 255
-            label = label - 1
-            label[label == 254] = 255
+    # modify if custom classes
+    if label_map is not None:
+        for old_id, new_id in label_map.items():
+            label[label == old_id] = new_id
+    if reduce_zero_label:
+        # avoid using underflow conversion
+        label[label == 0] = 255
+        label = label - 1
+        label[label == 254] = 255
 
     mask = (label != ignore_index)
     pred_label = pred_label[mask]