diff --git a/configs/apcnet/apcnet_r50-d8_512x512_160k_ade20k.py b/configs/apcnet/apcnet_r50-d8_512x512_160k_ade20k.py
index aa45e35d3..f7821c559 100644
--- a/configs/apcnet/apcnet_r50-d8_512x512_160k_ade20k.py
+++ b/configs/apcnet/apcnet_r50-d8_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/apcnet/apcnet_r50-d8_512x512_80k_ade20k.py b/configs/apcnet/apcnet_r50-d8_512x512_80k_ade20k.py
index 6b40d1f7a..daafa5fbc 100644
--- a/configs/apcnet/apcnet_r50-d8_512x512_80k_ade20k.py
+++ b/configs/apcnet/apcnet_r50-d8_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/deeplabv3/deeplabv3_r50-d8_512x512_160k_ade20k.py b/configs/deeplabv3/deeplabv3_r50-d8_512x512_160k_ade20k.py
index 742e17d74..b4a9d4e1b 100644
--- a/configs/deeplabv3/deeplabv3_r50-d8_512x512_160k_ade20k.py
+++ b/configs/deeplabv3/deeplabv3_r50-d8_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/deeplabv3/deeplabv3_r50-d8_512x512_80k_ade20k.py b/configs/deeplabv3/deeplabv3_r50-d8_512x512_80k_ade20k.py
index 5ddef212f..78f4d0d9d 100644
--- a/configs/deeplabv3/deeplabv3_r50-d8_512x512_80k_ade20k.py
+++ b/configs/deeplabv3/deeplabv3_r50-d8_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_160k_ade20k.py b/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_160k_ade20k.py
index e73488095..1491e3b82 100644
--- a/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_160k_ade20k.py
+++ b/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_80k_ade20k.py b/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_80k_ade20k.py
index 870597263..352d870bc 100644
--- a/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_80k_ade20k.py
+++ b/configs/deeplabv3plus/deeplabv3plus_r50-d8_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/dmnet/dmnet_r50-d8_512x512_160k_ade20k.py b/configs/dmnet/dmnet_r50-d8_512x512_160k_ade20k.py
index 69f4165c7..a8fbd9beb 100644
--- a/configs/dmnet/dmnet_r50-d8_512x512_160k_ade20k.py
+++ b/configs/dmnet/dmnet_r50-d8_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/dmnet/dmnet_r50-d8_512x512_80k_ade20k.py b/configs/dmnet/dmnet_r50-d8_512x512_80k_ade20k.py
index 513f58cbe..74f6d6a85 100644
--- a/configs/dmnet/dmnet_r50-d8_512x512_80k_ade20k.py
+++ b/configs/dmnet/dmnet_r50-d8_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/fcn/fcn_r50-d8_512x512_160k_ade20k.py b/configs/fcn/fcn_r50-d8_512x512_160k_ade20k.py
index db272d6b5..9ca7fd23c 100644
--- a/configs/fcn/fcn_r50-d8_512x512_160k_ade20k.py
+++ b/configs/fcn/fcn_r50-d8_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/fcn/fcn_r50-d8_512x512_80k_ade20k.py b/configs/fcn/fcn_r50-d8_512x512_80k_ade20k.py
index 64997c26f..ef194cb59 100644
--- a/configs/fcn/fcn_r50-d8_512x512_80k_ade20k.py
+++ b/configs/fcn/fcn_r50-d8_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/psanet/psanet_r50-d8_512x512_160k_ade20k.py b/configs/psanet/psanet_r50-d8_512x512_160k_ade20k.py
index d177d17e1..9c6364eb4 100644
--- a/configs/psanet/psanet_r50-d8_512x512_160k_ade20k.py
+++ b/configs/psanet/psanet_r50-d8_512x512_160k_ade20k.py
@@ -5,4 +5,3 @@ _base_ = [
 model = dict(
     decode_head=dict(mask_size=(66, 66), num_classes=150),
     auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/psanet/psanet_r50-d8_512x512_80k_ade20k.py b/configs/psanet/psanet_r50-d8_512x512_80k_ade20k.py
index 58a18a043..0141a6d09 100644
--- a/configs/psanet/psanet_r50-d8_512x512_80k_ade20k.py
+++ b/configs/psanet/psanet_r50-d8_512x512_80k_ade20k.py
@@ -5,4 +5,3 @@ _base_ = [
 model = dict(
     decode_head=dict(mask_size=(66, 66), num_classes=150),
     auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py b/configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py
index c34b66aaf..86584573a 100644
--- a/configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py
+++ b/configs/pspnet/pspnet_r50-d8_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/pspnet/pspnet_r50-d8_512x512_80k_ade20k.py b/configs/pspnet/pspnet_r50-d8_512x512_80k_ade20k.py
index 6922cc6d1..52efdf51d 100644
--- a/configs/pspnet/pspnet_r50-d8_512x512_80k_ade20k.py
+++ b/configs/pspnet/pspnet_r50-d8_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/upernet/upernet_r50_512x512_160k_ade20k.py b/configs/upernet/upernet_r50_512x512_160k_ade20k.py
index f259165fc..f5dd9aa4e 100644
--- a/configs/upernet/upernet_r50_512x512_160k_ade20k.py
+++ b/configs/upernet/upernet_r50_512x512_160k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/configs/upernet/upernet_r50_512x512_80k_ade20k.py b/configs/upernet/upernet_r50_512x512_80k_ade20k.py
index ce5d71f56..f561e309e 100644
--- a/configs/upernet/upernet_r50_512x512_80k_ade20k.py
+++ b/configs/upernet/upernet_r50_512x512_80k_ade20k.py
@@ -4,4 +4,3 @@ _base_ = [
 ]
 model = dict(
     decode_head=dict(num_classes=150), auxiliary_head=dict(num_classes=150))
-test_cfg = dict(mode='whole')
diff --git a/mmseg/core/evaluation/metrics.py b/mmseg/core/evaluation/metrics.py
index 86475a898..95b096e7a 100644
--- a/mmseg/core/evaluation/metrics.py
+++ b/mmseg/core/evaluation/metrics.py
@@ -34,15 +34,15 @@ def intersect_and_union(pred_label,
 
     if isinstance(label, str):
         label = mmcv.imread(label, flag='unchanged', backend='pillow')
-        # modify if custom classes
-        if label_map is not None:
-            for old_id, new_id in label_map.items():
-                label[label == old_id] = new_id
-        if reduce_zero_label:
-            # avoid using underflow conversion
-            label[label == 0] = 255
-            label = label - 1
-            label[label == 254] = 255
+    # modify if custom classes
+    if label_map is not None:
+        for old_id, new_id in label_map.items():
+            label[label == old_id] = new_id
+    if reduce_zero_label:
+        # avoid using underflow conversion
+        label[label == 0] = 255
+        label = label - 1
+        label[label == 254] = 255
 
     mask = (label != ignore_index)
     pred_label = pred_label[mask]