[Config] Add multiloop cfg (#1186)

configs/_base_/recog_datasets/academic_test.py

@@ -67,3 +67,102 @@ CUTE80 = dict(
     pipeline=None)
 
 test_list = [IIIT5K, SVT, IC13, IC15, SVTP, CUTE80]
+
+IIIT5K_val_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    persistent_workers=True,
+    drop_last=False,
+    sampler=dict(type='DefaultSampler', shuffle=False),
+    dataset=IIIT5K)
+
+SVT_val_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    persistent_workers=True,
+    drop_last=False,
+    sampler=dict(type='DefaultSampler', shuffle=False),
+    dataset=SVT)
+
+IC13_val_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    persistent_workers=True,
+    drop_last=False,
+    sampler=dict(type='DefaultSampler', shuffle=False),
+    dataset=IC13)
+
+IC15_val_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    persistent_workers=True,
+    drop_last=False,
+    sampler=dict(type='DefaultSampler', shuffle=False),
+    dataset=IC15)
+
+SVTP_val_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    persistent_workers=True,
+    drop_last=False,
+    sampler=dict(type='DefaultSampler', shuffle=False),
+    dataset=SVTP)
+
+CUTE80_val_dataloader = dict(
+    batch_size=1,
+    num_workers=4,
+    persistent_workers=True,
+    drop_last=False,
+    sampler=dict(type='DefaultSampler', shuffle=False),
+    dataset=CUTE80)
+
+val_dataloader = [
+    IIIT5K_val_dataloader, SVT_val_dataloader, IC13_val_dataloader,
+    IC15_val_dataloader, SVTP_val_dataloader, CUTE80_val_dataloader
+]
+
+test_dataloader = val_dataloader
+
+val_evaluator = [[
+    dict(
+        type='WordMetric',
+        mode=['exact', 'ignore_case', 'ignore_case_symbol'],
+        prefix='IIIT5K'),
+    dict(type='CharMetric', prefix='IIIT5K')
+],
+                 [
+                     dict(
+                         type='WordMetric',
+                         mode=['exact', 'ignore_case', 'ignore_case_symbol'],
+                         prefix='svt'),
+                     dict(type='CharMetric', prefix='svt')
+                 ],
+                 [
+                     dict(
+                         type='WordMetric',
+                         mode=['exact', 'ignore_case', 'ignore_case_symbol'],
+                         prefix='icdar_2013'),
+                     dict(type='CharMetric', prefix='icdar_2013')
+                 ],
+                 [
+                     dict(
+                         type='WordMetric',
+                         mode=['exact', 'ignore_case', 'ignore_case_symbol'],
+                         prefix='icdar_2015'),
+                     dict(type='CharMetric', prefix='icdar_2015')
+                 ],
+                 [
+                     dict(
+                         type='WordMetric',
+                         mode=['exact', 'ignore_case', 'ignore_case_symbol'],
+                         prefix='svtp'),
+                     dict(type='CharMetric', prefix='svtp')
+                 ],
+                 [
+                     dict(
+                         type='WordMetric',
+                         mode=['exact', 'ignore_case', 'ignore_case_symbol'],
+                         prefix='ct80'),
+                     dict(type='CharMetric', prefix='ct80')
+                 ]]
+test_evaluator = val_evaluator

configs/textrecog/abinet/base.py

@@ -7,7 +7,6 @@ _base_ = [
 
 # dataset settings
 train_list = {{_base_.train_list}}
-test_list = {{_base_.test_list}}
 file_client_args = dict(backend='disk')
 default_hooks = dict(logger=dict(type='LoggerHook', interval=100))
 
@@ -89,16 +88,13 @@ train_dataloader = dict(
     dataset=dict(
         type='ConcatDataset', datasets=train_list, pipeline=train_pipeline))
 
-val_dataloader = dict(
-    batch_size=192,
-    num_workers=16,
-    persistent_workers=True,
-    drop_last=False,
-    sampler=dict(type='DefaultSampler', shuffle=False),
-    dataset=dict(
-        type='ConcatDataset', datasets=test_list, pipeline=test_pipeline))
-test_dataloader = val_dataloader
-
-val_evaluator = dict(type='WordMetric', mode=['ignore_case_symbol'])
-test_evaluator = val_evaluator
 visualizer = dict(type='TextRecogLocalVisualizer', name='visualizer')
+
+test_cfg = dict(type='MultiTestLoop')
+val_cfg = dict(type='MultiValLoop')
+val_dataloader = _base_.val_dataloader
+test_dataloader = _base_.test_dataloader
+for dataloader in test_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
+for dataloader in val_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline

configs/textrecog/crnn/crnn_academic_dataset.py

@@ -9,7 +9,7 @@ _base_ = [
 
 # dataset settings
 train_list = {{_base_.train_list}}
-test_list = {{_base_.test_list}}
+
 file_client_args = dict(backend='disk')
 default_hooks = dict(logger=dict(type='LoggerHook', interval=50), )
 
@@ -50,21 +50,13 @@ train_dataloader = dict(
     dataset=dict(
         type='ConcatDataset', datasets=train_list, pipeline=train_pipeline))
 
-val_dataloader = dict(
-    batch_size=1,
-    num_workers=4,
-    persistent_workers=True,
-    drop_last=False,
-    sampler=dict(type='DefaultSampler', shuffle=False),
-    dataset=dict(
-        type='ConcatDataset', datasets=test_list, pipeline=test_pipeline))
-test_dataloader = val_dataloader
+test_cfg = dict(type='MultiTestLoop')
+val_cfg = dict(type='MultiValLoop')
+val_dataloader = _base_.val_dataloader
+test_dataloader = _base_.test_dataloader
+for dataloader in test_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
+for dataloader in val_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
 
-val_evaluator = [
-    dict(
-        type='WordMetric', mode=['exact', 'ignore_case',
-                                 'ignore_case_symbol']),
-    dict(type='CharMetric')
-]
-test_evaluator = val_evaluator
 visualizer = dict(type='TextRecogLocalVisualizer', name='visualizer')

configs/textrecog/master/master_r31_12e_ST_MJ_SA.py

@@ -8,7 +8,6 @@ _base_ = [
 
 # dataset settings
 train_list = {{_base_.train_list}}
-test_list = {{_base_.test_list}}
 file_client_args = dict(backend='disk')
 default_hooks = dict(logger=dict(type='LoggerHook', interval=50), )
 
@@ -50,21 +49,13 @@ train_dataloader = dict(
     dataset=dict(
         type='ConcatDataset', datasets=train_list, pipeline=train_pipeline))
 
-val_dataloader = dict(
-    batch_size=128,
-    num_workers=4,
-    persistent_workers=True,
-    drop_last=False,
-    sampler=dict(type='DefaultSampler', shuffle=False),
-    dataset=dict(
-        type='ConcatDataset', datasets=test_list, pipeline=train_pipeline))
-test_dataloader = val_dataloader
+test_cfg = dict(type='MultiTestLoop')
+val_cfg = dict(type='MultiValLoop')
+val_dataloader = _base_.val_dataloader
+test_dataloader = _base_.test_dataloader
+for dataloader in test_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
+for dataloader in val_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
 
-val_evaluator = [
-    dict(
-        type='WordMetric', mode=['exact', 'ignore_case',
-                                 'ignore_case_symbol']),
-    dict(type='CharMetric')
-]
-test_evaluator = val_evaluator
 visualizer = dict(type='TextRecogLocalVisualizer', name='visualizer')

configs/textrecog/nrtr/nrtr_modality_transform_academic.py

@@ -10,7 +10,6 @@ default_hooks = dict(logger=dict(type='LoggerHook', interval=50))
 
 # dataset settings
 train_list = {{_base_.train_list}}
-test_list = {{_base_.test_list}}
 file_client_args = dict(backend='disk')
 default_hooks = dict(logger=dict(type='LoggerHook', interval=100))
 
@@ -52,22 +51,13 @@ train_dataloader = dict(
     dataset=dict(
         type='ConcatDataset', datasets=train_list, pipeline=test_pipeline))
 
-val_dataloader = dict(
-    batch_size=128,
-    num_workers=4,
-    persistent_workers=True,
-    drop_last=False,
-    sampler=dict(type='DefaultSampler', shuffle=False),
-    dataset=dict(
-        type='ConcatDataset', datasets=test_list, pipeline=test_pipeline))
-test_dataloader = val_dataloader
+test_cfg = dict(type='MultiTestLoop')
+val_cfg = dict(type='MultiValLoop')
+val_dataloader = _base_.val_dataloader
+test_dataloader = _base_.test_dataloader
+for dataloader in test_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
+for dataloader in val_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
 
-val_evaluator = [
-    dict(
-        type='WordMetric', mode=['exact', 'ignore_case',
-                                 'ignore_case_symbol']),
-    dict(type='CharMetric')
-]
-
-test_evaluator = val_evaluator
 visualizer = dict(type='TextRecogLocalVisualizer', name='visualizer')

configs/textrecog/robust_scanner/robustscanner_r31_academic.py

@@ -7,7 +7,6 @@ _base_ = [
 
 # dataset settings
 train_list = {{_base_.train_list}}
-test_list = {{_base_.test_list}}
 file_client_args = dict(backend='disk')
 default_hooks = dict(logger=dict(type='LoggerHook', interval=100))
 
@@ -43,21 +42,12 @@ train_dataloader = dict(
     dataset=dict(
         type='ConcatDataset', datasets=train_list, pipeline=train_pipeline))
 
-val_dataloader = dict(
-    batch_size=1,
-    num_workers=4,
-    persistent_workers=True,
-    drop_last=False,
-    sampler=dict(type='DefaultSampler', shuffle=False),
-    dataset=dict(
-        type='ConcatDataset', datasets=test_list, pipeline=test_pipeline))
-test_dataloader = val_dataloader
-
-val_evaluator = [
-    dict(
-        type='WordMetric', mode=['exact', 'ignore_case',
-                                 'ignore_case_symbol']),
-    dict(type='CharMetric')
-]
-test_evaluator = val_evaluator
+test_cfg = dict(type='MultiTestLoop')
+val_cfg = dict(type='MultiValLoop')
+val_dataloader = _base_.val_dataloader
+test_dataloader = _base_.test_dataloader
+for dataloader in test_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
+for dataloader in val_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
 visualizer = dict(type='TextRecogLocalVisualizer', name='visualizer')

configs/textrecog/sar/sar_r31_parallel_decoder_academic.py

@@ -8,7 +8,6 @@ _base_ = [
 
 # dataset settings
 train_list = {{_base_.train_list}}
-test_list = {{_base_.test_list}}
 file_client_args = dict(backend='disk')
 default_hooks = dict(logger=dict(type='LoggerHook', interval=100))
 
@@ -50,21 +49,12 @@ train_dataloader = dict(
     dataset=dict(
         type='ConcatDataset', datasets=train_list, pipeline=train_pipeline))
 
-val_dataloader = dict(
-    batch_size=1,
-    num_workers=4,
-    persistent_workers=True,
-    drop_last=False,
-    sampler=dict(type='DefaultSampler', shuffle=False),
-    dataset=dict(
-        type='ConcatDataset', datasets=test_list, pipeline=test_pipeline))
-test_dataloader = val_dataloader
-
-val_evaluator = [
-    dict(
-        type='WordMetric', mode=['exact', 'ignore_case',
-                                 'ignore_case_symbol']),
-    dict(type='CharMetric')
-]
-test_evaluator = val_evaluator
+test_cfg = dict(type='MultiTestLoop')
+val_cfg = dict(type='MultiValLoop')
+val_dataloader = _base_.val_dataloader
+test_dataloader = _base_.test_dataloader
+for dataloader in test_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
+for dataloader in val_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
 visualizer = dict(type='TextRecogLocalVisualizer', name='visualizer')

configs/textrecog/satrn/satrn_academic.py

@@ -8,7 +8,6 @@ _base_ = [
 
 # dataset settings
 train_list = {{_base_.train_list}}
-test_list = {{_base_.test_list}}
 file_client_args = dict(backend='disk')
 default_hooks = dict(logger=dict(type='LoggerHook', interval=50))
 
@@ -69,21 +68,12 @@ train_dataloader = dict(
     dataset=dict(
         type='ConcatDataset', datasets=train_list, pipeline=train_pipeline))
 
-val_dataloader = dict(
-    batch_size=64,
-    num_workers=4,
-    persistent_workers=True,
-    drop_last=False,
-    sampler=dict(type='DefaultSampler', shuffle=False),
-    dataset=dict(
-        type='ConcatDataset', datasets=test_list, pipeline=test_pipeline))
-test_dataloader = val_dataloader
-
-val_evaluator = [
-    dict(
-        type='WordMetric', mode=['exact', 'ignore_case',
-                                 'ignore_case_symbol']),
-    dict(type='CharMetric')
-]
-test_evaluator = val_evaluator
+test_cfg = dict(type='MultiTestLoop')
+val_cfg = dict(type='MultiValLoop')
+val_dataloader = _base_.val_dataloader
+test_dataloader = _base_.test_dataloader
+for dataloader in test_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
+for dataloader in val_dataloader:
+    dataloader['dataset']['pipeline'] = test_pipeline
 visualizer = dict(type='TextRecogLocalVisualizer', name='visualizer')