Add benchmark options (#1654)

* update dev_scripts

* update metafile

* update multimodal floating range

* fix lint

* update

* update

* fix lint

* Update metric map

---------

Co-authored-by: mzr1996 <mzr1996@163.com>

.dev_scripts/benchmark_options.py

@@ -0,0 +1,10 @@
+third_part_libs = [
+    'pip install -r ../requirements.txt',
+]
+
+default_floating_range = 0.2
+model_floating_ranges = {
+    'blip/blip-base_8xb32_retrieval.py': 1.0,
+    'blip2/blip2-opt2.7b_8xb32_caption.py': 1.0,
+    'ofa/ofa-base_finetuned_caption.py': 1.0,
+}

.dev_scripts/benchmark_regression/3-benchmark_train.py

@@ -16,7 +16,8 @@ from modelindex.load_model_index import load
 from rich.console import Console
 from rich.syntax import Syntax
 from rich.table import Table
-from utils import METRICS_MAP, MMCLS_ROOT
+
+from .utils import METRICS_MAP, MMCLS_ROOT
 
 # Avoid to import MMPretrain to accelerate speed to show summary
 
@@ -147,6 +148,9 @@ def create_train_job_batch(model_info, args, port, pretrain_info=None):
     if not args.local:
         launcher = 'slurm'
         runner = 'srun python'
+        if gpus > 8:
+            gpus = 8
+            cfg_options.append('auto_scale_lr.enable=True')
     elif not args.non_distributed:
         launcher = 'pytorch'
         if gpus > 8:

.dev_scripts/benchmark_regression/bench_test.yml

@@ -0,0 +1,14 @@
+- Name: convnext-base_32xb128_in1k
+- Name: convnext-v2-atto_fcmae-pre_3rdparty_in1k
+- Name: mobilenet-v2_8xb32_in1k
+- Name: mobilenet-v3-small-050_8xb128_in1k
+- Name: swin-tiny_16xb64_in1k
+- Name: swinv2-tiny-w8_3rdparty_in1k-256px
+- Name: vit-base-p16_32xb128-mae_in1k
+- Name: resnet34_8xb32_in1k
+- Name: resnext50-32x4d_8xb32_in1k
+- Name: shufflenet-v2-1x_16xb64_in1k
+- Name: riformer-s12_in1k
+- Name: blip-base_3rdparty_retrieval
+- Name: blip2-opt2.7b_3rdparty-zeroshot_caption
+- Name: ofa-base_3rdparty-finetuned_caption

.dev_scripts/benchmark_regression/bench_train.yml

@@ -1,10 +1,7 @@
 - Name: mobilenet-v2_8xb32_in1k
   Cycle: month
 
-- Name: resnet50_8xb32_in1k
-  Cycle: month
-
-- Name: seresnet50_8xb32_in1k
+- Name: resnet50_8xb256-rsb-a1-600e_in1k
   Cycle: month
 
 - Name: swin-small_16xb64_in1k
@@ -13,6 +10,12 @@
 - Name: vit-base-p16_32xb128-mae_in1k
   Cycle: month
 
+- Name: seresnet50_8xb32_in1k
+  Cycle: quarter
+
+- Name: resnet50_8xb32_in1k
+  Cycle: quarter
+
 - Name: resnet50_8xb256-rsb-a1-600e_in1k
   Cycle: quarter
 

.dev_scripts/benchmark_regression/utils.py

@@ -6,6 +6,9 @@ METRICS_MAP = {
     'Top 1 Accuracy': 'accuracy/top1',
     'Top 5 Accuracy': 'accuracy/top5',
     'Recall@1': 'retrieval/Recall@1',
+    'Recall@5': 'retrieval/Recall@5',
+    'BLEU-4': 'Bleu_4',
+    'CIDER': 'CIDEr',
 }
 
 

configs/blip/metafile.yml

@@ -9,6 +9,7 @@ Collections:
         - SBU captions
       Architecture:
         - Transformer
+      Training Resources: 8x A100 GPUs
     Paper:
       Title: 'BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language
         Understanding and Generation'

configs/blip2/metafile.yml

@@ -8,6 +8,7 @@ Collections:
         - CC12M
         - SBU
         - LAION-400M
+      Training Resources: 8x A100 GPUs
       Architecture:
         - Transformer
         - Q-Former

configs/riformer/metafile.yml

@@ -2,6 +2,7 @@ Collections:
   - Name: RIFormer
     Metadata:
       Training Data: ImageNet-1k
+      Training Resources: 8x A100 GPUs
       Architecture:
         - Affine
         - 1x1 Convolution