[Dev] update benchmark script (#3116)

.dev_scripts/benchmark_full_models.txt

@@ -0,0 +1,56 @@
+ann/ann_r50-d8_4xb2-40k_cityscapes-512x1024.py
+apcnet/apcnet_r50-d8_4xb4-80k_ade20k-512x512.py
+beit/beit-large_upernet_8xb1-amp-160k_ade20k-640x640.py
+bisenetv1/bisenetv1_r18-d32_4xb4-160k_coco-stuff164k-512x512.py
+bisenetv2/bisenetv2_fcn_4xb4-ohem-160k_cityscapes-1024x1024.py
+ccnet/ccnet_r50-d8_4xb2-40k_cityscapes-512x1024.py
+cgnet/cgnet_fcn_4xb4-60k_cityscapes-680x680.py
+convnext/convnext-tiny_upernet_8xb2-amp-160k_ade20k-512x512.py
+danet/danet_r50-d8_4xb2-40k_cityscapes-512x1024.py
+deeplabv3/deeplabv3_r50-d8_4xb2-40k_cityscapes-512x1024.py
+deeplabv3plus/deeplabv3plus_r101-d8_4xb2-40k_cityscapes-769x769.py
+dmnet/dmnet_r50-d8_4xb2-40k_cityscapes-512x1024.py
+dnlnet/dnl_r50-d8_4xb2-40k_cityscapes-512x1024.py
+dpt/dpt_vit-b16_8xb2-160k_ade20k-512x512.py
+emanet/eemanet_r50-d8_4xb2-80k_cityscapes-512x1024.py
+encnet/encnet_r50-d8_4xb4-80k_ade20k-512x512.py
+erfnet/erfnet_fcn_4xb4-160k_cityscapes-512x1024.py
+fastfcn/fastfcn_r50-d32_jpu_aspp_4xb2-80k_cityscapes-512x1024.py
+fastscnn/fast_scnn_8xb4-160k_cityscapes-512x1024.py
+fcn/fcn_r50-d8_4xb2-40k_cityscapes-512x1024.py
+gcnet/gcnet_r50-d8_4xb2-40k_cityscapes-769x769.py
+hrnet/fcn_hr18s_4xb4-80k_ade20k-512x512.py
+icnet/icnet_r50-d8_4xb2-80k_cityscapes-832x832.py
+isanet/isanet_r50-d8_4xb2-80k_cityscapes-512x1024.py
+knet/knet-s3_swin-l_upernet_8xb2-adamw-80k_ade20k-640x640.py
+mae/mae-base_upernet_8xb2-amp-160k_ade20k-512x512.py
+mask2former/mask2former_r50_8xb2-90k_cityscapes-512x1024.py
+mask2former/mask2former_swin-t_8xb2-90k_cityscapes-512x1024.py
+mask2former/mask2former_swin-s_8xb2-160k_ade20k-512x512.py
+mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-160k_ade20k-640x640.py
+maskformer/maskformer_r50-d32_8xb2-160k_ade20k-512x512.py
+maskformer/maskformer_r101-d32_8xb2-160k_ade20k-512x512.py
+maskformer/maskformer_swin-t_upernet_8xb2-160k_ade20k-512x512.py
+maskformer/maskformer_swin-s_upernet_8xb2-160k_ade20k-512x512.py
+mobilenet_v2/mobilenet-v2-d8_pspnet_4xb2-80k_cityscapes-512x1024.py
+mobilenet_v3/mobilenet-v3-d8-scratch_lraspp_4xb4-320k_cityscapes-512x1024.py
+nonlocal_net/nonlocal_r50-d8_4xb2-40k_cityscapes-512x1024.py
+ocrnet/ocrnet_hr18_4xb4-80k_ade20k-512x512.py
+pidnet/pidnet-l_2xb6-120k_1024x1024-cityscapes.py
+point_rend/pointrend_r50_4xb2-80k_cityscapes-512x1024.py
+poolformer/fpn_poolformer_m48_8xb4-40k_ade20k-512x512.py
+psanet/psanet_r101-d8_4xb4-80k_ade20k-512x512.py
+pspnet/pspnet_r50-d8-rsb_4xb2-adamw-80k_cityscapes-512x1024.py
+resnest/resnest_s101-d8_deeplabv3_4xb4-160k_ade20k-512x512.py
+segformer/segformer_mit-b5_8xb1-160k_cityscapes-1024x1024.py
+segformer/segformer_mit-b5_8xb2-160k_ade20k-512x512.py
+segmenter/segmenter_vit-t_mask_8xb1-160k_ade20k-512x512.py
+segnext/segnext_mscan-t_1xb16-adamw-160k_ade20k-512x512.py
+sem_fpn/fpn_r101_4xb2-80k_cityscapes-512x1024.py
+setr/setr_vit-l-mla_8xb1-160k_ade20k-512x512.py
+stdc/stdc1_in1k-pre_4xb12-80k_cityscapes-512x1024.py
+swin/swin-small-patch4-window7-in1k-pre_upernet_8xb2-160k_ade20k-512x512.py
+twins/twins_pcpvt-s_uperhead_8xb4-160k_ade20k-512x512.py
+unet/unet-s5-d16_fcn_4xb4-160k_cityscapes-512x1024.py
+upernet/upernet_r50_4xb2-40k_cityscapes-769x769.py
+vit/vit_deit-s16_upernet_8xb2-80k_ade20k-512x512.py

.dev_scripts/benchmark_options.py

@@ -0,0 +1,10 @@
+third_part_libs = [
+    'pip install mmengine',
+    'pip install mmcv>=2.0.0',
+    'pip install mmcls==1.0.0rc6',
+    'pip install mmdet==3.0.0',
+    'pip install -r requirements.txt',
+    'pip install timm',
+]
+
+default_floating_range = 0.5

.dev_scripts/benchmark_train_models.txt

@@ -0,0 +1,25 @@
+bisenetv1/bisenetv1_r18-d32_4xb4-160k_coco-stuff164k-512x512.py
+bisenetv2/bisenetv2_fcn_4xb4-ohem-160k_cityscapes-1024x1024.py
+deeplabv3/deeplabv3_r50-d8_4xb2-40k_cityscapes-512x1024.py
+deeplabv3/deeplabv3_r101-d8_4xb4-160k_ade20k-512x512.py
+deeplabv3plus/deeplabv3plus_r101-d8_4xb2-40k_cityscapes-769x769.py
+deeplabv3plus/deeplabv3plus_r50-d8_4xb4-80k_ade20k-512x512.py
+hrnet/fcn_hr18s_4xb4-80k_ade20k-512x512.py
+hrnet/fcn_hr18_4xb2-160k_cityscapes-512x1024.py
+knet/knet-s3_swin-l_upernet_8xb2-adamw-80k_ade20k-512x512.py
+knet/knet-s3_r50-d8_pspnet_8xb2-adamw-80k_ade20k-512x512.py
+mae/mae-base_upernet_8xb2-amp-160k_ade20k-512x512.py
+mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-160k_ade20k-640x640.py
+maskformer/maskformer_swin-s_upernet_8xb2-160k_ade20k-512x512.py
+mobilenet_v2/mobilenet-v2-d8_deeplabv3_4xb2-80k_cityscapes-512x1024.py
+ocrnet/ocrnet_hr48_4xb4-80k_ade20k-512x512.py
+pidnet/pidnet-m_2xb6-120k_1024x1024-cityscapes.py
+pspnet/pspnet_r50-d8-rsb_4xb2-adamw-80k_cityscapes-512x1024.py
+pspnet/pspnet_r101-d8_4xb4-80k_ade20k-512x512.py
+segformer/segformer_mit-b5_8xb2-160k_ade20k-512x512.py
+segmenter/segmenter_vit-t_mask_8xb1-160k_ade20k-512x512.py
+segnext/segnext_mscan-t_1xb16-adamw-160k_ade20k-512x512.py
+swin/swin-base-patch4-window12-in22k-384x384-pre_upernet_8xb2-160k_ade20k-512x512.py
+twins/twins_pcpvt-l_uperhead_8xb2-160k_ade20k-512x512.py
+unet/unet-s5-d16_fcn_4xb4-160k_cityscapes-512x1024.py
+upernet/upernet_r101_4xb2-40k_cityscapes-512x1024.py

.dev_scripts/generate_benchmark_evaluation_script.py

@@ -17,7 +17,7 @@ def parse_args():
     parser.add_argument(
         '--out',
         type=str,
-        default='.dev/benchmark_evaluation.sh',
+        default='.dev_scripts/benchmark_evaluation.sh',
         help='path to save model benchmark script')
 
     args = parser.parse_args()

.dev_scripts/generate_benchmark_train_script.py

@@ -19,7 +19,7 @@ def parse_args():
     parser.add_argument(
         '--out',
         type=str,
-        default='.dev/benchmark_train.sh',
+        default='.dev_scripts/benchmark_train.sh',
         help='path to save model benchmark script')
 
     args = parser.parse_args()

.pre-commit-config.yaml

@@ -47,7 +47,7 @@ repos:
       - id: update-model-index
         name: update-model-index
         description: Collect model information and update model-index.yml
-        entry: .dev/update_model_index.py
+        entry: .dev_scripts/update_model_index.py
         additional_dependencies: [pyyaml]
         language: python
         require_serial: true

requirements/albu.txt

@@ -0,0 +1 @@
+albumentations>=0.3.2 --no-binary qudida,albumentations