From 20adcfa3554ed3ddfa35ce5417c56ff41e7c5a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E6=98=95=E8=BE=B0?= Date: Mon, 19 Jun 2023 16:59:33 +0800 Subject: [PATCH] [Dev] update benchmark script (#3116) --- {.dev => .dev_scripts}/batch_test_list.py | 0 {.dev => .dev_scripts}/batch_train_list.txt | 0 .../benchmark_evaluation.sh | 0 .dev_scripts/benchmark_full_models.txt | 56 +++++++++++++++++++ {.dev => .dev_scripts}/benchmark_inference.py | 0 .dev_scripts/benchmark_options.py | 10 ++++ {.dev => .dev_scripts}/benchmark_train.sh | 0 .dev_scripts/benchmark_train_models.txt | 25 +++++++++ {.dev => .dev_scripts}/check_urls.py | 0 .../gather_benchmark_evaluation_results.py | 0 .../gather_benchmark_train_results.py | 0 {.dev => .dev_scripts}/gather_models.py | 0 .../generate_benchmark_evaluation_script.py | 2 +- .../generate_benchmark_train_script.py | 2 +- .../log_collector/example_config.py | 0 .../log_collector/log_collector.py | 0 .../log_collector/readme.md | 0 {.dev => .dev_scripts}/log_collector/utils.py | 0 {.dev => .dev_scripts}/update_model_index.py | 0 {.dev => .dev_scripts}/upload_modelzoo.py | 0 .pre-commit-config.yaml | 2 +- requirements/albu.txt | 1 + 22 files changed, 95 insertions(+), 3 deletions(-) rename {.dev => .dev_scripts}/batch_test_list.py (100%) rename {.dev => .dev_scripts}/batch_train_list.txt (100%) rename {.dev => .dev_scripts}/benchmark_evaluation.sh (100%) create mode 100644 .dev_scripts/benchmark_full_models.txt rename {.dev => .dev_scripts}/benchmark_inference.py (100%) create mode 100644 .dev_scripts/benchmark_options.py rename {.dev => .dev_scripts}/benchmark_train.sh (100%) create mode 100644 .dev_scripts/benchmark_train_models.txt rename {.dev => .dev_scripts}/check_urls.py (100%) rename {.dev => .dev_scripts}/gather_benchmark_evaluation_results.py (100%) rename {.dev => .dev_scripts}/gather_benchmark_train_results.py (100%) rename {.dev => .dev_scripts}/gather_models.py (100%) rename {.dev => .dev_scripts}/generate_benchmark_evaluation_script.py (98%) rename {.dev => .dev_scripts}/generate_benchmark_train_script.py (98%) rename {.dev => .dev_scripts}/log_collector/example_config.py (100%) rename {.dev => .dev_scripts}/log_collector/log_collector.py (100%) rename {.dev => .dev_scripts}/log_collector/readme.md (100%) rename {.dev => .dev_scripts}/log_collector/utils.py (100%) rename {.dev => .dev_scripts}/update_model_index.py (100%) rename {.dev => .dev_scripts}/upload_modelzoo.py (100%) create mode 100644 requirements/albu.txt diff --git a/.dev/batch_test_list.py b/.dev_scripts/batch_test_list.py similarity index 100% rename from .dev/batch_test_list.py rename to .dev_scripts/batch_test_list.py diff --git a/.dev/batch_train_list.txt b/.dev_scripts/batch_train_list.txt similarity index 100% rename from .dev/batch_train_list.txt rename to .dev_scripts/batch_train_list.txt diff --git a/.dev/benchmark_evaluation.sh b/.dev_scripts/benchmark_evaluation.sh similarity index 100% rename from .dev/benchmark_evaluation.sh rename to .dev_scripts/benchmark_evaluation.sh diff --git a/.dev_scripts/benchmark_full_models.txt b/.dev_scripts/benchmark_full_models.txt new file mode 100644 index 000000000..c574d8f15 --- /dev/null +++ b/.dev_scripts/benchmark_full_models.txt @@ -0,0 +1,56 @@ +ann/ann_r50-d8_4xb2-40k_cityscapes-512x1024.py +apcnet/apcnet_r50-d8_4xb4-80k_ade20k-512x512.py +beit/beit-large_upernet_8xb1-amp-160k_ade20k-640x640.py +bisenetv1/bisenetv1_r18-d32_4xb4-160k_coco-stuff164k-512x512.py +bisenetv2/bisenetv2_fcn_4xb4-ohem-160k_cityscapes-1024x1024.py +ccnet/ccnet_r50-d8_4xb2-40k_cityscapes-512x1024.py +cgnet/cgnet_fcn_4xb4-60k_cityscapes-680x680.py +convnext/convnext-tiny_upernet_8xb2-amp-160k_ade20k-512x512.py +danet/danet_r50-d8_4xb2-40k_cityscapes-512x1024.py +deeplabv3/deeplabv3_r50-d8_4xb2-40k_cityscapes-512x1024.py +deeplabv3plus/deeplabv3plus_r101-d8_4xb2-40k_cityscapes-769x769.py +dmnet/dmnet_r50-d8_4xb2-40k_cityscapes-512x1024.py +dnlnet/dnl_r50-d8_4xb2-40k_cityscapes-512x1024.py +dpt/dpt_vit-b16_8xb2-160k_ade20k-512x512.py +emanet/eemanet_r50-d8_4xb2-80k_cityscapes-512x1024.py +encnet/encnet_r50-d8_4xb4-80k_ade20k-512x512.py +erfnet/erfnet_fcn_4xb4-160k_cityscapes-512x1024.py +fastfcn/fastfcn_r50-d32_jpu_aspp_4xb2-80k_cityscapes-512x1024.py +fastscnn/fast_scnn_8xb4-160k_cityscapes-512x1024.py +fcn/fcn_r50-d8_4xb2-40k_cityscapes-512x1024.py +gcnet/gcnet_r50-d8_4xb2-40k_cityscapes-769x769.py +hrnet/fcn_hr18s_4xb4-80k_ade20k-512x512.py +icnet/icnet_r50-d8_4xb2-80k_cityscapes-832x832.py +isanet/isanet_r50-d8_4xb2-80k_cityscapes-512x1024.py +knet/knet-s3_swin-l_upernet_8xb2-adamw-80k_ade20k-640x640.py +mae/mae-base_upernet_8xb2-amp-160k_ade20k-512x512.py +mask2former/mask2former_r50_8xb2-90k_cityscapes-512x1024.py +mask2former/mask2former_swin-t_8xb2-90k_cityscapes-512x1024.py +mask2former/mask2former_swin-s_8xb2-160k_ade20k-512x512.py +mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-160k_ade20k-640x640.py +maskformer/maskformer_r50-d32_8xb2-160k_ade20k-512x512.py +maskformer/maskformer_r101-d32_8xb2-160k_ade20k-512x512.py +maskformer/maskformer_swin-t_upernet_8xb2-160k_ade20k-512x512.py +maskformer/maskformer_swin-s_upernet_8xb2-160k_ade20k-512x512.py +mobilenet_v2/mobilenet-v2-d8_pspnet_4xb2-80k_cityscapes-512x1024.py +mobilenet_v3/mobilenet-v3-d8-scratch_lraspp_4xb4-320k_cityscapes-512x1024.py +nonlocal_net/nonlocal_r50-d8_4xb2-40k_cityscapes-512x1024.py +ocrnet/ocrnet_hr18_4xb4-80k_ade20k-512x512.py +pidnet/pidnet-l_2xb6-120k_1024x1024-cityscapes.py +point_rend/pointrend_r50_4xb2-80k_cityscapes-512x1024.py +poolformer/fpn_poolformer_m48_8xb4-40k_ade20k-512x512.py +psanet/psanet_r101-d8_4xb4-80k_ade20k-512x512.py +pspnet/pspnet_r50-d8-rsb_4xb2-adamw-80k_cityscapes-512x1024.py +resnest/resnest_s101-d8_deeplabv3_4xb4-160k_ade20k-512x512.py +segformer/segformer_mit-b5_8xb1-160k_cityscapes-1024x1024.py +segformer/segformer_mit-b5_8xb2-160k_ade20k-512x512.py +segmenter/segmenter_vit-t_mask_8xb1-160k_ade20k-512x512.py +segnext/segnext_mscan-t_1xb16-adamw-160k_ade20k-512x512.py +sem_fpn/fpn_r101_4xb2-80k_cityscapes-512x1024.py +setr/setr_vit-l-mla_8xb1-160k_ade20k-512x512.py +stdc/stdc1_in1k-pre_4xb12-80k_cityscapes-512x1024.py +swin/swin-small-patch4-window7-in1k-pre_upernet_8xb2-160k_ade20k-512x512.py +twins/twins_pcpvt-s_uperhead_8xb4-160k_ade20k-512x512.py +unet/unet-s5-d16_fcn_4xb4-160k_cityscapes-512x1024.py +upernet/upernet_r50_4xb2-40k_cityscapes-769x769.py +vit/vit_deit-s16_upernet_8xb2-80k_ade20k-512x512.py diff --git a/.dev/benchmark_inference.py b/.dev_scripts/benchmark_inference.py similarity index 100% rename from .dev/benchmark_inference.py rename to .dev_scripts/benchmark_inference.py diff --git a/.dev_scripts/benchmark_options.py b/.dev_scripts/benchmark_options.py new file mode 100644 index 000000000..51909a091 --- /dev/null +++ b/.dev_scripts/benchmark_options.py @@ -0,0 +1,10 @@ +third_part_libs = [ + 'pip install mmengine', + 'pip install mmcv>=2.0.0', + 'pip install mmcls==1.0.0rc6', + 'pip install mmdet==3.0.0', + 'pip install -r requirements.txt', + 'pip install timm', +] + +default_floating_range = 0.5 diff --git a/.dev/benchmark_train.sh b/.dev_scripts/benchmark_train.sh similarity index 100% rename from .dev/benchmark_train.sh rename to .dev_scripts/benchmark_train.sh diff --git a/.dev_scripts/benchmark_train_models.txt b/.dev_scripts/benchmark_train_models.txt new file mode 100644 index 000000000..887b2e376 --- /dev/null +++ b/.dev_scripts/benchmark_train_models.txt @@ -0,0 +1,25 @@ +bisenetv1/bisenetv1_r18-d32_4xb4-160k_coco-stuff164k-512x512.py +bisenetv2/bisenetv2_fcn_4xb4-ohem-160k_cityscapes-1024x1024.py +deeplabv3/deeplabv3_r50-d8_4xb2-40k_cityscapes-512x1024.py +deeplabv3/deeplabv3_r101-d8_4xb4-160k_ade20k-512x512.py +deeplabv3plus/deeplabv3plus_r101-d8_4xb2-40k_cityscapes-769x769.py +deeplabv3plus/deeplabv3plus_r50-d8_4xb4-80k_ade20k-512x512.py +hrnet/fcn_hr18s_4xb4-80k_ade20k-512x512.py +hrnet/fcn_hr18_4xb2-160k_cityscapes-512x1024.py +knet/knet-s3_swin-l_upernet_8xb2-adamw-80k_ade20k-512x512.py +knet/knet-s3_r50-d8_pspnet_8xb2-adamw-80k_ade20k-512x512.py +mae/mae-base_upernet_8xb2-amp-160k_ade20k-512x512.py +mask2former/mask2former_swin-l-in22k-384x384-pre_8xb2-160k_ade20k-640x640.py +maskformer/maskformer_swin-s_upernet_8xb2-160k_ade20k-512x512.py +mobilenet_v2/mobilenet-v2-d8_deeplabv3_4xb2-80k_cityscapes-512x1024.py +ocrnet/ocrnet_hr48_4xb4-80k_ade20k-512x512.py +pidnet/pidnet-m_2xb6-120k_1024x1024-cityscapes.py +pspnet/pspnet_r50-d8-rsb_4xb2-adamw-80k_cityscapes-512x1024.py +pspnet/pspnet_r101-d8_4xb4-80k_ade20k-512x512.py +segformer/segformer_mit-b5_8xb2-160k_ade20k-512x512.py +segmenter/segmenter_vit-t_mask_8xb1-160k_ade20k-512x512.py +segnext/segnext_mscan-t_1xb16-adamw-160k_ade20k-512x512.py +swin/swin-base-patch4-window12-in22k-384x384-pre_upernet_8xb2-160k_ade20k-512x512.py +twins/twins_pcpvt-l_uperhead_8xb2-160k_ade20k-512x512.py +unet/unet-s5-d16_fcn_4xb4-160k_cityscapes-512x1024.py +upernet/upernet_r101_4xb2-40k_cityscapes-512x1024.py diff --git a/.dev/check_urls.py b/.dev_scripts/check_urls.py similarity index 100% rename from .dev/check_urls.py rename to .dev_scripts/check_urls.py diff --git a/.dev/gather_benchmark_evaluation_results.py b/.dev_scripts/gather_benchmark_evaluation_results.py similarity index 100% rename from .dev/gather_benchmark_evaluation_results.py rename to .dev_scripts/gather_benchmark_evaluation_results.py diff --git a/.dev/gather_benchmark_train_results.py b/.dev_scripts/gather_benchmark_train_results.py similarity index 100% rename from .dev/gather_benchmark_train_results.py rename to .dev_scripts/gather_benchmark_train_results.py diff --git a/.dev/gather_models.py b/.dev_scripts/gather_models.py similarity index 100% rename from .dev/gather_models.py rename to .dev_scripts/gather_models.py diff --git a/.dev/generate_benchmark_evaluation_script.py b/.dev_scripts/generate_benchmark_evaluation_script.py similarity index 98% rename from .dev/generate_benchmark_evaluation_script.py rename to .dev_scripts/generate_benchmark_evaluation_script.py index bdc5a8fca..4c48f8542 100644 --- a/.dev/generate_benchmark_evaluation_script.py +++ b/.dev_scripts/generate_benchmark_evaluation_script.py @@ -17,7 +17,7 @@ def parse_args(): parser.add_argument( '--out', type=str, - default='.dev/benchmark_evaluation.sh', + default='.dev_scripts/benchmark_evaluation.sh', help='path to save model benchmark script') args = parser.parse_args() diff --git a/.dev/generate_benchmark_train_script.py b/.dev_scripts/generate_benchmark_train_script.py similarity index 98% rename from .dev/generate_benchmark_train_script.py rename to .dev_scripts/generate_benchmark_train_script.py index 4e6fa181f..4bfdfbf09 100644 --- a/.dev/generate_benchmark_train_script.py +++ b/.dev_scripts/generate_benchmark_train_script.py @@ -19,7 +19,7 @@ def parse_args(): parser.add_argument( '--out', type=str, - default='.dev/benchmark_train.sh', + default='.dev_scripts/benchmark_train.sh', help='path to save model benchmark script') args = parser.parse_args() diff --git a/.dev/log_collector/example_config.py b/.dev_scripts/log_collector/example_config.py similarity index 100% rename from .dev/log_collector/example_config.py rename to .dev_scripts/log_collector/example_config.py diff --git a/.dev/log_collector/log_collector.py b/.dev_scripts/log_collector/log_collector.py similarity index 100% rename from .dev/log_collector/log_collector.py rename to .dev_scripts/log_collector/log_collector.py diff --git a/.dev/log_collector/readme.md b/.dev_scripts/log_collector/readme.md similarity index 100% rename from .dev/log_collector/readme.md rename to .dev_scripts/log_collector/readme.md diff --git a/.dev/log_collector/utils.py b/.dev_scripts/log_collector/utils.py similarity index 100% rename from .dev/log_collector/utils.py rename to .dev_scripts/log_collector/utils.py diff --git a/.dev/update_model_index.py b/.dev_scripts/update_model_index.py similarity index 100% rename from .dev/update_model_index.py rename to .dev_scripts/update_model_index.py diff --git a/.dev/upload_modelzoo.py b/.dev_scripts/upload_modelzoo.py similarity index 100% rename from .dev/upload_modelzoo.py rename to .dev_scripts/upload_modelzoo.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3bc3fd09a..cf6ecdd8f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: - id: update-model-index name: update-model-index description: Collect model information and update model-index.yml - entry: .dev/update_model_index.py + entry: .dev_scripts/update_model_index.py additional_dependencies: [pyyaml] language: python require_serial: true diff --git a/requirements/albu.txt b/requirements/albu.txt new file mode 100644 index 000000000..f421fbbdc --- /dev/null +++ b/requirements/albu.txt @@ -0,0 +1 @@ +albumentations>=0.3.2 --no-binary qudida,albumentations