From 1fd3509f7b58dd2203260044a9ad04b9b9185a3d Mon Sep 17 00:00:00 2001 From: Yixiao Fang <36138628+fangyixiao18@users.noreply.github.com> Date: Fri, 30 Dec 2022 17:37:21 +0800 Subject: [PATCH] [Refactor] Refactor benchmark script and update files (#637) * update benchmark regression script * update yml files * fix lint --- .dev_scripts/benchmark_regression/models.yml | 54 +++++++++++++------ .../benchmark_regression/pretrain_with_cls.py | 41 ++++++++++---- configs/selfsup/cae/metafile.yml | 2 +- configs/selfsup/mae/metafile.yml | 2 +- configs/selfsup/milan/metafile.yaml | 4 +- configs/selfsup/mocov3/metafile.yml | 4 +- configs/selfsup/simmim/metafile.yml | 16 +++--- 7 files changed, 81 insertions(+), 42 deletions(-) diff --git a/.dev_scripts/benchmark_regression/models.yml b/.dev_scripts/benchmark_regression/models.yml index 5b576067..1965d41d 100644 --- a/.dev_scripts/benchmark_regression/models.yml +++ b/.dev_scripts/benchmark_regression/models.yml @@ -1,35 +1,55 @@ -- Name: mae_vit-base-p16_8xb512-coslr-400e_in1k - ClsConfig: configs/benchmarks/classification/imagenet/vit-base-p16_ft-8xb128-coslr-100e_in1k.py +- Name: mae_vit-base-p16_8xb512-amp-coslr-300e_in1k + Task: Linear Evaluation Cycle: month -- Name: mocov3_vit-small-p16_16xb256-fp16-coslr-300e_in1k-224 - ClsConfig: configs/benchmarks/classification/imagenet/vit-small-p16_linear-8xb128-coslr-90e_in1k.py +- Name: mocov3_vit-small-p16_16xb256-amp-coslr-300e_in1k + Task: Linear Evaluation Cycle: month -- Name: simmim_swin-base_8xb256-coslr-100e_in1k-192 - ClsConfig: configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k.py +- Name: simmim_swin-base_8xb256-amp-coslr-100e_in1k-192 + Task: Fine-tuning Cycle: month - Name: simclr_resnet50_8xb32-coslr-200e_in1k - ClsConfig: configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py - Cycle: month + Task: Linear Evaluation + Cycle: quarter -- Name: byol_resnet50_8xb32-accum16-coslr-200e_in1k - ClsConfig: configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py - Cycle: month +- Name: byol_resnet50_16xb256-coslr-200e_in1k + Task: Linear Evaluation + Cycle: quarter - Name: mocov2_resnet50_8xb32-coslr-200e_in1k - ClsConfig: configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py + Task: Linear Evaluation Cycle: quarter - Name: simsiam_resnet50_8xb32-coslr-100e_in1k - ClsConfig: configs/benchmarks/classification/imagenet/resnet50_linear-8xb512-coslr-90e_in1k.py + Task: Linear Evaluation + Cycle: quarter + +- Name: maskfeat_vit-base-p16_8xb256-amp-coslr-300e_in1k + Task: Fine-tuning + Cycle: quarter + +- Name: beit_vit-base-p16_8xb256-amp-coslr-300e_in1k + Task: Fine-tuning + Cycle: quarter + +- Name: beitv2_vit-base-p16_8xb256-amp-coslr-300e_in1k + Task: Fine-tuning + Cycle: quarter + +- Name: milan_vit-base-p16_16xb256-amp-coslr-400e_in1k + Task: Linear Evaluation Cycle: quarter - Name: densecl_resnet50_8xb32-coslr-200e_in1k - ClsConfig: configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py - Cycle: quarter + Task: Linear Evaluation + Cycle: half-year - Name: swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96 - ClsConfig: configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-coslr-100e_in1k.py - Cycle: quarter + Task: Linear Evaluation + Cycle: half-year + +- Name: cae_vit-base-p16_8xb256-fp16-coslr-300e_in1k + Task: Fine-tuning + Cycle: half-year diff --git a/.dev_scripts/benchmark_regression/pretrain_with_cls.py b/.dev_scripts/benchmark_regression/pretrain_with_cls.py index 66bd06db..38734ad0 100644 --- a/.dev_scripts/benchmark_regression/pretrain_with_cls.py +++ b/.dev_scripts/benchmark_regression/pretrain_with_cls.py @@ -113,11 +113,21 @@ def get_gpu_number(model_info): matches = re.match(r'.*[-_](\d+)xb(\d+).*', config) if matches is None: raise ValueError( - 'Cannot get gpu numbers from the config name {config}') + f'Cannot get gpu numbers from the config name {config}') gpus = int(matches.groups()[0]) return gpus +def get_pretrain_epoch(model_info): + config = osp.basename(model_info.config) + matches = re.match(r'.*[-_](\d+)e[-_].*', config) + if matches is None: + raise ValueError( + f'Cannot get epoch setting from the config name {config}') + epoch = int(matches.groups()[0]) + return epoch + + def create_train_job_batch(commands, model_info, args, port, script_name): fname = model_info.name @@ -137,7 +147,6 @@ def create_train_job_batch(commands, model_info, args, port, script_name): quota_cfg = '' launcher = 'none' if args.local else 'slurm' - job_name = f'{args.job_name}_{fname}' job_script = (f'#!/bin/bash\n' f'srun -p {args.partition} ' @@ -158,14 +167,23 @@ def create_train_job_batch(commands, model_info, args, port, script_name): commands.append(f'echo "{config}"') # downstream classification task - cls_config = getattr(model_info, 'cls_config', None) + cls_config = None + task = getattr(model_info, 'task', None) + if task is not None: + for downstream in model_info.data['Downstream']: + if task == downstream['Results'][0]['Task']: + cls_config = downstream['Config'] + break + else: + cls_config = None + if cls_config: fname = model_info.name gpus = get_gpu_number(model_info) gpus_per_node = min(gpus, 8) - cls_config_path = Path(model_info.cls_config) + cls_config_path = Path(cls_config) assert cls_config_path.exists(), f'"{fname}": {cls_config} not found.' job_name = f'{args.job_name}_{fname}' @@ -178,13 +196,11 @@ def create_train_job_batch(commands, model_info, args, port, script_name): if args.quotatype is not None: srun_args = srun_args.join(f'--quotatype {args.quotatype}') - # get pretrain weights - ckpt_path_file = work_dir / 'last_checkpoint' - with open(ckpt_path_file, 'r') as f: - ckpt = f.readlines()[0] + # get pretrain weights path + epoch = get_pretrain_epoch(model_info) + ckpt = work_dir / f'epoch_{epoch}.pth' launcher = 'none' if args.local else 'slurm' - cls_job_script = ( f'\n' f'mim train mmcls {cls_config} ' @@ -365,7 +381,10 @@ def summary(models, args): if len(val_logs) == 0: continue - expect_metrics = model_info.results[0].metrics + for downstream in model_info.data['Downstream']: + if model_info.task == downstream['Results'][0]['Task']: + expect_metrics = downstream['Results'][0]['Metrics'] + break # extract metrics summary = {'log_file': log_file} @@ -408,7 +427,7 @@ def main(): name = item['Name'] model_info = all_models[item['Name']] model_info.cycle = item.get('Cycle', None) - model_info.cls_config = item.get('ClsConfig', None) + model_info.task = item.get('Task', None) cycle = getattr(model_info, 'cycle', 'month') cycle_level = CYCLE_LEVELS.index(cycle) if cycle_level in args.range: diff --git a/configs/selfsup/cae/metafile.yml b/configs/selfsup/cae/metafile.yml index 78c3c48a..b106d37d 100644 --- a/configs/selfsup/cae/metafile.yml +++ b/configs/selfsup/cae/metafile.yml @@ -30,6 +30,6 @@ Models: - Task: Fine-tuning Dataset: ImageNet-1k Metrics: - Top 1 Accuracy: 60.8 + Top 1 Accuracy: 83.2 Config: configs/benchmarks/classification/imagenet/vit-base-p16_ft-8xb128-coslr-100e-rpe_in1k.py Weights: https://download.openmmlab.com/mmselfsup/1.x/cae/cae_vit-base-p16_16xb128-fp16-coslr-300e_in1k/vit-base-p16_ft-8xb128-coslr-100e-rpe_in1k/vit-base-p16_ft-8xb128-coslr-100e-rpe_in1k_20220825-f3d234cd.pth diff --git a/configs/selfsup/mae/metafile.yml b/configs/selfsup/mae/metafile.yml index 5de231c7..27078226 100644 --- a/configs/selfsup/mae/metafile.yml +++ b/configs/selfsup/mae/metafile.yml @@ -42,7 +42,7 @@ Models: Metrics: Top 1 Accuracy: 83.1 Config: configs/benchmarks/classification/imagenet/vit-base-p16_ft-8xb128-coslr-100e_in1k.py - - Name: mae_vit-base-p16_8xb512-amp-coslr-400e_in1k + - Name: mae_vit-base-p16_8xb512-coslr-400e_in1k In Collection: MAE Metadata: Epochs: 400 diff --git a/configs/selfsup/milan/metafile.yaml b/configs/selfsup/milan/metafile.yaml index 91c9f098..d5e4f364 100644 --- a/configs/selfsup/milan/metafile.yaml +++ b/configs/selfsup/milan/metafile.yaml @@ -31,14 +31,14 @@ Models: Dataset: ImageNet-1k Metrics: Top 1 Accuracy: 85.3 - Config: configs/selfsup/milan/classification/vit-base-p16_ft-8xb128-coslr-100e_in1k-milan.py + Config: configs/selfsup/milan/classification/vit-base-p16_ft-8xb128-coslr-100e_in1k.py Weights: https://download.openmmlab.com/mmselfsup/1.x/milan/milan_vit-base-p16_16xb256-amp-coslr-400e_in1k/vit-base-p16_ft-8xb128-coslr-100e_in1k/vit-base-p16_ft-8xb128-coslr-100e_in1k-milan_20221129-74ac94fa.pth - Type: Image Classification Metadata: Epochs: 100 Batch Size: 16384 Results: - - Task: Linear-probing + - Task: Linear Evaluation Dataset: ImageNet-1k Metrics: Top 1 Accuracy: 78.9 diff --git a/configs/selfsup/mocov3/metafile.yml b/configs/selfsup/mocov3/metafile.yml index 72e70840..ff97788b 100644 --- a/configs/selfsup/mocov3/metafile.yml +++ b/configs/selfsup/mocov3/metafile.yml @@ -93,7 +93,7 @@ Models: Dataset: ImageNet-1k Metrics: Top 1 Accuracy: 73.6 - Config: configs/benchmarks/classification/imagenet/vit-small-p16_linear-8xb128-coslr-90e.py + Config: configs/benchmarks/classification/imagenet/vit-small-p16_linear-8xb128-coslr-90e_in1k.py Weights: https://download.openmmlab.com/mmselfsup/1.x/mocov3/mocov3_vit-small-p16_16xb256-amp-coslr-300e_in1k/vit-small-p16_linear-8xb128-coslr-90e_in1k/vit-small-p16_linear-8xb128-coslr-90e_in1k_20220826-376674ef.pth - Name: mocov3_vit-base-p16_16xb256-amp-coslr-300e_in1k In Collection: MoCoV3 @@ -113,7 +113,7 @@ Models: Dataset: ImageNet-1k Metrics: Top 1 Accuracy: 76.9 - Config: configs/benchmarks/classification/imagenet/vit-base-p16_linear-8xb128-coslr-90e.py + Config: configs/benchmarks/classification/imagenet/vit-base-p16_linear-8xb128-coslr-90e_in1k.py Weights: https://download.openmmlab.com/mmselfsup/1.x/mocov3/mocov3_vit-base-p16_16xb256-amp-coslr-300e_in1k/vit-base-p16_linear-8xb128-coslr-90e_in1k/vit-base-p16_linear-8xb128-coslr-90e_in1k_20220826-83be7758.pth - Type: Image Classification Metadata: diff --git a/configs/selfsup/simmim/metafile.yml b/configs/selfsup/simmim/metafile.yml index df07a805..900377a3 100644 --- a/configs/selfsup/simmim/metafile.yml +++ b/configs/selfsup/simmim/metafile.yml @@ -13,13 +13,13 @@ Collections: README: configs/selfsup/simmim/README.md Models: - - Name: simmim_swin-base_16xb128-coslr-100e_in1k-192 + - Name: simmim_swin-base_8xb256-amp-coslr-100e_in1k-192 In Collection: SimMIM Metadata: Epochs: 100 Batch Size: 2048 Results: null - Config: configs/selfsup/simmim/simmim_swin-base_16xb128-coslr-100e_in1k-192.py + Config: configs/selfsup/simmim/simmim_swin-base_8xb256-amp-coslr-100e_in1k-192.py Weights: https://download.openmmlab.com/mmselfsup/1.x/simmim/simmim_swin-base_8xb256-amp-coslr-100e_in1k-192/simmim_swin-base_8xb256-amp-coslr-100e_in1k-192_20220829-0e15782d.pth Downstream: - Type: Image Classification @@ -31,7 +31,7 @@ Models: Dataset: ImageNet-1k Metrics: Top 1 Accuracy: 82.7 - Config: configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k.py + Config: configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k-192.py Weights: https://download.openmmlab.com/mmselfsup/1.x/simmim/simmim_swin-base_8xb256-amp-coslr-100e_in1k-192/swin-base_ft-8xb256-coslr-100e_in1k/swin-base_ft-8xb256-coslr-100e_in1k_20220829-9cf23aa1.pth - Type: Image Classification Metadata: @@ -43,13 +43,13 @@ Models: Metrics: Top 1 Accuracy: 83.5 Config: configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k-224.py - - Name: simmim_swin-base_16xb128-coslr-800e_in1k-192 + - Name: simmim_swin-base_16xb128-amp-coslr-800e_in1k-192 In Collection: SimMIM Metadata: Epochs: 100 Batch Size: 2048 Results: null - Config: configs/selfsup/simmim/simmim_swin-base_16xb128-coslr-800e_in1k-192.py + Config: configs/selfsup/simmim/simmim_swin-base_16xb128-amp-coslr-800e_in1k-192.py Weights: https://download.openmmlab.com/mmselfsup/1.x/simmim/simmim_swin-base_16xb128-amp-coslr-800e_in1k-192/simmim_swin-base_16xb128-amp-coslr-800e_in1k-192_20220916-a0e931ac.pth Downstream: - Type: Image Classification @@ -61,15 +61,15 @@ Models: Dataset: ImageNet-1k Metrics: Top 1 Accuracy: 83.8 - Config: configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k.py + Config: configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k-192.py Weights: https://download.openmmlab.com/mmselfsup/1.x/simmim/simmim_swin-base_8xb256-amp-coslr-100e_in1k-192/swin-base_ft-8xb256-coslr-100e_in1k/swin-base_ft-8xb256-coslr-100e_in1k_20220829-9cf23aa1.pth - - Name: simmim_swin-large_16xb128-coslr-800e_in1k-192 + - Name: simmim_swin-large_16xb128-amp-coslr-800e_in1k-192 In Collection: SimMIM Metadata: Epochs: 100 Batch Size: 2048 Results: null - Config: configs/selfsup/simmim/simmim_swin-base_16xb128-coslr-800e_in1k-192.py + Config: configs/selfsup/simmim/simmim_swin-base_16xb128-amp-coslr-800e_in1k-192.py Weights: https://download.openmmlab.com/mmselfsup/1.x/simmim/simmim_swin-large_16xb128-amp-coslr-800e_in1k-192/simmim_swin-large_16xb128-amp-coslr-800e_in1k-192_20220916-4ad216d3.pth Downstream: - Type: Image Classification