mirrors
/
mmselfsup
mirror of https://github.com/open-mmlab/mmselfsup.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Refactor] Refactor benchmark script and update files (#637) 

* update benchmark regression script

* update yml files

* fix lint
pull/681/head
Yixiao Fang 2022-12-30 17:37:21 +08:00 committed by GitHub
parent 90f6292d89
commit 1fd3509f7b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 81 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
.dev_scripts/benchmark_regression/models.yml
View File

@ -1,35 +1,55 @@
- Name: mae_vit-base-p16_8xb512-coslr-400e_in1k
ClsConfig: configs/benchmarks/classification/imagenet/vit-base-p16_ft-8xb128-coslr-100e_in1k.py
- Name: mae_vit-base-p16_8xb512-amp-coslr-300e_in1k
Task: Linear Evaluation
Cycle: month
- Name: mocov3_vit-small-p16_16xb256-fp16-coslr-300e_in1k-224
ClsConfig: configs/benchmarks/classification/imagenet/vit-small-p16_linear-8xb128-coslr-90e_in1k.py
- Name: mocov3_vit-small-p16_16xb256-amp-coslr-300e_in1k
Task: Linear Evaluation
Cycle: month
- Name: simmim_swin-base_8xb256-coslr-100e_in1k-192
ClsConfig: configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k.py
- Name: simmim_swin-base_8xb256-amp-coslr-100e_in1k-192
Task: Fine-tuning
Cycle: month
- Name: simclr_resnet50_8xb32-coslr-200e_in1k
ClsConfig: configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py
Cycle: month
Task: Linear Evaluation
Cycle: quarter
- Name: byol_resnet50_8xb32-accum16-coslr-200e_in1k
ClsConfig: configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py
Cycle: month
- Name: byol_resnet50_16xb256-coslr-200e_in1k
Task: Linear Evaluation
Cycle: quarter
- Name: mocov2_resnet50_8xb32-coslr-200e_in1k
ClsConfig: configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py
Task: Linear Evaluation
Cycle: quarter
- Name: simsiam_resnet50_8xb32-coslr-100e_in1k
ClsConfig: configs/benchmarks/classification/imagenet/resnet50_linear-8xb512-coslr-90e_in1k.py
Task: Linear Evaluation
Cycle: quarter
- Name: maskfeat_vit-base-p16_8xb256-amp-coslr-300e_in1k
Task: Fine-tuning
Cycle: quarter
- Name: beit_vit-base-p16_8xb256-amp-coslr-300e_in1k
Task: Fine-tuning
Cycle: quarter
- Name: beitv2_vit-base-p16_8xb256-amp-coslr-300e_in1k
Task: Fine-tuning
Cycle: quarter
- Name: milan_vit-base-p16_16xb256-amp-coslr-400e_in1k
Task: Linear Evaluation
Cycle: quarter
- Name: densecl_resnet50_8xb32-coslr-200e_in1k
ClsConfig: configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-steplr-100e_in1k.py
Cycle: quarter
Task: Linear Evaluation
Cycle: half-year
- Name: swav_resnet50_8xb32-mcrop-2-6-coslr-200e_in1k-224-96
ClsConfig: configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-coslr-100e_in1k.py
Cycle: quarter
Task: Linear Evaluation
Cycle: half-year
- Name: cae_vit-base-p16_8xb256-fp16-coslr-300e_in1k
Task: Fine-tuning
Cycle: half-year

41
.dev_scripts/benchmark_regression/pretrain_with_cls.py
View File

@ -113,11 +113,21 @@ def get_gpu_number(model_info):
matches = re.match(r'.*[-_](\d+)xb(\d+).*', config)
if matches is None:
raise ValueError(
'Cannot get gpu numbers from the config name {config}')
f'Cannot get gpu numbers from the config name {config}')
gpus = int(matches.groups()[0])
return gpus
def get_pretrain_epoch(model_info):
config = osp.basename(model_info.config)
matches = re.match(r'.*[-_](\d+)e[-_].*', config)
if matches is None:
raise ValueError(
f'Cannot get epoch setting from the config name {config}')
epoch = int(matches.groups()[0])
return epoch
def create_train_job_batch(commands, model_info, args, port, script_name):
fname = model_info.name
@ -137,7 +147,6 @@ def create_train_job_batch(commands, model_info, args, port, script_name):
quota_cfg = ''
launcher = 'none' if args.local else 'slurm'
job_name = f'{args.job_name}_{fname}'
job_script = (f'#!/bin/bash\n'
f'srun -p {args.partition} '
@ -158,14 +167,23 @@ def create_train_job_batch(commands, model_info, args, port, script_name):
commands.append(f'echo "{config}"')
# downstream classification task
cls_config = getattr(model_info, 'cls_config', None)
cls_config = None
task = getattr(model_info, 'task', None)
if task is not None:
for downstream in model_info.data['Downstream']:
if task == downstream['Results'][0]['Task']:
cls_config = downstream['Config']
break
else:
cls_config = None
if cls_config:
fname = model_info.name
gpus = get_gpu_number(model_info)
gpus_per_node = min(gpus, 8)
cls_config_path = Path(model_info.cls_config)
cls_config_path = Path(cls_config)
assert cls_config_path.exists(), f'"{fname}": {cls_config} not found.'
job_name = f'{args.job_name}_{fname}'
@ -178,13 +196,11 @@ def create_train_job_batch(commands, model_info, args, port, script_name):
if args.quotatype is not None:
srun_args = srun_args.join(f'--quotatype {args.quotatype}')
# get pretrain weights
ckpt_path_file = work_dir / 'last_checkpoint'
with open(ckpt_path_file, 'r') as f:
ckpt = f.readlines()[0]
# get pretrain weights path
epoch = get_pretrain_epoch(model_info)
ckpt = work_dir / f'epoch_{epoch}.pth'
launcher = 'none' if args.local else 'slurm'
cls_job_script = (
f'\n'
f'mim train mmcls {cls_config} '
@ -365,7 +381,10 @@ def summary(models, args):
if len(val_logs) == 0:
continue
expect_metrics = model_info.results[0].metrics
for downstream in model_info.data['Downstream']:
if model_info.task == downstream['Results'][0]['Task']:
expect_metrics = downstream['Results'][0]['Metrics']
break
# extract metrics
summary = {'log_file': log_file}
@ -408,7 +427,7 @@ def main():
name = item['Name']
model_info = all_models[item['Name']]
model_info.cycle = item.get('Cycle', None)
model_info.cls_config = item.get('ClsConfig', None)
model_info.task = item.get('Task', None)
cycle = getattr(model_info, 'cycle', 'month')
cycle_level = CYCLE_LEVELS.index(cycle)
if cycle_level in args.range:

2
configs/selfsup/cae/metafile.yml
View File

@ -30,6 +30,6 @@ Models:
- Task: Fine-tuning
Dataset: ImageNet-1k
Metrics:
Top 1 Accuracy: 60.8
Top 1 Accuracy: 83.2
Config: configs/benchmarks/classification/imagenet/vit-base-p16_ft-8xb128-coslr-100e-rpe_in1k.py
Weights: https://download.openmmlab.com/mmselfsup/1.x/cae/cae_vit-base-p16_16xb128-fp16-coslr-300e_in1k/vit-base-p16_ft-8xb128-coslr-100e-rpe_in1k/vit-base-p16_ft-8xb128-coslr-100e-rpe_in1k_20220825-f3d234cd.pth

2
configs/selfsup/mae/metafile.yml
View File

@ -42,7 +42,7 @@ Models:
Metrics:
Top 1 Accuracy: 83.1
Config: configs/benchmarks/classification/imagenet/vit-base-p16_ft-8xb128-coslr-100e_in1k.py
- Name: mae_vit-base-p16_8xb512-amp-coslr-400e_in1k
- Name: mae_vit-base-p16_8xb512-coslr-400e_in1k
In Collection: MAE
Metadata:
Epochs: 400

4
configs/selfsup/milan/metafile.yaml
View File

@ -31,14 +31,14 @@ Models:
Dataset: ImageNet-1k
Metrics:
Top 1 Accuracy: 85.3
Config: configs/selfsup/milan/classification/vit-base-p16_ft-8xb128-coslr-100e_in1k-milan.py
Config: configs/selfsup/milan/classification/vit-base-p16_ft-8xb128-coslr-100e_in1k.py
Weights: https://download.openmmlab.com/mmselfsup/1.x/milan/milan_vit-base-p16_16xb256-amp-coslr-400e_in1k/vit-base-p16_ft-8xb128-coslr-100e_in1k/vit-base-p16_ft-8xb128-coslr-100e_in1k-milan_20221129-74ac94fa.pth
- Type: Image Classification
Metadata:
Epochs: 100
Batch Size: 16384
Results:
- Task: Linear-probing
- Task: Linear Evaluation
Dataset: ImageNet-1k
Metrics:
Top 1 Accuracy: 78.9

4
configs/selfsup/mocov3/metafile.yml
View File

@ -93,7 +93,7 @@ Models:
Dataset: ImageNet-1k
Metrics:
Top 1 Accuracy: 73.6
Config: configs/benchmarks/classification/imagenet/vit-small-p16_linear-8xb128-coslr-90e.py
Config: configs/benchmarks/classification/imagenet/vit-small-p16_linear-8xb128-coslr-90e_in1k.py
Weights: https://download.openmmlab.com/mmselfsup/1.x/mocov3/mocov3_vit-small-p16_16xb256-amp-coslr-300e_in1k/vit-small-p16_linear-8xb128-coslr-90e_in1k/vit-small-p16_linear-8xb128-coslr-90e_in1k_20220826-376674ef.pth
- Name: mocov3_vit-base-p16_16xb256-amp-coslr-300e_in1k
In Collection: MoCoV3
@ -113,7 +113,7 @@ Models:
Dataset: ImageNet-1k
Metrics:
Top 1 Accuracy: 76.9
Config: configs/benchmarks/classification/imagenet/vit-base-p16_linear-8xb128-coslr-90e.py
Config: configs/benchmarks/classification/imagenet/vit-base-p16_linear-8xb128-coslr-90e_in1k.py
Weights: https://download.openmmlab.com/mmselfsup/1.x/mocov3/mocov3_vit-base-p16_16xb256-amp-coslr-300e_in1k/vit-base-p16_linear-8xb128-coslr-90e_in1k/vit-base-p16_linear-8xb128-coslr-90e_in1k_20220826-83be7758.pth
- Type: Image Classification
Metadata:

16
configs/selfsup/simmim/metafile.yml
View File

@ -13,13 +13,13 @@ Collections:
README: configs/selfsup/simmim/README.md
Models:
- Name: simmim_swin-base_16xb128-coslr-100e_in1k-192
- Name: simmim_swin-base_8xb256-amp-coslr-100e_in1k-192
In Collection: SimMIM
Metadata:
Epochs: 100
Batch Size: 2048
Results: null
Config: configs/selfsup/simmim/simmim_swin-base_16xb128-coslr-100e_in1k-192.py
Config: configs/selfsup/simmim/simmim_swin-base_8xb256-amp-coslr-100e_in1k-192.py
Weights: https://download.openmmlab.com/mmselfsup/1.x/simmim/simmim_swin-base_8xb256-amp-coslr-100e_in1k-192/simmim_swin-base_8xb256-amp-coslr-100e_in1k-192_20220829-0e15782d.pth
Downstream:
- Type: Image Classification
@ -31,7 +31,7 @@ Models:
Dataset: ImageNet-1k
Metrics:
Top 1 Accuracy: 82.7
Config: configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k.py
Config: configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k-192.py
Weights: https://download.openmmlab.com/mmselfsup/1.x/simmim/simmim_swin-base_8xb256-amp-coslr-100e_in1k-192/swin-base_ft-8xb256-coslr-100e_in1k/swin-base_ft-8xb256-coslr-100e_in1k_20220829-9cf23aa1.pth
- Type: Image Classification
Metadata:
@ -43,13 +43,13 @@ Models:
Metrics:
Top 1 Accuracy: 83.5
Config: configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k-224.py
- Name: simmim_swin-base_16xb128-coslr-800e_in1k-192
- Name: simmim_swin-base_16xb128-amp-coslr-800e_in1k-192
In Collection: SimMIM
Metadata:
Epochs: 100
Batch Size: 2048
Results: null
Config: configs/selfsup/simmim/simmim_swin-base_16xb128-coslr-800e_in1k-192.py
Config: configs/selfsup/simmim/simmim_swin-base_16xb128-amp-coslr-800e_in1k-192.py
Weights: https://download.openmmlab.com/mmselfsup/1.x/simmim/simmim_swin-base_16xb128-amp-coslr-800e_in1k-192/simmim_swin-base_16xb128-amp-coslr-800e_in1k-192_20220916-a0e931ac.pth
Downstream:
- Type: Image Classification
@ -61,15 +61,15 @@ Models:
Dataset: ImageNet-1k
Metrics:
Top 1 Accuracy: 83.8
Config: configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k.py
Config: configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k-192.py
Weights: https://download.openmmlab.com/mmselfsup/1.x/simmim/simmim_swin-base_8xb256-amp-coslr-100e_in1k-192/swin-base_ft-8xb256-coslr-100e_in1k/swin-base_ft-8xb256-coslr-100e_in1k_20220829-9cf23aa1.pth
- Name: simmim_swin-large_16xb128-coslr-800e_in1k-192
- Name: simmim_swin-large_16xb128-amp-coslr-800e_in1k-192
In Collection: SimMIM
Metadata:
Epochs: 100
Batch Size: 2048
Results: null
Config: configs/selfsup/simmim/simmim_swin-base_16xb128-coslr-800e_in1k-192.py
Config: configs/selfsup/simmim/simmim_swin-base_16xb128-amp-coslr-800e_in1k-192.py
Weights: https://download.openmmlab.com/mmselfsup/1.x/simmim/simmim_swin-large_16xb128-amp-coslr-800e_in1k-192/simmim_swin-large_16xb128-amp-coslr-800e_in1k-192_20220916-4ad216d3.pth
Downstream:
- Type: Image Classification