43 lines
1.1 KiB
Python
43 lines
1.1 KiB
Python
# optimizer
|
|
# In ClassyVision, the lr is set to 0.003 for bs4096.
|
|
# In this implementation(bs2048), lr = 0.003 / 4096 * (32bs * 64gpus) = 0.0015
|
|
optim_wrapper = dict(
|
|
optimizer=dict(type='AdamW', lr=0.0015, weight_decay=0.3),
|
|
# specific to vit pretrain
|
|
paramwise_cfg=dict(custom_keys={
|
|
'.cls_token': dict(decay_mult=0.0),
|
|
'.pos_embed': dict(decay_mult=0.0)
|
|
}),
|
|
)
|
|
|
|
# learning policy
|
|
param_scheduler = [
|
|
# warm up learning rate scheduler
|
|
dict(
|
|
type='LinearLR',
|
|
start_factor=1e-3,
|
|
by_epoch=True,
|
|
begin=0,
|
|
# about 10000 iterations for ImageNet-1k
|
|
end=15,
|
|
# update by iter
|
|
convert_to_iter_based=True),
|
|
# main learning rate scheduler
|
|
dict(
|
|
type='CosineAnnealingLR',
|
|
T_max=285,
|
|
eta_min=1e-5,
|
|
by_epoch=True,
|
|
begin=15,
|
|
end=300)
|
|
]
|
|
|
|
# train, val, test setting
|
|
train_cfg = dict(by_epoch=True, max_epochs=300, val_interval=1)
|
|
val_cfg = dict()
|
|
test_cfg = dict()
|
|
|
|
# NOTE: `auto_scale_lr` is for automatically scaling LR,
|
|
# based on the actual training batch size.
|
|
auto_scale_lr = dict(base_batch_size=2048)
|