59 lines
1.5 KiB
Python
59 lines
1.5 KiB
Python
_base_ = [
|
|
'../_base_/datasets/imagenet_bs64_swin_224.py',
|
|
'../_base_/schedules/imagenet_bs1024_adamw_swin.py',
|
|
'../_base_/default_runtime.py'
|
|
]
|
|
|
|
# model settings
|
|
model = dict(
|
|
type='ImageClassifier',
|
|
backbone=dict(
|
|
type='VisionTransformer',
|
|
arch='base',
|
|
img_size=224,
|
|
patch_size=16,
|
|
drop_path_rate=0.1),
|
|
neck=None,
|
|
head=dict(
|
|
type='VisionTransformerClsHead',
|
|
num_classes=1000,
|
|
in_channels=768,
|
|
loss=dict(
|
|
type='LabelSmoothLoss', label_smooth_val=0.1, mode='original'),
|
|
),
|
|
init_cfg=[
|
|
dict(type='TruncNormal', layer='Linear', std=.02),
|
|
dict(type='Constant', layer='LayerNorm', val=1., bias=0.),
|
|
],
|
|
train_cfg=dict(augments=[
|
|
dict(type='Mixup', alpha=0.8),
|
|
dict(type='CutMix', alpha=1.0)
|
|
]))
|
|
|
|
# dataset settings
|
|
train_dataloader = dict(batch_size=128)
|
|
|
|
# schedule settings
|
|
optim_wrapper = dict(
|
|
optimizer=dict(
|
|
type='AdamW',
|
|
lr=1e-4 * 4096 / 256,
|
|
weight_decay=0.3,
|
|
eps=1e-8,
|
|
betas=(0.9, 0.95)),
|
|
paramwise_cfg=dict(
|
|
norm_decay_mult=0.0,
|
|
bias_decay_mult=0.0,
|
|
custom_keys={
|
|
'.cls_token': dict(decay_mult=0.0),
|
|
'.pos_embed': dict(decay_mult=0.0)
|
|
}))
|
|
|
|
# runtime settings
|
|
custom_hooks = [dict(type='EMAHook', momentum=1e-4)]
|
|
|
|
# NOTE: `auto_scale_lr` is for automatically scaling LR
|
|
# based on the actual training batch size.
|
|
# base_batch_size = (32 GPUs) x (128 samples per GPU)
|
|
auto_scale_lr = dict(base_batch_size=4096)
|