Merge branch 'limengzhang/fix_optim_wrapper' into 'refactor_dev'

[Fix] Remove OptimizerHook and Add it in OptimWrapper

See merge request openmmlab-enterprise/openmmlab-ce/mmsegmentation!47

configs/_base_/schedules/schedule_160k.py

@@ -1,6 +1,6 @@
 # optimizer
 optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005)
-optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer)
+optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer, clip_grad=None)
 # learning policy
 param_scheduler = [
     dict(
@@ -17,7 +17,6 @@ train_cfg = dict(
 val_cfg = dict(type='ValLoop')
 test_cfg = dict(type='TestLoop')
 default_hooks = dict(
-    optimizer=dict(type='OptimizerHook', grad_clip=None),
     timer=dict(type='IterTimerHook'),
     logger=dict(type='LoggerHook', interval=50),
     param_scheduler=dict(type='ParamSchedulerHook'),

configs/_base_/schedules/schedule_20k.py

@@ -1,6 +1,6 @@
 # optimizer
 optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005)
-optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer)
+optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer, clip_grad=None)
 # learning policy
 param_scheduler = [
     dict(
@@ -16,7 +16,6 @@ train_cfg = dict(type='IterBasedTrainLoop', max_iters=20000, val_interval=2000)
 val_cfg = dict(type='ValLoop')
 test_cfg = dict(type='TestLoop')
 default_hooks = dict(
-    optimizer=dict(type='OptimizerHook', grad_clip=None),
     timer=dict(type='IterTimerHook'),
     logger=dict(type='LoggerHook', interval=50),
     param_scheduler=dict(type='ParamSchedulerHook'),

configs/_base_/schedules/schedule_320k.py

@@ -1,6 +1,6 @@
 # optimizer
 optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005)
-optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer)
+optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer, clip_grad=None)
 # learning policy
 param_scheduler = [
     dict(
@@ -17,7 +17,6 @@ train_cfg = dict(
 val_cfg = dict(type='ValLoop')
 test_cfg = dict(type='TestLoop')
 default_hooks = dict(
-    optimizer=dict(type='OptimizerHook', grad_clip=None),
     timer=dict(type='IterTimerHook'),
     logger=dict(type='LoggerHook', interval=50),
     param_scheduler=dict(type='ParamSchedulerHook'),

configs/_base_/schedules/schedule_40k.py

@@ -1,6 +1,6 @@
 # optimizer
 optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005)
-optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer)
+optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer, clip_grad=None)
 # learning policy
 param_scheduler = [
     dict(
@@ -16,7 +16,6 @@ train_cfg = dict(type='IterBasedTrainLoop', max_iters=40000, val_interval=4000)
 val_cfg = dict(type='ValLoop')
 test_cfg = dict(type='TestLoop')
 default_hooks = dict(
-    optimizer=dict(type='OptimizerHook', grad_clip=None),
     timer=dict(type='IterTimerHook'),
     logger=dict(type='LoggerHook', interval=50),
     param_scheduler=dict(type='ParamSchedulerHook'),

configs/_base_/schedules/schedule_80k.py

@@ -1,6 +1,6 @@
 # optimizer
 optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005)
-optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer)
+optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer, clip_grad=None)
 # learning policy
 param_scheduler = [
     dict(
@@ -16,7 +16,6 @@ train_cfg = dict(type='IterBasedTrainLoop', max_iters=80000, val_interval=8000)
 val_cfg = dict(type='ValLoop')
 test_cfg = dict(type='TestLoop')
 default_hooks = dict(
-    optimizer=dict(type='OptimizerHook', grad_clip=None),
     timer=dict(type='IterTimerHook'),
     logger=dict(type='LoggerHook', interval=50),
     param_scheduler=dict(type='ParamSchedulerHook'),

configs/beit/upernet_beit-large_fp16_8x1_640x640_160k_ade20k.py

@@ -31,10 +31,12 @@ optimizer = dict(
     weight_decay=0.05)
 
 optim_wrapper = dict(
-    type='OptimWrapper',
+    _delete_=True,
+    type='AmpOptimWrapper',
     optimizer=optimizer,
     constructor='LayerDecayOptimizerConstructor',
-    paramwise_cfg=dict(num_layers=24, layer_decay_rate=0.95))
+    paramwise_cfg=dict(num_layers=24, layer_decay_rate=0.95),
+    accumulative_counts=2)
 
 param_scheduler = [
     dict(
@@ -52,7 +54,3 @@ param_scheduler = [
 train_dataloader = dict(batch_size=1)
 val_dataloader = dict(batch_size=1)
 test_dataloader = val_dataloader
-optimizer_config = dict(
-    type='GradientCumulativeFp16OptimizerHook', cumulative_iters=2)
-
-fp16 = dict()

configs/bisenetv2/bisenetv2_fcn_fp16_4x4_1024x1024_160k_cityscapes.py

@@ -1,5 +1,7 @@
 _base_ = './bisenetv2_fcn_4x4_1024x1024_160k_cityscapes.py'
-# fp16 settings
-default_hooks = dict(optimizer=dict(type='Fp16OptimizerHook', loss_scale=512.))
-# fp16 placeholder
-fp16 = dict()
+optimizer = dict(type='SGD', lr=0.05, momentum=0.9, weight_decay=0.0005)
+optim_wrapper = dict(
+    _delete_=True,
+    type='AmpOptimWrapper',
+    optimizer=optimizer,
+    loss_scale=512.)

configs/convnext/upernet_convnext_base_fp16_512x512_160k_ade20k.py

@@ -19,14 +19,16 @@ optimizer = dict(
     weight_decay=0.05)
 
 optim_wrapper = dict(
-    type='OptimWrapper',
+    _delete_=True,
+    type='AmpOptimWrapper',
     optimizer=optimizer,
     paramwise_cfg={
         'decay_rate': 0.9,
         'decay_type': 'stage_wise',
         'num_layers': 12
     },
-    constructor='LearningRateDecayOptimizerConstructor')
+    constructor='LearningRateDecayOptimizerConstructor',
+    loss_scale='dynamic')
 
 param_scheduler = [
     dict(
@@ -45,8 +47,3 @@ param_scheduler = [
 train_dataloader = dict(batch_size=2)
 val_dataloader = dict(batch_size=1)
 test_dataloader = val_dataloader
-# fp16 settings
-default_hooks = dict(
-    optimizer=dict(type='Fp16OptimizerHook', loss_scale='dynamic'))
-# fp16 placeholder
-fp16 = dict()

configs/convnext/upernet_convnext_base_fp16_640x640_160k_ade20k.py

@@ -34,14 +34,16 @@ optimizer = dict(
     weight_decay=0.05)
 
 optim_wrapper = dict(
-    type='OptimWrapper',
+    _delete_=True,
+    type='AmpOptimWrapper',
     optimizer=optimizer,
     paramwise_cfg={
         'decay_rate': 0.9,
         'decay_type': 'stage_wise',
         'num_layers': 12
     },
-    constructor='LearningRateDecayOptimizerConstructor')
+    constructor='LearningRateDecayOptimizerConstructor',
+    loss_scale='dynamic')
 
 param_scheduler = [
     dict(
@@ -60,8 +62,3 @@ param_scheduler = [
 train_dataloader = dict(batch_size=2)
 val_dataloader = dict(batch_size=1)
 test_dataloader = val_dataloader
-# fp16 settings
-default_hooks = dict(
-    optimizer=dict(type='Fp16OptimizerHook', loss_scale='dynamic'))
-# fp16 placeholder
-fp16 = dict()

configs/convnext/upernet_convnext_large_fp16_640x640_160k_ade20k.py

@@ -34,14 +34,16 @@ optimizer = dict(
     weight_decay=0.05)
 
 optim_wrapper = dict(
-    type='OptimWrapper',
+    _delete_=True,
+    type='AmpOptimWrapper',
     optimizer=optimizer,
     paramwise_cfg={
         'decay_rate': 0.9,
         'decay_type': 'stage_wise',
         'num_layers': 12
     },
-    constructor='LearningRateDecayOptimizerConstructor')
+    constructor='LearningRateDecayOptimizerConstructor',
+    loss_scale='dynamic')
 
 param_scheduler = [
     dict(
@@ -60,8 +62,3 @@ param_scheduler = [
 train_dataloader = dict(batch_size=2)
 val_dataloader = dict(batch_size=1)
 test_dataloader = val_dataloader
-# fp16 settings
-default_hooks = dict(
-    optimizer=dict(type='Fp16OptimizerHook', loss_scale='dynamic'))
-# fp16 placeholder
-fp16 = dict()

configs/convnext/upernet_convnext_small_fp16_512x512_160k_ade20k.py

@@ -33,14 +33,16 @@ optimizer = dict(
     weight_decay=0.05)
 
 optim_wrapper = dict(
-    type='OptimWrapper',
+    _delete_=True,
+    type='AmpOptimWrapper',
     optimizer=optimizer,
     paramwise_cfg={
         'decay_rate': 0.9,
         'decay_type': 'stage_wise',
         'num_layers': 12
     },
-    constructor='LearningRateDecayOptimizerConstructor')
+    constructor='LearningRateDecayOptimizerConstructor',
+    loss_scale='dynamic')
 
 param_scheduler = [
     dict(
@@ -59,8 +61,3 @@ param_scheduler = [
 train_dataloader = dict(batch_size=2)
 val_dataloader = dict(batch_size=1)
 test_dataloader = val_dataloader
-# fp16 settings
-default_hooks = dict(
-    optimizer=dict(type='Fp16OptimizerHook', loss_scale='dynamic'))
-# fp16 placeholder
-fp16 = dict()

configs/convnext/upernet_convnext_tiny_fp16_512x512_160k_ade20k.py

@@ -33,14 +33,16 @@ optimizer = dict(
     weight_decay=0.05)
 
 optim_wrapper = dict(
-    type='OptimWrapper',
+    _delete_=True,
+    type='AmpOptimWrapper',
     optimizer=optimizer,
     paramwise_cfg={
         'decay_rate': 0.9,
         'decay_type': 'stage_wise',
         'num_layers': 6
     },
-    constructor='LearningRateDecayOptimizerConstructor')
+    constructor='LearningRateDecayOptimizerConstructor',
+    loss_scale='dynamic')
 
 param_scheduler = [
     dict(
@@ -59,8 +61,3 @@ param_scheduler = [
 train_dataloader = dict(batch_size=2)
 val_dataloader = dict(batch_size=1)
 test_dataloader = val_dataloader
-# fp16 settings
-default_hooks = dict(
-    optimizer=dict(type='Fp16OptimizerHook', loss_scale='dynamic'))
-# fp16 placeholder
-fp16 = dict()

configs/convnext/upernet_convnext_xlarge_fp16_640x640_160k_ade20k.py

@@ -34,14 +34,16 @@ optimizer = dict(
     weight_decay=0.05)
 
 optim_wrapper = dict(
-    type='OptimWrapper',
+    _delete_=True,
+    type='AmpOptimWrapper',
     optimizer=optimizer,
     paramwise_cfg={
         'decay_rate': 0.9,
         'decay_type': 'stage_wise',
         'num_layers': 12
     },
-    constructor='LearningRateDecayOptimizerConstructor')
+    constructor='LearningRateDecayOptimizerConstructor',
+    loss_scale='dynamic')
 
 param_scheduler = [
     dict(
@@ -60,8 +62,3 @@ param_scheduler = [
 train_dataloader = dict(batch_size=2)
 val_dataloader = dict(batch_size=1)
 test_dataloader = val_dataloader
-# fp16 settings
-default_hooks = dict(
-    optimizer=dict(type='Fp16OptimizerHook', loss_scale='dynamic'))
-# fp16 placeholder
-fp16 = dict()

configs/deeplabv3/deeplabv3_r101-d8_fp16_512x1024_80k_cityscapes.py

@@ -1,5 +1,7 @@
 _base_ = './deeplabv3_r101-d8_512x1024_80k_cityscapes.py'
-# fp16 settings
-default_hooks = dict(optimizer=dict(type='Fp16OptimizerHook', loss_scale=512.))
-# fp16 placeholder
-fp16 = dict()
+optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005)
+optim_wrapper = dict(
+    _delete_=True,
+    type='AmpOptimWrapper',
+    optimizer=optimizer,
+    loss_scale=512.)

configs/deeplabv3plus/deeplabv3plus_r101-d8_fp16_512x1024_80k_cityscapes.py

@@ -1,5 +1,7 @@
 _base_ = './deeplabv3plus_r101-d8_512x1024_80k_cityscapes.py'
-# fp16 settings
-default_hooks = dict(optimizer=dict(type='Fp16OptimizerHook', loss_scale=512.))
-# fp16 placeholder
-fp16 = dict()
+optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005)
+optim_wrapper = dict(
+    _delete_=True,
+    type='AmpOptimWrapper',
+    optimizer=optimizer,
+    loss_scale=512.)

configs/fcn/fcn_r101-d8_fp16_512x1024_80k_cityscapes.py

@@ -1,5 +1,7 @@
 _base_ = './fcn_r101-d8_512x1024_80k_cityscapes.py'
-# fp16 settings
-default_hooks = dict(optimizer=dict(type='Fp16OptimizerHook', loss_scale=512.))
-# fp16 placeholder
-fp16 = dict()
+optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005)
+optim_wrapper = dict(
+    _delete_=True,
+    type='AmpOptimWrapper',
+    optimizer=optimizer,
+    loss_scale=512.)

configs/knet/knet_s3_deeplabv3_r50-d8_8x2_512x512_adamw_80k_ade20k.py

@@ -81,12 +81,11 @@ model = dict(
 
 # optimizer
 optimizer = dict(_delete_=True, type='AdamW', lr=0.0001, weight_decay=0.0005)
-optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer)
-default_hooks = dict(
-    optimizer=dict(
-        _delete_=True,
-        type='OptimizerHook',
-        grad_clip=dict(max_norm=1, norm_type=2)))
+optim_wrapper = dict(
+    _delete_=True,
+    type='OptimWrapper',
+    optimizer=optimizer,
+    clip_grad=dict(max_norm=1, norm_type=2))
 # learning policy
 param_scheduler = [
     dict(

configs/knet/knet_s3_fcn_r50-d8_8x2_512x512_adamw_80k_ade20k.py

@@ -81,12 +81,12 @@ model = dict(
     test_cfg=dict(mode='whole'))
 # optimizer
 optimizer = dict(_delete_=True, type='AdamW', lr=0.0001, weight_decay=0.0005)
-optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer)
-default_hooks = dict(
-    optimizer=dict(
-        _delete_=True,
-        type='OptimizerHook',
-        grad_clip=dict(max_norm=1, norm_type=2)))
+optim_wrapper = dict(
+    _delete_=True,
+    type='OptimWrapper',
+    optimizer=optimizer,
+    clip_grad=dict(max_norm=1, norm_type=2))
+
 # learning policy
 param_scheduler = [
     dict(

configs/knet/knet_s3_pspnet_r50-d8_8x2_512x512_adamw_80k_ade20k.py

@@ -80,12 +80,11 @@ model = dict(
     test_cfg=dict(mode='whole'))
 # optimizer
 optimizer = dict(_delete_=True, type='AdamW', lr=0.0001, weight_decay=0.0005)
-optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer)
-default_hooks = dict(
-    optimizer=dict(
-        _delete_=True,
-        type='OptimizerHook',
-        grad_clip=dict(max_norm=1, norm_type=2)))
+optim_wrapper = dict(
+    _delete_=True,
+    type='OptimWrapper',
+    optimizer=optimizer,
+    clip_grad=dict(max_norm=1, norm_type=2))
 # learning policy
 param_scheduler = [
     dict(

configs/knet/knet_s3_upernet_r50-d8_8x2_512x512_adamw_80k_ade20k.py

@@ -81,12 +81,11 @@ model = dict(
     test_cfg=dict(mode='whole'))
 # optimizer
 optimizer = dict(_delete_=True, type='AdamW', lr=0.0001, weight_decay=0.0005)
-optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer)
-default_hooks = dict(
-    optimizer=dict(
-        _delete_=True,
-        type='OptimizerHook',
-        grad_clip=dict(max_norm=1, norm_type=2)))
+optim_wrapper = dict(
+    _delete_=True,
+    type='OptimWrapper',
+    optimizer=optimizer,
+    clip_grad=dict(max_norm=1, norm_type=2))
 # learning policy
 param_scheduler = [
     dict(

configs/knet/knet_s3_upernet_swin-t_8x2_512x512_adamw_80k_ade20k.py

@@ -39,6 +39,7 @@ optimizer = dict(
     weight_decay=0.0005)
 
 optim_wrapper = dict(
+    _delete_=True,
     type='OptimWrapper',
     optimizer=optimizer,
     paramwise_cfg=dict(
@@ -46,13 +47,9 @@ optim_wrapper = dict(
             'absolute_pos_embed': dict(decay_mult=0.),
             'relative_position_bias_table': dict(decay_mult=0.),
             'norm': dict(decay_mult=0.)
-        }))
+        }),
+    clip_grad=dict(max_norm=1, norm_type=2))
 
-default_hooks = dict(
-    optimizer=dict(
-        _delete_=True,
-        type='OptimizerHook',
-        grad_clip=dict(max_norm=1, norm_type=2)))
 # learning policy
 param_scheduler = [
     dict(

configs/pspnet/pspnet_r101-d8_fp16_512x1024_80k_cityscapes.py

@@ -1,5 +1,7 @@
 _base_ = './pspnet_r101-d8_512x1024_80k_cityscapes.py'
-# fp16 settings
-default_hooks = dict(optimizer=dict(type='Fp16OptimizerHook', loss_scale=512.))
-# fp16 placeholder
-fp16 = dict()
+optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005)
+optim_wrapper = dict(
+    _delete_=True,
+    type='AmpOptimWrapper',
+    optimizer=optimizer,
+    loss_scale=512.)

configs/pspnet/pspnet_r50-d32_rsb-pretrain_512x1024_adamw_80k_cityscapes.py

@@ -16,12 +16,11 @@ model = dict(
         strides=(1, 2, 2, 2)))
 
 optimizer = dict(_delete_=True, type='AdamW', lr=0.0005, weight_decay=0.05)
-optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer)
-default_hooks = dict(
-    optimizer=dict(
-        _delete_=True,
-        type='OptimizerHook',
-        grad_clip=dict(max_norm=1, norm_type=2)))
+optim_wrapper = dict(
+    _delete_=True,
+    type='OptimWrapper',
+    optimizer=optimizer,
+    clip_grad=dict(max_norm=1, norm_type=2))
 # learning policy
 param_scheduler = [
     dict(

configs/pspnet/pspnet_r50-d8_rsb-pretrain_512x1024_adamw_80k_cityscapes.py

@@ -14,12 +14,11 @@ model = dict(
             type='Pretrained', prefix='backbone.', checkpoint=checkpoint)))
 
 optimizer = dict(_delete_=True, type='AdamW', lr=0.0005, weight_decay=0.05)
-optim_wrapper = dict(type='OptimWrapper', optimizer=optimizer)
-default_hooks = dict(
-    optimizer=dict(
-        _delete_=True,
-        type='OptimizerHook',
-        grad_clip=dict(max_norm=1, norm_type=2)))
+optim_wrapper = dict(
+    _delete_=True,
+    type='OptimWrapper',
+    optimizer=optimizer,
+    clip_grad=dict(max_norm=1, norm_type=2))
 # learning policy
 param_scheduler = [
     dict(