mirror of
https://github.com/open-mmlab/mmengine.git
synced 2025-06-03 21:54:44 +08:00
3e3866c1b9
* Support multiple optimizers * minor refinement * improve unit tests * minor fix * Update unit tests for resuming or saving ckpt for multiple optimizers * refine docstring * refine docstring * fix typo * update docstring * refactor the logic to build multiple optimizers * resolve comments * ParamSchedulers spports multiple optimizers * add optimizer_wrapper * fix comment and docstirng * fix unit test * add unit test * refine docstring * RuntimeInfoHook supports printing multi learning rates * resolve comments * add optimizer_wrapper * fix mypy * fix lint * fix OptimizerWrapperDict docstring and add unit test * rename OptimizerWrapper to OptimWrapper, OptimWrapperDict inherit OptimWrapper, and fix as comment * Fix AmpOptimizerWrapper * rename build_optmizer_wrapper to build_optim_wrapper * refine optimizer wrapper * fix AmpOptimWrapper.step, docstring * resolve confict * rename DefaultOptimConstructor * fix as comment * rename clig grad auguments * refactor optim_wrapper config * fix docstring of DefaultOptimWrapperConstructor fix docstring of DefaultOptimWrapperConstructor * add get_lr method to OptimWrapper and OptimWrapperDict * skip some amp unit test * fix unit test * fix get_lr, get_momentum docstring * refactor get_lr, get_momentum, fix as comment * fix error message Co-authored-by: zhouzaida <zhouzaida@163.com>
116 lines
4.3 KiB
Python
116 lines
4.3 KiB
Python
# Copyright (c) OpenMMLab. All rights reserved.
|
|
from unittest.mock import MagicMock, Mock
|
|
|
|
import torch
|
|
from torch import nn
|
|
|
|
from mmengine.hooks import OptimizerHook
|
|
|
|
|
|
class TestOptimizerHook:
|
|
|
|
def test_after_train_iter(self):
|
|
|
|
class Model(nn.Module):
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.conv1 = nn.Conv2d(
|
|
in_channels=1,
|
|
out_channels=2,
|
|
kernel_size=3,
|
|
stride=1,
|
|
padding=1,
|
|
dilation=1)
|
|
self.conv2 = nn.Conv2d(
|
|
in_channels=2,
|
|
out_channels=2,
|
|
kernel_size=3,
|
|
stride=1,
|
|
padding=1,
|
|
dilation=1)
|
|
self.conv3 = nn.Conv2d(
|
|
in_channels=1,
|
|
out_channels=2,
|
|
kernel_size=3,
|
|
stride=1,
|
|
padding=1,
|
|
dilation=1)
|
|
|
|
def forward(self, x):
|
|
x1 = self.conv1(x)
|
|
x2 = self.conv2(x1)
|
|
return x1, x2
|
|
|
|
model = Model()
|
|
x = torch.rand(1, 1, 3, 3)
|
|
|
|
dummy_runner = MagicMock()
|
|
dummy_runner.optim_wrapper.zero_grad = Mock(return_value=None)
|
|
dummy_runner.optim_wrapper.step = Mock(return_value=None)
|
|
dummy_runner.model = model
|
|
dummy_runner.outputs = dict()
|
|
|
|
dummy_runner.outputs['num_samples'] = 0
|
|
|
|
class DummyLogger():
|
|
|
|
def __init__(self):
|
|
self.msg = ''
|
|
|
|
def log(self, msg=None, **kwargs):
|
|
self.msg += msg
|
|
|
|
dummy_runner.logger = DummyLogger()
|
|
optimizer_hook = OptimizerHook(
|
|
dict(max_norm=2), detect_anomalous_params=True)
|
|
|
|
dummy_runner.outputs['loss'] = model(x)[0].sum()
|
|
|
|
dummy_runner.outputs['loss'].backward = Mock(
|
|
wraps=dummy_runner.outputs['loss'].backward)
|
|
optimizer_hook.detect_anomalous_parameters = Mock(
|
|
wraps=optimizer_hook.detect_anomalous_parameters)
|
|
optimizer_hook.clip_grads = Mock(wraps=optimizer_hook.clip_grads)
|
|
|
|
optimizer_hook.after_train_iter(dummy_runner, 0)
|
|
# assert the parameters of conv2 and conv3 are not in the
|
|
# computational graph which is with x1.sum() as root.
|
|
assert 'conv2.weight' in dummy_runner.logger.msg
|
|
assert 'conv2.bias' in dummy_runner.logger.msg
|
|
assert 'conv3.weight' in dummy_runner.logger.msg
|
|
assert 'conv3.bias' in dummy_runner.logger.msg
|
|
assert 'conv1.weight' not in dummy_runner.logger.msg
|
|
assert 'conv1.bias' not in dummy_runner.logger.msg
|
|
dummy_runner.optim_wrapper.step.assert_called()
|
|
dummy_runner.outputs['loss'].backward.assert_called()
|
|
optimizer_hook.clip_grads.assert_called()
|
|
optimizer_hook.detect_anomalous_parameters.assert_called()
|
|
|
|
dummy_runner.outputs['loss'] = model(x)[1].sum()
|
|
dummy_runner.logger.msg = ''
|
|
optimizer_hook.after_train_iter(dummy_runner, 0)
|
|
# assert the parameters of conv3 are not in the computational graph
|
|
assert 'conv3.weight' in dummy_runner.logger.msg
|
|
assert 'conv3.bias' in dummy_runner.logger.msg
|
|
assert 'conv2.weight' not in dummy_runner.logger.msg
|
|
assert 'conv2.bias' not in dummy_runner.logger.msg
|
|
assert 'conv1.weight' not in dummy_runner.logger.msg
|
|
assert 'conv1.bias' not in dummy_runner.logger.msg
|
|
|
|
# grad_clip is None and detect_anomalous_parameters is False
|
|
optimizer_hook = OptimizerHook(detect_anomalous_params=False)
|
|
optimizer_hook.detect_anomalous_parameters = Mock(
|
|
wraps=optimizer_hook.detect_anomalous_parameters)
|
|
optimizer_hook.clip_grads = Mock(wraps=optimizer_hook.clip_grads)
|
|
dummy_runner.outputs['loss'] = model(x)[0].sum()
|
|
dummy_runner.outputs['loss'].backward = Mock(
|
|
wraps=dummy_runner.outputs['loss'].backward)
|
|
|
|
optimizer_hook.after_train_iter(dummy_runner, 0)
|
|
|
|
dummy_runner.optim_wrapper.step.assert_called()
|
|
dummy_runner.outputs['loss'].backward.assert_called()
|
|
optimizer_hook.clip_grads.assert_not_called()
|
|
optimizer_hook.detect_anomalous_parameters.assert_not_called()
|