mirrors/pytorch-image-models
1
0
Fork 0
mirror of https://github.com/huggingface/pytorch-image-models.git synced 2025-06-03 15:01:08 +08:00
Code Issues Projects Releases Wiki Activity

Need to init momentum with correct dtype

Browse Source
This commit is contained in:
Ross Wightman 2024-11-04 09:36:00 -08:00 committed by Ross Wightman
parent 484a88f4b4
commit 19090ea966
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
timm/optim/adafactor_bv.py
View File

@ -146,7 +146,7 @@ class AdafactorBigVision(Optimizer):
state['exp_avg_sq'] = torch.zeros_like(p.grad, memory_format=torch.preserve_format)
if self.defaults['momentum'] is not None:
state['exp_avg'] = torch.zeros_like(p.grad, dtype=torch.bfloat16)
state['exp_avg'] = torch.zeros_like(p.grad, dtype=self.defaults['momentum_dtype'])
state_steps.append(state['step'])
exp_avg_sq_rs.append(state.get('exp_avg_sq_r', None))