Remove hook `torch.nan_to_num(x)` (#8826)

* Remove hook `torch.nan_to_num(x)`

Observed erratic training behavior (green line) with the nan_to_num hook in classifier branch. I'm going to remove it from master.

* Update train.py
pull/8827/head
Glenn Jocher 2022-08-01 21:39:04 +02:00 committed by GitHub
parent 59578f2782
commit f3c78a387e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 1 deletions

View File

@ -131,7 +131,7 @@ def train(hyp, opt, device, callbacks): # hyp is path/to/hyp.yaml or hyp dictio
freeze = [f'model.{x}.' for x in (freeze if len(freeze) > 1 else range(freeze[0]))] # layers to freeze
for k, v in model.named_parameters():
v.requires_grad = True # train all layers
v.register_hook(lambda x: torch.nan_to_num(x)) # NaN to 0.0
# v.register_hook(lambda x: torch.nan_to_num(x)) # NaN to 0 (commented for erratic training results)
if any(x in k for x in freeze):
LOGGER.info(f'freezing {k}')
v.requires_grad = False