[Fix] Fix the resume of iteration (#1471)

This commit is contained in:
Zhihao Lin 2024-01-11 10:47:05 +08:00 committed by GitHub
parent 4a50213c69
commit b51bf60964
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -271,6 +271,14 @@ class IterBasedTrainLoop(BaseLoop):
# In iteration-based training loop, we treat the whole training process
# as a big epoch and execute the corresponding hook.
self.runner.call_hook('before_train_epoch')
if self._iter > 0:
print_log(
f'Advance dataloader {self._iter} steps to skip data '
'that has already been trained',
logger='current',
level=logging.WARNING)
for _ in range(self._iter):
next(self.dataloader_iterator)
while self._iter < self._max_iters and not self.stop_training:
self.runner.model.train()