[Fix] CheckpointHook should check whether file exists before removing it (#1198)

pull/1201/head
Zaida Zhou 2023-06-12 22:35:07 +08:00 committed by GitHub
parent 94e7a3bb57
commit 2c35c164d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 4 deletions

View File

@ -518,15 +518,19 @@ class CheckpointHook(Hook):
runner.message_hub.update_info(best_score_key, best_score)
if best_ckpt_path and is_main_process():
is_removed = False
if self.file_backend.isfile(best_ckpt_path):
self.file_backend.remove(best_ckpt_path)
else:
is_removed = True
elif self.file_backend.isdir(best_ckpt_path):
# checkpoints saved by deepspeed are directories
self.file_backend.rmtree(best_ckpt_path)
is_removed = True
runner.logger.info(
f'The previous best checkpoint {best_ckpt_path} '
'is removed')
if is_removed:
runner.logger.info(
f'The previous best checkpoint {best_ckpt_path} '
'is removed')
best_ckpt_name = f'best_{key_indicator}_{ckpt_filename}'
# Replace illegal characters for filename with `_`