mirrors
/
PaddleClas
mirror of https://github.com/PaddlePaddle/PaddleClas.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #45 from littletomatodonkey/fix_shuffle 

fix reader shuffle
pull/46/head
dyning 2020-04-19 12:56:41 +08:00 committed by GitHub
parent f895dfb4ee 8795569003
commit 5736d85b96
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 8 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ppcls/data/reader.py
View File

@ -206,6 +206,8 @@ def mp_reader(params):
check_params(params) check_params(params)
full_lines = get_file_list(params) full_lines = get_file_list(params)
if params["mode"] == "train":
full_lines = shuffle_lines(full_lines, seed=None)
part_num = 1 if 'num_workers' not in params else params['num_workers'] part_num = 1 if 'num_workers' not in params else params['num_workers']
@ -254,11 +256,10 @@ class Reader:
self.batch_ops = create_operators(self.params['mix']) self.batch_ops = create_operators(self.params['mix'])
def __call__(self): def __call__(self):
reader = mp_reader(self.params)
batch_size = int(self.params['batch_size']) // trainers_num batch_size = int(self.params['batch_size']) // trainers_num
def wrapper(): def wrapper():
reader = mp_reader(self.params)
batch = [] batch = []
for idx, sample in enumerate(reader()): for idx, sample in enumerate(reader()):
img, label = sample img, label = sample

12
ppcls/utils/save_load.py
View File

@ -106,22 +106,20 @@ def load_params(exe, prog, path, ignore_params=[]):
fluid.io.set_program_state(prog, state) fluid.io.set_program_state(prog, state)
def init_model(config, program, exe, prefix=""): def init_model(config, program, exe):
""" """
load model from checkpoint or pretrained_model load model from checkpoint or pretrained_model
""" """
checkpoints = config.get('checkpoints') checkpoints = config.get('checkpoints')
if checkpoints: if checkpoints:
path = os.path.join(checkpoints, prefix) fluid.load(program, checkpoints, exe)
fluid.load(program, path, exe) logger.info("Finish initing model from {}".format(checkpoints))
logger.info("Finish initing model from {}".format(path))
return return
pretrained_model = config.get('pretrained_model') pretrained_model = config.get('pretrained_model')
if pretrained_model: if pretrained_model:
path = os.path.join(pretrained_model, prefix) load_params(exe, program, pretrained_model)
load_params(exe, program, path) logger.info("Finish initing model from {}".format(pretrained_model))
logger.info("Finish initing model from {}".format(path))
def save_model(program, model_path, epoch_id, prefix='ppcls'): def save_model(program, model_path, epoch_id, prefix='ppcls'):