add acc benchmark
littletomatodonkey
parent
fa24f82344
commit
ebadacd7f8
|
|
@ -158,14 +158,22 @@ def _decompress(fname):
|
|||
else:
|
||||
raise TypeError("Unsupport compress file type {}".format(fname))
|
||||
|
||||
for f in os.listdir(fpath_tmp):
|
||||
src_dir = os.path.join(fpath_tmp, f)
|
||||
dst_dir = os.path.join(fpath, f)
|
||||
_move_and_merge_tree(src_dir, dst_dir)
|
||||
fs = os.listdir(fpath_tmp)
|
||||
assert len(
|
||||
fs
|
||||
) == 1, "There should just be 1 pretrained path in an archive file but got {}.".format(
|
||||
len(fs))
|
||||
|
||||
f = fs[0]
|
||||
src_dir = os.path.join(fpath_tmp, f)
|
||||
dst_dir = os.path.join(fpath, f)
|
||||
_move_and_merge_tree(src_dir, dst_dir)
|
||||
|
||||
shutil.rmtree(fpath_tmp)
|
||||
os.remove(fname)
|
||||
|
||||
return f
|
||||
|
||||
|
||||
def _get_pretrained():
|
||||
with open('./ppcls/utils/pretrained.list') as flist:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
python3.7 -m paddle.distributed.launch \
|
||||
--selected_gpus="5" \
|
||||
tools/benchmark/benchmark_acc.py
|
||||
|
|
@ -0,0 +1,114 @@
|
|||
# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
__dir__ = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.append(__dir__)
|
||||
sys.path.append(os.path.abspath(os.path.join(__dir__, '..')))
|
||||
sys.path.append(os.path.abspath(os.path.join(__dir__, '../..')))
|
||||
|
||||
from multiprocessing import Process, Manager
|
||||
import tools.eval as eval
|
||||
from ppcls.utils.model_zoo import _download, _decompress
|
||||
from ppcls.utils import logger
|
||||
|
||||
|
||||
def parse_args():
|
||||
def str2bool(v):
|
||||
return v.lower() in ("true", "t", "1")
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"-b",
|
||||
"--benchmark_file_list",
|
||||
type=str,
|
||||
default="./tools/benchmark/benchmark_list.txt")
|
||||
parser.add_argument(
|
||||
"-p", "--pretrained_dir", type=str, default="./pretrained/")
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def parse_model_infos(benchmark_file_list):
|
||||
model_infos = []
|
||||
with open(benchmark_file_list, "r") as fin:
|
||||
lines = fin.readlines()
|
||||
for idx, line in enumerate(lines):
|
||||
strs = line.strip("\n").strip("\r").split(" ")
|
||||
if len(strs) != 4:
|
||||
logger.info(
|
||||
"line {0}(info: {1}) format wrong, it should be splited into 4 parts, but got {2}".
|
||||
format(idx, line, len(strs)))
|
||||
model_infos.append({
|
||||
"top1_acc": float(strs[0]),
|
||||
"model_name": strs[1],
|
||||
"config_path": strs[2],
|
||||
"pretrain_path": strs[3],
|
||||
})
|
||||
return model_infos
|
||||
|
||||
|
||||
def main(args):
|
||||
benchmark_file_list = args.benchmark_file_list
|
||||
model_infos = parse_model_infos(benchmark_file_list)
|
||||
right_models = []
|
||||
wrong_models = []
|
||||
|
||||
for model_info in model_infos:
|
||||
try:
|
||||
pretrained_url = model_info["pretrain_path"]
|
||||
fname = _download(pretrained_url, args.pretrained_dir)
|
||||
pretrained_path = os.path.splitext(fname)[0]
|
||||
if pretrained_url.endswith("tar"):
|
||||
path = _decompress(fname)
|
||||
pretrained_path = os.path.join(
|
||||
os.path.dirname(pretrained_path), path)
|
||||
|
||||
args.config = model_info["config_path"]
|
||||
args.override = [
|
||||
"pretrained_model={}".format(pretrained_path),
|
||||
"VALID.batch_size=128",
|
||||
"VALID.num_workers=16",
|
||||
"load_static_weights=True",
|
||||
"print_interval=100",
|
||||
]
|
||||
manager = Manager()
|
||||
return_dict = manager.dict()
|
||||
p = Process(target=eval.main, args=(args, return_dict))
|
||||
p.start()
|
||||
p.join()
|
||||
top1_acc = return_dict.get("top1_acc", 0.0)
|
||||
except:
|
||||
top1_acc = 0.0
|
||||
diff = abs(top1_acc - model_info["top1_acc"])
|
||||
if diff > 0.001:
|
||||
err_info = "[{}]Top-1 acc diff should be <= 0.001 but got diff {}, gt acc: {}, eval acc: {}".format(
|
||||
model_info["model_name"], diff, model_info["top1_acc"],
|
||||
top1_acc)
|
||||
logger.warning(err_info)
|
||||
wrong_models.append(model_info["model_name"])
|
||||
else:
|
||||
right_models.append(model_info["model_name"])
|
||||
|
||||
logger.info("[number of right models: {}, they are: {}".format(
|
||||
len(right_models), right_models))
|
||||
logger.info("[number of wrong models: {}, they are: {}".format(
|
||||
len(wrong_models), wrong_models))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
args = parse_args()
|
||||
main(args)
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
0.7098 ResNet18 configs/ResNet/ResNet18.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNet18_pretrained.tar
|
||||
0.7650 ResNet50 configs/ResNet/ResNet50.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNet50_pretrained.tar
|
||||
0.7226 ResNet18_vd configs/ResNet/ResNet18_vd.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNet18_vd_pretrained.tar
|
||||
0.7912 ResNet50_vd configs/ResNet/ResNet50_vd.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNet50_vd_pretrained.tar
|
||||
0.7099 MobileNetV1 configs/MobileNetV1/MobileNetV1.yaml https://paddle-imagenet-models-name.bj.bcebos.com/MobileNetV1_pretrained.tar
|
||||
0.7215 MobileNetV2 configs/MobileNetV2/MobileNetV2.yaml https://paddle-imagenet-models-name.bj.bcebos.com/MobileNetV2_pretrained.tar
|
||||
0.7532 MobileNetV3_large_x1_0 configs/MobileNetV3/MobileNetV3_large_x1_0.yaml https://paddle-imagenet-models-name.bj.bcebos.com/MobileNetV3_large_x1_0_pretrained.tar
|
||||
0.6880 ShuffleNetV2 configs/ShuffleNet/ShuffleNetV2.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ShuffleNetV2_pretrained.tar
|
||||
0.7933 Res2Net50_26w_4s configs/Res2Net/Res2Net50_26w_4s.yaml https://paddle-imagenet-models-name.bj.bcebos.com/Res2Net50_26w_4s_pretrained.tar
|
||||
0.7775 ResNeXt50_32x4d configs/ResNeXt/ResNeXt50_32x4d.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNeXt50_32x4d_pretrained.tar
|
||||
0.7333 SE_ResNet18_vd configs/SENet/SE_ResNet18_vd.yaml https://paddle-imagenet-models-name.bj.bcebos.com/SE_ResNet18_vd_pretrained.tar
|
||||
0.7952 SE_ResNet50_vd configs/SENet/SE_ResNet50_vd.yaml https://paddle-imagenet-models-name.bj.bcebos.com/SE_ResNet50_vd_pretrained.tar
|
||||
0.8024 SE_ResNeXt50_vd_32x4d configs/SENet/SE_ResNeXt50_vd_32x4d.yaml https://paddle-imagenet-models-name.bj.bcebos.com/SE_ResNeXt50_vd_32x4d_pretrained.tar
|
||||
0.7566 DenseNet121 configs/DenseNet/DenseNet121.yaml https://paddle-imagenet-models-name.bj.bcebos.com/DenseNet121_pretrained.tar
|
||||
0.7678 DPN68 configs/DPN/DPN68.yaml https://paddle-imagenet-models-name.bj.bcebos.com/DPN68_pretrained.tar
|
||||
0.7692 HRNet_W18_C configs/HRNet/HRNet_W18_C.yaml https://paddle-imagenet-models-name.bj.bcebos.com/HRNet_W18_C_pretrained.tar
|
||||
0.7070 GoogLeNet configs/Inception/GoogLeNet.yaml https://paddle-imagenet-models-name.bj.bcebos.com/GoogLeNet_pretrained.tar
|
||||
0.7930 Xception41 configs/Xception/Xception41.yaml https://paddle-imagenet-models-name.bj.bcebos.com/Xception41_pretrained.tar
|
||||
0.7955 Xception41_deeplab configs/Xception/Xception41_deeplab.yaml https://paddle-imagenet-models-name.bj.bcebos.com/Xception41_deeplab_pretrained.tar
|
||||
0.8077 InceptionV4 configs/Inception/InceptionV4.yaml https://paddle-imagenet-models-name.bj.bcebos.com/InceptionV4_pretrained.tar
|
||||
0.8255 ResNeXt101_32x8d_wsl configs/ResNeXt101_wsl/ResNeXt101_32x8d_wsl.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNeXt101_32x8d_wsl_pretrained.tar
|
||||
0.7738 EfficientNetB0 configs/EfficientNet/EfficientNetB0.yaml https://paddle-imagenet-models-name.bj.bcebos.com/EfficientNetB0_pretrained.tar
|
||||
0.8035 ResNeSt50_fast_1s1x64d configs/ResNeSt/ResNeSt50_fast_1s1x64d.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNeSt50_fast_1s1x64d_pretrained.pdparams
|
||||
0.8102 ResNeSt50 configs/ResNeSt/ResNeSt50.yaml https://paddle-imagenet-models-name.bj.bcebos.com/ResNeSt50_pretrained.pdparams
|
||||
0.785 RegNetX_4GF configs/RegNet/RegNetX_4GF.yaml https://paddle-imagenet-models-name.bj.bcebos.com/RegNetX_4GF_pretrained.pdparams
|
||||
0.7402 GhostNet_x1_0 configs/GhostNet/GhostNet_x1_0.yaml https://paddle-imagenet-models-name.bj.bcebos.com/GhostNet_x1_0_pretrained.pdparams
|
||||
0.567 AlexNet configs/AlexNet/AlexNet.yaml https://paddle-imagenet-models-name.bj.bcebos.com/AlexNet_pretrained.tar
|
||||
0.596 SqueezeNet1_0 configs/SqueezeNet/SqueezeNet1_0.yaml https://paddle-imagenet-models-name.bj.bcebos.com/SqueezeNet1_0_pretrained.tar
|
||||
0.693 VGG11 configs/VGG/VGG11.yaml https://paddle-imagenet-models-name.bj.bcebos.com/VGG11_pretrained.tar
|
||||
0.780 DarkNet53 configs/DarkNet/DarkNet53.yaml https://paddle-imagenet-models-name.bj.bcebos.com/DarkNet53_ImageNet1k_pretrained.tar
|
||||
|
|
@ -12,7 +12,6 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from ppcls import model_zoo
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
|
@ -20,6 +19,8 @@ __dir__ = os.path.dirname(os.path.abspath(__file__))
|
|||
sys.path.append(__dir__)
|
||||
sys.path.append(os.path.abspath(os.path.join(__dir__, '..')))
|
||||
|
||||
from ppcls import model_zoo
|
||||
|
||||
|
||||
def parse_args():
|
||||
def str2bool(v):
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ def parse_args():
|
|||
return args
|
||||
|
||||
|
||||
def main(args):
|
||||
def main(args, return_dict={}):
|
||||
config = get_config(args.config, overrides=args.override, show=True)
|
||||
# assign place
|
||||
use_gpu = config.get("use_gpu", True)
|
||||
|
|
@ -68,6 +68,8 @@ def main(args):
|
|||
|
||||
top1_acc = program.run(valid_dataloader, config, net, None, None, 0,
|
||||
'valid')
|
||||
return_dict["top1_acc"] = top1_acc
|
||||
return top1_acc
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
Loading…
Reference in New Issue