From 86b5f9b4906fa613d6564cfb1154151064d081a4 Mon Sep 17 00:00:00 2001 From: dongshuilong Date: Fri, 17 Sep 2021 03:14:28 +0000 Subject: [PATCH 1/6] add version 1 for benchmark --- benchmark/README.md | 27 +++++++++++++++++++ benchmark/prepare_data.sh | 11 ++++++++ benchmark/run_all.sh | 25 +++++++++++++++++ benchmark/run_benchmark.sh | 55 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+) create mode 100644 benchmark/README.md create mode 100644 benchmark/prepare_data.sh create mode 100644 benchmark/run_all.sh create mode 100644 benchmark/run_benchmark.sh diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 000000000..3bdb8b112 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,27 @@ +# benchmark使用说明 + +此目录所有shell脚本是为了测试PaddleClas中不同模型的速度指标，如单卡训练速度指标、多卡训练速度指标等。 + +## 相关脚本说明 + +一共有3个脚本： + +- `prepare_data.sh`: 下载相应的测试数据，并配置好数据路径 +- `run_benchmark.sh`: 执行单独一个训练测试的脚本，具体调用方式，可查看脚本注释 +- `run_all.sh`: 执行所有训练测试的入口脚本 + +## 使用说明 + +******为了跟PaddleClas中其他的模块的执行目录保持一致，此模块的执行目录为`PaddleClas`的根目录。 + +### 1.准备数据 + +```shell +bash benchmark/prepare_data.sh +``` + +### 2.执行所有模型的测试 + +```shell +bash benchmark/run_all.sh +``` diff --git a/benchmark/prepare_data.sh b/benchmark/prepare_data.sh new file mode 100644 index 000000000..b6ad84e36 --- /dev/null +++ b/benchmark/prepare_data.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +cd dataset +rm -rf ILSVRC2012 +wget -nc https://paddle-imagenet-models-name.bj.bcebos.com/data/whole_chain/whole_chain_little_train.tar +tar xf whole_chain_little_train.tar +ln -s whole_chain_little_train ILSVRC2012 +cd ILSVRC2012 +mv train.txt train_list.txt +mv val.txt val_list.txt +cd ../../ diff --git a/benchmark/run_all.sh b/benchmark/run_all.sh new file mode 100644 index 000000000..7e7b5fe0a --- /dev/null +++ b/benchmark/run_all.sh @@ -0,0 +1,25 @@ +# 提供可稳定复现性能的脚本，默认在标准docker环境内py37执行： paddlepaddle/paddle:latest-gpu-cuda10.1-cudnn7 paddle=2.1.2 py=37 +# 执行目录：需说明 +# cd ** +# 1 安装该模型需要的依赖 (如需开启优化策略请注明) +# pip install ... +# 2 拷贝该模型需要数据、预训练模型 +# 3 批量运行（如不方便批量，1，2需放到单个模型中） + +model_mode_list=(MobileNetV1 MobileNetV2 MobileNetV3_large_x1_0 EfficientNetB0 ShuffleNetV2_x1_0 DenseNet121 HRNet_W48_C SwinTransformer_tiny_patch4_window7_224 alt_gvt_base) +fp_item_list=(fp32) +bs_list=(32 64 96 128) +for model_mode in ${model_mode_list[@]}; do + for fp_item in ${fp_item_list[@]}; do + for bs_item in ${bs_list[@]};do + echo "index is speed, 1gpus, begin, ${model_name}" + run_mode=sp + CUDA_VISIBLE_DEVICES=0 bash benchmark/run_benchmark.sh ${run_mode} ${bs_item} ${fp_item} 10 ${model_mode} # (5min) + sleep 10 + echo "index is speed, 8gpus, run_mode is multi_process, begin, ${model_name}" + run_mode=mp + CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 bash benchmark/run_benchmark.sh ${run_mode} ${bs_item} ${fp_item} 10 ${model_mode} + sleep 10 + done + done +done diff --git a/benchmark/run_benchmark.sh b/benchmark/run_benchmark.sh new file mode 100644 index 000000000..909f38f2c --- /dev/null +++ b/benchmark/run_benchmark.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -xe +# 运行示例：CUDA_VISIBLE_DEVICES=0 bash run_benchmark.sh ${run_mode} ${bs_item} ${fp_item} 500 ${model_mode} +# 参数说明 +function _set_params(){ + run_mode=${1:-"sp"} # 单卡sp|多卡mp + batch_size=${2:-"64"} + fp_item=${3:-"fp32"} # fp32|fp16 + epochs=${4:-"10"} # 可选，如果需要修改代码提前中断 + model_name=${5:-"model_name"} + run_log_path="${TRAIN_LOG_DIR:-$(pwd)}/benchmark" # TRAIN_LOG_DIR 后续QA设置该参数 + +# 以下不用修改 + device=${CUDA_VISIBLE_DEVICES//,/ } + arr=(${device}) + num_gpu_devices=${#arr[*]} + log_file=${run_log_path}/${model_name}_${run_mode}_bs${batch_size}_${fp_item}_${num_gpu_devices} +} +function _train(){ + echo "Train on ${num_gpu_devices} GPUs" + echo "current CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES, gpus=$num_gpu_devices, batch_size=$batch_size" + + if [ ${fp_item} = "fp32" ];then + model_config=`find ppcls/configs/ -name ${model_name}.yaml` + else + model_config=`find ppcls/configs/ -name ${model_name}_fp16.yaml` + fi + + train_cmd="-c ${model_config} -o DataLoader.Train.sampler.batch_size=${batch_size} -o Global.epochs=${epochs}" + case ${run_mode} in + sp) train_cmd="python -u tools/train.py ${train_cmd}" ;; + mp) + train_cmd="python -m paddle.distributed.launch --log_dir=./mylog --gpus=$CUDA_VISIBLE_DEVICES tools/train.py ${train_cmd}" + log_parse_file="mylog/workerlog.0" ;; + *) echo "choose run_mode(sp or mp)"; exit 1; + esac +# 以下不用修改 + timeout 15m ${train_cmd} > ${log_file} 2>&1 + if [ $? -ne 0 ];then + echo -e "${model_name}, FAIL" + export job_fail_flag=1 + else + echo -e "${model_name}, SUCCESS" + export job_fail_flag=0 + fi + kill -9 `ps -ef|grep 'python'|awk '{print $2}'` + + if [ $run_mode = "mp" -a -d mylog ]; then + rm ${log_file} + cp mylog/workerlog.0 ${log_file} + fi +} + +_set_params $@ +_train From 22c0a53c32bcbb95bcea4e9c297e8ee9140a3b37 Mon Sep 17 00:00:00 2001 From: Walter Date: Fri, 17 Sep 2021 11:16:35 +0800 Subject: [PATCH 2/6] Update README.md --- benchmark/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/README.md b/benchmark/README.md index 3bdb8b112..2e892d97e 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -12,7 +12,7 @@ ## 使用说明 -******为了跟PaddleClas中其他的模块的执行目录保持一致，此模块的执行目录为`PaddleClas`的根目录。 +**注意**：为了跟PaddleClas中其他的模块的执行目录保持一致，此模块的执行目录为`PaddleClas`的根目录。 ### 1.准备数据 From 9f2ab06ec2f0f9712a6b1a154d05af6fb12f97ca Mon Sep 17 00:00:00 2001 From: dongshuilong Date: Wed, 22 Sep 2021 08:29:44 +0000 Subject: [PATCH 3/6] add profiler --- ppcls/engine/train/train.py | 2 ++ ppcls/utils/config.py | 7 +++++++ tools/train.py | 1 + 3 files changed, 10 insertions(+) diff --git a/ppcls/engine/train/train.py b/ppcls/engine/train/train.py index 73f225087..d78508147 100644 --- a/ppcls/engine/train/train.py +++ b/ppcls/engine/train/train.py @@ -16,6 +16,7 @@ from __future__ import absolute_import, division, print_function import time import paddle from ppcls.engine.train.utils import update_loss, update_metric, log_info +from ppcls.utils import profiler def train_epoch(trainer, epoch_id, print_batch_step): @@ -26,6 +27,7 @@ def train_epoch(trainer, epoch_id, print_batch_step): for iter_id, batch in enumerate(train_dataloader): if iter_id >= trainer.max_iter: break + profiler.add_profiler_step(trainer.config["profiler_options"]) if iter_id == 5: for key in trainer.time_info: trainer.time_info[key].reset() diff --git a/ppcls/utils/config.py b/ppcls/utils/config.py index b92f0d945..e3277c480 100644 --- a/ppcls/utils/config.py +++ b/ppcls/utils/config.py @@ -199,5 +199,12 @@ def parse_args(): action='append', default=[], help='config options to be overridden') + parser.add_argument( + '-p', + '--profiler_options', + type=str, + default=None, + help='The option of profiler, which should be in format \"key1=value1;key2=value2;key3=value3\".' + ) args = parser.parse_args() return args diff --git a/tools/train.py b/tools/train.py index 1d8359036..e7c9d7bcc 100644 --- a/tools/train.py +++ b/tools/train.py @@ -27,5 +27,6 @@ if __name__ == "__main__": args = config.parse_args() config = config.get_config( args.config, overrides=args.override, show=False) + config.profiler_options = args.profiler_options engine = Engine(config, mode="train") engine.train() From d44bc7fd62cc65e2f44c08be9e9f9bb4bd9d5c3a Mon Sep 17 00:00:00 2001 From: dongshuilong Date: Thu, 30 Sep 2021 03:39:54 +0000 Subject: [PATCH 4/6] modify prepare_data.sh for benchmark --- benchmark/prepare_data.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/benchmark/prepare_data.sh b/benchmark/prepare_data.sh index b6ad84e36..83a685698 100644 --- a/benchmark/prepare_data.sh +++ b/benchmark/prepare_data.sh @@ -1,11 +1,11 @@ #!/bin/bash +dataset_url=$1 cd dataset rm -rf ILSVRC2012 -wget -nc https://paddle-imagenet-models-name.bj.bcebos.com/data/whole_chain/whole_chain_little_train.tar -tar xf whole_chain_little_train.tar -ln -s whole_chain_little_train ILSVRC2012 -cd ILSVRC2012 -mv train.txt train_list.txt -mv val.txt val_list.txt +wget -nc ${dataset_url} +tar xf ILSVRC2012_val.tar +ln -s ILSVRC2012_val ILSVRC2012 +cd ILSVRC2012 +ln -s val_list.txt train_list.txt cd ../../ From 6c6fe7deabc4a69baa5d86313232ee26ed15b755 Mon Sep 17 00:00:00 2001 From: dongshuilong Date: Thu, 30 Sep 2021 08:35:27 +0000 Subject: [PATCH 5/6] update benchmark --- benchmark/run_benchmark.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmark/run_benchmark.sh b/benchmark/run_benchmark.sh index 909f38f2c..1e51f9a66 100644 --- a/benchmark/run_benchmark.sh +++ b/benchmark/run_benchmark.sh @@ -14,7 +14,7 @@ function _set_params(){ device=${CUDA_VISIBLE_DEVICES//,/ } arr=(${device}) num_gpu_devices=${#arr[*]} - log_file=${run_log_path}/${model_name}_${run_mode}_bs${batch_size}_${fp_item}_${num_gpu_devices} + log_file=${run_log_path}/clas_${model_name}_${run_mode}_bs${batch_size}_${fp_item}_${num_gpu_devices} } function _train(){ echo "Train on ${num_gpu_devices} GPUs" @@ -34,6 +34,7 @@ function _train(){ log_parse_file="mylog/workerlog.0" ;; *) echo "choose run_mode(sp or mp)"; exit 1; esac + rm -rf mylog # 以下不用修改 timeout 15m ${train_cmd} > ${log_file} 2>&1 if [ $? -ne 0 ];then From c2a9f8830ed4da6417d301e5747035df1456600c Mon Sep 17 00:00:00 2001 From: dongshuilong Date: Wed, 13 Oct 2021 09:30:11 +0000 Subject: [PATCH 6/6] update profiler --- ppcls/engine/train/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ppcls/engine/train/train.py b/ppcls/engine/train/train.py index 8999476e2..540dd7512 100644 --- a/ppcls/engine/train/train.py +++ b/ppcls/engine/train/train.py @@ -24,7 +24,7 @@ def train_epoch(engine, epoch_id, print_batch_step): for iter_id, batch in enumerate(engine.train_dataloader): if iter_id >= engine.max_iter: break - profiler.add_profiler_step(trainer.config["profiler_options"]) + profiler.add_profiler_step(engine.config["profiler_options"]) if iter_id == 5: for key in engine.time_info: engine.time_info[key].reset()