keep model name and directory same
LDOUBLEV
parent
f744089c28
commit
44ea4d8698
|
|
@ -1,9 +1,19 @@
|
|||
#!/bin/bash
|
||||
source test_tipc/common_func.sh
|
||||
|
||||
# set env
|
||||
python=python
|
||||
export model_branch=`git symbolic-ref HEAD 2>/dev/null | cut -d"/" -f 3`
|
||||
export model_commit=$(git log|head -n1|awk '{print $2}')
|
||||
export str_tmp=$(echo `pip list|grep paddlepaddle-gpu|awk -F ' ' '{print $2}'`)
|
||||
export frame_version=${str_tmp%%.post*}
|
||||
export frame_commit=$(echo `${python} -c "import paddle;print(paddle.version.commit)"`)
|
||||
|
||||
# run benchmark sh
|
||||
# Usage:
|
||||
# bash run_benchmark_train.sh config.txt params
|
||||
# or
|
||||
# bash run_benchmark_train.sh config.txt
|
||||
|
||||
function func_parser_params(){
|
||||
strs=$1
|
||||
|
|
@ -55,30 +65,15 @@ function get_repo_name(){
|
|||
}
|
||||
|
||||
FILENAME=$1
|
||||
# MODE be one of ['benchmark_train']
|
||||
# copy FILENAME as new
|
||||
new_filename="./test_tipc/benchmark_train.txt"
|
||||
cmd=`yes|cp $FILENAME $new_filename`
|
||||
FILENAME=$new_filename
|
||||
# MODE must be one of ['benchmark_train']
|
||||
MODE=$2
|
||||
params=$3
|
||||
# bash test_tipc/benchmark_train.sh test_tipc/configs/det_mv3_db_v2.0/train_benchmark.txt benchmark_train dynamic_bs8_null_SingleP_DP_N1C1
|
||||
IFS="\n"
|
||||
|
||||
# parser params from input: modeltype_bs${bs_item}_${fp_item}_${run_process_type}_${run_mode}_${device_num}
|
||||
IFS="_"
|
||||
params_list=(${params})
|
||||
model_type=${params_list[0]}
|
||||
batch_size=${params_list[1]}
|
||||
batch_size=`echo ${batch_size} | tr -cd "[0-9]" `
|
||||
precision=${params_list[2]}
|
||||
run_process_type=${params_list[3]}
|
||||
run_mode=${params_list[4]}
|
||||
device_num=${params_list[5]}
|
||||
device_num_copy=$device_num
|
||||
IFS=";"
|
||||
|
||||
|
||||
# sed batchsize and precision
|
||||
func_sed_params "$FILENAME" "6" "$precision"
|
||||
func_sed_params "$FILENAME" "9" "$batch_size"
|
||||
|
||||
PARAMS=$3
|
||||
# bash test_tipc/benchmark_train.sh test_tipc/configs/det_mv3_db_v2_0/train_benchmark.txt benchmark_train dynamic_bs8_null_SingleP_DP_N1C1
|
||||
IFS=$'\n'
|
||||
# parser params from train_benchmark.txt
|
||||
dataline=`cat $FILENAME`
|
||||
# parser params
|
||||
|
|
@ -87,31 +82,22 @@ lines=(${dataline})
|
|||
model_name=$(func_parser_value "${lines[1]}")
|
||||
|
||||
# 获取benchmark_params所在的行数
|
||||
line_num=`grep -n "benchmark_params" $FILENAME | cut -d ":" -f 1`
|
||||
line_num=`grep -n "train_benchmark_params" $FILENAME | cut -d ":" -f 1`
|
||||
# for train log parser
|
||||
line_num=`expr $line_num + 3`
|
||||
speed_unit_value=$(func_parser_value "${lines[line_num]}")
|
||||
batch_size=$(func_parser_value "${lines[line_num]}")
|
||||
line_num=`expr $line_num + 1`
|
||||
fp_items=$(func_parser_value "${lines[line_num]}")
|
||||
line_num=`expr $line_num + 1`
|
||||
epoch=$(func_parser_value "${lines[line_num]}")
|
||||
|
||||
line_num=`expr $line_num + 1`
|
||||
skip_steps_value=$(func_parser_value "${lines[line_num]}")
|
||||
|
||||
line_num=`expr $line_num + 1`
|
||||
keyword_value=$(func_parser_value "${lines[line_num]}")
|
||||
echo $keyword_value
|
||||
|
||||
line_num=`expr $line_num + 1`
|
||||
convergence_key_value=$(func_parser_value "${lines[line_num]}")
|
||||
profile_option_key=$(func_parser_key "${lines[line_num]}")
|
||||
profile_option_params=$(func_parser_value "${lines[line_num]}")
|
||||
profile_option="${profile_option_key}:${profile_option_params}"
|
||||
|
||||
line_num=`expr $line_num + 1`
|
||||
flags_value=$(func_parser_value "${lines[line_num]}")
|
||||
|
||||
gpu_id=$(set_gpu_id $device_num)
|
||||
repo_name=$(get_repo_name )
|
||||
|
||||
SAVE_LOG=${BENCHMARK_LOG_DIR:-$(pwd)} # */benchmark_log
|
||||
status_log="${SAVE_LOG}/benchmark_log/results.log"
|
||||
|
||||
# set export
|
||||
# set flags
|
||||
IFS=";"
|
||||
flags_list=(${flags_value})
|
||||
for _flag in ${flags_list[*]}; do
|
||||
|
|
@ -119,97 +105,154 @@ for _flag in ${flags_list[*]}; do
|
|||
eval $cmd
|
||||
done
|
||||
|
||||
if [ ${precision} = "null" ];then
|
||||
precision="fp32"
|
||||
fi
|
||||
# set log_name
|
||||
repo_name=$(get_repo_name )
|
||||
SAVE_LOG=${BENCHMARK_LOG_DIR:-$(pwd)} # */benchmark_log
|
||||
mkdir -p "${SAVE_LOG}/benchmark_log/"
|
||||
status_log="${SAVE_LOG}/benchmark_log/results.log"
|
||||
|
||||
# set env
|
||||
export model_branch=`git symbolic-ref HEAD 2>/dev/null | cut -d"/" -f 3`
|
||||
export model_commit=$(git log|head -n1|awk '{print $2}')
|
||||
export str_tmp=$(echo `pip list|grep paddlepaddle-gpu|awk -F ' ' '{print $2}'`)
|
||||
export frame_version=${str_tmp%%.post*}
|
||||
export frame_commit=$(echo `python -c "import paddle;print(paddle.version.commit)"`)
|
||||
# The number of lines in which train params can be replaced.
|
||||
line_python=3
|
||||
line_gpuid=4
|
||||
line_precision=6
|
||||
line_epoch=7
|
||||
line_batchsize=9
|
||||
line_profile=13
|
||||
line_eval_py=24
|
||||
line_export_py=30
|
||||
|
||||
func_sed_params "$FILENAME" "${line_eval_py}" "null"
|
||||
func_sed_params "$FILENAME" "${line_export_py}" "null"
|
||||
func_sed_params "$FILENAME" "${line_python}" "$python"
|
||||
|
||||
if [ ${#gpu_id} -le 1 ];then
|
||||
log_path="$SAVE_LOG/profiling_log"
|
||||
mkdir -p $log_path
|
||||
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_profiling"
|
||||
func_sed_params "$FILENAME" "4" "0" # sed used gpu_id
|
||||
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
|
||||
echo $cmd
|
||||
eval $cmd
|
||||
eval "cat ${log_path}/${log_name}"
|
||||
|
||||
# without profile
|
||||
log_path="$SAVE_LOG/train_log"
|
||||
speed_log_path="$SAVE_LOG/index"
|
||||
mkdir -p $log_path
|
||||
mkdir -p $speed_log_path
|
||||
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_log"
|
||||
speed_log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_speed"
|
||||
func_sed_params "$FILENAME" "13" "null" # sed used gpu_id
|
||||
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
|
||||
echo $cmd
|
||||
job_bt=`date '+%Y%m%d%H%M%S'`
|
||||
eval $cmd
|
||||
job_et=`date '+%Y%m%d%H%M%S'`
|
||||
export model_run_time=$((${job_et}-${job_bt}))
|
||||
eval "cat ${log_path}/${log_name}"
|
||||
|
||||
# parser log
|
||||
_model_name="${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}"
|
||||
cmd="python ${BENCHMARK_ROOT}/scripts/analysis.py --filename ${log_path}/${log_name} \
|
||||
--speed_log_file '${speed_log_path}/${speed_log_name}' \
|
||||
--model_name ${_model_name} \
|
||||
--base_batch_size ${batch_size} \
|
||||
--run_mode ${run_mode} \
|
||||
--run_process_type ${run_process_type} \
|
||||
--fp_item ${precision} \
|
||||
--keyword ${keyword_value}: \
|
||||
--skip_steps ${skip_steps_value} \
|
||||
--device_num ${device_num} \
|
||||
--speed_unit ${speed_unit_value} \
|
||||
--convergence_key ${convergence_key_value}: "
|
||||
echo $cmd
|
||||
eval $cmd
|
||||
last_status=${PIPESTATUS[0]}
|
||||
status_check $last_status "${cmd}" "${status_log}"
|
||||
|
||||
# if params
|
||||
if [ ! -n "$PARAMS" ] ;then
|
||||
# PARAMS input is not a word.
|
||||
IFS="|"
|
||||
batch_size_list=(${batch_size})
|
||||
fp_items_list=(${fp_items})
|
||||
device_num_list=(N1C4)
|
||||
run_mode="DP"
|
||||
else
|
||||
log_path="$SAVE_LOG/train_log"
|
||||
speed_log_path="$SAVE_LOG/index"
|
||||
mkdir -p $log_path
|
||||
mkdir -p $speed_log_path
|
||||
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_log"
|
||||
speed_log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_speed"
|
||||
func_sed_params "$FILENAME" "4" "$gpu_id" # sed used gpu_id
|
||||
func_sed_params "$FILENAME" "13" "null" # sed --profile_option as null
|
||||
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
|
||||
echo $cmd
|
||||
job_bt=`date '+%Y%m%d%H%M%S'`
|
||||
eval $cmd
|
||||
job_et=`date '+%Y%m%d%H%M%S'`
|
||||
export model_run_time=$((${job_et}-${job_bt}))
|
||||
eval "cat ${log_path}/${log_name}"
|
||||
# parser log
|
||||
_model_name="${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}"
|
||||
|
||||
cmd="python ${BENCHMARK_ROOT}/scripts/analysis.py --filename ${log_path}/${log_name} \
|
||||
--speed_log_file '${speed_log_path}/${speed_log_name}' \
|
||||
--model_name ${_model_name} \
|
||||
--base_batch_size ${batch_size} \
|
||||
--run_mode ${run_mode} \
|
||||
--run_process_type ${run_process_type} \
|
||||
--fp_item ${precision} \
|
||||
--keyword ${keyword_value}: \
|
||||
--skip_steps ${skip_steps_value} \
|
||||
--device_num ${device_num} \
|
||||
--speed_unit ${speed_unit_value} \
|
||||
--convergence_key ${convergence_key_value}: "
|
||||
echo $cmd
|
||||
eval $cmd
|
||||
last_status=${PIPESTATUS[0]}
|
||||
status_check $last_status "${cmd}" "${status_log}"
|
||||
# parser params from input: modeltype_bs${bs_item}_${fp_item}_${run_process_type}_${run_mode}_${device_num}
|
||||
IFS="_"
|
||||
params_list=(${PARAMS})
|
||||
model_type=${params_list[0]}
|
||||
batch_size=${params_list[1]}
|
||||
batch_size=`echo ${batch_size} | tr -cd "[0-9]" `
|
||||
precision=${params_list[2]}
|
||||
run_process_type=${params_list[3]}
|
||||
run_mode=${params_list[4]}
|
||||
device_num=${params_list[5]}
|
||||
IFS=";"
|
||||
|
||||
if [ ${precision} = "null" ];then
|
||||
precision="fp32"
|
||||
fi
|
||||
|
||||
fp_items_list=($precision)
|
||||
batch_size_list=($batch_size)
|
||||
device_num_list=($device_num)
|
||||
fi
|
||||
|
||||
IFS="|"
|
||||
for batch_size in ${batch_size_list[*]}; do
|
||||
for precision in ${fp_items_list[*]}; do
|
||||
for device_num in ${device_num_list[*]}; do
|
||||
# sed batchsize and precision
|
||||
func_sed_params "$FILENAME" "${line_precision}" "$precision"
|
||||
func_sed_params "$FILENAME" "${line_batchsize}" "$MODE=$batch_size"
|
||||
func_sed_params "$FILENAME" "${line_epoch}" "$MODE=$epoch"
|
||||
gpu_id=$(set_gpu_id $device_num)
|
||||
|
||||
if [ ${#gpu_id} -le 1 ];then
|
||||
run_process_type="SingleP"
|
||||
log_path="$SAVE_LOG/profiling_log"
|
||||
mkdir -p $log_path
|
||||
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_profiling"
|
||||
func_sed_params "$FILENAME" "${line_gpuid}" "0" # sed used gpu_id
|
||||
# set profile_option params
|
||||
tmp=`sed -i "${line_profile}s/.*/${profile_option}/" "${FILENAME}"`
|
||||
|
||||
# run test_train_inference_python.sh
|
||||
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
|
||||
echo $cmd
|
||||
eval $cmd
|
||||
eval "cat ${log_path}/${log_name}"
|
||||
|
||||
# without profile
|
||||
log_path="$SAVE_LOG/train_log"
|
||||
speed_log_path="$SAVE_LOG/index"
|
||||
mkdir -p $log_path
|
||||
mkdir -p $speed_log_path
|
||||
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_log"
|
||||
speed_log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_speed"
|
||||
func_sed_params "$FILENAME" "${line_profile}" "null" # sed profile_id as null
|
||||
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
|
||||
echo $cmd
|
||||
job_bt=`date '+%Y%m%d%H%M%S'`
|
||||
eval $cmd
|
||||
job_et=`date '+%Y%m%d%H%M%S'`
|
||||
export model_run_time=$((${job_et}-${job_bt}))
|
||||
eval "cat ${log_path}/${log_name}"
|
||||
|
||||
# parser log
|
||||
_model_name="${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}"
|
||||
cmd="${python} ${BENCHMARK_ROOT}/scripts/analysis.py --filename ${log_path}/${log_name} \
|
||||
--speed_log_file '${speed_log_path}/${speed_log_name}' \
|
||||
--model_name ${_model_name} \
|
||||
--base_batch_size ${batch_size} \
|
||||
--run_mode ${run_mode} \
|
||||
--run_process_type ${run_process_type} \
|
||||
--fp_item ${precision} \
|
||||
--keyword ips: \
|
||||
--skip_steps 2 \
|
||||
--device_num ${device_num} \
|
||||
--speed_unit samples/s \
|
||||
--convergence_key loss: "
|
||||
echo $cmd
|
||||
eval $cmd
|
||||
last_status=${PIPESTATUS[0]}
|
||||
status_check $last_status "${cmd}" "${status_log}"
|
||||
else
|
||||
IFS=";"
|
||||
unset_env=`unset CUDA_VISIBLE_DEVICES`
|
||||
run_process_type="MultiP"
|
||||
log_path="$SAVE_LOG/train_log"
|
||||
speed_log_path="$SAVE_LOG/index"
|
||||
mkdir -p $log_path
|
||||
mkdir -p $speed_log_path
|
||||
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_log"
|
||||
speed_log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_speed"
|
||||
func_sed_params "$FILENAME" "${line_gpuid}" "$gpu_id" # sed used gpu_id
|
||||
func_sed_params "$FILENAME" "${line_profile}" "null" # sed --profile_option as null
|
||||
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
|
||||
echo $cmd
|
||||
job_bt=`date '+%Y%m%d%H%M%S'`
|
||||
eval $cmd
|
||||
job_et=`date '+%Y%m%d%H%M%S'`
|
||||
export model_run_time=$((${job_et}-${job_bt}))
|
||||
eval "cat ${log_path}/${log_name}"
|
||||
# parser log
|
||||
_model_name="${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}"
|
||||
|
||||
cmd="${python} ${BENCHMARK_ROOT}/scripts/analysis.py --filename ${log_path}/${log_name} \
|
||||
--speed_log_file '${speed_log_path}/${speed_log_name}' \
|
||||
--model_name ${_model_name} \
|
||||
--base_batch_size ${batch_size} \
|
||||
--run_mode ${run_mode} \
|
||||
--run_process_type ${run_process_type} \
|
||||
--fp_item ${precision} \
|
||||
--keyword ips: \
|
||||
--skip_steps 2 \
|
||||
--device_num ${device_num} \
|
||||
--speed_unit images/s \
|
||||
--convergence_key loss: "
|
||||
echo $cmd
|
||||
eval $cmd
|
||||
last_status=${PIPESTATUS[0]}
|
||||
status_check $last_status "${cmd}" "${status_log}"
|
||||
fi
|
||||
done
|
||||
done
|
||||
done
|
||||
|
|
@ -1,258 +0,0 @@
|
|||
#!/bin/bash
|
||||
source test_tipc/common_func.sh
|
||||
|
||||
# set env
|
||||
python=python3.7
|
||||
export model_branch=`git symbolic-ref HEAD 2>/dev/null | cut -d"/" -f 3`
|
||||
export model_commit=$(git log|head -n1|awk '{print $2}')
|
||||
export str_tmp=$(echo `pip list|grep paddlepaddle-gpu|awk -F ' ' '{print $2}'`)
|
||||
export frame_version=${str_tmp%%.post*}
|
||||
export frame_commit=$(echo `${python} -c "import paddle;print(paddle.version.commit)"`)
|
||||
|
||||
# run benchmark sh
|
||||
# Usage:
|
||||
# bash run_benchmark_train.sh config.txt params
|
||||
# or
|
||||
# bash run_benchmark_train.sh config.txt
|
||||
|
||||
function func_parser_params(){
|
||||
strs=$1
|
||||
IFS="="
|
||||
array=(${strs})
|
||||
tmp=${array[1]}
|
||||
echo ${tmp}
|
||||
}
|
||||
|
||||
function func_sed_params(){
|
||||
filename=$1
|
||||
line=$2
|
||||
param_value=$3
|
||||
params=`sed -n "${line}p" $filename`
|
||||
IFS=":"
|
||||
array=(${params})
|
||||
key=${array[0]}
|
||||
value=${array[1]}
|
||||
if [[ $value =~ 'benchmark_train' ]];then
|
||||
IFS='='
|
||||
_val=(${value})
|
||||
param_value="${_val[0]}=${param_value}"
|
||||
fi
|
||||
new_params="${key}:${param_value}"
|
||||
IFS=";"
|
||||
cmd="sed -i '${line}s/.*/${new_params}/' '${filename}'"
|
||||
eval $cmd
|
||||
}
|
||||
|
||||
function set_gpu_id(){
|
||||
string=$1
|
||||
_str=${string:1:6}
|
||||
IFS="C"
|
||||
arr=(${_str})
|
||||
M=${arr[0]}
|
||||
P=${arr[1]}
|
||||
gn=`expr $P - 1`
|
||||
gpu_num=`expr $gn / $M`
|
||||
seq=`seq -s "," 0 $gpu_num`
|
||||
echo $seq
|
||||
}
|
||||
|
||||
function get_repo_name(){
|
||||
IFS=";"
|
||||
cur_dir=$(pwd)
|
||||
IFS="/"
|
||||
arr=(${cur_dir})
|
||||
echo ${arr[-1]}
|
||||
}
|
||||
|
||||
FILENAME=$1
|
||||
# copy FILENAME as new
|
||||
new_filename="./test_tipc/benchmark_train.txt"
|
||||
cmd=`yes|cp $FILENAME $new_filename`
|
||||
FILENAME=$new_filename
|
||||
# MODE must be one of ['benchmark_train']
|
||||
MODE=$2
|
||||
PARAMS=$3
|
||||
# bash test_tipc/benchmark_train.sh test_tipc/configs/det_mv3_db_v2.0/train_benchmark.txt benchmark_train dynamic_bs8_null_SingleP_DP_N1C1
|
||||
IFS=$'\n'
|
||||
# parser params from train_benchmark.txt
|
||||
dataline=`cat $FILENAME`
|
||||
# parser params
|
||||
IFS=$'\n'
|
||||
lines=(${dataline})
|
||||
model_name=$(func_parser_value "${lines[1]}")
|
||||
|
||||
# 获取benchmark_params所在的行数
|
||||
line_num=`grep -n "train_benchmark_params" $FILENAME | cut -d ":" -f 1`
|
||||
# for train log parser
|
||||
batch_size=$(func_parser_value "${lines[line_num]}")
|
||||
line_num=`expr $line_num + 1`
|
||||
fp_items=$(func_parser_value "${lines[line_num]}")
|
||||
line_num=`expr $line_num + 1`
|
||||
epoch=$(func_parser_value "${lines[line_num]}")
|
||||
|
||||
line_num=`expr $line_num + 1`
|
||||
profile_option_key=$(func_parser_key "${lines[line_num]}")
|
||||
profile_option_params=$(func_parser_value "${lines[line_num]}")
|
||||
profile_option="${profile_option_key}:${profile_option_params}"
|
||||
|
||||
line_num=`expr $line_num + 1`
|
||||
flags_value=$(func_parser_value "${lines[line_num]}")
|
||||
# set flags
|
||||
IFS=";"
|
||||
flags_list=(${flags_value})
|
||||
for _flag in ${flags_list[*]}; do
|
||||
cmd="export ${_flag}"
|
||||
eval $cmd
|
||||
done
|
||||
|
||||
# set log_name
|
||||
repo_name=$(get_repo_name )
|
||||
SAVE_LOG=${BENCHMARK_LOG_DIR:-$(pwd)} # */benchmark_log
|
||||
mkdir -p "${SAVE_LOG}/benchmark_log/"
|
||||
status_log="${SAVE_LOG}/benchmark_log/results.log"
|
||||
|
||||
# The number of lines in which train params can be replaced.
|
||||
line_python=3
|
||||
line_gpuid=4
|
||||
line_precision=6
|
||||
line_epoch=7
|
||||
line_batchsize=9
|
||||
line_profile=13
|
||||
line_eval_py=24
|
||||
line_export_py=30
|
||||
|
||||
func_sed_params "$FILENAME" "${line_eval_py}" "null"
|
||||
func_sed_params "$FILENAME" "${line_export_py}" "null"
|
||||
func_sed_params "$FILENAME" "${line_python}" "$python"
|
||||
|
||||
# if params
|
||||
if [ ! -n "$PARAMS" ] ;then
|
||||
# PARAMS input is not a word.
|
||||
IFS="|"
|
||||
batch_size_list=(${batch_size})
|
||||
fp_items_list=(${fp_items})
|
||||
device_num_list=(N1C4)
|
||||
run_mode="DP"
|
||||
else
|
||||
# parser params from input: modeltype_bs${bs_item}_${fp_item}_${run_process_type}_${run_mode}_${device_num}
|
||||
IFS="_"
|
||||
params_list=(${PARAMS})
|
||||
model_type=${params_list[0]}
|
||||
batch_size=${params_list[1]}
|
||||
batch_size=`echo ${batch_size} | tr -cd "[0-9]" `
|
||||
precision=${params_list[2]}
|
||||
run_process_type=${params_list[3]}
|
||||
run_mode=${params_list[4]}
|
||||
device_num=${params_list[5]}
|
||||
IFS=";"
|
||||
|
||||
if [ ${precision} = "null" ];then
|
||||
precision="fp32"
|
||||
fi
|
||||
|
||||
fp_items_list=($precision)
|
||||
batch_size_list=($batch_size)
|
||||
device_num_list=($device_num)
|
||||
fi
|
||||
|
||||
IFS="|"
|
||||
for batch_size in ${batch_size_list[*]}; do
|
||||
for precision in ${fp_items_list[*]}; do
|
||||
for device_num in ${device_num_list[*]}; do
|
||||
# sed batchsize and precision
|
||||
func_sed_params "$FILENAME" "${line_precision}" "$precision"
|
||||
func_sed_params "$FILENAME" "${line_batchsize}" "$MODE=$batch_size"
|
||||
func_sed_params "$FILENAME" "${line_epoch}" "$MODE=$epoch"
|
||||
gpu_id=$(set_gpu_id $device_num)
|
||||
|
||||
if [ ${#gpu_id} -le 1 ];then
|
||||
run_process_type="SingleP"
|
||||
log_path="$SAVE_LOG/profiling_log"
|
||||
mkdir -p $log_path
|
||||
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_profiling"
|
||||
func_sed_params "$FILENAME" "${line_gpuid}" "0" # sed used gpu_id
|
||||
# set profile_option params
|
||||
tmp=`sed -i "${line_profile}s/.*/${profile_option}/" "${FILENAME}"`
|
||||
|
||||
# run test_train_inference_python.sh
|
||||
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
|
||||
echo $cmd
|
||||
eval $cmd
|
||||
eval "cat ${log_path}/${log_name}"
|
||||
|
||||
# without profile
|
||||
log_path="$SAVE_LOG/train_log"
|
||||
speed_log_path="$SAVE_LOG/index"
|
||||
mkdir -p $log_path
|
||||
mkdir -p $speed_log_path
|
||||
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_log"
|
||||
speed_log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_speed"
|
||||
func_sed_params "$FILENAME" "${line_profile}" "null" # sed profile_id as null
|
||||
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
|
||||
echo $cmd
|
||||
job_bt=`date '+%Y%m%d%H%M%S'`
|
||||
eval $cmd
|
||||
job_et=`date '+%Y%m%d%H%M%S'`
|
||||
export model_run_time=$((${job_et}-${job_bt}))
|
||||
eval "cat ${log_path}/${log_name}"
|
||||
|
||||
# parser log
|
||||
_model_name="${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}"
|
||||
cmd="${python} ${BENCHMARK_ROOT}/scripts/analysis.py --filename ${log_path}/${log_name} \
|
||||
--speed_log_file '${speed_log_path}/${speed_log_name}' \
|
||||
--model_name ${_model_name} \
|
||||
--base_batch_size ${batch_size} \
|
||||
--run_mode ${run_mode} \
|
||||
--run_process_type ${run_process_type} \
|
||||
--fp_item ${precision} \
|
||||
--keyword ips: \
|
||||
--skip_steps 2 \
|
||||
--device_num ${device_num} \
|
||||
--speed_unit samples/s \
|
||||
--convergence_key loss: "
|
||||
echo $cmd
|
||||
eval $cmd
|
||||
last_status=${PIPESTATUS[0]}
|
||||
status_check $last_status "${cmd}" "${status_log}"
|
||||
else
|
||||
IFS=";"
|
||||
unset_env=`unset CUDA_VISIBLE_DEVICES`
|
||||
run_process_type="MultiP"
|
||||
log_path="$SAVE_LOG/train_log"
|
||||
speed_log_path="$SAVE_LOG/index"
|
||||
mkdir -p $log_path
|
||||
mkdir -p $speed_log_path
|
||||
log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_log"
|
||||
speed_log_name="${repo_name}_${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}_${device_num}_speed"
|
||||
func_sed_params "$FILENAME" "${line_gpuid}" "$gpu_id" # sed used gpu_id
|
||||
func_sed_params "$FILENAME" "${line_profile}" "null" # sed --profile_option as null
|
||||
cmd="bash test_tipc/test_train_inference_python.sh ${FILENAME} benchmark_train > ${log_path}/${log_name} 2>&1 "
|
||||
echo $cmd
|
||||
job_bt=`date '+%Y%m%d%H%M%S'`
|
||||
eval $cmd
|
||||
job_et=`date '+%Y%m%d%H%M%S'`
|
||||
export model_run_time=$((${job_et}-${job_bt}))
|
||||
eval "cat ${log_path}/${log_name}"
|
||||
# parser log
|
||||
_model_name="${model_name}_bs${batch_size}_${precision}_${run_process_type}_${run_mode}"
|
||||
|
||||
cmd="${python} ${BENCHMARK_ROOT}/scripts/analysis.py --filename ${log_path}/${log_name} \
|
||||
--speed_log_file '${speed_log_path}/${speed_log_name}' \
|
||||
--model_name ${_model_name} \
|
||||
--base_batch_size ${batch_size} \
|
||||
--run_mode ${run_mode} \
|
||||
--run_process_type ${run_process_type} \
|
||||
--fp_item ${precision} \
|
||||
--keyword ips: \
|
||||
--skip_steps 2 \
|
||||
--device_num ${device_num} \
|
||||
--speed_unit images/s \
|
||||
--convergence_key loss: "
|
||||
echo $cmd
|
||||
eval $cmd
|
||||
last_status=${PIPESTATUS[0]}
|
||||
status_check $last_status "${cmd}" "${status_log}"
|
||||
fi
|
||||
done
|
||||
done
|
||||
done
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
===========================train_params===========================
|
||||
model_name:det_mv3_db_v2
|
||||
python:python
|
||||
gpu_list:0
|
||||
Global.use_gpu:True
|
||||
Global.auto_cast:null
|
||||
Global.epoch_num:benchmark_train=2
|
||||
Global.save_model_dir:./output/
|
||||
Train.loader.batch_size_per_card:benchmark_train=16
|
||||
Global.pretrained_model:null
|
||||
train_model_name:latest
|
||||
train_infer_img_dir:null
|
||||
--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
|
||||
##
|
||||
trainer:norm_train
|
||||
norm_train:tools/train.py -c configs/det/det_mv3_db.yml -o Global.pretrained_model=./pretrain_models/MobileNetV3_large_x0_5_pretrained
|
||||
pact_train:null
|
||||
fpgm_train:null
|
||||
distill_train:null
|
||||
null:null
|
||||
null:null
|
||||
##
|
||||
===========================eval_params===========================
|
||||
eval:null
|
||||
null:null
|
||||
##
|
||||
===========================infer_params===========================
|
||||
Global.save_inference_dir:./output/
|
||||
Global.checkpoints:
|
||||
norm_export:null
|
||||
===========================benchmark_params==========================
|
||||
device_num:N1C1|N1C8|N4C32
|
||||
run_process_type:MultiP
|
||||
run_mode:DP|DP1-MP1-PP1|DP2-MP2-PP2
|
||||
speed_unit:images/s
|
||||
skip_steps:2
|
||||
keyword:ips:
|
||||
convergence_key:loss:
|
||||
flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
|
||||
null:null
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
===========================train_params===========================
|
||||
model_name:det_mv3_db_v2_0
|
||||
python:python3.7
|
||||
gpu_list:0|0,1
|
||||
Global.use_gpu:True|True
|
||||
Global.auto_cast:null
|
||||
Global.epoch_num:lite_train_lite_infer=1|whole_train_whole_infer=300
|
||||
Global.save_model_dir:./output/
|
||||
Train.loader.batch_size_per_card:lite_train_lite_infer=2|whole_train_whole_infer=4
|
||||
Global.pretrained_model:null
|
||||
train_model_name:latest
|
||||
train_infer_img_dir:./train_data/icdar2015/text_localization/ch4_test_images/
|
||||
null:null
|
||||
##
|
||||
trainer:norm_train
|
||||
norm_train:tools/train.py -c configs/det/det_mv3_db.yml -o Global.pretrained_model=./pretrain_models/MobileNetV3_large_x0_5_pretrained
|
||||
pact_train:null
|
||||
fpgm_train:null
|
||||
distill_train:null
|
||||
null:null
|
||||
null:null
|
||||
##
|
||||
===========================eval_params===========================
|
||||
eval:null
|
||||
null:null
|
||||
##
|
||||
===========================infer_params===========================
|
||||
Global.save_inference_dir:./output/
|
||||
Global.checkpoints:
|
||||
norm_export:tools/export_model.py -c configs/det/det_mv3_db.yml -o
|
||||
quant_export:null
|
||||
fpgm_export:null
|
||||
distill_export:null
|
||||
export1:null
|
||||
export2:null
|
||||
inference_dir:null
|
||||
train_model:./inference/det_mv3_db_v2.0_train/best_accuracy
|
||||
infer_export:tools/export_model.py -c configs/det/det_mv3_db.yml -o
|
||||
infer_quant:False
|
||||
inference:tools/infer/predict_det.py
|
||||
--use_gpu:True|False
|
||||
--enable_mkldnn:True|False
|
||||
--cpu_threads:1|6
|
||||
--rec_batch_num:1
|
||||
--use_tensorrt:False|True
|
||||
--precision:fp32|fp16|int8
|
||||
--det_model_dir:
|
||||
--image_dir:./inference/ch_det_data_50/all-sum-510/
|
||||
null:null
|
||||
--benchmark:True
|
||||
null:null
|
||||
===========================train_benchmark_params==========================
|
||||
batch_size:8|16
|
||||
fp_items:fp32|fp16
|
||||
epoch:2
|
||||
--profiler_options:batch_range=[10,20];state=GPU;tracer_option=Default;profile_path=model.profile
|
||||
flags:FLAGS_eager_delete_tensor_gb=0.0;FLAGS_fraction_of_gpu_memory_to_use=0.98;FLAGS_conv_workspace_size_limit=4096
|
||||
Loading…
Reference in New Issue