From 10e5cf6e0f2f134ed208e08a173a2f1401730365 Mon Sep 17 00:00:00 2001
From: RunningLeon <mnsheng@yeah.net>
Date: Fri, 25 Nov 2022 21:16:22 +0800
Subject: [PATCH] update reg test  (#1440)

* give model path if stead of 'x' when conversion failed

* set --models with default value ['all']

* fix mmseg yml
---
 tests/regression/mmseg.yml |  2 --
 tools/regression_test.py   | 34 +++++++++++++---------------------
 2 files changed, 13 insertions(+), 23 deletions(-)

diff --git a/tests/regression/mmseg.yml b/tests/regression/mmseg.yml
index 70fb63763..9c5a5e179 100644
--- a/tests/regression/mmseg.yml
+++ b/tests/regression/mmseg.yml
@@ -194,7 +194,6 @@ models:
       - *pipeline_trt_dynamic_fp16
       - *pipeline_ncnn_static_fp32
       - *pipeline_pplnn_dynamic_fp32
-      - *pipeline_openvino_dynamic_fp32
 
   - name: ANN
     metafile: configs/ann/ann.yml
@@ -379,4 +378,3 @@ models:
       - *pipeline_trt_static_fp32_512x512
       - *pipeline_openvino_static_fp32_512x512
       - *pipeline_ncnn_static_fp32
-      - *pipeline_ts_fp32
diff --git a/tools/regression_test.py b/tools/regression_test.py
index 0c25ace0e..5fece5458 100644
--- a/tools/regression_test.py
+++ b/tools/regression_test.py
@@ -36,7 +36,8 @@ def parse_args():
         help='test performance if it set')
     parser.add_argument(
         '--backends', nargs='+', help='test specific backend(s)')
-    parser.add_argument('--models', nargs='+', help='test specific model(s)')
+    parser.add_argument(
+        '--models', nargs='+', default=['all'], help='test specific model(s)')
     parser.add_argument(
         '--work-dir',
         type=str,
@@ -230,20 +231,16 @@ def update_report(report_dict: dict, model_name: str, model_config: str,
         checkpoint = str(checkpoint).split(f'/{codebase_name}/')[-1]
         checkpoint = '${CHECKPOINT_DIR}' + f'/{codebase_name}/{checkpoint}'
     else:
-        if Path(checkpoint).exists():
-            # To invoice the path which is 'A.a B.b' when test sdk.
-            checkpoint = Path(checkpoint).absolute().resolve()
-        elif backend_name == 'ncnn':
-            # ncnn have 2 backend file but only need xxx.param
-            checkpoint = checkpoint.split('.param')[0] + '.param'
+        parent_dir, filename = os.path.split(checkpoint)
+        parent_dir = os.path.abspath(parent_dir)
         work_dir = report_txt_path.parent.absolute().resolve()
-        checkpoint = str(checkpoint).replace(str(work_dir), '${WORK_DIR}')
+        parent_dir = parent_dir.replace(str(work_dir), '${WORK_DIR}')
+        checkpoint = os.path.join(parent_dir, filename)
 
     # save to tmp file
     tmp_str = f'{model_name},{model_config},{task_name},{checkpoint},' \
               f'{dataset},{backend_name},{deploy_config},' \
-              f'{static_or_dynamic},{precision_type},{conversion_result},' \
-              f'{fps},'
+              f'{static_or_dynamic},{precision_type},{conversion_result},'
 
     # save to report
     report_dict.get('Model').append(model_name)
@@ -725,12 +722,14 @@ def get_backend_result(pipeline_info: dict, model_cfg_path: Path,
     logger.info(f'Got convert_result = {convert_result}')
 
     if isinstance(backend_file_name, list):
+        report_checkpoint = backend_output_path.joinpath(backend_file_name[0])
         convert_checkpoint_path = ''
         for backend_file in backend_file_name:
             backend_path = backend_output_path.joinpath(backend_file)
             backend_path = str(backend_path.absolute().resolve())
             convert_checkpoint_path += f'{str(backend_path)} '
     else:
+        report_checkpoint = backend_output_path.joinpath(backend_file_name)
         convert_checkpoint_path = \
             str(backend_output_path.joinpath(backend_file_name))
 
@@ -787,10 +786,6 @@ def get_backend_result(pipeline_info: dict, model_cfg_path: Path,
         metric_list = [{metric: '-'} for metric in metric_info]
         fps = '-'
         test_pass = convert_result
-        if convert_result:
-            report_checkpoint = convert_checkpoint_path
-        else:
-            report_checkpoint = 'x'
         dataset_type = metafile_dataset['dataset']
         task_name = metafile_dataset['task']
         # update the report
@@ -799,7 +794,7 @@ def get_backend_result(pipeline_info: dict, model_cfg_path: Path,
             model_name=model_name,
             model_config=str(model_cfg_path),
             task_name=task_name,
-            checkpoint=report_checkpoint,
+            checkpoint=str(report_checkpoint),
             dataset=dataset_type,
             backend_name=backend_name,
             deploy_config=str(deploy_cfg_path),
@@ -873,11 +868,8 @@ def main():
     assert isinstance(backend_list, list)
     logger.info(f'Regression test backend list = {backend_list}')
 
-    if args.models is None:
-        logger.info('Regression test for all models in test yaml.')
-    else:
-        args.models = tuple([_filter_string(s) for s in args.models])
-        logger.info(f'Regression test models list = {args.models}')
+    args.models = [_filter_string(s) for s in args.models]
+    logger.info(f'Regression test models list = {args.models}')
 
     assert ' ' not in args.work_dir, \
         f'No empty space included in {args.work_dir}'
@@ -944,7 +936,7 @@ def main():
                                f'skipping {model_name_origin}...')
                 continue
 
-            if args.models is not None and model_name_new not in args.models:
+            if args.models != ['all'] and model_name_new not in args.models:
                 logger.info(
                     f'Test specific model mode, skip {model_name_origin}...')
                 continue