From bce276ef24e4801c07b6ec43fca512875d18ab02 Mon Sep 17 00:00:00 2001 From: kaizhong <97659362+kaizhong2021@users.noreply.github.com> Date: Wed, 18 Jan 2023 16:32:26 +0800 Subject: [PATCH] [Feature]: add a tool to generate supported-backends markdown table (#1374) * convert2markdown * update yaml2mardown code * code update * add parse_args * add parse_args * add parse_args * add parse_args * add website list * add website list * add website list * add website list * add website list * add website list * add website list * add url in yaml * add table in convert * add table in convert * From yaml export markdown * From yaml export markdown * From yaml export markdown * From yaml export markdown * From yaml export markdown * From yaml export markdown * Rename convert.py to generate_md_table.py generate_markdownd_table * docs(project): sync en and zh docs * docs(project): sync en and zh docs * docs(project): sync en and zh docs * docs(project): sync en and zh docs * docs(project): sync en and zh docs * docs(project): sync en and zh docs * docs(project): sync en and zh docs * Update mmaction.yml * add backends parser * add backends parser * Add type for the codeblock. * move to useful tools --- docs/en/02-how-to-run/useful_tools.md | 35 ++++++++++ docs/zh_cn/02-how-to-run/useful_tools.md | 35 ++++++++++ requirements/tests.txt | 1 + tests/regression/mmaction.yml | 1 + tests/regression/mmcls.yml | 1 + tests/regression/mmdet.yml | 1 + tests/regression/mmdet3d.yml | 1 + tests/regression/mmedit.yml | 1 + tests/regression/mmocr.yml | 1 + tests/regression/mmpose.yml | 1 + tests/regression/mmrotate.yml | 1 + tests/regression/mmseg.yml | 1 + tools/generate_md_table.py | 81 ++++++++++++++++++++++++ 13 files changed, 161 insertions(+) create mode 100644 tools/generate_md_table.py diff --git a/docs/en/02-how-to-run/useful_tools.md b/docs/en/02-how-to-run/useful_tools.md index ee69c51a2..99e63aedd 100644 --- a/docs/en/02-how-to-run/useful_tools.md +++ b/docs/en/02-how-to-run/useful_tools.md @@ -202,3 +202,38 @@ And the output look like this: | Max | 1.689 | 591.983 | +--------+------------+---------+ ``` + +## generate_md_table + +This tool can be used to generate supported-backends markdown table. + +### Usage + +```shell +python tools/generate_md_table.py \ + ${yml_file} \ + ${output} \ + ${backends} +``` + +### Description of all arguments + +- `yml_file:` input yml config path +- `output:` output markdown file path +- `backends:` output backends list. If not specified, it will be set 'onnxruntime' 'tensorrt' 'torchscript' 'pplnn' 'openvino' 'ncnn'. + +### Example: + +Generate backends markdown table from mmocr.yml + +```shell +python tools/generate_md_table.py tests/regression/mmocr.yml tests/regression/mmocr.md onnxruntime tensorrt torchscript pplnn openvino ncnn +``` + +And the output look like this: + +| model | task | onnxruntime | tensorrt | torchscript | pplnn | openvino | ncnn | +| :--------------------------------------------------------------------------- | :-------------- | :---------- | :------- | :---------- | :---- | :------- | :--- | +| [DBNet](https://github.com/open-mmlab/mmocr/tree/main/configs/textdet/dbnet) | TextDetection | Y | Y | Y | Y | Y | Y | +| [CRNN](https://github.com/open-mmlab/mmocr/tree/main/configs/textrecog/crnn) | TextRecognition | Y | Y | Y | Y | N | Y | +| [SAR](https://github.com/open-mmlab/mmocr/tree/main/configs/textrecog/sar) | TextRecognition | Y | N | N | N | N | N | diff --git a/docs/zh_cn/02-how-to-run/useful_tools.md b/docs/zh_cn/02-how-to-run/useful_tools.md index bbd823c69..6cf881cec 100644 --- a/docs/zh_cn/02-how-to-run/useful_tools.md +++ b/docs/zh_cn/02-how-to-run/useful_tools.md @@ -202,3 +202,38 @@ python tools/profiler.py \ | Max | 1.689 | 591.983 | +--------+------------+---------+ ``` + +## generate_md_table + +生成mmdeploy支持的后端表。 + +### 用法 + +```shell +python tools/generate_md_table.py \ + ${yml_file} \ + ${output} \ + ${backends} +``` + +### 参数说明 + +- `yml_file:` 输入 yml 配置路径 +- `output:` 输出markdown文件路径 +- `backends:` 要输出的后端，默认为 onnxruntime tensorrt torchscript pplnn openvino ncnn + +### 使用举例 + +从 mmocr.yml 生成mmdeploy支持的后端表 + +```shell +python tools/generate_md_table.py tests/regression/mmocr.yml tests/regression/mmocr.md onnxruntime tensorrt torchscript pplnn openvino ncnn +``` + +输出： + +| model | task | onnxruntime | tensorrt | torchscript | pplnn | openvino | ncnn | +| :--------------------------------------------------------------------------- | :-------------- | :---------- | :------- | :---------- | :---- | :------- | :--- | +| [DBNet](https://github.com/open-mmlab/mmocr/tree/main/configs/textdet/dbnet) | TextDetection | Y | Y | Y | Y | Y | Y | +| [CRNN](https://github.com/open-mmlab/mmocr/tree/main/configs/textrecog/crnn) | TextRecognition | Y | Y | Y | Y | N | Y | +| [SAR](https://github.com/open-mmlab/mmocr/tree/main/configs/textrecog/sar) | TextRecognition | Y | N | N | N | N | N | diff --git a/requirements/tests.txt b/requirements/tests.txt index 581ae5c08..3e3d6b154 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,5 +1,6 @@ asynctest coverage +easydict flake8 interrogate isort==4.3.21 diff --git a/tests/regression/mmaction.yml b/tests/regression/mmaction.yml index 1b8cd3897..6f8815e5d 100644 --- a/tests/regression/mmaction.yml +++ b/tests/regression/mmaction.yml @@ -1,4 +1,5 @@ globals: + repo_url: https://github.com/open-mmlab/mmaction2/tree/master codebase_dir: ../mmaction2 checkpoint_force_download: False images: diff --git a/tests/regression/mmcls.yml b/tests/regression/mmcls.yml index 90a5e063c..a13f120c0 100644 --- a/tests/regression/mmcls.yml +++ b/tests/regression/mmcls.yml @@ -1,4 +1,5 @@ globals: + repo_url: https://github.com/open-mmlab/mmclassification/tree/master codebase_dir: ../mmclassification checkpoint_force_download: False images: diff --git a/tests/regression/mmdet.yml b/tests/regression/mmdet.yml index 0e0226a6e..2dc8408cc 100644 --- a/tests/regression/mmdet.yml +++ b/tests/regression/mmdet.yml @@ -1,4 +1,5 @@ globals: + repo_url: https://github.com/open-mmlab/mmdetection/tree/master codebase_dir: ../mmdetection checkpoint_force_download: False images: diff --git a/tests/regression/mmdet3d.yml b/tests/regression/mmdet3d.yml index 52ff03579..f188d0a51 100644 --- a/tests/regression/mmdet3d.yml +++ b/tests/regression/mmdet3d.yml @@ -1,4 +1,5 @@ globals: + repo_url: https://github.com/open-mmlab/mmdetection3d/tree/master codebase_dir: ../mmdetection3d checkpoint_force_download: False images: diff --git a/tests/regression/mmedit.yml b/tests/regression/mmedit.yml index ce510ce5e..019cfa814 100644 --- a/tests/regression/mmedit.yml +++ b/tests/regression/mmedit.yml @@ -1,4 +1,5 @@ globals: + repo_url: https://github.com/open-mmlab/mmediting/tree/master codebase_dir: ../mmediting checkpoint_force_download: False images: diff --git a/tests/regression/mmocr.yml b/tests/regression/mmocr.yml index 2da113cee..948fbab05 100644 --- a/tests/regression/mmocr.yml +++ b/tests/regression/mmocr.yml @@ -1,4 +1,5 @@ globals: + repo_url: https://github.com/open-mmlab/mmocr/tree/main codebase_dir: ../mmocr checkpoint_force_download: False images: diff --git a/tests/regression/mmpose.yml b/tests/regression/mmpose.yml index 895114796..16b8044e7 100644 --- a/tests/regression/mmpose.yml +++ b/tests/regression/mmpose.yml @@ -1,4 +1,5 @@ globals: + repo_url: https://github.com/open-mmlab/mmpose/tree/master codebase_dir: ../mmpose checkpoint_force_download: False images: diff --git a/tests/regression/mmrotate.yml b/tests/regression/mmrotate.yml index 5fcbb97f5..172703de4 100644 --- a/tests/regression/mmrotate.yml +++ b/tests/regression/mmrotate.yml @@ -1,4 +1,5 @@ globals: + repo_url: https://github.com/open-mmlab/mmrotate/tree/main codebase_dir: ../mmrotate checkpoint_force_download: False images: diff --git a/tests/regression/mmseg.yml b/tests/regression/mmseg.yml index 9c5a5e179..9682ea514 100644 --- a/tests/regression/mmseg.yml +++ b/tests/regression/mmseg.yml @@ -1,4 +1,5 @@ globals: + repo_url: https://github.com/open-mmlab/mmsegmentation/tree/master codebase_dir: ../mmsegmentation checkpoint_force_download: False images: diff --git a/tools/generate_md_table.py b/tools/generate_md_table.py new file mode 100644 index 000000000..e8a64eeb6 --- /dev/null +++ b/tools/generate_md_table.py @@ -0,0 +1,81 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import argparse +import os +import os.path as osp + +import yaml +from easydict import EasyDict as edict + +from mmdeploy.utils import get_backend, get_task_type, load_config + + +def parse_args(): + parser = argparse.ArgumentParser( + description='from yaml export markdown table') + parser.add_argument('yml_file', help='input yml config path') + parser.add_argument('output', help='output markdown file path') + parser.add_argument( + 'backends', + nargs='*', + help='backends you want to generate', + default=[ + 'onnxruntime', 'tensorrt', 'torchscript', 'pplnn', 'openvino', + 'ncnn' + ]) + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + assert osp.exists(args.yml_file), f'File not exists: {args.yml_file}' + output_dir, _ = osp.split(args.output) + if output_dir: + os.makedirs(output_dir, exist_ok=True) + header = ['model', 'task'] + args.backends + aligner = [':--'] * 2 + [':--'] * len(args.backends) + + def write_row_f(writer, row): + writer.write('|' + '|'.join(row) + '|\n') + + print(f'Processing{args.yml_file}') + with open(args.yml_file, 'r') as reader, open(args.output, 'w') as writer: + config = yaml.load(reader, Loader=yaml.FullLoader) + config = edict(config) + write_row_f(writer, header) + write_row_f(writer, aligner) + repo_url = config.globals.repo_url + for i in range(len(config.models)): + name = config.models[i].name + model_configs = config.models[i].model_configs + pipelines = config.models[i].pipelines + config_url = osp.join(repo_url, model_configs[0]) + config_url, _ = osp.split(config_url) + support_backends = {b: 'N' for b in args.backends} + deploy_config = [ + pipelines[i].deploy_config for i in range(len(pipelines)) + ] + cfg = [ + load_config(deploy_config[i]) + for i in range(len(deploy_config)) + ] + task = [ + get_task_type(cfg[i][0]).value + for i in range(len(deploy_config)) + ] + backend_type = [ + get_backend(cfg[i][0]).value + for i in range(len(deploy_config)) + ] + for i in range(len(deploy_config)): + support_backends[backend_type[i]] = 'Y' + support_backends = [support_backends[i] for i in args.backends] + model_name = f'[{name}]({config_url})' + row = [model_name, task[i]] + support_backends + + write_row_f(writer, row) + print(f'Save to {args.output}') + + +if __name__ == '__main__': + main()