Fix: skip tests for uninstalled codebases (#987)

* skip tests if codebase not installed

* skip ort run test

* fix mmseg

tests/regression/mmcls.yml

@@ -38,8 +38,7 @@ onnxruntime:
 
   pipeline_ort_dynamic_fp32: &pipeline_ort_dynamic_fp32
     convert_image: *convert_image
-    backend_test: *default_backend_test
-    sdk_config: *sdk_dynamic
+    backend_test: False
     deploy_config: configs/mmcls/classification_onnxruntime_dynamic.py
 
 

tests/regression/mmdet.yml

@@ -37,8 +37,7 @@ globals:
 onnxruntime:
   pipeline_ort_static_fp32: &pipeline_ort_static_fp32
     convert_image: *convert_image
-    backend_test: *default_backend_test
-    sdk_config: *sdk_dynamic
+    backend_test: False
     deploy_config: configs/mmdet/detection/detection_onnxruntime_static.py
 
   pipeline_ort_dynamic_fp32: &pipeline_ort_dynamic_fp32
@@ -168,13 +167,13 @@ pplnn:
 torchscript:
   pipeline_ts_fp32: &pipeline_ts_fp32
     convert_image: *convert_image
-    backend_test: False
+    backend_test: *default_backend_test
     deploy_config: configs/mmdet/detection/detection_torchscript.py
 
   # ============= seg ================
   pipeline_seg_ts_fp32: &pipeline_seg_ts_fp32
     convert_image: *convert_image
-    backend_test: False
+    backend_test: *default_backend_test
     deploy_config: configs/mmdet/instance-seg/instance-seg_torchscript.py
 
 models:
@@ -206,6 +205,7 @@ models:
       - deploy_config: configs/mmdet/detection/detection_tensorrt-fp16_dynamic-300x300-512x512.py
         convert_image: *convert_image
         backend_test: *default_backend_test
+        sdk_config: *sdk_dynamic
       - deploy_config: configs/mmdet/detection/single-stage_ncnn_static-300x300.py
         convert_image: *convert_image
         backend_test: False

tests/regression/mmedit.yml

@@ -27,8 +27,6 @@ globals:
 onnxruntime:
   pipeline_ort_static_fp32: &pipeline_ort_static_fp32
     convert_image: *convert_image
-    backend_test: *default_backend_test
-    sdk_config: *sdk_dynamic
     deploy_config: configs/mmedit/super-resolution/super-resolution_onnxruntime_static.py
 
   pipeline_ort_dynamic_fp32: &pipeline_ort_dynamic_fp32

tests/regression/mmocr.yml

@@ -34,8 +34,6 @@ onnxruntime:
   # ======= detection =======
   pipeline_ort_detection_static_fp32: &pipeline_ort_detection_static_fp32
     convert_image: *convert_image_det
-    backend_test: *default_backend_test
-    sdk_config: *sdk_detection_dynamic
     deploy_config: configs/mmocr/text-detection/text-detection_onnxruntime_static.py
 
   pipeline_ort_detection_dynamic_fp32: &pipeline_ort_detection_dynamic_fp32

tests/regression/mmrotate.yml

@@ -14,7 +14,7 @@ globals:
   convert_image_det: &convert_image_det
     input_img: *img_demo
     test_img: *img_dota_demo
-  backend_test: &default_backend_test True
+  backend_test: &default_backend_test False
 
 onnxruntime:
   # ======= detection =======

tests/test_codebase/test_mmcls/test_classification.py

@@ -15,7 +15,10 @@ from mmdeploy.codebase import import_codebase
 from mmdeploy.utils import Codebase, load_config
 from mmdeploy.utils.test import DummyModel, SwitchBackendWrapper
 
-import_codebase(Codebase.MMCLS)
+try:
+    import_codebase(Codebase.MMCLS)
+except ImportError:
+    pytest.skip(f'{Codebase.MMCLS} is not installed.', allow_module_level=True)
 
 model_cfg_path = 'tests/test_codebase/test_mmcls/data/model.py'
 model_cfg = load_config(model_cfg_path)[0]

tests/test_codebase/test_mmcls/test_classification_model.py

@@ -15,7 +15,10 @@ from mmdeploy.utils.test import SwitchBackendWrapper, backend_checker
 NUM_CLASS = 1000
 IMAGE_SIZE = 64
 
-import_codebase(Codebase.MMCLS)
+try:
+    import_codebase(Codebase.MMCLS)
+except ImportError:
+    pytest.skip(f'{Codebase.MMCLS} is not installed.', allow_module_level=True)
 
 
 @backend_checker(Backend.ONNXRUNTIME)

tests/test_codebase/test_mmcls/test_mmcls_models.py

@@ -9,7 +9,10 @@ from mmdeploy.core import RewriterContext
 from mmdeploy.utils import Backend, Codebase
 from mmdeploy.utils.test import WrapModel, check_backend, get_rewrite_outputs
 
-import_codebase(Codebase.MMCLS)
+try:
+    import_codebase(Codebase.MMCLS)
+except ImportError:
+    pytest.skip(f'{Codebase.MMCLS} is not installed.', allow_module_level=True)
 
 input = torch.rand(1)
 

tests/test_codebase/test_mmdet/test_mmdet_core.py

@@ -13,7 +13,10 @@ from mmdeploy.utils.test import (WrapFunction, WrapModel, backend_checker,
                                  check_backend, get_onnx_model,
                                  get_rewrite_outputs)
 
-import_codebase(Codebase.MMDET)
+try:
+    import_codebase(Codebase.MMDET)
+except ImportError:
+    pytest.skip(f'{Codebase.MMDET} is not installed.', allow_module_level=True)
 
 
 @backend_checker(Backend.TENSORRT)

tests/test_codebase/test_mmdet/test_mmdet_models.py

@@ -15,7 +15,10 @@ from mmdeploy.utils.config_utils import get_ir_config
 from mmdeploy.utils.test import (WrapModel, check_backend, get_model_outputs,
                                  get_rewrite_outputs)
 
-import_codebase(Codebase.MMDET)
+try:
+    import_codebase(Codebase.MMDET)
+except ImportError:
+    pytest.skip(f'{Codebase.MMDET} is not installed.', allow_module_level=True)
 
 
 def seed_everything(seed=1029):

tests/test_codebase/test_mmdet/test_mmdet_utils.py

@@ -1,6 +1,7 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import mmcv
 import numpy as np
+import pytest
 import torch
 
 from mmdeploy.codebase import import_codebase
@@ -9,7 +10,10 @@ from mmdeploy.codebase.mmdet import (clip_bboxes, get_post_processing_params,
                                      pad_with_value_if_necessary)
 from mmdeploy.utils import Codebase
 
-import_codebase(Codebase.MMDET)
+try:
+    import_codebase(Codebase.MMDET)
+except ImportError:
+    pytest.skip(f'{Codebase.MMDET} is not installed.', allow_module_level=True)
 
 
 def test_clip_bboxes():

tests/test_codebase/test_mmdet/test_object_detection.py

@@ -17,7 +17,10 @@ from mmdeploy.codebase import import_codebase
 from mmdeploy.utils import Codebase, load_config
 from mmdeploy.utils.test import DummyModel, SwitchBackendWrapper
 
-import_codebase(Codebase.MMDET)
+try:
+    import_codebase(Codebase.MMDET)
+except ImportError:
+    pytest.skip(f'{Codebase.MMDET} is not installed.', allow_module_level=True)
 
 model_cfg_path = 'tests/test_codebase/test_mmdet/data/model.py'
 model_cfg = load_config(model_cfg_path)[0]

tests/test_codebase/test_mmdet/test_object_detection_model.py

@@ -14,7 +14,10 @@ from mmdeploy.codebase.mmdet.deploy.object_detection_model import End2EndModel
 from mmdeploy.utils import Backend, Codebase
 from mmdeploy.utils.test import SwitchBackendWrapper, backend_checker
 
-import_codebase(Codebase.MMDET)
+try:
+    import_codebase(Codebase.MMDET)
+except ImportError:
+    pytest.skip(f'{Codebase.MMDET} is not installed.', allow_module_level=True)
 
 
 def assert_det_results(results, module_name: str = 'model'):

tests/test_codebase/test_mmedit/test_mmedit_models.py

@@ -5,14 +5,18 @@ from typing import Dict
 
 import mmcv
 import onnx
+import pytest
 import torch
-from mmedit.models.backbones.sr_backbones import SRCNN
 
 from mmdeploy.codebase import import_codebase
 from mmdeploy.core import RewriterContext
 from mmdeploy.utils import Backend, Codebase, get_onnx_config
 
-import_codebase(Codebase.MMEDIT)
+try:
+    import_codebase(Codebase.MMEDIT)
+except ImportError:
+    pytest.skip(
+        f'{Codebase.MMEDIT} is not installed.', allow_module_level=True)
 
 img = torch.rand(1, 3, 4, 4)
 model_file = tempfile.NamedTemporaryFile(suffix='.onnx').name
@@ -46,6 +50,7 @@ deploy_cfg = mmcv.Config(
 
 
 def test_srcnn():
+    from mmedit.models.backbones.sr_backbones import SRCNN
     pytorch_model = SRCNN()
     model_inputs = {'x': img}
 

tests/test_codebase/test_mmedit/test_super_resolution.py

@@ -14,7 +14,11 @@ from mmdeploy.codebase import import_codebase
 from mmdeploy.utils import Codebase, load_config
 from mmdeploy.utils.test import SwitchBackendWrapper
 
-import_codebase(Codebase.MMEDIT)
+try:
+    import_codebase(Codebase.MMEDIT)
+except ImportError:
+    pytest.skip(
+        f'{Codebase.MMEDIT} is not installed.', allow_module_level=True)
 
 model_cfg = 'tests/test_codebase/test_mmedit/data/model.py'
 model_cfg = load_config(model_cfg)[0]

tests/test_codebase/test_mmedit/test_super_resolution_model.py

@@ -1,6 +1,7 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import mmcv
 import numpy as np
+import pytest
 import torch
 
 import mmdeploy.backend.onnxruntime as ort_apis
@@ -8,7 +9,11 @@ from mmdeploy.codebase import import_codebase
 from mmdeploy.utils import Backend, Codebase, load_config
 from mmdeploy.utils.test import SwitchBackendWrapper, backend_checker
 
-import_codebase(Codebase.MMEDIT)
+try:
+    import_codebase(Codebase.MMEDIT)
+except ImportError:
+    pytest.skip(
+        f'{Codebase.MMEDIT} is not installed.', allow_module_level=True)
 
 
 @backend_checker(Backend.ONNXRUNTIME)

tests/test_codebase/test_mmocr/test_mmocr_models.py

@@ -5,7 +5,6 @@ import mmcv
 import numpy as np
 import pytest
 import torch
-from mmocr.models.textdet.necks import FPNC
 
 from mmdeploy.codebase import import_codebase
 from mmdeploy.core import RewriterContext, patch_model
@@ -13,7 +12,12 @@ from mmdeploy.utils import Backend, Codebase
 from mmdeploy.utils.test import (WrapModel, check_backend, get_model_outputs,
                                  get_rewrite_outputs)
 
-import_codebase(Codebase.MMOCR)
+try:
+    import_codebase(Codebase.MMOCR)
+except ImportError:
+    pytest.skip(f'{Codebase.MMOCR} is not installed.', allow_module_level=True)
+
+from mmocr.models.textdet.necks import FPNC
 
 
 class FPNCNeckModel(FPNC):

tests/test_codebase/test_mmocr/test_text_detection.py

@@ -14,7 +14,10 @@ from mmdeploy.codebase import import_codebase
 from mmdeploy.utils import Codebase, load_config
 from mmdeploy.utils.test import DummyModel, SwitchBackendWrapper
 
-import_codebase(Codebase.MMOCR)
+try:
+    import_codebase(Codebase.MMOCR)
+except ImportError:
+    pytest.skip(f'{Codebase.MMOCR} is not installed.', allow_module_level=True)
 
 model_cfg_path = 'tests/test_codebase/test_mmocr/data/dbnet.py'
 model_cfg = load_config(model_cfg_path)[0]

tests/test_codebase/test_mmocr/test_text_detection_models.py

@@ -12,7 +12,10 @@ from mmdeploy.codebase import import_codebase
 from mmdeploy.utils import Backend, Codebase, load_config
 from mmdeploy.utils.test import SwitchBackendWrapper, backend_checker
 
-import_codebase(Codebase.MMOCR)
+try:
+    import_codebase(Codebase.MMOCR)
+except ImportError:
+    pytest.skip(f'{Codebase.MMOCR} is not installed.', allow_module_level=True)
 
 IMAGE_SIZE = 32
 

tests/test_codebase/test_mmocr/test_text_recognition.py

@@ -14,7 +14,10 @@ from mmdeploy.codebase import import_codebase
 from mmdeploy.utils import Codebase, load_config
 from mmdeploy.utils.test import DummyModel, SwitchBackendWrapper
 
-import_codebase(Codebase.MMOCR)
+try:
+    import_codebase(Codebase.MMOCR)
+except ImportError:
+    pytest.skip(f'{Codebase.MMOCR} is not installed.', allow_module_level=True)
 
 model_cfg_path = 'tests/test_codebase/test_mmocr/data/crnn.py'
 model_cfg = load_config(model_cfg_path)[0]

tests/test_codebase/test_mmocr/test_text_recognition_models.py

@@ -12,7 +12,10 @@ from mmdeploy.codebase import import_codebase
 from mmdeploy.utils import Backend, Codebase, load_config
 from mmdeploy.utils.test import SwitchBackendWrapper, backend_checker
 
-import_codebase(Codebase.MMOCR)
+try:
+    import_codebase(Codebase.MMOCR)
+except ImportError:
+    pytest.skip(f'{Codebase.MMOCR} is not installed.', allow_module_level=True)
 
 IMAGE_SIZE = 32
 

tests/test_codebase/test_mmseg/test_mmseg_models.py

@@ -5,15 +5,19 @@ import pytest
 import torch
 import torch.nn as nn
 from mmcv import ConfigDict
-from mmseg.models import BACKBONES, HEADS
-from mmseg.models.decode_heads.decode_head import BaseDecodeHead
 
 from mmdeploy.codebase import import_codebase
 from mmdeploy.utils import Backend, Codebase, Task
 from mmdeploy.utils.test import (WrapModel, check_backend, get_model_outputs,
                                  get_rewrite_outputs)
 
-import_codebase(Codebase.MMSEG)
+try:
+    import_codebase(Codebase.MMSEG)
+except ImportError:
+    pytest.skip(f'{Codebase.MMSEG} is not installed.', allow_module_level=True)
+
+from mmseg.models import BACKBONES, HEADS
+from mmseg.models.decode_heads.decode_head import BaseDecodeHead
 
 
 @BACKBONES.register_module()

tests/test_codebase/test_mmseg/test_segmentation.py

@@ -16,7 +16,10 @@ from mmdeploy.codebase import import_codebase
 from mmdeploy.utils import Codebase, load_config
 from mmdeploy.utils.test import DummyModel, SwitchBackendWrapper
 
-import_codebase(Codebase.MMSEG)
+try:
+    import_codebase(Codebase.MMSEG)
+except ImportError:
+    pytest.skip(f'{Codebase.MMSEG} is not installed.', allow_module_level=True)
 
 model_cfg_path = 'tests/test_codebase/test_mmseg/data/model.py'
 model_cfg = load_config(model_cfg_path)[0]

tests/test_codebase/test_mmseg/test_segmentation_model.py

@@ -12,7 +12,10 @@ from mmdeploy.codebase import import_codebase
 from mmdeploy.utils import Backend, Codebase
 from mmdeploy.utils.test import SwitchBackendWrapper, backend_checker
 
-import_codebase(Codebase.MMSEG)
+try:
+    import_codebase(Codebase.MMSEG)
+except ImportError:
+    pytest.skip(f'{Codebase.MMSEG} is not installed.', allow_module_level=True)
 
 NUM_CLASS = 19
 IMAGE_SIZE = 32