diff --git a/mmseg/models/backbones/beit.py b/mmseg/models/backbones/beit.py
index 4553f72d8..e5da71e72 100644
--- a/mmseg/models/backbones/beit.py
+++ b/mmseg/models/backbones/beit.py
@@ -11,6 +11,7 @@ from mmengine.model import BaseModule, ModuleList
 from mmengine.model.weight_init import (constant_init, kaiming_init,
                                         trunc_normal_)
 from mmengine.runner.checkpoint import _load_checkpoint
+from scipy import interpolate
 from torch.nn.modules.batchnorm import _BatchNorm
 from torch.nn.modules.utils import _pair as to_2tuple
 
@@ -18,11 +19,6 @@ from mmseg.registry import MODELS
 from ..utils import PatchEmbed
 from .vit import TransformerEncoderLayer as VisionTransformerEncoderLayer
 
-try:
-    from scipy import interpolate
-except ImportError:
-    interpolate = None
-
 
 class BEiTAttention(BaseModule):
     """Window based multi-head self-attention (W-MSA) module with relative
diff --git a/requirements/runtime.txt b/requirements/runtime.txt
index 82b392e74..7aafb8b71 100644
--- a/requirements/runtime.txt
+++ b/requirements/runtime.txt
@@ -3,3 +3,4 @@ mmcls>=1.0.0rc0
 numpy
 packaging
 prettytable
+scipy
diff --git a/tests/test_models/test_backbones/test_beit.py b/tests/test_models/test_backbones/test_beit.py
index cf3960894..59a12c5d0 100644
--- a/tests/test_models/test_backbones/test_beit.py
+++ b/tests/test_models/test_backbones/test_beit.py
@@ -140,8 +140,11 @@ def test_beit_init():
         }
     }
     model = BEiT(img_size=(512, 512))
-    with pytest.raises(AttributeError):
-        model.resize_rel_pos_embed(ckpt)
+    # If scipy is installed, this AttributeError would not be raised.
+    from mmengine.utils import is_installed
+    if not is_installed('scipy'):
+        with pytest.raises(AttributeError):
+            model.resize_rel_pos_embed(ckpt)
 
     # pretrained=None
     # init_cfg=123, whose type is unsupported
diff --git a/tests/test_models/test_backbones/test_mae.py b/tests/test_models/test_backbones/test_mae.py
index 562d067a7..16f52b54b 100644
--- a/tests/test_models/test_backbones/test_mae.py
+++ b/tests/test_models/test_backbones/test_mae.py
@@ -138,8 +138,11 @@ def test_mae_init():
         }
     }
     model = MAE(img_size=(512, 512))
-    with pytest.raises(AttributeError):
-        model.resize_rel_pos_embed(ckpt)
+    # If scipy is installed, this AttributeError would not be raised.
+    from mmengine.utils import is_installed
+    if not is_installed('scipy'):
+        with pytest.raises(AttributeError):
+            model.resize_rel_pos_embed(ckpt)
 
     # test resize abs pos embed
     ckpt = model.resize_abs_pos_embed(ckpt['state_dict'])