diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7a60d49a..9bd750f5 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -32,10 +32,8 @@ jobs:
     strategy:
       matrix:
         python-version: [3.7]
-        torch: [1.3.0, 1.5.0, 1.6.0, 1.7.0]
+        torch: [1.5.0, 1.6.0, 1.7.0]
         include:
-          - torch: 1.3.0
-            torchvision: 0.4.1
           - torch: 1.5.0
             torchvision: 0.6.0
           - torch: 1.6.0
@@ -79,10 +77,8 @@ jobs:
     strategy:
       matrix:
         python-version: [3.7]
-        torch: [1.3.0, 1.5.0, 1.6.0, 1.7.0]
+        torch: [1.5.0, 1.6.0, 1.7.0]
         include:
-          - torch: 1.3.0
-            torchvision: 0.4.1
           - torch: 1.5.0
             torchvision: 0.6.0
           - torch: 1.6.0
@@ -122,12 +118,7 @@ jobs:
         if: ${{matrix.torchvision < 0.5}}
       - name: Install PyTorch
         run: |
-          if [ ${{matrix.torch}} == '1.3.0' ]
-          then
-            pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
-          else
-            pip install torch==${{matrix.torch}}+cu101 torchvision==${{matrix.torchvision}}+cu101 -f https://download.pytorch.org/whl/torch_stable.html
-          fi
+          pip install torch==${{matrix.torch}}+cu101 torchvision==${{matrix.torchvision}}+cu101 -f https://download.pytorch.org/whl/torch_stable.html
       - name: Install MMCV
         run: pip install mmcv-full==1.3.0 -f https://download.openmmlab.com/mmcv/dist/cu101/torch${{matrix.torch}}/index.html
       - name: Install MMDet