diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ac30118321..0136f2b4f8 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -22,7 +22,41 @@ jobs:
           pip install interrogate
           interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --exclude mmseg/ops --ignore-regex "__repr__" --fail-under 80 mmseg
 
-  build:
+  build_cpu:
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        python-version: [ 3.6, 3.7 ]
+        torch: [ 1.3.0, 1.5.0 ]
+        include:
+          - torch: 1.3.0
+            torchvision: 0.4.1
+          - torch: 1.5.0
+            torchvision: 0.6.0
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Pillow
+        if: ${{matrix.torchvision < 0.5}}
+        run: pip install Pillow==6.2.2
+      - name: Install PyTorch
+        run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
+      - name: Install mmseg dependencies
+        run: |
+          pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch${{matrix.torch}}/index.html
+          pip install -r requirements.txt
+      - name: Build and install
+        run: rm -rf .eggs && pip install -e .
+      - name: Run unittests and generate coverage report
+        run: |
+          coverage run --branch --source mmseg -m pytest tests/
+          coverage xml
+          coverage report -m
+
+  build_cuda:
     env:
       CUDA: 10.1.105-1
       CUDA_SHORT: 10.1
@@ -33,23 +67,16 @@ jobs:
     strategy:
       matrix:
         python-version: [ 3.6, 3.7 ]
-        torch: [ 1.3.0+cpu, 1.5.0+cpu, 1.5.0+cu101, 1.6.0+cu101, 1.7.0+cu101, 1.8.0+cu101 ]
+        torch: [ 1.5.0, 1.6.0, 1.7.0, 1.8.0 ]
         include:
-          - torch: 1.3.0+cpu
-            torchvision: 0.4.1+cpu
-          - torch: 1.5.0+cpu
-            torchvision: 0.6.0+cpu
-          - torch: 1.5.0+cu101
-            torchvision: 0.6.0+cu101
-          - torch: 1.6.0+cu101
-            torchvision: 0.7.0+cu101
-          - torch: 1.7.0+cu101
-            torchvision: 0.8.1+cu101
-          - torch: 1.8.0+cu101
-            torchvision: 0.9.0+cu101
-          - torch: 1.8.0+cu101
-            torchvision: 0.9.0+cu101
-
+          - torch: 1.5.0
+            torchvision: 0.6.0
+          - torch: 1.6.0
+            torchvision: 0.7.0
+          - torch: 1.7.0
+            torchvision: 0.8.1
+          - torch: 1.8.0
+            torchvision: 0.9.0
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
@@ -57,7 +84,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install CUDA
-        if: ${{matrix.torch == '1.5.0+cu101' || matrix.torch == '1.6.0+cu101' || matrix.torch == '1.7.0+cu101' || matrix.torch == '1.8.0+cu101'}}
         run: |
           export INSTALLER=cuda-repo-${UBUNTU_VERSION}_${CUDA}_amd64.deb
           wget http://developer.download.nvidia.com/compute/cuda/repos/${UBUNTU_VERSION}/x86_64/${INSTALLER}
@@ -72,13 +98,13 @@ jobs:
           export PATH=${CUDA_HOME}/bin:${PATH}
           sudo apt-get install -y ninja-build
       - name: Install Pillow
-        if: ${{matrix.torchvision == '0.4.1+cpu'}}
+        if: ${{matrix.torchvision < 0.5}}
         run: pip install Pillow==6.2.2
       - name: Install PyTorch
-        run: pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
+        run: pip install torch==${{matrix.torch}}+cu101 torchvision==${{matrix.torchvision}}+cu101 -f https://download.pytorch.org/whl/torch_stable.html
       - name: Install mmseg dependencies
         run: |
-          pip install mmcv-full==latest+torch${{matrix.torch}} -f https://download.openmmlab.com/mmcv/dist/index.html --use-deprecated=legacy-resolver
+          pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cu101/torch${{matrix.torch}}/index.html
           pip install -r requirements.txt
       - name: Build and install
         run: rm -rf .eggs && pip install -e .
@@ -89,7 +115,6 @@ jobs:
           coverage report -m
       # Only upload coverage report for python3.7 && pytorch1.5
       - name: Upload coverage to Codecov
-        if: ${{matrix.torch == '1.8.0+cu101' && matrix.python-version == '3.7'}}
         uses: codecov/codecov-action@v1.0.10
         with:
           file: ./coverage.xml