diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml
index 5892b24dd2..510d4c9dc3 100644
--- a/.github/actions/build_cmake/action.yml
+++ b/.github/actions/build_cmake/action.yml
@@ -25,21 +25,21 @@ runs:
       run: |
         conda config --set solver libmamba
         conda update -y -q conda
-    - name: Install env using main channel
-      if: inputs.raft == 'OFF'
+    - name: Configure conda environment
       shell: bash
       run: |
-        conda install -y -q python=3.11 cmake make swig mkl=2023 mkl-devel=2023 numpy scipy pytest gxx_linux-64=11.2 sysroot_linux-64
-    - name: Install env using conda-forge channel
-      if: inputs.raft == 'ON'
-      shell: bash
-      run: |
-        conda install -y -q python=3.11 cmake make swig mkl=2023 mkl-devel=2023 numpy scipy pytest gxx_linux-64=11.2 sysroot_linux-64=2.28 libraft cuda-version=11.8 cuda-toolkit -c rapidsai-nightly -c "nvidia/label/cuda-11.8.0" -c conda-forge
+        conda install -y -q -c conda-forge gxx_linux-64=11.2 sysroot_linux-64=2.28
+        conda install -y -q python=3.11 cmake make swig mkl=2023 mkl-devel=2023 numpy scipy pytest
     - name: Install CUDA
       if: inputs.gpu == 'ON' && inputs.raft == 'OFF'
       shell: bash
       run: |
         conda install -y -q cuda-toolkit -c "nvidia/label/cuda-11.8.0"
+    - name: Install RAFT
+      if: inputs.raft == 'ON'
+      shell: bash
+      run: |
+        conda install -y -q libraft cuda-version=11.8 cuda-toolkit -c rapidsai-nightly -c "nvidia/label/cuda-11.8.0" -c conda-forge
     - name: Build all targets
       shell: bash
       run: |