[CI] Enable CUDA SYCL CTS tests

.github/workflows/sycl_gen_test_matrix.yml

@@ -19,6 +19,10 @@ on:
         type: string
         required: true
         default: ""
+      cts_config:
+        type: string
+        required: false
+        default: ""
       uniq:
         description: Unique string to name dynamic runners in AWS
         type: string
@@ -33,6 +37,9 @@ on:
       lts_matrix:
         description: "Generated Matrix"
         value: ${{ jobs.test_matrix.outputs.lts_matrix }}
+      cts_matrix:
+        description: "Generated SYCL CTS Matrix"
+        value: ${{ jobs.test_matrix.outputs.cts_matrix }}
       lts_aws_matrix:
         description: "Generated Matrix AWS subset"
         value: ${{ jobs.test_matrix.outputs.lts_aws_matrix }}
@@ -42,6 +49,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       lts_matrix: ${{ steps.work.outputs.lts_matrix }}
+      cts_matrix: ${{ steps.work.outputs.cts_matrix }}
       lts_aws_matrix: ${{ steps.work.outputs.lts_aws_matrix }}
     steps:
     - name: Download scripts and configs

.github/workflows/sycl_linux_build_and_test.yml

@@ -56,6 +56,18 @@ on:
         type: number
         required: false
         default: 4
+      cts_matrix:
+        type: string
+        required: false
+        default: ""
+      cts_cmake_extra_args:
+        type: string
+        required: false
+        default: ""
+      cts_ref:
+        type: string
+        required: false
+        default: 'SYCL-2020'
 
 jobs:
   build:
@@ -230,6 +242,53 @@ jobs:
         check_sycl_all: ${{ matrix.check_sycl_all }}
         results_name_suffix: ${{ matrix.config }}_${{ inputs.build_artifact_suffix }}
         cmake_args: '${{ matrix.cmake_args }} ${{ inputs.lts_cmake_extra_args }}'
+
+  khronos_sycl_cts:
+    needs: build
+    if: ${{ inputs.cts_matrix != '' }}
+    strategy:
+      fail-fast: false
+      max-parallel: ${{ inputs.max_parallel }}
+      matrix:
+        include: ${{ fromJSON(inputs.cts_matrix) }}
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.runs-on }}
+    env: ${{ matrix.env }}
+    container:
+      image: ${{ matrix.image }}
+      options: ${{ matrix.container_options }}
+    steps:
+    - name: Reset GPU
+      if: ${{ contains(matrix.config, 'gen9') }}
+      run: |
+        sudo mount -t debugfs none /sys/kernel/debug
+        sudo bash -c 'echo 1 > /sys/kernel/debug/dri/0/i915_wedged'
+    - run: cp -r /actions .
+    - name: Register cleanup after job is finished
+      uses: ./actions/cleanup
+    - name: Install drivers
+      if: env.compute_runtime_tag != ''
+      run: |
+        if [ -e /opt/install_drivers.sh ]; then
+          # TODO pack this script into container
+          wget raw.githubusercontent.com/intel/llvm/${{ github.sha }}/devops/scripts/get_release.py
+          sudo mv get_release.py /opt/
+          sudo -E /opt/install_drivers.sh --all
+        fi
+    # FIXME cached_checkout fails here, but works everywhere else
+    # TODO: figure out if we remove this action
+    - uses: actions/checkout@v2
+      with:
+        path: llvm
+    # TODO should this action be packed into container as well?
+    - uses: ./llvm/devops/actions/khronos_cts_test
+      name: Run Khronos SYCL CTS
+      with:
+        test_ref: ${{ inputs.cts_ref }}
+        sycl_artifact: sycl_linux_${{ inputs.build_artifact_suffix }}
+        sycl_archive: llvm_sycl.tar.xz
+        sycl_device_filter: ${{ matrix.sycl_device_filter }}
+        cmake_args: '${{ matrix.cmake_args }} ${{ inputs.cts_cmake_extra_args }}'
 
   aws-stop:
     name: Stop AWS

.github/workflows/sycl_post_commit.yml

@@ -17,6 +17,7 @@ jobs:
     uses: ./.github/workflows/sycl_gen_test_matrix.yml
     with:
       lts_config: "l0_gen9"
+      cts_config: "cuda"
   linux_default:
     name: Linux Default
     needs: test_matrix
@@ -26,6 +27,7 @@ jobs:
       build_cache_root: "/__w/llvm"
       build_artifact_suffix: "post_commit"
       lts_matrix: ${{ needs.test_matrix.outputs.lts_matrix }}
+      cts_matrix: ${{ needs.test_matrix.outputs.cts_matrix }}
       lts_aws_matrix: ${{ needs.test_matrix.outputs.lts_aws_matrix }}
   linux_no_assert:
     name: Linux (no assert)

devops/actions/khronos_cts_test/action.yml

@@ -0,0 +1,113 @@
+name: 'Run khronos_sycl_cts testing'
+description: 'Run SYCL tests from khronos_sycl_cts'
+inputs:
+  test_ref:
+    description: 'commit-ish identifier for test repo'
+    required: true
+    default: 'SYCL-2020'
+  sycl_artifact:
+    description: 'Name of the artifact, that contains compiler toolchain'
+    required: true
+  sycl_archive:
+    description: 'Name of SYCL toolchain archive file'
+    required: false
+    default: 'llvm_sycl.tar.xz'
+  sycl_device_filter:
+    description: 'List of SYCL backends with set of target devices per each to be tested iteratively'
+    required: true
+  cmake_args:
+    description: 'Extra arguments to cmake command'
+    required: false
+
+post-if: false
+runs:
+  using: "composite"
+  steps:
+  - run: |
+      cp -r /actions .
+      git config --global --add safe.directory /__w/repo_cache/KhronosGroup/SYCL-CTS
+    shell: bash
+  - run: apt update && apt install -yqq opencl-headers ocl-icd-opencl-dev
+    shell: bash
+  - name: Checkout SYCL CTS
+    uses: ./actions/cached_checkout
+    with:
+      path: khronos_sycl_cts
+      repository: 'KhronosGroup/SYCL-CTS'
+      ref: ${{ inputs.test_ref }}
+      default_branch: 'SYCL-2020'
+      cache_path: "/__w/repo_cache/"
+  - run: |
+      cd khronos_sycl_cts
+      git submodule update --init
+    shell: bash
+  - name: Download compiler toolchain
+    uses: actions/download-artifact@v2
+    with:
+      name: ${{ inputs.sycl_artifact }}
+  - name: Extract SYCL toolchain
+    shell: bash
+    run: |
+      mkdir toolchain
+      tar -xf ${{ inputs.sycl_archive }} -C toolchain
+      rm -f ${{ inputs.sycl_archive }}
+  - name: Build SYCL-CTS
+    shell: bash
+    run: |
+      echo "::group::CMake configuration"
+      export LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH
+      mkdir build
+      cmake  -GNinja -B./build -S./khronos_sycl_cts -DSYCL_IMPLEMENTATION=DPCPP \
+      -DCMAKE_CXX_COMPILER="$PWD/toolchain/bin/clang++" \
+      -DSYCL_CTS_EXCLUDE_TEST_CATEGORIES="$PWD/llvm/devops/cts_exclude_filter" \
+      -DSYCL_CTS_ENABLE_OPENCL_INTEROP_TESTS=OFF \
+      -DSYCL_CTS_MEASURE_BUILD_TIMES=ON \
+      -DDPCPP_INSTALL_DIR="$PWD/toolchain" \
+      ${{ inputs.cmake_args }}
+      echo "::endgroup::"
+      echo "::group::Bulid testing"
+      cd build
+      ninja
+      echo "::endgroup::"
+  - name: Run SYCL-CTS
+    shell: bash
+    run: |
+      echo "::group::Configure execution environment"
+      export PATH=$PWD/toolchain/bin/:$PATH
+      export LD_LIBRARY_PATH=$PWD/toolchain/lib/:$LD_LIBRARY_PATH
+      # TODO make this part of container build
+      export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/hip/lib/:/opt/rocm/lib
+      if [ -e /runtimes/oneapi-tbb/env/vars.sh ]; then
+        source /runtimes/oneapi-tbb/env/vars.sh;
+      elif [ -e /opt/runtimes/oneapi-tbb/env/vars.sh ]; then
+        source /opt/runtimes/oneapi-tbb/env/vars.sh;
+      else
+        echo "no TBB vars in /opt/runtimes or /runtimes";
+      fi
+      # TODO remove workaround of FPGA emu bug
+      mkdir -p icd
+      echo /usr/lib/x86_64-linux-gnu/intel-opencl/libigdrcl.so > icd/gpu.icd
+      echo /runtimes/oclcpu/x64/libintelocl.so > icd/cpu.icd
+      echo /opt/runtimes/oclcpu/x64/libintelocl.so > icd/cpu2.icd
+      export OCL_ICD_VENDORS=$PWD/icd
+      echo "::endgroup::"
+      echo "::group::sycl-ls --verbose"
+      sycl-ls --verbose
+      echo "::endgroup::"
+      echo "::group::SYCL_PI_TRACE=-1 sycl-ls"
+      echo $LD_LIBRARY_PATH
+      SYCL_PI_TRACE=-1 sycl-ls
+      echo "::endgroup::"
+      echo "::group::Run testing"
+      export SYCL_PI_CUDA_ENABLE_IMAGE_SUPPORT=1
+      export SYCL_DEVICE_FILTER=${{ matrix.sycl_device_filter }}
+      ./build/bin/test_all --list-devices
+      ./build/bin/test_all
+      echo "::endgroup::"
+  - name: Cleanup
+    shell: bash
+    if: always()
+    run: |
+      rm -rf toolchain
+      rm -rf build
+      rm -rf khronos_sycl_cts

devops/cts_exclude_filter

@@ -0,0 +1,22 @@
+context
+device
+event
+exceptions
+kernel
+multi_ptr
+platform
+queue
+reduction
+optional_kernel_features
+accessor
+accessor_legacy
+vector_alias
+vector_api
+vector_constructors
+vector_load_store
+vector_operators
+vector_swizzle_assignment
+vector_swizzles
+kernel_bundle
+specialization_constants
+device_selector

devops/scripts/generate_test_matrix.js

@@ -69,6 +69,43 @@ module.exports = ({core, process}) => {
 
       core.setOutput('lts_matrix', ltsString);
       core.setOutput('lts_aws_matrix', ltsAWSString);
+
+      const ctsConfigs = inputs.cts_config.split(';');
+
+      const enabledCTSConfigs = [];
+
+      testConfigs.cts.forEach(v => {
+        if (ctsConfigs.includes(v.config)) {
+          if (needsDrivers) {
+            v["env"] = {
+              "compute_runtime_tag" :
+                  driverNew["linux"]["compute_runtime"]["github_tag"],
+              "igc_tag" : driverNew["linux"]["igc"]["github_tag"],
+              "cm_tag" : driverNew["linux"]["cm"]["github_tag"],
+              "tbb_tag" : driverNew["linux"]["tbb"]["github_tag"],
+              "cpu_tag" : driverNew["linux"]["oclcpu"]["github_tag"],
+              "fpgaemu_tag" : driverNew["linux"]["fpgaemu"]["github_tag"],
+            };
+          } else {
+            v["env"] = {};
+          }
+          enabledCTSConfigs.push(v);
+        }
+      });
+
+      let ctsString = JSON.stringify(enabledCTSConfigs);
+      console.log(ctsString);
+
+      for (let [key, value] of Object.entries(inputs)) {
+        ctsString = ctsString.replaceAll("${{ inputs." + key + " }}", value);
+      }
+      if (needsDrivers) {
+        ctsString = ctsString.replaceAll(
+            "ghcr.io/intel/llvm/ubuntu2004_intel_drivers:latest",
+            "ghcr.io/intel/llvm/ubuntu2004_base:latest");
+      }
+
+      core.setOutput('cts_matrix', ctsString);
     }
   });
 }

devops/test_configs.json

@@ -72,5 +72,19 @@
       "check_sycl_all": "cuda:gpu,host",
       "cmake_args": ""
     }
+  ],
+  "cts": [
+    {
+      "config": "cuda",
+      "name": "CUDA SYCL CTS",
+      "runs-on": [
+        "Linux",
+        "cuda"
+      ],
+      "image": "${{ inputs.cuda_image }}",
+      "container_options": "--gpus all",
+      "sycl_device_filter": "ext_oneapi_cuda:gpu,host",
+      "cmake_args": "-DDPCPP_TARGET_TRIPLES=nvptx64-nvidia-cuda"
+    }
   ]
 }