diff --git a/.github/workflows/sycl_containers.yaml b/.github/workflows/sycl_containers.yaml index 1115d5ef43239..1ae62ec5885c2 100644 --- a/.github/workflows/sycl_containers.yaml +++ b/.github/workflows/sycl_containers.yaml @@ -61,7 +61,7 @@ jobs: ghcr.io/${{ github.repository }}/ubuntu2204_build:${{ github.sha }} ghcr.io/${{ github.repository }}/ubuntu2204_build:latest - # This job produces a Docker container with the latest versions of Intel + # This job produces a Docker container with the released versions of Intel # drivers, that can be found on GitHub. drivers_image_ubuntu2204: if: github.repository == 'intel/llvm' @@ -99,8 +99,8 @@ jobs: tbb_tag=${{fromJson(steps.deps.outputs.deps).linux.tbb.github_tag}} fpgaemu_tag=${{fromJson(steps.deps.outputs.deps).linux.fpgaemu.github_tag}} cpu_tag=${{fromJson(steps.deps.outputs.deps).linux.oclcpu.github_tag}} - # This job produces a Docker container with the latest versions of Intel - # drivers, that can be found on GitHub. + # This job produces a Docker container with the released versions of Intel + # drivers and latest IGC version that can be found on GitHub. drivers_image_ubuntu2204_unstable: if: github.repository == 'intel/llvm' name: Intel Drivers (unstable) Ubuntu 22.04 Docker image @@ -123,11 +123,17 @@ jobs: uses: ./devops/actions/build_container with: push: ${{ github.event_name != 'pull_request' }} - file: ubuntu2204_intel_drivers + file: ubuntu2204_intel_drivers_dev_igfx username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} tags: | - ghcr.io/${{ github.repository }}/ubuntu2204_intel_drivers:unstable-${{ github.sha }} - ghcr.io/${{ github.repository }}/ubuntu2204_intel_drivers:unstable - # build-args:empty, so we automatically take the "latest" tags as - # unstable + ghcr.io/${{ github.repository }}/ubuntu2204_intel_drivers_dev_igfx:latest-${{ github.sha }} + ghcr.io/${{ github.repository }}/ubuntu2204_intel_drivers_dev_igfx:latest + build-args: | + compute_runtime_tag=${{fromJson(steps.deps.outputs.deps).linux.compute_runtime.github_tag}} + igc_dev_hash=${{fromJson(steps.deps.outputs.deps).linux.igc_dev.github_hash}} + cm_tag=${{fromJson(steps.deps.outputs.deps).linux.cm.github_tag}} + level_zero_tag=${{fromJson(steps.deps.outputs.deps).linux.level_zero.github_tag}} + tbb_tag=${{fromJson(steps.deps.outputs.deps).linux.tbb.github_tag}} + fpgaemu_tag=${{fromJson(steps.deps.outputs.deps).linux.fpgaemu.github_tag}} + cpu_tag=${{fromJson(steps.deps.outputs.deps).linux.oclcpu.github_tag}} diff --git a/.github/workflows/sycl_linux_precommit.yml b/.github/workflows/sycl_linux_precommit.yml index f0f4fa872fb33..a7908f51133a9 100644 --- a/.github/workflows/sycl_linux_precommit.yml +++ b/.github/workflows/sycl_linux_precommit.yml @@ -88,11 +88,12 @@ jobs: install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} - name: Matrix E2E tests on Intel Arc A-Series Graphics runner: '["Linux", "arc"]' - image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers:latest + image: ghcr.io/intel/llvm/ubuntu2204_intel_drivers_dev_igfx:latest image_options: -u 1001 --device=/dev/dri --privileged --cap-add SYS_ADMIN target_devices: ext_oneapi_level_zero:gpu;opencl:gpu reset_gpu: true install_drivers: ${{ contains(needs.detect_changes.outputs.filters, 'drivers') }} + use_dev_igfx_drivers: true extra_lit_opts: --param matrix-xmx8=True --param gpu-intel-dg2=True env: '{"LIT_FILTER":"Matrix/"}' uses: ./.github/workflows/sycl_linux_run_tests.yml @@ -104,6 +105,7 @@ jobs: target_devices: ${{ matrix.target_devices }} reset_gpu: ${{ matrix.reset_gpu }} install_drivers: ${{ matrix.install_drivers }} + use_dev_igfx_drivers: ${{ matrix.use_dev_igfx_drivers }} extra_lit_opts: ${{ matrix.extra_lit_opts }} env: ${{ matrix.env || '{}' }} diff --git a/.github/workflows/sycl_linux_run_tests.yml b/.github/workflows/sycl_linux_run_tests.yml index fb3e027f0f959..6512b53455fdd 100644 --- a/.github/workflows/sycl_linux_run_tests.yml +++ b/.github/workflows/sycl_linux_run_tests.yml @@ -68,6 +68,9 @@ on: install_drivers: type: string required: False + use_dev_igfx_drivers: + type: string + required: False env: type: string default: '{}' @@ -131,6 +134,12 @@ on: - false - true + use_dev_igfx_drivers: + type: choice + options: + - false + - true + jobs: run: name: ${{ inputs.name }} @@ -174,11 +183,17 @@ jobs: run: | git -C khronos_sycl_cts submodule update --init - name: Install drivers - if: inputs.install_drivers == 'true' + if: inputs.install_drivers == 'true' && inputs.use_dev_igfx_drivers == 'false' env: GITHUB_TOKEN: ${{ github.token }} run: | sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json --all + - name: Install drivers with development version of IGFX + if: inputs.install_drivers == 'true' && inputs.use_dev_igfx_drivers == 'true' + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + sudo -E bash devops/scripts/install_drivers.sh llvm/devops/dependencies.json --all --use-dev-igfx - name: Source OneAPI TBB vars.sh shell: bash run: | diff --git a/.github/workflows/sycl_update_dev_igc.yml b/.github/workflows/sycl_update_dev_igc.yml new file mode 100644 index 0000000000000..86f393b627c9a --- /dev/null +++ b/.github/workflows/sycl_update_dev_igc.yml @@ -0,0 +1,30 @@ +name: Update development IGC version + +on: + schedule: + - cron: '0 3 * * 2' + workflow_dispatch: + +jobs: + update_dev_igfx_linux: + runs-on: ubuntu-20.04 + if: github.repository == 'intel/llvm' + steps: + - uses: actions/checkout@v3 + - name: Update dependencies file + run: | + version="$(python3 devops/scripts/update_dev_igc.py linux)" + echo 'NEW_IGC_VERSION='$version >> $GITHUB_ENV + - name: Create Pull Request + env: + BRANCH: ci/update_dev_igc-linux-${{ env.NEW_DRIVER_VERSION }} + GITHUB_TOKEN: ${{ secrets.LLVM_MAIN_SYNC_BBSYCL_TOKEN }} + run: | + cd $GITHUB_WORKSPACE + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" + git checkout -B $BRANCH + git add -u + git commit -m "[GHA] Uplift Linux dev IGC version to $NEW_DRIVER_VERSION" || exit 0 # exit if commit is empty + git push https://$GITHUB_TOKEN@github.com/${{ github.repository }} ${BRANCH} + gh pr create --head $BRANCH --title "[GHA] Uplift Linux dev IGC version to $NEW_DRIVER_VERSION" --body "Scheduled IGC uplift" diff --git a/devops/containers/ubuntu2204_intel_drivers_dev_igfx.Dockerfile b/devops/containers/ubuntu2204_intel_drivers_dev_igfx.Dockerfile new file mode 100644 index 0000000000000..705eb5252efa0 --- /dev/null +++ b/devops/containers/ubuntu2204_intel_drivers_dev_igfx.Dockerfile @@ -0,0 +1,29 @@ +ARG base_tag=latest +ARG base_image=ghcr.io/intel/llvm/ubuntu2204_base + +FROM $base_image:$base_tag + +ENV DEBIAN_FRONTEND=noninteractive + +ARG compute_runtime_tag=latest +ARG igc_dev_hash=latest +ARG cm_tag=latest +ARG level_zero_tag=latest +ARG tbb_tag=latest +ARG fpgaemu_tag=latest +ARG cpu_tag=latest + +RUN apt update && apt install -yqq wget +RUN apt install -yqq libllvm14 + +COPY scripts/get_release.py / +COPY scripts/install_drivers.sh / + +RUN mkdir /runtimes +ENV INSTALL_LOCATION=/runtimes +RUN --mount=type=secret,id=github_token GITHUB_TOKEN=$(cat /run/secrets/github_token) /install_drivers.sh --all --use-dev-igfx + +COPY scripts/drivers_entrypoint.sh /drivers_entrypoint.sh + +ENTRYPOINT ["/bin/bash", "/drivers_entrypoint.sh"] + diff --git a/devops/dependencies.json b/devops/dependencies.json index 660c9c628cae4..0cd8fcae30b34 100644 --- a/devops/dependencies.json +++ b/devops/dependencies.json @@ -12,6 +12,9 @@ "url": "https://github.com/intel/intel-graphics-compiler/releases/tag/igc-1.0.14828.8", "root": "{DEPS_ROOT}/opencl/runtime/linux/oclgpu" }, + "igc_dev": { + "github_hash": "f43aace" + }, "cm": { "github_tag": "cmclang-1.0.144", "version": "1.0.144", diff --git a/devops/scripts/install_drivers.sh b/devops/scripts/install_drivers.sh index d9220050ee995..7c72b60a98e97 100755 --- a/devops/scripts/install_drivers.sh +++ b/devops/scripts/install_drivers.sh @@ -3,25 +3,16 @@ set -e set -x -if [ -f "$1" ]; then - # Read data from the dependencies.json passed as the first argument. - CONFIG_FILE=$1 - CR_TAG=$(jq -r '.linux.compute_runtime.github_tag' $CONFIG_FILE) - IGC_TAG=$(jq -r '.linux.igc.github_tag' $CONFIG_FILE) - CM_TAG=$(jq -r '.linux.cm.github_tag' $CONFIG_FILE) - L0_TAG=$(jq -r '.linux.level_zero.github_tag' $CONFIG_FILE) - TBB_TAG=$(jq -r '.linux.tbb.github_tag' $CONFIG_FILE) - FPGA_TAG=$(jq -r '.linux.fpgaemu.github_tag' $CONFIG_FILE) - CPU_TAG=$(jq -r '.linux.oclcpu.github_tag' $CONFIG_FILE) -else - CR_TAG=$compute_runtime_tag - IGC_TAG=$igc_tag - CM_TAG=$cm_tag - L0_TAG=$level_zero_tag - TBB_TAG=$tbb_tag - FPGA_TAG=$fpgaemu_tag - CPU_TAG=$cpu_tag -fi +# Read data from the dependencies.json passed as the first argument. +CONFIG_FILE=$1 +CR_TAG=$(jq -r '.linux.compute_runtime.github_tag' $CONFIG_FILE) +IGC_TAG=$(jq -r '.linux.igc.github_tag' $CONFIG_FILE) +IGC_DEV_HASH=$(jq -r '.linux.igc_dev.github_hash' $CONFIG_FILE) +CM_TAG=$(jq -r '.linux.cm.github_tag' $CONFIG_FILE) +L0_TAG=$(jq -r '.linux.level_zero.github_tag' $CONFIG_FILE) +TBB_TAG=$(jq -r '.linux.tbb.github_tag' $CONFIG_FILE) +FPGA_TAG=$(jq -r '.linux.fpgaemu.github_tag' $CONFIG_FILE) +CPU_TAG=$(jq -r '.linux.oclcpu.github_tag' $CONFIG_FILE) function get_release() { REPO=$1 @@ -39,6 +30,19 @@ function get_release() { | jq -r '. as $raw | try .assets[].browser_download_url catch error($raw)' } +function get_pre_release_igfx() { + HASH=$1 + URL="https://api.github.com/repos/intel/intel-graphics-compiler/actions/artifacts?name=IGC_Ubuntu22.04_llvm14_clang-${HASH}" + HEADER="" + if [ "$GITHUB_TOKEN" != "" ]; then + HEADER="Authorization: Bearer $GITHUB_TOKEN" + fi + ARCH_URL=$(curl -s -L -H "$HEADER" $URL \ + | jq -r '. as $raw | try .artifacts[0].archive_download_url catch error($raw)') + curl -s -L -H "$HEADER" $ARCH_URL > $HASH.zip + unzip $HASH.zip && rm $HASH.zip +} + TBB_INSTALLED=false if [[ -v INSTALL_LOCATION ]]; then @@ -65,18 +69,32 @@ InstallTBB () { } InstallIGFX () { - echo "Installing Intel Graphics driver..." + if [ "$1" == "dev" ]; then + echo "Installing development Intel Graphics driver..." + else + echo "Installing Intel Graphics driver..." + fi echo "Compute Runtime version $CR_TAG" - echo "IGC version $IGC_TAG" + if [ "$1" == "dev" ]; then + echo "IGC hash $IGC_DEV_HASH" + else + echo "IGC version $IGC_TAG" + fi echo "CM compiler version $CM_TAG" echo "Level Zero version $L0_TAG" - get_release intel/intel-graphics-compiler $IGC_TAG \ - | grep ".*deb" \ - | wget -qi - + if [ "$1" != "dev" ]; then + get_release intel/intel-graphics-compiler $IGC_TAG \ + | grep ".*deb" \ + | wget -qi - + else + get_pre_release_igfx $IGC_DEV_HASH + fi get_release intel/compute-runtime $CR_TAG \ | grep -E ".*((deb)|(sum))" \ | wget -qi - - sha256sum -c *.sum && \ + if [ "$1" != "dev" ]; then + sha256sum -c *.sum + fi get_release intel/cm-compiler $CM_TAG \ | grep ".*deb" \ | grep -v "u18" \ @@ -84,7 +102,11 @@ InstallIGFX () { get_release oneapi-src/level-zero $L0_TAG \ | grep ".*deb" \ | wget -qi - - dpkg -i *.deb && rm *.deb *.sum + if [ "$1" != "dev" ]; then + dpkg -i *.deb && rm *.deb *.sum + else + dpkg -i --force-depends-version *.deb && rm *.deb *.sum + fi } InstallCPURT () { @@ -129,24 +151,32 @@ InstallFPGAEmu () { if [[ $# -eq 0 ]] ; then echo "No options were specified. Please, specify one or more of the following:" - echo "--all - Install all Intel drivers" - echo "--igfx - Install Intel Graphics drivers" - echo "--cpu - Install Intel CPU OpenCL runtime" - echo "--fpga-emu - Install Intel FPGA Fast emulator" + echo "--all - Install all Intel drivers" + echo "--igfx - Install Intel Graphics drivers" + echo "--use-dev-igfx - Use with --all or --igfx. Install development version of Intel Graphics drivers" + echo "--cpu - Install Intel CPU OpenCL runtime" + echo "--fpga-emu - Install Intel FPGA Fast emulator" echo "Set INSTALL_LOCATION env variable to specify install location" exit 0 fi +if [[ "$*" == *"--use-dev-igfx"* ]] +then + IGFX_DEV="dev" +else + IGFX_DEV="" +fi + while [ "${1:-}" != "" ]; do case "$1" in "--all") - InstallIGFX + InstallIGFX $IGFX_DEV InstallTBB InstallCPURT InstallFPGAEmu ;; "--igfx") - InstallIGFX + InstallIGFX $IGFX_DEV ;; "--cpu") InstallTBB diff --git a/devops/scripts/update_dev_igc.py b/devops/scripts/update_dev_igc.py new file mode 100644 index 0000000000000..df75dbfdfcb8f --- /dev/null +++ b/devops/scripts/update_dev_igc.py @@ -0,0 +1,40 @@ +from urllib.request import urlopen +import json +import sys +import os + +def uplift_linux_dev_igfx_driver(config, platform_tag): + action_runs = urlopen("https://api.github.com/repos/intel/intel-graphics-compiler/actions/runs").read() + workflow_runs = json.loads(action_runs)['workflow_runs'] + + for run in workflow_runs: + if run['name'] != 'Build IGC': + continue + if run['status'] != 'completed': + continue + if run['conclusion'] != 'success': + continue + config[platform_tag]['igc_dev']['github_hash'] = run['head_sha'][:7] + break + + return config + +def main(platform_tag): + script = os.path.dirname(os.path.realpath(__file__)) + config_name = os.path.join(script, '..', 'dependencies.json') + config = {} + + with open(config_name, "r") as f: + config = json.loads(f.read()) + config = uplift_linux_dev_igfx_driver(config, platform_tag) + + with open(config_name, "w") as f: + json.dump(config, f, indent=2) + f.write('\n') + + return config[platform_tag]['igc_dev']['github_hash'] + + +if __name__ == '__main__': + platform_tag = sys.argv[1] if len(sys.argv) > 1 else "ERROR_PLATFORM" + sys.stdout.write(main(platform_tag) + '\n')