From 57d68630edcfe1c516ef490f5ad2cc3eda686174 Mon Sep 17 00:00:00 2001 From: "Havrylko, Yevhenii" Date: Mon, 22 May 2023 12:33:12 -0500 Subject: [PATCH] Update docker --- .github/workflows/docker.yml | 92 +++++++++++++++--------------------- Dockerfile | 64 ++++++++++++++++++++----- 2 files changed, 90 insertions(+), 66 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a243cd7768..5f7c2036ae 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,21 +4,23 @@ on: workflow_dispatch: inputs: version: - default: '0.20.0' + default: '0.21.0dev1' dpctl_version: - default: '0.14.2' + default: '0.14.3dev1' dpnp_version: - default: '0.11.1' + default: '0.11.2dev0' numpy_version: - default: '1.22.3' + default: '1.23.5' numba_version: - default: '0.56.4' + default: '0.57.0' cython_version: default: "0.29.33" scikit_build_version: default: "0.16.7" oneapi_version: - default: '2023.0.0' + default: '2023.1.0.46401' + python_versions: + default: '["3.8", "3.9", "3.10"]' jobs: @@ -30,6 +32,7 @@ jobs: DOCKER_TAG: ${{ steps.write.outputs.DOCKER_TAG }} DOCKER_STAGES_IMAGE: ${{ steps.write.outputs.DOCKER_STAGES_IMAGE }} DOCKER_REPO_IMAGE: ${{ steps.write.outputs.DOCKER_REPO_IMAGE }} + VERSION: ${{ steps.write.outputs.VERSION }} steps: - name: Write output env id: write @@ -53,18 +56,17 @@ jobs: echo "DOCKER_STAGES_IMAGE=ghcr.io/$REPOSITORY/stages" >> $GITHUB_OUTPUT echo "DOCKER_REPO_IMAGE=ghcr.io/$REPOSITORY" >> $GITHUB_OUTPUT + echo "VERSION=$(echo ${{ inputs.version }} | tr '/' '-')" >> $GITHUB_OUTPUT build-toolkit: needs: env runs-on: ubuntu-latest strategy: matrix: - # Numba does not support 3.11 - # https://github.com/numba/numba/issues/8304 - python_version: ['3.8', '3.9', '3.10'] + python_version: ${{ fromJSON(inputs.python_versions) }} env: BUILD_TARGET: toolkit - DOCKER_TAG: ${{ inputs.version }}-py${{ matrix.python_version }} + DOCKER_TAG: ${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} steps: - name: Checkout uses: actions/checkout@v3 @@ -101,12 +103,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Numba does not support 3.11 - # https://github.com/numba/numba/issues/8304 - python_version: ['3.8', '3.9', '3.10'] + python_version: ${{ fromJSON(inputs.python_versions) }} env: BUILD_TARGET: runtime-base - DOCKER_TAG: ${{ inputs.version }}-py${{ matrix.python_version }} + DOCKER_TAG: ${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} steps: - name: Checkout uses: actions/checkout@v3 @@ -145,12 +145,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Numba does not support 3.11 - # https://github.com/numba/numba/issues/8304 - python_version: ['3.8', '3.9', '3.10'] + python_version: ${{ fromJSON(inputs.python_versions) }} env: BUILD_TARGET: builder - DOCKER_TAG: ${{ inputs.version }}-py${{ matrix.python_version }} + DOCKER_TAG: ${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} steps: - name: Checkout uses: actions/checkout@v3 @@ -189,12 +187,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Numba does not support 3.11 - # https://github.com/numba/numba/issues/8304 - python_version: ['3.8', '3.9', '3.10'] + python_version: ${{ fromJSON(inputs.python_versions) }} env: BUILD_TARGET: dpctl-builder - DOCKER_TAG: ${{ inputs.version }}-py${{ matrix.python_version }} + DOCKER_TAG: ${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} steps: - name: Checkout uses: actions/checkout@v3 @@ -232,12 +228,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Numba does not support 3.11 - # https://github.com/numba/numba/issues/8304 - python_version: ['3.8', '3.9', '3.10'] + python_version: ${{ fromJSON(inputs.python_versions) }} env: BUILD_TARGET: dpnp-builder - DOCKER_TAG: ${{ inputs.version }}-py${{ matrix.python_version }} + DOCKER_TAG: ${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} steps: - name: Checkout uses: actions/checkout@v3 @@ -276,13 +270,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Numba does not support 3.11 - # https://github.com/numba/numba/issues/8304 - python_version: ['3.8', '3.9', '3.10'] + python_version: ${{ fromJSON(inputs.python_versions) }} env: BUILD_TARGET: numba-dpex-builder-runtime - DOCKER_TAG: ${{ inputs.version }}-py${{ matrix.python_version }} - DEST_TAG: ${{ needs.env.outputs.DOCKER_REPO_IMAGE }}/builder:${{ inputs.version }}-py${{ matrix.python_version }} + DOCKER_TAG: ${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} + DEST_TAG: ${{ needs.env.outputs.DOCKER_REPO_IMAGE }}/builder:${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} steps: - name: Checkout uses: actions/checkout@v3 @@ -322,13 +314,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Numba does not support 3.11 - # https://github.com/numba/numba/issues/8304 - python_version: ['3.8', '3.9', '3.10'] + python_version: ${{ fromJSON(inputs.python_versions) }} env: BUILD_TARGET: runtime - DOCKER_TAG: ${{ inputs.version }}-py${{ matrix.python_version }} - DEST_TAG: ${{ needs.env.outputs.DOCKER_REPO_IMAGE }}/runtime:${{ inputs.version }}-py${{ matrix.python_version }} + DOCKER_TAG: ${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} + DEST_TAG: ${{ needs.env.outputs.DOCKER_REPO_IMAGE }}/runtime:${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} steps: - name: Checkout uses: actions/checkout@v3 @@ -370,12 +360,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Numba does not support 3.11 - # https://github.com/numba/numba/issues/8304 - python_version: ['3.8', '3.9', '3.10'] + python_version: ${{ fromJSON(inputs.python_versions) }} env: BUILD_TARGET: drivers - DOCKER_TAG: ${{ inputs.version }}-py${{ matrix.python_version }} + DOCKER_TAG: ${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} steps: - name: Checkout uses: actions/checkout@v3 @@ -412,13 +400,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Numba does not support 3.11 - # https://github.com/numba/numba/issues/8304 - python_version: ['3.8', '3.9', '3.10'] + python_version: ${{ fromJSON(inputs.python_versions) }} env: BUILD_TARGET: runtime-gpu - DOCKER_TAG: ${{ inputs.version }}-py${{ matrix.python_version }} - DEST_TAG: ${{ needs.env.outputs.DOCKER_REPO_IMAGE }}/runtime:${{ inputs.version }}-py${{ matrix.python_version }}-gpu + DOCKER_TAG: ${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} + DEST_TAG: ${{ needs.env.outputs.DOCKER_REPO_IMAGE }}/runtime:${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }}-gpu steps: - name: Checkout uses: actions/checkout@v3 @@ -456,13 +442,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Numba does not support 3.11 - # https://github.com/numba/numba/issues/8304 - python_version: ['3.8', '3.9', '3.10'] + python_version: ${{ fromJSON(inputs.python_versions) }} env: BUILD_TARGET: numba-dpex-builder-runtime-gpu - DOCKER_TAG: ${{ inputs.version }}-py${{ matrix.python_version }} - DEST_TAG: ${{ needs.env.outputs.DOCKER_REPO_IMAGE }}/builder:${{ inputs.version }}-py${{ matrix.python_version }}-gpu + DOCKER_TAG: ${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} + DEST_TAG: ${{ needs.env.outputs.DOCKER_REPO_IMAGE }}/builder:${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }}-gpu steps: - name: Checkout uses: actions/checkout@v3 @@ -502,14 +486,12 @@ jobs: strategy: fail-fast: false matrix: - # Numba does not support 3.11 - # https://github.com/numba/numba/issues/8304 - python_version: ['3.8', '3.9', '3.10'] + python_version: ${{ fromJSON(inputs.python_versions) }} pyargs: ['dpctl', 'numba_dpex.tests', '/opt/dpnp/tests', '/opt/dpnp/tests/third_party'] env: - DOCKER_TAG: ${{ inputs.version }}-py${{ matrix.python_version }} + DOCKER_TAG: ${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }} container: - image: ${{ needs.env.outputs.DOCKER_STAGES_IMAGE }}:${{ inputs.version }}-py${{ matrix.python_version }}-runtime + image: ${{ needs.env.outputs.DOCKER_STAGES_IMAGE }}:${{ needs.env.outputs.VERSION }}-py${{ matrix.python_version }}-runtime credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index 40e5b3116c..a2be036a97 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,25 +4,33 @@ ARG PYTHON_VERSION=3.9.16 # Driver args # print on gpu is broken for 22.43.24595.30 + igc-1.0.12812.26 -ARG CR_TAG=22.43.24595.30 -ARG IGC_TAG=igc-1.0.12504.5 +# ARG CR_TAG=22.43.24595.30 +# ARG IGC_TAG=igc-1.0.12504.5 +ARG CR_TAG=latest +ARG IGC_TAG=latest ARG CM_TAG=latest ARG L0_TAG=latest # ONEAPI -ARG ONEAPI_INSTALLER_URL=https://registrationcenter-download.intel.com/akdlm/irc_nas/19079 -ARG ONEAPI_VERSION=2023.0.0 -ARG ONEAPI_INSTALL_BINARY_NAME=l_BaseKit_p_$ONEAPI_VERSION.25537.sh +ARG ONEAPI_INSTALLER_URL=https://registrationcenter-download.intel.com/akdlm/IRC_NAS/7deeaac4-f605-4bcf-a81b-ea7531577c61 +ARG ONEAPI_VERSION=2023.1.0.46401 +ARG ONEAPI_INSTALL_BINARY_NAME=l_BaseKit_p_$ONEAPI_VERSION.sh ARG ONEAPI_INSTALL_DIR=/opt/intel/oneapi # Versions of the intel python packages -ARG DPCTL_GIT_BRANCH=0.14.2 +ARG DPCTL_GIT_BRANCH=0.14.3dev1 ARG DPCTL_GIT_URL=https://github.com/IntelPython/dpctl.git -ARG DPNP_GIT_BRANCH=0.11.1 +ARG DPNP_GIT_BRANCH=0.11.2dev0 ARG DPNP_GIT_URL=https://github.com/IntelPython/dpnp.git -ARG NUMBA_DPEX_GIT_BRANCH=0.20.0 +ARG DPCPP_LLVM_SPIRV_GIT_BRANCH=main +ARG DPCPP_LLVM_SPIRV_GIT_URL=https://github.com/IntelPython/dpcpp-llvm-spirv.git + +#ARG NUMBA_DPEX_GIT_BRANCH=0.20.1 +#ARG NUMBA_DPEX_GIT_BRANCH=main +#ARG NUMBA_DPEX_GIT_BRANCH=fix/add_lower_function +ARG NUMBA_DPEX_GIT_BRANCH=main ARG NUMBA_DPEX_GIT_URL=https://github.com/IntelPython/numba-dpex.git # CMAKE @@ -30,8 +38,9 @@ ARG CMAKE_VERSION=3.25 ARG CMAKE_VERSION_BUILD=3 # Python -ARG INTEL_NUMPY_VERSION="==1.22.3" -ARG INTEL_NUMBA_VERSION="==0.56.4" +ARG INTEL_NUMPY_VERSION="==1.23.5" +ARG INTEL_NUMBA_VERSION="==0.57.0" +#ARG INTEL_NUMBA_VERSION="==0.56.4" ARG CYTHON_VERSION="==0.29.33" ARG SCIKIT_BUILD_VERSION="==0.16.7" @@ -51,6 +60,7 @@ ARG RUNTIME_BASE_IMAGE=runtime-base ARG BUILDER_IMAGE=builder ARG DPCTL_BUILDER_IMAGE=dpctl-builder ARG DPNP_BUILDER_IMAGE=dpnp-builder +ARG DPCPP_LLVM_SPIRV_BUILDER_IMAGE=dpcpp-llvm-spirv-builder ARG NUMBA_DPEX_BUILDER_IMAGE=numba-dpex-builder ARG TOOLKIT_IMAGE=toolkit ARG NUMBA_DPEX_BUILDER_RUNTIME_IMAGE=numba-dpex-builder-runtime @@ -345,6 +355,31 @@ RUN \ FROM $DPNP_BUILDER_IMAGE AS dpnp-builder-dist +FROM $BUILDER_IMAGE AS dpcpp-llvm-spirv-builder + +ARG ONEAPI_INSTALL_DIR +ARG DPCPP_LLVM_SPIRV_GIT_BRANCH +ARG DPCPP_LLVM_SPIRV_GIT_URL +ARG DPCPP_LLVM_SPIRV_BUILD_DIR=/build +ARG DPCPP_LLVM_SPIRV_DIST_DIR=/dist +ARG http_proxy +ARG https_proxy + +RUN \ + export http_proxy=$http_proxy https_proxy=$https_proxy \ + && mkdir -p $DPCPP_LLVM_SPIRV_BUILD_DIR \ + && mkdir $DPCPP_LLVM_SPIRV_DIST_DIR \ + && cd $DPCPP_LLVM_SPIRV_BUILD_DIR \ + && cd $DPCPP_LLVM_SPIRV_BUILD_DIR \ + && git clone --recursive -b $DPCPP_LLVM_SPIRV_GIT_BRANCH --depth 1 $DPCPP_LLVM_SPIRV_GIT_URL . \ + && cd pkg \ + && python setup.py bdist_wheel \ + && cp dist/dpcpp_llvm_spirv*.whl $DPCPP_LLVM_SPIRV_DIST_DIR + + +FROM $DPCPP_LLVM_SPIRV_BUILDER_IMAGE AS dpcpp-llvm-spirv-dist + + FROM $BUILDER_IMAGE AS numba-dpex-builder-runtime ARG ONEAPI_INSTALL_DIR ARG NUMBA_DPEX_BUILD_DIR=/build @@ -359,14 +394,18 @@ ARG https_proxy RUN \ --mount=type=bind,target=/mnt/dpctl,source=/dist,from=dpctl-builder-dist \ --mount=type=bind,target=/mnt/dpnp,source=/dist,from=dpnp-builder-dist \ + --mount=type=bind,target=/mnt/dpcpp_llvm_spirv,source=/dist,from=dpcpp-llvm-spirv-dist \ --mount=type=cache,target=/root/.cache/pip/ \ export http_proxy=$http_proxy https_proxy=$https_proxy \ && pip install -U \ /mnt/dpctl/dpctl*.whl /mnt/dpnp/dpnp*.whl \ + /mnt/dpcpp_llvm_spirv/dpcpp_llvm_spirv*.whl \ + && ln -s /usr/local/bin/llvm-spirv /usr/local/lib/python*/site-packages/dpcpp_llvm_spirv/ \ && mkdir -p $NUMBA_DPEX_BUILD_DIR \ && mkdir $NUMBA_DPEX_DIST_DIR \ && cd $NUMBA_DPEX_BUILD_DIR \ - && git clone --recursive -b $NUMBA_DPEX_GIT_BRANCH --depth 1 $NUMBA_DPEX_GIT_URL . + && git clone --recursive -b $NUMBA_DPEX_GIT_BRANCH --depth 1 $NUMBA_DPEX_GIT_URL . \ + && find $NUMBA_DPEX_BUILD_DIR -type f -exec sed -i 's/nrt_debug_print(char \*fmt, \.\.\.)/nrt_debug_print(const char \*fmt, \.\.\.)/g' {} + `TODO: remove once fixed` FROM $NUMBA_DPEX_BUILDER_RUNTIME_IMAGE AS numba-dpex-builder @@ -413,6 +452,7 @@ COPY --from=dpnp-builder-dist /build/tests /opt/dpnp/tests RUN \ --mount=type=bind,target=/mnt/dpctl,source=/dist,from=dpctl-builder-dist \ --mount=type=bind,target=/mnt/dpnp,source=/dist,from=dpnp-builder-dist \ + --mount=type=bind,target=/mnt/dpcpp_llvm_spirv,source=/dist,from=dpcpp-llvm-spirv-dist \ --mount=type=bind,target=/mnt/numba_dpex,source=/dist,from=numba-dpex-builder \ --mount=type=cache,target=/root/.cache/pip/ \ export http_proxy=$http_proxy https_proxy=$https_proxy \ @@ -422,7 +462,9 @@ RUN \ numba${INTEL_NUMBA_VERSION} \ /mnt/dpctl/dpctl*.whl \ /mnt/dpnp/dpnp*.whl \ + /mnt/dpcpp_llvm_spirv/dpcpp_llvm_spirv*.whl \ /mnt/numba_dpex/numba_dpex*.whl \ + && ln -s /usr/local/bin/llvm-spirv /usr/local/lib/python*/site-packages/dpcpp_llvm_spirv/ \ && fdupes -qio name /usr/local/lib/python*/site-packages/dpctl/ | \ awk '{if ($0=="") ln=""; else if (ln=="") ln = $0; else system("rm " $0 ";\tln -s " ln " " $0) }'