Add clang as CUDA FE compiler in CI

.github/workflows/canary.yml

@@ -4,7 +4,7 @@ on:
   push:
     branches:
       - master
-      - add-canary-release
+      - add-support-clang-12
   workflow_dispatch:
     inputs:
       oneflow-ref:
@@ -43,7 +43,7 @@ jobs:
       - name: Checkout Oneflow-Inc/oneflow
         if: ${{ github.event.inputs.oneflow-ref == '' }}
         uses: actions/checkout@v2
-      - uses: Oneflow-Inc/get-oneflow@canary-release
+      - uses: Oneflow-Inc/get-oneflow@support-clang-12
         name: Build manylinux
         id: build-cuda
         with:

.github/workflows/simple.yml

@@ -247,7 +247,7 @@ jobs:
           repository: Oneflow-Inc/conda-env
           ref: 30a7f00eb48ee9009d85a848e720823e5054c66b
           path: conda-env
-      - uses: Oneflow-Inc/get-oneflow@canary-release
+      - uses: Oneflow-Inc/get-oneflow@support-clang-12
         name: Build with gcc7
         if: ${{ matrix.build-type == 'gcc7'}}
         with:
@@ -256,7 +256,7 @@ jobs:
           oneflow-build-env: conda
           conda-env-file: conda-env/dev/gcc7/environment-v2.yml
           conda-env-name: oneflow-dev-gcc7-v2
-      - uses: Oneflow-Inc/get-oneflow@canary-release
+      - uses: Oneflow-Inc/get-oneflow@support-clang-12
         name: Build with clang10
         if: ${{ matrix.build-type == 'clang10'}}
         with:

.github/workflows/test.yml

@@ -182,7 +182,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           repository: ${{github.event.pull_request.head.repo.full_name}}
-      - uses: Oneflow-Inc/get-oneflow/cache-complete/matrix/build@canary-release
+      - uses: Oneflow-Inc/get-oneflow/cache-complete/matrix/build@support-clang-12
         name: find cache
         id: find-cache
         timeout-minutes: 5
@@ -228,7 +228,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           repository: ${{github.event.pull_request.head.repo.full_name}}
-      - uses: Oneflow-Inc/get-oneflow/cache-complete@canary-release
+      - uses: Oneflow-Inc/get-oneflow/cache-complete@support-clang-12
         name: Save cache if successful
         id: save-cache
         timeout-minutes: 5
@@ -242,7 +242,7 @@ jobs:
         run: |
           echo "::error file=test.yml,line=204,col=10::steps.save-cache.outputs.cache-hit != matrix.cache-hit"
           exit 1
-      - uses: Oneflow-Inc/get-oneflow@canary-release
+      - uses: Oneflow-Inc/get-oneflow@support-clang-12
         name: Build manylinux cpu only
         id: build-cpu
         if: ${{ matrix.entry =='cpu' && !matrix.cache-hit }}
@@ -263,7 +263,7 @@ jobs:
           python-versions: |
             3.6
             3.7
-      - uses: Oneflow-Inc/get-oneflow@canary-release
+      - uses: Oneflow-Inc/get-oneflow@support-clang-12
         name: Build manylinux cu102
         id: build-cuda
         if: ${{ matrix.entry =='cu102' && !matrix.cache-hit }}
@@ -284,7 +284,7 @@ jobs:
           python-versions: |
             3.6
             3.7
-      - uses: Oneflow-Inc/get-oneflow@canary-release
+      - uses: Oneflow-Inc/get-oneflow@support-clang-12
         name: Build manylinux cu101_xla
         id: build-xla
         if: ${{ matrix.entry =='cu101_xla' && !matrix.cache-hit && needs.changed_files.outputs.should_run_single_client_tests == '1' }}
@@ -306,7 +306,7 @@ jobs:
             3.6
       - name: Upload bin
         if: ${{ !fromJson(matrix.cache-hit) && contains(matrix.runs-on, 'self-hosted') && (steps.build-cpu.outcome == 'success' || steps.build-cuda.outcome == 'success' || steps.build-xla.outcome == 'success') }}
-        uses: Oneflow-Inc/get-oneflow/digest/upload@canary-release
+        uses: Oneflow-Inc/get-oneflow/digest/upload@support-clang-12
         timeout-minutes: 10
         with:
           digest: ${{ steps.save-cache.outputs.build-digest }}
@@ -317,7 +317,7 @@ jobs:
           dst-dir: bin
       - name: Upload whl
         if: ${{ !fromJson(matrix.cache-hit) && contains(matrix.runs-on, 'self-hosted') && (steps.build-cpu.outcome == 'success' || steps.build-cuda.outcome == 'success' || steps.build-xla.outcome == 'success') }}
-        uses: Oneflow-Inc/get-oneflow/digest/upload@canary-release
+        uses: Oneflow-Inc/get-oneflow/digest/upload@support-clang-12
         timeout-minutes: 10
         with:
           digest: ${{ steps.save-cache.outputs.build-digest }}
@@ -331,14 +331,19 @@ jobs:
     name: Build with clang
     if: github.event.pull_request.draft == false && github.base_ref == 'master' && contains(github.event.pull_request.requested_reviewers.*.login, 'oneflow-ci-bot')
     runs-on: [self-hosted, linux, build]
+    env:
+      ONEFLOW_SRC: .
+      MANYLINUX_CACHE_DIR: ~/manylinux-cache-dir/clang13
+      CUDA_VERSION: "10.1"
+      WHEELHOUSE_DIR: ./wheelhouse
     steps:
       - name: Fix permissions
         run: |
           set -x
           docker run --rm -v $PWD:$PWD -w $PWD busybox rm -rf *
       - name: Checkout Oneflow-Inc/oneflow
         uses: actions/checkout@v2
-      - uses: Oneflow-Inc/get-oneflow/cache-complete@canary-release
+      - uses: Oneflow-Inc/get-oneflow/cache-complete@support-clang-12
         name: Save cache if successful
         id: save-cache
         timeout-minutes: 5
@@ -347,25 +352,26 @@ jobs:
           entry: build-with-clang
           digest-type: build
           mark-as-completed: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
-      - name: Checkout Oneflow-Inc/conda-env
-        if: ${{ !fromJSON(steps.save-cache.outputs.cache-hit) }}
-        uses: actions/checkout@v2
-        with:
-          repository: Oneflow-Inc/conda-env
-          ref: 30a7f00eb48ee9009d85a848e720823e5054c66b
-          path: conda-env
-      - uses: Oneflow-Inc/get-oneflow@canary-release
+      - name: Build with Clang
+        uses: Oneflow-Inc/get-oneflow@support-clang-12
         if: ${{ !fromJSON(steps.save-cache.outputs.cache-hit) }}
-        name: Build with clang10
         with:
-          cmake-init-cache: cmake/caches/ci/gh-hosted/cpu-clang.cmake
-          oneflow-src: .
-          oneflow-build-env: conda
-          conda-env-file: conda-env/dev/clang10/environment-v2.yml
-          conda-env-name: oneflow-dev-clang10-v2
-          conda-installer-url: https://oneflow-static.oss-cn-beijing.aliyuncs.com/downloads/conda-installers/Miniconda3-py39_4.10.3-Linux-x86_64.sh
-          conda-prefix: ~/miniconda3-prefixes/py39_4.10.3
+          cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/llvm/cuda-75-clang.cmake
+          build-script: ${{ env.ONEFLOW_SRC }}/ci/clang/build-llvm.sh
+          oneflow-src: ${{ env.ONEFLOW_SRC }}
+          oneflow-build-env: llvm
+          wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }}
+          clear-wheelhouse-dir: true
           self-hosted: true
+          cuda-version: ${{ env.CUDA_VERSION }}
+          manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }}
+          docker-run-use-system-http-proxy: false
+          docker-run-use-lld: false
+          retry-failed-build: true
+          clean-ccache: ${{ contains(github.event.pull_request.labels.*.name, 'need-clean-ccache') }}
+          wheel-audit: false
+          python-versions: |
+            3.8
 
   find-test-cache:
     name: "Find test cache"
@@ -382,7 +388,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           repository: ${{github.event.pull_request.head.repo.full_name}}
-      - uses: Oneflow-Inc/get-oneflow/cache-complete/matrix/test@canary-release
+      - uses: Oneflow-Inc/get-oneflow/cache-complete/matrix/test@support-clang-12
         name: find cache
         id: find-cache
         timeout-minutes: 5
@@ -424,7 +430,7 @@ jobs:
         if: ${{ contains(matrix.runs-on, 'self-hosted') }}
         run: |
           docker rm -f ${{ env.TEST_CONTAINER_NAME }} || true
-      - uses: Oneflow-Inc/get-oneflow/cache-complete@canary-release
+      - uses: Oneflow-Inc/get-oneflow/cache-complete@support-clang-12
         name: Save cache if successful
         id: save-cache
         timeout-minutes: 5
@@ -440,7 +446,7 @@ jobs:
           exit 1
       - name: Download wheel and binary
         if: ${{ !fromJson(matrix.cache-hit) && contains(matrix.runs-on, 'self-hosted') && (!fromJson(matrix.is-xla) || (fromJson(matrix.is-xla) && needs.changed_files.outputs.should_run_single_client_tests == '1')) }}
-        uses: Oneflow-Inc/get-oneflow/digest/download@canary-release
+        uses: Oneflow-Inc/get-oneflow/digest/download@support-clang-12
         id: download-digest
         timeout-minutes: 10
         with:
@@ -744,7 +750,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           repository: ${{github.event.pull_request.head.repo.full_name}}
           fetch-depth: 0
-      - uses: Oneflow-Inc/get-oneflow/cache-complete@canary-release
+      - uses: Oneflow-Inc/get-oneflow/cache-complete@support-clang-12
         name: Save cache if successful
         id: save-cache
         timeout-minutes: 5

CMakeLists.txt

@@ -34,6 +34,7 @@ option(WITH_OPENVINO "Option to build with OpenVINO" OFF)
 option(WITH_MLIR "" OFF)
 option(WITH_MLIR_CUDA_CODEGEN "" OFF)
 option(WITH_COCOAPI "Option to build with COCO API" ON)
+option(WITH_ZLIB "" ON)
 option(BUILD_GIT_VERSION "" ON)
 option(BUILD_PROFILER "" OFF)
 option(OF_SOFTMAX_USE_FAST_MATH "" ON)

ci/clang/build-llvm.sh

@@ -0,0 +1,28 @@
+set -ex
+export PATH=/usr/lib/llvm-12/bin:/usr/lib/llvm-13/bin:/usr/lib64/ccache:/root/.local/bin:$PATH
+
+# clean python dir
+cd ${ONEFLOW_CI_SRC_DIR}
+${ONEFLOW_CI_PYTHON_EXE} -m pip install -i https://mirrors.aliyun.com/pypi/simple --user -r ci/fixed-dev-requirements.txt
+cd python
+git clean -nXd -e \!dist -e \!dist/**
+git clean -fXd -e \!dist -e \!dist/**
+
+# cmake config
+mkdir -p ${ONEFLOW_CI_BUILD_DIR}
+cd ${ONEFLOW_CI_BUILD_DIR}
+find ${ONEFLOW_CI_BUILD_DIR} -name CMakeCache.txt
+find ${ONEFLOW_CI_BUILD_DIR} -name CMakeCache.txt -delete
+if [ ! -f "$ONEFLOW_CI_CMAKE_INIT_CACHE" ]; then
+    echo "$ONEFLOW_CI_CMAKE_INIT_CACHE does not exist."
+    exit 1
+fi
+cmake -S ${ONEFLOW_CI_SRC_DIR} -C ${ONEFLOW_CI_CMAKE_INIT_CACHE} -DPython3_EXECUTABLE=${ONEFLOW_CI_PYTHON_EXE}
+# cmake build
+cd ${ONEFLOW_CI_BUILD_DIR}
+cmake --build . -j $(nproc)
+
+# build pip
+cd ${ONEFLOW_CI_SRC_DIR}
+cd python
+${ONEFLOW_CI_PYTHON_EXE} setup.py bdist_wheel

cmake/caches/ci/llvm/cuda-75-clang.cmake

@@ -0,0 +1,22 @@
+set(CMAKE_C_COMPILER "clang" CACHE STRING "")
+set(CMAKE_CXX_COMPILER "clang++" CACHE STRING "")
+set(CMAKE_CUDA_COMPILER "clang++" CACHE STRING "")
+set(CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld" CACHE STRING "")
+set(CMAKE_MODULE_LINKER_FLAGS_INIT "-fuse-ld=lld" CACHE STRING "")
+set(CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld" CACHE STRING "")
+set(BUILD_SHARED_LIBS YES CACHE BOOL "")
+set(BUILD_CUDA YES CACHE BOOL "")
+set(CMAKE_CUDA_ARCHITECTURES "75" CACHE STRING "")
+set(BUILD_TESTING YES CACHE BOOL "")
+set(THIRD_PARTY_MIRROR aliyun CACHE STRING "")
+set(PIP_INDEX_MIRROR "https://pypi.tuna.tsinghua.edu.cn/simple" CACHE STRING "")
+set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
+set(CMAKE_GENERATOR Ninja CACHE STRING "")
+set(CMAKE_C_COMPILER_LAUNCHER ccache CACHE STRING "")
+set(CMAKE_CXX_COMPILER_LAUNCHER ccache CACHE STRING "")
+set(CMAKE_CUDA_COMPILER_LAUNCHER ccache CACHE STRING "")
+set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF CACHE BOOL "")
+set(CUDAToolkit_ROOT /usr/local/cuda CACHE STRING "")
+set(CUDNN_ROOT_DIR /usr/local/cudnn CACHE STRING "")
+set(RPC_BACKEND "LOCAL" CACHE STRING "")
+set(BUILD_HWLOC NO CACHE BOOL "")

cmake/caches/cn/fast/cuda-61-clang.cmake

@@ -1,5 +1,6 @@
 set(CMAKE_C_COMPILER "clang" CACHE STRING "")
 set(CMAKE_CXX_COMPILER "clang++" CACHE STRING "")
+set(CMAKE_CUDA_COMPILER "clang++" CACHE STRING "")
 set(CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld" CACHE STRING "")
 set(CMAKE_MODULE_LINKER_FLAGS_INIT "-fuse-ld=lld" CACHE STRING "")
 set(CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld" CACHE STRING "")
@@ -13,5 +14,5 @@ set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
 set(CMAKE_GENERATOR Ninja CACHE STRING "")
 set(CMAKE_C_COMPILER_LAUNCHER ccache CACHE STRING "")
 set(CMAKE_CXX_COMPILER_LAUNCHER ccache CACHE STRING "")
+set(CMAKE_CUDA_COMPILER_LAUNCHER ccache CACHE STRING "")
 set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF CACHE BOOL "")
-

cmake/caches/cn/fast/cuda-75-clang.cmake

@@ -1,5 +1,6 @@
 set(CMAKE_C_COMPILER "clang" CACHE STRING "")
 set(CMAKE_CXX_COMPILER "clang++" CACHE STRING "")
+set(CMAKE_CUDA_COMPILER "clang++" CACHE STRING "")
 set(CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld" CACHE STRING "")
 set(CMAKE_MODULE_LINKER_FLAGS_INIT "-fuse-ld=lld" CACHE STRING "")
 set(CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld" CACHE STRING "")
@@ -13,6 +14,5 @@ set(CMAKE_BUILD_TYPE RelWithDebInfo CACHE STRING "")
 set(CMAKE_GENERATOR Ninja CACHE STRING "")
 set(CMAKE_C_COMPILER_LAUNCHER ccache CACHE STRING "")
 set(CMAKE_CXX_COMPILER_LAUNCHER ccache CACHE STRING "")
+set(CMAKE_CUDA_COMPILER_LAUNCHER ccache CACHE STRING "")
 set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF CACHE BOOL "")
-
-

cmake/third_party.cmake

@@ -3,7 +3,9 @@ if (NOT WIN32)
   find_package(Threads)
 endif()
 
-include(zlib)
+if (WITH_ZLIB)
+  include(zlib)
+endif()
 include(protobuf)
 include(googletest)
 include(gflags)
@@ -140,7 +142,6 @@ set(oneflow_third_party_libs
     ${OPENCV_STATIC_LIBRARIES}
     ${COCOAPI_STATIC_LIBRARIES}
     ${LIBJPEG_STATIC_LIBRARIES}
-    zlib_imported
     ${ABSL_STATIC_LIBRARIES}
     ${OPENSSL_STATIC_LIBRARIES}
     ${CMAKE_THREAD_LIBS_INIT}
@@ -155,14 +156,17 @@ if (NOT WITH_XLA)
   list(APPEND oneflow_third_party_libs ${RE2_LIBRARIES})
 endif()
 
+if (WITH_ZLIB)
+  list(APPEND oneflow_third_party_libs zlib_imported)
+endif()
+
 if(WIN32)
   # static gflags lib requires "PathMatchSpecA" defined in "ShLwApi.Lib"
   list(APPEND oneflow_third_party_libs "ShLwApi.Lib")
   list(APPEND oneflow_third_party_libs "Ws2_32.lib")
 endif()
 
 set(oneflow_third_party_dependencies
-  zlib
   protobuf
   gflags
   glog
@@ -181,7 +185,9 @@ set(oneflow_third_party_dependencies
 if (WITH_ONEDNN)
   list(APPEND oneflow_third_party_dependencies onednn)
 endif()
-
+if (WITH_ZLIB)
+  list(APPEND oneflow_third_party_dependencies zlib)
+endif()
 
 if (WITH_COCOAPI)
   list(APPEND oneflow_third_party_dependencies cocoapi_copy_headers_to_destination)

cmake/third_party/absl.cmake

@@ -34,12 +34,6 @@ if(THIRD_PARTY)
     URL_MD5 20126998c9b17e5f7a93711972f03f79
     UPDATE_COMMAND ""
     BUILD_BYPRODUCTS ${ABSL_STATIC_LIBRARIES}
-    CMAKE_ARGS
-        -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
-        -DBUILD_SHARED_LIBS:BOOL=OFF
-        -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
-        -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG}
-        -DCMAKE_CXX_FLAGS_RELEASE:STRING=${CMAKE_CXX_FLAGS_RELEASE}
     CMAKE_CACHE_ARGS
         -DCMAKE_C_COMPILER_LAUNCHER:STRING=${CMAKE_C_COMPILER_LAUNCHER}
         -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=${CMAKE_CXX_COMPILER_LAUNCHER}

cmake/third_party/opencv.cmake

@@ -46,7 +46,7 @@ else()
 endif()
 
 ExternalProject_Add(opencv
-    DEPENDS zlib libjpeg_copy_headers_to_destination libjpeg_copy_libs_to_destination
+    DEPENDS libjpeg_copy_headers_to_destination libjpeg_copy_libs_to_destination
     PREFIX opencv
     URL ${OPENCV_URL}
     URL_MD5 59870e55385f5202c1aa178fe37ed2de
@@ -143,6 +143,10 @@ ExternalProject_Add(opencv
         # -DLIB_SUFFIX:STRING=64
 )
 
+if (WITH_ZLIB)
+  add_dependencies(opencv zlib)
+endif()
+
 # put opencv includes in the 'THIRD_PARTY_DIR'
 add_copy_headers_target(NAME opencv SRC ${OPENCV_BUILD_INCLUDE_DIR} DST ${OPENCV_INCLUDE_DIR} DEPS opencv INDEX_FILE "${oneflow_cmake_dir}/third_party/header_index/opencv_headers.txt")
 

cmake/third_party/protobuf.cmake

@@ -45,7 +45,6 @@ if (THIRD_PARTY)
 
 ExternalProject_Add(protobuf
     PREFIX protobuf
-    DEPENDS zlib
     URL ${PROTOBUF_URL}
     URL_MD5 ${PROTOBUF_MD5}
     UPDATE_COMMAND ""
@@ -61,6 +60,7 @@ ExternalProject_Add(protobuf
         -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
         -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
         -DZLIB_ROOT:PATH=${ZLIB_INSTALL}
+        -Dprotobuf_WITH_ZLIB:BOOL=${WITH_ZLIB}
         -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG}
         -DBUILD_SHARED_LIBS:BOOL=${PROTOBUF_BUILD_SHARED_LIBS}
         -Dprotobuf_BUILD_SHARED_LIBS:BOOL=${PROTOBUF_BUILD_SHARED_LIBS}
@@ -72,6 +72,9 @@ ExternalProject_Add(protobuf
         -Dprotobuf_DEBUG_POSTFIX:STRING=
         ${PROTOBUF_ADDITIONAL_CMAKE_OPTIONS}
 )
+if (WITH_ZLIB)
+  add_dependencies(protobuf zlib)
+endif()
 else()
 add_custom_target(protobuf)
 endif(THIRD_PARTY)