[ci] Use sccache and mamba in MacOS/Android builds

This swaps from conda build to a plain mamba environment which speeds
the setup time from ~15-20 minutes to 2 minutes and enables sccache
which speeds up builds from ~1.5hr to ~5 minutes.

.github/workflows/main.yml

@@ -42,40 +42,94 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: 'recursive'
-      - name: Set up environment
-        uses: ./.github/actions/setup
-      - name: Conda Build
-        shell: bash -l {0}
-        run: >-
-          conda build --output-folder=conda/pkg  conda/recipe &&
-          conda install tvm -c ./conda/pkg
+      - name: Set up
+        run: |
+          set -x
+          brew update
+          brew install sccache
+          mkdir -p ~/.sccache
+          ln -s "$(which sccache)" ~/.sccache/cc
+          ln -s "$(which sccache)" ~/.sccache/c++
+          brew install --cask micromamba
+          eval "$(micromamba shell hook --shell=bash)"
+          micromamba create -f ci/micro.yaml
+          micromamba activate tvm-build
+          llvm-config --version
+          python3 --version
+      # - name: Setup tmate session
+      #   uses: mxschmitt/action-tmate@v3
+      - name: Build library
+        env:
+          target_platform: osx-64
+          PREFIX: /usr/local
+          SRC_DIR: '..'
+          CPU_COUNT: 2
+          AWS_ACCESS_KEY_ID: ${{ secrets.MACOS_SCCACHE_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.MACOS_SCCACHE_AWS_SECRET_ACCESS_KEY }}
+        shell: bash
+        run: |
+          eval "$(micromamba shell hook --shell=bash)"
+          micromamba activate tvm-build
+          set -eux
+          mkdir -p build
+          cp ci/macos_config.cmake build/config.cmake
+          python3 tests/scripts/task_build.py \
+            --cc ~/.sccache/cc \
+            --c++ ~/.sccache/c++ \
+            --sccache-bucket tvm-sccache-macos
+      - name: Install library
+        shell: bash
+        env:
+          SRC_DIR: '.'
+          PYTHON: python3
+        run: |
+          eval "$(micromamba shell hook --shell=bash)"
+          micromamba activate tvm-build
+          set -eux
+          bash ./conda/recipe/install_libtvm.sh
+          bash ./conda/recipe/install_tvm_python.sh
       - name: Build iOS RPC
+        shell: bash
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.MACOS_SCCACHE_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.MACOS_SCCACHE_AWS_SECRET_ACCESS_KEY }}
         run: |
+          eval "$(micromamba shell hook --shell=bash)"
+          micromamba activate tvm-build
+          set -eux
           IOS_VERSION="14.0"
-          CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \
-                       -DCMAKE_SYSTEM_NAME=iOS \
-                       -DCMAKE_SYSTEM_VERSION=${IOS_VERSION} \
-                       -DCMAKE_OSX_SYSROOT=iphonesimulator \
-                       -DCMAKE_OSX_ARCHITECTURES=x86_64 \
-                       -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
-                       -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON \
-                       -DUSE_IOS_RPC=ON"
-
           mkdir build-ios-simulator
-          cd build-ios-simulator
-          cmake .. ${CMAKE_FLAGS}
-          cmake --build . --target ios_rpc
+          python3 tests/scripts/task_build.py \
+            --build-dir build-ios-simulator \
+            --cmake-target ios_rpc \
+            --cc ~/.sccache/cc \
+            --c++ ~/.sccache/c++ \
+            --sccache-bucket tvm-sccache-macos \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_SYSTEM_NAME=iOS \
+            -DCMAKE_SYSTEM_VERSION=${IOS_VERSION} \
+            -DCMAKE_OSX_SYSROOT=iphonesimulator \
+            -DCMAKE_OSX_ARCHITECTURES=x86_64 \
+            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
+            -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON \
+            -DUSE_IOS_RPC=ON
       - name: Test
-        shell: bash -l {0}
-        run: >-
+        shell: bash
+        run: |
+          eval "$(micromamba shell hook --shell=bash)"
+          micromamba activate tvm-build
+          set -eux
           python -m pytest -v tests/python/all-platform-minimal-test
       - name: Test iOS RPC
-        shell: bash -l {0}
-        run: >-
-          python -m pip install tornado psutil cloudpickle &&
-          export PYTHONPATH=tests/python/contrib:${PYTHONPATH} &&
-          export BUNDLE_ID=org.apache.tvmrpc &&
-          export BUNDLE_PATH=build-ios-simulator/apps/ios_rpc/ios_rpc/src/ios_rpc-build/Release-iphonesimulator/tvmrpc.app &&
+        shell: bash
+        run: |
+          eval "$(micromamba shell hook --shell=bash)"
+          micromamba activate tvm-build
+          set -eux
+          python -m pip install tornado psutil cloudpickle
+          export PYTHONPATH=tests/python/contrib
+          export BUNDLE_ID=org.apache.tvmrpc
+          export BUNDLE_PATH=build-ios-simulator/apps/ios_rpc/ios_rpc/src/ios_rpc-build/Release-iphonesimulator/tvmrpc.app
           python -m pytest -v tests/python/contrib/test_rpc_server_device.py
 
   Windows:
@@ -104,21 +158,47 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: 'recursive'
-      - name: Set up environment
-        uses: ./.github/actions/setup
+      - name: Setup dependencies
+        run: |
+          set -x
+          mkdir ~/.sccache
+          pushd ~/.sccache
+          curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.2.15/sccache-v0.2.15-x86_64-unknown-linux-musl.tar.gz
+          tar -xzvf sccache.tar.gz
+          chmod +x sccache-v0.2.15-x86_64-unknown-linux-musl/sccache
+          ln -s "$(pwd)/sccache-v0.2.15-x86_64-unknown-linux-musl/sccache" /usr/local/bin/sccache
+          ln -s "$(pwd)/sccache-v0.2.15-x86_64-unknown-linux-musl/sccache" ~/.sccache/gcc
+          ln -s "$(pwd)/sccache-v0.2.15-x86_64-unknown-linux-musl/sccache" ~/.sccache/g++
+          popd
+          curl -L -o micromamba.tar.bz2 https://micro.mamba.pm/api/micromamba/linux-64/latest
+          tar -xvjf micromamba.tar.bz2
+          export PATH=$(pwd)/bin:$PATH
+          eval "$(micromamba shell hook --shell=bash)"
+          micromamba create -f ci/micro.yaml
+          micromamba activate tvm-build
+          llvm-config --version
+          python3 --version
       - name: Set up java
         uses: actions/setup-java@v3
         with:
           distribution: 'zulu'
           java-version: '11'
       - name: Build TVM
-        shell: bash -l {0}
+        shell: bash
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.MACOS_SCCACHE_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.MACOS_SCCACHE_AWS_SECRET_ACCESS_KEY }}
         run: |
-          mkdir build
-          cd build
-          ../tests/scripts/task_config_build_jvm.sh .
-          cmake ..
-          make
+          export PATH=$(pwd)/bin:$PATH
+          eval "$(micromamba shell hook --shell=bash)"
+          micromamba activate tvm-build
+          set -eux
+          bash ./tests/scripts/task_config_build_jvm.sh build
+          python3 tests/scripts/task_build.py \
+            --sccache-bucket tvm-sccache-macos \
+            --no-cc \
+            -DCMAKE_C_COMPILER_LAUNCHER=sccache \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
       - name: Build TVM4J
         run: |
           make jvmpkg

ci/macos_config.cmake

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set(CMAKE_BUILD_TYPE Release)
+set(USE_RPC ON)
+set(USE_CPP_RPC OFF)
+set(USE_SORT ON)
+set(USE_RANDOM ON)
+set(USE_PROFILER ON)
+set(USE_LLVM ON)
+set(INSTALL_DEV ON)
+set(USE_LIBBACKTRACE AUTO)
+set(USE_METAL ON)
+set(CMAKE_OSX_DEPLOYMENT_TARGET 10.13)
+set(SUMMARIZE ON)

ci/micro.yaml

@@ -0,0 +1,19 @@
+name: tvm-build
+channels:
+  - conda-forge
+dependencies:
+  - python = 3.7
+  - llvmdev >=11
+  - numpy
+  - pytest
+  - cython
+  - cmake
+  - bzip2
+  - ninja
+  - make
+  - scipy
+  - pillow
+  - python
+  - decorator
+  - psutil
+  - scipy

ci/windows_config.cmake

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set(USE_LLVM ON)
+set(USE_RPC ON)
+set(USE_CPP_RPC ON)
+set(USE_SORT ON)
+set(USE_RANDOM ON)
+set(USE_PROFILER ON)
+set(INSTALL_DEV ON)
+set(SUMMARIZE ON)

cmake/utils/Summary.cmake

@@ -42,7 +42,7 @@ macro(print_summary)
     message(STATUS "  C++ compiler ID       : ${CMAKE_CXX_COMPILER_ID}")
     message(STATUS "  C++ compiler version  : ${CMAKE_CXX_COMPILER_VERSION}")
     message(STATUS "  CXX flags             : ${CMAKE_CXX_FLAGS}")
-    message(STATUS "  CXX launcher          : ${CXX_COMPILER_LAUNCHER}")
+    message(STATUS "  CXX launcher          : ${CMAKE_CXX_COMPILER_LAUNCHER}")
     message(STATUS "  Linker flags          : ${CMAKE_SHARED_LINKER_FLAGS}")
     message(STATUS "  Build type            : ${CMAKE_BUILD_TYPE}")
     get_directory_property(READABLE_COMPILE_DEFS DIRECTORY ${PROJECT_SOURCE_DIR} COMPILE_DEFINITIONS)

conda/recipe/install_libtvm.sh

@@ -16,8 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
-set -u
+set -euxo pipefail
 
 cd build
-make install
+ninja install

conda/recipe/install_tvm_python.sh

@@ -16,8 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
-set -u
+set -euxo pipefail
 
 cd ${SRC_DIR}/python
 ${PYTHON} setup.py install --single-version-externally-managed --record=/tmp/record.txt

tests/scripts/task_build.py

@@ -20,7 +20,10 @@
 import os
 import logging
 import sys
+import shlex
 import multiprocessing
+import os
+
 
 from pathlib import Path
 
@@ -29,31 +32,53 @@
 sys.path.append(str(REPO_ROOT / "ci" / "scripts"))
 from cmd_utils import Sh, init_log, REPO_ROOT
 
+IS_WINDOWS = os.name == "nt"
+
 
 if __name__ == "__main__":
     init_log()
 
-    parser = argparse.ArgumentParser(description="List pytest nodeids for a folder")
+    parser = argparse.ArgumentParser(
+        description="Build TVM in CI (all unknown args are passed to CMake)"
+    )
     parser.add_argument("--sccache-bucket", required=False, help="sccache bucket name")
     parser.add_argument("--build-dir", default="build", help="build folder")
     parser.add_argument("--cmake-target", help="optional build target")
-    args = parser.parse_args()
+    parser.add_argument("--cc", help="c sccache wrapper", default="/opt/sccache/cc")
+    parser.add_argument("--c++", help="c++ sccache wrapper", default="/opt/sccache/c++")
+    parser.add_argument("--no-cc", help="dont set compiler wrappers", action="store_true")
+    args, other = parser.parse_known_args()
 
     env = {"VTA_HW_PATH": str(Path(os.getcwd()) / "3rdparty" / "vta-hw")}
     sccache_exe = shutil.which("sccache")
 
     use_sccache = sccache_exe is not None
     build_dir = Path(os.getcwd()) / args.build_dir
+    build_dir.mkdir(exist_ok=True, parents=True)
     build_dir = build_dir.relative_to(REPO_ROOT)
 
     if use_sccache:
         if args.sccache_bucket:
             env["SCCACHE_BUCKET"] = args.sccache_bucket
+            env["SCCACHE_REGION"] = "us-west-2"
+            if "AWS_ACCESS_KEY_ID" in os.environ and "AWS_SECRET_ACCESS_KEY" in os.environ:
+                logging.info("Found environment variable credentials, passing them through")
+                env["AWS_ACCESS_KEY_ID"] = os.environ["AWS_ACCESS_KEY_ID"]
+                env["AWS_SECRET_ACCESS_KEY"] = os.environ["AWS_SECRET_ACCESS_KEY"]
+            else:
+                logging.info("No AWS credentials found")
+                logging.info(f"env: {os.environ.keys()}")
+                logging.info(f"env: {os.environ}")
             logging.info(f"Using sccache bucket: {args.sccache_bucket}")
         else:
             logging.info(f"No sccache bucket set, using local cache")
-        env["CXX"] = "/opt/sccache/c++"
-        env["CC"] = "/opt/sccache/cc"
+        if IS_WINDOWS or args.no_cc:
+            logging.info(f"Detected Windows, not setting c compilers")
+        else:
+            logging.info(f"Using c compiler {args.cc}")
+            logging.info(f"Using c++ compiler {getattr(args, 'c++')}")
+            env["CXX"] = getattr(args, "c++")
+            env["CC"] = args.cc
 
     else:
         if sccache_exe is None:
@@ -76,7 +101,10 @@
     available_cpus = nproc // executors
     num_cpus = max(available_cpus, 1)
 
-    sh.run("cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo ..", cwd=build_dir)
+    command = ["cmake", "-GNinja", "-DCMAKE_BUILD_TYPE=RelWithDebInfo"]
+    command += other
+    command.append("..")
+    sh.run(" ".join([shlex.quote(arg) for arg in command]), cwd=build_dir)
 
     target = ""
     if args.cmake_target: